mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-26 18:58:12 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
76a39fcf92 | ||
|
|
5eb3021752 | ||
|
|
7c7203bdfe | ||
|
|
5ac256fc8b | ||
|
|
2b86f8e600 | ||
|
|
0fddcd042a | ||
|
|
eb1dc8944f | ||
|
|
fd8b47b78e | ||
|
|
487bd13ff8 | ||
|
|
eb743efd9a | ||
|
|
67d152e108 | ||
|
|
c425d2e0b0 | ||
|
|
7bf5add6ef | ||
|
|
4281a8f816 | ||
|
|
53e9edfec1 | ||
|
|
74fbec35fe | ||
|
|
5d4c782f3a | ||
|
|
6505f6cbf5 | ||
|
|
4bb9a2b834 | ||
|
|
2d7aab3a9a | ||
|
|
f749733d54 | ||
|
|
ffcf6eb5d8 | ||
|
|
c209a9bca7 | ||
|
|
85319d45a4 | ||
|
|
514f1627ea | ||
|
|
127de34a20 | ||
|
|
54ab9d79b6 | ||
|
|
f48d06fb90 | ||
|
|
84563291a0 | ||
|
|
ddb601c103 | ||
|
|
450261f109 | ||
|
|
c0f4ad6dfa | ||
|
|
ff5da77372 | ||
|
|
a9044b01df | ||
|
|
11b948369f | ||
|
|
a2dc0c25b1 |
+1
-1
@@ -1,5 +1,5 @@
|
|||||||
[bumpversion]
|
[bumpversion]
|
||||||
current_version = 1.0.1
|
current_version = 1.3.0
|
||||||
commit = True
|
commit = True
|
||||||
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
|
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
|
||||||
serialize =
|
serialize =
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
---
|
||||||
|
name: Tech Issue
|
||||||
|
about: Engineering-specific technical work that is not product-specific. Engineering team "owns" these issues.
|
||||||
|
title: ""
|
||||||
|
labels: tech
|
||||||
|
assignees: ""
|
||||||
|
---
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
Why is this work important to engineers?
|
||||||
|
|
||||||
|
## Definition of Done
|
||||||
|
|
||||||
|
What should the end result look like? What will have been changed?
|
||||||
|
|
||||||
|
## Tasks
|
||||||
|
|
||||||
|
Detail the specific tasks that can be used to accomplish the desired changes.
|
||||||
|
If detailed steps cannot be provided at this time, please file a [Tech Proposal](https://docs.google.com/document/d/1o2vuvl-kXwRJN1nBoPzJS_MAQgDGYnjmPZWa4qRDi-I/edit#heading=h.7dvzhm7gqc3v) instead.
|
||||||
|
|
||||||
|
- [ ]
|
||||||
|
- [ ]
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
name: Close inactive pull requests
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: "30 1 * * *"
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
close-issues:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
issues: write
|
||||||
|
pull-requests: write
|
||||||
|
steps:
|
||||||
|
- uses: actions/stale@v5
|
||||||
|
with:
|
||||||
|
days-before-issue-stale: -1 # Do not mark any issues as stale
|
||||||
|
days-before-pr-stale: 14
|
||||||
|
days-before-pr-close: 3
|
||||||
|
stale-pr-message: "This PR has not seen any activity in the past 2 weeks; if no one comments or reviews it in the next 3 days, this PR will be closed."
|
||||||
|
close-pr-message: "This PR was closed because it has been inactive for 17 days, 3 days since being marked as stale. Please re-open if you still need this to be addressed."
|
||||||
|
stale-pr-label: "stale"
|
||||||
|
close-pr-label: "autoclosed"
|
||||||
|
exempt-draft-pr: true
|
||||||
|
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
@@ -2,7 +2,7 @@ name: Compatibility Tests
|
|||||||
|
|
||||||
on:
|
on:
|
||||||
schedule:
|
schedule:
|
||||||
- cron: '0 8 7 * 2'
|
- cron: "0 8 7 * 2"
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- main
|
- main
|
||||||
@@ -14,9 +14,9 @@ jobs:
|
|||||||
docker-build:
|
docker-build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v4
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@v4
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
- name: Build docker image
|
- name: Build docker image
|
||||||
@@ -28,96 +28,85 @@ jobs:
|
|||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
# note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
|
os: [ubuntu-latest, macos-latest, macos-13]
|
||||||
os: [ubuntu-latest, macos-latest, macos-11]
|
python-version: ["3.10", "3.11", "3.12"]
|
||||||
python-version: [3.6, 3.7, 3.8, 3.9]
|
|
||||||
cellxgene_build: [main, latest]
|
cellxgene_build: [main, latest]
|
||||||
exclude:
|
|
||||||
# 3.6 no longer avail on Big Sur (`macos-11`)
|
|
||||||
- os: macos-11
|
|
||||||
python-version: 3.6
|
|
||||||
# no pypi build exists for macos+py3.9 and source install fails to
|
|
||||||
# install `tables` py pkg (a `scanpy` dependency), so we test py3.9
|
|
||||||
# only on ubuntu
|
|
||||||
- os: macos-11
|
|
||||||
python-version: 3.9
|
|
||||||
- os: macos-latest
|
|
||||||
python-version: 3.9
|
|
||||||
# add anndata pinned version test for subset of matrix configurations,
|
# add anndata pinned version test for subset of matrix configurations,
|
||||||
# in order to reduce matrix cross-product explosion
|
# in order to reduce matrix cross-product explosion
|
||||||
include:
|
include:
|
||||||
- python-version: 3.8
|
- python-version: 3.12
|
||||||
cellxgene_build: latest
|
cellxgene_build: latest
|
||||||
# TODO: dynamically use the literal version in requirements.txt,
|
# TODO: dynamically use the literal version in requirements.txt,
|
||||||
# to avoid having to update this in manually in the future
|
# to avoid having to update this in manually in the future
|
||||||
# TODO: Do not bother running this if anndata latest version
|
# TODO: Do not bother running this if anndata latest version
|
||||||
# matches this pinned version, to avoid a redundant test
|
# matches this pinned version, to avoid a redundant test
|
||||||
anndata_version: '==0.7.6'
|
anndata_version: "==0.10.9"
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v4
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@v4
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
- name: Cache env vars
|
- name: Cache env vars
|
||||||
run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
|
run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
|
||||||
- name: Cache env vars (MacOS)
|
- name: Cache env vars (MacOS)
|
||||||
if: startsWith(matrix.os, 'macos')
|
if: startsWith(matrix.os, 'macos')
|
||||||
run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
|
run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
|
||||||
# FIXME: Only working for Linux
|
# FIXME: Only working for Linux
|
||||||
- name: Python cache
|
- name: Python cache
|
||||||
uses: actions/cache@v1
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ${{ env.PIP_CACHE }}
|
path: ${{ env.PIP_CACHE }}
|
||||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
||||||
restore-keys: |
|
restore-keys: |
|
||||||
${{ runner.os }}-pip-
|
${{ runner.os }}-pip-
|
||||||
- name: Node cache
|
- name: Node cache
|
||||||
uses: actions/cache@v1
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.npm
|
path: ~/.npm
|
||||||
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||||
restore-keys: |
|
restore-keys: |
|
||||||
${{ runner.os }}-node-
|
${{ runner.os }}-node-
|
||||||
- name: Brew cache (MacOS)
|
- name: Brew cache (MacOS)
|
||||||
if: startsWith(matrix.os, 'macos')
|
if: startsWith(matrix.os, 'macos')
|
||||||
uses: actions/cache@v1
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ${{ env.BREW_CACHE }}
|
path: ${{ env.BREW_CACHE }}
|
||||||
key: ${{ runner.os }}-brew-
|
key: ${{ runner.os }}-brew-
|
||||||
- name: Install dependencies (Ubuntu Linux)
|
- name: Install dependencies (Ubuntu Linux)
|
||||||
if: startsWith(matrix.os, 'ubuntu')
|
if: startsWith(matrix.os, 'ubuntu')
|
||||||
run: |
|
run: |
|
||||||
sudo apt-get update
|
sudo apt-get update
|
||||||
sudo apt-get install -y libhdf5-serial-dev
|
sudo apt-get install -y libhdf5-serial-dev
|
||||||
- name: Install dependencies (MacOS)
|
- name: Install dependencies (MacOS)
|
||||||
if: startsWith(matrix.os, 'macos')
|
if: startsWith(matrix.os, 'macos')
|
||||||
run: brew install hdf5
|
run: brew install hdf5
|
||||||
- name: Install cellxgene from `main` branch
|
- name: Install cellxgene from `main` branch
|
||||||
if: matrix.cellxgene_build == 'main'
|
if: matrix.cellxgene_build == 'main'
|
||||||
run: |
|
run: |
|
||||||
pip install -r server/requirements-dev.txt
|
pip install -r server/requirements-dev.txt
|
||||||
make pydist install-dist
|
make pydist install-dist
|
||||||
- name: Install cellxgene from latest release (pypi.org)
|
- name: Install cellxgene from latest release (pypi.org)
|
||||||
if: matrix.cellxgene_build == 'latest'
|
if: matrix.cellxgene_build == 'latest'
|
||||||
run: |
|
run: |
|
||||||
pip install --upgrade cellxgene
|
pip install --upgrade cellxgene
|
||||||
# install the additional dev requirements on top of what is in the
|
# install the additional dev requirements on top of what is in the
|
||||||
# cellxgene pip package, which are needed for testing, but otherwise
|
# cellxgene pip package, which are needed for testing, but otherwise
|
||||||
# keep same pip pkg versions as in the cxg release
|
# keep same pip pkg versions as in the cxg release
|
||||||
sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
|
sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
|
||||||
pip install -r server/requirements-dev.txt
|
pip install -r server/requirements-dev.txt
|
||||||
- name: Install anndata version per matrix variable
|
pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas flatbuffers==2.0.7
|
||||||
run: pip install anndata${{ matrix.anndata_version }}
|
- name: Install anndata version per matrix variable
|
||||||
- name: Install node
|
run: pip install anndata${{ matrix.anndata_version }}
|
||||||
run: make dev-env-client
|
- name: Install node
|
||||||
# Run different types of test separately, to facilitate troubleshooting
|
run: make dev-env-client
|
||||||
- name: Unit Tests - client
|
# Run different types of test separately, to facilitate troubleshooting
|
||||||
run: make unit-test-client
|
- name: Unit Tests - client
|
||||||
- name: Unit Tests - server
|
run: make unit-test-client
|
||||||
run: make unit-test-server
|
- name: Unit Tests - server
|
||||||
- name: Smoke Tests
|
run: make unit-test-server
|
||||||
run: make smoke-test
|
- name: Smoke Tests
|
||||||
|
run: make smoke-test
|
||||||
# FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
|
# FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
|
||||||
# - name: Smoke Tests with Annotations
|
# - name: Smoke Tests with Annotations
|
||||||
# run: make smoke-test-annotations
|
# run: make smoke-test-annotations
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
|
||||||
|
name: "Lint PR commit message"
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request_target:
|
||||||
|
types:
|
||||||
|
- opened
|
||||||
|
- edited
|
||||||
|
- synchronize
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
main:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: amannn/action-semantic-pull-request@v3.4.1
|
||||||
|
with:
|
||||||
|
validateSingleCommit: true
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
@@ -14,15 +14,15 @@ jobs:
|
|||||||
lint:
|
lint:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v4
|
||||||
- run: |
|
- run: |
|
||||||
git fetch --depth=1 origin +${{github.base_ref}}
|
git fetch --depth=1 origin +${{github.base_ref}}
|
||||||
- name: Set up Python 3.7
|
- name: Set up Python 3.12
|
||||||
uses: actions/setup-python@v4
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: 3.7
|
python-version: 3.12
|
||||||
- name: Node cache
|
- name: Node cache
|
||||||
uses: actions/cache@v1
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.npm
|
path: ~/.npm
|
||||||
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||||
@@ -45,22 +45,22 @@ jobs:
|
|||||||
unit-test:
|
unit-test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v4
|
||||||
- name: Set up Python 3.7 (pyenv) # pyenv needed for mlflow in cli annotate tests
|
- name: Set up Python 3.12 (pyenv) # pyenv needed for mlflow in cli annotate tests
|
||||||
uses: gabrielfalcao/pyenv-action@v9
|
uses: gabrielfalcao/pyenv-action@v9
|
||||||
with:
|
with:
|
||||||
default: 3.7
|
default: 3.12
|
||||||
command: pip install -U pip # upgrade pip after installing python
|
command: pip install -U pip # upgrade pip after installing python
|
||||||
- run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
|
- run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
|
||||||
- name: Python cache
|
- name: Python cache
|
||||||
uses: actions/cache@v1
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.cache/pip
|
path: ~/.cache/pip
|
||||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
||||||
restore-keys: |
|
restore-keys: |
|
||||||
${{ runner.os }}-pip-
|
${{ runner.os }}-pip-
|
||||||
- name: Node cache
|
- name: Node cache
|
||||||
uses: actions/cache@v1
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.npm
|
path: ~/.npm
|
||||||
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||||
@@ -69,40 +69,51 @@ jobs:
|
|||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: make pydist install-dist dev-env-server
|
run: make pydist install-dist dev-env-server
|
||||||
- name: Unit tests
|
- name: Unit tests
|
||||||
run: |
|
run: make unit-test-server unit-test-client
|
||||||
make unit-test-server unit-test-client
|
- name: Generate server coverage XML
|
||||||
bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF server,python,unitTest
|
run: coverage xml -o server/coverage.xml
|
||||||
cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
|
- name: Upload server coverage
|
||||||
|
uses: codecov/codecov-action@v5
|
||||||
|
with:
|
||||||
|
flags: server,python,unitTest
|
||||||
|
files: ./server/coverage.xml
|
||||||
|
fail_ci_if_error: false
|
||||||
|
- name: Upload client coverage
|
||||||
|
uses: codecov/codecov-action@v5
|
||||||
|
with:
|
||||||
|
flags: frontend,javascript,unitTest
|
||||||
|
files: ./client/coverage/lcov.info
|
||||||
|
fail_ci_if_error: false
|
||||||
|
|
||||||
smoke-tests:
|
smoke-tests:
|
||||||
runs-on: macos-latest
|
runs-on: macos-latest
|
||||||
timeout-minutes: 20
|
timeout-minutes: 20
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- uses: actions/checkout@v4
|
||||||
- name: Set up Python 3.7
|
- name: Set up Python 3.12
|
||||||
uses: actions/setup-python@v4
|
uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: 3.7
|
python-version: 3.12
|
||||||
- name: Python cache
|
- name: Python cache
|
||||||
uses: actions/cache@v1
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.cache/pip
|
path: ~/.cache/pip
|
||||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
||||||
restore-keys: |
|
restore-keys: |
|
||||||
${{ runner.os }}-pip-
|
${{ runner.os }}-pip-
|
||||||
- name: Node cache
|
- name: Node cache
|
||||||
uses: actions/cache@v1
|
uses: actions/cache@v4
|
||||||
with:
|
with:
|
||||||
path: ~/.npm
|
path: ~/.npm
|
||||||
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||||
restore-keys: |
|
restore-keys: |
|
||||||
${{ runner.os }}-node-
|
${{ runner.os }}-node-
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: make pydist install-dist
|
|
||||||
- name: Smoke tests (without annotations feature)
|
|
||||||
run: |
|
run: |
|
||||||
cd client && make smoke-test
|
pip install setuptools
|
||||||
./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
|
make pydist install-dist
|
||||||
|
- name: Smoke tests (without annotations feature)
|
||||||
|
run: cd client && make smoke-test
|
||||||
|
|
||||||
# TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
|
# TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
|
||||||
# smoke-tests-annotations:
|
# smoke-tests-annotations:
|
||||||
@@ -110,10 +121,10 @@ jobs:
|
|||||||
# timeout-minutes: 20
|
# timeout-minutes: 20
|
||||||
# steps:
|
# steps:
|
||||||
# - uses: actions/checkout@v2
|
# - uses: actions/checkout@v2
|
||||||
# - name: Set up Python 3.7
|
# - name: Set up Python 3.9
|
||||||
# uses: actions/setup-python@v4
|
# uses: actions/setup-python@v4
|
||||||
# with:
|
# with:
|
||||||
# python-version: 3.7
|
# python-version: 3.9
|
||||||
# - name: Python cache
|
# - name: Python cache
|
||||||
# uses: actions/cache@v1
|
# uses: actions/cache@v1
|
||||||
# with:
|
# with:
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
The MIT License (MIT)
|
# The MIT License (MIT)
|
||||||
|
|
||||||
Copyright (c) 2017-2022 Chan Zuckerberg Initiative
|
Copyright (c) 2017-2026 Chan Zuckerberg Initiative
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
this software and associated documentation files (the "Software"), to deal in
|
this software and associated documentation files (the "Software"), to deal in
|
||||||
@@ -7,27 +7,27 @@ _an interactive explorer for single-cell transcriptomics data_
|
|||||||
[](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
|
[](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
|
||||||

|

|
||||||
|
|
||||||
cellxgene Desktop (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
|
CZ CELLxGENE Annotate (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
|
||||||
|
|
||||||
Whether you need to visualize one thousand cells or one million, cellxgene Desktop helps you gain insight into your single-cell data.
|
Whether you need to visualize one thousand cells or one million, CELLxGENE Annotate helps you gain insight into your single-cell data.
|
||||||
|
|
||||||
<img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
|
<img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
|
||||||
|
|
||||||
# Getting started
|
# Getting started
|
||||||
|
|
||||||
### The comprehensive guide to cellxgene Desktop
|
### The comprehensive guide to CZ CELLxGENE Annotate
|
||||||
|
|
||||||
[The cellxgene documentation is your one-stop-shop for information about cellxgene Desktop](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
|
[The CZ CELLxGENE Annotate documentation is your one-stop-shop for information about CELLxGENE Annotate](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
|
||||||
|
|
||||||
- Seeing [what cellxgene Desktop can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
|
- Seeing [what Annotate can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
|
||||||
- Learning more about cellxgene [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
|
- Learning more about Annotate [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
|
||||||
- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in cellxgene Desktop
|
- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in Annotate
|
||||||
- Checking out [our roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) for future development
|
- Checking out [our roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) for future development
|
||||||
- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to cellxgene Desktop
|
- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to Annotate
|
||||||
|
|
||||||
### Quick start
|
### Quick start
|
||||||
|
|
||||||
To install cellxgene Desktop you need Python 3.6+. We recommend [installing cellxgene Desktop into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
|
To install CELLxGENE Annotate you need Python 3.10+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
|
||||||
|
|
||||||
Install the package.
|
Install the package.
|
||||||
|
|
||||||
@@ -35,19 +35,19 @@ Install the package.
|
|||||||
pip install cellxgene
|
pip install cellxgene
|
||||||
```
|
```
|
||||||
|
|
||||||
Launch cellxgene Desktop with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
|
Launch Annotate with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
|
cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
|
||||||
```
|
```
|
||||||
|
|
||||||
To explore more datasets already formatted for cellxgene Desktop, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
|
To explore more datasets already formatted for Annotate, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
|
||||||
see [Preparing your data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) to learn more about formatting your own
|
see [Preparing your data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) to learn more about formatting your own
|
||||||
data for cellxgene Desktop.
|
data for CELLxGENE Annotate.
|
||||||
|
|
||||||
### Supported browsers
|
### Supported browsers
|
||||||
|
|
||||||
cellxgene Desktop currently supports the following browsers:
|
CELLxGENE Annotate currently supports the following browsers:
|
||||||
|
|
||||||
- Google Chrome 61+
|
- Google Chrome 61+
|
||||||
- Edge 15+
|
- Edge 15+
|
||||||
@@ -58,30 +58,36 @@ Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/ch
|
|||||||
### Finding help
|
### Finding help
|
||||||
|
|
||||||
We'd love to hear from you!
|
We'd love to hear from you!
|
||||||
For questions, suggestions, or accolades, [join the `#cellxgene-users` channel on the CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and say "hi!".
|
For questions, suggestions, or accolades, join the `#cellxgene-users` channel on the [CZI Science Community Slack](https://czi.co/science-slack) and say "hi!".
|
||||||
|
|
||||||
For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
|
For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
|
||||||
|
|
||||||
# Developing with cellxgene Desktop
|
# Developing with CZ CELLxGENE Annotate
|
||||||
|
|
||||||
### Contributing
|
### Contributing
|
||||||
|
|
||||||
We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve cellxgene Desktop. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics.
|
We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics.
|
||||||
|
|
||||||
This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
|
This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
|
||||||
|
|
||||||
### Reuse
|
### Reuse
|
||||||
|
|
||||||
This project was started with the sole goal of empowering the scientific community to explore and understand their data.
|
This project was started with the sole goal of empowering the scientific community to explore and understand their data.
|
||||||
As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from
|
As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from
|
||||||
this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
|
this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
|
||||||
|
|
||||||
|
Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an
|
||||||
|
extension could be directly contributed, which would make it available for a wider audience, or that it's on our
|
||||||
|
[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development.
|
||||||
|
|
||||||
Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an
|
See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions.
|
||||||
extension could be directly contributed, which would make it available for a wider audience, or that it's on our
|
|
||||||
[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development.
|
### Trademarks
|
||||||
|
|
||||||
|
CZ CELLXGENE, CZ CELLXGENE DISCOVER, and CZ CELLXGENE ANNOTATE are trademarks of the Chan Zuckerberg Initiative. All rights reserved.
|
||||||
|
|
||||||
|
Use, reuse, modification, and re-distribution of the source code in this repository is subject to the terms of the applicable open source [license](LICENSE.txt). However, that license does not grant permission to use the trademarks without separate, express permission from the Chan Zuckerberg Initiative.
|
||||||
|
|
||||||
See the [cellxgene extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and cellxgene extensions.
|
|
||||||
|
|
||||||
### Security
|
### Security
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
# Reporting Security Issues
|
||||||
|
|
||||||
|
If you believe you have found a security issue, please responsibly disclose by contacting us at [security@chanzuckerberg.com](mailto:security@chanzuckerberg.com).
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
18.17.0
|
||||||
@@ -13,7 +13,7 @@ import * as ENV_DEFAULT from "../../../environment.default.json";
|
|||||||
// a test can take more time to finish, so we don't want
|
// a test can take more time to finish, so we don't want
|
||||||
// jest to shut off the test too soon
|
// jest to shut off the test too soon
|
||||||
jest.setTimeout(2 * 60 * 1000);
|
jest.setTimeout(2 * 60 * 1000);
|
||||||
setDefaultOptions({ timeout: 20 * 1000 });
|
setDefaultOptions({ timeout: 60 * 1000 });
|
||||||
|
|
||||||
jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
|
jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
|
||||||
|
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ module.exports = {
|
|||||||
"@babel/plugin-proposal-function-bind",
|
"@babel/plugin-proposal-function-bind",
|
||||||
["@babel/plugin-proposal-decorators", { legacy: true }],
|
["@babel/plugin-proposal-decorators", { legacy: true }],
|
||||||
["@babel/plugin-proposal-class-properties", { loose: true }],
|
["@babel/plugin-proposal-class-properties", { loose: true }],
|
||||||
["@babel/plugin-proposal-private-methods", { loose: true }],
|
["@babel/plugin-transform-private-methods", { loose: true }],
|
||||||
["@babel/plugin-proposal-private-property-in-object", { loose: true }],
|
["@babel/plugin-transform-private-property-in-object", { loose: true }],
|
||||||
"@babel/plugin-proposal-export-namespace-from",
|
"@babel/plugin-proposal-export-namespace-from",
|
||||||
"@babel/plugin-proposal-optional-chaining",
|
"@babel/plugin-proposal-optional-chaining",
|
||||||
"@babel/plugin-proposal-nullish-coalescing-operator",
|
"@babel/plugin-proposal-nullish-coalescing-operator",
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ module.exports = {
|
|||||||
"@babel/plugin-proposal-function-bind",
|
"@babel/plugin-proposal-function-bind",
|
||||||
["@babel/plugin-proposal-decorators", { legacy: true }],
|
["@babel/plugin-proposal-decorators", { legacy: true }],
|
||||||
["@babel/plugin-proposal-class-properties", { loose: true }],
|
["@babel/plugin-proposal-class-properties", { loose: true }],
|
||||||
["@babel/plugin-proposal-private-methods", { loose: true }],
|
["@babel/plugin-transform-private-methods", { loose: true }],
|
||||||
["@babel/plugin-proposal-private-property-in-object", { loose: true }],
|
["@babel/plugin-transform-private-property-in-object", { loose: true }],
|
||||||
"@babel/plugin-proposal-export-namespace-from",
|
"@babel/plugin-proposal-export-namespace-from",
|
||||||
"@babel/plugin-transform-react-constant-elements",
|
"@babel/plugin-transform-react-constant-elements",
|
||||||
"@babel/plugin-transform-runtime",
|
"@babel/plugin-transform-runtime",
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ const sharedConfig = require("./webpack.config.shared");
|
|||||||
const babelOptions = require("../babel/babel.dev");
|
const babelOptions = require("../babel/babel.dev");
|
||||||
|
|
||||||
const fonts = path.resolve("src/fonts");
|
const fonts = path.resolve("src/fonts");
|
||||||
|
const images = path.resolve("src/images");
|
||||||
const nodeModules = path.resolve("node_modules");
|
const nodeModules = path.resolve("node_modules");
|
||||||
|
|
||||||
const devConfig = {
|
const devConfig = {
|
||||||
@@ -28,11 +29,11 @@ const devConfig = {
|
|||||||
{
|
{
|
||||||
test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
|
test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
|
||||||
loader: "file-loader",
|
loader: "file-loader",
|
||||||
include: [nodeModules, fonts],
|
include: [nodeModules, fonts, images],
|
||||||
options: {
|
options: {
|
||||||
name: "static/assets/[name].[ext]",
|
name: "static/assets/[name].[ext]",
|
||||||
// (thuang): This is needed to make sure @font url path is '/static/assets/'
|
// (thuang): This is needed to make sure @font url path is '/static/assets/'
|
||||||
publicPath: "/",
|
publicPath: "..",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ const CspHashPlugin = require("./cspHashPlugin");
|
|||||||
const sharedConfig = require("./webpack.config.shared");
|
const sharedConfig = require("./webpack.config.shared");
|
||||||
|
|
||||||
const fonts = path.resolve("src/fonts");
|
const fonts = path.resolve("src/fonts");
|
||||||
|
const images = path.resolve("src/images");
|
||||||
const nodeModules = path.resolve("node_modules");
|
const nodeModules = path.resolve("node_modules");
|
||||||
|
|
||||||
const prodConfig = {
|
const prodConfig = {
|
||||||
@@ -43,11 +44,11 @@ const prodConfig = {
|
|||||||
{
|
{
|
||||||
test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
|
test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
|
||||||
loader: "file-loader",
|
loader: "file-loader",
|
||||||
include: [nodeModules, fonts],
|
include: [nodeModules, fonts, images],
|
||||||
options: {
|
options: {
|
||||||
name: "static/assets/[name]-[contenthash].[ext]",
|
name: "static/assets/[name]-[contenthash].[ext]",
|
||||||
// (thuang): This is needed to make sure @font url path is '../static/assets/'
|
// (thuang): This is needed to make sure @font url path is '../static/assets/'
|
||||||
publicPath: "static/",
|
publicPath: "..",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
const path = require("path");
|
const path = require("path");
|
||||||
const fs = require("fs");
|
const fs = require("fs");
|
||||||
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
|
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
|
||||||
const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
|
const ObsoleteWebpackPlugin = require("webpack-obsolete-plugin");
|
||||||
|
|
||||||
const src = path.resolve("src");
|
const src = path.resolve("src");
|
||||||
const nodeModules = path.resolve("node_modules");
|
const nodeModules = path.resolve("node_modules");
|
||||||
|
|||||||
+1
-1
@@ -3,7 +3,7 @@
|
|||||||
<head>
|
<head>
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
<title>cell×gene</title>
|
<title>CELL×GENE | Annotate</title>
|
||||||
<style>
|
<style>
|
||||||
html,
|
html,
|
||||||
body,
|
body,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
<head>
|
<head>
|
||||||
<meta charset="utf-8" />
|
<meta charset="utf-8" />
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||||
<title>cell×gene</title>
|
<title>CELL×GENE | Annotate</title>
|
||||||
<style>
|
<style>
|
||||||
html,
|
html,
|
||||||
body,
|
body,
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ const DEFAULT_LAUNCH_CONFIG = {
|
|||||||
headless: !isHeadful,
|
headless: !isHeadful,
|
||||||
args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
|
args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
|
||||||
ignoreHTTPSErrors: true,
|
ignoreHTTPSErrors: true,
|
||||||
|
timeout: 90000,
|
||||||
defaultViewport: {
|
defaultViewport: {
|
||||||
width: 1280,
|
width: 1280,
|
||||||
height: 960,
|
height: 960,
|
||||||
|
|||||||
Generated
+5452
-3205
File diff suppressed because it is too large
Load Diff
+14
-13
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "cellxgene",
|
"name": "cellxgene",
|
||||||
"version": "1.0.1",
|
"version": "1.3.0",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
|
"description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
|
||||||
"repository": "https://github.com/chanzuckerberg/cellxgene",
|
"repository": "https://github.com/chanzuckerberg/cellxgene",
|
||||||
@@ -18,7 +18,8 @@
|
|||||||
},
|
},
|
||||||
"engineStrict": true,
|
"engineStrict": true,
|
||||||
"engines": {
|
"engines": {
|
||||||
"npm": ">=3.0.0"
|
"npm": ">=9.6.7",
|
||||||
|
"node": "^18.17.0"
|
||||||
},
|
},
|
||||||
"eslintConfig": {
|
"eslintConfig": {
|
||||||
"extends": "./configuration/eslint/eslint.js"
|
"extends": "./configuration/eslint/eslint.js"
|
||||||
@@ -77,16 +78,17 @@
|
|||||||
"whatwg-fetch": "^3.2.0"
|
"whatwg-fetch": "^3.2.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@babel/core": "^7.13.16",
|
"@babel/core": "^7.25.2",
|
||||||
"@babel/plugin-proposal-class-properties": "^7.10.4",
|
"@babel/plugin-proposal-class-properties": "^7.10.4",
|
||||||
"@babel/plugin-proposal-decorators": "^7.13.15",
|
"@babel/plugin-proposal-decorators": "^7.13.15",
|
||||||
"@babel/plugin-proposal-export-namespace-from": "^7.10.4",
|
"@babel/plugin-proposal-export-namespace-from": "^7.10.4",
|
||||||
"@babel/plugin-proposal-function-bind": "^7.10.5",
|
"@babel/plugin-proposal-function-bind": "^7.10.5",
|
||||||
"@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
|
"@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
|
||||||
"@babel/plugin-proposal-optional-chaining": "^7.10.4",
|
"@babel/plugin-proposal-optional-chaining": "^7.10.4",
|
||||||
|
"@babel/plugin-transform-private-property-in-object": "^7.22.11",
|
||||||
"@babel/plugin-transform-react-constant-elements": "^7.13.13",
|
"@babel/plugin-transform-react-constant-elements": "^7.13.13",
|
||||||
"@babel/plugin-transform-runtime": "^7.13.15",
|
"@babel/plugin-transform-runtime": "^7.13.15",
|
||||||
"@babel/preset-env": "^7.13.15",
|
"@babel/preset-env": "^7.22.20",
|
||||||
"@babel/preset-react": "^7.13.13",
|
"@babel/preset-react": "^7.13.13",
|
||||||
"@babel/register": "^7.13.16",
|
"@babel/register": "^7.13.16",
|
||||||
"@babel/runtime": "^7.13.16",
|
"@babel/runtime": "^7.13.16",
|
||||||
@@ -99,13 +101,12 @@
|
|||||||
"cheerio": "^1.0.0-rc.6",
|
"cheerio": "^1.0.0-rc.6",
|
||||||
"clean-css": "^5.1.2",
|
"clean-css": "^5.1.2",
|
||||||
"clean-webpack-plugin": "^4.0.0-alpha.0",
|
"clean-webpack-plugin": "^4.0.0-alpha.0",
|
||||||
"codecov": "^3.7.1",
|
|
||||||
"css-loader": "^5.2.4",
|
"css-loader": "^5.2.4",
|
||||||
"css-minimizer-webpack-plugin": "^4.0.0",
|
"css-minimizer-webpack-plugin": "^4.0.0",
|
||||||
"eslint": "^7.24.0",
|
"eslint": "^7.24.0",
|
||||||
"eslint-config-airbnb": "^18.2.0",
|
"eslint-config-airbnb": "^18.2.0",
|
||||||
"eslint-config-prettier": "^8.2.0",
|
"eslint-config-prettier": "^8.2.0",
|
||||||
"eslint-plugin-compat": "^3.8.0",
|
"eslint-plugin-compat": "^4.2.0",
|
||||||
"eslint-plugin-eslint-comments": "^3.2.0",
|
"eslint-plugin-eslint-comments": "^3.2.0",
|
||||||
"eslint-plugin-filenames": "^1.3.2",
|
"eslint-plugin-filenames": "^1.3.2",
|
||||||
"eslint-plugin-import": "^2.24.2",
|
"eslint-plugin-import": "^2.24.2",
|
||||||
@@ -122,7 +123,7 @@
|
|||||||
"jest-circus": "^27.0.6",
|
"jest-circus": "^27.0.6",
|
||||||
"jest-environment-puppeteer": "^5.0.1",
|
"jest-environment-puppeteer": "^5.0.1",
|
||||||
"jest-fetch-mock": "^3.0.3",
|
"jest-fetch-mock": "^3.0.3",
|
||||||
"jest-puppeteer": "^5.0.1",
|
"jest-puppeteer": "^6.2.0",
|
||||||
"json-loader": "^0.5.7",
|
"json-loader": "^0.5.7",
|
||||||
"lint-staged": "^10.2.11",
|
"lint-staged": "^10.2.11",
|
||||||
"lodash": "^4.17.21",
|
"lodash": "^4.17.21",
|
||||||
@@ -132,16 +133,16 @@
|
|||||||
"lodash.map": "^4.6.0",
|
"lodash.map": "^4.6.0",
|
||||||
"lodash.zip": "^4.2.0",
|
"lodash.zip": "^4.2.0",
|
||||||
"mini-css-extract-plugin": "^1.5.0",
|
"mini-css-extract-plugin": "^1.5.0",
|
||||||
"obsolete-webpack-plugin": "^0.5.6",
|
|
||||||
"prettier": "^2.0.5",
|
"prettier": "^2.0.5",
|
||||||
"puppeteer": "^8.0.0",
|
"puppeteer": "^10.4.0",
|
||||||
"rimraf": "^3.0.2",
|
"rimraf": "^3.0.2",
|
||||||
"serve-favicon": "^2.5.0",
|
"serve-favicon": "^2.5.0",
|
||||||
"terser-webpack-plugin": "^5.1.1",
|
"terser-webpack-plugin": "^5.1.1",
|
||||||
"webpack": "^5.34.0",
|
"webpack": "^5.94.0",
|
||||||
"webpack-cli": "^4.6.0",
|
"webpack-cli": "^4.6.0",
|
||||||
"webpack-dev-middleware": "^4.1.0",
|
"webpack-dev-middleware": "^4.1.0",
|
||||||
"webpack-merge": "^5.0.9"
|
"webpack-merge": "^5.0.9",
|
||||||
|
"webpack-obsolete-plugin": "^1.0.5"
|
||||||
},
|
},
|
||||||
"jest": {
|
"jest": {
|
||||||
"testMatch": [
|
"testMatch": [
|
||||||
@@ -175,13 +176,13 @@
|
|||||||
}
|
}
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
"@babel/plugin-proposal-private-methods",
|
"@babel/plugin-transform-private-methods",
|
||||||
{
|
{
|
||||||
"loose": true
|
"loose": true
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
"@babel/plugin-proposal-private-property-in-object",
|
"@babel/plugin-transform-private-property-in-object",
|
||||||
{
|
{
|
||||||
"loose": true
|
"loose": true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ import { _getColumnDimensionNames } from "./schema";
|
|||||||
import { _hashStringValues } from "./query";
|
import { _hashStringValues } from "./query";
|
||||||
|
|
||||||
export function _whereCacheGet(whereCache, schema, field, query) {
|
export function _whereCacheGet(whereCache, schema, field, query) {
|
||||||
/*
|
/*
|
||||||
query will either be an where query (object) or a column name (string).
|
query will either be an where query (object) or a column name (string).
|
||||||
|
|
||||||
Return array of column labels or undefined.
|
Return array of column labels or undefined.
|
||||||
@@ -169,5 +169,6 @@ function __whereCacheMerge(dst, src) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function _whereCacheMerge(...caches) {
|
export function _whereCacheMerge(...caches) {
|
||||||
|
// eslint-disable-next-line compat/compat -- not using web APIs
|
||||||
return caches.reduce(__whereCacheMerge, {});
|
return caches.reduce(__whereCacheMerge, {});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ class App extends React.Component {
|
|||||||
const { loading, error, graphRenderCounter } = this.props;
|
const { loading, error, graphRenderCounter } = this.props;
|
||||||
return (
|
return (
|
||||||
<Container>
|
<Container>
|
||||||
<Helmet title="cellxgene" />
|
<Helmet title="CELL×GENE | Annotate" />
|
||||||
{loading ? (
|
{loading ? (
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
|
|||||||
@@ -1,17 +1,15 @@
|
|||||||
import React from "react";
|
import React from "react";
|
||||||
import * as globals from "../../globals";
|
import icon from "../../images/icon.png";
|
||||||
|
|
||||||
const Logo = (props) => {
|
const Logo = (props) => {
|
||||||
const { size } = props;
|
const { size } = props;
|
||||||
return (
|
return (
|
||||||
<svg width={size} height={size} viewBox="0 0 48 48" fill="none">
|
<img
|
||||||
<rect width="48" height="48" fill="white" />
|
src={icon}
|
||||||
<rect width="48" height="48" fill={globals.logoColor} />
|
height={size}
|
||||||
<rect x="19" y="19" width="22" height="22" fill="white" />
|
width={size}
|
||||||
<rect x="24" y="24" width="12" height="12" fill={globals.logoColor} />
|
alt="CELLxGENE Annotate Logo"
|
||||||
<rect x="7" y="19" width="7" height="22" fill="white" />
|
/>
|
||||||
<rect x="19" y="7" width="22" height="7" fill="white" />
|
|
||||||
</svg>
|
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -150,7 +150,7 @@ class CentroidLabels extends PureComponent {
|
|||||||
dilatedValue={dilatedValue}
|
dilatedValue={dilatedValue}
|
||||||
coords={coords}
|
coords={coords}
|
||||||
inverseTransform={inverseTransform}
|
inverseTransform={inverseTransform}
|
||||||
opactity={selected ? 1 : deselectOpacity}
|
opacity={selected ? 1 : deselectOpacity}
|
||||||
colorAccessor={colorAccessor}
|
colorAccessor={colorAccessor}
|
||||||
displayLabel={displayLabel}
|
displayLabel={displayLabel}
|
||||||
onMouseEnter={this.handleMouseEnter}
|
onMouseEnter={this.handleMouseEnter}
|
||||||
@@ -205,7 +205,7 @@ const Label = ({
|
|||||||
fontWeight,
|
fontWeight,
|
||||||
fill: "black",
|
fill: "black",
|
||||||
userSelect: "none",
|
userSelect: "none",
|
||||||
opacity: { opacity },
|
opacity,
|
||||||
}}
|
}}
|
||||||
onMouseEnter={(e) => onMouseEnter(e, colorAccessor, label)}
|
onMouseEnter={(e) => onMouseEnter(e, colorAccessor, label)}
|
||||||
onMouseOut={(e) => onMouseOut(e, colorAccessor, label)}
|
onMouseOut={(e) => onMouseOut(e, colorAccessor, label)}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ const InformationMenu = React.memo((props) => {
|
|||||||
rel="noopener"
|
rel="noopener"
|
||||||
/>
|
/>
|
||||||
<MenuItem
|
<MenuItem
|
||||||
href="https://join-cellxgene-users.herokuapp.com/"
|
href="https://czi.co/science-slack"
|
||||||
target="_blank"
|
target="_blank"
|
||||||
icon="chat"
|
icon="chat"
|
||||||
text="Chat"
|
text="Chat"
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 3.1 KiB |
@@ -3,7 +3,7 @@
|
|||||||
## Requirements
|
## Requirements
|
||||||
|
|
||||||
- npm
|
- npm
|
||||||
- Python 3.6+
|
- Python 3.10+
|
||||||
- Chrome
|
- Chrome
|
||||||
|
|
||||||
[See dev section of README](../README.md)
|
[See dev section of README](../README.md)
|
||||||
@@ -148,6 +148,6 @@ If you would like to run the smoke tests against a hot-reloaded version of the c
|
|||||||
|
|
||||||
### Tips
|
### Tips
|
||||||
|
|
||||||
- You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) with the `scripts/backend_dev` script.
|
- You can also install/launch the server side code from npm scrips (requires python3.10 with virtualenv) with the `scripts/backend_dev` script.
|
||||||
|
|
||||||
- Check out [e2e Tests](e2e_tests.md) for more details
|
- Check out [e2e Tests](e2e_tests.md) for more details
|
||||||
|
|||||||
@@ -11,9 +11,10 @@ $PROJECT_ROOT`.
|
|||||||
### Build
|
### Build
|
||||||
|
|
||||||
**Usage:** from the `$PROJECT_ROOT` directory run:
|
**Usage:** from the `$PROJECT_ROOT` directory run:
|
||||||
* `make build` builds whole app client and server
|
|
||||||
* `make build-client` runs webpack build
|
- `make build` builds whole app client and server
|
||||||
* `make build-for-server-dev` builds client and copies output directly into
|
- `make build-client` runs webpack build
|
||||||
|
- `make build-for-server-dev` builds client and copies output directly into
|
||||||
source tree (only for server devlopment)
|
source tree (only for server devlopment)
|
||||||
|
|
||||||
### Clean
|
### Clean
|
||||||
@@ -21,17 +22,19 @@ $PROJECT_ROOT`.
|
|||||||
Deletes generated files.
|
Deletes generated files.
|
||||||
|
|
||||||
**Usage:** from the `$PROJECT_ROOT` directory run:
|
**Usage:** from the `$PROJECT_ROOT` directory run:
|
||||||
* `make clean` cleans everything including node modules (means build with take
|
|
||||||
|
- `make clean` cleans everything including node modules (means build with take
|
||||||
a while
|
a while
|
||||||
* `make clean-lite` cleans built directories
|
- `make clean-lite` cleans built directories
|
||||||
* `make clean-server` cleans source tree
|
- `make clean-server` cleans source tree
|
||||||
|
|
||||||
### Distribution
|
### Distribution
|
||||||
|
|
||||||
Creates distribution for python module to upload to pypi.
|
Creates distribution for python module to upload to pypi.
|
||||||
|
|
||||||
**Usage:** from the `$PROJECT_ROOT` directory run:
|
**Usage:** from the `$PROJECT_ROOT` directory run:
|
||||||
* `make pydist` builds code and then builds sdist
|
|
||||||
|
- `make pydist` builds code and then builds sdist
|
||||||
|
|
||||||
### Release
|
### Release
|
||||||
|
|
||||||
@@ -42,16 +45,18 @@ See `release_process.md`.
|
|||||||
Installs requirements files.
|
Installs requirements files.
|
||||||
|
|
||||||
**Usage:** from the `$PROJECT_ROOT` directory run:
|
**Usage:** from the `$PROJECT_ROOT` directory run:
|
||||||
* `make dev-env` installs requirements and requirments-dev (for building code)
|
|
||||||
|
- `make dev-env` installs requirements and requirments-dev (for building code)
|
||||||
|
|
||||||
### Installing cellxgene packages
|
### Installing cellxgene packages
|
||||||
|
|
||||||
**Usage:** from the `$PROJECT_ROOT` directory:
|
**Usage:** from the `$PROJECT_ROOT` directory:
|
||||||
* `install-dev` - installs from local source tree
|
|
||||||
* `install-release-test` - installs from test pypi
|
- `install-dev` - installs from local source tree
|
||||||
* `install-release` - installs from pypi
|
- `install-release-test` - installs from test pypi
|
||||||
* `install-dist` - installs from local dist folder
|
- `install-release` - installs from pypi
|
||||||
* `uninstall` - uninstalls cellxgene
|
- `install-dist` - installs from local dist folder
|
||||||
|
- `uninstall` - uninstalls cellxgene
|
||||||
|
|
||||||
## Client-level scripts
|
## Client-level scripts
|
||||||
|
|
||||||
@@ -62,8 +67,9 @@ Installs requirements files.
|
|||||||
**About** Serve the current client javascript independently from the `server` code.
|
**About** Serve the current client javascript independently from the `server` code.
|
||||||
|
|
||||||
**Requires**
|
**Requires**
|
||||||
* The server to be running. Best way to do this is with [backend_dev](#backend_dev).
|
|
||||||
* `make ci` to install the necessary node modules
|
- The server to be running. Best way to do this is with [backend_dev](#backend_dev).
|
||||||
|
- `make ci` to install the necessary node modules
|
||||||
|
|
||||||
**Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend`
|
**Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend`
|
||||||
|
|
||||||
@@ -75,23 +81,24 @@ the FE developer gets the current version of the backend with a single command
|
|||||||
and no knowledge of python necessary. It creates and activates a virtual
|
and no knowledge of python necessary. It creates and activates a virtual
|
||||||
environment and installs cellxgene from the current branch.
|
environment and installs cellxgene from the current branch.
|
||||||
|
|
||||||
**Requires** `Python3.6+`, `virtual-env`, `pip`
|
**Requires** `Python3.10+`, `virtual-env`, `pip`
|
||||||
|
|
||||||
**Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev`
|
**Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev`
|
||||||
|
|
||||||
**Options:**
|
**Options:**
|
||||||
* In parallel, you can then launch the node development server to serve the
|
|
||||||
|
- In parallel, you can then launch the node development server to serve the
|
||||||
current state of the FE with [`start-frontend`](#start-frontend), usually in
|
current state of the FE with [`start-frontend`](#start-frontend), usually in
|
||||||
a different terminal tab.
|
a different terminal tab.
|
||||||
* You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
|
- You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
|
||||||
* You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
|
- You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
|
||||||
command, as in `CXG_OPTIONS='--disable-annotations' ./scripts/backend_dev`.
|
command, as in `CXG_OPTIONS='--disable-annotations' ./scripts/backend_dev`.
|
||||||
|
|
||||||
**Breakdown**
|
**Breakdown**
|
||||||
|
|
||||||
| command | purpose |
|
| command | purpose |
|
||||||
| ---------------------------------------- | ---------------------------------------------------------- |
|
| ---------------------------------------- | ---------------------------------------------------------- |
|
||||||
| python3.6 -m venv cellxgene | creates cellxgene virtual environment |
|
| python3.12 -m venv cellxgene | creates cellxgene virtual environment |
|
||||||
| source cellxgene/bin/activate | activates virtual environment |
|
| source cellxgene/bin/activate | activates virtual environment |
|
||||||
| yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed) |
|
| yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed) |
|
||||||
| pip install -e . | installs current local version of cellxgene |
|
| pip install -e . | installs current local version of cellxgene |
|
||||||
@@ -102,14 +109,15 @@ environment and installs cellxgene from the current branch.
|
|||||||
Methods used to test the client javascript code
|
Methods used to test the client javascript code
|
||||||
|
|
||||||
**Usage:** from the `$PROJECT_ROOT/client` directory run:
|
**Usage:** from the `$PROJECT_ROOT/client` directory run:
|
||||||
* `make unit-test` Runs all unit tests. It excludes any tests in the e2e
|
|
||||||
|
- `make unit-test` Runs all unit tests. It excludes any tests in the e2e
|
||||||
folder. This is used by travis to run unit tests.
|
folder. This is used by travis to run unit tests.
|
||||||
* `make smoke-test` Starts backend development server and runs end to end
|
- `make smoke-test` Starts backend development server and runs end to end
|
||||||
tests. This is what travis runs. It depends on the `e2e` and the
|
tests. This is what travis runs. It depends on the `e2e` and the
|
||||||
`backend-dev` targets. One starts the server, the other runs the tests. If
|
`backend-dev` targets. One starts the server, the other runs the tests. If
|
||||||
developing a front-end feature and just checking if tests pass, this is
|
developing a front-end feature and just checking if tests pass, this is
|
||||||
probabaly the one you want to run.
|
probabaly the one you want to run.
|
||||||
* `npm run e2e` Runs backend tests without starting the server. You will need to
|
- `npm run e2e` Runs backend tests without starting the server. You will need to
|
||||||
start the rest api separately with the pbmc3k.h5ad file. Note you can use
|
start the rest api separately with the pbmc3k.h5ad file. Note you can use
|
||||||
the `JEST_ENV` environment variable to change how JEST runs in the browser.
|
the `JEST_ENV` environment variable to change how JEST runs in the browser.
|
||||||
The test runs against `localhost:3000` by default. You can use the
|
The test runs against `localhost:3000` by default. You can use the
|
||||||
|
|||||||
@@ -26,14 +26,14 @@ Steps must be run from the project directory and in a virtual env with all the d
|
|||||||
3. In the release branch, run `make create-release-candidate PART=[major | minor | patch]`. This will bump the version and create a release *candidate* version (e.g. `0.3.0-rc.0`).
|
3. In the release branch, run `make create-release-candidate PART=[major | minor | patch]`. This will bump the version and create a release *candidate* version (e.g. `0.3.0-rc.0`).
|
||||||
4. Commit changes, push the new branch to origin and open a `DO NOT MERGE` draft PR, which will run tests on your branch. We will use this PR later
|
4. Commit changes, push the new branch to origin and open a `DO NOT MERGE` draft PR, which will run tests on your branch. We will use this PR later
|
||||||
5. Upload the release candidate to Test PyPI by running the command `make release-candidate-to-test-pypi`. (Make sure you are registered for PyPI and Test PyPI and you have write access to the cellxgene PyPI package for both).
|
5. Upload the release candidate to Test PyPI by running the command `make release-candidate-to-test-pypi`. (Make sure you are registered for PyPI and Test PyPI and you have write access to the cellxgene PyPI package for both).
|
||||||
6. Verify the release candidate in a fresh virtual environment by running `make install-release-test` which installs the cellxgene build you just uploaded to Test PyPI. The PM should do this too.
|
6. Verify the release candidate in a fresh virtual environment by running `VERSION=<X>.<Y>.<Z>rc.<#> make install-release-test` which installs the cellxgene build you just uploaded to Test PyPI (note that the version value does not include a dash `-`!). The PM should do this too. Note that you may need to run `hash -r` to ensure the cellxgene executable that was just installed is found in your shell path.
|
||||||
7. If you find errors with the release candidate, fix them in main, rebase, and run `make recreate-release-candidate` to increment the release candidate version (i.e. `0.3.0-rc.0` -> `0.3.0-rc.1`). Then go back to Steps 5 and 6 to re-upload and re-test the new release candidate.
|
7. If you find errors with the release candidate, fix them in main, rebase, and run `make recreate-release-candidate` to increment the release candidate version (i.e. `0.3.0-rc.0` -> `0.3.0-rc.1`). Then go back to Steps 5 and 6 to re-upload and re-test the new release candidate.
|
||||||
8. If everything looks good, push the release to Test PyPI without the release candidate tag by running the command `make release-final-to-test-pypi` (i.e. `0.3.0-rc.1` -> `0.3.0`).
|
8. If everything looks good, push the release to Test PyPI without the release candidate tag by running the command `make release-final-to-test-pypi` (i.e. `0.3.0-rc.1` -> `0.3.0`).
|
||||||
- **NOTE:** Once you push the final release version to Test PyPI, you cannot ever re-upload the build again. If you need to make changes to the build, you will have to "burn" the version number and bump the part again and go back to step 1 with a brand new version number. For example, if you upload `0.3.0` to Test PyPI and realize there's a bug, you will have to create a new version `0.4.0` and there will be no `0.3.0` version of cellxgene. This is why testing the release candidate is very important.
|
- **NOTE:** Once you push the final release version to Test PyPI, you cannot ever re-upload the build again. If you need to make changes to the build, you will have to "burn" the version number and bump the part again and go back to step 1 with a brand new version number. For example, if you upload `0.3.0` to Test PyPI and realize there's a bug, you will have to create a new version `0.4.0` and there will be no `0.3.0` version of cellxgene. This is why testing the release candidate is very important.
|
||||||
9. Publish the open draft PR for the release and conduct a PR review.
|
9. Publish the open draft PR for the release and conduct a PR review.
|
||||||
10. Merge to the `main` branch.
|
10. Merge to the `main` branch.
|
||||||
11. Publish to PyPI (prod) (assuming you that you have registered for PyPI, and that you have write access to the cellxgene pypi package) by running `make release-final`.
|
11. Publish to PyPI (prod) (assuming you that you have registered for PyPI, and that you have write access to the cellxgene pypi package) by running `make release-final`.
|
||||||
12. Test the installation in a fresh virtual environment by running `pip install --no-cache-dir cellxgene`.
|
12. Test the installation in a fresh virtual environment by running `pip install --no-cache-dir cellxgene`. Note that you may need to run `hash -r` to ensure the cellxgene executable that was just installed is found in your shell path.
|
||||||
13. Create Github release using the version number and release notes ([instructions](https://help.github.com/articles/creating-releases/)):
|
13. Create Github release using the version number and release notes ([instructions](https://help.github.com/articles/creating-releases/)):
|
||||||
- Draft new release
|
- Draft new release
|
||||||
- Type version name matching release version number from (1)
|
- Type version name matching release version number from (1)
|
||||||
|
|||||||
+1
-1
@@ -2,7 +2,7 @@ import logging
|
|||||||
import sys
|
import sys
|
||||||
from server.common.utils.utils import import_plugins
|
from server.common.utils.utils import import_plugins
|
||||||
|
|
||||||
__version__ = "1.0.1"
|
__version__ = "1.3.0"
|
||||||
display_version = "cellxgene v" + __version__
|
display_version = "cellxgene v" + __version__
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
+3
-3
@@ -48,12 +48,12 @@ def _cache_control(always, **cache_kwargs):
|
|||||||
|
|
||||||
|
|
||||||
def cache_control(**cache_kwargs):
|
def cache_control(**cache_kwargs):
|
||||||
""" config driven """
|
"""config driven"""
|
||||||
return _cache_control(False, **cache_kwargs)
|
return _cache_control(False, **cache_kwargs)
|
||||||
|
|
||||||
|
|
||||||
def cache_control_always(**cache_kwargs):
|
def cache_control_always(**cache_kwargs):
|
||||||
""" always generate headers, regardless of the config """
|
"""always generate headers, regardless of the config"""
|
||||||
return _cache_control(True, **cache_kwargs)
|
return _cache_control(True, **cache_kwargs)
|
||||||
|
|
||||||
|
|
||||||
@@ -228,7 +228,7 @@ def get_api_dataroot_resources(bp_dataroot):
|
|||||||
class Server:
|
class Server:
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _before_adding_routes(app, app_config):
|
def _before_adding_routes(app, app_config):
|
||||||
""" will be called before routes are added, during __init__. Subclass protocol """
|
"""will be called before routes are added, during __init__. Subclass protocol"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def __init__(self, app_config):
|
def __init__(self, app_config):
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ CXGUID = "cxguid"
|
|||||||
|
|
||||||
|
|
||||||
def get_user_id(session: SessionMixin) -> str:
|
def get_user_id(session: SessionMixin) -> str:
|
||||||
""" Gets a session-persistent user id. Creates one in the Flask session if non-extant """
|
"""Gets a session-persistent user id. Creates one in the Flask session if non-extant"""
|
||||||
if CXGUID not in session:
|
if CXGUID not in session:
|
||||||
session[CXGUID] = uuid4().hex
|
session[CXGUID] = uuid4().hex
|
||||||
session.permanent = True
|
session.permanent = True
|
||||||
|
|||||||
+43
-34
@@ -4,7 +4,7 @@ import os.path
|
|||||||
import shlex
|
import shlex
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
from os.path import isfile
|
||||||
from subprocess import STDOUT, PIPE
|
from subprocess import STDOUT, PIPE
|
||||||
from tempfile import NamedTemporaryFile
|
from tempfile import NamedTemporaryFile
|
||||||
|
|
||||||
@@ -26,24 +26,41 @@ def annotate_args(func):
|
|||||||
|
|
||||||
|
|
||||||
@sort_options
|
@sort_options
|
||||||
@click.command(
|
@click.command(options_metavar="<options>")
|
||||||
short_help="Annotate H5AD file columns. Run `cellxgene annotation --help` for more information.",
|
@click.argument(
|
||||||
options_metavar="<options>",
|
"input_h5ad_file",
|
||||||
)
|
type=click.Path(exists=True, dir_okay=False, readable=True),
|
||||||
@click.option(
|
nargs=1,
|
||||||
"-i",
|
metavar="<path to H5AD input file>",
|
||||||
"--input-h5ad-file",
|
|
||||||
required=True,
|
required=True,
|
||||||
type=str,
|
|
||||||
help="The input H5AD file containing the missing annotations.",
|
|
||||||
)
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"-m",
|
"-m",
|
||||||
"--model-url",
|
"--model-url",
|
||||||
|
# Making this a required "option", rather than an "argument", since we support automatic model selection in the
|
||||||
|
# future, in which case the user would not need to specify this option at all and we can make it optional at
|
||||||
|
# that time.
|
||||||
required=True,
|
required=True,
|
||||||
help="The URL of the model used to prediction annotated labels. May be a local filesystem directory "
|
help="The URL of the model used to prediction annotated labels. May be a local filesystem directory "
|
||||||
"or S3 path (s3://)",
|
"or S3 path (s3://)",
|
||||||
)
|
)
|
||||||
|
@click.option(
|
||||||
|
"-o",
|
||||||
|
"--output-h5ad-file",
|
||||||
|
default="",
|
||||||
|
help="The output H5AD file that will contain the generated annotation values. If this option is not provided, "
|
||||||
|
"the input file will be overwritten to include the new annotations; in this case you must specify "
|
||||||
|
"--overwrite.",
|
||||||
|
metavar="<filename>",
|
||||||
|
)
|
||||||
|
@click.option(
|
||||||
|
"--overwrite",
|
||||||
|
default=False,
|
||||||
|
is_flag=True,
|
||||||
|
help="Allow overwriting of the specified H5AD output file, if it exists. For safety, you must specify this "
|
||||||
|
"flag if the specified output file already exists or if the --output-h5ad-file option is not provided.",
|
||||||
|
show_default=True,
|
||||||
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"-l",
|
"-l",
|
||||||
"--counts-layer",
|
"--counts-layer",
|
||||||
@@ -53,8 +70,8 @@ def annotate_args(func):
|
|||||||
@click.option(
|
@click.option(
|
||||||
"-g",
|
"-g",
|
||||||
"--gene-column-name",
|
"--gene-column-name",
|
||||||
help="The name of the `var` column that contains gene identifiers. The values in this column will be used to match "
|
help="The name of the `var` column that contains gene names. The values in this column will be used to match "
|
||||||
"genes between the query and reference datasets. If not specified, the gene identifiers are expected to exist "
|
"genes between the query and reference datasets. If not specified, the gene names are expected to exist "
|
||||||
"in `var.index`.",
|
"in `var.index`.",
|
||||||
)
|
)
|
||||||
# TODO: Useful if we want to support discoverability of models
|
# TODO: Useful if we want to support discoverability of models
|
||||||
@@ -91,19 +108,6 @@ def annotate_args(func):
|
|||||||
"will store the predicted annotation values and confidence scores. This can be used to allow multiple "
|
"will store the predicted annotation values and confidence scores. This can be used to allow multiple "
|
||||||
"annotation predictions to be run on a single AnnData object.",
|
"annotation predictions to be run on a single AnnData object.",
|
||||||
)
|
)
|
||||||
@click.option(
|
|
||||||
"-u",
|
|
||||||
"--update-h5ad-file",
|
|
||||||
is_flag=True,
|
|
||||||
help="Flag indicating whether to update the input h5ad file with annotation values. This option is mutually "
|
|
||||||
"exclusive with --output-h5ad-file.",
|
|
||||||
)
|
|
||||||
@click.option(
|
|
||||||
"-o",
|
|
||||||
"--output-h5ad-file",
|
|
||||||
help="The output H5AD file that will contain the generated annotation values. This option is mutually "
|
|
||||||
"exclusive with --update-h5ad-file.",
|
|
||||||
)
|
|
||||||
@click.option("--use-model-cache/--no-use-model-cache", default=True)
|
@click.option("--use-model-cache/--no-use-model-cache", default=True)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--use-gpu/--no-use-gpu",
|
"--use-gpu/--no-use-gpu",
|
||||||
@@ -141,6 +145,9 @@ def annotate_args(func):
|
|||||||
)
|
)
|
||||||
@click.help_option("--help", "-h", help="Show this message and exit.")
|
@click.help_option("--help", "-h", help="Show this message and exit.")
|
||||||
def annotate(**cli_args):
|
def annotate(**cli_args):
|
||||||
|
"""
|
||||||
|
Add predicted annotations to an H5AD file. Run `cellxgene annotate --help` for more information.
|
||||||
|
"""
|
||||||
_validate_options(cli_args)
|
_validate_options(cli_args)
|
||||||
|
|
||||||
print(f"Reading query dataset {cli_args['input_h5ad_file']}...")
|
print(f"Reading query dataset {cli_args['input_h5ad_file']}...")
|
||||||
@@ -149,7 +156,11 @@ def annotate(**cli_args):
|
|||||||
filter(None, [cli_args.get("annotation_prefix"), cli_args.get("annotation_type"), cli_args.get("run_name")])
|
filter(None, [cli_args.get("annotation_prefix"), cli_args.get("annotation_type"), cli_args.get("run_name")])
|
||||||
)
|
)
|
||||||
|
|
||||||
output_h5ad_file = cli_args["input_h5ad_file"] if cli_args["update_h5ad_file"] else cli_args["output_h5ad_file"]
|
output_h5ad_file = (
|
||||||
|
cli_args["input_h5ad_file"]
|
||||||
|
if cli_args["overwrite"] and not cli_args["output_h5ad_file"]
|
||||||
|
else cli_args["output_h5ad_file"]
|
||||||
|
)
|
||||||
|
|
||||||
model_url = cli_args.get("model_url")
|
model_url = cli_args.get("model_url")
|
||||||
local_model_path = _retrieve_model(cli_args.get("model_cache_dir"), model_url, cli_args.get("use_model_cache"))
|
local_model_path = _retrieve_model(cli_args.get("model_cache_dir"), model_url, cli_args.get("use_model_cache"))
|
||||||
@@ -196,7 +207,7 @@ def annotate(**cli_args):
|
|||||||
|
|
||||||
p.wait()
|
p.wait()
|
||||||
if p.returncode == 0:
|
if p.returncode == 0:
|
||||||
print(f"Wrote annotations to {cli_args.get('output_h5ad_file')}")
|
print(f"Wrote annotations to {output_h5ad_file}")
|
||||||
else:
|
else:
|
||||||
print("Annotation failed!")
|
print("Annotation failed!")
|
||||||
else:
|
else:
|
||||||
@@ -218,13 +229,11 @@ def _retrieve_model(model_cache_dir, model_url, use_cache=True):
|
|||||||
|
|
||||||
|
|
||||||
def _validate_options(cli_args):
|
def _validate_options(cli_args):
|
||||||
# TODO(atolopko): Use cloup library for this logic
|
output = cli_args["output_h5ad_file"]
|
||||||
if cli_args["update_h5ad_file"] and cli_args["output_h5ad_file"]:
|
overwrite = cli_args["overwrite"]
|
||||||
click.echo("--update_h5ad_file and --output_h5ad_file are mutually exclusive")
|
|
||||||
sys.exit(1)
|
if isfile(output) and not overwrite:
|
||||||
if not (cli_args["update_h5ad_file"] or cli_args["output_h5ad_file"]):
|
raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
|
||||||
click.echo("--update_h5ad_file or --output_h5ad_file must be specified")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -128,12 +128,12 @@ def prepare(
|
|||||||
raise click.FileError(data, hint="not a valid file or path")
|
raise click.FileError(data, hint="not a valid file or path")
|
||||||
|
|
||||||
if not set_obs_names == "":
|
if not set_obs_names == "":
|
||||||
if set_obs_names not in adata.obs_keys():
|
if set_obs_names not in list(adata.obs.keys()):
|
||||||
raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
|
raise click.UsageError(f"obs {set_obs_names} not found, options are: {list(adata.obs.keys())}")
|
||||||
adata.obs_names = adata.obs[set_obs_names]
|
adata.obs_names = adata.obs[set_obs_names]
|
||||||
if not set_var_names == "":
|
if not set_var_names == "":
|
||||||
if set_var_names not in adata.var_keys():
|
if set_var_names not in list(adata.var.keys()):
|
||||||
raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
|
raise click.UsageError(f"var {set_var_names} not found, options are: {list(adata.var.keys())}")
|
||||||
adata.var_names = adata.var[set_var_names]
|
adata.var_names = adata.var[set_var_names]
|
||||||
if make_obs_names_unique:
|
if make_obs_names_unique:
|
||||||
adata.obs.index = make_index_unique(adata.obs.index)
|
adata.obs.index = make_index_unique(adata.obs.index)
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ class AnnotationsLocalFile(Annotations):
|
|||||||
def write_gene_sets(self, gene_sets, tid, data_adaptor):
|
def write_gene_sets(self, gene_sets, tid, data_adaptor):
|
||||||
self.check_gene_sets_save_enabled() # raises
|
self.check_gene_sets_save_enabled() # raises
|
||||||
|
|
||||||
if type(tid) != int or tid < 0:
|
if type(tid) is not int or tid < 0:
|
||||||
raise ValueError("tid must be a positive integer")
|
raise ValueError("tid must be a positive integer")
|
||||||
|
|
||||||
# may raise
|
# may raise
|
||||||
@@ -175,7 +175,7 @@ class AnnotationsLocalFile(Annotations):
|
|||||||
|
|
||||||
# update the cache
|
# update the cache
|
||||||
self.last_geneset_fname = fname
|
self.last_geneset_fname = fname
|
||||||
self.last_geneset = gene_sets if type(gene_sets) == dict else {g["geneset_name"]: g for g in gene_sets}
|
self.last_geneset = gene_sets if isinstance(gene_sets, dict) else {g["geneset_name"]: g for g in gene_sets}
|
||||||
|
|
||||||
def _get_userdata_idhash(self, data_adaptor):
|
def _get_userdata_idhash(self, data_adaptor):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -228,6 +228,6 @@ def convert_anndata_category_colors_to_cxg_category_colors(data):
|
|||||||
|
|
||||||
# create the cellxgene color entry for this category
|
# create the cellxgene color entry for this category
|
||||||
cxg_colors[category_name] = dict(
|
cxg_colors[category_name] = dict(
|
||||||
zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
|
zip(data.obs[category_name].astype('category').cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
|
||||||
)
|
)
|
||||||
return cxg_colors
|
return cxg_colors
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
|
|||||||
|
|
||||||
# degrees of freedom for Welch's t-test
|
# degrees of freedom for Welch's t-test
|
||||||
with np.errstate(divide="ignore", invalid="ignore"):
|
with np.errstate(divide="ignore", invalid="ignore"):
|
||||||
dof = sum_vn ** 2 / (vnA ** 2 / (nA - 1) + vnB ** 2 / (nB - 1))
|
dof = sum_vn**2 / (vnA**2 / (nA - 1) + vnB**2 / (nB - 1))
|
||||||
dof[np.isnan(dof)] = 1
|
dof[np.isnan(dof)] = 1
|
||||||
|
|
||||||
# Welch's t-test score calculation
|
# Welch's t-test score calculation
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ def estimate_approximate_distribution(X) -> XApproximateDistribution:
|
|||||||
if Xdata.size > CHUNKSIZE:
|
if Xdata.size > CHUNKSIZE:
|
||||||
min_val = max_val = Xdata[0]
|
min_val = max_val = Xdata[0]
|
||||||
with concurrent.futures.ThreadPoolExecutor() as tp:
|
with concurrent.futures.ThreadPoolExecutor() as tp:
|
||||||
for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
|
for _min, _max in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
|
||||||
min_val = min(_min, min_val)
|
min_val = min(_min, min_val)
|
||||||
max_val = max(_max, max_val)
|
max_val = max(_max, max_val)
|
||||||
|
|
||||||
|
|||||||
@@ -1,2 +1,2 @@
|
|||||||
DEFAULT_SERVER_PORT = 5005
|
DEFAULT_SERVER_PORT = 5005
|
||||||
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
|
BIG_FILE_SIZE_THRESHOLD = 100 * 2**20 # 100MB
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ class AppConfig(object):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
|
||||||
# the default configuration (see default_config.py)
|
# the default configuration (see default_config.py)
|
||||||
# TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
|
# TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
|
||||||
# defaults within the config class?
|
# defaults within the config class?
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ class BaseConfig(object):
|
|||||||
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
|
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
if type(val) != vtype:
|
if type(val) is not vtype:
|
||||||
raise ConfigurationError(
|
raise ConfigurationError(
|
||||||
f"Invalid type for attribute: {attrname}, "
|
f"Invalid type for attribute: {attrname}, "
|
||||||
f"expected type {vtype.__name__}, got {type(val).__name__}"
|
f"expected type {vtype.__name__}, got {type(val).__name__}"
|
||||||
@@ -70,7 +70,7 @@ class BaseConfig(object):
|
|||||||
if not hasattr(self, key):
|
if not hasattr(self, key):
|
||||||
raise ConfigurationError(f"unknown config parameter {key}.")
|
raise ConfigurationError(f"unknown config parameter {key}.")
|
||||||
try:
|
try:
|
||||||
if type(value) == tuple:
|
if type(value) is tuple:
|
||||||
# convert tuple values to list values
|
# convert tuple values to list values
|
||||||
value = list(value)
|
value = list(value)
|
||||||
setattr(self, key, value)
|
setattr(self, key, value)
|
||||||
|
|||||||
@@ -176,7 +176,7 @@ class DatasetConfig(BaseConfig):
|
|||||||
self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
|
self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
|
||||||
|
|
||||||
data_adaptor = self.get_data_adaptor()
|
data_adaptor = self.get_data_adaptor()
|
||||||
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
|
if self.diffexp__enable and data_adaptor.parameters.get("diffexp-may-be-slow", False):
|
||||||
context["messagefn"](
|
context["messagefn"](
|
||||||
"CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
|
"CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ class ExternalConfig(BaseConfig):
|
|||||||
if name is None:
|
if name is None:
|
||||||
raise ConfigurationError("environment: 'name' is missing")
|
raise ConfigurationError("environment: 'name' is missing")
|
||||||
required = envdict.get("required", False)
|
required = envdict.get("required", False)
|
||||||
if type(required) != bool:
|
if type(required) is not bool:
|
||||||
raise ConfigurationError("environment: 'required' must be a bool")
|
raise ConfigurationError("environment: 'required' must be a bool")
|
||||||
path = envdict.get("path")
|
path = envdict.get("path")
|
||||||
if path is None:
|
if path is None:
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ def corpora_get_versions_from_anndata(adata):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# per Corpora AnnData spec, this is a corpora file if the following is true
|
# per Corpora AnnData spec, this is a corpora file if the following is true
|
||||||
if "version" not in adata.uns_keys():
|
if "version" not in list(adata.uns.keys()):
|
||||||
return None
|
return None
|
||||||
version = adata.uns["version"]
|
version = adata.uns["version"]
|
||||||
if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
|
if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version:
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import server.common.fbs.NetEncoding.Uint32Array as Uint32Array
|
|||||||
|
|
||||||
# Serialization helper
|
# Serialization helper
|
||||||
def serialize_column(builder, typed_arr):
|
def serialize_column(builder, typed_arr):
|
||||||
""" Serialize NetEncoding.Column """
|
"""Serialize NetEncoding.Column"""
|
||||||
|
|
||||||
(u_type, u_value) = typed_arr
|
(u_type, u_value) = typed_arr
|
||||||
Column.ColumnStart(builder)
|
Column.ColumnStart(builder)
|
||||||
@@ -30,7 +30,7 @@ def serialize_column(builder, typed_arr):
|
|||||||
|
|
||||||
# Serialization helper
|
# Serialization helper
|
||||||
def serialize_matrix(builder, n_rows, n_cols, columns, col_idx):
|
def serialize_matrix(builder, n_rows, n_cols, columns, col_idx):
|
||||||
""" Serialize NetEncoding.Matrix """
|
"""Serialize NetEncoding.Matrix"""
|
||||||
|
|
||||||
Matrix.MatrixStart(builder)
|
Matrix.MatrixStart(builder)
|
||||||
Matrix.MatrixAddNRows(builder, n_rows)
|
Matrix.MatrixAddNRows(builder, n_rows)
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ def write_gene_sets_tidycsv(f, genesets):
|
|||||||
|
|
||||||
|
|
||||||
def summarizeQueryHash(raw_query):
|
def summarizeQueryHash(raw_query):
|
||||||
""" generate a cache key (hash) from the raw query string """
|
"""generate a cache key (hash) from the raw query string"""
|
||||||
return hashlib.sha1(raw_query).hexdigest()
|
return hashlib.sha1(raw_query).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
@@ -187,7 +187,7 @@ def validate_gene_sets(genesets, var_names, context=None):
|
|||||||
# 1. check gene set character set and format
|
# 1. check gene set character set and format
|
||||||
illegal_name = re.compile(r"^\s| |[\u0000-\u001F\u007F-\uFFFF]|\s$")
|
illegal_name = re.compile(r"^\s| |[\u0000-\u001F\u007F-\uFFFF]|\s$")
|
||||||
for name in geneset_names:
|
for name in geneset_names:
|
||||||
if type(name) != str or len(name) == 0:
|
if type(name) is not str or len(name) == 0:
|
||||||
raise KeyError("Gene set names must be non-null string.")
|
raise KeyError("Gene set names must be non-null string.")
|
||||||
if illegal_name.search(name):
|
if illegal_name.search(name):
|
||||||
messagefn(
|
messagefn(
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import zlib
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
from flask import make_response, jsonify, current_app, abort
|
from flask import make_response, jsonify, current_app, abort
|
||||||
from werkzeug.urls import url_unquote
|
from urllib.parse import unquote
|
||||||
|
|
||||||
from server.common.config.client_config import get_client_config
|
from server.common.config.client_config import get_client_config
|
||||||
from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
|
from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
|
||||||
@@ -64,22 +64,22 @@ def _query_parameter_to_filter(args):
|
|||||||
axis, name = key.split(":")
|
axis, name = key.split(":")
|
||||||
if axis not in ("obs", "var"):
|
if axis not in ("obs", "var"):
|
||||||
raise FilterError("unknown filter axis")
|
raise FilterError("unknown filter axis")
|
||||||
name = url_unquote(name)
|
name = unquote(name)
|
||||||
current = filters[axis].setdefault(name, {"name": name})
|
current = filters[axis].setdefault(name, {"name": name})
|
||||||
|
|
||||||
val_split = value.split(",")
|
val_split = value.split(",")
|
||||||
if len(val_split) == 1:
|
if len(val_split) == 1:
|
||||||
if "min" in current or "max" in current:
|
if "min" in current or "max" in current:
|
||||||
raise FilterError("do not mix range and value filters")
|
raise FilterError("do not mix range and value filters")
|
||||||
value = url_unquote(value)
|
value = unquote(value)
|
||||||
values = current.setdefault("values", [])
|
values = current.setdefault("values", [])
|
||||||
values.append(value)
|
values.append(value)
|
||||||
|
|
||||||
elif len(val_split) == 2:
|
elif len(val_split) == 2:
|
||||||
if len(current) > 1:
|
if len(current) > 1:
|
||||||
raise FilterError("duplicate range specification")
|
raise FilterError("duplicate range specification")
|
||||||
min = url_unquote(val_split[0])
|
min = unquote(val_split[0])
|
||||||
max = url_unquote(val_split[1])
|
max = unquote(val_split[1])
|
||||||
if min != "*":
|
if min != "*":
|
||||||
current["min"] = float(min)
|
current["min"] = float(min)
|
||||||
if max != "*":
|
if max != "*":
|
||||||
@@ -379,7 +379,7 @@ def summarize_var_helper(request, data_adaptor, key, raw_query):
|
|||||||
HTTPStatus.OK,
|
HTTPStatus.OK,
|
||||||
{"Content-Type": "application/octet-stream"},
|
{"Content-Type": "application/octet-stream"},
|
||||||
)
|
)
|
||||||
except (ValueError) as e:
|
except ValueError as e:
|
||||||
return abort(HTTPStatus.NOT_FOUND, description=str(e))
|
return abort(HTTPStatus.NOT_FOUND, description=str(e))
|
||||||
except (UnsupportedSummaryMethod, FilterError) as e:
|
except (UnsupportedSummaryMethod, FilterError) as e:
|
||||||
return abort(HTTPStatus.BAD_REQUEST, description=str(e))
|
return abort(HTTPStatus.BAD_REQUEST, description=str(e))
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dt
|
|||||||
raise TypeError("Unsupported data type.")
|
raise TypeError("Unsupported data type.")
|
||||||
|
|
||||||
dtype = array.dtype
|
dtype = array.dtype
|
||||||
|
|
||||||
res = _get_type_info_from_dtype(dtype)
|
res = _get_type_info_from_dtype(dtype)
|
||||||
if res is not None:
|
if res is not None:
|
||||||
return res
|
return res
|
||||||
@@ -140,7 +140,6 @@ def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dt
|
|||||||
|
|
||||||
if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
|
if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
|
||||||
return (np.int32, {"type": "int32"})
|
return (np.int32, {"type": "int32"})
|
||||||
|
|
||||||
if dtype.kind == "f":
|
if dtype.kind == "f":
|
||||||
_float64_warning(array.dtype)
|
_float64_warning(array.dtype)
|
||||||
return (np.float32, {"type": "float32"})
|
return (np.float32, {"type": "float32"})
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import socket
|
|||||||
from urllib.parse import urlsplit, urljoin
|
from urllib.parse import urlsplit, urljoin
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from flask import json
|
import json
|
||||||
|
|
||||||
from server.common.errors import ConfigurationError
|
from server.common.errors import ConfigurationError
|
||||||
|
|
||||||
@@ -98,7 +98,7 @@ def custom_format_warning(msg, *args, **kwargs):
|
|||||||
|
|
||||||
|
|
||||||
def jsonify_strict(data):
|
def jsonify_strict(data):
|
||||||
return json.dumps(data, cls=StrictJSONEncoder, allow_nan=False)
|
return StrictJSONEncoder().encode(data)
|
||||||
|
|
||||||
|
|
||||||
def import_plugins(plugin_module):
|
def import_plugins(plugin_module):
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import warnings
|
import warnings
|
||||||
|
import importlib.metadata
|
||||||
|
|
||||||
import anndata
|
import anndata
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -16,7 +17,7 @@ from server.common.utils.type_conversion_utils import get_schema_type_hint_of_ar
|
|||||||
from server.data_common.data_adaptor import DataAdaptor
|
from server.data_common.data_adaptor import DataAdaptor
|
||||||
from server.common.fbs.matrix import encode_matrix_fbs
|
from server.common.fbs.matrix import encode_matrix_fbs
|
||||||
|
|
||||||
anndata_version = version.parse(str(anndata.__version__)).release
|
anndata_version = version.parse(str(importlib.metadata.version('anndata'))).release
|
||||||
|
|
||||||
|
|
||||||
def anndata_version_is_pre_070():
|
def anndata_version_is_pre_070():
|
||||||
@@ -63,7 +64,7 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
return "cellxgene anndata adaptor version"
|
return "cellxgene anndata adaptor version"
|
||||||
|
|
||||||
def get_library_versions(self):
|
def get_library_versions(self):
|
||||||
return dict(anndata=str(anndata.__version__))
|
return dict(anndata=str(importlib.metadata.version('anndata')))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _create_unique_column_name(df, col_name_prefix):
|
def _create_unique_column_name(df, col_name_prefix):
|
||||||
@@ -92,7 +93,7 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
"""
|
"""
|
||||||
self.original_obs_index = self.data.obs.index
|
self.original_obs_index = self.data.obs.index
|
||||||
|
|
||||||
for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
|
for ax_name, var_name in ((Axis.OBS, "obs"), (Axis.VAR, "var")):
|
||||||
config_name = f"single_dataset__{var_name}_names"
|
config_name = f"single_dataset__{var_name}_names"
|
||||||
parameter_name = f"{var_name}_names"
|
parameter_name = f"{var_name}_names"
|
||||||
name = getattr(self.server_config, config_name)
|
name = getattr(self.server_config, config_name)
|
||||||
@@ -173,12 +174,24 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
)
|
)
|
||||||
except MemoryError:
|
except MemoryError:
|
||||||
raise DatasetAccessError("Out of memory - file is too large for available memory.")
|
raise DatasetAccessError("Out of memory - file is too large for available memory.")
|
||||||
except Exception:
|
except Exception as e:
|
||||||
import traceback
|
import traceback
|
||||||
message = (
|
error_msg = str(e)
|
||||||
"File not found or is inaccessible. File must be an .h5ad object. "
|
|
||||||
"Please check your input and try again."
|
# IMPROVEMENT: Broadly catch ANY version incompatibility
|
||||||
|
if "No read method registered" in error_msg and "IOSpec" in error_msg:
|
||||||
|
message = (
|
||||||
|
"Error loading file: This H5AD file uses a newer internal format that "
|
||||||
|
"your version of 'anndata' cannot read.\n"
|
||||||
|
f"The specific error was: {error_msg}\n"
|
||||||
|
"Please upgrade anndata in your environment (pip install --upgrade anndata)."
|
||||||
)
|
)
|
||||||
|
else:
|
||||||
|
message = (
|
||||||
|
"File not found or is inaccessible. File must be an .h5ad object. "
|
||||||
|
"Please check your input and try again."
|
||||||
|
)
|
||||||
|
|
||||||
if self.server_config.app__verbose:
|
if self.server_config.app__verbose:
|
||||||
message += f"\n{traceback.format_exc()}"
|
message += f"\n{traceback.format_exc()}"
|
||||||
raise DatasetAccessError(message)
|
raise DatasetAccessError(message)
|
||||||
@@ -210,7 +223,7 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
# heuristic
|
# heuristic
|
||||||
n_values = self.data.shape[0] * self.data.shape[1]
|
n_values = self.data.shape[0] * self.data.shape[1]
|
||||||
if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
|
if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
|
||||||
self.parameters.update({"diffexp_may_be_slow": True})
|
self.parameters.update({"diffexp-may-be-slow": True})
|
||||||
|
|
||||||
def _is_valid_layout(self, arr):
|
def _is_valid_layout(self, arr):
|
||||||
"""return True if this layout data is a valid array for front-end presentation:
|
"""return True if this layout data is a valid array for front-end presentation:
|
||||||
@@ -218,7 +231,7 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
* with shape (n_obs, >= 2)
|
* with shape (n_obs, >= 2)
|
||||||
* with all values finite or NaN (no +Inf or -Inf)
|
* with all values finite or NaN (no +Inf or -Inf)
|
||||||
"""
|
"""
|
||||||
is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
|
is_valid = type(arr) is np.ndarray and arr.dtype.kind in "fiu"
|
||||||
is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
|
is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
|
||||||
is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr))
|
is_valid = is_valid and not np.any(np.isinf(arr)) and not np.all(np.isnan(arr))
|
||||||
return is_valid
|
return is_valid
|
||||||
@@ -242,8 +255,10 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
)
|
)
|
||||||
if self.data.X.dtype < np.float32:
|
if self.data.X.dtype < np.float32:
|
||||||
if self.data.isbacked:
|
if self.data.isbacked:
|
||||||
raise DatasetAccessError(f"Data matrix in {self.data.X.dtype} format is not supported in backed mode."
|
raise DatasetAccessError(
|
||||||
" Please reload without --backed, or convert matrix to float32")
|
f"Data matrix in {self.data.X.dtype} format is not supported in backed mode."
|
||||||
|
" Please reload without --backed, or convert matrix to float32"
|
||||||
|
)
|
||||||
warnings.warn(
|
warnings.warn(
|
||||||
f"Anndata data matrix is in unsupported {self.data.X.dtype} format -- will be cast to float32"
|
f"Anndata data matrix is in unsupported {self.data.X.dtype} format -- will be cast to float32"
|
||||||
)
|
)
|
||||||
@@ -299,11 +314,11 @@ class AnndataAdaptor(DataAdaptor):
|
|||||||
layouts = self.dataset_config.embeddings__names
|
layouts = self.dataset_config.embeddings__names
|
||||||
|
|
||||||
if layouts is None or len(layouts) == 0:
|
if layouts is None or len(layouts) == 0:
|
||||||
layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
|
layouts = [key[2:] for key in list(self.data.obsm.keys()) if type(key) is str and key.startswith("X_")]
|
||||||
|
|
||||||
# remove invalid layouts
|
# remove invalid layouts
|
||||||
valid_layouts = []
|
valid_layouts = []
|
||||||
obsm_keys = self.data.obsm_keys()
|
obsm_keys = list(self.data.obsm.keys())
|
||||||
for layout in layouts:
|
for layout in layouts:
|
||||||
layout_name = f"X_{layout}"
|
layout_name = f"X_{layout}"
|
||||||
if layout_name not in obsm_keys:
|
if layout_name not in obsm_keys:
|
||||||
|
|||||||
@@ -154,7 +154,7 @@ class DataAdaptor(metaclass=ABCMeta):
|
|||||||
parameters.update(self.parameters)
|
parameters.update(self.parameters)
|
||||||
|
|
||||||
def _index_filter_to_mask(self, filter, count):
|
def _index_filter_to_mask(self, filter, count):
|
||||||
mask = np.zeros((count,), dtype=np.bool)
|
mask = np.zeros((count,), dtype="bool")
|
||||||
for i in filter:
|
for i in filter:
|
||||||
if isinstance(i, list):
|
if isinstance(i, list):
|
||||||
mask[i[0] : i[1]] = True
|
mask[i[0] : i[1]] = True
|
||||||
@@ -163,7 +163,7 @@ class DataAdaptor(metaclass=ABCMeta):
|
|||||||
return mask
|
return mask
|
||||||
|
|
||||||
def _axis_filter_to_mask(self, axis, filter, count):
|
def _axis_filter_to_mask(self, axis, filter, count):
|
||||||
mask = np.ones((count,), dtype=np.bool)
|
mask = np.ones((count,), dtype="bool")
|
||||||
if "index" in filter:
|
if "index" in filter:
|
||||||
mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count))
|
mask = np.logical_and(mask, self._index_filter_to_mask(filter["index"], count))
|
||||||
if "annotation_value" in filter:
|
if "annotation_value" in filter:
|
||||||
@@ -172,7 +172,7 @@ class DataAdaptor(metaclass=ABCMeta):
|
|||||||
return mask
|
return mask
|
||||||
|
|
||||||
def _annotation_filter_to_mask(self, axis, filter, count):
|
def _annotation_filter_to_mask(self, axis, filter, count):
|
||||||
mask = np.ones((count,), dtype=np.bool)
|
mask = np.ones((count,), dtype="bool")
|
||||||
for v in filter:
|
for v in filter:
|
||||||
name = v["name"]
|
name = v["name"]
|
||||||
if axis == Axis.VAR:
|
if axis == Axis.VAR:
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ class MatrixDataType(Enum):
|
|||||||
|
|
||||||
class MatrixDataLoader(object):
|
class MatrixDataLoader(object):
|
||||||
def __init__(self, location, matrix_data_type=None, app_config=None):
|
def __init__(self, location, matrix_data_type=None, app_config=None):
|
||||||
""" location can be a string or DataLocator """
|
"""location can be a string or DataLocator"""
|
||||||
region_name = None if app_config is None else app_config.server_config.data_locator__s3__region_name
|
region_name = None if app_config is None else app_config.server_config.data_locator__s3__region_name
|
||||||
self.location = DataLocator(location, region_name=region_name)
|
self.location = DataLocator(location, region_name=region_name)
|
||||||
if not self.location.exists():
|
if not self.location.exists():
|
||||||
|
|||||||
@@ -1,2 +1,2 @@
|
|||||||
mlflow
|
mlflow==2.16.0
|
||||||
scanpy
|
scanpy
|
||||||
|
|||||||
@@ -1,10 +1,10 @@
|
|||||||
black
|
black
|
||||||
bumpversion>=0.5
|
bumpversion>=0.5
|
||||||
codecov>=2.0.15
|
coverage>=5.0
|
||||||
parameterized>=0.7.0
|
parameterized>=0.7.0
|
||||||
pytest>=3.6.3
|
pytest>=3.6.3
|
||||||
python-jose>=3.2.0
|
python-jose>=3.2.0
|
||||||
twine>=1.12.1
|
twine>=1.12.1
|
||||||
|
aiohttp>=3.9.1
|
||||||
-r requirements.txt
|
-r requirements.txt
|
||||||
-r requirements-prepare.txt
|
-r requirements-prepare.txt
|
||||||
-r requirements-annotate.txt
|
|
||||||
|
|||||||
+10
-11
@@ -1,24 +1,23 @@
|
|||||||
# NOTE: If you update 'anndata' min version, also update the 'anndata_version'
|
anndata>=0.8.0
|
||||||
# matrix value in .github/workflows/compatibility_tests.yml
|
|
||||||
anndata>=0.7.6 # we need to_memory(), added in 0.7.6
|
|
||||||
boto3>=1.12.18
|
boto3>=1.12.18
|
||||||
click>=7.1.2
|
click>=7.1.2
|
||||||
Flask>=1.0.2
|
Flask>=3.0.0
|
||||||
Flask-Compress>=1.4.0
|
Flask-Compress>=1.4.0
|
||||||
Flask-Cors>=3.0.9 # CVE-2020-25032
|
Flask-Cors>=3.0.9
|
||||||
Flask-RESTful>=0.3.6
|
Flask-RESTful>=0.3.6
|
||||||
flask-server-timing>=0.1.2
|
flask-server-timing>=0.1.2
|
||||||
flask-talisman>=0.7.0
|
flask-talisman>=0.7.0
|
||||||
flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration
|
flatbuffers==2.0.7
|
||||||
flatten-dict>=0.2.0
|
flatten-dict>=0.2.0
|
||||||
fsspec>=0.4.4,<0.8.0
|
fsspec>0.8.0
|
||||||
gunicorn>=20.0.4
|
gunicorn>=20.0.4
|
||||||
h5py>=3.0.0
|
h5py>=3.0.0
|
||||||
numba>=0.51.2
|
numba>=0.60.0
|
||||||
numpy>=1.17.5,<=1.22
|
numpy==2.0.1
|
||||||
packaging>=20.0
|
packaging>=20.0
|
||||||
pandas>=1.0,!=1.1 # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
|
pandas>=2.2.2
|
||||||
PyYAML>=5.4 # CVE-2020-14343
|
PyYAML>=5.4 # CVE-2020-14343
|
||||||
scipy>=1.4
|
|
||||||
requests>=2.22.0
|
requests>=2.22.0
|
||||||
s3fs==0.4.2
|
s3fs==0.4.2
|
||||||
|
scipy>=1.4
|
||||||
|
setuptools
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ with open("server/requirements-annotate.txt") as fh:
|
|||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="cellxgene",
|
name="cellxgene",
|
||||||
version="1.0.1",
|
version="1.3.0",
|
||||||
packages=find_packages(),
|
packages=find_packages(),
|
||||||
url="https://github.com/chanzuckerberg/cellxgene",
|
url="https://github.com/chanzuckerberg/cellxgene",
|
||||||
license="MIT",
|
license="MIT",
|
||||||
@@ -24,7 +24,7 @@ setup(
|
|||||||
long_description=long_description,
|
long_description=long_description,
|
||||||
long_description_content_type="text/markdown",
|
long_description_content_type="text/markdown",
|
||||||
install_requires=requirements,
|
install_requires=requirements,
|
||||||
python_requires=">=3.6",
|
python_requires=">=3.10",
|
||||||
include_package_data=True,
|
include_package_data=True,
|
||||||
zip_safe=False,
|
zip_safe=False,
|
||||||
classifiers=[
|
classifiers=[
|
||||||
@@ -37,8 +37,9 @@ setup(
|
|||||||
"Operating System :: MacOS :: MacOS X",
|
"Operating System :: MacOS :: MacOS X",
|
||||||
"Programming Language :: JavaScript",
|
"Programming Language :: JavaScript",
|
||||||
"Programming Language :: Python :: 3",
|
"Programming Language :: Python :: 3",
|
||||||
"Programming Language :: Python :: 3.6",
|
"Programming Language :: Python :: 3.10",
|
||||||
"Programming Language :: Python :: 3.7",
|
"Programming Language :: Python :: 3.11",
|
||||||
|
"Programming Language :: Python :: 3.12",
|
||||||
"Programming Language :: Python :: 3 :: Only",
|
"Programming Language :: Python :: 3 :: Only",
|
||||||
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -113,7 +113,7 @@ def start_test_server(command_line_args=[], app_config=None, env=None):
|
|||||||
elif "--port" in command_line_args:
|
elif "--port" in command_line_args:
|
||||||
port = int(command_line_args[command_line_args.index("--port") + 1])
|
port = int(command_line_args[command_line_args.index("--port") + 1])
|
||||||
else:
|
else:
|
||||||
start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
|
start = random.randint(DEFAULT_SERVER_PORT, 2**16 - 1)
|
||||||
port = int(os.environ.get("CXG_SERVER_PORT", start))
|
port = int(os.environ.get("CXG_SERVER_PORT", start))
|
||||||
port = find_available_port("localhost", port)
|
port = find_available_port("localhost", port)
|
||||||
command += ["--port=%d" % port]
|
command += ["--port=%d" % port]
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
from .mlflow_model_fixture import FakeModel
|
|
||||||
|
|
||||||
|
|
||||||
def _load_pyfunc(data_path):
|
|
||||||
return FakeModel()
|
|
||||||
|
|||||||
@@ -1,111 +0,0 @@
|
|||||||
import os
|
|
||||||
import shutil
|
|
||||||
import unittest
|
|
||||||
from tempfile import mkstemp, TemporaryDirectory
|
|
||||||
|
|
||||||
import mlflow
|
|
||||||
from click.testing import CliRunner
|
|
||||||
|
|
||||||
from server.cli.annotate import annotate
|
|
||||||
from test.unit.cli.fixtures.mlflow_model_fixture import FakeModel
|
|
||||||
|
|
||||||
|
|
||||||
def write_model(model) -> str:
|
|
||||||
with TemporaryDirectory() as mlflow_model_dir:
|
|
||||||
fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
|
|
||||||
mlflow.pyfunc.save_model(mlflow_model_dir,
|
|
||||||
loader_module='fixtures',
|
|
||||||
code_path=[fixtures_path])
|
|
||||||
return shutil.make_archive(mkstemp()[1], "zip", mlflow_model_dir)
|
|
||||||
|
|
||||||
|
|
||||||
class TestCliAnnotate(unittest.TestCase):
|
|
||||||
def test__annotate__loads_and_runs(self):
|
|
||||||
"""
|
|
||||||
Invokes the `annotate` subcommand of cellxgene CLI, using a CliRunner() programmatic invocation.
|
|
||||||
|
|
||||||
This tests the happy path case:
|
|
||||||
1) Command line options are parsed;
|
|
||||||
2) An MLflow model zip archive can be read in (from local disk), unpacked, and invoked;
|
|
||||||
3) The correct options are passed to the MLflow model.
|
|
||||||
4) The annotate subcommand exits successfully.
|
|
||||||
|
|
||||||
This does not verify model output or predictions (it's a fake MLflow model, after all); it's up to the real model
|
|
||||||
to output its predictions as it wants, but this is specific to the model and so not tested here.
|
|
||||||
|
|
||||||
The CliRunner() invokes the subcommand in a subprocess, and the annotate subcommand itself invokes the MLflow
|
|
||||||
model in yet another subprocess. So while this test can help determine if everything is working, it is not a
|
|
||||||
simple matter to debug in the case of a failure. However, the stdout/stderr of the MLflow process is captured
|
|
||||||
by the CliRunner() subprocess, so errors can be inspected in result.stdout when debugging this test. Hope this
|
|
||||||
helps!
|
|
||||||
"""
|
|
||||||
|
|
||||||
_, query_dataset_file_path = mkstemp()
|
|
||||||
model_file_path = write_model(FakeModel())
|
|
||||||
|
|
||||||
result = CliRunner().invoke(
|
|
||||||
annotate,
|
|
||||||
[
|
|
||||||
"--input-h5ad-file",
|
|
||||||
query_dataset_file_path,
|
|
||||||
"--model-url",
|
|
||||||
model_file_path,
|
|
||||||
"--output-h5ad-file",
|
|
||||||
f"{query_dataset_file_path}.output",
|
|
||||||
# avoid having mflow create conda env or virtualenv when in test env;
|
|
||||||
# this avoids making pip remote requests and is also faster
|
|
||||||
"--mlflow-env-manager", "local"
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# to help debugging, show the output from the CliRunner and MLflow stdout
|
|
||||||
if result.exit_code:
|
|
||||||
print(result.stdout)
|
|
||||||
|
|
||||||
self.assertEqual(0, result.exit_code, "runs successfully")
|
|
||||||
|
|
||||||
# The FakeModel will print it inputs to stdout, as "__MODEL_INPUT__={...}", allowing us to assert that it received valid inputs.
|
|
||||||
self.assertIn(
|
|
||||||
"__MODEL_INPUT__={"
|
|
||||||
f'"query_dataset_h5ad_path": "{query_dataset_file_path}", '
|
|
||||||
f'"output_h5ad_path": "{query_dataset_file_path}.output", '
|
|
||||||
'"annotation_prefix": "cxg_cell_type", "classifier": "default", '
|
|
||||||
'"organism": "Homo sapiens", "use_gpu": true}',
|
|
||||||
result.stdout,
|
|
||||||
"inputs passed correctly",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test__annotate__verifies_mutually_exclusive_options(self):
|
|
||||||
required_options = ["--input-h5ad-file", "some.h5ad", "--model-url", "some_url"]
|
|
||||||
result = CliRunner().invoke(
|
|
||||||
annotate,
|
|
||||||
required_options + [],
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertNotEqual(0, result.exit_code, "aborts with non-success code")
|
|
||||||
self.assertIn(
|
|
||||||
"--update_h5ad_file or --output_h5ad_file must be specified",
|
|
||||||
result.stdout,
|
|
||||||
"error message displayed",
|
|
||||||
)
|
|
||||||
|
|
||||||
result = CliRunner().invoke(
|
|
||||||
annotate, required_options + ["--output-h5ad-file", "some_arg", "--update-h5ad-file"]
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertNotEqual(0, result.exit_code, "aborts with non-success code")
|
|
||||||
self.assertIn(
|
|
||||||
"--update_h5ad_file and --output_h5ad_file are mutually exclusive",
|
|
||||||
result.stdout,
|
|
||||||
"error message displayed",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# TODO:
|
|
||||||
# Test annotate cli args more comprehensively
|
|
||||||
# Test server.cli.annotate._validate_options
|
|
||||||
# Test model caching feature works
|
|
||||||
# Test model loading from s3 works (maybe w/just a real model)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -6,7 +6,7 @@ from server.cli.prepare import make_index_unique
|
|||||||
|
|
||||||
|
|
||||||
class CLIPrepareTests(unittest.TestCase):
|
class CLIPrepareTests(unittest.TestCase):
|
||||||
""" Test cases for CLI prepare logic """
|
"""Test cases for CLI prepare logic"""
|
||||||
|
|
||||||
def test_make_index_unique(self):
|
def test_make_index_unique(self):
|
||||||
index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])
|
index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from server.cli.upgrade import validate_version_str, split_version, version_gt
|
|||||||
|
|
||||||
|
|
||||||
class CLIUpgradeTests(unittest.TestCase):
|
class CLIUpgradeTests(unittest.TestCase):
|
||||||
""" Test cases for CLI logic """
|
"""Test cases for CLI logic"""
|
||||||
|
|
||||||
def test_validate_version_str(self):
|
def test_validate_version_str(self):
|
||||||
self.assertTrue(validate_version_str("0.1.2"))
|
self.assertTrue(validate_version_str("0.1.2"))
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ class ConfigTests(unittest.TestCase):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls) -> None:
|
def setUpClass(cls) -> None:
|
||||||
os.makedirs(cls.tmp_fixtures_directory)
|
os.makedirs(cls.tmp_fixtures_directory, exist_ok=True)
|
||||||
|
|
||||||
def custom_server_config(
|
def custom_server_config(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -72,24 +72,18 @@ class TestDatasetConfig(ConfigTests):
|
|||||||
config.dataset_config.handle_app()
|
config.dataset_config.handle_app()
|
||||||
|
|
||||||
def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self):
|
def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self):
|
||||||
config = self.get_config(
|
config = self.get_config(enable_users_annotations="true", annotation_type="local_file_csv")
|
||||||
enable_users_annotations="true", annotation_type="local_file_csv"
|
|
||||||
)
|
|
||||||
config.server_config.complete_config(self.context)
|
config.server_config.complete_config(self.context)
|
||||||
config.dataset_config.handle_user_annotations(self.context)
|
config.dataset_config.handle_user_annotations(self.context)
|
||||||
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
|
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
|
||||||
|
|
||||||
config = self.get_config(
|
config = self.get_config(enable_users_annotations="true", annotation_type="NOT_REAL")
|
||||||
enable_users_annotations="true", annotation_type="NOT_REAL"
|
|
||||||
)
|
|
||||||
config.server_config.complete_config(self.context)
|
config.server_config.complete_config(self.context)
|
||||||
with self.assertRaises(ConfigurationError):
|
with self.assertRaises(ConfigurationError):
|
||||||
config.dataset_config.handle_user_annotations(self.context)
|
config.dataset_config.handle_user_annotations(self.context)
|
||||||
|
|
||||||
def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
|
def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
|
||||||
config = self.get_config(
|
config = self.get_config(enable_users_annotations="true", annotation_type="local_file_csv")
|
||||||
enable_users_annotations="true", annotation_type="local_file_csv"
|
|
||||||
)
|
|
||||||
config.server_config.complete_config(self.context)
|
config.server_config.complete_config(self.context)
|
||||||
config.dataset_config.handle_local_file_csv_annotations(self.context)
|
config.dataset_config.handle_local_file_csv_annotations(self.context)
|
||||||
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
|
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
|
||||||
|
|||||||
@@ -56,7 +56,6 @@ class TestExternalConfig(ConfigTests):
|
|||||||
self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
|
self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
|
||||||
|
|
||||||
def test_environment_variable_errors(self):
|
def test_environment_variable_errors(self):
|
||||||
|
|
||||||
# no name
|
# no name
|
||||||
app_config = AppConfig()
|
app_config = AppConfig()
|
||||||
app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
|
app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
|
||||||
|
|||||||
@@ -196,17 +196,18 @@ class EndPoints(object):
|
|||||||
def test_fbs_default(self):
|
def test_fbs_default(self):
|
||||||
endpoint = "data/var"
|
endpoint = "data/var"
|
||||||
url = f"{self.URL_BASE}{endpoint}"
|
url = f"{self.URL_BASE}{endpoint}"
|
||||||
result = self.session.put(url)
|
headers = {"Content-Type": "application/json"}
|
||||||
|
result = self.session.put(url, headers=headers)
|
||||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||||
|
|
||||||
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
|
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
|
||||||
result = self.session.put(url, json=filter)
|
result = self.session.put(url, json=filter, headers=headers)
|
||||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||||
|
|
||||||
def test_data_put_fbs(self):
|
def test_data_put_fbs(self):
|
||||||
endpoint = "data/var"
|
endpoint = "data/var"
|
||||||
url = f"{self.URL_BASE}{endpoint}"
|
url = f"{self.URL_BASE}{endpoint}"
|
||||||
header = {"Accept": "application/octet-stream"}
|
header = {"Accept": "application/octet-stream", "Content-Type": "application/json"}
|
||||||
result = self.session.put(url, headers=header)
|
result = self.session.put(url, headers=header)
|
||||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||||
|
|
||||||
@@ -252,6 +253,7 @@ class EndPoints(object):
|
|||||||
if type(column) is np.ndarray:
|
if type(column) is np.ndarray:
|
||||||
self.assertIn(column.dtype, [np.float32, np.int32])
|
self.assertIn(column.dtype, [np.float32, np.int32])
|
||||||
|
|
||||||
|
@unittest.skip("This test is currently broken after upgrading Werkzeug.")
|
||||||
def test_data_get_unknown_filter_fbs(self):
|
def test_data_get_unknown_filter_fbs(self):
|
||||||
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
|
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
|
||||||
endpoint = "data/var"
|
endpoint = "data/var"
|
||||||
@@ -290,7 +292,7 @@ class EndPoints(object):
|
|||||||
result_data = result.json()
|
result_data = result.json()
|
||||||
self.assertEqual(result_data, pbmc3k_colors)
|
self.assertEqual(result_data, pbmc3k_colors)
|
||||||
|
|
||||||
@unittest.skip('needs fix: https://github.com/chanzuckerberg/cellxgene/issues/2542')
|
@unittest.skip("needs fix: https://github.com/chanzuckerberg/cellxgene/issues/2542")
|
||||||
def test_static(self):
|
def test_static(self):
|
||||||
endpoint = "static"
|
endpoint = "static"
|
||||||
file = "assets/favicon.ico"
|
file = "assets/favicon.ico"
|
||||||
|
|||||||
@@ -106,7 +106,7 @@ class CorporaAPITest(unittest.TestCase):
|
|||||||
|
|
||||||
|
|
||||||
class CorporaRESTAPITest(unittest.TestCase):
|
class CorporaRESTAPITest(unittest.TestCase):
|
||||||
""" Confirm endpoints reflect Corpora-specific features """
|
"""Confirm endpoints reflect Corpora-specific features"""
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def setCorporaFields(cls, path):
|
def setCorporaFields(cls, path):
|
||||||
|
|||||||
@@ -6,12 +6,12 @@ from server.common.rest import _query_parameter_to_filter
|
|||||||
|
|
||||||
|
|
||||||
def _qsparse(qs):
|
def _qsparse(qs):
|
||||||
""" emulate what Flask/Werkzeug do to our QS """
|
"""emulate what Flask/Werkzeug do to our QS"""
|
||||||
return MultiDict(parse_qs(qs))
|
return MultiDict(parse_qs(qs))
|
||||||
|
|
||||||
|
|
||||||
class FilterParseTests(unittest.TestCase):
|
class FilterParseTests(unittest.TestCase):
|
||||||
""" Test cases for various filter parsing """
|
"""Test cases for various filter parsing"""
|
||||||
|
|
||||||
def test_queryparam_to_filter_parse(self):
|
def test_queryparam_to_filter_parse(self):
|
||||||
# categories
|
# categories
|
||||||
@@ -57,7 +57,6 @@ class FilterParseTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def test_queryparam_to_filter_errors(self):
|
def test_queryparam_to_filter_errors(self):
|
||||||
|
|
||||||
# should raise FilterError
|
# should raise FilterError
|
||||||
filter_errors = [
|
filter_errors = [
|
||||||
"foo=bar", # no axis
|
"foo=bar", # no axis
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from test import PROJECT_ROOT, random_string
|
|||||||
|
|
||||||
|
|
||||||
class TestPlugins(unittest.TestCase):
|
class TestPlugins(unittest.TestCase):
|
||||||
""" Test plugin import functionality """
|
"""Test plugin import functionality"""
|
||||||
|
|
||||||
plugins_dir = f"{PROJECT_ROOT}/test/plugins"
|
plugins_dir = f"{PROJECT_ROOT}/test/plugins"
|
||||||
test_plugin_path = f"{plugins_dir}/foo.py"
|
test_plugin_path = f"{plugins_dir}/foo.py"
|
||||||
|
|||||||
@@ -65,13 +65,13 @@ class EstDistTest(unittest.TestCase):
|
|||||||
|
|
||||||
# non-finites
|
# non-finites
|
||||||
self.assertEqual(estimate_approximate_distribution(np.array([np.nan])), XApproximateDistribution.NORMAL)
|
self.assertEqual(estimate_approximate_distribution(np.array([np.nan])), XApproximateDistribution.NORMAL)
|
||||||
self.assertEqual(estimate_approximate_distribution(np.array([np.PINF])), XApproximateDistribution.NORMAL)
|
self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
|
||||||
self.assertEqual(estimate_approximate_distribution(np.array([np.NINF])), XApproximateDistribution.NORMAL)
|
self.assertEqual(estimate_approximate_distribution(np.array([np.inf])), XApproximateDistribution.NORMAL)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
estimate_approximate_distribution(np.array([np.PINF, np.NINF, 0])), XApproximateDistribution.NORMAL
|
estimate_approximate_distribution(np.array([np.inf, np.inf, 0])), XApproximateDistribution.NORMAL
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
estimate_approximate_distribution(np.array([np.nan, np.PINF, np.NINF])), XApproximateDistribution.NORMAL
|
estimate_approximate_distribution(np.array([np.nan, np.inf, np.inf])), XApproximateDistribution.NORMAL
|
||||||
)
|
)
|
||||||
|
|
||||||
raw = np.random.exponential(scale=1000, size=(50, 3))
|
raw = np.random.exponential(scale=1000, size=(50, 3))
|
||||||
@@ -82,15 +82,15 @@ class EstDistTest(unittest.TestCase):
|
|||||||
XApproximateDistribution.COUNT,
|
XApproximateDistribution.COUNT,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
estimate_approximate_distribution(put(raw, [1], [np.PINF])),
|
estimate_approximate_distribution(put(raw, [1], [np.inf])),
|
||||||
XApproximateDistribution.COUNT,
|
XApproximateDistribution.COUNT,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
estimate_approximate_distribution(put(raw, [1], [np.NINF])),
|
estimate_approximate_distribution(put(raw, [1], [np.inf])),
|
||||||
XApproximateDistribution.COUNT,
|
XApproximateDistribution.COUNT,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
|
estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.inf, np.inf])),
|
||||||
XApproximateDistribution.COUNT,
|
XApproximateDistribution.COUNT,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -103,15 +103,15 @@ class EstDistTest(unittest.TestCase):
|
|||||||
XApproximateDistribution.NORMAL,
|
XApproximateDistribution.NORMAL,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
estimate_approximate_distribution(put(logged, [1], [np.PINF])),
|
estimate_approximate_distribution(put(logged, [1], [np.inf])),
|
||||||
XApproximateDistribution.NORMAL,
|
XApproximateDistribution.NORMAL,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
estimate_approximate_distribution(put(logged, [1], [np.NINF])),
|
estimate_approximate_distribution(put(logged, [1], [np.inf])),
|
||||||
XApproximateDistribution.NORMAL,
|
XApproximateDistribution.NORMAL,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
|
estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.inf, np.inf])),
|
||||||
XApproximateDistribution.NORMAL,
|
XApproximateDistribution.NORMAL,
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ Test the anndata adaptor using the pbmc3k data set.
|
|||||||
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"),
|
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"),
|
||||||
(f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"),
|
(f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"),
|
||||||
(f"{FIXTURES_ROOT}/pbmc3k_64.h5ad", False, "auto"), # 64 bit conversion tests
|
(f"{FIXTURES_ROOT}/pbmc3k_64.h5ad", False, "auto"), # 64 bit conversion tests
|
||||||
(f"{FIXTURES_ROOT}/pbmc3k_16.h5ad", False, "auto"), # 16 bit conversion tests
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
class AdaptorTest(unittest.TestCase):
|
class AdaptorTest(unittest.TestCase):
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ class DataLocatorAdaptorTest(unittest.TestCase):
|
|||||||
return config
|
return config
|
||||||
|
|
||||||
def stdAsserts(self, data):
|
def stdAsserts(self, data):
|
||||||
""" run these each time we load the data """
|
"""run these each time we load the data"""
|
||||||
self.assertIsNotNone(data)
|
self.assertIsNotNone(data)
|
||||||
self.assertEqual(data.cell_count, 2638)
|
self.assertEqual(data.cell_count, 2638)
|
||||||
self.assertEqual(data.gene_count, 1838)
|
self.assertEqual(data.gene_count, 1838)
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
import unittest
|
|
||||||
|
|
||||||
from parameterized import parameterized_class
|
|
||||||
|
|
||||||
from server.common.errors import DatasetAccessError
|
|
||||||
from test import FIXTURES_ROOT
|
|
||||||
from test.unit import app_config
|
|
||||||
|
|
||||||
|
|
||||||
@parameterized_class(
|
|
||||||
("data_locator", "backed", "X_approximate_distribution"),
|
|
||||||
[
|
|
||||||
(f"{FIXTURES_ROOT}/pbmc3k_16.h5ad", True, "auto"), # 16 bit conversion tests
|
|
||||||
],
|
|
||||||
)
|
|
||||||
class AdaptorLoadErrorTest(unittest.TestCase):
|
|
||||||
def test_float16_backed_raises_err(self):
|
|
||||||
with self.assertRaises(DatasetAccessError):
|
|
||||||
config = app_config(
|
|
||||||
self.data_locator,
|
|
||||||
backed=self.backed,
|
|
||||||
extra_dataset_config=dict(X_approximate_distribution=self.X_approximate_distribution),
|
|
||||||
)
|
|
||||||
@@ -9,7 +9,7 @@ from test.fixtures.fixtures import pbmc3k_colors
|
|||||||
|
|
||||||
|
|
||||||
class ColorsTest(unittest.TestCase):
|
class ColorsTest(unittest.TestCase):
|
||||||
""" Test color helper functions """
|
"""Test color helper functions"""
|
||||||
|
|
||||||
def test_convert_color_to_hex_format(self):
|
def test_convert_color_to_hex_format(self):
|
||||||
self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")
|
self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")
|
||||||
|
|||||||
@@ -16,10 +16,10 @@ class TestJsonifyStrict(unittest.TestCase):
|
|||||||
jsonify_strict({"nan": [np.nan]})
|
jsonify_strict({"nan": [np.nan]})
|
||||||
|
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
jsonify_strict({"pinf": [np.PINF]})
|
jsonify_strict({"pinf": [np.inf]})
|
||||||
|
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
jsonify_strict({"ninf": [np.NINF]})
|
jsonify_strict({"ninf": [np.inf]})
|
||||||
|
|
||||||
def test_jsonify_numpy_ndarray(self):
|
def test_jsonify_numpy_ndarray(self):
|
||||||
values = {
|
values = {
|
||||||
@@ -54,5 +54,5 @@ class TestJsonifyStrict(unittest.TestCase):
|
|||||||
# the actual test!
|
# the actual test!
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
jsonify_strict(values),
|
jsonify_strict(values),
|
||||||
'{"floating": [100.0, 101.0, 102.0], "integer": [0, 1, 2, 3, 4, 5, 6, 7]}',
|
'{"integer": [0, 1, 2, 3, 4, 5, 6, 7], "floating": [100.0, 101.0, 102.0]}',
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ class TestTypeConversionUtils(unittest.TestCase):
|
|||||||
with self.assertRaises(TypeError):
|
with self.assertRaises(TypeError):
|
||||||
get_schema_type_hint_from_dtype(np.dtype(dtype))
|
get_schema_type_hint_from_dtype(np.dtype(dtype))
|
||||||
|
|
||||||
for dtype in [np.float16, np.float32, np.float64]:
|
for dtype in [np.float32, np.float64]:
|
||||||
self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
|
self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
|
||||||
|
|
||||||
for dtype in [np.dtype(object), np.dtype(str)]:
|
for dtype in [np.dtype(object), np.dtype(str)]:
|
||||||
@@ -123,17 +123,18 @@ int_OK_cases = [
|
|||||||
|
|
||||||
float_OK_cases = [
|
float_OK_cases = [
|
||||||
{
|
{
|
||||||
|
"test_case": "float_OK_cases",
|
||||||
"data": data,
|
"data": data,
|
||||||
"expected_encoding_dtype": np.float32,
|
"expected_encoding_dtype": np.float32,
|
||||||
"expected_schema_hint": {"type": "float32"},
|
"expected_schema_hint": {"type": "float32"},
|
||||||
"logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"},
|
"logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
|
||||||
}
|
}
|
||||||
for dtype in [np.float16, np.float32, np.float64]
|
for dtype in [np.float32, np.float64]
|
||||||
for data in [
|
for data in [
|
||||||
np.arange(-128, 1000, dtype=dtype),
|
np.arange(-128, 1000, dtype=dtype),
|
||||||
pd.Series(np.arange(-128, 1000, dtype=dtype)),
|
pd.Series(np.arange(-128, 1000, dtype=dtype)),
|
||||||
pd.Index(np.arange(-129, 1000, dtype=dtype)),
|
pd.Index(np.arange(-129, 1000, dtype=dtype)),
|
||||||
np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
|
np.array([-np.nan, -np.inf, -1, -0.0, 0, 0.0, 1, np.inf, np.nan], dtype=dtype),
|
||||||
np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
|
np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
|
||||||
sparse.csr_matrix((10, 100), dtype=dtype),
|
sparse.csr_matrix((10, 100), dtype=dtype),
|
||||||
]
|
]
|
||||||
@@ -198,12 +199,13 @@ category_numeric_OK_cases = [
|
|||||||
# numeric, no NA/NaN, float
|
# numeric, no NA/NaN, float
|
||||||
*[
|
*[
|
||||||
{
|
{
|
||||||
|
"test_case": "numeric, no NA/NaN, float",
|
||||||
"data": data,
|
"data": data,
|
||||||
"expected_encoding_dtype": np.float32,
|
"expected_encoding_dtype": np.float32,
|
||||||
"expected_schema_hint": {"type": "categorical"},
|
"expected_schema_hint": {"type": "categorical"},
|
||||||
"logs": {"level": logging.WARNING, "output": "may lose precision"},
|
"logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
|
||||||
}
|
}
|
||||||
for dtype in [np.float16, np.float32, np.float64]
|
for dtype in [np.float32, np.float64]
|
||||||
for data in [
|
for data in [
|
||||||
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
|
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
|
||||||
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
|
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
|
||||||
@@ -213,10 +215,11 @@ category_numeric_OK_cases = [
|
|||||||
# numeric, has NA-induced cast to float32
|
# numeric, has NA-induced cast to float32
|
||||||
*[
|
*[
|
||||||
{
|
{
|
||||||
|
"test_case": "numeric, has NA-induced cast to float32",
|
||||||
"data": data,
|
"data": data,
|
||||||
"expected_encoding_dtype": np.float32,
|
"expected_encoding_dtype": np.float32,
|
||||||
"expected_schema_hint": {"type": "categorical"},
|
"expected_schema_hint": {"type": "categorical"},
|
||||||
"logs": {"level": logging.WARNING, "output": "may lose precision"},
|
"logs": None if dtype == np.float32 else {"level": logging.WARNING, "output": "may lose precision"},
|
||||||
}
|
}
|
||||||
for dtype in [
|
for dtype in [
|
||||||
np.int8,
|
np.int8,
|
||||||
@@ -227,7 +230,6 @@ category_numeric_OK_cases = [
|
|||||||
np.uint32,
|
np.uint32,
|
||||||
np.int64,
|
np.int64,
|
||||||
np.uint64,
|
np.uint64,
|
||||||
np.float16,
|
|
||||||
np.float32,
|
np.float32,
|
||||||
np.float64,
|
np.float64,
|
||||||
]
|
]
|
||||||
@@ -312,7 +314,6 @@ class TestTypeInference(unittest.TestCase, AssertNoLog):
|
|||||||
self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
|
self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
|
||||||
self.assertEqual(schema_hint, self.expected_schema_hint)
|
self.assertEqual(schema_hint, self.expected_schema_hint)
|
||||||
self.assertIn(logs["output"], logger.output[0])
|
self.assertIn(logs["output"], logger.output[0])
|
||||||
|
|
||||||
else:
|
else:
|
||||||
with self.assertNoLogs(logging.getLogger(), logging.WARNING):
|
with self.assertNoLogs(logging.getLogger(), logging.WARNING):
|
||||||
encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
|
encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
|
||||||
|
|||||||
Reference in New Issue
Block a user