fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-03 13:38:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Makefile modularity, test targets, and auto-formatting (#1070)

Browse Source 
* Fix Makefile whitespace and .PHONY use

* Fix Makefile filename

* Modularize Makefile into client and server Makefiles

Part of the reason that the Makefile in the root directory is a bit
complicated is that it tries to handle tasks that can be handled
separately in the client and server modules.

This commit pushes some of the make logic specific to each module into
their own makefiles and calls out to those makefiles from that in the
project root.

* Add auto-formatting to client and server modules

One thing that can make linting faster is auto-formatting. This commit
adds the yapf auto-formatting tool to the server module and uses
eslint's "fix" functionality to speed up the linting/formatting process.

* Add yapf for automatic code formatting

* Add a root test target that calls sub-tests

* Apply yapf to python files

* Do not duplicate npm commands, simply pass through

* Update documentation

* Do not shadow reserved word len

* Add general test target

* Fix make call in dev-env

* Use black instead of yapf

* Run flake8 from the root directory

* Revert "Apply yapf to python files"

This reverts commit cdca128a01.

* Apply black to python code

* Resolve lint errors resulting from black format

* Add explanation of server unit tests in dev guidelines
This commit is contained in:
Matt Weiden
2019-12-27 14:43:37 -08:00
committed by GitHub
parent ec79995be8
commit f3015cb9df
37 changed files with 738 additions and 806 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+4 -7
View File
@@ -9,23 +9,20 @@ cache:
install: install:
- set -eo pipefail - set -eo pipefail
- pip install flake8 - pip install flake8
- make pydist - make pydist install-dist dev-env
- make install-dist
- pip install -r server/requirements-dev.txt
jobs: jobs:
include: include:
- name: "Branch Tests 3.7" - name: "Branch Tests 3.7"
python: "3.7" python: "3.7"
script: ./travis-build.sh script: make build-client lint unit-test
- name: "Branch Tests 3.6" - name: "Branch Tests 3.6"
python: "3.6" python: "3.6"
script: ./travis-build.sh script: make build-client lint unit-test
- name: "Docker Build" - name: "Docker Build"
install: skip install: skip
python: "3.6" python: "3.6"
script: docker build . script: docker build .
- name: "Smoke Tests" - name: "Smoke Tests"
python: "3.6" python: "3.6"
script: script: make smoke-test
- npm run --prefix client/ smoke-test
makefile → Makefile
+96 -43
View File
@@ -5,12 +5,32 @@ CLEANFILES := $(BUILDDIR)/ client/build build dist cellxgene.egg-info
PART ?= patch PART ?= patch
# CLEANING
.PHONY: clean
clean: clean-lite clean-server clean-client
# cleaning the client's node_modules is the longest one, so we avoid that if possible
.PHONY: clean-lite
clean-lite:
rm -rf $(CLEANFILES)
clean-%:
cd $(*) && $(MAKE) clean
# BUILDING PACKAGE # BUILDING PACKAGE
build : clean build-server .PHONY: build
build: clean build-server
@echo "done" @echo "done"
build-server : build-client .PHONY: build-client
build-client:
cd client && $(MAKE) install build
.PHONY: build-server
build-server: build-client
mkdir -p $(SERVERBUILD) mkdir -p $(SERVERBUILD)
cp -r server/* $(SERVERBUILD) cp -r server/* $(SERVERBUILD)
cp -r client/build/ $(CLIENTBUILD) cp -r client/build/ $(CLIENTBUILD)
@@ -22,12 +42,9 @@ build-server : build-client
cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/ cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/
cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR) cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
build-client :
npm install --prefix client/ client
npm run --prefix client build
# If you are actively developing in the server folder use this, dirties the source tree # If you are actively developing in the server folder use this, dirties the source tree
build-for-server-dev : clean-server build-client .PHONY: build-for-server-dev
build-for-server-dev: clean-server build-client
mkdir -p server/app/web/static/img mkdir -p server/app/web/static/img
mkdir -p server/app/web/static/js mkdir -p server/app/web/static/js
mkdir -p server/app/web/templates/ mkdir -p server/app/web/templates/
@@ -36,124 +53,160 @@ build-for-server-dev : clean-server build-client
cp client/build/favicon.png server/app/web/static/img cp client/build/favicon.png server/app/web/static/img
cp client/build/service-worker.js server/app/web/static/js/ cp client/build/service-worker.js server/app/web/static/js/
clean : clean-lite clean-server
rm -rf client/node_modules
# cleaning node_modules is the longest one, so we avoid that if possible # TESTING
clean-lite : .PHONY: test
rm -rf $(CLEANFILES) test: unit-test smoke-test
clean-server : .PHONY: unit-test
rm -f server/app/web/templates/index.html unit-test: unit-test-server unit-test-client
rm -rf server/app/web/static
unit-test-%:
cd $(*) && $(MAKE) unit-test
.PHONY: smoke-test
smoke-test:
cd client && $(MAKE) smoke-test
# FORMATTING CODE
.PHOHY: fmt
fmt: fmt-client fmt-py
fmt-client:
cd client && $(MAKE) fmt
fmt-py:
black .
.PHONY: lint
lint:
flake8 server
.PHONY : build build-server build-client build-for-server-dev clean clean-lite clean-server
# CREATING DISTRIBUTION RELEASE # CREATING DISTRIBUTION RELEASE
pydist : build .PHONY: pydist
pydist: build
cd $(BUILDDIR); python setup.py sdist -d ../dist cd $(BUILDDIR); python setup.py sdist -d ../dist
@echo "done" @echo "done"
.PHONY : pydist
# RELEASE HELPERS # RELEASE HELPERS
# create new version to commit to master # create new version to commit to master
release-stage-1 : dev-env bump clean-lite gen-package-lock .PHONY: release-stage-1
release-stage-1: dev-env bump clean-lite gen-package-lock
@echo "Version bumped part:$(PART) and client built. Ready to commit and push" @echo "Version bumped part:$(PART) and client built. Ready to commit and push"
# build dist and release to dev pypi # build dist and release to dev pypi
release-stage-2 : dev-env pydist twine .PHONY: release-stage-2
release-stage-2: dev-env pydist twine
@echo "Dist built and uploaded to test.pypi.org" @echo "Dist built and uploaded to test.pypi.org"
@echo "Test the install:" @echo "Test the install:"
@echo " make install-release-test" @echo " make install-release-test"
@echo "Then upload to Pypi prod:" @echo "Then upload to Pypi prod:"
@echo " make twine-prod" @echo " make twine-prod"
.PHONY: release-stage-final
release-stage-final: twine-prod release-stage-final: twine-prod
@echo "Release uploaded to pypi.org" @echo "Release uploaded to pypi.org"
# DANGER: releases directly to prod # DANGER: releases directly to prod
# use this if you accidently burned a test release version number, # use this if you accidently burned a test release version number,
release-directly-to-prod : dev-env pydist twine-prod .PHONY: release-directly-to-prod
release-directly-to-prod: dev-env pydist twine-prod
@echo "Dist built and uploaded to pypi.org" @echo "Dist built and uploaded to pypi.org"
@echo "Test the install:" @echo "Test the install:"
@echo " make install-release" @echo " make install-release"
dev-env : .PHONY: dev-env
dev-env:
cd client && $(MAKE) install
pip install -r server/requirements-dev.txt pip install -r server/requirements-dev.txt
gui-env : dev-env .PHONY: gui-env
gui-env: dev-env
pip install -r server/requirements-gui.txt pip install -r server/requirements-gui.txt
# give PART=[major, minor, part] as param to make bump # give PART=[major, minor, part] as param to make bump
bump : .PHONY: bump
bump:
bumpversion --config-file .bumpversion.cfg $(PART) bumpversion --config-file .bumpversion.cfg $(PART)
twine : .PHONY: twine
twine:
twine upload --repository-url https://test.pypi.org/legacy/ dist/* twine upload --repository-url https://test.pypi.org/legacy/ dist/*
twine-prod : .PHONY: twine-prod
twine-prod:
twine upload dist/* twine upload dist/*
# quicker than re-building client # quicker than re-building client
gen-package-lock : .PHONY: gen-package-lock
npm install --prefix client/ client gen-package-lock:
cd client && $(MAKE) install
.PHONY : release-stage-1 release-stage-2 release-stage-final release-burned dev-env bump twine twine-prod gen-package-lock
# INSTALL # INSTALL
# setup.py sucks when you have your library in a separate folder, adding these in to help setup envs # setup.py sucks when you have your library in a separate folder, adding these in to help setup envs
# install from build directory # install from build directory
install : uninstall .PHONY: install
install: uninstall
cd $(BUILDDIR); pip install -e . cd $(BUILDDIR); pip install -e .
# install from source tree for development # install from source tree for development
install-dev : uninstall .PHONY: install-dev
install-dev: uninstall
pip install -e . pip install -e .
# install from test.pypi to test your release # install from test.pypi to test your release
install-release-test : uninstall .PHONY: install-release-test
install-release-test: uninstall
pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
@echo "Installed cellxgene from test.pypi.org, now run and smoke test" @echo "Installed cellxgene from test.pypi.org, now run and smoke test"
# install from pypi to test your release # install from pypi to test your release
install-release : uninstall .PHONY: install-release
install-release: uninstall
pip install --no-cache-dir cellxgene pip install --no-cache-dir cellxgene
@echo "Installed cellxgene from pypi.org" @echo "Installed cellxgene from pypi.org"
# install from dist # install from dist
install-dist : uninstall .PHONY: install-dist
install-dist: uninstall
pip install dist/cellxgene*.tar.gz pip install dist/cellxgene*.tar.gz
uninstall : .PHONY: uninstall
uninstall:
pip uninstall -y cellxgene || : pip uninstall -y cellxgene || :
.PHONY : install install-dev install-release-test install-release uninstall
# GUI # GUI
build-assets : .PHONY: build-assets
build-assets:
pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py
gui-spec-osx : clean-lite gui-env .PHONY: gui-spec-osx
gui-spec-osx: clean-lite gui-env
pip install -e .[gui] pip install -e .[gui]
pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl' --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl' --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py
mv cellxgene.spec cellxgene-osx.spec mv cellxgene.spec cellxgene-osx.spec
gui-spec-windows : clean-lite dev-env .PHONY: gui-spec-windows
gui-spec-windows: clean-lite dev-env
pip install -e .[gui] pip install -e .[gui]
pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py
mv cellxgene.spec cellxgene-windows.spec mv cellxgene.spec cellxgene-windows.spec
gui-build-osx : clean-lite .PHONY: gui-build-osx
gui-build-osx: clean-lite
pyinstaller --clean cellxgene-osx.spec pyinstaller --clean cellxgene-osx.spec
gui-build-windows : clean-lite .PHONY: gui-build-windows
gui-build-windows: clean-lite
pyinstaller --clean cellxgene-windows.spec pyinstaller --clean cellxgene-windows.spec
.PHONY : build-assets gui-spec-osx gui-spec-windows gui-build-osx gui-build-windows
client/Makefile
+16
View File
@@ -0,0 +1,16 @@
.PHONY: clean
clean:
rm -rf node_modules
.PHONY: install
install:
npm install client
.PHONY: build
build:
npm run build
# pass remaining commands through to npm run
%:
npm run $(*)
client/package.json
+1
View File
@@ -11,6 +11,7 @@
"clean": "rimraf build", "clean": "rimraf build",
"dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js", "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
"e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js", "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
"fmt": "eslint --fix src",
"lint": "eslint src", "lint": "eslint src",
"smoke-test": "start-server-and-test start-server-for-test :5000 e2e", "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
"start": "node server/development.js", "start": "node server/development.js",
dev_docs/developer_guidelines.md
+45 -40
View File
@@ -9,6 +9,38 @@
**All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.** **All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.**
## Running test suite
Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only.
### Unit tests
Steps to run the all unit tests:
1. Start in the project root directory
1. `make dev-env`
1. `make unit-test`
### End to end tests
End to end tests use two env variables:
* `JEST_ENV` - environment to run end to end tests. Default `dev`
* `prod` - run headless with no slowdown, chromium will not open.
* `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
* `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` make target.
To run end to end tests as they will be run on CI
1. cellxgene should be built and installed as [specified in server dev](#install)
2. `export JEST_ENV='prod'`
3. `export JEST_CXG_PORT=5000`
4. Run `npm run --prefix client/ smoke-test`
Run end to end tests interactively during development
1. cellxgene should be installed as [specified in client dev](#install-1)
2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k`
3. Run `make smoke-test`
4. To debug a failing test `export JEST_ENV='debug'` and re-run.
## Server dev ## Server dev
### Install ### Install
* Build the client and put static files in place: `make build-for-server-dev` * Build the client and put static files in place: `make build-for-server-dev`
@@ -21,11 +53,15 @@
If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded. If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded.
### Linter ### Linter
We use `flake8` to lint code. Travis CI runs `flake8 server`.
We use [`flake8`](https://github.com/PyCQA/flake8) to lint python and [`black`](https://pypi.org/project/black/) for auto-formatting.
To auto-format code run `make fmt`. To run lint checks on the code run `make lint`.
### Test ### Test
1. Install development requirements `pip install -r server/requirements-dev.txt` If you would like to run the server tests individually, follow the steps below
2. Run tests `pytest server/test` 1. Install development requirements `make dev-env`
1. Run `make unit-test` in the `server` directory or `make unit-test-server` in the root directory.
### Tips ### Tips
* Install in a virtualenv * Install in a virtualenv
@@ -33,8 +69,8 @@ We use `flake8` to lint code. Travis CI runs `flake8 server`.
## Client dev ## Client dev
### Install ### Install
1. Install prereqs for client: `npm install --prefix client/ client` 1. Install prereqs for client: `make dev-env`
2. Install cellxgene server: `pip install -e .` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets. 2. Install cellxgene server: `make install-dev` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets.
### Launch ### Launch
To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made. To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made.
@@ -49,43 +85,12 @@ To build only the client: `make build-client`
We use `eslint` to lint the code and `prettier` as our code formatter. We use `eslint` to lint the code and `prettier` as our code formatter.
### Test ### Test
In `client/` directory run `npm run unit-test`
If you would like to run the client tests individually, follow the steps below in the `client` directory
1. For unit tests run `npm run unit-test` or `make unit-test`
1. For the smoke test run `npm run smoke-test` or `make smoke-test`
### Tips ### Tips
* You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) in `client/` directory run `npm run backend-dev` * You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) in `client/` directory run `npm run backend-dev`
## Running tests
Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only.
### Server unit tests
Install development requirements `pip install -r server/requirements-dev.txt`
Run tests `pytest server/test`
### Client unit tests
In `client/` directory run `npm run unit-test`
### End to end tests
End to end tests use two env variables:
* `JEST_ENV` - environment to run end to end tests. Default `dev`
* `prod` - run headless with no slowdown, chromium will not open.
* `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
* `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` npm script
To run end to end tests as they will be run on CI
1. cellxgene should be built and installed as [specified in server dev](#install)
2. `export JEST_ENV='prod'`
3. `export JEST_CXG_PORT='5000'`
4. Run `npm run --prefix client/ smoke-test`
Run end to end tests interactively during development
1. cellxgene should be installed as [specified in client dev](#install-1)
2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k`
3. Run `npm run --prefix client/ e2e`
4. To debug a failing test `export JEST_ENV='debug'` and re-run.
pyproject.toml
+25
View File
@@ -0,0 +1,25 @@
[tool.black]
line-length = 120
target_version = ['py37']
include = '\.pyi?$'
exclude = '''
(
/(
\.eggs # exclude a few common directories in the
| \.git # root of the project
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| venv
| _build
| buck-out
| build
| dist
| server/app/util/fbs/NetEncoding
)/
| server/gui/cellxgene_rc.py
)
'''
server/Makefile
+8
View File
@@ -0,0 +1,8 @@
.PHONY: clean
clean:
rm -f app/web/templates/index.html
rm -rf app/web/static
.PHONY: unit-test
unit-test:
pytest -s test
server/__main__.py
+2 -2
View File
@@ -5,11 +5,11 @@ if __package__ is None:
PKG_PATH = Path(__file__).parent PKG_PATH = Path(__file__).parent
sys.path.insert(0, str(PKG_PATH.parent)) sys.path.insert(0, str(PKG_PATH.parent))
import server # noqa F401 import server # noqa F401
__package__ = PKG_PATH.name __package__ = PKG_PATH.name
# Main thing # Main thing
from .cli.cli import cli # noqa F402 from .cli.cli import cli # noqa F402
cli() cli()
server/app/driver/driver.py
+2 -2
View File
@@ -33,7 +33,7 @@ class CXGDriver(metaclass=ABCMeta):
"max_category_items": None, "max_category_items": None,
"diffexp_lfc_cutoff": None, "diffexp_lfc_cutoff": None,
"disable_diffexp": False, "disable_diffexp": False,
"diffexp_may_be_slow": False "diffexp_may_be_slow": False,
} }
@abstractmethod @abstractmethod
@@ -51,7 +51,7 @@ class CXGDriver(metaclass=ABCMeta):
features = { features = {
"cluster": {"available": False}, "cluster": {"available": False},
"layout": {"obs": {"available": False}, "var": {"available": False}}, "layout": {"obs": {"available": False}, "var": {"available": False}},
"diffexp": {"available": True, "interactiveLimit": 50000} "diffexp": {"available": True, "interactiveLimit": 50000},
} }
# TODO - Interactive limit should be generated from the actual available methods see GH issue #94 # TODO - Interactive limit should be generated from the actual available methods see GH issue #94
if self.config["layout"]: if self.config["layout"]:
server/app/rest_api/rest.py
+34 -91
View File
@@ -8,13 +8,7 @@ from flask_restful import Api, Resource
from server import __version__ as cellxgene_version from server import __version__ as cellxgene_version
from anndata import __version__ as anndata_version from anndata import __version__ as anndata_version
from server.app.util.constants import ( from server.app.util.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg, CXGUID, CXG_ANNO_COLLECTION
Axis,
DiffExpMode,
JSON_NaN_to_num_warning_msg,
CXGUID,
CXG_ANNO_COLLECTION
)
from server.app.util.errors import ( from server.app.util.errors import (
FilterError, FilterError,
InteractiveError, InteractiveError,
@@ -40,41 +34,18 @@ class ConfigAPI(Resource):
config = { config = {
"config": { "config": {
"features": [ "features": [
{ {"method": "POST", "path": "/cluster/", **current_app.data.features["cluster"]},
"method": "POST", {"method": "POST", "path": "/layout/obs", **current_app.data.features["layout"]["obs"]},
"path": "/cluster/", {"method": "POST", "path": "/layout/var", **current_app.data.features["layout"]["var"]},
**current_app.data.features["cluster"], {"method": "POST", "path": "/diffexp/", **current_app.data.features["diffexp"]},
},
{
"method": "POST",
"path": "/layout/obs",
**current_app.data.features["layout"]["obs"],
},
{
"method": "POST",
"path": "/layout/var",
**current_app.data.features["layout"]["var"],
},
{
"method": "POST",
"path": "/diffexp/",
**current_app.data.features["diffexp"],
},
], ],
"displayNames": { "displayNames": {
"engine": f"cellxgene Scanpy engine version ", "engine": f"cellxgene Scanpy engine version ",
"dataset": current_app.config["DATASET_TITLE"], "dataset": current_app.config["DATASET_TITLE"],
}, },
"links": { "links": {"about-dataset": current_app.config["ABOUT_DATASET"]},
"about-dataset": current_app.config["ABOUT_DATASET"] "parameters": {**current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection)},
}, "library_versions": {"cellxgene": cellxgene_version, "anndata": anndata_version},
"parameters": {
**current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection)
},
"library_versions": {
"cellxgene": cellxgene_version,
"anndata": anndata_version
}
} }
} }
@@ -84,17 +55,13 @@ class ConfigAPI(Resource):
class AnnotationsObsAPI(Resource): class AnnotationsObsAPI(Resource):
def get(self): def get(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
["application/octet-stream"]
)
cxguid = get_userid(session) cxguid = get_userid(session)
anno_collection = get_anno_collection(session) anno_collection = get_anno_collection(session)
try: try:
if preferred_mimetype == "application/octet-stream": if preferred_mimetype == "application/octet-stream":
fbs = current_app.data.annotation_to_fbs_matrix("obs", fields, uid=cxguid, collection=anno_collection) fbs = current_app.data.annotation_to_fbs_matrix("obs", fields, uid=cxguid, collection=anno_collection)
return make_response(fbs, return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else: else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError: except KeyError:
@@ -115,9 +82,7 @@ class AnnotationsObsAPI(Resource):
try: try:
fbs = request.get_data() fbs = request.get_data()
res = current_app.data.annotation_put_fbs("obs", fbs, uid=cxguid, collection=anno_collection) res = current_app.data.annotation_put_fbs("obs", fbs, uid=cxguid, collection=anno_collection)
return make_response( return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"})
res, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except (ValueError, DisabledFeatureError, KeyError) as e: except (ValueError, DisabledFeatureError, KeyError) as e:
return make_response(str(e), HTTPStatus.BAD_REQUEST) return make_response(str(e), HTTPStatus.BAD_REQUEST)
except Exception as e: except Exception as e:
@@ -127,14 +92,14 @@ class AnnotationsObsAPI(Resource):
class AnnotationsVarAPI(Resource): class AnnotationsVarAPI(Resource):
def get(self): def get(self):
fields = request.args.getlist("annotation-name", None) fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
["application/octet-stream"]
)
try: try:
if preferred_mimetype == "application/octet-stream": if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("var", fields), return make_response(
HTTPStatus.OK, current_app.data.annotation_to_fbs_matrix("var", fields),
{"Content-Type": "application/octet-stream"}) HTTPStatus.OK,
{"Content-Type": "application/octet-stream"},
)
else: else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError: except KeyError:
@@ -145,19 +110,16 @@ class AnnotationsVarAPI(Resource):
class DataVarAPI(Resource): class DataVarAPI(Resource):
def put(self): def put(self):
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
["application/octet-stream"]
)
try: try:
if preferred_mimetype == "application/octet-stream": if preferred_mimetype == "application/octet-stream":
filter_json = request.get_json() filter_json = request.get_json()
filter = filter_json["filter"] if filter_json else None filter = filter_json["filter"] if filter_json else None
return make_response( return make_response(
current_app.data.data_frame_to_fbs_matrix( current_app.data.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
filter, axis=Axis.VAR
),
HTTPStatus.OK, HTTPStatus.OK,
{"Content-Type": "application/octet-stream"}) {"Content-Type": "application/octet-stream"},
)
else: else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except FilterError as e: except FilterError as e:
@@ -175,35 +137,23 @@ class DiffExpObsAPI(Resource):
except KeyError: except KeyError:
return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST) return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST)
except ValueError: except ValueError:
return make_response( return make_response(f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST)
f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST
)
# Validate filters # Validate filters
if mode == DiffExpMode.VAR_FILTER or "varFilter" in args: if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
# not NOT_IMPLEMENTED # not NOT_IMPLEMENTED
return make_response( return make_response("mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED)
"mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED
)
if mode == DiffExpMode.TOP_N and "count" not in args: if mode == DiffExpMode.TOP_N and "count" not in args:
return make_response( return make_response("mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST)
"mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST
)
if "set1" not in args: if "set1" not in args:
return make_response("set1 is required.", HTTPStatus.BAD_REQUEST) return make_response("set1 is required.", HTTPStatus.BAD_REQUEST)
if Axis.VAR in args["set1"]["filter"]: if Axis.VAR in args["set1"]["filter"]:
return make_response( return make_response("Var filter not allowed for set1", HTTPStatus.BAD_REQUEST)
"Var filter not allowed for set1", HTTPStatus.BAD_REQUEST
)
# set2 # set2
if "set2" not in args: if "set2" not in args:
return make_response( return make_response("Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED)
"Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED
)
if Axis.VAR in args["set2"]["filter"]: if Axis.VAR in args["set2"]["filter"]:
return make_response( return make_response("Var filter not allowed for set2", HTTPStatus.BAD_REQUEST)
"Var filter not allowed for set2", HTTPStatus.BAD_REQUEST
)
set1_filter = args["set1"]["filter"] set1_filter = args["set1"]["filter"]
set2_filter = args.get("set2", {"filter": {}})["filter"] set2_filter = args.get("set2", {"filter": {}})["filter"]
@@ -214,14 +164,9 @@ class DiffExpObsAPI(Resource):
count = args.get("count", None) count = args.get("count", None)
try: try:
diffexp = current_app.data.diffexp_topN( diffexp = current_app.data.diffexp_topN(
set1_filter, set1_filter, set2_filter, count, current_app.data.features["diffexp"]["interactiveLimit"],
set2_filter,
count,
current_app.data.features["diffexp"]["interactiveLimit"],
)
return make_response(
diffexp, HTTPStatus.OK, {"Content-Type": "application/json"}
) )
return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"})
except (ValueError, FilterError) as e: except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST) return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError: except InteractiveError:
@@ -236,14 +181,12 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource): class LayoutObsAPI(Resource):
def get(self): def get(self):
preferred_mimetype = request.accept_mimetypes.best_match( preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
["application/octet-stream"]
)
try: try:
if preferred_mimetype == "application/octet-stream": if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.layout_to_fbs_matrix(), return make_response(
HTTPStatus.OK, current_app.data.layout_to_fbs_matrix(), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
{"Content-Type": "application/octet-stream"}) )
else: else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except PrepareError as e: except PrepareError as e:
@@ -278,7 +221,7 @@ def is_safe_collection_name(name):
""" """
if name is None: if name is None:
return False return False
return re.match(r'^\w+$', name) is not None return re.match(r"^\w+$", name) is not None
def get_api_resources(): def get_api_resources():
server/app/scanpy_engine/diffexp.py
+2 -2
View File
@@ -32,8 +32,8 @@ def _mean_var_n(X):
v = sumsq / (n - 1) v = sumsq / (n - 1)
if fp_err_occurred: if fp_err_occurred:
mean[np.isfinite(mean) == False] = 0 # noqa: E712 mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712 v[np.isfinite(v) == False] = 0 # noqa: E712
return mean, v, n return mean, v, n
server/app/scanpy_engine/labels.py
+4 -4
View File
@@ -9,7 +9,7 @@ import pandas as pd
def read_labels(fname): def read_labels(fname):
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#') return pd.read_csv(fname, dtype="category", index_col=0, header=0, comment="#")
else: else:
return pd.DataFrame() return pd.DataFrame()
@@ -19,12 +19,12 @@ def write_labels(fname, df, header=None, backup_dir=None):
backup(fname, backup_dir) backup(fname, backup_dir)
# rotate_fname(fname, backup_dir) # rotate_fname(fname, backup_dir)
if not df.empty: if not df.empty:
with open(fname, 'w', newline="") as f: with open(fname, "w", newline="") as f:
if header is not None: if header is not None:
f.write(header) f.write(header)
df.to_csv(f) df.to_csv(f)
else: else:
open(fname, 'w').close() open(fname, "w").close()
def backup(fname, backup_dir, max_backups=9): def backup(fname, backup_dir, max_backups=9):
@@ -46,7 +46,7 @@ def backup(fname, backup_dir, max_backups=9):
fname_base = os.path.basename(fname) fname_base = os.path.basename(fname)
fname_base_root, fname_base_ext = os.path.splitext(fname_base) fname_base_root, fname_base_ext = os.path.splitext(fname_base)
# don't use ISO standard time format, as it contains characters illegal on some filesytems. # don't use ISO standard time format, as it contains characters illegal on some filesytems.
nowish = datetime.now().strftime('%Y-%m-%dT%H-%M-%S') nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}") backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
if os.path.exists(backup_fname): if os.path.exists(backup_fname):
os.remove(backup_fname) os.remove(backup_fname)
server/app/scanpy_engine/matrix_proxy.py
+8 -5
View File
@@ -1,4 +1,3 @@
from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView
""" """
@@ -17,6 +16,7 @@ class ArrayProxyView_anndata_h5py(ArrayProxyView):
override to handle sparse getitem semantics, which differ override to handle sparse getitem semantics, which differ
from numpy. from numpy.
""" """
def toarray(self): def toarray(self):
""" sadly, sparse indexing doesn't drop dimensions like numpy! """ """ sadly, sparse indexing doesn't drop dimensions like numpy! """
arr = self.m[self._index[0], self._index[1]] arr = self.m[self._index[0], self._index[1]]
@@ -30,12 +30,15 @@ class MatrixProxy_anndata_h5py(MatrixProxyView):
AnnData sparse array stored in H5AD, or proxies for backed data. AnnData sparse array stored in H5AD, or proxies for backed data.
None of these handle indexing very well, so we plop a proxy on top. None of these handle indexing very well, so we plop a proxy on top.
""" """
@classmethod @classmethod
def __supports__(cls): def __supports__(cls):
return ("anndata.h5py.h5sparse.SparseDataset", return (
"anndata.h5py.h5sparse.backed_csc_matrix", "anndata.h5py.h5sparse.SparseDataset",
"anndata.h5py.h5sparse.backed_csr_matrix", "anndata.h5py.h5sparse.backed_csc_matrix",
"h5py._hl.dataset.Dataset") "anndata.h5py.h5sparse.backed_csr_matrix",
"h5py._hl.dataset.Dataset",
)
@classmethod @classmethod
def create_array(cls, *args, **kwargs): def create_array(cls, *args, **kwargs):
server/app/scanpy_engine/scanpy_engine.py
+65 -99
View File
@@ -62,7 +62,7 @@ class ScanpyEngine(CXGDriver):
"annotations_output_dir": None, "annotations_output_dir": None,
"backed": False, "backed": False,
"disable_diffexp": False, "disable_diffexp": False,
"diffexp_may_be_slow": False "diffexp_may_be_slow": False,
} }
def get_config_parameters(self, uid=None, collection=None): def get_config_parameters(self, uid=None, collection=None):
@@ -70,26 +70,25 @@ class ScanpyEngine(CXGDriver):
"max-category-items": self.config["max_category_items"], "max-category-items": self.config["max_category_items"],
"disable-diffexp": self.config["disable_diffexp"], "disable-diffexp": self.config["disable_diffexp"],
"diffexp-may-be-slow": self.config["diffexp_may_be_slow"], "diffexp-may-be-slow": self.config["diffexp_may_be_slow"],
"annotations": self.config["annotations"] "annotations": self.config["annotations"],
} }
if self.config["annotations"]: if self.config["annotations"]:
if uid is not None: if uid is not None:
params.update({ params.update({"annotations-user-data-idhash": self.get_userdata_idhash(uid)})
"annotations-user-data-idhash": self.get_userdata_idhash(uid) if self.config["annotations_file"] is not None:
})
if self.config['annotations_file'] is not None:
# user has hard-wired the name of the annotation data collection # user has hard-wired the name of the annotation data collection
fname = os.path.basename(self.config['annotations_file']) fname = os.path.basename(self.config["annotations_file"])
collection_fname = os.path.splitext(fname)[0] collection_fname = os.path.splitext(fname)[0]
params.update({ params.update(
'annotations-data-collection-is-read-only': True, {
'annotations-data-collection-name': collection_fname "annotations-data-collection-is-read-only": True,
}) "annotations-data-collection-name": collection_fname,
}
)
elif collection is not None: elif collection is not None:
params.update({ params.update(
'annotations-data-collection-is-read-only': False, {"annotations-data-collection-is-read-only": False, "annotations-data-collection-name": collection}
'annotations-data-collection-name': collection )
})
return params return params
@staticmethod @staticmethod
@@ -140,23 +139,18 @@ class ScanpyEngine(CXGDriver):
# User has specified alternative column for unique names, and it exists # User has specified alternative column for unique names, and it exists
if not df_axis[name].is_unique: if not df_axis[name].is_unique:
raise KeyError( raise KeyError(
f"Values in {ax_name}.{name} must be unique. " f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values."
"Please prepare data to contain unique values."
) )
df_axis.reset_index(drop=True, inplace=True) df_axis.reset_index(drop=True, inplace=True)
else: else:
# user specified a non-existent column name # user specified a non-existent column name
raise KeyError( raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
f"Annotation name {name}, specified in --{ax_name}-name does not exist."
)
@staticmethod @staticmethod
def _can_cast_to_float32(ann): def _can_cast_to_float32(ann):
if ann.dtype.kind == "f": if ann.dtype.kind == "f":
if not np.can_cast(ann.dtype, np.float32): if not np.can_cast(ann.dtype, np.float32):
warnings.warn( warnings.warn(f"Annotation {ann.name} will be converted to 32 bit float and may lose precision.")
f"Annotation {ann.name} will be converted to 32 bit float and may lose precision."
)
return True return True
return False return False
@@ -188,30 +182,18 @@ class ScanpyEngine(CXGDriver):
schema["type"] = "categorical" schema["type"] = "categorical"
schema["categories"] = dtype.categories.tolist() schema["categories"] = dtype.categories.tolist()
else: else:
raise TypeError( raise TypeError(f"Annotations of type {dtype} are unsupported by cellxgene.")
f"Annotations of type {dtype} are unsupported by cellxgene."
)
return schema return schema
@requires_data @requires_data
def _create_schema(self): def _create_schema(self):
self.schema = { self.schema = {
"dataframe": { "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
"nObs": self.cell_count,
"nVar": self.gene_count,
"type": str(self.data.X.dtype),
},
"annotations": { "annotations": {
"obs": { "obs": {"index": self.config["obs_names"], "columns": []},
"index": self.config["obs_names"], "var": {"index": self.config["var_names"], "columns": []},
"columns": []
},
"var": {
"index": self.config["var_names"],
"columns": []
}
}, },
"layout": {"obs": []} "layout": {"obs": []},
} }
for ax in Axis: for ax in Axis:
curr_axis = getattr(self.data, str(ax)) curr_axis = getattr(self.data, str(ax))
@@ -220,12 +202,8 @@ class ScanpyEngine(CXGDriver):
ann_schema.update(self._get_col_type(curr_axis[ann])) ann_schema.update(self._get_col_type(curr_axis[ann]))
self.schema["annotations"][ax]["columns"].append(ann_schema) self.schema["annotations"][ax]["columns"].append(ann_schema)
for layout in self.config['layout']: for layout in self.config["layout"]:
layout_schema = { layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]}
"name": layout,
"type": "float32",
"dims": [f"{layout}_0", f"{layout}_1"]
}
self.schema["layout"]["obs"].append(layout_schema) self.schema["layout"]["obs"].append(layout_schema)
@requires_data @requires_data
@@ -250,7 +228,7 @@ class ScanpyEngine(CXGDriver):
Used to create safe annotations output file names. Used to create safe annotations output file names.
""" """
id = (uid + self.data_locator.abspath()).encode() id = (uid + self.data_locator.abspath()).encode()
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode('utf-8') idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
return idhash return idhash
def get_anno_fname(self, uid=None, collection=None): def get_anno_fname(self, uid=None, collection=None):
@@ -272,11 +250,11 @@ class ScanpyEngine(CXGDriver):
if not self.config["annotations"]: if not self.config["annotations"]:
return None return None
if self.config['annotations_output_dir']: if self.config["annotations_output_dir"]:
return self.config['annotations_output_dir'] return self.config["annotations_output_dir"]
if self.config['annotations_file']: if self.config["annotations_file"]:
return os.path.dirname(os.path.abspath(self.config['annotations_file'])) return os.path.dirname(os.path.abspath(self.config["annotations_file"]))
return os.getcwd() return os.getcwd()
@@ -299,7 +277,7 @@ class ScanpyEngine(CXGDriver):
with data_locator.local_handle() as lh: with data_locator.local_handle() as lh:
# as of AnnData 0.6.19, backed mode performs initial load fast, but at the # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
# cost of significantly slower access to X data. # cost of significantly slower access to X data.
backed = 'r' if self.config['backed'] else None backed = "r" if self.config["backed"] else None
self.data = anndata.read_h5ad(lh, backed=backed) self.data = anndata.read_h5ad(lh, backed=backed)
except ValueError: except ValueError:
@@ -338,7 +316,7 @@ class ScanpyEngine(CXGDriver):
# heuristic # heuristic
n_values = self.data.shape[0] * self.data.shape[1] n_values = self.data.shape[0] * self.data.shape[1]
if (n_values > 1e8 and self.config['backed'] is True) or (n_values > 5e8): if (n_values > 1e8 and self.config["backed"] is True) or (n_values > 5e8):
self.config.update({"diffexp_may_be_slow": True}) self.config.update({"diffexp_may_be_slow": True})
@requires_data @requires_data
@@ -348,7 +326,7 @@ class ScanpyEngine(CXGDriver):
b) validate layouts are legal. remove/warn on any that are not b) validate layouts are legal. remove/warn on any that are not
c) cap total list of layouts at global const MAX_LAYOUTS c) cap total list of layouts at global const MAX_LAYOUTS
""" """
layouts = self.config['layout'] layouts = self.config["layout"]
# handle default # handle default
if layouts is None or len(layouts) == 0: if layouts is None or len(layouts) == 0:
# load default layouts from the data. # load default layouts from the data.
@@ -372,7 +350,7 @@ class ScanpyEngine(CXGDriver):
raise PrepareError(f"No valid layout data.") raise PrepareError(f"No valid layout data.")
# cap layouts to MAX_LAYOUTS # cap layouts to MAX_LAYOUTS
self.config['layout'] = valid_layouts[0:MAX_LAYOUTS] self.config["layout"] = valid_layouts[0:MAX_LAYOUTS]
@requires_data @requires_data
def _is_valid_layout(self, arr): def _is_valid_layout(self, arr):
@@ -394,8 +372,7 @@ class ScanpyEngine(CXGDriver):
) )
if self.data.X.dtype != "float32": if self.data.X.dtype != "float32":
warnings.warn( warnings.warn(
f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
f"Precision may be truncated."
) )
for ax in Axis: for ax in Axis:
curr_axis = getattr(self.data, str(ax)) curr_axis = getattr(self.data, str(ax))
@@ -414,7 +391,7 @@ class ScanpyEngine(CXGDriver):
) )
if isinstance(datatype, CategoricalDtype): if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories) category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.config['max_category_items']: if category_num > 500 and category_num > self.config["max_category_items"]:
warnings.warn( warnings.warn(
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be " f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the " f"cumbersome or slow to display. We recommend setting the "
@@ -439,14 +416,17 @@ class ScanpyEngine(CXGDriver):
raise KeyError(f"All row index values specified in user annotations must be unique.") raise KeyError(f"All row index values specified in user annotations must be unique.")
if not labels.index.equals(self.original_obs_index): if not labels.index.equals(self.original_obs_index):
raise KeyError("Label file row index does not match H5AD file index. " raise KeyError(
"Please ensure that column zero (0) in the label file contain the same " "Label file row index does not match H5AD file index. "
"index values as the H5AD file.") "Please ensure that column zero (0) in the label file contain the same "
"index values as the H5AD file."
)
duplicate_columns = list(set(labels.columns) & set(self.data.obs.columns)) duplicate_columns = list(set(labels.columns) & set(self.data.obs.columns))
if len(duplicate_columns) > 0: if len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap " raise KeyError(
f"with h5ad obs columns {duplicate_columns}") f"Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}"
)
# labels must have same count as obs annotations # labels must have same count as obs annotations
if labels.shape[0] != self.data.obs.shape[0]: if labels.shape[0] != self.data.obs.shape[0]:
@@ -475,7 +455,7 @@ class ScanpyEngine(CXGDriver):
mask = np.zeros((count,), dtype=bool) mask = np.zeros((count,), dtype=bool)
for i in filter: for i in filter:
if type(i) == list: if type(i) == list:
mask[i[0]: i[1]] = True mask[i[0] : i[1]] = True
else: else:
mask[i] = True mask[i] = True
return mask return mask
@@ -484,15 +464,10 @@ class ScanpyEngine(CXGDriver):
def _axis_filter_to_mask(filter, d_axis, count): def _axis_filter_to_mask(filter, d_axis, count):
mask = np.ones((count,), dtype=bool) mask = np.ones((count,), dtype=bool)
if "index" in filter: if "index" in filter:
mask = np.logical_and( mask = np.logical_and(mask, ScanpyEngine._index_filter_to_mask(filter["index"], count))
mask, ScanpyEngine._index_filter_to_mask(filter["index"], count)
)
if "annotation_value" in filter: if "annotation_value" in filter:
mask = np.logical_and( mask = np.logical_and(
mask, mask, ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"], d_axis, count),
ScanpyEngine._annotation_filter_to_mask(
filter["annotation_value"], d_axis, count
),
) )
return mask return mask
@@ -507,13 +482,9 @@ class ScanpyEngine(CXGDriver):
if filter is not None: if filter is not None:
if Axis.OBS in filter: if Axis.OBS in filter:
obs_selector = self._axis_filter_to_mask( obs_selector = self._axis_filter_to_mask(filter["obs"], self.data.obs, self.data.n_obs)
filter["obs"], self.data.obs, self.data.n_obs
)
if Axis.VAR in filter: if Axis.VAR in filter:
var_selector = self._axis_filter_to_mask( var_selector = self._axis_filter_to_mask(filter["var"], self.data.var, self.data.n_vars)
filter["var"], self.data.var, self.data.n_vars
)
return obs_selector, var_selector return obs_selector, var_selector
@requires_data @requires_data
@@ -531,7 +502,7 @@ class ScanpyEngine(CXGDriver):
labels = None labels = None
if labels is not None and not labels.empty: if labels is not None and not labels.empty:
df = self.data.obs.join(labels, self.config['obs_names']) df = self.data.obs.join(labels, self.config["obs_names"])
else: else:
df = self.data.obs df = self.data.obs
else: else:
@@ -560,18 +531,21 @@ class ScanpyEngine(CXGDriver):
# if any of the new column labels overlap with our existing labels, raise error # if any of the new column labels overlap with our existing labels, raise error
duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns)) duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns))
if not new_label_df.columns.is_unique or len(duplicate_columns) > 0: if not new_label_df.columns.is_unique or len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap " raise KeyError(
f"with h5ad obs columns {duplicate_columns}") f"Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}"
)
# update our internal state and save it. Multi-threading often enabled, # update our internal state and save it. Multi-threading often enabled,
# so treat this as a critical section. # so treat this as a critical section.
with self.label_lock: with self.label_lock:
lastmod = self.data_locator.lastmodtime() lastmod = self.data_locator.lastmodtime()
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \ header = (
f"using cellxgene version {cellxgene_version}\n" \ f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} "
f"# Input data file was {self.data_locator.uri_or_path}, " \ f"using cellxgene version {cellxgene_version}\n"
f"which was last modified on {lastmodstr}\n" f"# Input data file was {self.data_locator.uri_or_path}, "
f"which was last modified on {lastmodstr}\n"
)
write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection)) write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection))
return jsonify_scanpy({"status": "OK"}) return jsonify_scanpy({"status": "OK"})
@@ -598,8 +572,7 @@ class ScanpyEngine(CXGDriver):
raise FilterError("filtering on obs unsupported") raise FilterError("filtering on obs unsupported")
# Currently only handles VAR dimension # Currently only handles VAR dimension
X = MatrixProxy.create(self.data.X if var_selector is None X = MatrixProxy.create(self.data.X if var_selector is None else self.data.X[:, var_selector])
else self.data.X[:, var_selector])
return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None) return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None)
@requires_data @requires_data
@@ -607,25 +580,17 @@ class ScanpyEngine(CXGDriver):
if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB: if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
raise FilterError("Observation filters may not contain vaiable conditions") raise FilterError("Observation filters may not contain vaiable conditions")
try: try:
obs_mask_A = self._axis_filter_to_mask( obs_mask_A = self._axis_filter_to_mask(obsFilterA["obs"], self.data.obs, self.data.n_obs)
obsFilterA["obs"], self.data.obs, self.data.n_obs obs_mask_B = self._axis_filter_to_mask(obsFilterB["obs"], self.data.obs, self.data.n_obs)
)
obs_mask_B = self._axis_filter_to_mask(
obsFilterB["obs"], self.data.obs, self.data.n_obs
)
except (KeyError, IndexError) as e: except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e raise FilterError(f"Error parsing filter: {e}") from e
if top_n is None: if top_n is None:
top_n = DEFAULT_TOP_N top_n = DEFAULT_TOP_N
result = diffexp_ttest( result = diffexp_ttest(self.data, obs_mask_A, obs_mask_B, top_n, self.config["diffexp_lfc_cutoff"])
self.data, obs_mask_A, obs_mask_B, top_n, self.config['diffexp_lfc_cutoff']
)
try: try:
return jsonify_scanpy(result) return jsonify_scanpy(result)
except ValueError: except ValueError:
raise JSONEncodingValueError( raise JSONEncodingValueError("Error encoding differential expression to JSON")
"Error encoding differential expression to JSON"
)
@requires_data @requires_data
def layout_to_fbs_matrix(self): def layout_to_fbs_matrix(self):
@@ -661,7 +626,8 @@ class ScanpyEngine(CXGDriver):
except ValueError as e: except ValueError as e:
raise PrepareError( raise PrepareError(
f"Layout has not been calculated using {self.config['layout']}, " f"Layout has not been calculated using {self.config['layout']}, "
f"please prepare your datafile and relaunch cellxgene") from e f"please prepare your datafile and relaunch cellxgene"
) from e
df = pandas.concat(layout_data, axis=1, copy=False) df = pandas.concat(layout_data, axis=1, copy=False)
return encode_matrix_fbs(df, col_idx=df.columns, row_idx=None) return encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)
server/app/util/constants.py
+1 -3
View File
@@ -27,9 +27,7 @@ class DiffExpMode(AugmentedEnum):
VAR_FILTER = "varFilter" VAR_FILTER = "varFilter"
JSON_NaN_to_num_warning_msg = ( JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
"JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
)
REACTIVE_LIMIT = 1_000_000 REACTIVE_LIMIT = 1_000_000
MAX_LAYOUTS = 30 MAX_LAYOUTS = 30
server/app/util/data_locator.py
+6 -6
View File
@@ -4,7 +4,7 @@ import fsspec
from datetime import datetime from datetime import datetime
class DataLocator(): class DataLocator:
""" """
DataLocator is a simple wrapper around fsspec functionality, and provides a DataLocator is a simple wrapper around fsspec functionality, and provides a
set of functions to encapsulate a data location (URI or path), interogate set of functions to encapsulate a data location (URI or path), interogate
@@ -29,7 +29,7 @@ class DataLocator():
self.uri_or_path = uri_or_path self.uri_or_path = uri_or_path
self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path) self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
# work-around for LocalFileSystem not treating file: and None as the same scheme/protocol # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
self.cname = self.path if self.protocol == 'file' else self.uri_or_path self.cname = self.path if self.protocol == "file" else self.uri_or_path
# will throw RuntimeError if the protocol is unsupported # will throw RuntimeError if the protocol is unsupported
self.fs = fsspec.filesystem(self.protocol) self.fs = fsspec.filesystem(self.protocol)
@@ -53,9 +53,9 @@ class DataLocator():
""" return datetime object representing last modification time, or None if unavailable """ """ return datetime object representing last modification time, or None if unavailable """
info = self.fs.info(self.cname) info = self.fs.info(self.cname)
if self.islocal() and info is not None: if self.islocal() and info is not None:
return datetime.fromtimestamp(info['mtime']) return datetime.fromtimestamp(info["mtime"])
else: else:
return getattr(info, 'LastModified', None) return getattr(info, "LastModified", None)
def abspath(self): def abspath(self):
""" """
@@ -74,7 +74,7 @@ class DataLocator():
return self.fs.open(self.uri_or_path, *args) return self.fs.open(self.uri_or_path, *args)
def islocal(self): def islocal(self):
return self.protocol is None or self.protocol == 'file' return self.protocol is None or self.protocol == "file"
def local_handle(self): def local_handle(self):
if self.islocal(): if self.islocal():
@@ -90,7 +90,7 @@ class DataLocator():
return LocalFilePath(tmp_path, delete=True) return LocalFilePath(tmp_path, delete=True)
class LocalFilePath(): class LocalFilePath:
def __init__(self, tmp_path, delete=False): def __init__(self, tmp_path, delete=False):
self.tmp_path = tmp_path self.tmp_path = tmp_path
self.delete = delete self.delete = delete
server/app/util/fbs/matrix.py
+14 -17
View File
@@ -27,7 +27,7 @@ def CreateNumpyVector(builder, x):
if not isinstance(x, np.ndarray): if not isinstance(x, np.ndarray):
raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}") raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
if x.dtype.kind not in ['b', 'i', 'u', 'f']: if x.dtype.kind not in ["b", "i", "u", "f"]:
raise TypeError("numpy-ndarray holds elements of unsupported datatype") raise TypeError("numpy-ndarray holds elements of unsupported datatype")
if x.ndim > 1: if x.ndim > 1:
@@ -42,11 +42,11 @@ def CreateNumpyVector(builder, x):
x_little_endian = x.byteswap(inplace=False) x_little_endian = x.byteswap(inplace=False)
# Calculate total length # Calculate total length
len = int(x_little_endian.itemsize * x_little_endian.size) length = int(x_little_endian.itemsize * x_little_endian.size)
builder.head = int(builder.Head() - len) builder.head = int(builder.Head() - length)
# tobytes ensures c_contiguous ordering # tobytes ensures c_contiguous ordering
builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C') builder.Bytes[builder.Head() : builder.Head() + length] = x_little_endian.tobytes(order="C")
return builder.EndVector(x.size) return builder.EndVector(x.size)
@@ -88,9 +88,9 @@ def serialize_typed_array(builder, source_array, encoding_info):
arr = arr.to_series() arr = arr.to_series()
# convert to a simple ndarray # convert to a simple ndarray
if as_type == 'json': if as_type == "json":
as_json = arr.to_json(orient='records') as_json = arr.to_json(orient="records")
arr = np.array(bytearray(as_json, 'utf-8')) arr = np.array(bytearray(as_json, "utf-8"))
else: else:
if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr): if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr):
arr = arr.toarray() arr = arr.toarray()
@@ -119,18 +119,16 @@ column_encoding_type_map = {
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32) np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
} }
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json') column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
def column_encoding(arr): def column_encoding(arr):
@@ -141,11 +139,10 @@ index_encoding_type_map = {
# array protocol string: ( array_type, as_type ) # array protocol string: ( array_type, as_type )
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32) np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
} }
index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json') index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
def index_encoding(arr): def index_encoding(arr):
@@ -163,7 +160,7 @@ def guess_at_mem_needed(matrix):
guess = 1 guess = 1
# round up to nearest 1024 bytes # round up to nearest 1024 bytes
guess = (guess + 0x400) & (~0x3ff) guess = (guess + 0x400) & (~0x3FF)
return guess return guess
@@ -223,7 +220,7 @@ def deserialize_typed_array(tarr):
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
} }
(u_type, u) = tarr (u_type, u) = tarr
if u_type is TypedArray.TypedArray.NONE: if u_type is TypedArray.TypedArray.NONE:
@@ -237,7 +234,7 @@ def deserialize_typed_array(tarr):
arr.Init(u.Bytes, u.Pos) arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy() narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray: if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode('utf-8')) narr = json.loads(narr.tostring().decode("utf-8"))
return narr return narr
server/app/util/matrix_proxy.py
+30 -32
View File
@@ -18,6 +18,7 @@ class _ArrayProxyBase(abc.ABC):
Private base class for array or matrix proxy. This summarizes Private base class for array or matrix proxy. This summarizes
the interface used by the rest of cellxgene. the interface used by the rest of cellxgene.
""" """
@property @property
@abc.abstractmethod @abc.abstractmethod
def dtype(self): def dtype(self):
@@ -68,10 +69,10 @@ class MatrixProxy(_ArrayProxyBase):
Sub-classes automatically register. Sub-classes automatically register.
""" """
base_proxy_registry = { base_proxy_registry = {
'pandas.core.frame.DataFrame': True, "pandas.core.frame.DataFrame": True,
'numpy.ndarray': True, "numpy.ndarray": True,
'scipy.sparse.csc.csc_matrix': True, "scipy.sparse.csc.csc_matrix": True,
'scipy.sparse.csr.csr_matrix': True, "scipy.sparse.csr.csr_matrix": True,
} }
proxy_registry = None proxy_registry = None
last_cache_token = None last_cache_token = None
@@ -103,7 +104,7 @@ class MatrixProxy(_ArrayProxyBase):
""" """
cls.build_proxy_registry() cls.build_proxy_registry()
t = type(matrix) t = type(matrix)
fqtn = t.__module__ + '.' + t.__name__ fqtn = t.__module__ + "." + t.__name__
proxy_cls = cls.proxy_registry.get(fqtn, None) proxy_cls = cls.proxy_registry.get(fqtn, None)
if proxy_cls is None: if proxy_cls is None:
raise Exception(f"Matrix format `{fqtn}` is unsupported by proxy.") raise Exception(f"Matrix format `{fqtn}` is unsupported by proxy.")
@@ -128,21 +129,17 @@ class MatrixProxyView(MatrixProxy):
""" """
2D matrix view to a 2D matrix 2D matrix view to a 2D matrix
""" """
def __init__(self, arg1, shape=None, index=(),
transposed=False, copy=False): def __init__(self, arg1, shape=None, index=(), transposed=False, copy=False):
if not copy: if not copy:
m = arg1 m = arg1
super().__init__(m) super().__init__(m)
if shape is None: if shape is None:
shape = m.shape shape = m.shape
assert(len(shape) == 2) assert len(shape) == 2
index = tuple( index = tuple(map(lambda s_i: slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], zip_longest(shape, index)))
map(lambda s_i:
slice(0, s_i[0], 1) if s_i[1] is None else s_i[1],
zip_longest(shape, index))
)
self._shape = shape self._shape = shape
self._index = index self._index = index
@@ -234,20 +231,20 @@ class MatrixProxyView(MatrixProxy):
NOTE: these follow the numpy rules for dimensionality reduction NOTE: these follow the numpy rules for dimensionality reduction
when an integer index is specified. when an integer index is specified.
""" """
def _getitem_intXint(self, row, col): def _getitem_intXint(self, row, col):
return self.m[row, col] return self.m[row, col]
def _getitem_intXslice(self, row, col): def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), ) shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__.create_array(self.m, shape=shape, index=(row, col)) return self.__class__.create_array(self.m, shape=shape, index=(row, col))
def _getitem_sliceXint(self, row, col): def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), ) shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__.create_array(self.m, shape=shape, index=(row, col)) return self.__class__.create_array(self.m, shape=shape, index=(row, col))
def _getitem_sliceXslice(self, row, col): def _getitem_sliceXslice(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), shape = (_slice_length(row, self.m.shape[0]), _slice_length(col, self.m.shape[1]))
_slice_length(col, self.m.shape[1]))
return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed) return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed)
def toarray(self): def toarray(self):
@@ -261,22 +258,23 @@ class ArrayProxyView(_ArrayProxyBase):
""" """
1D array view to a 2D matrix 1D array view to a 2D matrix
""" """
def __init__(self, arg1, shape=None, index=None, copy=False): def __init__(self, arg1, shape=None, index=None, copy=False):
super().__init__() super().__init__()
if not copy: if not copy:
m = arg1 m = arg1
# one index MUST be an integer and the other MUST be a slice # one index MUST be an integer and the other MUST be a slice
assert(len(index) == 2) assert len(index) == 2
assert(all(isinstance(idx, INT_TYPES + (slice, )) for idx in index)) assert all(isinstance(idx, INT_TYPES + (slice,)) for idx in index)
assert(isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES)) assert isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES)
if shape is None: if shape is None:
if isinstance(index[0], INT_TYPES): if isinstance(index[0], INT_TYPES):
shape = (m.shape[0], ) shape = (m.shape[0],)
else: else:
shape = (m.shape[1], ) shape = (m.shape[1],)
assert(len(shape) == 1) assert len(shape) == 1
self._shape = shape self._shape = shape
self.m = m self.m = m
@@ -336,7 +334,7 @@ class ArrayProxyView(_ArrayProxyBase):
elif isinstance(col, slice): elif isinstance(col, slice):
return self._getitem_intXslice(row, col) return self._getitem_intXslice(row, col)
elif isinstance(row, slice): elif isinstance(row, slice):
assert(isinstance(col, INT_TYPES)) assert isinstance(col, INT_TYPES)
return self._getitem_sliceXint(row, col) return self._getitem_sliceXint(row, col)
raise IndexError("unsupported column index types") raise IndexError("unsupported column index types")
@@ -345,11 +343,11 @@ class ArrayProxyView(_ArrayProxyBase):
return self.m[row, col] return self.m[row, col]
def _getitem_intXslice(self, row, col): def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), ) shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__(self.m, shape=shape, index=(row, col)) return self.__class__(self.m, shape=shape, index=(row, col))
def _getitem_sliceXint(self, row, col): def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), ) shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__(self.m, shape=shape, index=(row, col)) return self.__class__(self.m, shape=shape, index=(row, col))
def toarray(self): def toarray(self):
@@ -358,7 +356,7 @@ class ArrayProxyView(_ArrayProxyBase):
def _unpack_index(index, shape): def _unpack_index(index, shape):
if not isinstance(index, tuple): if not isinstance(index, tuple):
index = (index, ) index = (index,)
if len(shape) < len(index): if len(shape) < len(index):
raise IndexError("invalid index dimensionality - must be 2") raise IndexError("invalid index dimensionality - must be 2")
@@ -366,7 +364,7 @@ def _unpack_index(index, shape):
for shp, idx in zip_longest(shape, index): for shp, idx in zip_longest(shape, index):
idx = slice(None) if idx is None else idx idx = slice(None) if idx is None else idx
idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx
unpacked += (idx, ) unpacked += (idx,)
return unpacked return unpacked
@@ -376,7 +374,7 @@ def _slice_slice(outer, outer_len, inner, inner_len):
slice a slice - we take advantage of Python 3 range's support slice a slice - we take advantage of Python 3 range's support
for indexing. for indexing.
""" """
assert(outer_len >= inner_len) assert outer_len >= inner_len
outer_rng = range(*outer.indices(outer_len)) outer_rng = range(*outer.indices(outer_len))
rng = outer_rng[inner] rng = outer_rng[inner]
start, stop, step = rng.start, rng.stop, rng.step start, stop, step = rng.start, rng.stop, rng.step
@@ -387,8 +385,8 @@ def _slice_slice(outer, outer_len, inner, inner_len):
def _range_length(start, stop, step): def _range_length(start, stop, step):
""" return length of range """ """ return length of range """
assert(step != 0) assert step != 0
assert(start is not None and stop is not None and step is not None) assert start is not None and stop is not None and step is not None
if step > 0 and start < stop: if step > 0 and start < stop:
return 1 + (stop - 1 - start) // step return 1 + (stop - 1 - start) // step
elif step < 0 and start > stop: elif step < 0 and start > stop:
@@ -404,7 +402,7 @@ def _slice_length(s, length):
def _slice_defaults(s, length): def _slice_defaults(s, length):
""" apply slice defaulting conventions """ """ apply slice defaulting conventions """
assert(length >= 0) assert length >= 0
step = 1 if s.step is None else s.step step = 1 if s.step is None else s.step
server/app/util/utils.py
+1
View File
@@ -40,4 +40,5 @@ def requires_data(func):
if self.data is None: if self.data is None:
raise DriverError(f"error data must be loaded before you call {func.__name__}") raise DriverError(f"error data must be loaded before you call {func.__name__}")
return func(self, *args, **kwargs) return func(self, *args, **kwargs)
return wrapped_function return wrapped_function
server/cli/cli.py
+8 -6
View File
@@ -4,17 +4,19 @@ from .launch import launch
from .prepare import prepare from .prepare import prepare
@click.group(name="cellxgene", @click.group(
subcommand_metavar="COMMAND <args>", name="cellxgene",
options_metavar="<options>", subcommand_metavar="COMMAND <args>",
context_settings=dict(max_content_width=85, options_metavar="<options>",
help_option_names=['-h', '--help'])) context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
)
@click.help_option("--help", "-h", help="Show this message and exit.") @click.help_option("--help", "-h", help="Show this message and exit.")
@click.version_option( @click.version_option(
version="0.13.0", version="0.13.0",
prog_name="cellxgene", prog_name="cellxgene",
message="[%(prog)s] Version %(version)s", message="[%(prog)s] Version %(version)s",
help="Show the software version and exit.") help="Show the software version and exit.",
)
def cli(): def cli():
pass pass
server/cli/launch.py
+98 -66
View File
@@ -25,16 +25,12 @@ def common_args(func):
Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args. Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args.
""" """
@click.option( @click.option("--title", "-t", metavar="<text>", help="Title to display. If omitted will use file name.")
"--title",
"-t",
metavar="<text>",
help="Title to display. If omitted will use file name.")
@click.option( @click.option(
"--about", "--about",
metavar="<URL>", metavar="<URL>",
help="URL providing more information about the dataset " help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).",
"(hint: must be a fully specified absolute URL).") )
@click.option( @click.option(
"--embedding", "--embedding",
"-e", "-e",
@@ -42,39 +38,43 @@ def common_args(func):
multiple=True, multiple=True,
show_default=False, show_default=False,
metavar="<text>", metavar="<text>",
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all." help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
) )
@click.option( @click.option(
"--obs-names", "--obs-names",
"-obs", "-obs",
default=None, default=None,
metavar="<text>", metavar="<text>",
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.") help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
)
@click.option( @click.option(
"--var-names", "--var-names",
"-var", "-var",
default=None, default=None,
metavar="<text>", metavar="<text>",
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.") help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
)
@click.option( @click.option(
"--max-category-items", "--max-category-items",
default=1000, default=1000,
metavar="<integer>", metavar="<integer>",
show_default=True, show_default=True,
help="Will not display categories with more distinct values than specified.",) help="Will not display categories with more distinct values than specified.",
)
@click.option( @click.option(
"--diffexp-lfc-cutoff", "--diffexp-lfc-cutoff",
"-de", "-de",
default=0.01, default=0.01,
show_default=True, show_default=True,
metavar="<float>", metavar="<float>",
help="Minimum log fold change threshold for differential expression.",) help="Minimum log fold change threshold for differential expression.",
)
@click.option( @click.option(
"--experimental-annotations", "--experimental-annotations",
is_flag=True, is_flag=True,
default=False, default=False,
show_default=True, show_default=True,
help="Enable user annotation of data." help="Enable user annotation of data.",
) )
@click.option( @click.option(
"--experimental-annotations-file", "--experimental-annotations-file",
@@ -83,7 +83,8 @@ def common_args(func):
multiple=False, multiple=False,
metavar="<path>", metavar="<path>",
help="CSV file to initialize editing of existing annotations; will be altered in-place. " help="CSV file to initialize editing of existing annotations; will be altered in-place. "
"Incompatible with --annotations-output-dir.",) "Incompatible with --annotations-output-dir.",
)
@click.option( @click.option(
"--experimental-annotations-output-dir", "--experimental-annotations-output-dir",
default=None, default=None,
@@ -91,20 +92,23 @@ def common_args(func):
multiple=False, multiple=False,
metavar="<directory path>", metavar="<directory path>",
help="Directory of where to save output annotations; filename will be specified in the application. " help="Directory of where to save output annotations; filename will be specified in the application. "
"Incompatible with --annotations-input-file.",) "Incompatible with --annotations-input-file.",
)
@click.option( @click.option(
"--backed", "--backed",
"-b", "-b",
is_flag=True, is_flag=True,
default=False, default=False,
show_default=False, show_default=False,
help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.") help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.",
)
@click.option( @click.option(
"--disable-diffexp", "--disable-diffexp",
is_flag=True, is_flag=True,
default=False, default=False,
show_default=False, show_default=False,
help="Disable on-demand differential expression.") help="Disable on-demand differential expression.",
)
@functools.wraps(func) @functools.wraps(func)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
return func(*args, **kwargs) return func(*args, **kwargs)
@@ -112,9 +116,18 @@ def common_args(func):
return wrapper return wrapper
def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, def parse_engine_args(
experimental_annotations, experimental_annotations_file, embedding,
experimental_annotations_output_dir, backed, disable_diffexp): obs_names,
var_names,
max_category_items,
diffexp_lfc_cutoff,
experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
disable_diffexp,
):
annotations_file = experimental_annotations_file if experimental_annotations else None annotations_file = experimental_annotations_file if experimental_annotations else None
annotations_output_dir = experimental_annotations_output_dir if experimental_annotations else None annotations_output_dir = experimental_annotations_output_dir if experimental_annotations else None
return { return {
@@ -127,14 +140,15 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
"annotations_file": annotations_file, "annotations_file": annotations_file,
"annotations_output_dir": annotations_output_dir, "annotations_output_dir": annotations_output_dir,
"backed": backed, "backed": backed,
"disable_diffexp": disable_diffexp "disable_diffexp": disable_diffexp,
} }
@sort_options @sort_options
@click.command(short_help="Launch the cellxgene data viewer. " @click.command(
"Run `cellxgene launch --help` for more information.", short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
options_metavar="<options>",) options_metavar="<options>",
)
@click.argument("data", nargs=1, metavar="<path to data file>", required=True) @click.argument("data", nargs=1, metavar="<path to data file>", required=True)
@click.option( @click.option(
"--verbose", "--verbose",
@@ -142,7 +156,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
is_flag=True, is_flag=True,
default=False, default=False,
show_default=True, show_default=True,
help="Provide verbose output, including warnings and all server requests.",) help="Provide verbose output, including warnings and all server requests.",
)
@click.option( @click.option(
"--debug", "--debug",
"-d", "-d",
@@ -150,7 +165,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
default=False, default=False,
show_default=True, show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, " help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",) "or when you want more information about an error condition.",
)
@click.option( @click.option(
"--open", "--open",
"-o", "-o",
@@ -158,19 +174,22 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
is_flag=True, is_flag=True,
default=False, default=False,
show_default=True, show_default=True,
help="Open web browser after launch.",) help="Open web browser after launch.",
)
@click.option( @click.option(
"--port", "--port",
"-p", "-p",
metavar="<port>", metavar="<port>",
show_default=True, show_default=True,
help="Port to run server on. If not specified cellxgene will find an available port.",) help="Port to run server on. If not specified cellxgene will find an available port.",
)
@click.option( @click.option(
"--host", "--host",
metavar="<IP address>", metavar="<IP address>",
default="127.0.0.1", default="127.0.0.1",
show_default=False, show_default=False,
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).") help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
)
@click.option( @click.option(
"--scripts", "--scripts",
"-s", "-s",
@@ -178,30 +197,31 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
multiple=True, multiple=True,
metavar="<text>", metavar="<text>",
help="Additional script files to include in HTML page. If not specified, " help="Additional script files to include in HTML page. If not specified, "
"no additional script files will be included.", "no additional script files will be included.",
show_default=False,) show_default=False,
)
@click.help_option("--help", "-h", help="Show this message and exit.") @click.help_option("--help", "-h", help="Show this message and exit.")
@common_args @common_args
def launch( def launch(
data, data,
verbose, verbose,
debug, debug,
open_browser, open_browser,
port, port,
host, host,
embedding, embedding,
obs_names, obs_names,
var_names, var_names,
max_category_items, max_category_items,
diffexp_lfc_cutoff, diffexp_lfc_cutoff,
title, title,
scripts, scripts,
about, about,
experimental_annotations, experimental_annotations,
experimental_annotations_file, experimental_annotations_file,
experimental_annotations_output_dir, experimental_annotations_output_dir,
backed, backed,
disable_diffexp disable_diffexp,
): ):
"""Launch the cellxgene data viewer. """Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data. This web app lets you explore single-cell expression data.
@@ -217,13 +237,18 @@ def launch(
> cellxgene launch <url>""" > cellxgene launch <url>"""
e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items, e_args = parse_engine_args(
diffexp_lfc_cutoff, embedding,
experimental_annotations, obs_names,
experimental_annotations_file, var_names,
experimental_annotations_output_dir, max_category_items,
backed, diffexp_lfc_cutoff,
disable_diffexp) experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
disable_diffexp,
)
try: try:
data_locator = DataLocator(data) data_locator = DataLocator(data)
except RuntimeError as re: except RuntimeError as re:
@@ -256,7 +281,8 @@ def launch(
sys.tracebacklimit = 0 sys.tracebacklimit = 0
if scripts: if scripts:
click.echo(r""" click.echo(
r"""
/ / /\ \ \__ _ _ __ _ __ (_)_ __ __ _ / / /\ \ \__ _ _ __ _ __ (_)_ __ __ _
\ \/ \/ / _` | '__| '_ \| | '_ \ / _` | \ \/ \/ / _` | '__| '_ \| | '_ \ / _` |
\ /\ / (_| | | | | | | | | | | (_| | \ /\ / (_| | | | | | | | | | | (_| |
@@ -264,7 +290,8 @@ def launch(
|___/ |___/
The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a
security risk by including the --scripts flag. Make sure you trust the scripts that you are including. security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
""") """
)
scripts_pretty = ", ".join(scripts) scripts_pretty = ", ".join(scripts)
click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
@@ -289,8 +316,9 @@ def launch(
click.echo("Warning: --experimental-annotations-output-dir ignored as --annotations not enabled.") click.echo("Warning: --experimental-annotations-output-dir ignored as --annotations not enabled.")
else: else:
if experimental_annotations_file is not None and experimental_annotations_output_dir is not None: if experimental_annotations_file is not None and experimental_annotations_output_dir is not None:
raise click.ClickException("--experimental-annotations-file and --experimental-annotations-output-dir " raise click.ClickException(
"may not be used together.") "--experimental-annotations-file and --experimental-annotations-output-dir " "may not be used together."
)
if experimental_annotations_file is not None: if experimental_annotations_file is not None:
lf_name, lf_ext = splitext(experimental_annotations_file) lf_name, lf_ext = splitext(experimental_annotations_file)
@@ -301,10 +329,12 @@ def launch(
try: try:
mkdir(experimental_annotations_output_dir) mkdir(experimental_annotations_output_dir)
except OSError: except OSError:
raise click.ClickException("Unable to create directory specified by " raise click.ClickException(
"--experimental-annotations-output-dir") "Unable to create directory specified by " "--experimental-annotations-output-dir"
)
if about: if about:
def url_check(url): def url_check(url):
try: try:
result = urlparse(url) result = urlparse(url)
@@ -346,9 +376,11 @@ def launch(
except ScanpyFileError as e: except ScanpyFileError as e:
raise click.ClickException(f"{e}") raise click.ClickException(f"{e}")
if not disable_diffexp and server.app.data.config['diffexp_may_be_slow']: if not disable_diffexp and server.app.data.config["diffexp_may_be_slow"]:
click.echo(f"[cellxgene] CAUTION: due to the size of your dataset, " click.echo(
f"running differential expression may take longer or fail.") f"[cellxgene] CAUTION: due to the size of your dataset, "
f"running differential expression may take longer or fail."
)
if open_browser: if open_browser:
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.") click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
server/cli/prepare.py
+24 -27
View File
@@ -8,9 +8,10 @@ from server.utils.utils import sort_options
@sort_options @sort_options
@click.command(short_help="Preprocess data for use with cellxgene. " @click.command(
"Run `cellxgene prepare --help` for more information.", short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.",
options_metavar="<options>",) options_metavar="<options>",
)
@click.argument("data", nargs=1, metavar="<path to data file>", required=True) @click.argument("data", nargs=1, metavar="<path to data file>", required=True)
@click.option( @click.option(
"--embedding", "--embedding",
@@ -35,37 +36,33 @@ from server.utils.utils import sort_options
@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True) @click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>") @click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>") @click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
@click.option("--skip-qc", default=False, is_flag=True,
help="Do not run quality control metrics. By default cellxgene runs them "
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).")
@click.option( @click.option(
"--make-obs-names-unique", "--skip-qc",
default=True, default=False,
is_flag=True, is_flag=True,
help="Ensure obs index is unique.", help="Do not run quality control metrics. By default cellxgene runs them "
show_default=True "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
) )
@click.option( @click.option(
"--make-var-names-unique", "--make-obs-names-unique", default=True, is_flag=True, help="Ensure obs index is unique.", show_default=True
default=True, )
is_flag=True, @click.option(
help="Ensure var index is unique.", "--make-var-names-unique", default=True, is_flag=True, help="Ensure var index is unique.", show_default=True
show_default=True
) )
@click.help_option("--help", "-h", help="Show this message and exit.") @click.help_option("--help", "-h", help="Show this message and exit.")
def prepare( def prepare(
data, data,
embedding, embedding,
recipe, recipe,
output, output,
plotting, plotting,
sparse, sparse,
overwrite, overwrite,
set_obs_names, set_obs_names,
set_var_names, set_var_names,
skip_qc, skip_qc,
make_obs_names_unique, make_obs_names_unique,
make_var_names_unique, make_var_names_unique,
): ):
""" """
Preprocess data for use with cellxgene. Preprocess data for use with cellxgene.
server/gui/browser.py
+3 -4
View File
@@ -12,6 +12,7 @@ WindowUtils = cef.WindowUtils()
# noinspection PyUnresolvedReferences # noinspection PyUnresolvedReferences
CefWidgetParent = QWidget CefWidgetParent = QWidget
class CefWidget(CefWidgetParent): class CefWidget(CefWidgetParent):
def __init__(self, parent=None): def __init__(self, parent=None):
super(CefWidget, self).__init__(parent) super(CefWidget, self).__init__(parent)
@@ -58,8 +59,7 @@ class CefWidget(CefWidgetParent):
if WINDOWS: if WINDOWS:
WindowUtils.OnSize(self.getHandle(), 0, 0, 0) WindowUtils.OnSize(self.getHandle(), 0, 0, 0)
elif LINUX: elif LINUX:
self.browser.SetBounds(self.x, self.y, self.browser.SetBounds(self.x, self.y, self.width(), self.height())
self.width(), self.height())
self.browser.NotifyMoveOrResizeStarted() self.browser.NotifyMoveOrResizeStarted()
def resizeEvent(self, event): def resizeEvent(self, event):
@@ -68,8 +68,7 @@ class CefWidget(CefWidgetParent):
if WINDOWS: if WINDOWS:
WindowUtils.OnSize(self.getHandle(), 0, 0, 0) WindowUtils.OnSize(self.getHandle(), 0, 0, 0)
elif LINUX: elif LINUX:
self.browser.SetBounds(self.x, self.y, self.browser.SetBounds(self.x, self.y, size.width(), size.height())
size.width(), size.height())
self.browser.NotifyMoveOrResizeStarted() self.browser.NotifyMoveOrResizeStarted()
server/gui/hook-cefpython3.py
+22 -26
View File
@@ -37,8 +37,7 @@ logger = logging.getLogger(__name__)
# Functions # Functions
def check_platforms(): def check_platforms():
if not is_win and not is_darwin and not is_linux: if not is_win and not is_darwin and not is_linux:
raise SystemExit("Error: Currently only Windows, Linux and Darwin " raise SystemExit("Error: Currently only Windows, Linux and Darwin " "platforms are supported, see Issue #135.")
"platforms are supported, see Issue #135.")
def check_pyinstaller_version(): def check_pyinstaller_version():
@@ -50,22 +49,19 @@ def check_pyinstaller_version():
version = PyInstaller.__version__ version = PyInstaller.__version__
match = re.search(r"^\d+\.\d+(\.\d+)?", version) match = re.search(r"^\d+\.\d+(\.\d+)?", version)
if not (match.group(0) >= PYINSTALLER_MIN_VERSION): if not (match.group(0) >= PYINSTALLER_MIN_VERSION):
raise SystemExit("Error: pyinstaller %s or higher is required" raise SystemExit("Error: pyinstaller %s or higher is required" % PYINSTALLER_MIN_VERSION)
% PYINSTALLER_MIN_VERSION)
def check_cefpython3_version(): def check_cefpython3_version():
if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION): if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION):
raise SystemExit("Error: cefpython3 %s or higher is required" raise SystemExit("Error: cefpython3 %s or higher is required" % CEFPYTHON_MIN_VERSION)
% CEFPYTHON_MIN_VERSION)
def get_cefpython_modules(): def get_cefpython_modules():
"""Get all cefpython Cython modules in the cefpython3 package. """Get all cefpython Cython modules in the cefpython3 package.
It returns a list of names without file extension. Eg. It returns a list of names without file extension. Eg.
'cefpython_py27'. """ 'cefpython_py27'. """
pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, "cefpython_py*" + CYTHON_MODULE_EXT))
"cefpython_py*" + CYTHON_MODULE_EXT))
assert len(pyds) > 1, "Missing cefpython3 Cython modules" assert len(pyds) > 1, "Missing cefpython3 Cython modules"
modules = [] modules = []
for path in pyds: for path in pyds:
@@ -130,14 +126,21 @@ def get_cefpython3_datas():
for filename in os.listdir(CEFPYTHON3_DIR): for filename in os.listdir(CEFPYTHON3_DIR):
# Ignore Cython modules which are already handled by # Ignore Cython modules which are already handled by
# pyinstaller automatically. # pyinstaller automatically.
if filename[:-len(CYTHON_MODULE_EXT)] in get_cefpython_modules(): if filename[: -len(CYTHON_MODULE_EXT)] in get_cefpython_modules():
continue continue
# CEF binaries and datas # CEF binaries and datas
extension = os.path.splitext(filename)[1] extension = os.path.splitext(filename)[1]
if extension in \ if extension in [
[".exe", ".dll", ".pak", ".dat", ".bin", ".txt", ".so", ".plist"] \ ".exe",
or filename.lower().startswith("license"): ".dll",
".pak",
".dat",
".bin",
".txt",
".so",
".plist",
] or filename.lower().startswith("license"):
logger.info("Include cefpython3 data: {}".format(filename)) logger.info("Include cefpython3 data: {}".format(filename))
ret.append((os.path.join(CEFPYTHON3_DIR, filename), cefdatadir)) ret.append((os.path.join(CEFPYTHON3_DIR, filename), cefdatadir))
@@ -145,11 +148,9 @@ def get_cefpython3_datas():
# "Chromium Embedded Framework.framework/Resources" with subdirectories # "Chromium Embedded Framework.framework/Resources" with subdirectories
# is required. Contain .pak files and locales (each locale in separate # is required. Contain .pak files and locales (each locale in separate
# subdirectory). # subdirectory).
resources_subdir = \ resources_subdir = os.path.join("Chromium Embedded Framework.framework", "Resources")
os.path.join("Chromium Embedded Framework.framework", "Resources")
base_path = os.path.join(CEFPYTHON3_DIR, resources_subdir) base_path = os.path.join(CEFPYTHON3_DIR, resources_subdir)
assert os.path.exists(base_path), \ assert os.path.exists(base_path), "{} dir not found in cefpython3".format(resources_subdir)
"{} dir not found in cefpython3".format(resources_subdir)
for path, dirs, files in os.walk(base_path): for path, dirs, files in os.walk(base_path):
for file in files: for file in files:
absolute_file_path = os.path.join(path, file) absolute_file_path = os.path.join(path, file)
@@ -159,22 +160,17 @@ def get_cefpython3_datas():
elif is_win or is_linux: elif is_win or is_linux:
# The .pak files in cefpython3/locales/ directory # The .pak files in cefpython3/locales/ directory
locales_dir = os.path.join(CEFPYTHON3_DIR, "locales") locales_dir = os.path.join(CEFPYTHON3_DIR, "locales")
assert os.path.exists(locales_dir), \ assert os.path.exists(locales_dir), "locales/ dir not found in cefpython3"
"locales/ dir not found in cefpython3"
for filename in os.listdir(locales_dir): for filename in os.listdir(locales_dir):
logger.info("Include cefpython3 data: {}/{}".format( logger.info("Include cefpython3 data: {}/{}".format(os.path.basename(locales_dir), filename))
os.path.basename(locales_dir), filename)) ret.append((os.path.join(locales_dir, filename), os.path.join(cefdatadir, "locales")))
ret.append((os.path.join(locales_dir, filename),
os.path.join(cefdatadir, "locales")))
# Optional .so/.dll files in cefpython3/swiftshader/ directory # Optional .so/.dll files in cefpython3/swiftshader/ directory
swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader") swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader")
if os.path.isdir(swiftshader_dir): if os.path.isdir(swiftshader_dir):
for filename in os.listdir(swiftshader_dir): for filename in os.listdir(swiftshader_dir):
logger.info("Include cefpython3 data: {}/{}".format( logger.info("Include cefpython3 data: {}/{}".format(os.path.basename(swiftshader_dir), filename))
os.path.basename(swiftshader_dir), filename)) ret.append((os.path.join(swiftshader_dir, filename), os.path.join(cefdatadir, "swiftshader")))
ret.append((os.path.join(swiftshader_dir, filename),
os.path.join(cefdatadir, "swiftshader")))
return ret return ret
server/gui/main.py
+17 -11
View File
@@ -20,8 +20,8 @@ from server.utils.utils import find_available_port
if WINDOWS or LINUX: if WINDOWS or LINUX:
dirname = dirname(PySide2.__file__) dirname = dirname(PySide2.__file__)
plugin_path = join(dirname, 'plugins', 'platforms') plugin_path = join(dirname, "plugins", "platforms")
environ['QT_QPA_PLATFORM_PLUGIN_PATH'] = plugin_path environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = plugin_path
# Configuration # Configuration
# TODO remember this or calculate it? # TODO remember this or calculate it?
@@ -94,8 +94,7 @@ class MainWindow(QMainWindow):
# a hidden window, embed CEF browser in it and then # a hidden window, embed CEF browser in it and then
# create a container for that hidden window and replace # create a container for that hidden window and replace
# cef widget in the layout with the container. # cef widget in the layout with the container.
self.container = QWidget.createWindowContainer( self.container = QWidget.createWindowContainer(self.cef_widget.hidden_window, parent=self)
self.cef_widget.hidden_window, parent=self)
self.stacked_layout.replaceWidget(self.cef_widget, self.container) self.stacked_layout.replaceWidget(self.cef_widget, self.container)
self.stacked_layout.setCurrentIndex(LOAD_INDEX) self.stacked_layout.setCurrentIndex(LOAD_INDEX)
@@ -117,7 +116,7 @@ class MainWindow(QMainWindow):
def setupMenu(self): def setupMenu(self):
# TODO add communication to subprocess on reload # TODO add communication to subprocess on reload
main_menu = self.menuBar() main_menu = self.menuBar()
file_menu = main_menu.addMenu('File') file_menu = main_menu.addMenu("File")
load_action = QAction("Load file...", self) load_action = QAction("Load file...", self)
load_action.setStatusTip("Load file") load_action.setStatusTip("Load file")
load_action.setShortcut("Ctrl+O") load_action.setShortcut("Ctrl+O")
@@ -201,7 +200,7 @@ class LoadWidget(QFrame):
for l in [logo_layout, file_layout, message_layout]: for l in [logo_layout, file_layout, message_layout]:
load_ui_layout.addLayout(l) load_ui_layout.addLayout(l)
#TODO remove magic number # TODO remove magic number
load_ui_layout.setStretch(1, 10) load_ui_layout.setStretch(1, 10)
self.setLayout(load_ui_layout) self.setLayout(load_ui_layout)
@@ -240,8 +239,15 @@ class LoadWidget(QFrame):
def createScanpyEngine(self, file_name): def createScanpyEngine(self, file_name):
title = splitext(basename(file_name))[0] title = splitext(basename(file_name))[0]
self.window().setupServer() self.window().setupServer()
worker = Worker(self.window().parent_conn, self.window().child_conn, file_name, host="127.0.0.1", worker = Worker(
port=GUI_PORT, title=title, engine_options={}) self.window().parent_conn,
self.window().child_conn,
file_name,
host="127.0.0.1",
port=GUI_PORT,
title=title,
engine_options={},
)
self.window().load_emitter.signals.ready.connect(self.onDataReady) self.window().load_emitter.signals.ready.connect(self.onDataReady)
self.window().load_emitter.signals.engine_error.connect(self.onServerError) self.window().load_emitter.signals.engine_error.connect(self.onServerError)
self.window().load_emitter.signals.server_error.connect(self.onServerError) self.window().load_emitter.signals.server_error.connect(self.onServerError)
@@ -289,6 +295,7 @@ class LoadWidget(QFrame):
onServerError = partialmethod(onError, server_error=True) onServerError = partialmethod(onError, server_error=True)
class FilePath(QObject): class FilePath(QObject):
def __init__(self): def __init__(self):
super(FilePath, self).__init__() super(FilePath, self).__init__()
@@ -320,8 +327,7 @@ class FileArea(QFrame):
def fileBrowse(self): def fileBrowse(self):
options = QFileDialog.Options() options = QFileDialog.Options()
# options |= QFileDialog.DontUseNativeDialog # options |= QFileDialog.DontUseNativeDialog
file_name, _ = QFileDialog.getOpenFileName(self, file_name, _ = QFileDialog.getOpenFileName(self, "Open H5AD File", "", "H5AD Files (*.h5ad)", options=options)
"Open H5AD File", "", "H5AD Files (*.h5ad)", options=options)
if file_name: if file_name:
self.parent().file_name.updateValue(file_name) self.parent().file_name.updateValue(file_name)
self.parent().onLoad() self.parent().onLoad()
@@ -391,5 +397,5 @@ def main():
sys.exit(0) sys.exit(0)
if __name__ == '__main__': if __name__ == "__main__":
main() main()
server/gui/utils.py
+5 -3
View File
@@ -4,9 +4,9 @@ import platform
from PySide2.QtCore import QObject, Signal from PySide2.QtCore import QObject, Signal
# Detect OS # Detect OS
WINDOWS = (platform.system() == "Windows") WINDOWS = platform.system() == "Windows"
LINUX = (platform.system() == "Linux") LINUX = platform.system() == "Linux"
MAC = (platform.system() == "Darwin") MAC = platform.system() == "Darwin"
class WorkerSignals(QObject): class WorkerSignals(QObject):
@@ -18,6 +18,7 @@ class WorkerSignals(QObject):
error - `str` error message error - `str` error message
result - `object` data returned from processing, anything result - `object` data returned from processing, anything
""" """
finished = Signal() finished = Signal()
engine_error = Signal(str) engine_error = Signal(str)
server_error = Signal(str) server_error = Signal(str)
@@ -34,6 +35,7 @@ class SiteReadySignals(QObject):
ready ready
error - `str` error message error - `str` error message
""" """
ready = Signal() ready = Signal()
timeout = Signal() timeout = Signal()
error = Signal(str) error = Signal(str)
server/gui/workers.py
+1
View File
@@ -36,6 +36,7 @@ class Worker(EmittingProcess):
return return
from server.app.app import Server from server.app.app import Server
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
# create server # create server
try: try:
server = Server() server = Server()
server/test/decode_fbs.py
+4 -11
View File
@@ -1,4 +1,3 @@
""" """
Code to decode, for testing purposes, the flatbuffer encoded blobs. Code to decode, for testing purposes, the flatbuffer encoded blobs.
This code will need to be updated if fbs/matrix.fbs changes. This code will need to be updated if fbs/matrix.fbs changes.
@@ -22,20 +21,20 @@ def decode_typed_array(tarr):
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
} }
(u_type, u) = tarr (u_type, u) = tarr
if u_type == TypedArray.TypedArray.NONE: if u_type == TypedArray.TypedArray.NONE:
return None return None
TarType = type_map.get(u_type, None) TarType = type_map.get(u_type, None)
assert(TarType is not None) assert TarType is not None
arr = TarType() arr = TarType()
arr.Init(u.Bytes, u.Pos) arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy() narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray: if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode('utf-8')) narr = json.loads(narr.tostring().decode("utf-8"))
return narr return narr
@@ -60,10 +59,4 @@ def decode_matrix_FBS(buf):
cidx = decode_typed_array((df.ColIndexType(), df.ColIndex())) cidx = decode_typed_array((df.ColIndexType(), df.ColIndex()))
return { return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None}
"n_rows": n_rows,
"n_cols": n_cols,
"columns": decoded_columns,
"col_idx": cidx,
"row_idx": None
}
server/test/test_api.py
+52 -57
View File
@@ -19,7 +19,7 @@ class EndPoints(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"]) cls.ps = Popen(["cellxgene", "launch", "../example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
session = requests.Session() session = requests.Session()
for i in range(90): for i in range(90):
try: try:
@@ -68,14 +68,15 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df['n_cols'], 8) self.assertEqual(df["n_cols"], 8)
self.assertIsNotNone(df['columns']) self.assertIsNotNone(df["columns"])
self.assertListEqual(df['col_idx'], [ self.assertListEqual(
'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1' df["col_idx"],
]) ["pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"],
self.assertIsNone(df['row_idx']) )
self.assertEqual(len(df['columns']), df['n_cols']) self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_bad_filter(self): def test_bad_filter(self):
endpoint = "data/var" endpoint = "data/var"
@@ -91,14 +92,14 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df['n_cols'], 5) self.assertEqual(df["n_cols"], 5)
self.assertIsNotNone(df['columns']) self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df['col_idx']) self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df['row_idx']) self.assertIsNone(df["row_idx"])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df["columns"]), df["n_cols"])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"] obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain']) self.assertListEqual(df["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
def test_get_annotations_obs_keys_fbs(self): def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs" endpoint = "annotations/obs"
@@ -109,13 +110,13 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df['n_cols'], 2) self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df['columns']) self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df['col_idx']) self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df['row_idx']) self.assertIsNone(df["row_idx"])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito']) self.assertListEqual(df["col_idx"], ["n_genes", "percent_mito"])
def test_get_annotations_obs_error(self): def test_get_annotations_obs_error(self):
endpoint = "annotations/obs" endpoint = "annotations/obs"
@@ -162,14 +163,14 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838) self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df['n_cols'], 2) self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df['columns']) self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df['col_idx']) self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df['row_idx']) self.assertIsNone(df["row_idx"])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df["columns"]), df["n_cols"])
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"] var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells']) self.assertListEqual(df["col_idx"], [var_index_col_name, "n_cells"])
def test_get_annotations_var_keys_fbs(self): def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var" endpoint = "annotations/var"
@@ -180,13 +181,13 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838) self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df['n_cols'], 1) self.assertEqual(df["n_cols"], 1)
self.assertIsNotNone(df['columns']) self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df['col_idx']) self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df['row_idx']) self.assertIsNone(df["row_idx"])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df['col_idx'], ['n_cells']) self.assertListEqual(df["col_idx"], ["n_cells"])
def test_get_annotations_var_error(self): def test_get_annotations_var_error(self):
endpoint = "annotations/var" endpoint = "annotations/var"
@@ -217,35 +218,29 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df['n_cols'], 1838) self.assertEqual(df["n_cols"], 1838)
self.assertIsNotNone(df['columns']) self.assertIsNotNone(df["columns"])
self.assertListEqual(df['col_idx'].tolist(), []) self.assertListEqual(df["col_idx"].tolist(), [])
self.assertIsNone(df['row_idx']) self.assertIsNone(df["row_idx"])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df["columns"]), df["n_cols"])
def test_data_put_filter_fbs(self): def test_data_put_filter_fbs(self):
endpoint = f"data/var" endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}" url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"} header = {"Accept": "application/octet-stream"}
filter = { filter = {"filter": {"var": {"index": [0, 1, 4]}}}
"filter": {
"var": {
"index": [0, 1, 4]
}
}
}
result = self.session.put(url, headers=header, json=filter) result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream") self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content) df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638) self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df['n_cols'], 3) self.assertEqual(df["n_cols"], 3)
self.assertIsNotNone(df['columns']) self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df['col_idx']) self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df['row_idx']) self.assertIsNone(df["row_idx"])
self.assertEqual(len(df['columns']), df['n_cols']) self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4]) self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
def test_data_put_single_var(self): def test_data_put_single_var(self):
endpoint = f"data/var" endpoint = f"data/var"
server/test/test_fbs.py
+16 -27
View File
@@ -32,7 +32,7 @@ class FbsTests(unittest.TestCase):
for i in range(0, len(d["columns"])): for i in range(0, len(d["columns"])):
self.assertEqual(len(d["columns"][i]), dims[0]) self.assertEqual(len(d["columns"][i]), dims[0])
self.assertIsInstance(d["columns"][i], expected_types[i][0]) self.assertIsInstance(d["columns"][i], expected_types[i][0])
if (expected_types[i][1] is not None): if expected_types[i][1] is not None:
self.assertEqual(d["columns"][i].dtype, expected_types[i][1]) self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
if expected_column_idx is not None: if expected_column_idx is not None:
self.assertSetEqual(set(expected_column_idx), set(d["col_idx"])) self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
@@ -40,48 +40,37 @@ class FbsTests(unittest.TestCase):
def test_encode_DataFrame(self): def test_encode_DataFrame(self):
df = pd.DataFrame( df = pd.DataFrame(
data={ data={
'a': np.zeros((10,), dtype=np.float32), "a": np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64), "b": np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16), "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}) }
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.int32),
(np.ndarray, np.uint32),
(list, None)
) )
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd']) self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
def test_encode_ndarray(self): def test_encode_ndarray(self):
arr = np.zeros((3, 2), dtype=np.float32) arr = np.zeros((3, 2), dtype=np.float32)
expected_types = ( expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
(np.ndarray, np.float32),
(np.ndarray, np.float32),
(np.ndarray, np.float32)
)
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None) fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (3, 2), expected_types, None) self.fbs_checks(fbs, (3, 2), expected_types, None)
def test_encode_sparse(self): def test_encode_sparse(self):
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]])) csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
expected_types = ( expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
(np.ndarray, np.int32),
(np.ndarray, np.int32),
(np.ndarray, np.int32)
)
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None) fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (2, 3), expected_types, None) self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_roundtrip(self): def test_roundtrip(self):
dfSrc = pd.DataFrame( dfSrc = pd.DataFrame(
data={ data={
'a': np.zeros((10,), dtype=np.float32), "a": np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64), "b": np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16), "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}) }
)
dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns)) dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
self.assertEqual(dfSrc.shape, dfDst.shape) self.assertEqual(dfSrc.shape, dfDst.shape)
self.assertEqual(set(dfSrc.columns), set(dfDst.columns)) self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
server/test/test_matrix_proxy.py
+39 -95
View File
@@ -7,9 +7,10 @@ class NdArrayProxyView(MatrixProxyView):
""" """
Fake test class for matrix proxy - wraps ndarray Fake test class for matrix proxy - wraps ndarray
""" """
@classmethod @classmethod
def __supports__(cls): def __supports__(cls):
return ('numpy.ndarray', ) return ("numpy.ndarray",)
class MatrixProxyViewTest(unittest.TestCase): class MatrixProxyViewTest(unittest.TestCase):
@@ -41,18 +42,17 @@ class MatrixProxyViewTest(unittest.TestCase):
def test_toarray(self): def test_toarray(self):
n = np.arange(15, dtype=np.float32).reshape((3, 5)) n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n) mp = MatrixProxy.create(n)
self.assertTrue(np.all(mp.toarray() == [ self.assertTrue(
[0., 1., 2., 3., 4.], np.all(
[5., 6., 7., 8., 9.], mp.toarray() == [[0.0, 1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0, 9.0], [10.0, 11.0, 12.0, 13.0, 14.0]]
[10., 11., 12., 13., 14.] )
])) )
self.assertTrue(np.all(mp.T.toarray() == [ self.assertTrue(
[0., 5., 10.], np.all(
[1., 6., 11.], mp.T.toarray()
[2., 7., 12.], == [[0.0, 5.0, 10.0], [1.0, 6.0, 11.0], [2.0, 7.0, 12.0], [3.0, 8.0, 13.0], [4.0, 9.0, 14.0]]
[3., 8., 13.], )
[4., 9., 14.] )
]))
def test_indexing(self): def test_indexing(self):
""" """
@@ -95,47 +95,19 @@ class MatrixProxyViewTest(unittest.TestCase):
# slice, slice # slice, slice
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [ self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]]))
[7, 8], self.assertTrue(
[12, 13] np.all(mp[:3, :4].toarray() == [[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0], [10.0, 11.0, 12.0, 13.0]])
])) )
self.assertTrue(np.all(mp[:3, :4].toarray() == [ self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]]))
[0., 1., 2., 3.], self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]]))
[5., 6., 7., 8.],
[10., 11., 12., 13.]
]))
self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [
[14, 13, 12, 11, 10],
[9, 8, 7, 6, 5],
[4, 3, 2, 1, 0]
]))
self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [
[14, 12, 10],
[4, 2, 0]
]))
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [ self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]]))
[7, 12], self.assertTrue(np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8, 13]]))
[8, 13] self.assertTrue(
])) np.all(mp.T[::-1, ::-1].toarray() == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]])
self.assertTrue(np.all(mp.T[:4, :3].toarray() == [ )
[0, 5, 10], self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]]))
[1, 6, 11],
[2, 7, 12],
[3, 8, 13]
]))
self.assertTrue(np.all(mp.T[::-1, ::-1].toarray() == [
[14, 9, 4],
[13, 8, 3],
[12, 7, 2],
[11, 6, 1],
[10, 5, 0]
]))
self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [
[14, 4],
[12, 2],
[10, 0]
]))
def test_repeated_indexing(self): def test_repeated_indexing(self):
""" """
@@ -149,31 +121,15 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0][1], 1) self.assertEqual(mp[0][1], 1)
self.assertEqual(mp.T[0][1], 5) self.assertEqual(mp.T[0][1], 5)
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [ self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1]))
2, 1 self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3]))
])) self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [ self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2]))
2, 3 self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [2, 3]))
]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [
2, 3
]))
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [ self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1]))
2, 1 self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10]))
])) self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == [])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == []))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == []))
@@ -190,20 +146,12 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0, 0], 0) self.assertEqual(mp[0, 0], 0)
# drop 1 dimension, to an array # drop 1 dimension, to an array
self.assertTrue(np.all(mp[0, :].toarray() == [ self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4]))
0, 1, 2, 3, 4 self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10]))
]))
self.assertTrue(np.all(mp[:, 0].toarray() == [
0, 5, 10
]))
# with .T # with .T
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [ self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]]))
[4, 9] self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9]))
]))
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [
4, 9
]))
def test_iter(self): def test_iter(self):
""" """
@@ -214,17 +162,13 @@ class MatrixProxyViewTest(unittest.TestCase):
rows = [r for r in mp] rows = [r for r in mp]
self.assertEqual(len(rows), 3) self.assertEqual(len(rows), 3)
self.assertTrue(np.all(rows[0].toarray() == [ self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4]))
0, 1, 2, 3, 4
]))
for i, r in enumerate(rows): for i, r in enumerate(rows):
self.assertTrue(np.all(mp[i].toarray() == r.toarray())) self.assertTrue(np.all(mp[i].toarray() == r.toarray()))
cols = [c for c in mp.T] cols = [c for c in mp.T]
self.assertEqual(len(cols), 5) self.assertEqual(len(cols), 5)
self.assertTrue(np.all(cols[0].toarray() == [ self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10]))
0, 5, 10
]))
for i, c in enumerate(cols): for i, c in enumerate(cols):
self.assertTrue(np.all(mp.T[i].toarray() == c.toarray())) self.assertTrue(np.all(mp.T[i].toarray() == c.toarray()))
server/test/test_nan_rest.py
+1 -3
View File
@@ -20,9 +20,7 @@ class WithNaNs(unittest.TestCase):
@classmethod @classmethod
def setUpClass(cls): def setUpClass(cls):
cls.ps = Popen( cls.ps = Popen(["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"])
["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]
)
session = requests.Session() session = requests.Session()
for i in range(90): for i in range(90):
try: try:
server/test/test_nan_scanpy_engine.py
+4 -11
View File
@@ -21,12 +21,12 @@ class NaNTest(unittest.TestCase):
} }
with warnings.catch_warnings(): with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning) warnings.simplefilter("ignore", category=UserWarning)
self.data = ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args) self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
self.data._create_schema() self.data._create_schema()
def test_load(self): def test_load(self):
with self.assertWarns(UserWarning): with self.assertWarns(UserWarning):
ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args) ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
def test_init(self): def test_init(self):
self.assertEqual(self.data.cell_count, 100) self.assertEqual(self.data.cell_count, 100)
@@ -44,11 +44,7 @@ class NaNTest(unittest.TestCase):
with pytest.raises(FilterError): with pytest.raises(FilterError):
self.data.data_frame_to_fbs_matrix("an erroneous filter", "var") self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
with pytest.raises(FilterError): with pytest.raises(FilterError):
filter_ = { filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
"filter": {
"obs": {"index": [1, 99, [200, 300]]}
}
}
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_dataframe_obs_not_implemented(self): def test_dataframe_obs_not_implemented(self):
@@ -59,10 +55,7 @@ class NaNTest(unittest.TestCase):
def test_annotation(self): def test_annotation(self):
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs")) annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual( self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
)
self.assertEqual(annotations["n_rows"], 100) self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0])) self.assertTrue(math.isnan(annotations["columns"][2][0]))
server/test/test_scanpy_engine.py
+26 -48
View File
@@ -18,15 +18,17 @@ Test the scanpy engine using the pbmc3k data set.
""" """
@parameterized_class(("data_locator", "backed"), [ @parameterized_class(
("example-dataset/pbmc3k.h5ad", False), ("data_locator", "backed"),
("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False), [
("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False), ("../example-dataset/pbmc3k.h5ad", False),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
("example-dataset/pbmc3k.h5ad", True), ("test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True), ("../example-dataset/pbmc3k.h5ad", True),
("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True), ("test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
]) ("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
],
)
class EngineTest(unittest.TestCase): class EngineTest(unittest.TestCase):
def setUp(self): def setUp(self):
args = { args = {
@@ -36,7 +38,7 @@ class EngineTest(unittest.TestCase):
"var_names": None, "var_names": None,
"diffexp_lfc_cutoff": 0.01, "diffexp_lfc_cutoff": 0.01,
"layout_file": None, "layout_file": None,
"backed": self.backed "backed": self.backed,
} }
self.data = ScanpyEngine(DataLocator(self.data_locator), args) self.data = ScanpyEngine(DataLocator(self.data_locator), args)
@@ -64,11 +66,7 @@ class EngineTest(unittest.TestCase):
self.data._validate_data_types() self.data._validate_data_types()
def test_filter_idx(self): def test_filter_idx(self):
filter_ = { filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
"filter": {
"var": {"index": [1, 99, [200, 300]]}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs) data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_rows"], 2638)
@@ -76,14 +74,7 @@ class EngineTest(unittest.TestCase):
def test_filter_complex(self): def test_filter_complex(self):
filter_ = { filter_ = {
"filter": { "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}}
"var": {
"annotation_value": [
{"name": "n_cells", "min": 10}
],
"index": [1, 99, [200, 300]]
}
}
} }
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs) data = decode_fbs.decode_matrix_FBS(fbs)
@@ -101,16 +92,14 @@ class EngineTest(unittest.TestCase):
def test_schema_produces_error(self): def test_schema_produces_error(self):
self.data.data.obs["time"] = pd.Series( self.data.data.obs["time"] = pd.Series(
list([time.time() for i in range(self.data.cell_count)]), list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
dtype="datetime64[ns]",
) )
with pytest.raises(TypeError): with pytest.raises(TypeError):
self.data._create_schema() self.data._create_schema()
def test_config(self): def test_config(self):
self.assertEqual( self.assertEqual(
self.data.features["layout"]["obs"], self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000},
{"available": True, "interactiveLimit": 50000},
) )
def test_layout(self): def test_layout(self):
@@ -131,14 +120,13 @@ class EngineTest(unittest.TestCase):
self.assertEqual(annotations["n_cols"], 5) self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
self.assertEqual( self.assertEqual(
annotations["col_idx"], annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
) )
fbs = self.data.annotation_to_fbs_matrix("var") fbs = self.data.annotation_to_fbs_matrix("var")
annotations = decode_fbs.decode_matrix_FBS(fbs) annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838) self.assertEqual(annotations["n_rows"], 1838)
self.assertEqual(annotations['n_cols'], 2) self.assertEqual(annotations["n_cols"], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"]) self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
@@ -146,13 +134,13 @@ class EngineTest(unittest.TestCase):
fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
annotations = decode_fbs.decode_matrix_FBS(fbs) annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638) self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations['n_cols'], 2) self.assertEqual(annotations["n_cols"], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name]) fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs) annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838) self.assertEqual(annotations["n_rows"], 1838)
self.assertEqual(annotations['n_cols'], 1) self.assertEqual(annotations["n_cols"], 1)
def test_annotation_put(self): def test_annotation_put(self):
with self.assertRaises(DisabledFeatureError): with self.assertRaises(DisabledFeatureError):
@@ -177,27 +165,19 @@ class EngineTest(unittest.TestCase):
self.data.data_frame_to_fbs_matrix(None, "obs") self.data.data_frame_to_fbs_matrix(None, "obs")
def test_filtered_data_frame(self): def test_filtered_data_frame(self):
filter_ = { filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}}
"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs) data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 1040) self.assertEqual(data["n_cols"], 1040)
filter_ = { filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
}
with self.assertRaises(FilterError): with self.assertRaises(FilterError):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_named_gene(self): def test_data_named_gene(self):
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
filter_ = { filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}}
"filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs) data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_rows"], 2638)
@@ -205,9 +185,7 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["col_idx"], [4]) self.assertEqual(data["col_idx"], [4])
filter_ = { filter_ = {
"filter": { "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}}
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
}
} }
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs) data = decode_fbs.decode_matrix_FBS(fbs)
server/test/test_scanpy_engine_data_load.py
+5 -3
View File
@@ -10,8 +10,9 @@ class DataLoadEngineTest(unittest.TestCase):
""" """
Test file loading, including deferred loading/update. Test file loading, including deferred loading/update.
""" """
def setUp(self): def setUp(self):
self.data_file = DataLocator("example-dataset/pbmc3k.h5ad") self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
self.data = ScanpyEngine() self.data = ScanpyEngine()
def test_init(self): def test_init(self):
@@ -29,7 +30,7 @@ class DataLoadEngineTest(unittest.TestCase):
"annotations_output_dir": None, "annotations_output_dir": None,
"backed": False, "backed": False,
"diffexp_may_be_slow": False, "diffexp_may_be_slow": False,
"disable_diffexp": False "disable_diffexp": False,
} }
self.data.update(args=args) self.data.update(args=args)
self.assertEqual(args, self.data.config) self.assertEqual(args, self.data.config)
@@ -60,6 +61,7 @@ class DataLocatorEngineTest(unittest.TestCase):
""" """
Test various types of data locators we expect to consume Test various types of data locators we expect to consume
""" """
def setUp(self): def setUp(self):
self.args = { self.args = {
"layout": ["umap"], "layout": ["umap"],
@@ -76,7 +78,7 @@ class DataLocatorEngineTest(unittest.TestCase):
self.assertEqual(data.gene_count, 1838) self.assertEqual(data.gene_count, 1838)
def test_posix_file(self): def test_posix_file(self):
locator = DataLocator("example-dataset/pbmc3k.h5ad") locator = DataLocator("../example-dataset/pbmc3k.h5ad")
data = ScanpyEngine(locator, self.args) data = ScanpyEngine(locator, self.args)
self.stdAsserts(data) self.stdAsserts(data)
server/test/test_writable_annotation.py
+49 -50
View File
@@ -25,9 +25,9 @@ class WritableAnnotationTest(unittest.TestCase):
"diffexp_lfc_cutoff": 0.01, "diffexp_lfc_cutoff": 0.01,
"annotations": True, "annotations": True,
"annotations_file": self.annotations_file, "annotations_file": self.annotations_file,
"annotations_output_dir": None "annotations_output_dir": None,
} }
self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args) self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args)
def tearDown(self): def tearDown(self):
shutil.rmtree(self.tmpDir) shutil.rmtree(self.tmpDir)
@@ -40,9 +40,7 @@ class WritableAnnotationTest(unittest.TestCase):
# verify that the expected errors are generated # verify that the expected errors are generated
n_rows = self.data.data.obs.shape[0] n_rows = self.data.data.obs.shape[0]
fbs_bad = self.make_fbs({ fbs_bad = self.make_fbs({"louvain": pd.Series(["undefined" for l in range(0, n_rows)], dtype="category")})
'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category')
})
# ensure attempt to change VAR annotation # ensure attempt to change VAR annotation
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
@@ -55,32 +53,36 @@ class WritableAnnotationTest(unittest.TestCase):
def test_write_to_file(self): def test_write_to_file(self):
# verify the file is written as expected # verify the file is written as expected
n_rows = self.data.data.obs.shape[0] n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({ fbs = self.make_fbs(
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), {
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') "cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"),
}) "cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
res = self.data.annotation_put_fbs("obs", fbs) res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"})) self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file)) self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
self.assertEqual(df.shape, (n_rows, 2)) self.assertEqual(df.shape, (n_rows, 2))
self.assertEqual(set(df.columns), set(['cat_A', 'cat_B'])) self.assertEqual(set(df.columns), set(["cat_A", "cat_B"]))
self.assertTrue(self.data.original_obs_index.equals(df.index)) self.assertTrue(self.data.original_obs_index.equals(df.index))
self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)])) self.assertTrue(np.all(df["cat_A"] == ["label_A" for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)])) self.assertTrue(np.all(df["cat_B"] == ["label_B" for l in range(0, n_rows)]))
# verify complete overwrite on second attempt, AND rotation occurs # verify complete overwrite on second attempt, AND rotation occurs
fbs = self.make_fbs({ fbs = self.make_fbs(
'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'), {
'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category') "cat_A": pd.Series(["label_A1" for l in range(0, n_rows)], dtype="category"),
}) "cat_C": pd.Series(["label_C" for l in range(0, n_rows)], dtype="category"),
}
)
res = self.data.annotation_put_fbs("obs", fbs) res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"})) self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file)) self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
self.assertEqual(set(df.columns), set(['cat_A', 'cat_C'])) self.assertEqual(set(df.columns), set(["cat_A", "cat_C"]))
self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)])) self.assertTrue(np.all(df["cat_A"] == ["label_A1" for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)])) self.assertTrue(np.all(df["cat_C"] == ["label_C" for l in range(0, n_rows)]))
# rotation # rotation
name, ext = path.splitext(self.annotations_file) name, ext = path.splitext(self.annotations_file)
@@ -92,10 +94,12 @@ class WritableAnnotationTest(unittest.TestCase):
def test_file_rotation_to_max_9(self): def test_file_rotation_to_max_9(self):
# verify we stop rotation at 9 # verify we stop rotation at 9
n_rows = self.data.data.obs.shape[0] n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({ fbs = self.make_fbs(
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), {
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') "cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"),
}) "cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
for i in range(0, 11): for i in range(0, 11):
res = self.data.annotation_put_fbs("obs", fbs) res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"})) self.assertEqual(res, json.dumps({"status": "OK"}))
@@ -111,10 +115,12 @@ class WritableAnnotationTest(unittest.TestCase):
# GET (annotation_to_fbs_matrix) # GET (annotation_to_fbs_matrix)
n_rows = self.data.data.obs.shape[0] n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({ fbs = self.make_fbs(
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), {
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') "cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"),
}) "cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
# put # put
res = self.data.annotation_put_fbs("obs", fbs) res = self.data.annotation_put_fbs("obs", fbs)
@@ -128,28 +134,21 @@ class WritableAnnotationTest(unittest.TestCase):
self.assertEqual(annotations["n_rows"], n_rows) self.assertEqual(annotations["n_rows"], n_rows)
self.assertEqual(annotations["n_cols"], 7) self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"]) self.assertIsNone(annotations["row_idx"])
self.assertEqual(annotations["col_idx"], [ self.assertEqual(
obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B" annotations["col_idx"],
]) [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
)
col_idx = annotations["col_idx"] col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [ self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A" for l in range(0, n_rows)])
'label_A' for l in range(0, n_rows) self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B" for l in range(0, n_rows)])
])
self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [
'label_B' for l in range(0, n_rows)
])
# verify the schema was updated # verify the schema was updated
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(all_col_schema["cat_A"], { self.assertEqual(
"name": "cat_A", all_col_schema["cat_A"],
"type": "categorical", {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
"categories": ["label_A"], )
"writable": True self.assertEqual(
}) all_col_schema["cat_B"],
self.assertEqual(all_col_schema["cat_B"], { {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
"name": "cat_B", )
"type": "categorical",
"categories": ["label_B"],
"writable": True
})
travis-build.sh
-5
View File
@@ -1,5 +0,0 @@
set -eo pipefail
flake8 server
npm run --prefix client/ build
npm run --prefix client/ unit-test
pytest -s server/test