fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-15 12:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Makefile modularity, test targets, and auto-formatting (#1070)

Browse Source 
* Fix Makefile whitespace and .PHONY use

* Fix Makefile filename

* Modularize Makefile into client and server Makefiles

Part of the reason that the Makefile in the root directory is a bit
complicated is that it tries to handle tasks that can be handled
separately in the client and server modules.

This commit pushes some of the make logic specific to each module into
their own makefiles and calls out to those makefiles from that in the
project root.

* Add auto-formatting to client and server modules

One thing that can make linting faster is auto-formatting. This commit
adds the yapf auto-formatting tool to the server module and uses
eslint's "fix" functionality to speed up the linting/formatting process.

* Add yapf for automatic code formatting

* Add a root test target that calls sub-tests

* Apply yapf to python files

* Do not duplicate npm commands, simply pass through

* Update documentation

* Do not shadow reserved word len

* Add general test target

* Fix make call in dev-env

* Use black instead of yapf

* Run flake8 from the root directory

* Revert "Apply yapf to python files"

This reverts commit cdca128a01.

* Apply black to python code

* Resolve lint errors resulting from black format

* Add explanation of server unit tests in dev guidelines
This commit is contained in:
Matt Weiden
2019-12-27 14:43:37 -08:00
committed by GitHub
parent ec79995be8
commit f3015cb9df
37 changed files with 738 additions and 806 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.travis.yml
View File

@@ -9,23 +9,20 @@ cache:
install:
- set -eo pipefail
- pip install flake8
- make pydist
- make install-dist
- pip install -r server/requirements-dev.txt
- make pydist install-dist dev-env
jobs:
include:
- name: "Branch Tests 3.7"
python: "3.7"
script: ./travis-build.sh
script: make build-client lint unit-test
- name: "Branch Tests 3.6"
python: "3.6"
script: ./travis-build.sh
script: make build-client lint unit-test
- name: "Docker Build"
install: skip
python: "3.6"
script: docker build .
- name: "Smoke Tests"
python: "3.6"
script:
- npm run --prefix client/ smoke-test
script: make smoke-test

139
makefile → Makefile
View File

@@ -5,12 +5,32 @@ CLEANFILES := $(BUILDDIR)/ client/build build dist cellxgene.egg-info
PART ?= patch
# CLEANING
.PHONY: clean
clean: clean-lite clean-server clean-client
# cleaning the client's node_modules is the longest one, so we avoid that if possible
.PHONY: clean-lite
clean-lite:
rm -rf $(CLEANFILES)
clean-%:
cd $(*) && $(MAKE) clean
# BUILDING PACKAGE
build : clean build-server
.PHONY: build
build: clean build-server
@echo "done"
build-server : build-client
.PHONY: build-client
build-client:
cd client && $(MAKE) install build
.PHONY: build-server
build-server: build-client
mkdir -p $(SERVERBUILD)
cp -r server/* $(SERVERBUILD)
cp -r client/build/ $(CLIENTBUILD)
@@ -22,12 +42,9 @@ build-server : build-client
cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/
cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
build-client :
npm install --prefix client/ client
npm run --prefix client build
# If you are actively developing in the server folder use this, dirties the source tree
build-for-server-dev : clean-server build-client
.PHONY: build-for-server-dev
build-for-server-dev: clean-server build-client
mkdir -p server/app/web/static/img
mkdir -p server/app/web/static/js
mkdir -p server/app/web/templates/
@@ -36,124 +53,160 @@ build-for-server-dev : clean-server build-client
cp client/build/favicon.png server/app/web/static/img
cp client/build/service-worker.js server/app/web/static/js/
clean : clean-lite clean-server
rm -rf client/node_modules
# cleaning node_modules is the longest one, so we avoid that if possible
clean-lite :
rm -rf $(CLEANFILES)
# TESTING
.PHONY: test
test: unit-test smoke-test
clean-server :
rm -f server/app/web/templates/index.html
rm -rf server/app/web/static
.PHONY: unit-test
unit-test: unit-test-server unit-test-client
unit-test-%:
cd $(*) && $(MAKE) unit-test
.PHONY: smoke-test
smoke-test:
cd client && $(MAKE) smoke-test
# FORMATTING CODE
.PHOHY: fmt
fmt: fmt-client fmt-py
fmt-client:
cd client && $(MAKE) fmt
fmt-py:
black .
.PHONY: lint
lint:
flake8 server
.PHONY : build build-server build-client build-for-server-dev clean clean-lite clean-server
# CREATING DISTRIBUTION RELEASE
pydist : build
.PHONY: pydist
pydist: build
cd $(BUILDDIR); python setup.py sdist -d ../dist
@echo "done"
.PHONY : pydist
# RELEASE HELPERS
# create new version to commit to master
release-stage-1 : dev-env bump clean-lite gen-package-lock
.PHONY: release-stage-1
release-stage-1: dev-env bump clean-lite gen-package-lock
@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
# build dist and release to dev pypi
release-stage-2 : dev-env pydist twine
.PHONY: release-stage-2
release-stage-2: dev-env pydist twine
@echo "Dist built and uploaded to test.pypi.org"
@echo "Test the install:"
@echo " make install-release-test"
@echo "Then upload to Pypi prod:"
@echo " make twine-prod"
.PHONY: release-stage-final
release-stage-final: twine-prod
@echo "Release uploaded to pypi.org"
# DANGER: releases directly to prod
# use this if you accidently burned a test release version number,
release-directly-to-prod : dev-env pydist twine-prod
.PHONY: release-directly-to-prod
release-directly-to-prod: dev-env pydist twine-prod
@echo "Dist built and uploaded to pypi.org"
@echo "Test the install:"
@echo " make install-release"
dev-env :
.PHONY: dev-env
dev-env:
cd client && $(MAKE) install
pip install -r server/requirements-dev.txt
gui-env : dev-env
.PHONY: gui-env
gui-env: dev-env
pip install -r server/requirements-gui.txt
# give PART=[major, minor, part] as param to make bump
bump :
.PHONY: bump
bump:
bumpversion --config-file .bumpversion.cfg $(PART)
twine :
.PHONY: twine
twine:
twine upload --repository-url https://test.pypi.org/legacy/ dist/*
twine-prod :
.PHONY: twine-prod
twine-prod:
twine upload dist/*
# quicker than re-building client
gen-package-lock :
npm install --prefix client/ client
.PHONY: gen-package-lock
gen-package-lock:
cd client && $(MAKE) install
.PHONY : release-stage-1 release-stage-2 release-stage-final release-burned dev-env bump twine twine-prod gen-package-lock
# INSTALL
# setup.py sucks when you have your library in a separate folder, adding these in to help setup envs
# install from build directory
install : uninstall
.PHONY: install
install: uninstall
cd $(BUILDDIR); pip install -e .
# install from source tree for development
install-dev : uninstall
.PHONY: install-dev
install-dev: uninstall
pip install -e .
# install from test.pypi to test your release
install-release-test : uninstall
.PHONY: install-release-test
install-release-test: uninstall
pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
# install from pypi to test your release
install-release : uninstall
.PHONY: install-release
install-release: uninstall
pip install --no-cache-dir cellxgene
@echo "Installed cellxgene from pypi.org"
# install from dist
install-dist : uninstall
.PHONY: install-dist
install-dist: uninstall
pip install dist/cellxgene*.tar.gz
uninstall :
.PHONY: uninstall
uninstall:
pip uninstall -y cellxgene || :
.PHONY : install install-dev install-release-test install-release uninstall
# GUI
build-assets :
.PHONY: build-assets
build-assets:
pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py
gui-spec-osx : clean-lite gui-env
.PHONY: gui-spec-osx
gui-spec-osx: clean-lite gui-env
pip install -e .[gui]
pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl' --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py
mv cellxgene.spec cellxgene-osx.spec
gui-spec-windows : clean-lite dev-env
.PHONY: gui-spec-windows
gui-spec-windows: clean-lite dev-env
pip install -e .[gui]
pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py
mv cellxgene.spec cellxgene-windows.spec
gui-build-osx : clean-lite
.PHONY: gui-build-osx
gui-build-osx: clean-lite
pyinstaller --clean cellxgene-osx.spec
gui-build-windows : clean-lite
.PHONY: gui-build-windows
gui-build-windows: clean-lite
pyinstaller --clean cellxgene-windows.spec
.PHONY : build-assets gui-spec-osx gui-spec-windows gui-build-osx gui-build-windows

16
client/Makefile Normal file
View File

@@ -0,0 +1,16 @@
.PHONY: clean
clean:
rm -rf node_modules
.PHONY: install
install:
npm install client
.PHONY: build
build:
npm run build
# pass remaining commands through to npm run
%:
npm run $(*)

1
client/package.json
View File

@@ -11,6 +11,7 @@
"clean": "rimraf build",
"dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
"e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
"fmt": "eslint --fix src",
"lint": "eslint src",
"smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
"start": "node server/development.js",

85
dev_docs/developer_guidelines.md
View File

@@ -9,6 +9,38 @@
**All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.**
## Running test suite
Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only.
### Unit tests
Steps to run the all unit tests:
1. Start in the project root directory
1. `make dev-env`
1. `make unit-test`
### End to end tests
End to end tests use two env variables:
* `JEST_ENV` - environment to run end to end tests. Default `dev`
* `prod` - run headless with no slowdown, chromium will not open.
* `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
* `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` make target.
To run end to end tests as they will be run on CI
1. cellxgene should be built and installed as [specified in server dev](#install)
2. `export JEST_ENV='prod'`
3. `export JEST_CXG_PORT=5000`
4. Run `npm run --prefix client/ smoke-test`
Run end to end tests interactively during development
1. cellxgene should be installed as [specified in client dev](#install-1)
2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k`
3. Run `make smoke-test`
4. To debug a failing test `export JEST_ENV='debug'` and re-run.
## Server dev
### Install
* Build the client and put static files in place: `make build-for-server-dev`
@@ -21,11 +53,15 @@
If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded.
### Linter
We use `flake8` to lint code. Travis CI runs `flake8 server`.
We use [`flake8`](https://github.com/PyCQA/flake8) to lint python and [`black`](https://pypi.org/project/black/) for auto-formatting.
To auto-format code run `make fmt`. To run lint checks on the code run `make lint`.
### Test
1. Install development requirements `pip install -r server/requirements-dev.txt`
2. Run tests `pytest server/test`
If you would like to run the server tests individually, follow the steps below
1. Install development requirements `make dev-env`
1. Run `make unit-test` in the `server` directory or `make unit-test-server` in the root directory.
### Tips
* Install in a virtualenv
@@ -33,8 +69,8 @@ We use `flake8` to lint code. Travis CI runs `flake8 server`.
## Client dev
### Install
1. Install prereqs for client: `npm install --prefix client/ client`
2. Install cellxgene server: `pip install -e .` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets.
1. Install prereqs for client: `make dev-env`
2. Install cellxgene server: `make install-dev` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets.
### Launch
To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made.
@@ -49,43 +85,12 @@ To build only the client: `make build-client`
We use `eslint` to lint the code and `prettier` as our code formatter.
### Test
In `client/` directory run `npm run unit-test`
If you would like to run the client tests individually, follow the steps below in the `client` directory
1. For unit tests run `npm run unit-test` or `make unit-test`
1. For the smoke test run `npm run smoke-test` or `make smoke-test`
### Tips
* You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) in `client/` directory run `npm run backend-dev`
## Running tests
Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only.
### Server unit tests
Install development requirements `pip install -r server/requirements-dev.txt`
Run tests `pytest server/test`
### Client unit tests
In `client/` directory run `npm run unit-test`
### End to end tests
End to end tests use two env variables:
* `JEST_ENV` - environment to run end to end tests. Default `dev`
* `prod` - run headless with no slowdown, chromium will not open.
* `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
* `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` npm script
To run end to end tests as they will be run on CI
1. cellxgene should be built and installed as [specified in server dev](#install)
2. `export JEST_ENV='prod'`
3. `export JEST_CXG_PORT='5000'`
4. Run `npm run --prefix client/ smoke-test`
Run end to end tests interactively during development
1. cellxgene should be installed as [specified in client dev](#install-1)
2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k`
3. Run `npm run --prefix client/ e2e`
4. To debug a failing test `export JEST_ENV='debug'` and re-run.

25
pyproject.toml Normal file
View File

@@ -0,0 +1,25 @@
[tool.black]
line-length = 120
target_version = ['py37']
include = '\.pyi?$'
exclude = '''
(
/(
\.eggs # exclude a few common directories in the
| \.git # root of the project
| \.hg
| \.mypy_cache
| \.tox
| \.venv
| venv
| _build
| buck-out
| build
| dist
| server/app/util/fbs/NetEncoding
)/
| server/gui/cellxgene_rc.py
)
'''

8
server/Makefile Normal file
View File

@@ -0,0 +1,8 @@
.PHONY: clean
clean:
rm -f app/web/templates/index.html
rm -rf app/web/static
.PHONY: unit-test
unit-test:
pytest -s test

4
server/__main__.py
View File

@@ -5,11 +5,11 @@ if __package__ is None:
PKG_PATH = Path(__file__).parent
sys.path.insert(0, str(PKG_PATH.parent))
import server # noqa F401
import server # noqa F401
__package__ = PKG_PATH.name
# Main thing
from .cli.cli import cli # noqa F402
from .cli.cli import cli # noqa F402
cli()

4
server/app/driver/driver.py
View File

@@ -33,7 +33,7 @@ class CXGDriver(metaclass=ABCMeta):
"max_category_items": None,
"diffexp_lfc_cutoff": None,
"disable_diffexp": False,
"diffexp_may_be_slow": False
"diffexp_may_be_slow": False,
}
@abstractmethod
@@ -51,7 +51,7 @@ class CXGDriver(metaclass=ABCMeta):
features = {
"cluster": {"available": False},
"layout": {"obs": {"available": False}, "var": {"available": False}},
"diffexp": {"available": True, "interactiveLimit": 50000}
"diffexp": {"available": True, "interactiveLimit": 50000},
}
# TODO - Interactive limit should be generated from the actual available methods see GH issue #94
if self.config["layout"]:

125
server/app/rest_api/rest.py
View File

@@ -8,13 +8,7 @@ from flask_restful import Api, Resource
from server import __version__ as cellxgene_version
from anndata import __version__ as anndata_version
from server.app.util.constants import (
Axis,
DiffExpMode,
JSON_NaN_to_num_warning_msg,
CXGUID,
CXG_ANNO_COLLECTION
)
from server.app.util.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg, CXGUID, CXG_ANNO_COLLECTION
from server.app.util.errors import (
FilterError,
InteractiveError,
@@ -40,41 +34,18 @@ class ConfigAPI(Resource):
config = {
"config": {
"features": [
{
"method": "POST",
"path": "/cluster/",
**current_app.data.features["cluster"],
},
{
"method": "POST",
"path": "/layout/obs",
**current_app.data.features["layout"]["obs"],
},
{
"method": "POST",
"path": "/layout/var",
**current_app.data.features["layout"]["var"],
},
{
"method": "POST",
"path": "/diffexp/",
**current_app.data.features["diffexp"],
},
{"method": "POST", "path": "/cluster/", **current_app.data.features["cluster"]},
{"method": "POST", "path": "/layout/obs", **current_app.data.features["layout"]["obs"]},
{"method": "POST", "path": "/layout/var", **current_app.data.features["layout"]["var"]},
{"method": "POST", "path": "/diffexp/", **current_app.data.features["diffexp"]},
],
"displayNames": {
"engine": f"cellxgene Scanpy engine version ",
"dataset": current_app.config["DATASET_TITLE"],
},
"links": {
"about-dataset": current_app.config["ABOUT_DATASET"]
},
"parameters": {
**current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection)
},
"library_versions": {
"cellxgene": cellxgene_version,
"anndata": anndata_version
}
"links": {"about-dataset": current_app.config["ABOUT_DATASET"]},
"parameters": {**current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection)},
"library_versions": {"cellxgene": cellxgene_version, "anndata": anndata_version},
}
}
@@ -84,17 +55,13 @@ class ConfigAPI(Resource):
class AnnotationsObsAPI(Resource):
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
cxguid = get_userid(session)
anno_collection = get_anno_collection(session)
try:
if preferred_mimetype == "application/octet-stream":
fbs = current_app.data.annotation_to_fbs_matrix("obs", fields, uid=cxguid, collection=anno_collection)
return make_response(fbs,
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
@@ -115,9 +82,7 @@ class AnnotationsObsAPI(Resource):
try:
fbs = request.get_data()
res = current_app.data.annotation_put_fbs("obs", fbs, uid=cxguid, collection=anno_collection)
return make_response(
res, HTTPStatus.OK, {"Content-Type": "application/json"}
)
return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"})
except (ValueError, DisabledFeatureError, KeyError) as e:
return make_response(str(e), HTTPStatus.BAD_REQUEST)
except Exception as e:
@@ -127,14 +92,14 @@ class AnnotationsObsAPI(Resource):
class AnnotationsVarAPI(Resource):
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
try:
if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("var", fields),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
return make_response(
current_app.data.annotation_to_fbs_matrix("var", fields),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"},
)
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
@@ -145,19 +110,16 @@ class AnnotationsVarAPI(Resource):
class DataVarAPI(Resource):
def put(self):
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
try:
if preferred_mimetype == "application/octet-stream":
filter_json = request.get_json()
filter = filter_json["filter"] if filter_json else None
return make_response(
current_app.data.data_frame_to_fbs_matrix(
filter, axis=Axis.VAR
),
current_app.data.data_frame_to_fbs_matrix(filter, axis=Axis.VAR),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
{"Content-Type": "application/octet-stream"},
)
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except FilterError as e:
@@ -175,35 +137,23 @@ class DiffExpObsAPI(Resource):
except KeyError:
return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST)
except ValueError:
return make_response(
f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST
)
return make_response(f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST)
# Validate filters
if mode == DiffExpMode.VAR_FILTER or "varFilter" in args:
# not NOT_IMPLEMENTED
return make_response(
"mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED
)
return make_response("mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED)
if mode == DiffExpMode.TOP_N and "count" not in args:
return make_response(
"mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST
)
return make_response("mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST)
if "set1" not in args:
return make_response("set1 is required.", HTTPStatus.BAD_REQUEST)
if Axis.VAR in args["set1"]["filter"]:
return make_response(
"Var filter not allowed for set1", HTTPStatus.BAD_REQUEST
)
return make_response("Var filter not allowed for set1", HTTPStatus.BAD_REQUEST)
# set2
if "set2" not in args:
return make_response(
"Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED
)
return make_response("Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED)
if Axis.VAR in args["set2"]["filter"]:
return make_response(
"Var filter not allowed for set2", HTTPStatus.BAD_REQUEST
)
return make_response("Var filter not allowed for set2", HTTPStatus.BAD_REQUEST)
set1_filter = args["set1"]["filter"]
set2_filter = args.get("set2", {"filter": {}})["filter"]
@@ -214,14 +164,9 @@ class DiffExpObsAPI(Resource):
count = args.get("count", None)
try:
diffexp = current_app.data.diffexp_topN(
set1_filter,
set2_filter,
count,
current_app.data.features["diffexp"]["interactiveLimit"],
)
return make_response(
diffexp, HTTPStatus.OK, {"Content-Type": "application/json"}
set1_filter, set2_filter, count, current_app.data.features["diffexp"]["interactiveLimit"],
)
return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"})
except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except InteractiveError:
@@ -236,14 +181,12 @@ class DiffExpObsAPI(Resource):
class LayoutObsAPI(Resource):
def get(self):
preferred_mimetype = request.accept_mimetypes.best_match(
["application/octet-stream"]
)
preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"])
try:
if preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.layout_to_fbs_matrix(),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
return make_response(
current_app.data.layout_to_fbs_matrix(), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
)
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except PrepareError as e:
@@ -278,7 +221,7 @@ def is_safe_collection_name(name):
"""
if name is None:
return False
return re.match(r'^\w+$', name) is not None
return re.match(r"^\w+$", name) is not None
def get_api_resources():

4
server/app/scanpy_engine/diffexp.py
View File

@@ -32,8 +32,8 @@ def _mean_var_n(X):
v = sumsq / (n - 1)
if fp_err_occurred:
mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712
mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712
return mean, v, n

8
server/app/scanpy_engine/labels.py
View File

@@ -9,7 +9,7 @@ import pandas as pd
def read_labels(fname):
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#')
return pd.read_csv(fname, dtype="category", index_col=0, header=0, comment="#")
else:
return pd.DataFrame()
@@ -19,12 +19,12 @@ def write_labels(fname, df, header=None, backup_dir=None):
backup(fname, backup_dir)
# rotate_fname(fname, backup_dir)
if not df.empty:
with open(fname, 'w', newline="") as f:
with open(fname, "w", newline="") as f:
if header is not None:
f.write(header)
df.to_csv(f)
else:
open(fname, 'w').close()
open(fname, "w").close()
def backup(fname, backup_dir, max_backups=9):
@@ -46,7 +46,7 @@ def backup(fname, backup_dir, max_backups=9):
fname_base = os.path.basename(fname)
fname_base_root, fname_base_ext = os.path.splitext(fname_base)
# don't use ISO standard time format, as it contains characters illegal on some filesytems.
nowish = datetime.now().strftime('%Y-%m-%dT%H-%M-%S')
nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
if os.path.exists(backup_fname):
os.remove(backup_fname)

13
server/app/scanpy_engine/matrix_proxy.py
View File

@@ -1,4 +1,3 @@
from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView
"""
@@ -17,6 +16,7 @@ class ArrayProxyView_anndata_h5py(ArrayProxyView):
override to handle sparse getitem semantics, which differ
from numpy.
"""
def toarray(self):
""" sadly, sparse indexing doesn't drop dimensions like numpy! """
arr = self.m[self._index[0], self._index[1]]
@@ -30,12 +30,15 @@ class MatrixProxy_anndata_h5py(MatrixProxyView):
AnnData sparse array stored in H5AD, or proxies for backed data.
None of these handle indexing very well, so we plop a proxy on top.
"""
@classmethod
def __supports__(cls):
return ("anndata.h5py.h5sparse.SparseDataset",
"anndata.h5py.h5sparse.backed_csc_matrix",
"anndata.h5py.h5sparse.backed_csr_matrix",
"h5py._hl.dataset.Dataset")
return (
"anndata.h5py.h5sparse.SparseDataset",
"anndata.h5py.h5sparse.backed_csc_matrix",
"anndata.h5py.h5sparse.backed_csr_matrix",
"h5py._hl.dataset.Dataset",
)
@classmethod
def create_array(cls, *args, **kwargs):

164
server/app/scanpy_engine/scanpy_engine.py
View File

@@ -62,7 +62,7 @@ class ScanpyEngine(CXGDriver):
"annotations_output_dir": None,
"backed": False,
"disable_diffexp": False,
"diffexp_may_be_slow": False
"diffexp_may_be_slow": False,
}
def get_config_parameters(self, uid=None, collection=None):
@@ -70,26 +70,25 @@ class ScanpyEngine(CXGDriver):
"max-category-items": self.config["max_category_items"],
"disable-diffexp": self.config["disable_diffexp"],
"diffexp-may-be-slow": self.config["diffexp_may_be_slow"],
"annotations": self.config["annotations"]
"annotations": self.config["annotations"],
}
if self.config["annotations"]:
if uid is not None:
params.update({
"annotations-user-data-idhash": self.get_userdata_idhash(uid)
})
if self.config['annotations_file'] is not None:
params.update({"annotations-user-data-idhash": self.get_userdata_idhash(uid)})
if self.config["annotations_file"] is not None:
# user has hard-wired the name of the annotation data collection
fname = os.path.basename(self.config['annotations_file'])
fname = os.path.basename(self.config["annotations_file"])
collection_fname = os.path.splitext(fname)[0]
params.update({
'annotations-data-collection-is-read-only': True,
'annotations-data-collection-name': collection_fname
})
params.update(
{
"annotations-data-collection-is-read-only": True,
"annotations-data-collection-name": collection_fname,
}
)
elif collection is not None:
params.update({
'annotations-data-collection-is-read-only': False,
'annotations-data-collection-name': collection
})
params.update(
{"annotations-data-collection-is-read-only": False, "annotations-data-collection-name": collection}
)
return params
@staticmethod
@@ -140,23 +139,18 @@ class ScanpyEngine(CXGDriver):
# User has specified alternative column for unique names, and it exists
if not df_axis[name].is_unique:
raise KeyError(
f"Values in {ax_name}.{name} must be unique. "
"Please prepare data to contain unique values."
f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values."
)
df_axis.reset_index(drop=True, inplace=True)
else:
# user specified a non-existent column name
raise KeyError(
f"Annotation name {name}, specified in --{ax_name}-name does not exist."
)
raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.")
@staticmethod
def _can_cast_to_float32(ann):
if ann.dtype.kind == "f":
if not np.can_cast(ann.dtype, np.float32):
warnings.warn(
f"Annotation {ann.name} will be converted to 32 bit float and may lose precision."
)
warnings.warn(f"Annotation {ann.name} will be converted to 32 bit float and may lose precision.")
return True
return False
@@ -188,30 +182,18 @@ class ScanpyEngine(CXGDriver):
schema["type"] = "categorical"
schema["categories"] = dtype.categories.tolist()
else:
raise TypeError(
f"Annotations of type {dtype} are unsupported by cellxgene."
)
raise TypeError(f"Annotations of type {dtype} are unsupported by cellxgene.")
return schema
@requires_data
def _create_schema(self):
self.schema = {
"dataframe": {
"nObs": self.cell_count,
"nVar": self.gene_count,
"type": str(self.data.X.dtype),
},
"dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
"annotations": {
"obs": {
"index": self.config["obs_names"],
"columns": []
},
"var": {
"index": self.config["var_names"],
"columns": []
}
"obs": {"index": self.config["obs_names"], "columns": []},
"var": {"index": self.config["var_names"], "columns": []},
},
"layout": {"obs": []}
"layout": {"obs": []},
}
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
@@ -220,12 +202,8 @@ class ScanpyEngine(CXGDriver):
ann_schema.update(self._get_col_type(curr_axis[ann]))
self.schema["annotations"][ax]["columns"].append(ann_schema)
for layout in self.config['layout']:
layout_schema = {
"name": layout,
"type": "float32",
"dims": [f"{layout}_0", f"{layout}_1"]
}
for layout in self.config["layout"]:
layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]}
self.schema["layout"]["obs"].append(layout_schema)
@requires_data
@@ -250,7 +228,7 @@ class ScanpyEngine(CXGDriver):
Used to create safe annotations output file names.
"""
id = (uid + self.data_locator.abspath()).encode()
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode('utf-8')
idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8")
return idhash
def get_anno_fname(self, uid=None, collection=None):
@@ -272,11 +250,11 @@ class ScanpyEngine(CXGDriver):
if not self.config["annotations"]:
return None
if self.config['annotations_output_dir']:
return self.config['annotations_output_dir']
if self.config["annotations_output_dir"]:
return self.config["annotations_output_dir"]
if self.config['annotations_file']:
return os.path.dirname(os.path.abspath(self.config['annotations_file']))
if self.config["annotations_file"]:
return os.path.dirname(os.path.abspath(self.config["annotations_file"]))
return os.getcwd()
@@ -299,7 +277,7 @@ class ScanpyEngine(CXGDriver):
with data_locator.local_handle() as lh:
# as of AnnData 0.6.19, backed mode performs initial load fast, but at the
# cost of significantly slower access to X data.
backed = 'r' if self.config['backed'] else None
backed = "r" if self.config["backed"] else None
self.data = anndata.read_h5ad(lh, backed=backed)
except ValueError:
@@ -338,7 +316,7 @@ class ScanpyEngine(CXGDriver):
# heuristic
n_values = self.data.shape[0] * self.data.shape[1]
if (n_values > 1e8 and self.config['backed'] is True) or (n_values > 5e8):
if (n_values > 1e8 and self.config["backed"] is True) or (n_values > 5e8):
self.config.update({"diffexp_may_be_slow": True})
@requires_data
@@ -348,7 +326,7 @@ class ScanpyEngine(CXGDriver):
b) validate layouts are legal. remove/warn on any that are not
c) cap total list of layouts at global const MAX_LAYOUTS
"""
layouts = self.config['layout']
layouts = self.config["layout"]
# handle default
if layouts is None or len(layouts) == 0:
# load default layouts from the data.
@@ -372,7 +350,7 @@ class ScanpyEngine(CXGDriver):
raise PrepareError(f"No valid layout data.")
# cap layouts to MAX_LAYOUTS
self.config['layout'] = valid_layouts[0:MAX_LAYOUTS]
self.config["layout"] = valid_layouts[0:MAX_LAYOUTS]
@requires_data
def _is_valid_layout(self, arr):
@@ -394,8 +372,7 @@ class ScanpyEngine(CXGDriver):
)
if self.data.X.dtype != "float32":
warnings.warn(
f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
f"Precision may be truncated."
f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
)
for ax in Axis:
curr_axis = getattr(self.data, str(ax))
@@ -414,7 +391,7 @@ class ScanpyEngine(CXGDriver):
)
if isinstance(datatype, CategoricalDtype):
category_num = len(curr_axis[ann].dtype.categories)
if category_num > 500 and category_num > self.config['max_category_items']:
if category_num > 500 and category_num > self.config["max_category_items"]:
warnings.warn(
f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be "
f"cumbersome or slow to display. We recommend setting the "
@@ -439,14 +416,17 @@ class ScanpyEngine(CXGDriver):
raise KeyError(f"All row index values specified in user annotations must be unique.")
if not labels.index.equals(self.original_obs_index):
raise KeyError("Label file row index does not match H5AD file index. "
"Please ensure that column zero (0) in the label file contain the same "
"index values as the H5AD file.")
raise KeyError(
"Label file row index does not match H5AD file index. "
"Please ensure that column zero (0) in the label file contain the same "
"index values as the H5AD file."
)
duplicate_columns = list(set(labels.columns) & set(self.data.obs.columns))
if len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
raise KeyError(
f"Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}"
)
# labels must have same count as obs annotations
if labels.shape[0] != self.data.obs.shape[0]:
@@ -475,7 +455,7 @@ class ScanpyEngine(CXGDriver):
mask = np.zeros((count,), dtype=bool)
for i in filter:
if type(i) == list:
mask[i[0]: i[1]] = True
mask[i[0] : i[1]] = True
else:
mask[i] = True
return mask
@@ -484,15 +464,10 @@ class ScanpyEngine(CXGDriver):
def _axis_filter_to_mask(filter, d_axis, count):
mask = np.ones((count,), dtype=bool)
if "index" in filter:
mask = np.logical_and(
mask, ScanpyEngine._index_filter_to_mask(filter["index"], count)
)
mask = np.logical_and(mask, ScanpyEngine._index_filter_to_mask(filter["index"], count))
if "annotation_value" in filter:
mask = np.logical_and(
mask,
ScanpyEngine._annotation_filter_to_mask(
filter["annotation_value"], d_axis, count
),
mask, ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"], d_axis, count),
)
return mask
@@ -507,13 +482,9 @@ class ScanpyEngine(CXGDriver):
if filter is not None:
if Axis.OBS in filter:
obs_selector = self._axis_filter_to_mask(
filter["obs"], self.data.obs, self.data.n_obs
)
obs_selector = self._axis_filter_to_mask(filter["obs"], self.data.obs, self.data.n_obs)
if Axis.VAR in filter:
var_selector = self._axis_filter_to_mask(
filter["var"], self.data.var, self.data.n_vars
)
var_selector = self._axis_filter_to_mask(filter["var"], self.data.var, self.data.n_vars)
return obs_selector, var_selector
@requires_data
@@ -531,7 +502,7 @@ class ScanpyEngine(CXGDriver):
labels = None
if labels is not None and not labels.empty:
df = self.data.obs.join(labels, self.config['obs_names'])
df = self.data.obs.join(labels, self.config["obs_names"])
else:
df = self.data.obs
else:
@@ -560,18 +531,21 @@ class ScanpyEngine(CXGDriver):
# if any of the new column labels overlap with our existing labels, raise error
duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns))
if not new_label_df.columns.is_unique or len(duplicate_columns) > 0:
raise KeyError(f"Labels file may not contain column names which overlap "
f"with h5ad obs columns {duplicate_columns}")
raise KeyError(
f"Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}"
)
# update our internal state and save it. Multi-threading often enabled,
# so treat this as a critical section.
with self.label_lock:
lastmod = self.data_locator.lastmodtime()
lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds")
header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \
f"using cellxgene version {cellxgene_version}\n" \
f"# Input data file was {self.data_locator.uri_or_path}, " \
f"which was last modified on {lastmodstr}\n"
header = (
f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} "
f"using cellxgene version {cellxgene_version}\n"
f"# Input data file was {self.data_locator.uri_or_path}, "
f"which was last modified on {lastmodstr}\n"
)
write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection))
return jsonify_scanpy({"status": "OK"})
@@ -598,8 +572,7 @@ class ScanpyEngine(CXGDriver):
raise FilterError("filtering on obs unsupported")
# Currently only handles VAR dimension
X = MatrixProxy.create(self.data.X if var_selector is None
else self.data.X[:, var_selector])
X = MatrixProxy.create(self.data.X if var_selector is None else self.data.X[:, var_selector])
return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None)
@requires_data
@@ -607,25 +580,17 @@ class ScanpyEngine(CXGDriver):
if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
raise FilterError("Observation filters may not contain vaiable conditions")
try:
obs_mask_A = self._axis_filter_to_mask(
obsFilterA["obs"], self.data.obs, self.data.n_obs
)
obs_mask_B = self._axis_filter_to_mask(
obsFilterB["obs"], self.data.obs, self.data.n_obs
)
obs_mask_A = self._axis_filter_to_mask(obsFilterA["obs"], self.data.obs, self.data.n_obs)
obs_mask_B = self._axis_filter_to_mask(obsFilterB["obs"], self.data.obs, self.data.n_obs)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if top_n is None:
top_n = DEFAULT_TOP_N
result = diffexp_ttest(
self.data, obs_mask_A, obs_mask_B, top_n, self.config['diffexp_lfc_cutoff']
)
result = diffexp_ttest(self.data, obs_mask_A, obs_mask_B, top_n, self.config["diffexp_lfc_cutoff"])
try:
return jsonify_scanpy(result)
except ValueError:
raise JSONEncodingValueError(
"Error encoding differential expression to JSON"
)
raise JSONEncodingValueError("Error encoding differential expression to JSON")
@requires_data
def layout_to_fbs_matrix(self):
@@ -661,7 +626,8 @@ class ScanpyEngine(CXGDriver):
except ValueError as e:
raise PrepareError(
f"Layout has not been calculated using {self.config['layout']}, "
f"please prepare your datafile and relaunch cellxgene") from e
f"please prepare your datafile and relaunch cellxgene"
) from e
df = pandas.concat(layout_data, axis=1, copy=False)
return encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)

4
server/app/util/constants.py
View File

@@ -27,9 +27,7 @@ class DiffExpMode(AugmentedEnum):
VAR_FILTER = "varFilter"
JSON_NaN_to_num_warning_msg = (
"JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
)
JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
REACTIVE_LIMIT = 1_000_000
MAX_LAYOUTS = 30

12
server/app/util/data_locator.py
View File

@@ -4,7 +4,7 @@ import fsspec
from datetime import datetime
class DataLocator():
class DataLocator:
"""
DataLocator is a simple wrapper around fsspec functionality, and provides a
set of functions to encapsulate a data location (URI or path), interogate
@@ -29,7 +29,7 @@ class DataLocator():
self.uri_or_path = uri_or_path
self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path)
# work-around for LocalFileSystem not treating file: and None as the same scheme/protocol
self.cname = self.path if self.protocol == 'file' else self.uri_or_path
self.cname = self.path if self.protocol == "file" else self.uri_or_path
# will throw RuntimeError if the protocol is unsupported
self.fs = fsspec.filesystem(self.protocol)
@@ -53,9 +53,9 @@ class DataLocator():
""" return datetime object representing last modification time, or None if unavailable """
info = self.fs.info(self.cname)
if self.islocal() and info is not None:
return datetime.fromtimestamp(info['mtime'])
return datetime.fromtimestamp(info["mtime"])
else:
return getattr(info, 'LastModified', None)
return getattr(info, "LastModified", None)
def abspath(self):
"""
@@ -74,7 +74,7 @@ class DataLocator():
return self.fs.open(self.uri_or_path, *args)
def islocal(self):
return self.protocol is None or self.protocol == 'file'
return self.protocol is None or self.protocol == "file"
def local_handle(self):
if self.islocal():
@@ -90,7 +90,7 @@ class DataLocator():
return LocalFilePath(tmp_path, delete=True)
class LocalFilePath():
class LocalFilePath:
def __init__(self, tmp_path, delete=False):
self.tmp_path = tmp_path
self.delete = delete

31
server/app/util/fbs/matrix.py
View File

@@ -27,7 +27,7 @@ def CreateNumpyVector(builder, x):
if not isinstance(x, np.ndarray):
raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}")
if x.dtype.kind not in ['b', 'i', 'u', 'f']:
if x.dtype.kind not in ["b", "i", "u", "f"]:
raise TypeError("numpy-ndarray holds elements of unsupported datatype")
if x.ndim > 1:
@@ -42,11 +42,11 @@ def CreateNumpyVector(builder, x):
x_little_endian = x.byteswap(inplace=False)
# Calculate total length
len = int(x_little_endian.itemsize * x_little_endian.size)
builder.head = int(builder.Head() - len)
length = int(x_little_endian.itemsize * x_little_endian.size)
builder.head = int(builder.Head() - length)
# tobytes ensures c_contiguous ordering
builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C')
builder.Bytes[builder.Head() : builder.Head() + length] = x_little_endian.tobytes(order="C")
return builder.EndVector(x.size)
@@ -88,9 +88,9 @@ def serialize_typed_array(builder, source_array, encoding_info):
arr = arr.to_series()
# convert to a simple ndarray
if as_type == 'json':
as_json = arr.to_json(orient='records')
arr = np.array(bytearray(as_json, 'utf-8'))
if as_type == "json":
as_json = arr.to_json(orient="records")
arr = np.array(bytearray(as_json, "utf-8"))
else:
if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr):
arr = arr.toarray()
@@ -119,18 +119,16 @@ column_encoding_type_map = {
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
}
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
def column_encoding(arr):
@@ -141,11 +139,10 @@ index_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32),
np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32)
np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32),
}
index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
def index_encoding(arr):
@@ -163,7 +160,7 @@ def guess_at_mem_needed(matrix):
guess = 1
# round up to nearest 1024 bytes
guess = (guess + 0x400) & (~0x3ff)
guess = (guess + 0x400) & (~0x3FF)
return guess
@@ -223,7 +220,7 @@ def deserialize_typed_array(tarr):
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
}
(u_type, u) = tarr
if u_type is TypedArray.TypedArray.NONE:
@@ -237,7 +234,7 @@ def deserialize_typed_array(tarr):
arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode('utf-8'))
narr = json.loads(narr.tostring().decode("utf-8"))
return narr

62
server/app/util/matrix_proxy.py
View File

@@ -18,6 +18,7 @@ class _ArrayProxyBase(abc.ABC):
Private base class for array or matrix proxy. This summarizes
the interface used by the rest of cellxgene.
"""
@property
@abc.abstractmethod
def dtype(self):
@@ -68,10 +69,10 @@ class MatrixProxy(_ArrayProxyBase):
Sub-classes automatically register.
"""
base_proxy_registry = {
'pandas.core.frame.DataFrame': True,
'numpy.ndarray': True,
'scipy.sparse.csc.csc_matrix': True,
'scipy.sparse.csr.csr_matrix': True,
"pandas.core.frame.DataFrame": True,
"numpy.ndarray": True,
"scipy.sparse.csc.csc_matrix": True,
"scipy.sparse.csr.csr_matrix": True,
}
proxy_registry = None
last_cache_token = None
@@ -103,7 +104,7 @@ class MatrixProxy(_ArrayProxyBase):
"""
cls.build_proxy_registry()
t = type(matrix)
fqtn = t.__module__ + '.' + t.__name__
fqtn = t.__module__ + "." + t.__name__
proxy_cls = cls.proxy_registry.get(fqtn, None)
if proxy_cls is None:
raise Exception(f"Matrix format `{fqtn}` is unsupported by proxy.")
@@ -128,21 +129,17 @@ class MatrixProxyView(MatrixProxy):
"""
2D matrix view to a 2D matrix
"""
def __init__(self, arg1, shape=None, index=(),
transposed=False, copy=False):
def __init__(self, arg1, shape=None, index=(), transposed=False, copy=False):
if not copy:
m = arg1
super().__init__(m)
if shape is None:
shape = m.shape
assert(len(shape) == 2)
assert len(shape) == 2
index = tuple(
map(lambda s_i:
slice(0, s_i[0], 1) if s_i[1] is None else s_i[1],
zip_longest(shape, index))
)
index = tuple(map(lambda s_i: slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], zip_longest(shape, index)))
self._shape = shape
self._index = index
@@ -234,20 +231,20 @@ class MatrixProxyView(MatrixProxy):
NOTE: these follow the numpy rules for dimensionality reduction
when an integer index is specified.
"""
def _getitem_intXint(self, row, col):
return self.m[row, col]
def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), )
shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), )
shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__.create_array(self.m, shape=shape, index=(row, col))
def _getitem_sliceXslice(self, row, col):
shape = (_slice_length(row, self.m.shape[0]),
_slice_length(col, self.m.shape[1]))
shape = (_slice_length(row, self.m.shape[0]), _slice_length(col, self.m.shape[1]))
return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed)
def toarray(self):
@@ -261,22 +258,23 @@ class ArrayProxyView(_ArrayProxyBase):
"""
1D array view to a 2D matrix
"""
def __init__(self, arg1, shape=None, index=None, copy=False):
super().__init__()
if not copy:
m = arg1
# one index MUST be an integer and the other MUST be a slice
assert(len(index) == 2)
assert(all(isinstance(idx, INT_TYPES + (slice, )) for idx in index))
assert(isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES))
assert len(index) == 2
assert all(isinstance(idx, INT_TYPES + (slice,)) for idx in index)
assert isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES)
if shape is None:
if isinstance(index[0], INT_TYPES):
shape = (m.shape[0], )
shape = (m.shape[0],)
else:
shape = (m.shape[1], )
assert(len(shape) == 1)
shape = (m.shape[1],)
assert len(shape) == 1
self._shape = shape
self.m = m
@@ -336,7 +334,7 @@ class ArrayProxyView(_ArrayProxyBase):
elif isinstance(col, slice):
return self._getitem_intXslice(row, col)
elif isinstance(row, slice):
assert(isinstance(col, INT_TYPES))
assert isinstance(col, INT_TYPES)
return self._getitem_sliceXint(row, col)
raise IndexError("unsupported column index types")
@@ -345,11 +343,11 @@ class ArrayProxyView(_ArrayProxyBase):
return self.m[row, col]
def _getitem_intXslice(self, row, col):
shape = (_slice_length(col, self.m.shape[1]), )
shape = (_slice_length(col, self.m.shape[1]),)
return self.__class__(self.m, shape=shape, index=(row, col))
def _getitem_sliceXint(self, row, col):
shape = (_slice_length(row, self.m.shape[0]), )
shape = (_slice_length(row, self.m.shape[0]),)
return self.__class__(self.m, shape=shape, index=(row, col))
def toarray(self):
@@ -358,7 +356,7 @@ class ArrayProxyView(_ArrayProxyBase):
def _unpack_index(index, shape):
if not isinstance(index, tuple):
index = (index, )
index = (index,)
if len(shape) < len(index):
raise IndexError("invalid index dimensionality - must be 2")
@@ -366,7 +364,7 @@ def _unpack_index(index, shape):
for shp, idx in zip_longest(shape, index):
idx = slice(None) if idx is None else idx
idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx
unpacked += (idx, )
unpacked += (idx,)
return unpacked
@@ -376,7 +374,7 @@ def _slice_slice(outer, outer_len, inner, inner_len):
slice a slice - we take advantage of Python 3 range's support
for indexing.
"""
assert(outer_len >= inner_len)
assert outer_len >= inner_len
outer_rng = range(*outer.indices(outer_len))
rng = outer_rng[inner]
start, stop, step = rng.start, rng.stop, rng.step
@@ -387,8 +385,8 @@ def _slice_slice(outer, outer_len, inner, inner_len):
def _range_length(start, stop, step):
""" return length of range """
assert(step != 0)
assert(start is not None and stop is not None and step is not None)
assert step != 0
assert start is not None and stop is not None and step is not None
if step > 0 and start < stop:
return 1 + (stop - 1 - start) // step
elif step < 0 and start > stop:
@@ -404,7 +402,7 @@ def _slice_length(s, length):
def _slice_defaults(s, length):
""" apply slice defaulting conventions """
assert(length >= 0)
assert length >= 0
step = 1 if s.step is None else s.step

1
server/app/util/utils.py
View File

@@ -40,4 +40,5 @@ def requires_data(func):
if self.data is None:
raise DriverError(f"error data must be loaded before you call {func.__name__}")
return func(self, *args, **kwargs)
return wrapped_function

14
server/cli/cli.py
View File

@@ -4,17 +4,19 @@ from .launch import launch
from .prepare import prepare
@click.group(name="cellxgene",
subcommand_metavar="COMMAND <args>",
options_metavar="<options>",
context_settings=dict(max_content_width=85,
help_option_names=['-h', '--help']))
@click.group(
name="cellxgene",
subcommand_metavar="COMMAND <args>",
options_metavar="<options>",
context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
)
@click.help_option("--help", "-h", help="Show this message and exit.")
@click.version_option(
version="0.13.0",
prog_name="cellxgene",
message="[%(prog)s] Version %(version)s",
help="Show the software version and exit.")
help="Show the software version and exit.",
)
def cli():
pass

164
server/cli/launch.py
View File

@@ -25,16 +25,12 @@ def common_args(func):
Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args.
"""
@click.option(
"--title",
"-t",
metavar="<text>",
help="Title to display. If omitted will use file name.")
@click.option("--title", "-t", metavar="<text>", help="Title to display. If omitted will use file name.")
@click.option(
"--about",
metavar="<URL>",
help="URL providing more information about the dataset "
"(hint: must be a fully specified absolute URL).")
help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).",
)
@click.option(
"--embedding",
"-e",
@@ -42,39 +38,43 @@ def common_args(func):
multiple=True,
show_default=False,
metavar="<text>",
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all."
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
)
@click.option(
"--obs-names",
"-obs",
default=None,
metavar="<text>",
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.")
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
)
@click.option(
"--var-names",
"-var",
default=None,
metavar="<text>",
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.")
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
)
@click.option(
"--max-category-items",
default=1000,
metavar="<integer>",
show_default=True,
help="Will not display categories with more distinct values than specified.",)
help="Will not display categories with more distinct values than specified.",
)
@click.option(
"--diffexp-lfc-cutoff",
"-de",
default=0.01,
show_default=True,
metavar="<float>",
help="Minimum log fold change threshold for differential expression.",)
help="Minimum log fold change threshold for differential expression.",
)
@click.option(
"--experimental-annotations",
is_flag=True,
default=False,
show_default=True,
help="Enable user annotation of data."
help="Enable user annotation of data.",
)
@click.option(
"--experimental-annotations-file",
@@ -83,7 +83,8 @@ def common_args(func):
multiple=False,
metavar="<path>",
help="CSV file to initialize editing of existing annotations; will be altered in-place. "
"Incompatible with --annotations-output-dir.",)
"Incompatible with --annotations-output-dir.",
)
@click.option(
"--experimental-annotations-output-dir",
default=None,
@@ -91,20 +92,23 @@ def common_args(func):
multiple=False,
metavar="<directory path>",
help="Directory of where to save output annotations; filename will be specified in the application. "
"Incompatible with --annotations-input-file.",)
"Incompatible with --annotations-input-file.",
)
@click.option(
"--backed",
"-b",
is_flag=True,
default=False,
show_default=False,
help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.")
help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.",
)
@click.option(
"--disable-diffexp",
is_flag=True,
default=False,
show_default=False,
help="Disable on-demand differential expression.")
help="Disable on-demand differential expression.",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
@@ -112,9 +116,18 @@ def common_args(func):
return wrapper
def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff,
experimental_annotations, experimental_annotations_file,
experimental_annotations_output_dir, backed, disable_diffexp):
def parse_engine_args(
embedding,
obs_names,
var_names,
max_category_items,
diffexp_lfc_cutoff,
experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
disable_diffexp,
):
annotations_file = experimental_annotations_file if experimental_annotations else None
annotations_output_dir = experimental_annotations_output_dir if experimental_annotations else None
return {
@@ -127,14 +140,15 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
"annotations_file": annotations_file,
"annotations_output_dir": annotations_output_dir,
"backed": backed,
"disable_diffexp": disable_diffexp
"disable_diffexp": disable_diffexp,
}
@sort_options
@click.command(short_help="Launch the cellxgene data viewer. "
"Run `cellxgene launch --help` for more information.",
options_metavar="<options>",)
@click.command(
short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
options_metavar="<options>",
)
@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
@click.option(
"--verbose",
@@ -142,7 +156,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
is_flag=True,
default=False,
show_default=True,
help="Provide verbose output, including warnings and all server requests.",)
help="Provide verbose output, including warnings and all server requests.",
)
@click.option(
"--debug",
"-d",
@@ -150,7 +165,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
default=False,
show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",)
"or when you want more information about an error condition.",
)
@click.option(
"--open",
"-o",
@@ -158,19 +174,22 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
is_flag=True,
default=False,
show_default=True,
help="Open web browser after launch.",)
help="Open web browser after launch.",
)
@click.option(
"--port",
"-p",
metavar="<port>",
show_default=True,
help="Port to run server on. If not specified cellxgene will find an available port.",)
help="Port to run server on. If not specified cellxgene will find an available port.",
)
@click.option(
"--host",
metavar="<IP address>",
default="127.0.0.1",
show_default=False,
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).")
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
)
@click.option(
"--scripts",
"-s",
@@ -178,30 +197,31 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe
multiple=True,
metavar="<text>",
help="Additional script files to include in HTML page. If not specified, "
"no additional script files will be included.",
show_default=False,)
"no additional script files will be included.",
show_default=False,
)
@click.help_option("--help", "-h", help="Show this message and exit.")
@common_args
def launch(
data,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
diffexp_lfc_cutoff,
title,
scripts,
about,
experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
disable_diffexp
data,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
diffexp_lfc_cutoff,
title,
scripts,
about,
experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
disable_diffexp,
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
@@ -217,13 +237,18 @@ def launch(
> cellxgene launch <url>"""
e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items,
diffexp_lfc_cutoff,
experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
disable_diffexp)
e_args = parse_engine_args(
embedding,
obs_names,
var_names,
max_category_items,
diffexp_lfc_cutoff,
experimental_annotations,
experimental_annotations_file,
experimental_annotations_output_dir,
backed,
disable_diffexp,
)
try:
data_locator = DataLocator(data)
except RuntimeError as re:
@@ -256,7 +281,8 @@ def launch(
sys.tracebacklimit = 0
if scripts:
click.echo(r"""
click.echo(
r"""
/ / /\ \ \__ _ _ __ _ __ (_)_ __ __ _
\ \/ \/ / _` | '__| '_ \| | '_ \ / _` |
\ /\ / (_| | | | | | | | | | | (_| |
@@ -264,7 +290,8 @@ def launch(
|___/
The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a
security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
""")
"""
)
scripts_pretty = ", ".join(scripts)
click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
@@ -289,8 +316,9 @@ def launch(
click.echo("Warning: --experimental-annotations-output-dir ignored as --annotations not enabled.")
else:
if experimental_annotations_file is not None and experimental_annotations_output_dir is not None:
raise click.ClickException("--experimental-annotations-file and --experimental-annotations-output-dir "
"may not be used together.")
raise click.ClickException(
"--experimental-annotations-file and --experimental-annotations-output-dir " "may not be used together."
)
if experimental_annotations_file is not None:
lf_name, lf_ext = splitext(experimental_annotations_file)
@@ -301,10 +329,12 @@ def launch(
try:
mkdir(experimental_annotations_output_dir)
except OSError:
raise click.ClickException("Unable to create directory specified by "
"--experimental-annotations-output-dir")
raise click.ClickException(
"Unable to create directory specified by " "--experimental-annotations-output-dir"
)
if about:
def url_check(url):
try:
result = urlparse(url)
@@ -346,9 +376,11 @@ def launch(
except ScanpyFileError as e:
raise click.ClickException(f"{e}")
if not disable_diffexp and server.app.data.config['diffexp_may_be_slow']:
click.echo(f"[cellxgene] CAUTION: due to the size of your dataset, "
f"running differential expression may take longer or fail.")
if not disable_diffexp and server.app.data.config["diffexp_may_be_slow"]:
click.echo(
f"[cellxgene] CAUTION: due to the size of your dataset, "
f"running differential expression may take longer or fail."
)
if open_browser:
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")

51
server/cli/prepare.py
View File

@@ -8,9 +8,10 @@ from server.utils.utils import sort_options
@sort_options
@click.command(short_help="Preprocess data for use with cellxgene. "
"Run `cellxgene prepare --help` for more information.",
options_metavar="<options>",)
@click.command(
short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.",
options_metavar="<options>",
)
@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
@click.option(
"--embedding",
@@ -35,37 +36,33 @@ from server.utils.utils import sort_options
@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
@click.option("--skip-qc", default=False, is_flag=True,
help="Do not run quality control metrics. By default cellxgene runs them "
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).")
@click.option(
"--make-obs-names-unique",
default=True,
"--skip-qc",
default=False,
is_flag=True,
help="Ensure obs index is unique.",
show_default=True
help="Do not run quality control metrics. By default cellxgene runs them "
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
)
@click.option(
"--make-var-names-unique",
default=True,
is_flag=True,
help="Ensure var index is unique.",
show_default=True
"--make-obs-names-unique", default=True, is_flag=True, help="Ensure obs index is unique.", show_default=True
)
@click.option(
"--make-var-names-unique", default=True, is_flag=True, help="Ensure var index is unique.", show_default=True
)
@click.help_option("--help", "-h", help="Show this message and exit.")
def prepare(
data,
embedding,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
skip_qc,
make_obs_names_unique,
make_var_names_unique,
data,
embedding,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
skip_qc,
make_obs_names_unique,
make_var_names_unique,
):
"""
Preprocess data for use with cellxgene.

7
server/gui/browser.py
View File

@@ -12,6 +12,7 @@ WindowUtils = cef.WindowUtils()
# noinspection PyUnresolvedReferences
CefWidgetParent = QWidget
class CefWidget(CefWidgetParent):
def __init__(self, parent=None):
super(CefWidget, self).__init__(parent)
@@ -58,8 +59,7 @@ class CefWidget(CefWidgetParent):
if WINDOWS:
WindowUtils.OnSize(self.getHandle(), 0, 0, 0)
elif LINUX:
self.browser.SetBounds(self.x, self.y,
self.width(), self.height())
self.browser.SetBounds(self.x, self.y, self.width(), self.height())
self.browser.NotifyMoveOrResizeStarted()
def resizeEvent(self, event):
@@ -68,8 +68,7 @@ class CefWidget(CefWidgetParent):
if WINDOWS:
WindowUtils.OnSize(self.getHandle(), 0, 0, 0)
elif LINUX:
self.browser.SetBounds(self.x, self.y,
size.width(), size.height())
self.browser.SetBounds(self.x, self.y, size.width(), size.height())
self.browser.NotifyMoveOrResizeStarted()

48
server/gui/hook-cefpython3.py
View File

@@ -37,8 +37,7 @@ logger = logging.getLogger(__name__)
# Functions
def check_platforms():
if not is_win and not is_darwin and not is_linux:
raise SystemExit("Error: Currently only Windows, Linux and Darwin "
"platforms are supported, see Issue #135.")
raise SystemExit("Error: Currently only Windows, Linux and Darwin " "platforms are supported, see Issue #135.")
def check_pyinstaller_version():
@@ -50,22 +49,19 @@ def check_pyinstaller_version():
version = PyInstaller.__version__
match = re.search(r"^\d+\.\d+(\.\d+)?", version)
if not (match.group(0) >= PYINSTALLER_MIN_VERSION):
raise SystemExit("Error: pyinstaller %s or higher is required"
% PYINSTALLER_MIN_VERSION)
raise SystemExit("Error: pyinstaller %s or higher is required" % PYINSTALLER_MIN_VERSION)
def check_cefpython3_version():
if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION):
raise SystemExit("Error: cefpython3 %s or higher is required"
% CEFPYTHON_MIN_VERSION)
raise SystemExit("Error: cefpython3 %s or higher is required" % CEFPYTHON_MIN_VERSION)
def get_cefpython_modules():
"""Get all cefpython Cython modules in the cefpython3 package.
It returns a list of names without file extension. Eg.
'cefpython_py27'. """
pyds = glob.glob(os.path.join(CEFPYTHON3_DIR,
"cefpython_py*" + CYTHON_MODULE_EXT))
pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, "cefpython_py*" + CYTHON_MODULE_EXT))
assert len(pyds) > 1, "Missing cefpython3 Cython modules"
modules = []
for path in pyds:
@@ -130,14 +126,21 @@ def get_cefpython3_datas():
for filename in os.listdir(CEFPYTHON3_DIR):
# Ignore Cython modules which are already handled by
# pyinstaller automatically.
if filename[:-len(CYTHON_MODULE_EXT)] in get_cefpython_modules():
if filename[: -len(CYTHON_MODULE_EXT)] in get_cefpython_modules():
continue
# CEF binaries and datas
extension = os.path.splitext(filename)[1]
if extension in \
[".exe", ".dll", ".pak", ".dat", ".bin", ".txt", ".so", ".plist"] \
or filename.lower().startswith("license"):
if extension in [
".exe",
".dll",
".pak",
".dat",
".bin",
".txt",
".so",
".plist",
] or filename.lower().startswith("license"):
logger.info("Include cefpython3 data: {}".format(filename))
ret.append((os.path.join(CEFPYTHON3_DIR, filename), cefdatadir))
@@ -145,11 +148,9 @@ def get_cefpython3_datas():
# "Chromium Embedded Framework.framework/Resources" with subdirectories
# is required. Contain .pak files and locales (each locale in separate
# subdirectory).
resources_subdir = \
os.path.join("Chromium Embedded Framework.framework", "Resources")
resources_subdir = os.path.join("Chromium Embedded Framework.framework", "Resources")
base_path = os.path.join(CEFPYTHON3_DIR, resources_subdir)
assert os.path.exists(base_path), \
"{} dir not found in cefpython3".format(resources_subdir)
assert os.path.exists(base_path), "{} dir not found in cefpython3".format(resources_subdir)
for path, dirs, files in os.walk(base_path):
for file in files:
absolute_file_path = os.path.join(path, file)
@@ -159,22 +160,17 @@ def get_cefpython3_datas():
elif is_win or is_linux:
# The .pak files in cefpython3/locales/ directory
locales_dir = os.path.join(CEFPYTHON3_DIR, "locales")
assert os.path.exists(locales_dir), \
"locales/ dir not found in cefpython3"
assert os.path.exists(locales_dir), "locales/ dir not found in cefpython3"
for filename in os.listdir(locales_dir):
logger.info("Include cefpython3 data: {}/{}".format(
os.path.basename(locales_dir), filename))
ret.append((os.path.join(locales_dir, filename),
os.path.join(cefdatadir, "locales")))
logger.info("Include cefpython3 data: {}/{}".format(os.path.basename(locales_dir), filename))
ret.append((os.path.join(locales_dir, filename), os.path.join(cefdatadir, "locales")))
# Optional .so/.dll files in cefpython3/swiftshader/ directory
swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader")
if os.path.isdir(swiftshader_dir):
for filename in os.listdir(swiftshader_dir):
logger.info("Include cefpython3 data: {}/{}".format(
os.path.basename(swiftshader_dir), filename))
ret.append((os.path.join(swiftshader_dir, filename),
os.path.join(cefdatadir, "swiftshader")))
logger.info("Include cefpython3 data: {}/{}".format(os.path.basename(swiftshader_dir), filename))
ret.append((os.path.join(swiftshader_dir, filename), os.path.join(cefdatadir, "swiftshader")))
return ret

28
server/gui/main.py
View File

@@ -20,8 +20,8 @@ from server.utils.utils import find_available_port
if WINDOWS or LINUX:
dirname = dirname(PySide2.__file__)
plugin_path = join(dirname, 'plugins', 'platforms')
environ['QT_QPA_PLATFORM_PLUGIN_PATH'] = plugin_path
plugin_path = join(dirname, "plugins", "platforms")
environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = plugin_path
# Configuration
# TODO remember this or calculate it?
@@ -94,8 +94,7 @@ class MainWindow(QMainWindow):
# a hidden window, embed CEF browser in it and then
# create a container for that hidden window and replace
# cef widget in the layout with the container.
self.container = QWidget.createWindowContainer(
self.cef_widget.hidden_window, parent=self)
self.container = QWidget.createWindowContainer(self.cef_widget.hidden_window, parent=self)
self.stacked_layout.replaceWidget(self.cef_widget, self.container)
self.stacked_layout.setCurrentIndex(LOAD_INDEX)
@@ -117,7 +116,7 @@ class MainWindow(QMainWindow):
def setupMenu(self):
# TODO add communication to subprocess on reload
main_menu = self.menuBar()
file_menu = main_menu.addMenu('File')
file_menu = main_menu.addMenu("File")
load_action = QAction("Load file...", self)
load_action.setStatusTip("Load file")
load_action.setShortcut("Ctrl+O")
@@ -201,7 +200,7 @@ class LoadWidget(QFrame):
for l in [logo_layout, file_layout, message_layout]:
load_ui_layout.addLayout(l)
#TODO remove magic number
# TODO remove magic number
load_ui_layout.setStretch(1, 10)
self.setLayout(load_ui_layout)
@@ -240,8 +239,15 @@ class LoadWidget(QFrame):
def createScanpyEngine(self, file_name):
title = splitext(basename(file_name))[0]
self.window().setupServer()
worker = Worker(self.window().parent_conn, self.window().child_conn, file_name, host="127.0.0.1",
port=GUI_PORT, title=title, engine_options={})
worker = Worker(
self.window().parent_conn,
self.window().child_conn,
file_name,
host="127.0.0.1",
port=GUI_PORT,
title=title,
engine_options={},
)
self.window().load_emitter.signals.ready.connect(self.onDataReady)
self.window().load_emitter.signals.engine_error.connect(self.onServerError)
self.window().load_emitter.signals.server_error.connect(self.onServerError)
@@ -289,6 +295,7 @@ class LoadWidget(QFrame):
onServerError = partialmethod(onError, server_error=True)
class FilePath(QObject):
def __init__(self):
super(FilePath, self).__init__()
@@ -320,8 +327,7 @@ class FileArea(QFrame):
def fileBrowse(self):
options = QFileDialog.Options()
# options |= QFileDialog.DontUseNativeDialog
file_name, _ = QFileDialog.getOpenFileName(self,
"Open H5AD File", "", "H5AD Files (*.h5ad)", options=options)
file_name, _ = QFileDialog.getOpenFileName(self, "Open H5AD File", "", "H5AD Files (*.h5ad)", options=options)
if file_name:
self.parent().file_name.updateValue(file_name)
self.parent().onLoad()
@@ -391,5 +397,5 @@ def main():
sys.exit(0)
if __name__ == '__main__':
if __name__ == "__main__":
main()

8
server/gui/utils.py
View File

@@ -4,9 +4,9 @@ import platform
from PySide2.QtCore import QObject, Signal
# Detect OS
WINDOWS = (platform.system() == "Windows")
LINUX = (platform.system() == "Linux")
MAC = (platform.system() == "Darwin")
WINDOWS = platform.system() == "Windows"
LINUX = platform.system() == "Linux"
MAC = platform.system() == "Darwin"
class WorkerSignals(QObject):
@@ -18,6 +18,7 @@ class WorkerSignals(QObject):
error - `str` error message
result - `object` data returned from processing, anything
"""
finished = Signal()
engine_error = Signal(str)
server_error = Signal(str)
@@ -34,6 +35,7 @@ class SiteReadySignals(QObject):
ready
error - `str` error message
"""
ready = Signal()
timeout = Signal()
error = Signal(str)

1
server/gui/workers.py
View File

@@ -36,6 +36,7 @@ class Worker(EmittingProcess):
return
from server.app.app import Server
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
# create server
try:
server = Server()

15
server/test/decode_fbs.py
View File

@@ -1,4 +1,3 @@
"""
Code to decode, for testing purposes, the flatbuffer encoded blobs.
This code will need to be updated if fbs/matrix.fbs changes.
@@ -22,20 +21,20 @@ def decode_typed_array(tarr):
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
}
(u_type, u) = tarr
if u_type == TypedArray.TypedArray.NONE:
return None
TarType = type_map.get(u_type, None)
assert(TarType is not None)
assert TarType is not None
arr = TarType()
arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode('utf-8'))
narr = json.loads(narr.tostring().decode("utf-8"))
return narr
@@ -60,10 +59,4 @@ def decode_matrix_FBS(buf):
cidx = decode_typed_array((df.ColIndexType(), df.ColIndex()))
return {
"n_rows": n_rows,
"n_cols": n_cols,
"columns": decoded_columns,
"col_idx": cidx,
"row_idx": None
}
return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None}

109
server/test/test_api.py
View File

@@ -19,7 +19,7 @@ class EndPoints(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
cls.ps = Popen(["cellxgene", "launch", "../example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
session = requests.Session()
for i in range(90):
try:
@@ -68,14 +68,15 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 8)
self.assertIsNotNone(df['columns'])
self.assertListEqual(df['col_idx'], [
'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1'
])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 8)
self.assertIsNotNone(df["columns"])
self.assertListEqual(
df["col_idx"],
["pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"],
)
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_bad_filter(self):
endpoint = "data/var"
@@ -91,14 +92,14 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 5)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 5)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
self.assertListEqual(df["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
@@ -109,13 +110,13 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito'])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"], ["n_genes", "percent_mito"])
def test_get_annotations_obs_error(self):
endpoint = "annotations/obs"
@@ -162,14 +163,14 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])
self.assertListEqual(df["col_idx"], [var_index_col_name, "n_cells"])
def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var"
@@ -180,13 +181,13 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 1)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['n_cells'])
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 1)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"], ["n_cells"])
def test_get_annotations_var_error(self):
endpoint = "annotations/var"
@@ -217,35 +218,29 @@ class EndPoints(unittest.TestCase):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 1838)
self.assertIsNotNone(df['columns'])
self.assertListEqual(df['col_idx'].tolist(), [])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1838)
self.assertIsNotNone(df["columns"])
self.assertListEqual(df["col_idx"].tolist(), [])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_data_put_filter_fbs(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
filter = {
"filter": {
"var": {
"index": [0, 1, 4]
}
}
}
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 3)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4])
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 3)
self.assertIsNotNone(df["columns"])
self.assertIsNotNone(df["col_idx"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
def test_data_put_single_var(self):
endpoint = f"data/var"

43
server/test/test_fbs.py
View File

@@ -32,7 +32,7 @@ class FbsTests(unittest.TestCase):
for i in range(0, len(d["columns"])):
self.assertEqual(len(d["columns"][i]), dims[0])
self.assertIsInstance(d["columns"][i], expected_types[i][0])
if (expected_types[i][1] is not None):
if expected_types[i][1] is not None:
self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
if expected_column_idx is not None:
self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
@@ -40,48 +40,37 @@ class FbsTests(unittest.TestCase):
def test_encode_DataFrame(self):
df = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
})
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.int32),
(np.ndarray, np.uint32),
(list, None)
"a": np.zeros((10,), dtype=np.float32),
"b": np.ones((10,), dtype=np.int64),
"c": np.array([i for i in range(0, 10)], dtype=np.uint16),
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}
)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd'])
self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
def test_encode_ndarray(self):
arr = np.zeros((3, 2), dtype=np.float32)
expected_types = (
(np.ndarray, np.float32),
(np.ndarray, np.float32),
(np.ndarray, np.float32)
)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (3, 2), expected_types, None)
def test_encode_sparse(self):
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
expected_types = (
(np.ndarray, np.int32),
(np.ndarray, np.int32),
(np.ndarray, np.int32)
)
expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_roundtrip(self):
dfSrc = pd.DataFrame(
data={
'a': np.zeros((10,), dtype=np.float32),
'b': np.ones((10,), dtype=np.int64),
'c': np.array([i for i in range(0, 10)], dtype=np.uint16),
'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category')
})
"a": np.zeros((10,), dtype=np.float32),
"b": np.ones((10,), dtype=np.int64),
"c": np.array([i for i in range(0, 10)], dtype=np.uint16),
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}
)
dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
self.assertEqual(dfSrc.shape, dfDst.shape)
self.assertEqual(set(dfSrc.columns), set(dfDst.columns))

134
server/test/test_matrix_proxy.py
View File

@@ -7,9 +7,10 @@ class NdArrayProxyView(MatrixProxyView):
"""
Fake test class for matrix proxy - wraps ndarray
"""
@classmethod
def __supports__(cls):
return ('numpy.ndarray', )
return ("numpy.ndarray",)
class MatrixProxyViewTest(unittest.TestCase):
@@ -41,18 +42,17 @@ class MatrixProxyViewTest(unittest.TestCase):
def test_toarray(self):
n = np.arange(15, dtype=np.float32).reshape((3, 5))
mp = MatrixProxy.create(n)
self.assertTrue(np.all(mp.toarray() == [
[0., 1., 2., 3., 4.],
[5., 6., 7., 8., 9.],
[10., 11., 12., 13., 14.]
]))
self.assertTrue(np.all(mp.T.toarray() == [
[0., 5., 10.],
[1., 6., 11.],
[2., 7., 12.],
[3., 8., 13.],
[4., 9., 14.]
]))
self.assertTrue(
np.all(
mp.toarray() == [[0.0, 1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0, 9.0], [10.0, 11.0, 12.0, 13.0, 14.0]]
)
)
self.assertTrue(
np.all(
mp.T.toarray()
== [[0.0, 5.0, 10.0], [1.0, 6.0, 11.0], [2.0, 7.0, 12.0], [3.0, 8.0, 13.0], [4.0, 9.0, 14.0]]
)
)
def test_indexing(self):
"""
@@ -95,47 +95,19 @@ class MatrixProxyViewTest(unittest.TestCase):
# slice, slice
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [
[7, 8],
[12, 13]
]))
self.assertTrue(np.all(mp[:3, :4].toarray() == [
[0., 1., 2., 3.],
[5., 6., 7., 8.],
[10., 11., 12., 13.]
]))
self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [
[14, 13, 12, 11, 10],
[9, 8, 7, 6, 5],
[4, 3, 2, 1, 0]
]))
self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [
[14, 12, 10],
[4, 2, 0]
]))
self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]]))
self.assertTrue(
np.all(mp[:3, :4].toarray() == [[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0], [10.0, 11.0, 12.0, 13.0]])
)
self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]]))
self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]]))
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [
[7, 12],
[8, 13]
]))
self.assertTrue(np.all(mp.T[:4, :3].toarray() == [
[0, 5, 10],
[1, 6, 11],
[2, 7, 12],
[3, 8, 13]
]))
self.assertTrue(np.all(mp.T[::-1, ::-1].toarray() == [
[14, 9, 4],
[13, 8, 3],
[12, 7, 2],
[11, 6, 1],
[10, 5, 0]
]))
self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [
[14, 4],
[12, 2],
[10, 0]
]))
self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]]))
self.assertTrue(np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8, 13]]))
self.assertTrue(
np.all(mp.T[::-1, ::-1].toarray() == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]])
)
self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]]))
def test_repeated_indexing(self):
"""
@@ -149,31 +121,15 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0][1], 1)
self.assertEqual(mp.T[0][1], 5)
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [
2, 1
]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [
2, 3
]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [
3, 2
]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [
2, 3
]))
self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3]))
self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2]))
self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [2, 3]))
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [
2, 1
]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [
10
]))
self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10]))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == []))
self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == []))
@@ -190,20 +146,12 @@ class MatrixProxyViewTest(unittest.TestCase):
self.assertEqual(mp[0, 0], 0)
# drop 1 dimension, to an array
self.assertTrue(np.all(mp[0, :].toarray() == [
0, 1, 2, 3, 4
]))
self.assertTrue(np.all(mp[:, 0].toarray() == [
0, 5, 10
]))
self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4]))
self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10]))
# with .T
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [
[4, 9]
]))
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [
4, 9
]))
self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]]))
self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9]))
def test_iter(self):
"""
@@ -214,17 +162,13 @@ class MatrixProxyViewTest(unittest.TestCase):
rows = [r for r in mp]
self.assertEqual(len(rows), 3)
self.assertTrue(np.all(rows[0].toarray() == [
0, 1, 2, 3, 4
]))
self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4]))
for i, r in enumerate(rows):
self.assertTrue(np.all(mp[i].toarray() == r.toarray()))
cols = [c for c in mp.T]
self.assertEqual(len(cols), 5)
self.assertTrue(np.all(cols[0].toarray() == [
0, 5, 10
]))
self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10]))
for i, c in enumerate(cols):
self.assertTrue(np.all(mp.T[i].toarray() == c.toarray()))

4
server/test/test_nan_rest.py
View File

@@ -20,9 +20,7 @@ class WithNaNs(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.ps = Popen(
["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]
)
cls.ps = Popen(["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"])
session = requests.Session()
for i in range(90):
try:

15
server/test/test_nan_scanpy_engine.py
View File

@@ -21,12 +21,12 @@ class NaNTest(unittest.TestCase):
}
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
self.data = ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args)
self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
self.data._create_schema()
def test_load(self):
with self.assertWarns(UserWarning):
ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args)
ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args)
def test_init(self):
self.assertEqual(self.data.cell_count, 100)
@@ -44,11 +44,7 @@ class NaNTest(unittest.TestCase):
with pytest.raises(FilterError):
self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
with pytest.raises(FilterError):
filter_ = {
"filter": {
"obs": {"index": [1, 99, [200, 300]]}
}
}
filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_dataframe_obs_not_implemented(self):
@@ -59,10 +55,7 @@ class NaNTest(unittest.TestCase):
def test_annotation(self):
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
)
self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))

74
server/test/test_scanpy_engine.py
View File

@@ -18,15 +18,17 @@ Test the scanpy engine using the pbmc3k data set.
"""
@parameterized_class(("data_locator", "backed"), [
("example-dataset/pbmc3k.h5ad", False),
("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
("example-dataset/pbmc3k.h5ad", True),
("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
])
@parameterized_class(
("data_locator", "backed"),
[
("../example-dataset/pbmc3k.h5ad", False),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
("../example-dataset/pbmc3k.h5ad", True),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
],
)
class EngineTest(unittest.TestCase):
def setUp(self):
args = {
@@ -36,7 +38,7 @@ class EngineTest(unittest.TestCase):
"var_names": None,
"diffexp_lfc_cutoff": 0.01,
"layout_file": None,
"backed": self.backed
"backed": self.backed,
}
self.data = ScanpyEngine(DataLocator(self.data_locator), args)
@@ -64,11 +66,7 @@ class EngineTest(unittest.TestCase):
self.data._validate_data_types()
def test_filter_idx(self):
filter_ = {
"filter": {
"var": {"index": [1, 99, [200, 300]]}
}
}
filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
@@ -76,14 +74,7 @@ class EngineTest(unittest.TestCase):
def test_filter_complex(self):
filter_ = {
"filter": {
"var": {
"annotation_value": [
{"name": "n_cells", "min": 10}
],
"index": [1, 99, [200, 300]]
}
}
"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
@@ -101,16 +92,14 @@ class EngineTest(unittest.TestCase):
def test_schema_produces_error(self):
self.data.data.obs["time"] = pd.Series(
list([time.time() for i in range(self.data.cell_count)]),
dtype="datetime64[ns]",
list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
)
with pytest.raises(TypeError):
self.data._create_schema()
def test_config(self):
self.assertEqual(
self.data.features["layout"]["obs"],
{"available": True, "interactiveLimit": 50000},
self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000},
)
def test_layout(self):
@@ -131,14 +120,13 @@ class EngineTest(unittest.TestCase):
self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
)
fbs = self.data.annotation_to_fbs_matrix("var")
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 2)
self.assertEqual(annotations["n_rows"], 1838)
self.assertEqual(annotations["n_cols"], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
@@ -146,13 +134,13 @@ class EngineTest(unittest.TestCase):
fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations['n_cols'], 2)
self.assertEqual(annotations["n_cols"], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations['n_rows'], 1838)
self.assertEqual(annotations['n_cols'], 1)
self.assertEqual(annotations["n_rows"], 1838)
self.assertEqual(annotations["n_cols"], 1)
def test_annotation_put(self):
with self.assertRaises(DisabledFeatureError):
@@ -177,27 +165,19 @@ class EngineTest(unittest.TestCase):
self.data.data_frame_to_fbs_matrix(None, "obs")
def test_filtered_data_frame(self):
filter_ = {
"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}
}
filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 1040)
filter_ = {
"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}
}
filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
with self.assertRaises(FilterError):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_named_gene(self):
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
}
}
filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
@@ -205,9 +185,7 @@ class EngineTest(unittest.TestCase):
self.assertEqual(data["col_idx"], [4])
filter_ = {
"filter": {
"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
}
"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)

8
server/test/test_scanpy_engine_data_load.py
View File

@@ -10,8 +10,9 @@ class DataLoadEngineTest(unittest.TestCase):
"""
Test file loading, including deferred loading/update.
"""
def setUp(self):
self.data_file = DataLocator("example-dataset/pbmc3k.h5ad")
self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
self.data = ScanpyEngine()
def test_init(self):
@@ -29,7 +30,7 @@ class DataLoadEngineTest(unittest.TestCase):
"annotations_output_dir": None,
"backed": False,
"diffexp_may_be_slow": False,
"disable_diffexp": False
"disable_diffexp": False,
}
self.data.update(args=args)
self.assertEqual(args, self.data.config)
@@ -60,6 +61,7 @@ class DataLocatorEngineTest(unittest.TestCase):
"""
Test various types of data locators we expect to consume
"""
def setUp(self):
self.args = {
"layout": ["umap"],
@@ -76,7 +78,7 @@ class DataLocatorEngineTest(unittest.TestCase):
self.assertEqual(data.gene_count, 1838)
def test_posix_file(self):
locator = DataLocator("example-dataset/pbmc3k.h5ad")
locator = DataLocator("../example-dataset/pbmc3k.h5ad")
data = ScanpyEngine(locator, self.args)
self.stdAsserts(data)

99
server/test/test_writable_annotation.py
View File

@@ -25,9 +25,9 @@ class WritableAnnotationTest(unittest.TestCase):
"diffexp_lfc_cutoff": 0.01,
"annotations": True,
"annotations_file": self.annotations_file,
"annotations_output_dir": None
"annotations_output_dir": None,
}
self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args)
self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args)
def tearDown(self):
shutil.rmtree(self.tmpDir)
@@ -40,9 +40,7 @@ class WritableAnnotationTest(unittest.TestCase):
# verify that the expected errors are generated
n_rows = self.data.data.obs.shape[0]
fbs_bad = self.make_fbs({
'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category')
})
fbs_bad = self.make_fbs({"louvain": pd.Series(["undefined" for l in range(0, n_rows)], dtype="category")})
# ensure attempt to change VAR annotation
with self.assertRaises(ValueError):
@@ -55,32 +53,36 @@ class WritableAnnotationTest(unittest.TestCase):
def test_write_to_file(self):
# verify the file is written as expected
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
fbs = self.make_fbs(
{
"cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"),
"cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#')
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
self.assertEqual(df.shape, (n_rows, 2))
self.assertEqual(set(df.columns), set(['cat_A', 'cat_B']))
self.assertEqual(set(df.columns), set(["cat_A", "cat_B"]))
self.assertTrue(self.data.original_obs_index.equals(df.index))
self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)]))
self.assertTrue(np.all(df["cat_A"] == ["label_A" for l in range(0, n_rows)]))
self.assertTrue(np.all(df["cat_B"] == ["label_B" for l in range(0, n_rows)]))
# verify complete overwrite on second attempt, AND rotation occurs
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'),
'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category')
})
fbs = self.make_fbs(
{
"cat_A": pd.Series(["label_A1" for l in range(0, n_rows)], dtype="category"),
"cat_C": pd.Series(["label_C" for l in range(0, n_rows)], dtype="category"),
}
)
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations_file))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#')
self.assertEqual(set(df.columns), set(['cat_A', 'cat_C']))
self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)]))
self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)]))
df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#")
self.assertEqual(set(df.columns), set(["cat_A", "cat_C"]))
self.assertTrue(np.all(df["cat_A"] == ["label_A1" for l in range(0, n_rows)]))
self.assertTrue(np.all(df["cat_C"] == ["label_C" for l in range(0, n_rows)]))
# rotation
name, ext = path.splitext(self.annotations_file)
@@ -92,10 +94,12 @@ class WritableAnnotationTest(unittest.TestCase):
def test_file_rotation_to_max_9(self):
# verify we stop rotation at 9
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
fbs = self.make_fbs(
{
"cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"),
"cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
for i in range(0, 11):
res = self.data.annotation_put_fbs("obs", fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
@@ -111,10 +115,12 @@ class WritableAnnotationTest(unittest.TestCase):
# GET (annotation_to_fbs_matrix)
n_rows = self.data.data.obs.shape[0]
fbs = self.make_fbs({
'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'),
'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category')
})
fbs = self.make_fbs(
{
"cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"),
"cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"),
}
)
# put
res = self.data.annotation_put_fbs("obs", fbs)
@@ -128,28 +134,21 @@ class WritableAnnotationTest(unittest.TestCase):
self.assertEqual(annotations["n_rows"], n_rows)
self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"])
self.assertEqual(annotations["col_idx"], [
obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"
])
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
)
col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [
'label_A' for l in range(0, n_rows)
])
self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [
'label_B' for l in range(0, n_rows)
])
self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A" for l in range(0, n_rows)])
self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B" for l in range(0, n_rows)])
# verify the schema was updated
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(all_col_schema["cat_A"], {
"name": "cat_A",
"type": "categorical",
"categories": ["label_A"],
"writable": True
})
self.assertEqual(all_col_schema["cat_B"], {
"name": "cat_B",
"type": "categorical",
"categories": ["label_B"],
"writable": True
})
self.assertEqual(
all_col_schema["cat_A"],
{"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
)
self.assertEqual(
all_col_schema["cat_B"],
{"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
)

5
travis-build.sh
View File

@@ -1,5 +0,0 @@
set -eo pipefail
flake8 server
npm run --prefix client/ build
npm run --prefix client/ unit-test
pytest -s server/test