From f3015cb9dff20c48c8ff12f03b4edcd77945ac80 Mon Sep 17 00:00:00 2001 From: Matt Weiden <538456+mweiden@users.noreply.github.com> Date: Fri, 27 Dec 2019 14:43:37 -0800 Subject: [PATCH] Makefile modularity, test targets, and auto-formatting (#1070) * Fix Makefile whitespace and .PHONY use * Fix Makefile filename * Modularize Makefile into client and server Makefiles Part of the reason that the Makefile in the root directory is a bit complicated is that it tries to handle tasks that can be handled separately in the client and server modules. This commit pushes some of the make logic specific to each module into their own makefiles and calls out to those makefiles from that in the project root. * Add auto-formatting to client and server modules One thing that can make linting faster is auto-formatting. This commit adds the yapf auto-formatting tool to the server module and uses eslint's "fix" functionality to speed up the linting/formatting process. * Add yapf for automatic code formatting * Add a root test target that calls sub-tests * Apply yapf to python files * Do not duplicate npm commands, simply pass through * Update documentation * Do not shadow reserved word len * Add general test target * Fix make call in dev-env * Use black instead of yapf * Run flake8 from the root directory * Revert "Apply yapf to python files" This reverts commit cdca128a01f484c28c3264f8d9fa411103220c2c. * Apply black to python code * Resolve lint errors resulting from black format * Add explanation of server unit tests in dev guidelines --- .travis.yml | 11 +- makefile => Makefile | 139 ++++++++++++----- client/Makefile | 16 ++ client/package.json | 1 + dev_docs/developer_guidelines.md | 85 +++++----- pyproject.toml | 25 +++ server/Makefile | 8 + server/__main__.py | 4 +- server/app/driver/driver.py | 4 +- server/app/rest_api/rest.py | 125 ++++----------- server/app/scanpy_engine/diffexp.py | 4 +- server/app/scanpy_engine/labels.py | 8 +- server/app/scanpy_engine/matrix_proxy.py | 13 +- server/app/scanpy_engine/scanpy_engine.py | 164 ++++++++------------ server/app/util/constants.py | 4 +- server/app/util/data_locator.py | 12 +- server/app/util/fbs/matrix.py | 31 ++-- server/app/util/matrix_proxy.py | 62 ++++---- server/app/util/utils.py | 1 + server/cli/cli.py | 14 +- server/cli/launch.py | 164 ++++++++++++-------- server/cli/prepare.py | 51 +++--- server/gui/browser.py | 7 +- server/gui/hook-cefpython3.py | 48 +++--- server/gui/main.py | 28 ++-- server/gui/utils.py | 8 +- server/gui/workers.py | 1 + server/test/decode_fbs.py | 15 +- server/test/test_api.py | 109 +++++++------ server/test/test_fbs.py | 43 ++--- server/test/test_matrix_proxy.py | 134 +++++----------- server/test/test_nan_rest.py | 4 +- server/test/test_nan_scanpy_engine.py | 15 +- server/test/test_scanpy_engine.py | 74 ++++----- server/test/test_scanpy_engine_data_load.py | 8 +- server/test/test_writable_annotation.py | 99 ++++++------ travis-build.sh | 5 - 37 files changed, 738 insertions(+), 806 deletions(-) rename makefile => Makefile (67%) create mode 100644 client/Makefile create mode 100644 pyproject.toml create mode 100644 server/Makefile delete mode 100755 travis-build.sh diff --git a/.travis.yml b/.travis.yml index 20e7159c..1a05cc8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,23 +9,20 @@ cache: install: - set -eo pipefail - pip install flake8 - - make pydist - - make install-dist - - pip install -r server/requirements-dev.txt + - make pydist install-dist dev-env jobs: include: - name: "Branch Tests 3.7" python: "3.7" - script: ./travis-build.sh + script: make build-client lint unit-test - name: "Branch Tests 3.6" python: "3.6" - script: ./travis-build.sh + script: make build-client lint unit-test - name: "Docker Build" install: skip python: "3.6" script: docker build . - name: "Smoke Tests" python: "3.6" - script: - - npm run --prefix client/ smoke-test + script: make smoke-test diff --git a/makefile b/Makefile similarity index 67% rename from makefile rename to Makefile index 47be5ecd..e97af83a 100644 --- a/makefile +++ b/Makefile @@ -5,12 +5,32 @@ CLEANFILES := $(BUILDDIR)/ client/build build dist cellxgene.egg-info PART ?= patch + +# CLEANING +.PHONY: clean +clean: clean-lite clean-server clean-client + +# cleaning the client's node_modules is the longest one, so we avoid that if possible +.PHONY: clean-lite +clean-lite: + rm -rf $(CLEANFILES) + +clean-%: + cd $(*) && $(MAKE) clean + + # BUILDING PACKAGE -build : clean build-server +.PHONY: build +build: clean build-server @echo "done" -build-server : build-client +.PHONY: build-client +build-client: + cd client && $(MAKE) install build + +.PHONY: build-server +build-server: build-client mkdir -p $(SERVERBUILD) cp -r server/* $(SERVERBUILD) cp -r client/build/ $(CLIENTBUILD) @@ -22,12 +42,9 @@ build-server : build-client cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/ cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR) -build-client : - npm install --prefix client/ client - npm run --prefix client build - # If you are actively developing in the server folder use this, dirties the source tree -build-for-server-dev : clean-server build-client +.PHONY: build-for-server-dev +build-for-server-dev: clean-server build-client mkdir -p server/app/web/static/img mkdir -p server/app/web/static/js mkdir -p server/app/web/templates/ @@ -36,124 +53,160 @@ build-for-server-dev : clean-server build-client cp client/build/favicon.png server/app/web/static/img cp client/build/service-worker.js server/app/web/static/js/ -clean : clean-lite clean-server - rm -rf client/node_modules -# cleaning node_modules is the longest one, so we avoid that if possible -clean-lite : - rm -rf $(CLEANFILES) +# TESTING +.PHONY: test +test: unit-test smoke-test -clean-server : - rm -f server/app/web/templates/index.html - rm -rf server/app/web/static +.PHONY: unit-test +unit-test: unit-test-server unit-test-client + +unit-test-%: + cd $(*) && $(MAKE) unit-test + +.PHONY: smoke-test +smoke-test: + cd client && $(MAKE) smoke-test + +# FORMATTING CODE + +.PHOHY: fmt +fmt: fmt-client fmt-py + +fmt-client: + cd client && $(MAKE) fmt + +fmt-py: + black . + +.PHONY: lint +lint: + flake8 server -.PHONY : build build-server build-client build-for-server-dev clean clean-lite clean-server # CREATING DISTRIBUTION RELEASE -pydist : build +.PHONY: pydist +pydist: build cd $(BUILDDIR); python setup.py sdist -d ../dist @echo "done" -.PHONY : pydist # RELEASE HELPERS # create new version to commit to master -release-stage-1 : dev-env bump clean-lite gen-package-lock +.PHONY: release-stage-1 +release-stage-1: dev-env bump clean-lite gen-package-lock @echo "Version bumped part:$(PART) and client built. Ready to commit and push" # build dist and release to dev pypi -release-stage-2 : dev-env pydist twine +.PHONY: release-stage-2 +release-stage-2: dev-env pydist twine @echo "Dist built and uploaded to test.pypi.org" @echo "Test the install:" @echo " make install-release-test" @echo "Then upload to Pypi prod:" @echo " make twine-prod" +.PHONY: release-stage-final release-stage-final: twine-prod @echo "Release uploaded to pypi.org" # DANGER: releases directly to prod # use this if you accidently burned a test release version number, -release-directly-to-prod : dev-env pydist twine-prod +.PHONY: release-directly-to-prod +release-directly-to-prod: dev-env pydist twine-prod @echo "Dist built and uploaded to pypi.org" @echo "Test the install:" @echo " make install-release" -dev-env : +.PHONY: dev-env +dev-env: + cd client && $(MAKE) install pip install -r server/requirements-dev.txt -gui-env : dev-env +.PHONY: gui-env +gui-env: dev-env pip install -r server/requirements-gui.txt # give PART=[major, minor, part] as param to make bump -bump : +.PHONY: bump +bump: bumpversion --config-file .bumpversion.cfg $(PART) -twine : +.PHONY: twine +twine: twine upload --repository-url https://test.pypi.org/legacy/ dist/* -twine-prod : +.PHONY: twine-prod +twine-prod: twine upload dist/* # quicker than re-building client -gen-package-lock : - npm install --prefix client/ client +.PHONY: gen-package-lock +gen-package-lock: + cd client && $(MAKE) install -.PHONY : release-stage-1 release-stage-2 release-stage-final release-burned dev-env bump twine twine-prod gen-package-lock # INSTALL # setup.py sucks when you have your library in a separate folder, adding these in to help setup envs # install from build directory -install : uninstall +.PHONY: install +install: uninstall cd $(BUILDDIR); pip install -e . # install from source tree for development -install-dev : uninstall +.PHONY: install-dev +install-dev: uninstall pip install -e . # install from test.pypi to test your release -install-release-test : uninstall +.PHONY: install-release-test +install-release-test: uninstall pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene @echo "Installed cellxgene from test.pypi.org, now run and smoke test" # install from pypi to test your release -install-release : uninstall +.PHONY: install-release +install-release: uninstall pip install --no-cache-dir cellxgene @echo "Installed cellxgene from pypi.org" # install from dist -install-dist : uninstall +.PHONY: install-dist +install-dist: uninstall pip install dist/cellxgene*.tar.gz -uninstall : +.PHONY: uninstall +uninstall: pip uninstall -y cellxgene || : -.PHONY : install install-dev install-release-test install-release uninstall # GUI -build-assets : +.PHONY: build-assets +build-assets: pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py -gui-spec-osx : clean-lite gui-env +.PHONY: gui-spec-osx +gui-spec-osx: clean-lite gui-env pip install -e .[gui] pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl' --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py mv cellxgene.spec cellxgene-osx.spec -gui-spec-windows : clean-lite dev-env +.PHONY: gui-spec-windows +gui-spec-windows: clean-lite dev-env pip install -e .[gui] pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py mv cellxgene.spec cellxgene-windows.spec -gui-build-osx : clean-lite +.PHONY: gui-build-osx +gui-build-osx: clean-lite pyinstaller --clean cellxgene-osx.spec -gui-build-windows : clean-lite +.PHONY: gui-build-windows +gui-build-windows: clean-lite pyinstaller --clean cellxgene-windows.spec -.PHONY : build-assets gui-spec-osx gui-spec-windows gui-build-osx gui-build-windows - diff --git a/client/Makefile b/client/Makefile new file mode 100644 index 00000000..47825cee --- /dev/null +++ b/client/Makefile @@ -0,0 +1,16 @@ +.PHONY: clean +clean: + rm -rf node_modules + +.PHONY: install +install: + npm install client + +.PHONY: build +build: + npm run build + +# pass remaining commands through to npm run +%: + npm run $(*) + diff --git a/client/package.json b/client/package.json index bb1e4f71..b6f60d7b 100644 --- a/client/package.json +++ b/client/package.json @@ -11,6 +11,7 @@ "clean": "rimraf build", "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js", "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js", + "fmt": "eslint --fix src", "lint": "eslint src", "smoke-test": "start-server-and-test start-server-for-test :5000 e2e", "start": "node server/development.js", diff --git a/dev_docs/developer_guidelines.md b/dev_docs/developer_guidelines.md index c9ce3a8c..d2951f05 100644 --- a/dev_docs/developer_guidelines.md +++ b/dev_docs/developer_guidelines.md @@ -9,6 +9,38 @@ **All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.** +## Running test suite +Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. + +### Unit tests +Steps to run the all unit tests: +1. Start in the project root directory +1. `make dev-env` +1. `make unit-test` + +### End to end tests + +End to end tests use two env variables: +* `JEST_ENV` - environment to run end to end tests. Default `dev` + * `prod` - run headless with no slowdown, chromium will not open. + * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit. + * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit. +* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port). + +On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` make target. + +To run end to end tests as they will be run on CI +1. cellxgene should be built and installed as [specified in server dev](#install) +2. `export JEST_ENV='prod'` +3. `export JEST_CXG_PORT=5000` +4. Run `npm run --prefix client/ smoke-test` + +Run end to end tests interactively during development +1. cellxgene should be installed as [specified in client dev](#install-1) +2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k` +3. Run `make smoke-test` +4. To debug a failing test `export JEST_ENV='debug'` and re-run. + ## Server dev ### Install * Build the client and put static files in place: `make build-for-server-dev` @@ -21,11 +53,15 @@ If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded. ### Linter -We use `flake8` to lint code. Travis CI runs `flake8 server`. + +We use [`flake8`](https://github.com/PyCQA/flake8) to lint python and [`black`](https://pypi.org/project/black/) for auto-formatting. + +To auto-format code run `make fmt`. To run lint checks on the code run `make lint`. ### Test -1. Install development requirements `pip install -r server/requirements-dev.txt` -2. Run tests `pytest server/test` +If you would like to run the server tests individually, follow the steps below +1. Install development requirements `make dev-env` +1. Run `make unit-test` in the `server` directory or `make unit-test-server` in the root directory. ### Tips * Install in a virtualenv @@ -33,8 +69,8 @@ We use `flake8` to lint code. Travis CI runs `flake8 server`. ## Client dev ### Install -1. Install prereqs for client: `npm install --prefix client/ client` -2. Install cellxgene server: `pip install -e .` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets. +1. Install prereqs for client: `make dev-env` +2. Install cellxgene server: `make install-dev` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets. ### Launch To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made. @@ -49,43 +85,12 @@ To build only the client: `make build-client` We use `eslint` to lint the code and `prettier` as our code formatter. ### Test -In `client/` directory run `npm run unit-test` + +If you would like to run the client tests individually, follow the steps below in the `client` directory +1. For unit tests run `npm run unit-test` or `make unit-test` +1. For the smoke test run `npm run smoke-test` or `make smoke-test` ### Tips * You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) in `client/` directory run `npm run backend-dev` -## Running tests -Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. - -### Server unit tests -Install development requirements `pip install -r server/requirements-dev.txt` -Run tests `pytest server/test` - -### Client unit tests -In `client/` directory run `npm run unit-test` - -### End to end tests - -End to end tests use two env variables: -* `JEST_ENV` - environment to run end to end tests. Default `dev` - * `prod` - run headless with no slowdown, chromium will not open. - * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit. - * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit. -* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port). - -On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` npm script - -To run end to end tests as they will be run on CI -1. cellxgene should be built and installed as [specified in server dev](#install) -2. `export JEST_ENV='prod'` -3. `export JEST_CXG_PORT='5000'` -4. Run `npm run --prefix client/ smoke-test` - -Run end to end tests interactively during development -1. cellxgene should be installed as [specified in client dev](#install-1) -2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k` -3. Run `npm run --prefix client/ e2e` -4. To debug a failing test `export JEST_ENV='debug'` and re-run. - - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..a30d4e82 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[tool.black] +line-length = 120 +target_version = ['py37'] +include = '\.pyi?$' +exclude = ''' + +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | venv + | _build + | buck-out + | build + | dist + | server/app/util/fbs/NetEncoding + )/ + | server/gui/cellxgene_rc.py + +) +''' diff --git a/server/Makefile b/server/Makefile new file mode 100644 index 00000000..271f48c4 --- /dev/null +++ b/server/Makefile @@ -0,0 +1,8 @@ +.PHONY: clean +clean: + rm -f app/web/templates/index.html + rm -rf app/web/static + +.PHONY: unit-test +unit-test: + pytest -s test diff --git a/server/__main__.py b/server/__main__.py index 1f0b8730..b582e892 100644 --- a/server/__main__.py +++ b/server/__main__.py @@ -5,11 +5,11 @@ if __package__ is None: PKG_PATH = Path(__file__).parent sys.path.insert(0, str(PKG_PATH.parent)) - import server # noqa F401 + import server # noqa F401 __package__ = PKG_PATH.name # Main thing -from .cli.cli import cli # noqa F402 +from .cli.cli import cli # noqa F402 cli() diff --git a/server/app/driver/driver.py b/server/app/driver/driver.py index 30aa8f06..38bdf484 100644 --- a/server/app/driver/driver.py +++ b/server/app/driver/driver.py @@ -33,7 +33,7 @@ class CXGDriver(metaclass=ABCMeta): "max_category_items": None, "diffexp_lfc_cutoff": None, "disable_diffexp": False, - "diffexp_may_be_slow": False + "diffexp_may_be_slow": False, } @abstractmethod @@ -51,7 +51,7 @@ class CXGDriver(metaclass=ABCMeta): features = { "cluster": {"available": False}, "layout": {"obs": {"available": False}, "var": {"available": False}}, - "diffexp": {"available": True, "interactiveLimit": 50000} + "diffexp": {"available": True, "interactiveLimit": 50000}, } # TODO - Interactive limit should be generated from the actual available methods see GH issue #94 if self.config["layout"]: diff --git a/server/app/rest_api/rest.py b/server/app/rest_api/rest.py index 780dd38f..32342e18 100644 --- a/server/app/rest_api/rest.py +++ b/server/app/rest_api/rest.py @@ -8,13 +8,7 @@ from flask_restful import Api, Resource from server import __version__ as cellxgene_version from anndata import __version__ as anndata_version -from server.app.util.constants import ( - Axis, - DiffExpMode, - JSON_NaN_to_num_warning_msg, - CXGUID, - CXG_ANNO_COLLECTION -) +from server.app.util.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg, CXGUID, CXG_ANNO_COLLECTION from server.app.util.errors import ( FilterError, InteractiveError, @@ -40,41 +34,18 @@ class ConfigAPI(Resource): config = { "config": { "features": [ - { - "method": "POST", - "path": "/cluster/", - **current_app.data.features["cluster"], - }, - { - "method": "POST", - "path": "/layout/obs", - **current_app.data.features["layout"]["obs"], - }, - { - "method": "POST", - "path": "/layout/var", - **current_app.data.features["layout"]["var"], - }, - { - "method": "POST", - "path": "/diffexp/", - **current_app.data.features["diffexp"], - }, + {"method": "POST", "path": "/cluster/", **current_app.data.features["cluster"]}, + {"method": "POST", "path": "/layout/obs", **current_app.data.features["layout"]["obs"]}, + {"method": "POST", "path": "/layout/var", **current_app.data.features["layout"]["var"]}, + {"method": "POST", "path": "/diffexp/", **current_app.data.features["diffexp"]}, ], "displayNames": { "engine": f"cellxgene Scanpy engine version ", "dataset": current_app.config["DATASET_TITLE"], }, - "links": { - "about-dataset": current_app.config["ABOUT_DATASET"] - }, - "parameters": { - **current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection) - }, - "library_versions": { - "cellxgene": cellxgene_version, - "anndata": anndata_version - } + "links": {"about-dataset": current_app.config["ABOUT_DATASET"]}, + "parameters": {**current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection)}, + "library_versions": {"cellxgene": cellxgene_version, "anndata": anndata_version}, } } @@ -84,17 +55,13 @@ class ConfigAPI(Resource): class AnnotationsObsAPI(Resource): def get(self): fields = request.args.getlist("annotation-name", None) - preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) cxguid = get_userid(session) anno_collection = get_anno_collection(session) try: if preferred_mimetype == "application/octet-stream": fbs = current_app.data.annotation_to_fbs_matrix("obs", fields, uid=cxguid, collection=anno_collection) - return make_response(fbs, - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) else: return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except KeyError: @@ -115,9 +82,7 @@ class AnnotationsObsAPI(Resource): try: fbs = request.get_data() res = current_app.data.annotation_put_fbs("obs", fbs, uid=cxguid, collection=anno_collection) - return make_response( - res, HTTPStatus.OK, {"Content-Type": "application/json"} - ) + return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"}) except (ValueError, DisabledFeatureError, KeyError) as e: return make_response(str(e), HTTPStatus.BAD_REQUEST) except Exception as e: @@ -127,14 +92,14 @@ class AnnotationsObsAPI(Resource): class AnnotationsVarAPI(Resource): def get(self): fields = request.args.getlist("annotation-name", None) - preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) try: if preferred_mimetype == "application/octet-stream": - return make_response(current_app.data.annotation_to_fbs_matrix("var", fields), - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + return make_response( + current_app.data.annotation_to_fbs_matrix("var", fields), + HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}, + ) else: return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except KeyError: @@ -145,19 +110,16 @@ class AnnotationsVarAPI(Resource): class DataVarAPI(Resource): def put(self): - preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) try: if preferred_mimetype == "application/octet-stream": filter_json = request.get_json() filter = filter_json["filter"] if filter_json else None return make_response( - current_app.data.data_frame_to_fbs_matrix( - filter, axis=Axis.VAR - ), + current_app.data.data_frame_to_fbs_matrix(filter, axis=Axis.VAR), HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + {"Content-Type": "application/octet-stream"}, + ) else: return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except FilterError as e: @@ -175,35 +137,23 @@ class DiffExpObsAPI(Resource): except KeyError: return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST) except ValueError: - return make_response( - f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST - ) + return make_response(f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST) # Validate filters if mode == DiffExpMode.VAR_FILTER or "varFilter" in args: # not NOT_IMPLEMENTED - return make_response( - "mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED - ) + return make_response("mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED) if mode == DiffExpMode.TOP_N and "count" not in args: - return make_response( - "mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST - ) + return make_response("mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST) if "set1" not in args: return make_response("set1 is required.", HTTPStatus.BAD_REQUEST) if Axis.VAR in args["set1"]["filter"]: - return make_response( - "Var filter not allowed for set1", HTTPStatus.BAD_REQUEST - ) + return make_response("Var filter not allowed for set1", HTTPStatus.BAD_REQUEST) # set2 if "set2" not in args: - return make_response( - "Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED - ) + return make_response("Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED) if Axis.VAR in args["set2"]["filter"]: - return make_response( - "Var filter not allowed for set2", HTTPStatus.BAD_REQUEST - ) + return make_response("Var filter not allowed for set2", HTTPStatus.BAD_REQUEST) set1_filter = args["set1"]["filter"] set2_filter = args.get("set2", {"filter": {}})["filter"] @@ -214,14 +164,9 @@ class DiffExpObsAPI(Resource): count = args.get("count", None) try: diffexp = current_app.data.diffexp_topN( - set1_filter, - set2_filter, - count, - current_app.data.features["diffexp"]["interactiveLimit"], - ) - return make_response( - diffexp, HTTPStatus.OK, {"Content-Type": "application/json"} + set1_filter, set2_filter, count, current_app.data.features["diffexp"]["interactiveLimit"], ) + return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"}) except (ValueError, FilterError) as e: return make_response(e.message, HTTPStatus.BAD_REQUEST) except InteractiveError: @@ -236,14 +181,12 @@ class DiffExpObsAPI(Resource): class LayoutObsAPI(Resource): def get(self): - preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) try: if preferred_mimetype == "application/octet-stream": - return make_response(current_app.data.layout_to_fbs_matrix(), - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + return make_response( + current_app.data.layout_to_fbs_matrix(), HTTPStatus.OK, {"Content-Type": "application/octet-stream"} + ) else: return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except PrepareError as e: @@ -278,7 +221,7 @@ def is_safe_collection_name(name): """ if name is None: return False - return re.match(r'^\w+$', name) is not None + return re.match(r"^\w+$", name) is not None def get_api_resources(): diff --git a/server/app/scanpy_engine/diffexp.py b/server/app/scanpy_engine/diffexp.py index d991f1c6..af267636 100644 --- a/server/app/scanpy_engine/diffexp.py +++ b/server/app/scanpy_engine/diffexp.py @@ -32,8 +32,8 @@ def _mean_var_n(X): v = sumsq / (n - 1) if fp_err_occurred: - mean[np.isfinite(mean) == False] = 0 # noqa: E712 - v[np.isfinite(v) == False] = 0 # noqa: E712 + mean[np.isfinite(mean) == False] = 0 # noqa: E712 + v[np.isfinite(v) == False] = 0 # noqa: E712 return mean, v, n diff --git a/server/app/scanpy_engine/labels.py b/server/app/scanpy_engine/labels.py index 3a5f7d08..1cdaeb42 100644 --- a/server/app/scanpy_engine/labels.py +++ b/server/app/scanpy_engine/labels.py @@ -9,7 +9,7 @@ import pandas as pd def read_labels(fname): if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: - return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#') + return pd.read_csv(fname, dtype="category", index_col=0, header=0, comment="#") else: return pd.DataFrame() @@ -19,12 +19,12 @@ def write_labels(fname, df, header=None, backup_dir=None): backup(fname, backup_dir) # rotate_fname(fname, backup_dir) if not df.empty: - with open(fname, 'w', newline="") as f: + with open(fname, "w", newline="") as f: if header is not None: f.write(header) df.to_csv(f) else: - open(fname, 'w').close() + open(fname, "w").close() def backup(fname, backup_dir, max_backups=9): @@ -46,7 +46,7 @@ def backup(fname, backup_dir, max_backups=9): fname_base = os.path.basename(fname) fname_base_root, fname_base_ext = os.path.splitext(fname_base) # don't use ISO standard time format, as it contains characters illegal on some filesytems. - nowish = datetime.now().strftime('%Y-%m-%dT%H-%M-%S') + nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}") if os.path.exists(backup_fname): os.remove(backup_fname) diff --git a/server/app/scanpy_engine/matrix_proxy.py b/server/app/scanpy_engine/matrix_proxy.py index 137c676b..29e0e4fe 100644 --- a/server/app/scanpy_engine/matrix_proxy.py +++ b/server/app/scanpy_engine/matrix_proxy.py @@ -1,4 +1,3 @@ - from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView """ @@ -17,6 +16,7 @@ class ArrayProxyView_anndata_h5py(ArrayProxyView): override to handle sparse getitem semantics, which differ from numpy. """ + def toarray(self): """ sadly, sparse indexing doesn't drop dimensions like numpy! """ arr = self.m[self._index[0], self._index[1]] @@ -30,12 +30,15 @@ class MatrixProxy_anndata_h5py(MatrixProxyView): AnnData sparse array stored in H5AD, or proxies for backed data. None of these handle indexing very well, so we plop a proxy on top. """ + @classmethod def __supports__(cls): - return ("anndata.h5py.h5sparse.SparseDataset", - "anndata.h5py.h5sparse.backed_csc_matrix", - "anndata.h5py.h5sparse.backed_csr_matrix", - "h5py._hl.dataset.Dataset") + return ( + "anndata.h5py.h5sparse.SparseDataset", + "anndata.h5py.h5sparse.backed_csc_matrix", + "anndata.h5py.h5sparse.backed_csr_matrix", + "h5py._hl.dataset.Dataset", + ) @classmethod def create_array(cls, *args, **kwargs): diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index a1a2b32a..a77bb48c 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -62,7 +62,7 @@ class ScanpyEngine(CXGDriver): "annotations_output_dir": None, "backed": False, "disable_diffexp": False, - "diffexp_may_be_slow": False + "diffexp_may_be_slow": False, } def get_config_parameters(self, uid=None, collection=None): @@ -70,26 +70,25 @@ class ScanpyEngine(CXGDriver): "max-category-items": self.config["max_category_items"], "disable-diffexp": self.config["disable_diffexp"], "diffexp-may-be-slow": self.config["diffexp_may_be_slow"], - "annotations": self.config["annotations"] + "annotations": self.config["annotations"], } if self.config["annotations"]: if uid is not None: - params.update({ - "annotations-user-data-idhash": self.get_userdata_idhash(uid) - }) - if self.config['annotations_file'] is not None: + params.update({"annotations-user-data-idhash": self.get_userdata_idhash(uid)}) + if self.config["annotations_file"] is not None: # user has hard-wired the name of the annotation data collection - fname = os.path.basename(self.config['annotations_file']) + fname = os.path.basename(self.config["annotations_file"]) collection_fname = os.path.splitext(fname)[0] - params.update({ - 'annotations-data-collection-is-read-only': True, - 'annotations-data-collection-name': collection_fname - }) + params.update( + { + "annotations-data-collection-is-read-only": True, + "annotations-data-collection-name": collection_fname, + } + ) elif collection is not None: - params.update({ - 'annotations-data-collection-is-read-only': False, - 'annotations-data-collection-name': collection - }) + params.update( + {"annotations-data-collection-is-read-only": False, "annotations-data-collection-name": collection} + ) return params @staticmethod @@ -140,23 +139,18 @@ class ScanpyEngine(CXGDriver): # User has specified alternative column for unique names, and it exists if not df_axis[name].is_unique: raise KeyError( - f"Values in {ax_name}.{name} must be unique. " - "Please prepare data to contain unique values." + f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values." ) df_axis.reset_index(drop=True, inplace=True) else: # user specified a non-existent column name - raise KeyError( - f"Annotation name {name}, specified in --{ax_name}-name does not exist." - ) + raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.") @staticmethod def _can_cast_to_float32(ann): if ann.dtype.kind == "f": if not np.can_cast(ann.dtype, np.float32): - warnings.warn( - f"Annotation {ann.name} will be converted to 32 bit float and may lose precision." - ) + warnings.warn(f"Annotation {ann.name} will be converted to 32 bit float and may lose precision.") return True return False @@ -188,30 +182,18 @@ class ScanpyEngine(CXGDriver): schema["type"] = "categorical" schema["categories"] = dtype.categories.tolist() else: - raise TypeError( - f"Annotations of type {dtype} are unsupported by cellxgene." - ) + raise TypeError(f"Annotations of type {dtype} are unsupported by cellxgene.") return schema @requires_data def _create_schema(self): self.schema = { - "dataframe": { - "nObs": self.cell_count, - "nVar": self.gene_count, - "type": str(self.data.X.dtype), - }, + "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)}, "annotations": { - "obs": { - "index": self.config["obs_names"], - "columns": [] - }, - "var": { - "index": self.config["var_names"], - "columns": [] - } + "obs": {"index": self.config["obs_names"], "columns": []}, + "var": {"index": self.config["var_names"], "columns": []}, }, - "layout": {"obs": []} + "layout": {"obs": []}, } for ax in Axis: curr_axis = getattr(self.data, str(ax)) @@ -220,12 +202,8 @@ class ScanpyEngine(CXGDriver): ann_schema.update(self._get_col_type(curr_axis[ann])) self.schema["annotations"][ax]["columns"].append(ann_schema) - for layout in self.config['layout']: - layout_schema = { - "name": layout, - "type": "float32", - "dims": [f"{layout}_0", f"{layout}_1"] - } + for layout in self.config["layout"]: + layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]} self.schema["layout"]["obs"].append(layout_schema) @requires_data @@ -250,7 +228,7 @@ class ScanpyEngine(CXGDriver): Used to create safe annotations output file names. """ id = (uid + self.data_locator.abspath()).encode() - idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode('utf-8') + idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8") return idhash def get_anno_fname(self, uid=None, collection=None): @@ -272,11 +250,11 @@ class ScanpyEngine(CXGDriver): if not self.config["annotations"]: return None - if self.config['annotations_output_dir']: - return self.config['annotations_output_dir'] + if self.config["annotations_output_dir"]: + return self.config["annotations_output_dir"] - if self.config['annotations_file']: - return os.path.dirname(os.path.abspath(self.config['annotations_file'])) + if self.config["annotations_file"]: + return os.path.dirname(os.path.abspath(self.config["annotations_file"])) return os.getcwd() @@ -299,7 +277,7 @@ class ScanpyEngine(CXGDriver): with data_locator.local_handle() as lh: # as of AnnData 0.6.19, backed mode performs initial load fast, but at the # cost of significantly slower access to X data. - backed = 'r' if self.config['backed'] else None + backed = "r" if self.config["backed"] else None self.data = anndata.read_h5ad(lh, backed=backed) except ValueError: @@ -338,7 +316,7 @@ class ScanpyEngine(CXGDriver): # heuristic n_values = self.data.shape[0] * self.data.shape[1] - if (n_values > 1e8 and self.config['backed'] is True) or (n_values > 5e8): + if (n_values > 1e8 and self.config["backed"] is True) or (n_values > 5e8): self.config.update({"diffexp_may_be_slow": True}) @requires_data @@ -348,7 +326,7 @@ class ScanpyEngine(CXGDriver): b) validate layouts are legal. remove/warn on any that are not c) cap total list of layouts at global const MAX_LAYOUTS """ - layouts = self.config['layout'] + layouts = self.config["layout"] # handle default if layouts is None or len(layouts) == 0: # load default layouts from the data. @@ -372,7 +350,7 @@ class ScanpyEngine(CXGDriver): raise PrepareError(f"No valid layout data.") # cap layouts to MAX_LAYOUTS - self.config['layout'] = valid_layouts[0:MAX_LAYOUTS] + self.config["layout"] = valid_layouts[0:MAX_LAYOUTS] @requires_data def _is_valid_layout(self, arr): @@ -394,8 +372,7 @@ class ScanpyEngine(CXGDriver): ) if self.data.X.dtype != "float32": warnings.warn( - f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " - f"Precision may be truncated." + f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated." ) for ax in Axis: curr_axis = getattr(self.data, str(ax)) @@ -414,7 +391,7 @@ class ScanpyEngine(CXGDriver): ) if isinstance(datatype, CategoricalDtype): category_num = len(curr_axis[ann].dtype.categories) - if category_num > 500 and category_num > self.config['max_category_items']: + if category_num > 500 and category_num > self.config["max_category_items"]: warnings.warn( f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be " f"cumbersome or slow to display. We recommend setting the " @@ -439,14 +416,17 @@ class ScanpyEngine(CXGDriver): raise KeyError(f"All row index values specified in user annotations must be unique.") if not labels.index.equals(self.original_obs_index): - raise KeyError("Label file row index does not match H5AD file index. " - "Please ensure that column zero (0) in the label file contain the same " - "index values as the H5AD file.") + raise KeyError( + "Label file row index does not match H5AD file index. " + "Please ensure that column zero (0) in the label file contain the same " + "index values as the H5AD file." + ) duplicate_columns = list(set(labels.columns) & set(self.data.obs.columns)) if len(duplicate_columns) > 0: - raise KeyError(f"Labels file may not contain column names which overlap " - f"with h5ad obs columns {duplicate_columns}") + raise KeyError( + f"Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}" + ) # labels must have same count as obs annotations if labels.shape[0] != self.data.obs.shape[0]: @@ -475,7 +455,7 @@ class ScanpyEngine(CXGDriver): mask = np.zeros((count,), dtype=bool) for i in filter: if type(i) == list: - mask[i[0]: i[1]] = True + mask[i[0] : i[1]] = True else: mask[i] = True return mask @@ -484,15 +464,10 @@ class ScanpyEngine(CXGDriver): def _axis_filter_to_mask(filter, d_axis, count): mask = np.ones((count,), dtype=bool) if "index" in filter: - mask = np.logical_and( - mask, ScanpyEngine._index_filter_to_mask(filter["index"], count) - ) + mask = np.logical_and(mask, ScanpyEngine._index_filter_to_mask(filter["index"], count)) if "annotation_value" in filter: mask = np.logical_and( - mask, - ScanpyEngine._annotation_filter_to_mask( - filter["annotation_value"], d_axis, count - ), + mask, ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"], d_axis, count), ) return mask @@ -507,13 +482,9 @@ class ScanpyEngine(CXGDriver): if filter is not None: if Axis.OBS in filter: - obs_selector = self._axis_filter_to_mask( - filter["obs"], self.data.obs, self.data.n_obs - ) + obs_selector = self._axis_filter_to_mask(filter["obs"], self.data.obs, self.data.n_obs) if Axis.VAR in filter: - var_selector = self._axis_filter_to_mask( - filter["var"], self.data.var, self.data.n_vars - ) + var_selector = self._axis_filter_to_mask(filter["var"], self.data.var, self.data.n_vars) return obs_selector, var_selector @requires_data @@ -531,7 +502,7 @@ class ScanpyEngine(CXGDriver): labels = None if labels is not None and not labels.empty: - df = self.data.obs.join(labels, self.config['obs_names']) + df = self.data.obs.join(labels, self.config["obs_names"]) else: df = self.data.obs else: @@ -560,18 +531,21 @@ class ScanpyEngine(CXGDriver): # if any of the new column labels overlap with our existing labels, raise error duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns)) if not new_label_df.columns.is_unique or len(duplicate_columns) > 0: - raise KeyError(f"Labels file may not contain column names which overlap " - f"with h5ad obs columns {duplicate_columns}") + raise KeyError( + f"Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}" + ) # update our internal state and save it. Multi-threading often enabled, # so treat this as a critical section. with self.label_lock: lastmod = self.data_locator.lastmodtime() lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") - header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \ - f"using cellxgene version {cellxgene_version}\n" \ - f"# Input data file was {self.data_locator.uri_or_path}, " \ - f"which was last modified on {lastmodstr}\n" + header = ( + f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " + f"using cellxgene version {cellxgene_version}\n" + f"# Input data file was {self.data_locator.uri_or_path}, " + f"which was last modified on {lastmodstr}\n" + ) write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection)) return jsonify_scanpy({"status": "OK"}) @@ -598,8 +572,7 @@ class ScanpyEngine(CXGDriver): raise FilterError("filtering on obs unsupported") # Currently only handles VAR dimension - X = MatrixProxy.create(self.data.X if var_selector is None - else self.data.X[:, var_selector]) + X = MatrixProxy.create(self.data.X if var_selector is None else self.data.X[:, var_selector]) return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None) @requires_data @@ -607,25 +580,17 @@ class ScanpyEngine(CXGDriver): if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB: raise FilterError("Observation filters may not contain vaiable conditions") try: - obs_mask_A = self._axis_filter_to_mask( - obsFilterA["obs"], self.data.obs, self.data.n_obs - ) - obs_mask_B = self._axis_filter_to_mask( - obsFilterB["obs"], self.data.obs, self.data.n_obs - ) + obs_mask_A = self._axis_filter_to_mask(obsFilterA["obs"], self.data.obs, self.data.n_obs) + obs_mask_B = self._axis_filter_to_mask(obsFilterB["obs"], self.data.obs, self.data.n_obs) except (KeyError, IndexError) as e: raise FilterError(f"Error parsing filter: {e}") from e if top_n is None: top_n = DEFAULT_TOP_N - result = diffexp_ttest( - self.data, obs_mask_A, obs_mask_B, top_n, self.config['diffexp_lfc_cutoff'] - ) + result = diffexp_ttest(self.data, obs_mask_A, obs_mask_B, top_n, self.config["diffexp_lfc_cutoff"]) try: return jsonify_scanpy(result) except ValueError: - raise JSONEncodingValueError( - "Error encoding differential expression to JSON" - ) + raise JSONEncodingValueError("Error encoding differential expression to JSON") @requires_data def layout_to_fbs_matrix(self): @@ -661,7 +626,8 @@ class ScanpyEngine(CXGDriver): except ValueError as e: raise PrepareError( f"Layout has not been calculated using {self.config['layout']}, " - f"please prepare your datafile and relaunch cellxgene") from e + f"please prepare your datafile and relaunch cellxgene" + ) from e df = pandas.concat(layout_data, axis=1, copy=False) return encode_matrix_fbs(df, col_idx=df.columns, row_idx=None) diff --git a/server/app/util/constants.py b/server/app/util/constants.py index b53d476c..545a8d50 100644 --- a/server/app/util/constants.py +++ b/server/app/util/constants.py @@ -27,9 +27,7 @@ class DiffExpMode(AugmentedEnum): VAR_FILTER = "varFilter" -JSON_NaN_to_num_warning_msg = ( - "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)" -) +JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)" REACTIVE_LIMIT = 1_000_000 MAX_LAYOUTS = 30 diff --git a/server/app/util/data_locator.py b/server/app/util/data_locator.py index 8a44f06d..2b7a8150 100644 --- a/server/app/util/data_locator.py +++ b/server/app/util/data_locator.py @@ -4,7 +4,7 @@ import fsspec from datetime import datetime -class DataLocator(): +class DataLocator: """ DataLocator is a simple wrapper around fsspec functionality, and provides a set of functions to encapsulate a data location (URI or path), interogate @@ -29,7 +29,7 @@ class DataLocator(): self.uri_or_path = uri_or_path self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path) # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol - self.cname = self.path if self.protocol == 'file' else self.uri_or_path + self.cname = self.path if self.protocol == "file" else self.uri_or_path # will throw RuntimeError if the protocol is unsupported self.fs = fsspec.filesystem(self.protocol) @@ -53,9 +53,9 @@ class DataLocator(): """ return datetime object representing last modification time, or None if unavailable """ info = self.fs.info(self.cname) if self.islocal() and info is not None: - return datetime.fromtimestamp(info['mtime']) + return datetime.fromtimestamp(info["mtime"]) else: - return getattr(info, 'LastModified', None) + return getattr(info, "LastModified", None) def abspath(self): """ @@ -74,7 +74,7 @@ class DataLocator(): return self.fs.open(self.uri_or_path, *args) def islocal(self): - return self.protocol is None or self.protocol == 'file' + return self.protocol is None or self.protocol == "file" def local_handle(self): if self.islocal(): @@ -90,7 +90,7 @@ class DataLocator(): return LocalFilePath(tmp_path, delete=True) -class LocalFilePath(): +class LocalFilePath: def __init__(self, tmp_path, delete=False): self.tmp_path = tmp_path self.delete = delete diff --git a/server/app/util/fbs/matrix.py b/server/app/util/fbs/matrix.py index b29f4eb4..637c39d7 100644 --- a/server/app/util/fbs/matrix.py +++ b/server/app/util/fbs/matrix.py @@ -27,7 +27,7 @@ def CreateNumpyVector(builder, x): if not isinstance(x, np.ndarray): raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}") - if x.dtype.kind not in ['b', 'i', 'u', 'f']: + if x.dtype.kind not in ["b", "i", "u", "f"]: raise TypeError("numpy-ndarray holds elements of unsupported datatype") if x.ndim > 1: @@ -42,11 +42,11 @@ def CreateNumpyVector(builder, x): x_little_endian = x.byteswap(inplace=False) # Calculate total length - len = int(x_little_endian.itemsize * x_little_endian.size) - builder.head = int(builder.Head() - len) + length = int(x_little_endian.itemsize * x_little_endian.size) + builder.head = int(builder.Head() - length) # tobytes ensures c_contiguous ordering - builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C') + builder.Bytes[builder.Head() : builder.Head() + length] = x_little_endian.tobytes(order="C") return builder.EndVector(x.size) @@ -88,9 +88,9 @@ def serialize_typed_array(builder, source_array, encoding_info): arr = arr.to_series() # convert to a simple ndarray - if as_type == 'json': - as_json = arr.to_json(orient='records') - arr = np.array(bytearray(as_json, 'utf-8')) + if as_type == "json": + as_json = arr.to_json(orient="records") + arr = np.array(bytearray(as_json, "utf-8")) else: if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr): arr = arr.toarray() @@ -119,18 +119,16 @@ column_encoding_type_map = { np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), - np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), - np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32) + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), } -column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json') +column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") def column_encoding(arr): @@ -141,11 +139,10 @@ index_encoding_type_map = { # array protocol string: ( array_type, as_type ) np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), - np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32) + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), } -index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json') +index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") def index_encoding(arr): @@ -163,7 +160,7 @@ def guess_at_mem_needed(matrix): guess = 1 # round up to nearest 1024 bytes - guess = (guess + 0x400) & (~0x3ff) + guess = (guess + 0x400) & (~0x3FF) return guess @@ -223,7 +220,7 @@ def deserialize_typed_array(tarr): TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, - TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray + TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray, } (u_type, u) = tarr if u_type is TypedArray.TypedArray.NONE: @@ -237,7 +234,7 @@ def deserialize_typed_array(tarr): arr.Init(u.Bytes, u.Pos) narr = arr.DataAsNumpy() if u_type == TypedArray.TypedArray.JSONEncodedArray: - narr = json.loads(narr.tostring().decode('utf-8')) + narr = json.loads(narr.tostring().decode("utf-8")) return narr diff --git a/server/app/util/matrix_proxy.py b/server/app/util/matrix_proxy.py index 32414c1d..ad81c43f 100644 --- a/server/app/util/matrix_proxy.py +++ b/server/app/util/matrix_proxy.py @@ -18,6 +18,7 @@ class _ArrayProxyBase(abc.ABC): Private base class for array or matrix proxy. This summarizes the interface used by the rest of cellxgene. """ + @property @abc.abstractmethod def dtype(self): @@ -68,10 +69,10 @@ class MatrixProxy(_ArrayProxyBase): Sub-classes automatically register. """ base_proxy_registry = { - 'pandas.core.frame.DataFrame': True, - 'numpy.ndarray': True, - 'scipy.sparse.csc.csc_matrix': True, - 'scipy.sparse.csr.csr_matrix': True, + "pandas.core.frame.DataFrame": True, + "numpy.ndarray": True, + "scipy.sparse.csc.csc_matrix": True, + "scipy.sparse.csr.csr_matrix": True, } proxy_registry = None last_cache_token = None @@ -103,7 +104,7 @@ class MatrixProxy(_ArrayProxyBase): """ cls.build_proxy_registry() t = type(matrix) - fqtn = t.__module__ + '.' + t.__name__ + fqtn = t.__module__ + "." + t.__name__ proxy_cls = cls.proxy_registry.get(fqtn, None) if proxy_cls is None: raise Exception(f"Matrix format `{fqtn}` is unsupported by proxy.") @@ -128,21 +129,17 @@ class MatrixProxyView(MatrixProxy): """ 2D matrix view to a 2D matrix """ - def __init__(self, arg1, shape=None, index=(), - transposed=False, copy=False): + + def __init__(self, arg1, shape=None, index=(), transposed=False, copy=False): if not copy: m = arg1 super().__init__(m) if shape is None: shape = m.shape - assert(len(shape) == 2) + assert len(shape) == 2 - index = tuple( - map(lambda s_i: - slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], - zip_longest(shape, index)) - ) + index = tuple(map(lambda s_i: slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], zip_longest(shape, index))) self._shape = shape self._index = index @@ -234,20 +231,20 @@ class MatrixProxyView(MatrixProxy): NOTE: these follow the numpy rules for dimensionality reduction when an integer index is specified. """ + def _getitem_intXint(self, row, col): return self.m[row, col] def _getitem_intXslice(self, row, col): - shape = (_slice_length(col, self.m.shape[1]), ) + shape = (_slice_length(col, self.m.shape[1]),) return self.__class__.create_array(self.m, shape=shape, index=(row, col)) def _getitem_sliceXint(self, row, col): - shape = (_slice_length(row, self.m.shape[0]), ) + shape = (_slice_length(row, self.m.shape[0]),) return self.__class__.create_array(self.m, shape=shape, index=(row, col)) def _getitem_sliceXslice(self, row, col): - shape = (_slice_length(row, self.m.shape[0]), - _slice_length(col, self.m.shape[1])) + shape = (_slice_length(row, self.m.shape[0]), _slice_length(col, self.m.shape[1])) return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed) def toarray(self): @@ -261,22 +258,23 @@ class ArrayProxyView(_ArrayProxyBase): """ 1D array view to a 2D matrix """ + def __init__(self, arg1, shape=None, index=None, copy=False): super().__init__() if not copy: m = arg1 # one index MUST be an integer and the other MUST be a slice - assert(len(index) == 2) - assert(all(isinstance(idx, INT_TYPES + (slice, )) for idx in index)) - assert(isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES)) + assert len(index) == 2 + assert all(isinstance(idx, INT_TYPES + (slice,)) for idx in index) + assert isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES) if shape is None: if isinstance(index[0], INT_TYPES): - shape = (m.shape[0], ) + shape = (m.shape[0],) else: - shape = (m.shape[1], ) - assert(len(shape) == 1) + shape = (m.shape[1],) + assert len(shape) == 1 self._shape = shape self.m = m @@ -336,7 +334,7 @@ class ArrayProxyView(_ArrayProxyBase): elif isinstance(col, slice): return self._getitem_intXslice(row, col) elif isinstance(row, slice): - assert(isinstance(col, INT_TYPES)) + assert isinstance(col, INT_TYPES) return self._getitem_sliceXint(row, col) raise IndexError("unsupported column index types") @@ -345,11 +343,11 @@ class ArrayProxyView(_ArrayProxyBase): return self.m[row, col] def _getitem_intXslice(self, row, col): - shape = (_slice_length(col, self.m.shape[1]), ) + shape = (_slice_length(col, self.m.shape[1]),) return self.__class__(self.m, shape=shape, index=(row, col)) def _getitem_sliceXint(self, row, col): - shape = (_slice_length(row, self.m.shape[0]), ) + shape = (_slice_length(row, self.m.shape[0]),) return self.__class__(self.m, shape=shape, index=(row, col)) def toarray(self): @@ -358,7 +356,7 @@ class ArrayProxyView(_ArrayProxyBase): def _unpack_index(index, shape): if not isinstance(index, tuple): - index = (index, ) + index = (index,) if len(shape) < len(index): raise IndexError("invalid index dimensionality - must be 2") @@ -366,7 +364,7 @@ def _unpack_index(index, shape): for shp, idx in zip_longest(shape, index): idx = slice(None) if idx is None else idx idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx - unpacked += (idx, ) + unpacked += (idx,) return unpacked @@ -376,7 +374,7 @@ def _slice_slice(outer, outer_len, inner, inner_len): slice a slice - we take advantage of Python 3 range's support for indexing. """ - assert(outer_len >= inner_len) + assert outer_len >= inner_len outer_rng = range(*outer.indices(outer_len)) rng = outer_rng[inner] start, stop, step = rng.start, rng.stop, rng.step @@ -387,8 +385,8 @@ def _slice_slice(outer, outer_len, inner, inner_len): def _range_length(start, stop, step): """ return length of range """ - assert(step != 0) - assert(start is not None and stop is not None and step is not None) + assert step != 0 + assert start is not None and stop is not None and step is not None if step > 0 and start < stop: return 1 + (stop - 1 - start) // step elif step < 0 and start > stop: @@ -404,7 +402,7 @@ def _slice_length(s, length): def _slice_defaults(s, length): """ apply slice defaulting conventions """ - assert(length >= 0) + assert length >= 0 step = 1 if s.step is None else s.step diff --git a/server/app/util/utils.py b/server/app/util/utils.py index 4bf57612..c931ae1b 100644 --- a/server/app/util/utils.py +++ b/server/app/util/utils.py @@ -40,4 +40,5 @@ def requires_data(func): if self.data is None: raise DriverError(f"error data must be loaded before you call {func.__name__}") return func(self, *args, **kwargs) + return wrapped_function diff --git a/server/cli/cli.py b/server/cli/cli.py index ba0576b6..439587d0 100644 --- a/server/cli/cli.py +++ b/server/cli/cli.py @@ -4,17 +4,19 @@ from .launch import launch from .prepare import prepare -@click.group(name="cellxgene", - subcommand_metavar="COMMAND ", - options_metavar="", - context_settings=dict(max_content_width=85, - help_option_names=['-h', '--help'])) +@click.group( + name="cellxgene", + subcommand_metavar="COMMAND ", + options_metavar="", + context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]), +) @click.help_option("--help", "-h", help="Show this message and exit.") @click.version_option( version="0.13.0", prog_name="cellxgene", message="[%(prog)s] Version %(version)s", - help="Show the software version and exit.") + help="Show the software version and exit.", +) def cli(): pass diff --git a/server/cli/launch.py b/server/cli/launch.py index c5fa3178..c7f0f006 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -25,16 +25,12 @@ def common_args(func): Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args. """ - @click.option( - "--title", - "-t", - metavar="", - help="Title to display. If omitted will use file name.") + @click.option("--title", "-t", metavar="", help="Title to display. If omitted will use file name.") @click.option( "--about", metavar="", - help="URL providing more information about the dataset " - "(hint: must be a fully specified absolute URL).") + help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).", + ) @click.option( "--embedding", "-e", @@ -42,39 +38,43 @@ def common_args(func): multiple=True, show_default=False, metavar="", - help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all." + help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.", ) @click.option( "--obs-names", "-obs", default=None, metavar="", - help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.") + help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.", + ) @click.option( "--var-names", "-var", default=None, metavar="", - help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.") + help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.", + ) @click.option( "--max-category-items", default=1000, metavar="", show_default=True, - help="Will not display categories with more distinct values than specified.",) + help="Will not display categories with more distinct values than specified.", + ) @click.option( "--diffexp-lfc-cutoff", "-de", default=0.01, show_default=True, metavar="", - help="Minimum log fold change threshold for differential expression.",) + help="Minimum log fold change threshold for differential expression.", + ) @click.option( "--experimental-annotations", is_flag=True, default=False, show_default=True, - help="Enable user annotation of data." + help="Enable user annotation of data.", ) @click.option( "--experimental-annotations-file", @@ -83,7 +83,8 @@ def common_args(func): multiple=False, metavar="", help="CSV file to initialize editing of existing annotations; will be altered in-place. " - "Incompatible with --annotations-output-dir.",) + "Incompatible with --annotations-output-dir.", + ) @click.option( "--experimental-annotations-output-dir", default=None, @@ -91,20 +92,23 @@ def common_args(func): multiple=False, metavar="", help="Directory of where to save output annotations; filename will be specified in the application. " - "Incompatible with --annotations-input-file.",) + "Incompatible with --annotations-input-file.", + ) @click.option( "--backed", "-b", is_flag=True, default=False, show_default=False, - help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.") + help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.", + ) @click.option( "--disable-diffexp", is_flag=True, default=False, show_default=False, - help="Disable on-demand differential expression.") + help="Disable on-demand differential expression.", + ) @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -112,9 +116,18 @@ def common_args(func): return wrapper -def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, - experimental_annotations, experimental_annotations_file, - experimental_annotations_output_dir, backed, disable_diffexp): +def parse_engine_args( + embedding, + obs_names, + var_names, + max_category_items, + diffexp_lfc_cutoff, + experimental_annotations, + experimental_annotations_file, + experimental_annotations_output_dir, + backed, + disable_diffexp, +): annotations_file = experimental_annotations_file if experimental_annotations else None annotations_output_dir = experimental_annotations_output_dir if experimental_annotations else None return { @@ -127,14 +140,15 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe "annotations_file": annotations_file, "annotations_output_dir": annotations_output_dir, "backed": backed, - "disable_diffexp": disable_diffexp + "disable_diffexp": disable_diffexp, } @sort_options -@click.command(short_help="Launch the cellxgene data viewer. " - "Run `cellxgene launch --help` for more information.", - options_metavar="",) +@click.command( + short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.", + options_metavar="", +) @click.argument("data", nargs=1, metavar="", required=True) @click.option( "--verbose", @@ -142,7 +156,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe is_flag=True, default=False, show_default=True, - help="Provide verbose output, including warnings and all server requests.",) + help="Provide verbose output, including warnings and all server requests.", +) @click.option( "--debug", "-d", @@ -150,7 +165,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe default=False, show_default=True, help="Run in debug mode. This is helpful for cellxgene developers, " - "or when you want more information about an error condition.",) + "or when you want more information about an error condition.", +) @click.option( "--open", "-o", @@ -158,19 +174,22 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe is_flag=True, default=False, show_default=True, - help="Open web browser after launch.",) + help="Open web browser after launch.", +) @click.option( "--port", "-p", metavar="", show_default=True, - help="Port to run server on. If not specified cellxgene will find an available port.",) + help="Port to run server on. If not specified cellxgene will find an available port.", +) @click.option( "--host", metavar="", default="127.0.0.1", show_default=False, - help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).") + help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).", +) @click.option( "--scripts", "-s", @@ -178,30 +197,31 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe multiple=True, metavar="", help="Additional script files to include in HTML page. If not specified, " - "no additional script files will be included.", - show_default=False,) + "no additional script files will be included.", + show_default=False, +) @click.help_option("--help", "-h", help="Show this message and exit.") @common_args def launch( - data, - verbose, - debug, - open_browser, - port, - host, - embedding, - obs_names, - var_names, - max_category_items, - diffexp_lfc_cutoff, - title, - scripts, - about, - experimental_annotations, - experimental_annotations_file, - experimental_annotations_output_dir, - backed, - disable_diffexp + data, + verbose, + debug, + open_browser, + port, + host, + embedding, + obs_names, + var_names, + max_category_items, + diffexp_lfc_cutoff, + title, + scripts, + about, + experimental_annotations, + experimental_annotations_file, + experimental_annotations_output_dir, + backed, + disable_diffexp, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. @@ -217,13 +237,18 @@ def launch( > cellxgene launch """ - e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items, - diffexp_lfc_cutoff, - experimental_annotations, - experimental_annotations_file, - experimental_annotations_output_dir, - backed, - disable_diffexp) + e_args = parse_engine_args( + embedding, + obs_names, + var_names, + max_category_items, + diffexp_lfc_cutoff, + experimental_annotations, + experimental_annotations_file, + experimental_annotations_output_dir, + backed, + disable_diffexp, + ) try: data_locator = DataLocator(data) except RuntimeError as re: @@ -256,7 +281,8 @@ def launch( sys.tracebacklimit = 0 if scripts: - click.echo(r""" + click.echo( + r""" / / /\ \ \__ _ _ __ _ __ (_)_ __ __ _ \ \/ \/ / _` | '__| '_ \| | '_ \ / _` | \ /\ / (_| | | | | | | | | | | (_| | @@ -264,7 +290,8 @@ def launch( |___/ The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a security risk by including the --scripts flag. Make sure you trust the scripts that you are including. - """) + """ + ) scripts_pretty = ", ".join(scripts) click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) @@ -289,8 +316,9 @@ def launch( click.echo("Warning: --experimental-annotations-output-dir ignored as --annotations not enabled.") else: if experimental_annotations_file is not None and experimental_annotations_output_dir is not None: - raise click.ClickException("--experimental-annotations-file and --experimental-annotations-output-dir " - "may not be used together.") + raise click.ClickException( + "--experimental-annotations-file and --experimental-annotations-output-dir " "may not be used together." + ) if experimental_annotations_file is not None: lf_name, lf_ext = splitext(experimental_annotations_file) @@ -301,10 +329,12 @@ def launch( try: mkdir(experimental_annotations_output_dir) except OSError: - raise click.ClickException("Unable to create directory specified by " - "--experimental-annotations-output-dir") + raise click.ClickException( + "Unable to create directory specified by " "--experimental-annotations-output-dir" + ) if about: + def url_check(url): try: result = urlparse(url) @@ -346,9 +376,11 @@ def launch( except ScanpyFileError as e: raise click.ClickException(f"{e}") - if not disable_diffexp and server.app.data.config['diffexp_may_be_slow']: - click.echo(f"[cellxgene] CAUTION: due to the size of your dataset, " - f"running differential expression may take longer or fail.") + if not disable_diffexp and server.app.data.config["diffexp_may_be_slow"]: + click.echo( + f"[cellxgene] CAUTION: due to the size of your dataset, " + f"running differential expression may take longer or fail." + ) if open_browser: click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.") diff --git a/server/cli/prepare.py b/server/cli/prepare.py index 224fa17c..8fd9e0bf 100644 --- a/server/cli/prepare.py +++ b/server/cli/prepare.py @@ -8,9 +8,10 @@ from server.utils.utils import sort_options @sort_options -@click.command(short_help="Preprocess data for use with cellxgene. " - "Run `cellxgene prepare --help` for more information.", - options_metavar="",) +@click.command( + short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.", + options_metavar="", +) @click.argument("data", nargs=1, metavar="", required=True) @click.option( "--embedding", @@ -35,37 +36,33 @@ from server.utils.utils import sort_options @click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True) @click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="") @click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="") -@click.option("--skip-qc", default=False, is_flag=True, - help="Do not run quality control metrics. By default cellxgene runs them " - "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).") @click.option( - "--make-obs-names-unique", - default=True, + "--skip-qc", + default=False, is_flag=True, - help="Ensure obs index is unique.", - show_default=True + help="Do not run quality control metrics. By default cellxgene runs them " + "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", ) @click.option( - "--make-var-names-unique", - default=True, - is_flag=True, - help="Ensure var index is unique.", - show_default=True + "--make-obs-names-unique", default=True, is_flag=True, help="Ensure obs index is unique.", show_default=True +) +@click.option( + "--make-var-names-unique", default=True, is_flag=True, help="Ensure var index is unique.", show_default=True ) @click.help_option("--help", "-h", help="Show this message and exit.") def prepare( - data, - embedding, - recipe, - output, - plotting, - sparse, - overwrite, - set_obs_names, - set_var_names, - skip_qc, - make_obs_names_unique, - make_var_names_unique, + data, + embedding, + recipe, + output, + plotting, + sparse, + overwrite, + set_obs_names, + set_var_names, + skip_qc, + make_obs_names_unique, + make_var_names_unique, ): """ Preprocess data for use with cellxgene. diff --git a/server/gui/browser.py b/server/gui/browser.py index d09bc621..aa2224e4 100644 --- a/server/gui/browser.py +++ b/server/gui/browser.py @@ -12,6 +12,7 @@ WindowUtils = cef.WindowUtils() # noinspection PyUnresolvedReferences CefWidgetParent = QWidget + class CefWidget(CefWidgetParent): def __init__(self, parent=None): super(CefWidget, self).__init__(parent) @@ -58,8 +59,7 @@ class CefWidget(CefWidgetParent): if WINDOWS: WindowUtils.OnSize(self.getHandle(), 0, 0, 0) elif LINUX: - self.browser.SetBounds(self.x, self.y, - self.width(), self.height()) + self.browser.SetBounds(self.x, self.y, self.width(), self.height()) self.browser.NotifyMoveOrResizeStarted() def resizeEvent(self, event): @@ -68,8 +68,7 @@ class CefWidget(CefWidgetParent): if WINDOWS: WindowUtils.OnSize(self.getHandle(), 0, 0, 0) elif LINUX: - self.browser.SetBounds(self.x, self.y, - size.width(), size.height()) + self.browser.SetBounds(self.x, self.y, size.width(), size.height()) self.browser.NotifyMoveOrResizeStarted() diff --git a/server/gui/hook-cefpython3.py b/server/gui/hook-cefpython3.py index 4c6e0a9d..6e3b66b1 100644 --- a/server/gui/hook-cefpython3.py +++ b/server/gui/hook-cefpython3.py @@ -37,8 +37,7 @@ logger = logging.getLogger(__name__) # Functions def check_platforms(): if not is_win and not is_darwin and not is_linux: - raise SystemExit("Error: Currently only Windows, Linux and Darwin " - "platforms are supported, see Issue #135.") + raise SystemExit("Error: Currently only Windows, Linux and Darwin " "platforms are supported, see Issue #135.") def check_pyinstaller_version(): @@ -50,22 +49,19 @@ def check_pyinstaller_version(): version = PyInstaller.__version__ match = re.search(r"^\d+\.\d+(\.\d+)?", version) if not (match.group(0) >= PYINSTALLER_MIN_VERSION): - raise SystemExit("Error: pyinstaller %s or higher is required" - % PYINSTALLER_MIN_VERSION) + raise SystemExit("Error: pyinstaller %s or higher is required" % PYINSTALLER_MIN_VERSION) def check_cefpython3_version(): if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION): - raise SystemExit("Error: cefpython3 %s or higher is required" - % CEFPYTHON_MIN_VERSION) + raise SystemExit("Error: cefpython3 %s or higher is required" % CEFPYTHON_MIN_VERSION) def get_cefpython_modules(): """Get all cefpython Cython modules in the cefpython3 package. It returns a list of names without file extension. Eg. 'cefpython_py27'. """ - pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, - "cefpython_py*" + CYTHON_MODULE_EXT)) + pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, "cefpython_py*" + CYTHON_MODULE_EXT)) assert len(pyds) > 1, "Missing cefpython3 Cython modules" modules = [] for path in pyds: @@ -130,14 +126,21 @@ def get_cefpython3_datas(): for filename in os.listdir(CEFPYTHON3_DIR): # Ignore Cython modules which are already handled by # pyinstaller automatically. - if filename[:-len(CYTHON_MODULE_EXT)] in get_cefpython_modules(): + if filename[: -len(CYTHON_MODULE_EXT)] in get_cefpython_modules(): continue # CEF binaries and datas extension = os.path.splitext(filename)[1] - if extension in \ - [".exe", ".dll", ".pak", ".dat", ".bin", ".txt", ".so", ".plist"] \ - or filename.lower().startswith("license"): + if extension in [ + ".exe", + ".dll", + ".pak", + ".dat", + ".bin", + ".txt", + ".so", + ".plist", + ] or filename.lower().startswith("license"): logger.info("Include cefpython3 data: {}".format(filename)) ret.append((os.path.join(CEFPYTHON3_DIR, filename), cefdatadir)) @@ -145,11 +148,9 @@ def get_cefpython3_datas(): # "Chromium Embedded Framework.framework/Resources" with subdirectories # is required. Contain .pak files and locales (each locale in separate # subdirectory). - resources_subdir = \ - os.path.join("Chromium Embedded Framework.framework", "Resources") + resources_subdir = os.path.join("Chromium Embedded Framework.framework", "Resources") base_path = os.path.join(CEFPYTHON3_DIR, resources_subdir) - assert os.path.exists(base_path), \ - "{} dir not found in cefpython3".format(resources_subdir) + assert os.path.exists(base_path), "{} dir not found in cefpython3".format(resources_subdir) for path, dirs, files in os.walk(base_path): for file in files: absolute_file_path = os.path.join(path, file) @@ -159,22 +160,17 @@ def get_cefpython3_datas(): elif is_win or is_linux: # The .pak files in cefpython3/locales/ directory locales_dir = os.path.join(CEFPYTHON3_DIR, "locales") - assert os.path.exists(locales_dir), \ - "locales/ dir not found in cefpython3" + assert os.path.exists(locales_dir), "locales/ dir not found in cefpython3" for filename in os.listdir(locales_dir): - logger.info("Include cefpython3 data: {}/{}".format( - os.path.basename(locales_dir), filename)) - ret.append((os.path.join(locales_dir, filename), - os.path.join(cefdatadir, "locales"))) + logger.info("Include cefpython3 data: {}/{}".format(os.path.basename(locales_dir), filename)) + ret.append((os.path.join(locales_dir, filename), os.path.join(cefdatadir, "locales"))) # Optional .so/.dll files in cefpython3/swiftshader/ directory swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader") if os.path.isdir(swiftshader_dir): for filename in os.listdir(swiftshader_dir): - logger.info("Include cefpython3 data: {}/{}".format( - os.path.basename(swiftshader_dir), filename)) - ret.append((os.path.join(swiftshader_dir, filename), - os.path.join(cefdatadir, "swiftshader"))) + logger.info("Include cefpython3 data: {}/{}".format(os.path.basename(swiftshader_dir), filename)) + ret.append((os.path.join(swiftshader_dir, filename), os.path.join(cefdatadir, "swiftshader"))) return ret diff --git a/server/gui/main.py b/server/gui/main.py index 72f3e2a7..9963c684 100644 --- a/server/gui/main.py +++ b/server/gui/main.py @@ -20,8 +20,8 @@ from server.utils.utils import find_available_port if WINDOWS or LINUX: dirname = dirname(PySide2.__file__) - plugin_path = join(dirname, 'plugins', 'platforms') - environ['QT_QPA_PLATFORM_PLUGIN_PATH'] = plugin_path + plugin_path = join(dirname, "plugins", "platforms") + environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = plugin_path # Configuration # TODO remember this or calculate it? @@ -94,8 +94,7 @@ class MainWindow(QMainWindow): # a hidden window, embed CEF browser in it and then # create a container for that hidden window and replace # cef widget in the layout with the container. - self.container = QWidget.createWindowContainer( - self.cef_widget.hidden_window, parent=self) + self.container = QWidget.createWindowContainer(self.cef_widget.hidden_window, parent=self) self.stacked_layout.replaceWidget(self.cef_widget, self.container) self.stacked_layout.setCurrentIndex(LOAD_INDEX) @@ -117,7 +116,7 @@ class MainWindow(QMainWindow): def setupMenu(self): # TODO add communication to subprocess on reload main_menu = self.menuBar() - file_menu = main_menu.addMenu('File') + file_menu = main_menu.addMenu("File") load_action = QAction("Load file...", self) load_action.setStatusTip("Load file") load_action.setShortcut("Ctrl+O") @@ -201,7 +200,7 @@ class LoadWidget(QFrame): for l in [logo_layout, file_layout, message_layout]: load_ui_layout.addLayout(l) - #TODO remove magic number + # TODO remove magic number load_ui_layout.setStretch(1, 10) self.setLayout(load_ui_layout) @@ -240,8 +239,15 @@ class LoadWidget(QFrame): def createScanpyEngine(self, file_name): title = splitext(basename(file_name))[0] self.window().setupServer() - worker = Worker(self.window().parent_conn, self.window().child_conn, file_name, host="127.0.0.1", - port=GUI_PORT, title=title, engine_options={}) + worker = Worker( + self.window().parent_conn, + self.window().child_conn, + file_name, + host="127.0.0.1", + port=GUI_PORT, + title=title, + engine_options={}, + ) self.window().load_emitter.signals.ready.connect(self.onDataReady) self.window().load_emitter.signals.engine_error.connect(self.onServerError) self.window().load_emitter.signals.server_error.connect(self.onServerError) @@ -289,6 +295,7 @@ class LoadWidget(QFrame): onServerError = partialmethod(onError, server_error=True) + class FilePath(QObject): def __init__(self): super(FilePath, self).__init__() @@ -320,8 +327,7 @@ class FileArea(QFrame): def fileBrowse(self): options = QFileDialog.Options() # options |= QFileDialog.DontUseNativeDialog - file_name, _ = QFileDialog.getOpenFileName(self, - "Open H5AD File", "", "H5AD Files (*.h5ad)", options=options) + file_name, _ = QFileDialog.getOpenFileName(self, "Open H5AD File", "", "H5AD Files (*.h5ad)", options=options) if file_name: self.parent().file_name.updateValue(file_name) self.parent().onLoad() @@ -391,5 +397,5 @@ def main(): sys.exit(0) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/server/gui/utils.py b/server/gui/utils.py index b2149461..7873f49d 100644 --- a/server/gui/utils.py +++ b/server/gui/utils.py @@ -4,9 +4,9 @@ import platform from PySide2.QtCore import QObject, Signal # Detect OS -WINDOWS = (platform.system() == "Windows") -LINUX = (platform.system() == "Linux") -MAC = (platform.system() == "Darwin") +WINDOWS = platform.system() == "Windows" +LINUX = platform.system() == "Linux" +MAC = platform.system() == "Darwin" class WorkerSignals(QObject): @@ -18,6 +18,7 @@ class WorkerSignals(QObject): error - `str` error message result - `object` data returned from processing, anything """ + finished = Signal() engine_error = Signal(str) server_error = Signal(str) @@ -34,6 +35,7 @@ class SiteReadySignals(QObject): ready error - `str` error message """ + ready = Signal() timeout = Signal() error = Signal(str) diff --git a/server/gui/workers.py b/server/gui/workers.py index 62320ab8..e77e057d 100644 --- a/server/gui/workers.py +++ b/server/gui/workers.py @@ -36,6 +36,7 @@ class Worker(EmittingProcess): return from server.app.app import Server from server.app.scanpy_engine.scanpy_engine import ScanpyEngine + # create server try: server = Server() diff --git a/server/test/decode_fbs.py b/server/test/decode_fbs.py index a9b847ad..ea029a1c 100644 --- a/server/test/decode_fbs.py +++ b/server/test/decode_fbs.py @@ -1,4 +1,3 @@ - """ Code to decode, for testing purposes, the flatbuffer encoded blobs. This code will need to be updated if fbs/matrix.fbs changes. @@ -22,20 +21,20 @@ def decode_typed_array(tarr): TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, - TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray + TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray, } (u_type, u) = tarr if u_type == TypedArray.TypedArray.NONE: return None TarType = type_map.get(u_type, None) - assert(TarType is not None) + assert TarType is not None arr = TarType() arr.Init(u.Bytes, u.Pos) narr = arr.DataAsNumpy() if u_type == TypedArray.TypedArray.JSONEncodedArray: - narr = json.loads(narr.tostring().decode('utf-8')) + narr = json.loads(narr.tostring().decode("utf-8")) return narr @@ -60,10 +59,4 @@ def decode_matrix_FBS(buf): cidx = decode_typed_array((df.ColIndexType(), df.ColIndex())) - return { - "n_rows": n_rows, - "n_cols": n_cols, - "columns": decoded_columns, - "col_idx": cidx, - "row_idx": None - } + return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None} diff --git a/server/test/test_api.py b/server/test/test_api.py index 7f396431..dea00132 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -19,7 +19,7 @@ class EndPoints(unittest.TestCase): @classmethod def setUpClass(cls): - cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"]) + cls.ps = Popen(["cellxgene", "launch", "../example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"]) session = requests.Session() for i in range(90): try: @@ -68,14 +68,15 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 8) - self.assertIsNotNone(df['columns']) - self.assertListEqual(df['col_idx'], [ - 'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1' - ]) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 8) + self.assertIsNotNone(df["columns"]) + self.assertListEqual( + df["col_idx"], + ["pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"], + ) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) def test_bad_filter(self): endpoint = "data/var" @@ -91,14 +92,14 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 5) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 5) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"] - self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain']) + self.assertListEqual(df["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]) def test_get_annotations_obs_keys_fbs(self): endpoint = "annotations/obs" @@ -109,13 +110,13 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 2) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) - self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito']) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 2) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertListEqual(df["col_idx"], ["n_genes", "percent_mito"]) def test_get_annotations_obs_error(self): endpoint = "annotations/obs" @@ -162,14 +163,14 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 1838) - self.assertEqual(df['n_cols'], 2) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) + self.assertEqual(df["n_rows"], 1838) + self.assertEqual(df["n_cols"], 2) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"] - self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells']) + self.assertListEqual(df["col_idx"], [var_index_col_name, "n_cells"]) def test_get_annotations_var_keys_fbs(self): endpoint = "annotations/var" @@ -180,13 +181,13 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 1838) - self.assertEqual(df['n_cols'], 1) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) - self.assertListEqual(df['col_idx'], ['n_cells']) + self.assertEqual(df["n_rows"], 1838) + self.assertEqual(df["n_cols"], 1) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertListEqual(df["col_idx"], ["n_cells"]) def test_get_annotations_var_error(self): endpoint = "annotations/var" @@ -217,35 +218,29 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 1838) - self.assertIsNotNone(df['columns']) - self.assertListEqual(df['col_idx'].tolist(), []) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1838) + self.assertIsNotNone(df["columns"]) + self.assertListEqual(df["col_idx"].tolist(), []) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) def test_data_put_filter_fbs(self): endpoint = f"data/var" url = f"{URL_BASE}{endpoint}" header = {"Accept": "application/octet-stream"} - filter = { - "filter": { - "var": { - "index": [0, 1, 4] - } - } - } + filter = {"filter": {"var": {"index": [0, 1, 4]}}} result = self.session.put(url, headers=header, json=filter) self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 3) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) - self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4]) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 3) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4]) def test_data_put_single_var(self): endpoint = f"data/var" diff --git a/server/test/test_fbs.py b/server/test/test_fbs.py index da32903a..36c3ec59 100644 --- a/server/test/test_fbs.py +++ b/server/test/test_fbs.py @@ -32,7 +32,7 @@ class FbsTests(unittest.TestCase): for i in range(0, len(d["columns"])): self.assertEqual(len(d["columns"][i]), dims[0]) self.assertIsInstance(d["columns"][i], expected_types[i][0]) - if (expected_types[i][1] is not None): + if expected_types[i][1] is not None: self.assertEqual(d["columns"][i].dtype, expected_types[i][1]) if expected_column_idx is not None: self.assertSetEqual(set(expected_column_idx), set(d["col_idx"])) @@ -40,48 +40,37 @@ class FbsTests(unittest.TestCase): def test_encode_DataFrame(self): df = pd.DataFrame( data={ - 'a': np.zeros((10,), dtype=np.float32), - 'b': np.ones((10,), dtype=np.int64), - 'c': np.array([i for i in range(0, 10)], dtype=np.uint16), - 'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') - }) - expected_types = ( - (np.ndarray, np.float32), - (np.ndarray, np.int32), - (np.ndarray, np.uint32), - (list, None) + "a": np.zeros((10,), dtype=np.float32), + "b": np.ones((10,), dtype=np.int64), + "c": np.array([i for i in range(0, 10)], dtype=np.uint16), + "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), + } ) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None)) fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) - self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd']) + self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"]) def test_encode_ndarray(self): arr = np.zeros((3, 2), dtype=np.float32) - expected_types = ( - (np.ndarray, np.float32), - (np.ndarray, np.float32), - (np.ndarray, np.float32) - ) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32)) fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None) self.fbs_checks(fbs, (3, 2), expected_types, None) def test_encode_sparse(self): csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]])) - expected_types = ( - (np.ndarray, np.int32), - (np.ndarray, np.int32), - (np.ndarray, np.int32) - ) + expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32)) fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None) self.fbs_checks(fbs, (2, 3), expected_types, None) def test_roundtrip(self): dfSrc = pd.DataFrame( data={ - 'a': np.zeros((10,), dtype=np.float32), - 'b': np.ones((10,), dtype=np.int64), - 'c': np.array([i for i in range(0, 10)], dtype=np.uint16), - 'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') - }) + "a": np.zeros((10,), dtype=np.float32), + "b": np.ones((10,), dtype=np.int64), + "c": np.array([i for i in range(0, 10)], dtype=np.uint16), + "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), + } + ) dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns)) self.assertEqual(dfSrc.shape, dfDst.shape) self.assertEqual(set(dfSrc.columns), set(dfDst.columns)) diff --git a/server/test/test_matrix_proxy.py b/server/test/test_matrix_proxy.py index 624cbda2..2c3ad5d9 100644 --- a/server/test/test_matrix_proxy.py +++ b/server/test/test_matrix_proxy.py @@ -7,9 +7,10 @@ class NdArrayProxyView(MatrixProxyView): """ Fake test class for matrix proxy - wraps ndarray """ + @classmethod def __supports__(cls): - return ('numpy.ndarray', ) + return ("numpy.ndarray",) class MatrixProxyViewTest(unittest.TestCase): @@ -41,18 +42,17 @@ class MatrixProxyViewTest(unittest.TestCase): def test_toarray(self): n = np.arange(15, dtype=np.float32).reshape((3, 5)) mp = MatrixProxy.create(n) - self.assertTrue(np.all(mp.toarray() == [ - [0., 1., 2., 3., 4.], - [5., 6., 7., 8., 9.], - [10., 11., 12., 13., 14.] - ])) - self.assertTrue(np.all(mp.T.toarray() == [ - [0., 5., 10.], - [1., 6., 11.], - [2., 7., 12.], - [3., 8., 13.], - [4., 9., 14.] - ])) + self.assertTrue( + np.all( + mp.toarray() == [[0.0, 1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0, 9.0], [10.0, 11.0, 12.0, 13.0, 14.0]] + ) + ) + self.assertTrue( + np.all( + mp.T.toarray() + == [[0.0, 5.0, 10.0], [1.0, 6.0, 11.0], [2.0, 7.0, 12.0], [3.0, 8.0, 13.0], [4.0, 9.0, 14.0]] + ) + ) def test_indexing(self): """ @@ -95,47 +95,19 @@ class MatrixProxyViewTest(unittest.TestCase): # slice, slice - self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [ - [7, 8], - [12, 13] - ])) - self.assertTrue(np.all(mp[:3, :4].toarray() == [ - [0., 1., 2., 3.], - [5., 6., 7., 8.], - [10., 11., 12., 13.] - ])) - self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [ - [14, 13, 12, 11, 10], - [9, 8, 7, 6, 5], - [4, 3, 2, 1, 0] - ])) - self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [ - [14, 12, 10], - [4, 2, 0] - ])) + self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]])) + self.assertTrue( + np.all(mp[:3, :4].toarray() == [[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0], [10.0, 11.0, 12.0, 13.0]]) + ) + self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]])) + self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]])) - self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [ - [7, 12], - [8, 13] - ])) - self.assertTrue(np.all(mp.T[:4, :3].toarray() == [ - [0, 5, 10], - [1, 6, 11], - [2, 7, 12], - [3, 8, 13] - ])) - self.assertTrue(np.all(mp.T[::-1, ::-1].toarray() == [ - [14, 9, 4], - [13, 8, 3], - [12, 7, 2], - [11, 6, 1], - [10, 5, 0] - ])) - self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [ - [14, 4], - [12, 2], - [10, 0] - ])) + self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]])) + self.assertTrue(np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8, 13]])) + self.assertTrue( + np.all(mp.T[::-1, ::-1].toarray() == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]]) + ) + self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]])) def test_repeated_indexing(self): """ @@ -149,31 +121,15 @@ class MatrixProxyViewTest(unittest.TestCase): self.assertEqual(mp[0][1], 1) self.assertEqual(mp.T[0][1], 5) - self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [ - 2, 1 - ])) - self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [ - 2, 3 - ])) - self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [ - 3, 2 - ])) - self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [ - 3, 2 - ])) - self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [ - 2, 3 - ])) + self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1])) + self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3])) + self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2])) + self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2])) + self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [2, 3])) - self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [ - 2, 1 - ])) - self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [ - 10 - ])) - self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [ - 10 - ])) + self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1])) + self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10])) + self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == [])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [])) @@ -190,20 +146,12 @@ class MatrixProxyViewTest(unittest.TestCase): self.assertEqual(mp[0, 0], 0) # drop 1 dimension, to an array - self.assertTrue(np.all(mp[0, :].toarray() == [ - 0, 1, 2, 3, 4 - ])) - self.assertTrue(np.all(mp[:, 0].toarray() == [ - 0, 5, 10 - ])) + self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4])) + self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10])) # with .T - self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [ - [4, 9] - ])) - self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [ - 4, 9 - ])) + self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]])) + self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9])) def test_iter(self): """ @@ -214,17 +162,13 @@ class MatrixProxyViewTest(unittest.TestCase): rows = [r for r in mp] self.assertEqual(len(rows), 3) - self.assertTrue(np.all(rows[0].toarray() == [ - 0, 1, 2, 3, 4 - ])) + self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4])) for i, r in enumerate(rows): self.assertTrue(np.all(mp[i].toarray() == r.toarray())) cols = [c for c in mp.T] self.assertEqual(len(cols), 5) - self.assertTrue(np.all(cols[0].toarray() == [ - 0, 5, 10 - ])) + self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10])) for i, c in enumerate(cols): self.assertTrue(np.all(mp.T[i].toarray() == c.toarray())) diff --git a/server/test/test_nan_rest.py b/server/test/test_nan_rest.py index a9ba4dd7..5dcf964a 100644 --- a/server/test/test_nan_rest.py +++ b/server/test/test_nan_rest.py @@ -20,9 +20,7 @@ class WithNaNs(unittest.TestCase): @classmethod def setUpClass(cls): - cls.ps = Popen( - ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"] - ) + cls.ps = Popen(["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]) session = requests.Session() for i in range(90): try: diff --git a/server/test/test_nan_scanpy_engine.py b/server/test/test_nan_scanpy_engine.py index ec85d2aa..84fa255e 100644 --- a/server/test/test_nan_scanpy_engine.py +++ b/server/test/test_nan_scanpy_engine.py @@ -21,12 +21,12 @@ class NaNTest(unittest.TestCase): } with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) - self.data = ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args) + self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args) self.data._create_schema() def test_load(self): with self.assertWarns(UserWarning): - ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args) + ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args) def test_init(self): self.assertEqual(self.data.cell_count, 100) @@ -44,11 +44,7 @@ class NaNTest(unittest.TestCase): with pytest.raises(FilterError): self.data.data_frame_to_fbs_matrix("an erroneous filter", "var") with pytest.raises(FilterError): - filter_ = { - "filter": { - "obs": {"index": [1, 99, [200, 300]]} - } - } + filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}} self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") def test_dataframe_obs_not_implemented(self): @@ -59,10 +55,7 @@ class NaNTest(unittest.TestCase): def test_annotation(self): annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs")) obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] - self.assertEqual( - annotations["col_idx"], - [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"] - ) + self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]) self.assertEqual(annotations["n_rows"], 100) self.assertTrue(math.isnan(annotations["columns"][2][0])) diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 73ed8de0..6fca92cc 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -18,15 +18,17 @@ Test the scanpy engine using the pbmc3k data set. """ -@parameterized_class(("data_locator", "backed"), [ - ("example-dataset/pbmc3k.h5ad", False), - ("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False), - ("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False), - - ("example-dataset/pbmc3k.h5ad", True), - ("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True), - ("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True), -]) +@parameterized_class( + ("data_locator", "backed"), + [ + ("../example-dataset/pbmc3k.h5ad", False), + ("test/test_datasets/pbmc3k-CSC-gz.h5ad", False), + ("test/test_datasets/pbmc3k-CSR-gz.h5ad", False), + ("../example-dataset/pbmc3k.h5ad", True), + ("test/test_datasets/pbmc3k-CSC-gz.h5ad", True), + ("test/test_datasets/pbmc3k-CSR-gz.h5ad", True), + ], +) class EngineTest(unittest.TestCase): def setUp(self): args = { @@ -36,7 +38,7 @@ class EngineTest(unittest.TestCase): "var_names": None, "diffexp_lfc_cutoff": 0.01, "layout_file": None, - "backed": self.backed + "backed": self.backed, } self.data = ScanpyEngine(DataLocator(self.data_locator), args) @@ -64,11 +66,7 @@ class EngineTest(unittest.TestCase): self.data._validate_data_types() def test_filter_idx(self): - filter_ = { - "filter": { - "var": {"index": [1, 99, [200, 300]]} - } - } + filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}} fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) @@ -76,14 +74,7 @@ class EngineTest(unittest.TestCase): def test_filter_complex(self): filter_ = { - "filter": { - "var": { - "annotation_value": [ - {"name": "n_cells", "min": 10} - ], - "index": [1, 99, [200, 300]] - } - } + "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}} } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) @@ -101,16 +92,14 @@ class EngineTest(unittest.TestCase): def test_schema_produces_error(self): self.data.data.obs["time"] = pd.Series( - list([time.time() for i in range(self.data.cell_count)]), - dtype="datetime64[ns]", + list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]", ) with pytest.raises(TypeError): self.data._create_schema() def test_config(self): self.assertEqual( - self.data.features["layout"]["obs"], - {"available": True, "interactiveLimit": 50000}, + self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000}, ) def test_layout(self): @@ -131,14 +120,13 @@ class EngineTest(unittest.TestCase): self.assertEqual(annotations["n_cols"], 5) obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] self.assertEqual( - annotations["col_idx"], - [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], + annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], ) fbs = self.data.annotation_to_fbs_matrix("var") annotations = decode_fbs.decode_matrix_FBS(fbs) - self.assertEqual(annotations['n_rows'], 1838) - self.assertEqual(annotations['n_cols'], 2) + self.assertEqual(annotations["n_rows"], 1838) + self.assertEqual(annotations["n_cols"], 2) var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"]) @@ -146,13 +134,13 @@ class EngineTest(unittest.TestCase): fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations["n_rows"], 2638) - self.assertEqual(annotations['n_cols'], 2) + self.assertEqual(annotations["n_cols"], 2) var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name]) annotations = decode_fbs.decode_matrix_FBS(fbs) - self.assertEqual(annotations['n_rows'], 1838) - self.assertEqual(annotations['n_cols'], 1) + self.assertEqual(annotations["n_rows"], 1838) + self.assertEqual(annotations["n_cols"], 1) def test_annotation_put(self): with self.assertRaises(DisabledFeatureError): @@ -177,27 +165,19 @@ class EngineTest(unittest.TestCase): self.data.data_frame_to_fbs_matrix(None, "obs") def test_filtered_data_frame(self): - filter_ = { - "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}} - } + filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}} fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_cols"], 1040) - filter_ = { - "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}} - } + filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}} with self.assertRaises(FilterError): self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") def test_data_named_gene(self): var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] - filter_ = { - "filter": { - "var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]} - } - } + filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}} fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) @@ -205,9 +185,7 @@ class EngineTest(unittest.TestCase): self.assertEqual(data["col_idx"], [4]) filter_ = { - "filter": { - "var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]} - } + "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}} } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) diff --git a/server/test/test_scanpy_engine_data_load.py b/server/test/test_scanpy_engine_data_load.py index 2be0b400..65d6522f 100644 --- a/server/test/test_scanpy_engine_data_load.py +++ b/server/test/test_scanpy_engine_data_load.py @@ -10,8 +10,9 @@ class DataLoadEngineTest(unittest.TestCase): """ Test file loading, including deferred loading/update. """ + def setUp(self): - self.data_file = DataLocator("example-dataset/pbmc3k.h5ad") + self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad") self.data = ScanpyEngine() def test_init(self): @@ -29,7 +30,7 @@ class DataLoadEngineTest(unittest.TestCase): "annotations_output_dir": None, "backed": False, "diffexp_may_be_slow": False, - "disable_diffexp": False + "disable_diffexp": False, } self.data.update(args=args) self.assertEqual(args, self.data.config) @@ -60,6 +61,7 @@ class DataLocatorEngineTest(unittest.TestCase): """ Test various types of data locators we expect to consume """ + def setUp(self): self.args = { "layout": ["umap"], @@ -76,7 +78,7 @@ class DataLocatorEngineTest(unittest.TestCase): self.assertEqual(data.gene_count, 1838) def test_posix_file(self): - locator = DataLocator("example-dataset/pbmc3k.h5ad") + locator = DataLocator("../example-dataset/pbmc3k.h5ad") data = ScanpyEngine(locator, self.args) self.stdAsserts(data) diff --git a/server/test/test_writable_annotation.py b/server/test/test_writable_annotation.py index 5df57f1c..c659e00a 100644 --- a/server/test/test_writable_annotation.py +++ b/server/test/test_writable_annotation.py @@ -25,9 +25,9 @@ class WritableAnnotationTest(unittest.TestCase): "diffexp_lfc_cutoff": 0.01, "annotations": True, "annotations_file": self.annotations_file, - "annotations_output_dir": None + "annotations_output_dir": None, } - self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args) + self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args) def tearDown(self): shutil.rmtree(self.tmpDir) @@ -40,9 +40,7 @@ class WritableAnnotationTest(unittest.TestCase): # verify that the expected errors are generated n_rows = self.data.data.obs.shape[0] - fbs_bad = self.make_fbs({ - 'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category') - }) + fbs_bad = self.make_fbs({"louvain": pd.Series(["undefined" for l in range(0, n_rows)], dtype="category")}) # ensure attempt to change VAR annotation with self.assertRaises(ValueError): @@ -55,32 +53,36 @@ class WritableAnnotationTest(unittest.TestCase): def test_write_to_file(self): # verify the file is written as expected n_rows = self.data.data.obs.shape[0] - fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), - 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') - }) + fbs = self.make_fbs( + { + "cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"), + "cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"), + } + ) res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.annotations_file)) - df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') + df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#") self.assertEqual(df.shape, (n_rows, 2)) - self.assertEqual(set(df.columns), set(['cat_A', 'cat_B'])) + self.assertEqual(set(df.columns), set(["cat_A", "cat_B"])) self.assertTrue(self.data.original_obs_index.equals(df.index)) - self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)])) - self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)])) + self.assertTrue(np.all(df["cat_A"] == ["label_A" for l in range(0, n_rows)])) + self.assertTrue(np.all(df["cat_B"] == ["label_B" for l in range(0, n_rows)])) # verify complete overwrite on second attempt, AND rotation occurs - fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'), - 'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category') - }) + fbs = self.make_fbs( + { + "cat_A": pd.Series(["label_A1" for l in range(0, n_rows)], dtype="category"), + "cat_C": pd.Series(["label_C" for l in range(0, n_rows)], dtype="category"), + } + ) res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.annotations_file)) - df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') - self.assertEqual(set(df.columns), set(['cat_A', 'cat_C'])) - self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)])) - self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)])) + df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#") + self.assertEqual(set(df.columns), set(["cat_A", "cat_C"])) + self.assertTrue(np.all(df["cat_A"] == ["label_A1" for l in range(0, n_rows)])) + self.assertTrue(np.all(df["cat_C"] == ["label_C" for l in range(0, n_rows)])) # rotation name, ext = path.splitext(self.annotations_file) @@ -92,10 +94,12 @@ class WritableAnnotationTest(unittest.TestCase): def test_file_rotation_to_max_9(self): # verify we stop rotation at 9 n_rows = self.data.data.obs.shape[0] - fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), - 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') - }) + fbs = self.make_fbs( + { + "cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"), + "cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"), + } + ) for i in range(0, 11): res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) @@ -111,10 +115,12 @@ class WritableAnnotationTest(unittest.TestCase): # GET (annotation_to_fbs_matrix) n_rows = self.data.data.obs.shape[0] - fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), - 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') - }) + fbs = self.make_fbs( + { + "cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"), + "cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"), + } + ) # put res = self.data.annotation_put_fbs("obs", fbs) @@ -128,28 +134,21 @@ class WritableAnnotationTest(unittest.TestCase): self.assertEqual(annotations["n_rows"], n_rows) self.assertEqual(annotations["n_cols"], 7) self.assertIsNone(annotations["row_idx"]) - self.assertEqual(annotations["col_idx"], [ - obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B" - ]) + self.assertEqual( + annotations["col_idx"], + [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"], + ) col_idx = annotations["col_idx"] - self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [ - 'label_A' for l in range(0, n_rows) - ]) - self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [ - 'label_B' for l in range(0, n_rows) - ]) + self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A" for l in range(0, n_rows)]) + self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B" for l in range(0, n_rows)]) # verify the schema was updated all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} - self.assertEqual(all_col_schema["cat_A"], { - "name": "cat_A", - "type": "categorical", - "categories": ["label_A"], - "writable": True - }) - self.assertEqual(all_col_schema["cat_B"], { - "name": "cat_B", - "type": "categorical", - "categories": ["label_B"], - "writable": True - }) + self.assertEqual( + all_col_schema["cat_A"], + {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True}, + ) + self.assertEqual( + all_col_schema["cat_B"], + {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True}, + ) diff --git a/travis-build.sh b/travis-build.sh deleted file mode 100755 index 9f043d0e..00000000 --- a/travis-build.sh +++ /dev/null @@ -1,5 +0,0 @@ -set -eo pipefail -flake8 server -npm run --prefix client/ build -npm run --prefix client/ unit-test -pytest -s server/test