diff --git a/.travis.yml b/.travis.yml index 20e7159c..1a05cc8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,23 +9,20 @@ cache: install: - set -eo pipefail - pip install flake8 - - make pydist - - make install-dist - - pip install -r server/requirements-dev.txt + - make pydist install-dist dev-env jobs: include: - name: "Branch Tests 3.7" python: "3.7" - script: ./travis-build.sh + script: make build-client lint unit-test - name: "Branch Tests 3.6" python: "3.6" - script: ./travis-build.sh + script: make build-client lint unit-test - name: "Docker Build" install: skip python: "3.6" script: docker build . - name: "Smoke Tests" python: "3.6" - script: - - npm run --prefix client/ smoke-test + script: make smoke-test diff --git a/makefile b/Makefile similarity index 67% rename from makefile rename to Makefile index 47be5ecd..e97af83a 100644 --- a/makefile +++ b/Makefile @@ -5,12 +5,32 @@ CLEANFILES := $(BUILDDIR)/ client/build build dist cellxgene.egg-info PART ?= patch + +# CLEANING +.PHONY: clean +clean: clean-lite clean-server clean-client + +# cleaning the client's node_modules is the longest one, so we avoid that if possible +.PHONY: clean-lite +clean-lite: + rm -rf $(CLEANFILES) + +clean-%: + cd $(*) && $(MAKE) clean + + # BUILDING PACKAGE -build : clean build-server +.PHONY: build +build: clean build-server @echo "done" -build-server : build-client +.PHONY: build-client +build-client: + cd client && $(MAKE) install build + +.PHONY: build-server +build-server: build-client mkdir -p $(SERVERBUILD) cp -r server/* $(SERVERBUILD) cp -r client/build/ $(CLIENTBUILD) @@ -22,12 +42,9 @@ build-server : build-client cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/ cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR) -build-client : - npm install --prefix client/ client - npm run --prefix client build - # If you are actively developing in the server folder use this, dirties the source tree -build-for-server-dev : clean-server build-client +.PHONY: build-for-server-dev +build-for-server-dev: clean-server build-client mkdir -p server/app/web/static/img mkdir -p server/app/web/static/js mkdir -p server/app/web/templates/ @@ -36,124 +53,160 @@ build-for-server-dev : clean-server build-client cp client/build/favicon.png server/app/web/static/img cp client/build/service-worker.js server/app/web/static/js/ -clean : clean-lite clean-server - rm -rf client/node_modules -# cleaning node_modules is the longest one, so we avoid that if possible -clean-lite : - rm -rf $(CLEANFILES) +# TESTING +.PHONY: test +test: unit-test smoke-test -clean-server : - rm -f server/app/web/templates/index.html - rm -rf server/app/web/static +.PHONY: unit-test +unit-test: unit-test-server unit-test-client + +unit-test-%: + cd $(*) && $(MAKE) unit-test + +.PHONY: smoke-test +smoke-test: + cd client && $(MAKE) smoke-test + +# FORMATTING CODE + +.PHOHY: fmt +fmt: fmt-client fmt-py + +fmt-client: + cd client && $(MAKE) fmt + +fmt-py: + black . + +.PHONY: lint +lint: + flake8 server -.PHONY : build build-server build-client build-for-server-dev clean clean-lite clean-server # CREATING DISTRIBUTION RELEASE -pydist : build +.PHONY: pydist +pydist: build cd $(BUILDDIR); python setup.py sdist -d ../dist @echo "done" -.PHONY : pydist # RELEASE HELPERS # create new version to commit to master -release-stage-1 : dev-env bump clean-lite gen-package-lock +.PHONY: release-stage-1 +release-stage-1: dev-env bump clean-lite gen-package-lock @echo "Version bumped part:$(PART) and client built. Ready to commit and push" # build dist and release to dev pypi -release-stage-2 : dev-env pydist twine +.PHONY: release-stage-2 +release-stage-2: dev-env pydist twine @echo "Dist built and uploaded to test.pypi.org" @echo "Test the install:" @echo " make install-release-test" @echo "Then upload to Pypi prod:" @echo " make twine-prod" +.PHONY: release-stage-final release-stage-final: twine-prod @echo "Release uploaded to pypi.org" # DANGER: releases directly to prod # use this if you accidently burned a test release version number, -release-directly-to-prod : dev-env pydist twine-prod +.PHONY: release-directly-to-prod +release-directly-to-prod: dev-env pydist twine-prod @echo "Dist built and uploaded to pypi.org" @echo "Test the install:" @echo " make install-release" -dev-env : +.PHONY: dev-env +dev-env: + cd client && $(MAKE) install pip install -r server/requirements-dev.txt -gui-env : dev-env +.PHONY: gui-env +gui-env: dev-env pip install -r server/requirements-gui.txt # give PART=[major, minor, part] as param to make bump -bump : +.PHONY: bump +bump: bumpversion --config-file .bumpversion.cfg $(PART) -twine : +.PHONY: twine +twine: twine upload --repository-url https://test.pypi.org/legacy/ dist/* -twine-prod : +.PHONY: twine-prod +twine-prod: twine upload dist/* # quicker than re-building client -gen-package-lock : - npm install --prefix client/ client +.PHONY: gen-package-lock +gen-package-lock: + cd client && $(MAKE) install -.PHONY : release-stage-1 release-stage-2 release-stage-final release-burned dev-env bump twine twine-prod gen-package-lock # INSTALL # setup.py sucks when you have your library in a separate folder, adding these in to help setup envs # install from build directory -install : uninstall +.PHONY: install +install: uninstall cd $(BUILDDIR); pip install -e . # install from source tree for development -install-dev : uninstall +.PHONY: install-dev +install-dev: uninstall pip install -e . # install from test.pypi to test your release -install-release-test : uninstall +.PHONY: install-release-test +install-release-test: uninstall pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene @echo "Installed cellxgene from test.pypi.org, now run and smoke test" # install from pypi to test your release -install-release : uninstall +.PHONY: install-release +install-release: uninstall pip install --no-cache-dir cellxgene @echo "Installed cellxgene from pypi.org" # install from dist -install-dist : uninstall +.PHONY: install-dist +install-dist: uninstall pip install dist/cellxgene*.tar.gz -uninstall : +.PHONY: uninstall +uninstall: pip uninstall -y cellxgene || : -.PHONY : install install-dev install-release-test install-release uninstall # GUI -build-assets : +.PHONY: build-assets +build-assets: pyside2-rcc server/gui/cellxgene.qrc -o server/gui/cellxgene_rc.py -gui-spec-osx : clean-lite gui-env +.PHONY: gui-spec-osx +gui-spec-osx: clean-lite gui-env pip install -e .[gui] pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-binary='/System/Library/Frameworks/Tk.framework/Tk':'tk' --add-binary='/System/Library/Frameworks/Tcl.framework/Tcl':'tcl' --add-data server/app/web/templates/:server/app/web/templates/ --add-data server/app/web/static/:server/app/web/static/ --icon server/gui/images/cxg_icons.icns server/gui/main.py mv cellxgene.spec cellxgene-osx.spec -gui-spec-windows : clean-lite dev-env +.PHONY: gui-spec-windows +gui-spec-windows: clean-lite dev-env pip install -e .[gui] pyi-makespec -D -w --additional-hooks-dir server/gui/ -n cellxgene --add-data server/app/web/templates;server/app/web/templates --add-data server/app/web/static;server/app/web/static --icon server/gui/images/icon.ico server/gui/main.py mv cellxgene.spec cellxgene-windows.spec -gui-build-osx : clean-lite +.PHONY: gui-build-osx +gui-build-osx: clean-lite pyinstaller --clean cellxgene-osx.spec -gui-build-windows : clean-lite +.PHONY: gui-build-windows +gui-build-windows: clean-lite pyinstaller --clean cellxgene-windows.spec -.PHONY : build-assets gui-spec-osx gui-spec-windows gui-build-osx gui-build-windows - diff --git a/client/Makefile b/client/Makefile new file mode 100644 index 00000000..47825cee --- /dev/null +++ b/client/Makefile @@ -0,0 +1,16 @@ +.PHONY: clean +clean: + rm -rf node_modules + +.PHONY: install +install: + npm install client + +.PHONY: build +build: + npm run build + +# pass remaining commands through to npm run +%: + npm run $(*) + diff --git a/client/package.json b/client/package.json index bb1e4f71..b6f60d7b 100644 --- a/client/package.json +++ b/client/package.json @@ -11,6 +11,7 @@ "clean": "rimraf build", "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js", "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js", + "fmt": "eslint --fix src", "lint": "eslint src", "smoke-test": "start-server-and-test start-server-for-test :5000 e2e", "start": "node server/development.js", diff --git a/dev_docs/developer_guidelines.md b/dev_docs/developer_guidelines.md index c9ce3a8c..d2951f05 100644 --- a/dev_docs/developer_guidelines.md +++ b/dev_docs/developer_guidelines.md @@ -9,6 +9,38 @@ **All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.** +## Running test suite +Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. + +### Unit tests +Steps to run the all unit tests: +1. Start in the project root directory +1. `make dev-env` +1. `make unit-test` + +### End to end tests + +End to end tests use two env variables: +* `JEST_ENV` - environment to run end to end tests. Default `dev` + * `prod` - run headless with no slowdown, chromium will not open. + * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit. + * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit. +* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port). + +On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` make target. + +To run end to end tests as they will be run on CI +1. cellxgene should be built and installed as [specified in server dev](#install) +2. `export JEST_ENV='prod'` +3. `export JEST_CXG_PORT=5000` +4. Run `npm run --prefix client/ smoke-test` + +Run end to end tests interactively during development +1. cellxgene should be installed as [specified in client dev](#install-1) +2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k` +3. Run `make smoke-test` +4. To debug a failing test `export JEST_ENV='debug'` and re-run. + ## Server dev ### Install * Build the client and put static files in place: `make build-for-server-dev` @@ -21,11 +53,15 @@ If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded. ### Linter -We use `flake8` to lint code. Travis CI runs `flake8 server`. + +We use [`flake8`](https://github.com/PyCQA/flake8) to lint python and [`black`](https://pypi.org/project/black/) for auto-formatting. + +To auto-format code run `make fmt`. To run lint checks on the code run `make lint`. ### Test -1. Install development requirements `pip install -r server/requirements-dev.txt` -2. Run tests `pytest server/test` +If you would like to run the server tests individually, follow the steps below +1. Install development requirements `make dev-env` +1. Run `make unit-test` in the `server` directory or `make unit-test-server` in the root directory. ### Tips * Install in a virtualenv @@ -33,8 +69,8 @@ We use `flake8` to lint code. Travis CI runs `flake8 server`. ## Client dev ### Install -1. Install prereqs for client: `npm install --prefix client/ client` -2. Install cellxgene server: `pip install -e .` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets. +1. Install prereqs for client: `make dev-env` +2. Install cellxgene server: `make install-dev` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets. ### Launch To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made. @@ -49,43 +85,12 @@ To build only the client: `make build-client` We use `eslint` to lint the code and `prettier` as our code formatter. ### Test -In `client/` directory run `npm run unit-test` + +If you would like to run the client tests individually, follow the steps below in the `client` directory +1. For unit tests run `npm run unit-test` or `make unit-test` +1. For the smoke test run `npm run smoke-test` or `make smoke-test` ### Tips * You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) in `client/` directory run `npm run backend-dev` -## Running tests -Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. - -### Server unit tests -Install development requirements `pip install -r server/requirements-dev.txt` -Run tests `pytest server/test` - -### Client unit tests -In `client/` directory run `npm run unit-test` - -### End to end tests - -End to end tests use two env variables: -* `JEST_ENV` - environment to run end to end tests. Default `dev` - * `prod` - run headless with no slowdown, chromium will not open. - * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit. - * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit. -* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port). - -On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` npm script - -To run end to end tests as they will be run on CI -1. cellxgene should be built and installed as [specified in server dev](#install) -2. `export JEST_ENV='prod'` -3. `export JEST_CXG_PORT='5000'` -4. Run `npm run --prefix client/ smoke-test` - -Run end to end tests interactively during development -1. cellxgene should be installed as [specified in client dev](#install-1) -2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k` -3. Run `npm run --prefix client/ e2e` -4. To debug a failing test `export JEST_ENV='debug'` and re-run. - - diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..a30d4e82 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[tool.black] +line-length = 120 +target_version = ['py37'] +include = '\.pyi?$' +exclude = ''' + +( + /( + \.eggs # exclude a few common directories in the + | \.git # root of the project + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | venv + | _build + | buck-out + | build + | dist + | server/app/util/fbs/NetEncoding + )/ + | server/gui/cellxgene_rc.py + +) +''' diff --git a/server/Makefile b/server/Makefile new file mode 100644 index 00000000..271f48c4 --- /dev/null +++ b/server/Makefile @@ -0,0 +1,8 @@ +.PHONY: clean +clean: + rm -f app/web/templates/index.html + rm -rf app/web/static + +.PHONY: unit-test +unit-test: + pytest -s test diff --git a/server/__main__.py b/server/__main__.py index 1f0b8730..b582e892 100644 --- a/server/__main__.py +++ b/server/__main__.py @@ -5,11 +5,11 @@ if __package__ is None: PKG_PATH = Path(__file__).parent sys.path.insert(0, str(PKG_PATH.parent)) - import server # noqa F401 + import server # noqa F401 __package__ = PKG_PATH.name # Main thing -from .cli.cli import cli # noqa F402 +from .cli.cli import cli # noqa F402 cli() diff --git a/server/app/driver/driver.py b/server/app/driver/driver.py index 30aa8f06..38bdf484 100644 --- a/server/app/driver/driver.py +++ b/server/app/driver/driver.py @@ -33,7 +33,7 @@ class CXGDriver(metaclass=ABCMeta): "max_category_items": None, "diffexp_lfc_cutoff": None, "disable_diffexp": False, - "diffexp_may_be_slow": False + "diffexp_may_be_slow": False, } @abstractmethod @@ -51,7 +51,7 @@ class CXGDriver(metaclass=ABCMeta): features = { "cluster": {"available": False}, "layout": {"obs": {"available": False}, "var": {"available": False}}, - "diffexp": {"available": True, "interactiveLimit": 50000} + "diffexp": {"available": True, "interactiveLimit": 50000}, } # TODO - Interactive limit should be generated from the actual available methods see GH issue #94 if self.config["layout"]: diff --git a/server/app/rest_api/rest.py b/server/app/rest_api/rest.py index 780dd38f..32342e18 100644 --- a/server/app/rest_api/rest.py +++ b/server/app/rest_api/rest.py @@ -8,13 +8,7 @@ from flask_restful import Api, Resource from server import __version__ as cellxgene_version from anndata import __version__ as anndata_version -from server.app.util.constants import ( - Axis, - DiffExpMode, - JSON_NaN_to_num_warning_msg, - CXGUID, - CXG_ANNO_COLLECTION -) +from server.app.util.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg, CXGUID, CXG_ANNO_COLLECTION from server.app.util.errors import ( FilterError, InteractiveError, @@ -40,41 +34,18 @@ class ConfigAPI(Resource): config = { "config": { "features": [ - { - "method": "POST", - "path": "/cluster/", - **current_app.data.features["cluster"], - }, - { - "method": "POST", - "path": "/layout/obs", - **current_app.data.features["layout"]["obs"], - }, - { - "method": "POST", - "path": "/layout/var", - **current_app.data.features["layout"]["var"], - }, - { - "method": "POST", - "path": "/diffexp/", - **current_app.data.features["diffexp"], - }, + {"method": "POST", "path": "/cluster/", **current_app.data.features["cluster"]}, + {"method": "POST", "path": "/layout/obs", **current_app.data.features["layout"]["obs"]}, + {"method": "POST", "path": "/layout/var", **current_app.data.features["layout"]["var"]}, + {"method": "POST", "path": "/diffexp/", **current_app.data.features["diffexp"]}, ], "displayNames": { "engine": f"cellxgene Scanpy engine version ", "dataset": current_app.config["DATASET_TITLE"], }, - "links": { - "about-dataset": current_app.config["ABOUT_DATASET"] - }, - "parameters": { - **current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection) - }, - "library_versions": { - "cellxgene": cellxgene_version, - "anndata": anndata_version - } + "links": {"about-dataset": current_app.config["ABOUT_DATASET"]}, + "parameters": {**current_app.data.get_config_parameters(uid=cxguid, collection=anno_collection)}, + "library_versions": {"cellxgene": cellxgene_version, "anndata": anndata_version}, } } @@ -84,17 +55,13 @@ class ConfigAPI(Resource): class AnnotationsObsAPI(Resource): def get(self): fields = request.args.getlist("annotation-name", None) - preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) cxguid = get_userid(session) anno_collection = get_anno_collection(session) try: if preferred_mimetype == "application/octet-stream": fbs = current_app.data.annotation_to_fbs_matrix("obs", fields, uid=cxguid, collection=anno_collection) - return make_response(fbs, - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + return make_response(fbs, HTTPStatus.OK, {"Content-Type": "application/octet-stream"}) else: return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except KeyError: @@ -115,9 +82,7 @@ class AnnotationsObsAPI(Resource): try: fbs = request.get_data() res = current_app.data.annotation_put_fbs("obs", fbs, uid=cxguid, collection=anno_collection) - return make_response( - res, HTTPStatus.OK, {"Content-Type": "application/json"} - ) + return make_response(res, HTTPStatus.OK, {"Content-Type": "application/json"}) except (ValueError, DisabledFeatureError, KeyError) as e: return make_response(str(e), HTTPStatus.BAD_REQUEST) except Exception as e: @@ -127,14 +92,14 @@ class AnnotationsObsAPI(Resource): class AnnotationsVarAPI(Resource): def get(self): fields = request.args.getlist("annotation-name", None) - preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) try: if preferred_mimetype == "application/octet-stream": - return make_response(current_app.data.annotation_to_fbs_matrix("var", fields), - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + return make_response( + current_app.data.annotation_to_fbs_matrix("var", fields), + HTTPStatus.OK, + {"Content-Type": "application/octet-stream"}, + ) else: return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except KeyError: @@ -145,19 +110,16 @@ class AnnotationsVarAPI(Resource): class DataVarAPI(Resource): def put(self): - preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) try: if preferred_mimetype == "application/octet-stream": filter_json = request.get_json() filter = filter_json["filter"] if filter_json else None return make_response( - current_app.data.data_frame_to_fbs_matrix( - filter, axis=Axis.VAR - ), + current_app.data.data_frame_to_fbs_matrix(filter, axis=Axis.VAR), HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + {"Content-Type": "application/octet-stream"}, + ) else: return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except FilterError as e: @@ -175,35 +137,23 @@ class DiffExpObsAPI(Resource): except KeyError: return make_response("Error: mode is required", HTTPStatus.BAD_REQUEST) except ValueError: - return make_response( - f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST - ) + return make_response(f"Error: invalid mode option {args['mode']}", HTTPStatus.BAD_REQUEST) # Validate filters if mode == DiffExpMode.VAR_FILTER or "varFilter" in args: # not NOT_IMPLEMENTED - return make_response( - "mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED - ) + return make_response("mode=varfilter not implemented", HTTPStatus.NOT_IMPLEMENTED) if mode == DiffExpMode.TOP_N and "count" not in args: - return make_response( - "mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST - ) + return make_response("mode=topN requires a count parameter", HTTPStatus.BAD_REQUEST) if "set1" not in args: return make_response("set1 is required.", HTTPStatus.BAD_REQUEST) if Axis.VAR in args["set1"]["filter"]: - return make_response( - "Var filter not allowed for set1", HTTPStatus.BAD_REQUEST - ) + return make_response("Var filter not allowed for set1", HTTPStatus.BAD_REQUEST) # set2 if "set2" not in args: - return make_response( - "Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED - ) + return make_response("Set2 as inverse of set1 is not implemented", HTTPStatus.NOT_IMPLEMENTED) if Axis.VAR in args["set2"]["filter"]: - return make_response( - "Var filter not allowed for set2", HTTPStatus.BAD_REQUEST - ) + return make_response("Var filter not allowed for set2", HTTPStatus.BAD_REQUEST) set1_filter = args["set1"]["filter"] set2_filter = args.get("set2", {"filter": {}})["filter"] @@ -214,14 +164,9 @@ class DiffExpObsAPI(Resource): count = args.get("count", None) try: diffexp = current_app.data.diffexp_topN( - set1_filter, - set2_filter, - count, - current_app.data.features["diffexp"]["interactiveLimit"], - ) - return make_response( - diffexp, HTTPStatus.OK, {"Content-Type": "application/json"} + set1_filter, set2_filter, count, current_app.data.features["diffexp"]["interactiveLimit"], ) + return make_response(diffexp, HTTPStatus.OK, {"Content-Type": "application/json"}) except (ValueError, FilterError) as e: return make_response(e.message, HTTPStatus.BAD_REQUEST) except InteractiveError: @@ -236,14 +181,12 @@ class DiffExpObsAPI(Resource): class LayoutObsAPI(Resource): def get(self): - preferred_mimetype = request.accept_mimetypes.best_match( - ["application/octet-stream"] - ) + preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) try: if preferred_mimetype == "application/octet-stream": - return make_response(current_app.data.layout_to_fbs_matrix(), - HTTPStatus.OK, - {"Content-Type": "application/octet-stream"}) + return make_response( + current_app.data.layout_to_fbs_matrix(), HTTPStatus.OK, {"Content-Type": "application/octet-stream"} + ) else: return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) except PrepareError as e: @@ -278,7 +221,7 @@ def is_safe_collection_name(name): """ if name is None: return False - return re.match(r'^\w+$', name) is not None + return re.match(r"^\w+$", name) is not None def get_api_resources(): diff --git a/server/app/scanpy_engine/diffexp.py b/server/app/scanpy_engine/diffexp.py index d991f1c6..af267636 100644 --- a/server/app/scanpy_engine/diffexp.py +++ b/server/app/scanpy_engine/diffexp.py @@ -32,8 +32,8 @@ def _mean_var_n(X): v = sumsq / (n - 1) if fp_err_occurred: - mean[np.isfinite(mean) == False] = 0 # noqa: E712 - v[np.isfinite(v) == False] = 0 # noqa: E712 + mean[np.isfinite(mean) == False] = 0 # noqa: E712 + v[np.isfinite(v) == False] = 0 # noqa: E712 return mean, v, n diff --git a/server/app/scanpy_engine/labels.py b/server/app/scanpy_engine/labels.py index 3a5f7d08..1cdaeb42 100644 --- a/server/app/scanpy_engine/labels.py +++ b/server/app/scanpy_engine/labels.py @@ -9,7 +9,7 @@ import pandas as pd def read_labels(fname): if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: - return pd.read_csv(fname, dtype='category', index_col=0, header=0, comment='#') + return pd.read_csv(fname, dtype="category", index_col=0, header=0, comment="#") else: return pd.DataFrame() @@ -19,12 +19,12 @@ def write_labels(fname, df, header=None, backup_dir=None): backup(fname, backup_dir) # rotate_fname(fname, backup_dir) if not df.empty: - with open(fname, 'w', newline="") as f: + with open(fname, "w", newline="") as f: if header is not None: f.write(header) df.to_csv(f) else: - open(fname, 'w').close() + open(fname, "w").close() def backup(fname, backup_dir, max_backups=9): @@ -46,7 +46,7 @@ def backup(fname, backup_dir, max_backups=9): fname_base = os.path.basename(fname) fname_base_root, fname_base_ext = os.path.splitext(fname_base) # don't use ISO standard time format, as it contains characters illegal on some filesytems. - nowish = datetime.now().strftime('%Y-%m-%dT%H-%M-%S') + nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S") backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}") if os.path.exists(backup_fname): os.remove(backup_fname) diff --git a/server/app/scanpy_engine/matrix_proxy.py b/server/app/scanpy_engine/matrix_proxy.py index 137c676b..29e0e4fe 100644 --- a/server/app/scanpy_engine/matrix_proxy.py +++ b/server/app/scanpy_engine/matrix_proxy.py @@ -1,4 +1,3 @@ - from server.app.util.matrix_proxy import MatrixProxyView, ArrayProxyView """ @@ -17,6 +16,7 @@ class ArrayProxyView_anndata_h5py(ArrayProxyView): override to handle sparse getitem semantics, which differ from numpy. """ + def toarray(self): """ sadly, sparse indexing doesn't drop dimensions like numpy! """ arr = self.m[self._index[0], self._index[1]] @@ -30,12 +30,15 @@ class MatrixProxy_anndata_h5py(MatrixProxyView): AnnData sparse array stored in H5AD, or proxies for backed data. None of these handle indexing very well, so we plop a proxy on top. """ + @classmethod def __supports__(cls): - return ("anndata.h5py.h5sparse.SparseDataset", - "anndata.h5py.h5sparse.backed_csc_matrix", - "anndata.h5py.h5sparse.backed_csr_matrix", - "h5py._hl.dataset.Dataset") + return ( + "anndata.h5py.h5sparse.SparseDataset", + "anndata.h5py.h5sparse.backed_csc_matrix", + "anndata.h5py.h5sparse.backed_csr_matrix", + "h5py._hl.dataset.Dataset", + ) @classmethod def create_array(cls, *args, **kwargs): diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index a1a2b32a..a77bb48c 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -62,7 +62,7 @@ class ScanpyEngine(CXGDriver): "annotations_output_dir": None, "backed": False, "disable_diffexp": False, - "diffexp_may_be_slow": False + "diffexp_may_be_slow": False, } def get_config_parameters(self, uid=None, collection=None): @@ -70,26 +70,25 @@ class ScanpyEngine(CXGDriver): "max-category-items": self.config["max_category_items"], "disable-diffexp": self.config["disable_diffexp"], "diffexp-may-be-slow": self.config["diffexp_may_be_slow"], - "annotations": self.config["annotations"] + "annotations": self.config["annotations"], } if self.config["annotations"]: if uid is not None: - params.update({ - "annotations-user-data-idhash": self.get_userdata_idhash(uid) - }) - if self.config['annotations_file'] is not None: + params.update({"annotations-user-data-idhash": self.get_userdata_idhash(uid)}) + if self.config["annotations_file"] is not None: # user has hard-wired the name of the annotation data collection - fname = os.path.basename(self.config['annotations_file']) + fname = os.path.basename(self.config["annotations_file"]) collection_fname = os.path.splitext(fname)[0] - params.update({ - 'annotations-data-collection-is-read-only': True, - 'annotations-data-collection-name': collection_fname - }) + params.update( + { + "annotations-data-collection-is-read-only": True, + "annotations-data-collection-name": collection_fname, + } + ) elif collection is not None: - params.update({ - 'annotations-data-collection-is-read-only': False, - 'annotations-data-collection-name': collection - }) + params.update( + {"annotations-data-collection-is-read-only": False, "annotations-data-collection-name": collection} + ) return params @staticmethod @@ -140,23 +139,18 @@ class ScanpyEngine(CXGDriver): # User has specified alternative column for unique names, and it exists if not df_axis[name].is_unique: raise KeyError( - f"Values in {ax_name}.{name} must be unique. " - "Please prepare data to contain unique values." + f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values." ) df_axis.reset_index(drop=True, inplace=True) else: # user specified a non-existent column name - raise KeyError( - f"Annotation name {name}, specified in --{ax_name}-name does not exist." - ) + raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.") @staticmethod def _can_cast_to_float32(ann): if ann.dtype.kind == "f": if not np.can_cast(ann.dtype, np.float32): - warnings.warn( - f"Annotation {ann.name} will be converted to 32 bit float and may lose precision." - ) + warnings.warn(f"Annotation {ann.name} will be converted to 32 bit float and may lose precision.") return True return False @@ -188,30 +182,18 @@ class ScanpyEngine(CXGDriver): schema["type"] = "categorical" schema["categories"] = dtype.categories.tolist() else: - raise TypeError( - f"Annotations of type {dtype} are unsupported by cellxgene." - ) + raise TypeError(f"Annotations of type {dtype} are unsupported by cellxgene.") return schema @requires_data def _create_schema(self): self.schema = { - "dataframe": { - "nObs": self.cell_count, - "nVar": self.gene_count, - "type": str(self.data.X.dtype), - }, + "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)}, "annotations": { - "obs": { - "index": self.config["obs_names"], - "columns": [] - }, - "var": { - "index": self.config["var_names"], - "columns": [] - } + "obs": {"index": self.config["obs_names"], "columns": []}, + "var": {"index": self.config["var_names"], "columns": []}, }, - "layout": {"obs": []} + "layout": {"obs": []}, } for ax in Axis: curr_axis = getattr(self.data, str(ax)) @@ -220,12 +202,8 @@ class ScanpyEngine(CXGDriver): ann_schema.update(self._get_col_type(curr_axis[ann])) self.schema["annotations"][ax]["columns"].append(ann_schema) - for layout in self.config['layout']: - layout_schema = { - "name": layout, - "type": "float32", - "dims": [f"{layout}_0", f"{layout}_1"] - } + for layout in self.config["layout"]: + layout_schema = {"name": layout, "type": "float32", "dims": [f"{layout}_0", f"{layout}_1"]} self.schema["layout"]["obs"].append(layout_schema) @requires_data @@ -250,7 +228,7 @@ class ScanpyEngine(CXGDriver): Used to create safe annotations output file names. """ id = (uid + self.data_locator.abspath()).encode() - idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode('utf-8') + idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8") return idhash def get_anno_fname(self, uid=None, collection=None): @@ -272,11 +250,11 @@ class ScanpyEngine(CXGDriver): if not self.config["annotations"]: return None - if self.config['annotations_output_dir']: - return self.config['annotations_output_dir'] + if self.config["annotations_output_dir"]: + return self.config["annotations_output_dir"] - if self.config['annotations_file']: - return os.path.dirname(os.path.abspath(self.config['annotations_file'])) + if self.config["annotations_file"]: + return os.path.dirname(os.path.abspath(self.config["annotations_file"])) return os.getcwd() @@ -299,7 +277,7 @@ class ScanpyEngine(CXGDriver): with data_locator.local_handle() as lh: # as of AnnData 0.6.19, backed mode performs initial load fast, but at the # cost of significantly slower access to X data. - backed = 'r' if self.config['backed'] else None + backed = "r" if self.config["backed"] else None self.data = anndata.read_h5ad(lh, backed=backed) except ValueError: @@ -338,7 +316,7 @@ class ScanpyEngine(CXGDriver): # heuristic n_values = self.data.shape[0] * self.data.shape[1] - if (n_values > 1e8 and self.config['backed'] is True) or (n_values > 5e8): + if (n_values > 1e8 and self.config["backed"] is True) or (n_values > 5e8): self.config.update({"diffexp_may_be_slow": True}) @requires_data @@ -348,7 +326,7 @@ class ScanpyEngine(CXGDriver): b) validate layouts are legal. remove/warn on any that are not c) cap total list of layouts at global const MAX_LAYOUTS """ - layouts = self.config['layout'] + layouts = self.config["layout"] # handle default if layouts is None or len(layouts) == 0: # load default layouts from the data. @@ -372,7 +350,7 @@ class ScanpyEngine(CXGDriver): raise PrepareError(f"No valid layout data.") # cap layouts to MAX_LAYOUTS - self.config['layout'] = valid_layouts[0:MAX_LAYOUTS] + self.config["layout"] = valid_layouts[0:MAX_LAYOUTS] @requires_data def _is_valid_layout(self, arr): @@ -394,8 +372,7 @@ class ScanpyEngine(CXGDriver): ) if self.data.X.dtype != "float32": warnings.warn( - f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " - f"Precision may be truncated." + f"Scanpy data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated." ) for ax in Axis: curr_axis = getattr(self.data, str(ax)) @@ -414,7 +391,7 @@ class ScanpyEngine(CXGDriver): ) if isinstance(datatype, CategoricalDtype): category_num = len(curr_axis[ann].dtype.categories) - if category_num > 500 and category_num > self.config['max_category_items']: + if category_num > 500 and category_num > self.config["max_category_items"]: warnings.warn( f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be " f"cumbersome or slow to display. We recommend setting the " @@ -439,14 +416,17 @@ class ScanpyEngine(CXGDriver): raise KeyError(f"All row index values specified in user annotations must be unique.") if not labels.index.equals(self.original_obs_index): - raise KeyError("Label file row index does not match H5AD file index. " - "Please ensure that column zero (0) in the label file contain the same " - "index values as the H5AD file.") + raise KeyError( + "Label file row index does not match H5AD file index. " + "Please ensure that column zero (0) in the label file contain the same " + "index values as the H5AD file." + ) duplicate_columns = list(set(labels.columns) & set(self.data.obs.columns)) if len(duplicate_columns) > 0: - raise KeyError(f"Labels file may not contain column names which overlap " - f"with h5ad obs columns {duplicate_columns}") + raise KeyError( + f"Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}" + ) # labels must have same count as obs annotations if labels.shape[0] != self.data.obs.shape[0]: @@ -475,7 +455,7 @@ class ScanpyEngine(CXGDriver): mask = np.zeros((count,), dtype=bool) for i in filter: if type(i) == list: - mask[i[0]: i[1]] = True + mask[i[0] : i[1]] = True else: mask[i] = True return mask @@ -484,15 +464,10 @@ class ScanpyEngine(CXGDriver): def _axis_filter_to_mask(filter, d_axis, count): mask = np.ones((count,), dtype=bool) if "index" in filter: - mask = np.logical_and( - mask, ScanpyEngine._index_filter_to_mask(filter["index"], count) - ) + mask = np.logical_and(mask, ScanpyEngine._index_filter_to_mask(filter["index"], count)) if "annotation_value" in filter: mask = np.logical_and( - mask, - ScanpyEngine._annotation_filter_to_mask( - filter["annotation_value"], d_axis, count - ), + mask, ScanpyEngine._annotation_filter_to_mask(filter["annotation_value"], d_axis, count), ) return mask @@ -507,13 +482,9 @@ class ScanpyEngine(CXGDriver): if filter is not None: if Axis.OBS in filter: - obs_selector = self._axis_filter_to_mask( - filter["obs"], self.data.obs, self.data.n_obs - ) + obs_selector = self._axis_filter_to_mask(filter["obs"], self.data.obs, self.data.n_obs) if Axis.VAR in filter: - var_selector = self._axis_filter_to_mask( - filter["var"], self.data.var, self.data.n_vars - ) + var_selector = self._axis_filter_to_mask(filter["var"], self.data.var, self.data.n_vars) return obs_selector, var_selector @requires_data @@ -531,7 +502,7 @@ class ScanpyEngine(CXGDriver): labels = None if labels is not None and not labels.empty: - df = self.data.obs.join(labels, self.config['obs_names']) + df = self.data.obs.join(labels, self.config["obs_names"]) else: df = self.data.obs else: @@ -560,18 +531,21 @@ class ScanpyEngine(CXGDriver): # if any of the new column labels overlap with our existing labels, raise error duplicate_columns = list(set(new_label_df.columns) & set(self.data.obs.columns)) if not new_label_df.columns.is_unique or len(duplicate_columns) > 0: - raise KeyError(f"Labels file may not contain column names which overlap " - f"with h5ad obs columns {duplicate_columns}") + raise KeyError( + f"Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}" + ) # update our internal state and save it. Multi-threading often enabled, # so treat this as a critical section. with self.label_lock: lastmod = self.data_locator.lastmodtime() lastmodstr = "'unknown'" if lastmod is None else lastmod.isoformat(timespec="seconds") - header = f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " \ - f"using cellxgene version {cellxgene_version}\n" \ - f"# Input data file was {self.data_locator.uri_or_path}, " \ - f"which was last modified on {lastmodstr}\n" + header = ( + f"# Annotations generated on {datetime.now().isoformat(timespec='seconds')} " + f"using cellxgene version {cellxgene_version}\n" + f"# Input data file was {self.data_locator.uri_or_path}, " + f"which was last modified on {lastmodstr}\n" + ) write_labels(fname, new_label_df, header, backup_dir=self.get_anno_backup_dir(uid, collection)) return jsonify_scanpy({"status": "OK"}) @@ -598,8 +572,7 @@ class ScanpyEngine(CXGDriver): raise FilterError("filtering on obs unsupported") # Currently only handles VAR dimension - X = MatrixProxy.create(self.data.X if var_selector is None - else self.data.X[:, var_selector]) + X = MatrixProxy.create(self.data.X if var_selector is None else self.data.X[:, var_selector]) return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None) @requires_data @@ -607,25 +580,17 @@ class ScanpyEngine(CXGDriver): if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB: raise FilterError("Observation filters may not contain vaiable conditions") try: - obs_mask_A = self._axis_filter_to_mask( - obsFilterA["obs"], self.data.obs, self.data.n_obs - ) - obs_mask_B = self._axis_filter_to_mask( - obsFilterB["obs"], self.data.obs, self.data.n_obs - ) + obs_mask_A = self._axis_filter_to_mask(obsFilterA["obs"], self.data.obs, self.data.n_obs) + obs_mask_B = self._axis_filter_to_mask(obsFilterB["obs"], self.data.obs, self.data.n_obs) except (KeyError, IndexError) as e: raise FilterError(f"Error parsing filter: {e}") from e if top_n is None: top_n = DEFAULT_TOP_N - result = diffexp_ttest( - self.data, obs_mask_A, obs_mask_B, top_n, self.config['diffexp_lfc_cutoff'] - ) + result = diffexp_ttest(self.data, obs_mask_A, obs_mask_B, top_n, self.config["diffexp_lfc_cutoff"]) try: return jsonify_scanpy(result) except ValueError: - raise JSONEncodingValueError( - "Error encoding differential expression to JSON" - ) + raise JSONEncodingValueError("Error encoding differential expression to JSON") @requires_data def layout_to_fbs_matrix(self): @@ -661,7 +626,8 @@ class ScanpyEngine(CXGDriver): except ValueError as e: raise PrepareError( f"Layout has not been calculated using {self.config['layout']}, " - f"please prepare your datafile and relaunch cellxgene") from e + f"please prepare your datafile and relaunch cellxgene" + ) from e df = pandas.concat(layout_data, axis=1, copy=False) return encode_matrix_fbs(df, col_idx=df.columns, row_idx=None) diff --git a/server/app/util/constants.py b/server/app/util/constants.py index b53d476c..545a8d50 100644 --- a/server/app/util/constants.py +++ b/server/app/util/constants.py @@ -27,9 +27,7 @@ class DiffExpMode(AugmentedEnum): VAR_FILTER = "varFilter" -JSON_NaN_to_num_warning_msg = ( - "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)" -) +JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)" REACTIVE_LIMIT = 1_000_000 MAX_LAYOUTS = 30 diff --git a/server/app/util/data_locator.py b/server/app/util/data_locator.py index 8a44f06d..2b7a8150 100644 --- a/server/app/util/data_locator.py +++ b/server/app/util/data_locator.py @@ -4,7 +4,7 @@ import fsspec from datetime import datetime -class DataLocator(): +class DataLocator: """ DataLocator is a simple wrapper around fsspec functionality, and provides a set of functions to encapsulate a data location (URI or path), interogate @@ -29,7 +29,7 @@ class DataLocator(): self.uri_or_path = uri_or_path self.protocol, self.path = DataLocator._get_protocol_and_path(uri_or_path) # work-around for LocalFileSystem not treating file: and None as the same scheme/protocol - self.cname = self.path if self.protocol == 'file' else self.uri_or_path + self.cname = self.path if self.protocol == "file" else self.uri_or_path # will throw RuntimeError if the protocol is unsupported self.fs = fsspec.filesystem(self.protocol) @@ -53,9 +53,9 @@ class DataLocator(): """ return datetime object representing last modification time, or None if unavailable """ info = self.fs.info(self.cname) if self.islocal() and info is not None: - return datetime.fromtimestamp(info['mtime']) + return datetime.fromtimestamp(info["mtime"]) else: - return getattr(info, 'LastModified', None) + return getattr(info, "LastModified", None) def abspath(self): """ @@ -74,7 +74,7 @@ class DataLocator(): return self.fs.open(self.uri_or_path, *args) def islocal(self): - return self.protocol is None or self.protocol == 'file' + return self.protocol is None or self.protocol == "file" def local_handle(self): if self.islocal(): @@ -90,7 +90,7 @@ class DataLocator(): return LocalFilePath(tmp_path, delete=True) -class LocalFilePath(): +class LocalFilePath: def __init__(self, tmp_path, delete=False): self.tmp_path = tmp_path self.delete = delete diff --git a/server/app/util/fbs/matrix.py b/server/app/util/fbs/matrix.py index b29f4eb4..637c39d7 100644 --- a/server/app/util/fbs/matrix.py +++ b/server/app/util/fbs/matrix.py @@ -27,7 +27,7 @@ def CreateNumpyVector(builder, x): if not isinstance(x, np.ndarray): raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}") - if x.dtype.kind not in ['b', 'i', 'u', 'f']: + if x.dtype.kind not in ["b", "i", "u", "f"]: raise TypeError("numpy-ndarray holds elements of unsupported datatype") if x.ndim > 1: @@ -42,11 +42,11 @@ def CreateNumpyVector(builder, x): x_little_endian = x.byteswap(inplace=False) # Calculate total length - len = int(x_little_endian.itemsize * x_little_endian.size) - builder.head = int(builder.Head() - len) + length = int(x_little_endian.itemsize * x_little_endian.size) + builder.head = int(builder.Head() - length) # tobytes ensures c_contiguous ordering - builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C') + builder.Bytes[builder.Head() : builder.Head() + length] = x_little_endian.tobytes(order="C") return builder.EndVector(x.size) @@ -88,9 +88,9 @@ def serialize_typed_array(builder, source_array, encoding_info): arr = arr.to_series() # convert to a simple ndarray - if as_type == 'json': - as_json = arr.to_json(orient='records') - arr = np.array(bytearray(as_json, 'utf-8')) + if as_type == "json": + as_json = arr.to_json(orient="records") + arr = np.array(bytearray(as_json, "utf-8")) else: if MatrixProxy.ismatrixproxy(arr) or sparse.issparse(arr): arr = arr.toarray() @@ -119,18 +119,16 @@ column_encoding_type_map = { np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), - np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32), np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), - np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32) + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), } -column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json') +column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") def column_encoding(arr): @@ -141,11 +139,10 @@ index_encoding_type_map = { # array protocol string: ( array_type, as_type ) np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), - np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32) + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), } -index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, 'json') +index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") def index_encoding(arr): @@ -163,7 +160,7 @@ def guess_at_mem_needed(matrix): guess = 1 # round up to nearest 1024 bytes - guess = (guess + 0x400) & (~0x3ff) + guess = (guess + 0x400) & (~0x3FF) return guess @@ -223,7 +220,7 @@ def deserialize_typed_array(tarr): TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, - TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray + TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray, } (u_type, u) = tarr if u_type is TypedArray.TypedArray.NONE: @@ -237,7 +234,7 @@ def deserialize_typed_array(tarr): arr.Init(u.Bytes, u.Pos) narr = arr.DataAsNumpy() if u_type == TypedArray.TypedArray.JSONEncodedArray: - narr = json.loads(narr.tostring().decode('utf-8')) + narr = json.loads(narr.tostring().decode("utf-8")) return narr diff --git a/server/app/util/matrix_proxy.py b/server/app/util/matrix_proxy.py index 32414c1d..ad81c43f 100644 --- a/server/app/util/matrix_proxy.py +++ b/server/app/util/matrix_proxy.py @@ -18,6 +18,7 @@ class _ArrayProxyBase(abc.ABC): Private base class for array or matrix proxy. This summarizes the interface used by the rest of cellxgene. """ + @property @abc.abstractmethod def dtype(self): @@ -68,10 +69,10 @@ class MatrixProxy(_ArrayProxyBase): Sub-classes automatically register. """ base_proxy_registry = { - 'pandas.core.frame.DataFrame': True, - 'numpy.ndarray': True, - 'scipy.sparse.csc.csc_matrix': True, - 'scipy.sparse.csr.csr_matrix': True, + "pandas.core.frame.DataFrame": True, + "numpy.ndarray": True, + "scipy.sparse.csc.csc_matrix": True, + "scipy.sparse.csr.csr_matrix": True, } proxy_registry = None last_cache_token = None @@ -103,7 +104,7 @@ class MatrixProxy(_ArrayProxyBase): """ cls.build_proxy_registry() t = type(matrix) - fqtn = t.__module__ + '.' + t.__name__ + fqtn = t.__module__ + "." + t.__name__ proxy_cls = cls.proxy_registry.get(fqtn, None) if proxy_cls is None: raise Exception(f"Matrix format `{fqtn}` is unsupported by proxy.") @@ -128,21 +129,17 @@ class MatrixProxyView(MatrixProxy): """ 2D matrix view to a 2D matrix """ - def __init__(self, arg1, shape=None, index=(), - transposed=False, copy=False): + + def __init__(self, arg1, shape=None, index=(), transposed=False, copy=False): if not copy: m = arg1 super().__init__(m) if shape is None: shape = m.shape - assert(len(shape) == 2) + assert len(shape) == 2 - index = tuple( - map(lambda s_i: - slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], - zip_longest(shape, index)) - ) + index = tuple(map(lambda s_i: slice(0, s_i[0], 1) if s_i[1] is None else s_i[1], zip_longest(shape, index))) self._shape = shape self._index = index @@ -234,20 +231,20 @@ class MatrixProxyView(MatrixProxy): NOTE: these follow the numpy rules for dimensionality reduction when an integer index is specified. """ + def _getitem_intXint(self, row, col): return self.m[row, col] def _getitem_intXslice(self, row, col): - shape = (_slice_length(col, self.m.shape[1]), ) + shape = (_slice_length(col, self.m.shape[1]),) return self.__class__.create_array(self.m, shape=shape, index=(row, col)) def _getitem_sliceXint(self, row, col): - shape = (_slice_length(row, self.m.shape[0]), ) + shape = (_slice_length(row, self.m.shape[0]),) return self.__class__.create_array(self.m, shape=shape, index=(row, col)) def _getitem_sliceXslice(self, row, col): - shape = (_slice_length(row, self.m.shape[0]), - _slice_length(col, self.m.shape[1])) + shape = (_slice_length(row, self.m.shape[0]), _slice_length(col, self.m.shape[1])) return self.__class__(self.m, shape=shape, index=(row, col), transposed=self.transposed) def toarray(self): @@ -261,22 +258,23 @@ class ArrayProxyView(_ArrayProxyBase): """ 1D array view to a 2D matrix """ + def __init__(self, arg1, shape=None, index=None, copy=False): super().__init__() if not copy: m = arg1 # one index MUST be an integer and the other MUST be a slice - assert(len(index) == 2) - assert(all(isinstance(idx, INT_TYPES + (slice, )) for idx in index)) - assert(isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES)) + assert len(index) == 2 + assert all(isinstance(idx, INT_TYPES + (slice,)) for idx in index) + assert isinstance(index[0], INT_TYPES) != isinstance(index[1], INT_TYPES) if shape is None: if isinstance(index[0], INT_TYPES): - shape = (m.shape[0], ) + shape = (m.shape[0],) else: - shape = (m.shape[1], ) - assert(len(shape) == 1) + shape = (m.shape[1],) + assert len(shape) == 1 self._shape = shape self.m = m @@ -336,7 +334,7 @@ class ArrayProxyView(_ArrayProxyBase): elif isinstance(col, slice): return self._getitem_intXslice(row, col) elif isinstance(row, slice): - assert(isinstance(col, INT_TYPES)) + assert isinstance(col, INT_TYPES) return self._getitem_sliceXint(row, col) raise IndexError("unsupported column index types") @@ -345,11 +343,11 @@ class ArrayProxyView(_ArrayProxyBase): return self.m[row, col] def _getitem_intXslice(self, row, col): - shape = (_slice_length(col, self.m.shape[1]), ) + shape = (_slice_length(col, self.m.shape[1]),) return self.__class__(self.m, shape=shape, index=(row, col)) def _getitem_sliceXint(self, row, col): - shape = (_slice_length(row, self.m.shape[0]), ) + shape = (_slice_length(row, self.m.shape[0]),) return self.__class__(self.m, shape=shape, index=(row, col)) def toarray(self): @@ -358,7 +356,7 @@ class ArrayProxyView(_ArrayProxyBase): def _unpack_index(index, shape): if not isinstance(index, tuple): - index = (index, ) + index = (index,) if len(shape) < len(index): raise IndexError("invalid index dimensionality - must be 2") @@ -366,7 +364,7 @@ def _unpack_index(index, shape): for shp, idx in zip_longest(shape, index): idx = slice(None) if idx is None else idx idx = _slice_defaults(idx, shp) if isinstance(idx, slice) else idx - unpacked += (idx, ) + unpacked += (idx,) return unpacked @@ -376,7 +374,7 @@ def _slice_slice(outer, outer_len, inner, inner_len): slice a slice - we take advantage of Python 3 range's support for indexing. """ - assert(outer_len >= inner_len) + assert outer_len >= inner_len outer_rng = range(*outer.indices(outer_len)) rng = outer_rng[inner] start, stop, step = rng.start, rng.stop, rng.step @@ -387,8 +385,8 @@ def _slice_slice(outer, outer_len, inner, inner_len): def _range_length(start, stop, step): """ return length of range """ - assert(step != 0) - assert(start is not None and stop is not None and step is not None) + assert step != 0 + assert start is not None and stop is not None and step is not None if step > 0 and start < stop: return 1 + (stop - 1 - start) // step elif step < 0 and start > stop: @@ -404,7 +402,7 @@ def _slice_length(s, length): def _slice_defaults(s, length): """ apply slice defaulting conventions """ - assert(length >= 0) + assert length >= 0 step = 1 if s.step is None else s.step diff --git a/server/app/util/utils.py b/server/app/util/utils.py index 4bf57612..c931ae1b 100644 --- a/server/app/util/utils.py +++ b/server/app/util/utils.py @@ -40,4 +40,5 @@ def requires_data(func): if self.data is None: raise DriverError(f"error data must be loaded before you call {func.__name__}") return func(self, *args, **kwargs) + return wrapped_function diff --git a/server/cli/cli.py b/server/cli/cli.py index ba0576b6..439587d0 100644 --- a/server/cli/cli.py +++ b/server/cli/cli.py @@ -4,17 +4,19 @@ from .launch import launch from .prepare import prepare -@click.group(name="cellxgene", - subcommand_metavar="COMMAND ", - options_metavar="", - context_settings=dict(max_content_width=85, - help_option_names=['-h', '--help'])) +@click.group( + name="cellxgene", + subcommand_metavar="COMMAND ", + options_metavar="", + context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]), +) @click.help_option("--help", "-h", help="Show this message and exit.") @click.version_option( version="0.13.0", prog_name="cellxgene", message="[%(prog)s] Version %(version)s", - help="Show the software version and exit.") + help="Show the software version and exit.", +) def cli(): pass diff --git a/server/cli/launch.py b/server/cli/launch.py index c5fa3178..c7f0f006 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -25,16 +25,12 @@ def common_args(func): Decorator to contain CLI args that will be common to both CLI and GUI: title and engine args. """ - @click.option( - "--title", - "-t", - metavar="", - help="Title to display. If omitted will use file name.") + @click.option("--title", "-t", metavar="", help="Title to display. If omitted will use file name.") @click.option( "--about", metavar="", - help="URL providing more information about the dataset " - "(hint: must be a fully specified absolute URL).") + help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).", + ) @click.option( "--embedding", "-e", @@ -42,39 +38,43 @@ def common_args(func): multiple=True, show_default=False, metavar="", - help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all." + help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.", ) @click.option( "--obs-names", "-obs", default=None, metavar="", - help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.") + help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.", + ) @click.option( "--var-names", "-var", default=None, metavar="", - help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.") + help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.", + ) @click.option( "--max-category-items", default=1000, metavar="", show_default=True, - help="Will not display categories with more distinct values than specified.",) + help="Will not display categories with more distinct values than specified.", + ) @click.option( "--diffexp-lfc-cutoff", "-de", default=0.01, show_default=True, metavar="", - help="Minimum log fold change threshold for differential expression.",) + help="Minimum log fold change threshold for differential expression.", + ) @click.option( "--experimental-annotations", is_flag=True, default=False, show_default=True, - help="Enable user annotation of data." + help="Enable user annotation of data.", ) @click.option( "--experimental-annotations-file", @@ -83,7 +83,8 @@ def common_args(func): multiple=False, metavar="", help="CSV file to initialize editing of existing annotations; will be altered in-place. " - "Incompatible with --annotations-output-dir.",) + "Incompatible with --annotations-output-dir.", + ) @click.option( "--experimental-annotations-output-dir", default=None, @@ -91,20 +92,23 @@ def common_args(func): multiple=False, metavar="", help="Directory of where to save output annotations; filename will be specified in the application. " - "Incompatible with --annotations-input-file.",) + "Incompatible with --annotations-input-file.", + ) @click.option( "--backed", "-b", is_flag=True, default=False, show_default=False, - help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.") + help="Load data in file-backed mode. This may save memory, but may result in slower overall performance.", + ) @click.option( "--disable-diffexp", is_flag=True, default=False, show_default=False, - help="Disable on-demand differential expression.") + help="Disable on-demand differential expression.", + ) @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -112,9 +116,18 @@ def common_args(func): return wrapper -def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffexp_lfc_cutoff, - experimental_annotations, experimental_annotations_file, - experimental_annotations_output_dir, backed, disable_diffexp): +def parse_engine_args( + embedding, + obs_names, + var_names, + max_category_items, + diffexp_lfc_cutoff, + experimental_annotations, + experimental_annotations_file, + experimental_annotations_output_dir, + backed, + disable_diffexp, +): annotations_file = experimental_annotations_file if experimental_annotations else None annotations_output_dir = experimental_annotations_output_dir if experimental_annotations else None return { @@ -127,14 +140,15 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe "annotations_file": annotations_file, "annotations_output_dir": annotations_output_dir, "backed": backed, - "disable_diffexp": disable_diffexp + "disable_diffexp": disable_diffexp, } @sort_options -@click.command(short_help="Launch the cellxgene data viewer. " - "Run `cellxgene launch --help` for more information.", - options_metavar="",) +@click.command( + short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.", + options_metavar="", +) @click.argument("data", nargs=1, metavar="", required=True) @click.option( "--verbose", @@ -142,7 +156,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe is_flag=True, default=False, show_default=True, - help="Provide verbose output, including warnings and all server requests.",) + help="Provide verbose output, including warnings and all server requests.", +) @click.option( "--debug", "-d", @@ -150,7 +165,8 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe default=False, show_default=True, help="Run in debug mode. This is helpful for cellxgene developers, " - "or when you want more information about an error condition.",) + "or when you want more information about an error condition.", +) @click.option( "--open", "-o", @@ -158,19 +174,22 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe is_flag=True, default=False, show_default=True, - help="Open web browser after launch.",) + help="Open web browser after launch.", +) @click.option( "--port", "-p", metavar="", show_default=True, - help="Port to run server on. If not specified cellxgene will find an available port.",) + help="Port to run server on. If not specified cellxgene will find an available port.", +) @click.option( "--host", metavar="", default="127.0.0.1", show_default=False, - help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).") + help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).", +) @click.option( "--scripts", "-s", @@ -178,30 +197,31 @@ def parse_engine_args(embedding, obs_names, var_names, max_category_items, diffe multiple=True, metavar="", help="Additional script files to include in HTML page. If not specified, " - "no additional script files will be included.", - show_default=False,) + "no additional script files will be included.", + show_default=False, +) @click.help_option("--help", "-h", help="Show this message and exit.") @common_args def launch( - data, - verbose, - debug, - open_browser, - port, - host, - embedding, - obs_names, - var_names, - max_category_items, - diffexp_lfc_cutoff, - title, - scripts, - about, - experimental_annotations, - experimental_annotations_file, - experimental_annotations_output_dir, - backed, - disable_diffexp + data, + verbose, + debug, + open_browser, + port, + host, + embedding, + obs_names, + var_names, + max_category_items, + diffexp_lfc_cutoff, + title, + scripts, + about, + experimental_annotations, + experimental_annotations_file, + experimental_annotations_output_dir, + backed, + disable_diffexp, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. @@ -217,13 +237,18 @@ def launch( > cellxgene launch """ - e_args = parse_engine_args(embedding, obs_names, var_names, max_category_items, - diffexp_lfc_cutoff, - experimental_annotations, - experimental_annotations_file, - experimental_annotations_output_dir, - backed, - disable_diffexp) + e_args = parse_engine_args( + embedding, + obs_names, + var_names, + max_category_items, + diffexp_lfc_cutoff, + experimental_annotations, + experimental_annotations_file, + experimental_annotations_output_dir, + backed, + disable_diffexp, + ) try: data_locator = DataLocator(data) except RuntimeError as re: @@ -256,7 +281,8 @@ def launch( sys.tracebacklimit = 0 if scripts: - click.echo(r""" + click.echo( + r""" / / /\ \ \__ _ _ __ _ __ (_)_ __ __ _ \ \/ \/ / _` | '__| '_ \| | '_ \ / _` | \ /\ / (_| | | | | | | | | | | (_| | @@ -264,7 +290,8 @@ def launch( |___/ The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a security risk by including the --scripts flag. Make sure you trust the scripts that you are including. - """) + """ + ) scripts_pretty = ", ".join(scripts) click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) @@ -289,8 +316,9 @@ def launch( click.echo("Warning: --experimental-annotations-output-dir ignored as --annotations not enabled.") else: if experimental_annotations_file is not None and experimental_annotations_output_dir is not None: - raise click.ClickException("--experimental-annotations-file and --experimental-annotations-output-dir " - "may not be used together.") + raise click.ClickException( + "--experimental-annotations-file and --experimental-annotations-output-dir " "may not be used together." + ) if experimental_annotations_file is not None: lf_name, lf_ext = splitext(experimental_annotations_file) @@ -301,10 +329,12 @@ def launch( try: mkdir(experimental_annotations_output_dir) except OSError: - raise click.ClickException("Unable to create directory specified by " - "--experimental-annotations-output-dir") + raise click.ClickException( + "Unable to create directory specified by " "--experimental-annotations-output-dir" + ) if about: + def url_check(url): try: result = urlparse(url) @@ -346,9 +376,11 @@ def launch( except ScanpyFileError as e: raise click.ClickException(f"{e}") - if not disable_diffexp and server.app.data.config['diffexp_may_be_slow']: - click.echo(f"[cellxgene] CAUTION: due to the size of your dataset, " - f"running differential expression may take longer or fail.") + if not disable_diffexp and server.app.data.config["diffexp_may_be_slow"]: + click.echo( + f"[cellxgene] CAUTION: due to the size of your dataset, " + f"running differential expression may take longer or fail." + ) if open_browser: click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.") diff --git a/server/cli/prepare.py b/server/cli/prepare.py index 224fa17c..8fd9e0bf 100644 --- a/server/cli/prepare.py +++ b/server/cli/prepare.py @@ -8,9 +8,10 @@ from server.utils.utils import sort_options @sort_options -@click.command(short_help="Preprocess data for use with cellxgene. " - "Run `cellxgene prepare --help` for more information.", - options_metavar="",) +@click.command( + short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.", + options_metavar="", +) @click.argument("data", nargs=1, metavar="", required=True) @click.option( "--embedding", @@ -35,37 +36,33 @@ from server.utils.utils import sort_options @click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True) @click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="") @click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="") -@click.option("--skip-qc", default=False, is_flag=True, - help="Do not run quality control metrics. By default cellxgene runs them " - "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).") @click.option( - "--make-obs-names-unique", - default=True, + "--skip-qc", + default=False, is_flag=True, - help="Ensure obs index is unique.", - show_default=True + help="Do not run quality control metrics. By default cellxgene runs them " + "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", ) @click.option( - "--make-var-names-unique", - default=True, - is_flag=True, - help="Ensure var index is unique.", - show_default=True + "--make-obs-names-unique", default=True, is_flag=True, help="Ensure obs index is unique.", show_default=True +) +@click.option( + "--make-var-names-unique", default=True, is_flag=True, help="Ensure var index is unique.", show_default=True ) @click.help_option("--help", "-h", help="Show this message and exit.") def prepare( - data, - embedding, - recipe, - output, - plotting, - sparse, - overwrite, - set_obs_names, - set_var_names, - skip_qc, - make_obs_names_unique, - make_var_names_unique, + data, + embedding, + recipe, + output, + plotting, + sparse, + overwrite, + set_obs_names, + set_var_names, + skip_qc, + make_obs_names_unique, + make_var_names_unique, ): """ Preprocess data for use with cellxgene. diff --git a/server/gui/browser.py b/server/gui/browser.py index d09bc621..aa2224e4 100644 --- a/server/gui/browser.py +++ b/server/gui/browser.py @@ -12,6 +12,7 @@ WindowUtils = cef.WindowUtils() # noinspection PyUnresolvedReferences CefWidgetParent = QWidget + class CefWidget(CefWidgetParent): def __init__(self, parent=None): super(CefWidget, self).__init__(parent) @@ -58,8 +59,7 @@ class CefWidget(CefWidgetParent): if WINDOWS: WindowUtils.OnSize(self.getHandle(), 0, 0, 0) elif LINUX: - self.browser.SetBounds(self.x, self.y, - self.width(), self.height()) + self.browser.SetBounds(self.x, self.y, self.width(), self.height()) self.browser.NotifyMoveOrResizeStarted() def resizeEvent(self, event): @@ -68,8 +68,7 @@ class CefWidget(CefWidgetParent): if WINDOWS: WindowUtils.OnSize(self.getHandle(), 0, 0, 0) elif LINUX: - self.browser.SetBounds(self.x, self.y, - size.width(), size.height()) + self.browser.SetBounds(self.x, self.y, size.width(), size.height()) self.browser.NotifyMoveOrResizeStarted() diff --git a/server/gui/hook-cefpython3.py b/server/gui/hook-cefpython3.py index 4c6e0a9d..6e3b66b1 100644 --- a/server/gui/hook-cefpython3.py +++ b/server/gui/hook-cefpython3.py @@ -37,8 +37,7 @@ logger = logging.getLogger(__name__) # Functions def check_platforms(): if not is_win and not is_darwin and not is_linux: - raise SystemExit("Error: Currently only Windows, Linux and Darwin " - "platforms are supported, see Issue #135.") + raise SystemExit("Error: Currently only Windows, Linux and Darwin " "platforms are supported, see Issue #135.") def check_pyinstaller_version(): @@ -50,22 +49,19 @@ def check_pyinstaller_version(): version = PyInstaller.__version__ match = re.search(r"^\d+\.\d+(\.\d+)?", version) if not (match.group(0) >= PYINSTALLER_MIN_VERSION): - raise SystemExit("Error: pyinstaller %s or higher is required" - % PYINSTALLER_MIN_VERSION) + raise SystemExit("Error: pyinstaller %s or higher is required" % PYINSTALLER_MIN_VERSION) def check_cefpython3_version(): if not is_module_satisfies("cefpython3 >= %s" % CEFPYTHON_MIN_VERSION): - raise SystemExit("Error: cefpython3 %s or higher is required" - % CEFPYTHON_MIN_VERSION) + raise SystemExit("Error: cefpython3 %s or higher is required" % CEFPYTHON_MIN_VERSION) def get_cefpython_modules(): """Get all cefpython Cython modules in the cefpython3 package. It returns a list of names without file extension. Eg. 'cefpython_py27'. """ - pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, - "cefpython_py*" + CYTHON_MODULE_EXT)) + pyds = glob.glob(os.path.join(CEFPYTHON3_DIR, "cefpython_py*" + CYTHON_MODULE_EXT)) assert len(pyds) > 1, "Missing cefpython3 Cython modules" modules = [] for path in pyds: @@ -130,14 +126,21 @@ def get_cefpython3_datas(): for filename in os.listdir(CEFPYTHON3_DIR): # Ignore Cython modules which are already handled by # pyinstaller automatically. - if filename[:-len(CYTHON_MODULE_EXT)] in get_cefpython_modules(): + if filename[: -len(CYTHON_MODULE_EXT)] in get_cefpython_modules(): continue # CEF binaries and datas extension = os.path.splitext(filename)[1] - if extension in \ - [".exe", ".dll", ".pak", ".dat", ".bin", ".txt", ".so", ".plist"] \ - or filename.lower().startswith("license"): + if extension in [ + ".exe", + ".dll", + ".pak", + ".dat", + ".bin", + ".txt", + ".so", + ".plist", + ] or filename.lower().startswith("license"): logger.info("Include cefpython3 data: {}".format(filename)) ret.append((os.path.join(CEFPYTHON3_DIR, filename), cefdatadir)) @@ -145,11 +148,9 @@ def get_cefpython3_datas(): # "Chromium Embedded Framework.framework/Resources" with subdirectories # is required. Contain .pak files and locales (each locale in separate # subdirectory). - resources_subdir = \ - os.path.join("Chromium Embedded Framework.framework", "Resources") + resources_subdir = os.path.join("Chromium Embedded Framework.framework", "Resources") base_path = os.path.join(CEFPYTHON3_DIR, resources_subdir) - assert os.path.exists(base_path), \ - "{} dir not found in cefpython3".format(resources_subdir) + assert os.path.exists(base_path), "{} dir not found in cefpython3".format(resources_subdir) for path, dirs, files in os.walk(base_path): for file in files: absolute_file_path = os.path.join(path, file) @@ -159,22 +160,17 @@ def get_cefpython3_datas(): elif is_win or is_linux: # The .pak files in cefpython3/locales/ directory locales_dir = os.path.join(CEFPYTHON3_DIR, "locales") - assert os.path.exists(locales_dir), \ - "locales/ dir not found in cefpython3" + assert os.path.exists(locales_dir), "locales/ dir not found in cefpython3" for filename in os.listdir(locales_dir): - logger.info("Include cefpython3 data: {}/{}".format( - os.path.basename(locales_dir), filename)) - ret.append((os.path.join(locales_dir, filename), - os.path.join(cefdatadir, "locales"))) + logger.info("Include cefpython3 data: {}/{}".format(os.path.basename(locales_dir), filename)) + ret.append((os.path.join(locales_dir, filename), os.path.join(cefdatadir, "locales"))) # Optional .so/.dll files in cefpython3/swiftshader/ directory swiftshader_dir = os.path.join(CEFPYTHON3_DIR, "swiftshader") if os.path.isdir(swiftshader_dir): for filename in os.listdir(swiftshader_dir): - logger.info("Include cefpython3 data: {}/{}".format( - os.path.basename(swiftshader_dir), filename)) - ret.append((os.path.join(swiftshader_dir, filename), - os.path.join(cefdatadir, "swiftshader"))) + logger.info("Include cefpython3 data: {}/{}".format(os.path.basename(swiftshader_dir), filename)) + ret.append((os.path.join(swiftshader_dir, filename), os.path.join(cefdatadir, "swiftshader"))) return ret diff --git a/server/gui/main.py b/server/gui/main.py index 72f3e2a7..9963c684 100644 --- a/server/gui/main.py +++ b/server/gui/main.py @@ -20,8 +20,8 @@ from server.utils.utils import find_available_port if WINDOWS or LINUX: dirname = dirname(PySide2.__file__) - plugin_path = join(dirname, 'plugins', 'platforms') - environ['QT_QPA_PLATFORM_PLUGIN_PATH'] = plugin_path + plugin_path = join(dirname, "plugins", "platforms") + environ["QT_QPA_PLATFORM_PLUGIN_PATH"] = plugin_path # Configuration # TODO remember this or calculate it? @@ -94,8 +94,7 @@ class MainWindow(QMainWindow): # a hidden window, embed CEF browser in it and then # create a container for that hidden window and replace # cef widget in the layout with the container. - self.container = QWidget.createWindowContainer( - self.cef_widget.hidden_window, parent=self) + self.container = QWidget.createWindowContainer(self.cef_widget.hidden_window, parent=self) self.stacked_layout.replaceWidget(self.cef_widget, self.container) self.stacked_layout.setCurrentIndex(LOAD_INDEX) @@ -117,7 +116,7 @@ class MainWindow(QMainWindow): def setupMenu(self): # TODO add communication to subprocess on reload main_menu = self.menuBar() - file_menu = main_menu.addMenu('File') + file_menu = main_menu.addMenu("File") load_action = QAction("Load file...", self) load_action.setStatusTip("Load file") load_action.setShortcut("Ctrl+O") @@ -201,7 +200,7 @@ class LoadWidget(QFrame): for l in [logo_layout, file_layout, message_layout]: load_ui_layout.addLayout(l) - #TODO remove magic number + # TODO remove magic number load_ui_layout.setStretch(1, 10) self.setLayout(load_ui_layout) @@ -240,8 +239,15 @@ class LoadWidget(QFrame): def createScanpyEngine(self, file_name): title = splitext(basename(file_name))[0] self.window().setupServer() - worker = Worker(self.window().parent_conn, self.window().child_conn, file_name, host="127.0.0.1", - port=GUI_PORT, title=title, engine_options={}) + worker = Worker( + self.window().parent_conn, + self.window().child_conn, + file_name, + host="127.0.0.1", + port=GUI_PORT, + title=title, + engine_options={}, + ) self.window().load_emitter.signals.ready.connect(self.onDataReady) self.window().load_emitter.signals.engine_error.connect(self.onServerError) self.window().load_emitter.signals.server_error.connect(self.onServerError) @@ -289,6 +295,7 @@ class LoadWidget(QFrame): onServerError = partialmethod(onError, server_error=True) + class FilePath(QObject): def __init__(self): super(FilePath, self).__init__() @@ -320,8 +327,7 @@ class FileArea(QFrame): def fileBrowse(self): options = QFileDialog.Options() # options |= QFileDialog.DontUseNativeDialog - file_name, _ = QFileDialog.getOpenFileName(self, - "Open H5AD File", "", "H5AD Files (*.h5ad)", options=options) + file_name, _ = QFileDialog.getOpenFileName(self, "Open H5AD File", "", "H5AD Files (*.h5ad)", options=options) if file_name: self.parent().file_name.updateValue(file_name) self.parent().onLoad() @@ -391,5 +397,5 @@ def main(): sys.exit(0) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/server/gui/utils.py b/server/gui/utils.py index b2149461..7873f49d 100644 --- a/server/gui/utils.py +++ b/server/gui/utils.py @@ -4,9 +4,9 @@ import platform from PySide2.QtCore import QObject, Signal # Detect OS -WINDOWS = (platform.system() == "Windows") -LINUX = (platform.system() == "Linux") -MAC = (platform.system() == "Darwin") +WINDOWS = platform.system() == "Windows" +LINUX = platform.system() == "Linux" +MAC = platform.system() == "Darwin" class WorkerSignals(QObject): @@ -18,6 +18,7 @@ class WorkerSignals(QObject): error - `str` error message result - `object` data returned from processing, anything """ + finished = Signal() engine_error = Signal(str) server_error = Signal(str) @@ -34,6 +35,7 @@ class SiteReadySignals(QObject): ready error - `str` error message """ + ready = Signal() timeout = Signal() error = Signal(str) diff --git a/server/gui/workers.py b/server/gui/workers.py index 62320ab8..e77e057d 100644 --- a/server/gui/workers.py +++ b/server/gui/workers.py @@ -36,6 +36,7 @@ class Worker(EmittingProcess): return from server.app.app import Server from server.app.scanpy_engine.scanpy_engine import ScanpyEngine + # create server try: server = Server() diff --git a/server/test/decode_fbs.py b/server/test/decode_fbs.py index a9b847ad..ea029a1c 100644 --- a/server/test/decode_fbs.py +++ b/server/test/decode_fbs.py @@ -1,4 +1,3 @@ - """ Code to decode, for testing purposes, the flatbuffer encoded blobs. This code will need to be updated if fbs/matrix.fbs changes. @@ -22,20 +21,20 @@ def decode_typed_array(tarr): TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, - TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray + TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray, } (u_type, u) = tarr if u_type == TypedArray.TypedArray.NONE: return None TarType = type_map.get(u_type, None) - assert(TarType is not None) + assert TarType is not None arr = TarType() arr.Init(u.Bytes, u.Pos) narr = arr.DataAsNumpy() if u_type == TypedArray.TypedArray.JSONEncodedArray: - narr = json.loads(narr.tostring().decode('utf-8')) + narr = json.loads(narr.tostring().decode("utf-8")) return narr @@ -60,10 +59,4 @@ def decode_matrix_FBS(buf): cidx = decode_typed_array((df.ColIndexType(), df.ColIndex())) - return { - "n_rows": n_rows, - "n_cols": n_cols, - "columns": decoded_columns, - "col_idx": cidx, - "row_idx": None - } + return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None} diff --git a/server/test/test_api.py b/server/test/test_api.py index 7f396431..dea00132 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -19,7 +19,7 @@ class EndPoints(unittest.TestCase): @classmethod def setUpClass(cls): - cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"]) + cls.ps = Popen(["cellxgene", "launch", "../example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"]) session = requests.Session() for i in range(90): try: @@ -68,14 +68,15 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 8) - self.assertIsNotNone(df['columns']) - self.assertListEqual(df['col_idx'], [ - 'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1' - ]) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 8) + self.assertIsNotNone(df["columns"]) + self.assertListEqual( + df["col_idx"], + ["pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"], + ) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) def test_bad_filter(self): endpoint = "data/var" @@ -91,14 +92,14 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 5) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 5) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"] - self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain']) + self.assertListEqual(df["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]) def test_get_annotations_obs_keys_fbs(self): endpoint = "annotations/obs" @@ -109,13 +110,13 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 2) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) - self.assertListEqual(df['col_idx'], ['n_genes', 'percent_mito']) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 2) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertListEqual(df["col_idx"], ["n_genes", "percent_mito"]) def test_get_annotations_obs_error(self): endpoint = "annotations/obs" @@ -162,14 +163,14 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 1838) - self.assertEqual(df['n_cols'], 2) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) + self.assertEqual(df["n_rows"], 1838) + self.assertEqual(df["n_cols"], 2) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"] - self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells']) + self.assertListEqual(df["col_idx"], [var_index_col_name, "n_cells"]) def test_get_annotations_var_keys_fbs(self): endpoint = "annotations/var" @@ -180,13 +181,13 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 1838) - self.assertEqual(df['n_cols'], 1) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) - self.assertListEqual(df['col_idx'], ['n_cells']) + self.assertEqual(df["n_rows"], 1838) + self.assertEqual(df["n_cols"], 1) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertListEqual(df["col_idx"], ["n_cells"]) def test_get_annotations_var_error(self): endpoint = "annotations/var" @@ -217,35 +218,29 @@ class EndPoints(unittest.TestCase): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 1838) - self.assertIsNotNone(df['columns']) - self.assertListEqual(df['col_idx'].tolist(), []) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 1838) + self.assertIsNotNone(df["columns"]) + self.assertListEqual(df["col_idx"].tolist(), []) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) def test_data_put_filter_fbs(self): endpoint = f"data/var" url = f"{URL_BASE}{endpoint}" header = {"Accept": "application/octet-stream"} - filter = { - "filter": { - "var": { - "index": [0, 1, 4] - } - } - } + filter = {"filter": {"var": {"index": [0, 1, 4]}}} result = self.session.put(url, headers=header, json=filter) self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df['n_rows'], 2638) - self.assertEqual(df['n_cols'], 3) - self.assertIsNotNone(df['columns']) - self.assertIsNotNone(df['col_idx']) - self.assertIsNone(df['row_idx']) - self.assertEqual(len(df['columns']), df['n_cols']) - self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4]) + self.assertEqual(df["n_rows"], 2638) + self.assertEqual(df["n_cols"], 3) + self.assertIsNotNone(df["columns"]) + self.assertIsNotNone(df["col_idx"]) + self.assertIsNone(df["row_idx"]) + self.assertEqual(len(df["columns"]), df["n_cols"]) + self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4]) def test_data_put_single_var(self): endpoint = f"data/var" diff --git a/server/test/test_fbs.py b/server/test/test_fbs.py index da32903a..36c3ec59 100644 --- a/server/test/test_fbs.py +++ b/server/test/test_fbs.py @@ -32,7 +32,7 @@ class FbsTests(unittest.TestCase): for i in range(0, len(d["columns"])): self.assertEqual(len(d["columns"][i]), dims[0]) self.assertIsInstance(d["columns"][i], expected_types[i][0]) - if (expected_types[i][1] is not None): + if expected_types[i][1] is not None: self.assertEqual(d["columns"][i].dtype, expected_types[i][1]) if expected_column_idx is not None: self.assertSetEqual(set(expected_column_idx), set(d["col_idx"])) @@ -40,48 +40,37 @@ class FbsTests(unittest.TestCase): def test_encode_DataFrame(self): df = pd.DataFrame( data={ - 'a': np.zeros((10,), dtype=np.float32), - 'b': np.ones((10,), dtype=np.int64), - 'c': np.array([i for i in range(0, 10)], dtype=np.uint16), - 'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') - }) - expected_types = ( - (np.ndarray, np.float32), - (np.ndarray, np.int32), - (np.ndarray, np.uint32), - (list, None) + "a": np.zeros((10,), dtype=np.float32), + "b": np.ones((10,), dtype=np.int64), + "c": np.array([i for i in range(0, 10)], dtype=np.uint16), + "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), + } ) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None)) fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) - self.fbs_checks(fbs, (10, 4), expected_types, ['a', 'b', 'c', 'd']) + self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"]) def test_encode_ndarray(self): arr = np.zeros((3, 2), dtype=np.float32) - expected_types = ( - (np.ndarray, np.float32), - (np.ndarray, np.float32), - (np.ndarray, np.float32) - ) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32)) fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None) self.fbs_checks(fbs, (3, 2), expected_types, None) def test_encode_sparse(self): csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]])) - expected_types = ( - (np.ndarray, np.int32), - (np.ndarray, np.int32), - (np.ndarray, np.int32) - ) + expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32)) fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None) self.fbs_checks(fbs, (2, 3), expected_types, None) def test_roundtrip(self): dfSrc = pd.DataFrame( data={ - 'a': np.zeros((10,), dtype=np.float32), - 'b': np.ones((10,), dtype=np.int64), - 'c': np.array([i for i in range(0, 10)], dtype=np.uint16), - 'd': pd.Series(['x', 'y', 'z', 'x', 'y', 'z', 'a', 'x', 'y', 'z'], dtype='category') - }) + "a": np.zeros((10,), dtype=np.float32), + "b": np.ones((10,), dtype=np.int64), + "c": np.array([i for i in range(0, 10)], dtype=np.uint16), + "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), + } + ) dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns)) self.assertEqual(dfSrc.shape, dfDst.shape) self.assertEqual(set(dfSrc.columns), set(dfDst.columns)) diff --git a/server/test/test_matrix_proxy.py b/server/test/test_matrix_proxy.py index 624cbda2..2c3ad5d9 100644 --- a/server/test/test_matrix_proxy.py +++ b/server/test/test_matrix_proxy.py @@ -7,9 +7,10 @@ class NdArrayProxyView(MatrixProxyView): """ Fake test class for matrix proxy - wraps ndarray """ + @classmethod def __supports__(cls): - return ('numpy.ndarray', ) + return ("numpy.ndarray",) class MatrixProxyViewTest(unittest.TestCase): @@ -41,18 +42,17 @@ class MatrixProxyViewTest(unittest.TestCase): def test_toarray(self): n = np.arange(15, dtype=np.float32).reshape((3, 5)) mp = MatrixProxy.create(n) - self.assertTrue(np.all(mp.toarray() == [ - [0., 1., 2., 3., 4.], - [5., 6., 7., 8., 9.], - [10., 11., 12., 13., 14.] - ])) - self.assertTrue(np.all(mp.T.toarray() == [ - [0., 5., 10.], - [1., 6., 11.], - [2., 7., 12.], - [3., 8., 13.], - [4., 9., 14.] - ])) + self.assertTrue( + np.all( + mp.toarray() == [[0.0, 1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0, 9.0], [10.0, 11.0, 12.0, 13.0, 14.0]] + ) + ) + self.assertTrue( + np.all( + mp.T.toarray() + == [[0.0, 5.0, 10.0], [1.0, 6.0, 11.0], [2.0, 7.0, 12.0], [3.0, 8.0, 13.0], [4.0, 9.0, 14.0]] + ) + ) def test_indexing(self): """ @@ -95,47 +95,19 @@ class MatrixProxyViewTest(unittest.TestCase): # slice, slice - self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [ - [7, 8], - [12, 13] - ])) - self.assertTrue(np.all(mp[:3, :4].toarray() == [ - [0., 1., 2., 3.], - [5., 6., 7., 8.], - [10., 11., 12., 13.] - ])) - self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [ - [14, 13, 12, 11, 10], - [9, 8, 7, 6, 5], - [4, 3, 2, 1, 0] - ])) - self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [ - [14, 12, 10], - [4, 2, 0] - ])) + self.assertTrue(np.all(mp[1:3, 2:4].toarray() == [[7, 8], [12, 13]])) + self.assertTrue( + np.all(mp[:3, :4].toarray() == [[0.0, 1.0, 2.0, 3.0], [5.0, 6.0, 7.0, 8.0], [10.0, 11.0, 12.0, 13.0]]) + ) + self.assertTrue(np.all(mp[::-1, ::-1].toarray() == [[14, 13, 12, 11, 10], [9, 8, 7, 6, 5], [4, 3, 2, 1, 0]])) + self.assertTrue(np.all(mp[::-2, ::-2].toarray() == [[14, 12, 10], [4, 2, 0]])) - self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [ - [7, 12], - [8, 13] - ])) - self.assertTrue(np.all(mp.T[:4, :3].toarray() == [ - [0, 5, 10], - [1, 6, 11], - [2, 7, 12], - [3, 8, 13] - ])) - self.assertTrue(np.all(mp.T[::-1, ::-1].toarray() == [ - [14, 9, 4], - [13, 8, 3], - [12, 7, 2], - [11, 6, 1], - [10, 5, 0] - ])) - self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [ - [14, 4], - [12, 2], - [10, 0] - ])) + self.assertTrue(np.all(mp.T[2:4, 1:3].toarray() == [[7, 12], [8, 13]])) + self.assertTrue(np.all(mp.T[:4, :3].toarray() == [[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8, 13]])) + self.assertTrue( + np.all(mp.T[::-1, ::-1].toarray() == [[14, 9, 4], [13, 8, 3], [12, 7, 2], [11, 6, 1], [10, 5, 0]]) + ) + self.assertTrue(np.all(mp.T[::-2, ::-2].toarray() == [[14, 4], [12, 2], [10, 0]])) def test_repeated_indexing(self): """ @@ -149,31 +121,15 @@ class MatrixProxyViewTest(unittest.TestCase): self.assertEqual(mp[0][1], 1) self.assertEqual(mp.T[0][1], 5) - self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [ - 2, 1 - ])) - self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [ - 2, 3 - ])) - self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [ - 3, 2 - ])) - self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [ - 3, 2 - ])) - self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [ - 2, 3 - ])) + self.assertTrue(np.all(mp[0::-1, ::-1][0, 2:4].toarray() == [2, 1])) + self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 1:3:1].toarray() == [2, 3])) + self.assertTrue(np.all(mp[0::-1, 1:5:1][0, 2:0:-1].toarray() == [3, 2])) + self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 1:3:1].toarray() == [3, 2])) + self.assertTrue(np.all(mp[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [2, 3])) - self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [ - 2, 1 - ])) - self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [ - 10 - ])) - self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [ - 10 - ])) + self.assertTrue(np.all(mp.T[::-1, 0::-1][2:4, 0].toarray() == [2, 1])) + self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 1:3:1].toarray() == [10])) + self.assertTrue(np.all(mp.T[0::-1, 1:5:1][0, 2:0:-1].toarray() == [10])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 1:3:1].toarray() == [])) self.assertTrue(np.all(mp.T[0::-1, 5:1:-1][0, 2:0:-1].toarray() == [])) @@ -190,20 +146,12 @@ class MatrixProxyViewTest(unittest.TestCase): self.assertEqual(mp[0, 0], 0) # drop 1 dimension, to an array - self.assertTrue(np.all(mp[0, :].toarray() == [ - 0, 1, 2, 3, 4 - ])) - self.assertTrue(np.all(mp[:, 0].toarray() == [ - 0, 5, 10 - ])) + self.assertTrue(np.all(mp[0, :].toarray() == [0, 1, 2, 3, 4])) + self.assertTrue(np.all(mp[:, 0].toarray() == [0, 5, 10])) # with .T - self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [ - [4, 9] - ])) - self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [ - 4, 9 - ])) + self.assertTrue(np.all(mp[0:2].T[-1:].toarray() == [[4, 9]])) + self.assertTrue(np.all(mp[0:2].T[-1].toarray() == [4, 9])) def test_iter(self): """ @@ -214,17 +162,13 @@ class MatrixProxyViewTest(unittest.TestCase): rows = [r for r in mp] self.assertEqual(len(rows), 3) - self.assertTrue(np.all(rows[0].toarray() == [ - 0, 1, 2, 3, 4 - ])) + self.assertTrue(np.all(rows[0].toarray() == [0, 1, 2, 3, 4])) for i, r in enumerate(rows): self.assertTrue(np.all(mp[i].toarray() == r.toarray())) cols = [c for c in mp.T] self.assertEqual(len(cols), 5) - self.assertTrue(np.all(cols[0].toarray() == [ - 0, 5, 10 - ])) + self.assertTrue(np.all(cols[0].toarray() == [0, 5, 10])) for i, c in enumerate(cols): self.assertTrue(np.all(mp.T[i].toarray() == c.toarray())) diff --git a/server/test/test_nan_rest.py b/server/test/test_nan_rest.py index a9ba4dd7..5dcf964a 100644 --- a/server/test/test_nan_rest.py +++ b/server/test/test_nan_rest.py @@ -20,9 +20,7 @@ class WithNaNs(unittest.TestCase): @classmethod def setUpClass(cls): - cls.ps = Popen( - ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"] - ) + cls.ps = Popen(["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]) session = requests.Session() for i in range(90): try: diff --git a/server/test/test_nan_scanpy_engine.py b/server/test/test_nan_scanpy_engine.py index ec85d2aa..84fa255e 100644 --- a/server/test/test_nan_scanpy_engine.py +++ b/server/test/test_nan_scanpy_engine.py @@ -21,12 +21,12 @@ class NaNTest(unittest.TestCase): } with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) - self.data = ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args) + self.data = ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args) self.data._create_schema() def test_load(self): with self.assertWarns(UserWarning): - ScanpyEngine(DataLocator("server/test/test_datasets/nan.h5ad"), self.args) + ScanpyEngine(DataLocator("test/test_datasets/nan.h5ad"), self.args) def test_init(self): self.assertEqual(self.data.cell_count, 100) @@ -44,11 +44,7 @@ class NaNTest(unittest.TestCase): with pytest.raises(FilterError): self.data.data_frame_to_fbs_matrix("an erroneous filter", "var") with pytest.raises(FilterError): - filter_ = { - "filter": { - "obs": {"index": [1, 99, [200, 300]]} - } - } + filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}} self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") def test_dataframe_obs_not_implemented(self): @@ -59,10 +55,7 @@ class NaNTest(unittest.TestCase): def test_annotation(self): annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs")) obs_index_col_name = self.data.schema["annotations"]["obs"]["index"] - self.assertEqual( - annotations["col_idx"], - [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"] - ) + self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]) self.assertEqual(annotations["n_rows"], 100) self.assertTrue(math.isnan(annotations["columns"][2][0])) diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 73ed8de0..6fca92cc 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -18,15 +18,17 @@ Test the scanpy engine using the pbmc3k data set. """ -@parameterized_class(("data_locator", "backed"), [ - ("example-dataset/pbmc3k.h5ad", False), - ("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False), - ("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False), - - ("example-dataset/pbmc3k.h5ad", True), - ("server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True), - ("server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True), -]) +@parameterized_class( + ("data_locator", "backed"), + [ + ("../example-dataset/pbmc3k.h5ad", False), + ("test/test_datasets/pbmc3k-CSC-gz.h5ad", False), + ("test/test_datasets/pbmc3k-CSR-gz.h5ad", False), + ("../example-dataset/pbmc3k.h5ad", True), + ("test/test_datasets/pbmc3k-CSC-gz.h5ad", True), + ("test/test_datasets/pbmc3k-CSR-gz.h5ad", True), + ], +) class EngineTest(unittest.TestCase): def setUp(self): args = { @@ -36,7 +38,7 @@ class EngineTest(unittest.TestCase): "var_names": None, "diffexp_lfc_cutoff": 0.01, "layout_file": None, - "backed": self.backed + "backed": self.backed, } self.data = ScanpyEngine(DataLocator(self.data_locator), args) @@ -64,11 +66,7 @@ class EngineTest(unittest.TestCase): self.data._validate_data_types() def test_filter_idx(self): - filter_ = { - "filter": { - "var": {"index": [1, 99, [200, 300]]} - } - } + filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}} fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) @@ -76,14 +74,7 @@ class EngineTest(unittest.TestCase): def test_filter_complex(self): filter_ = { - "filter": { - "var": { - "annotation_value": [ - {"name": "n_cells", "min": 10} - ], - "index": [1, 99, [200, 300]] - } - } + "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}} } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) @@ -101,16 +92,14 @@ class EngineTest(unittest.TestCase): def test_schema_produces_error(self): self.data.data.obs["time"] = pd.Series( - list([time.time() for i in range(self.data.cell_count)]), - dtype="datetime64[ns]", + list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]", ) with pytest.raises(TypeError): self.data._create_schema() def test_config(self): self.assertEqual( - self.data.features["layout"]["obs"], - {"available": True, "interactiveLimit": 50000}, + self.data.features["layout"]["obs"], {"available": True, "interactiveLimit": 50000}, ) def test_layout(self): @@ -131,14 +120,13 @@ class EngineTest(unittest.TestCase): self.assertEqual(annotations["n_cols"], 5) obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"] self.assertEqual( - annotations["col_idx"], - [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], + annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"], ) fbs = self.data.annotation_to_fbs_matrix("var") annotations = decode_fbs.decode_matrix_FBS(fbs) - self.assertEqual(annotations['n_rows'], 1838) - self.assertEqual(annotations['n_cols'], 2) + self.assertEqual(annotations["n_rows"], 1838) + self.assertEqual(annotations["n_cols"], 2) var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"]) @@ -146,13 +134,13 @@ class EngineTest(unittest.TestCase): fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"]) annotations = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(annotations["n_rows"], 2638) - self.assertEqual(annotations['n_cols'], 2) + self.assertEqual(annotations["n_cols"], 2) var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name]) annotations = decode_fbs.decode_matrix_FBS(fbs) - self.assertEqual(annotations['n_rows'], 1838) - self.assertEqual(annotations['n_cols'], 1) + self.assertEqual(annotations["n_rows"], 1838) + self.assertEqual(annotations["n_cols"], 1) def test_annotation_put(self): with self.assertRaises(DisabledFeatureError): @@ -177,27 +165,19 @@ class EngineTest(unittest.TestCase): self.data.data_frame_to_fbs_matrix(None, "obs") def test_filtered_data_frame(self): - filter_ = { - "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}} - } + filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}} fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) self.assertEqual(data["n_cols"], 1040) - filter_ = { - "filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}} - } + filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}} with self.assertRaises(FilterError): self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") def test_data_named_gene(self): var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"] - filter_ = { - "filter": { - "var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]} - } - } + filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}} fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) self.assertEqual(data["n_rows"], 2638) @@ -205,9 +185,7 @@ class EngineTest(unittest.TestCase): self.assertEqual(data["col_idx"], [4]) filter_ = { - "filter": { - "var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]} - } + "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}} } fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var") data = decode_fbs.decode_matrix_FBS(fbs) diff --git a/server/test/test_scanpy_engine_data_load.py b/server/test/test_scanpy_engine_data_load.py index 2be0b400..65d6522f 100644 --- a/server/test/test_scanpy_engine_data_load.py +++ b/server/test/test_scanpy_engine_data_load.py @@ -10,8 +10,9 @@ class DataLoadEngineTest(unittest.TestCase): """ Test file loading, including deferred loading/update. """ + def setUp(self): - self.data_file = DataLocator("example-dataset/pbmc3k.h5ad") + self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad") self.data = ScanpyEngine() def test_init(self): @@ -29,7 +30,7 @@ class DataLoadEngineTest(unittest.TestCase): "annotations_output_dir": None, "backed": False, "diffexp_may_be_slow": False, - "disable_diffexp": False + "disable_diffexp": False, } self.data.update(args=args) self.assertEqual(args, self.data.config) @@ -60,6 +61,7 @@ class DataLocatorEngineTest(unittest.TestCase): """ Test various types of data locators we expect to consume """ + def setUp(self): self.args = { "layout": ["umap"], @@ -76,7 +78,7 @@ class DataLocatorEngineTest(unittest.TestCase): self.assertEqual(data.gene_count, 1838) def test_posix_file(self): - locator = DataLocator("example-dataset/pbmc3k.h5ad") + locator = DataLocator("../example-dataset/pbmc3k.h5ad") data = ScanpyEngine(locator, self.args) self.stdAsserts(data) diff --git a/server/test/test_writable_annotation.py b/server/test/test_writable_annotation.py index 5df57f1c..c659e00a 100644 --- a/server/test/test_writable_annotation.py +++ b/server/test/test_writable_annotation.py @@ -25,9 +25,9 @@ class WritableAnnotationTest(unittest.TestCase): "diffexp_lfc_cutoff": 0.01, "annotations": True, "annotations_file": self.annotations_file, - "annotations_output_dir": None + "annotations_output_dir": None, } - self.data = ScanpyEngine(DataLocator("example-dataset/pbmc3k.h5ad"), args) + self.data = ScanpyEngine(DataLocator("../example-dataset/pbmc3k.h5ad"), args) def tearDown(self): shutil.rmtree(self.tmpDir) @@ -40,9 +40,7 @@ class WritableAnnotationTest(unittest.TestCase): # verify that the expected errors are generated n_rows = self.data.data.obs.shape[0] - fbs_bad = self.make_fbs({ - 'louvain': pd.Series(['undefined' for l in range(0, n_rows)], dtype='category') - }) + fbs_bad = self.make_fbs({"louvain": pd.Series(["undefined" for l in range(0, n_rows)], dtype="category")}) # ensure attempt to change VAR annotation with self.assertRaises(ValueError): @@ -55,32 +53,36 @@ class WritableAnnotationTest(unittest.TestCase): def test_write_to_file(self): # verify the file is written as expected n_rows = self.data.data.obs.shape[0] - fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), - 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') - }) + fbs = self.make_fbs( + { + "cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"), + "cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"), + } + ) res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.annotations_file)) - df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') + df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#") self.assertEqual(df.shape, (n_rows, 2)) - self.assertEqual(set(df.columns), set(['cat_A', 'cat_B'])) + self.assertEqual(set(df.columns), set(["cat_A", "cat_B"])) self.assertTrue(self.data.original_obs_index.equals(df.index)) - self.assertTrue(np.all(df['cat_A'] == ['label_A' for l in range(0, n_rows)])) - self.assertTrue(np.all(df['cat_B'] == ['label_B' for l in range(0, n_rows)])) + self.assertTrue(np.all(df["cat_A"] == ["label_A" for l in range(0, n_rows)])) + self.assertTrue(np.all(df["cat_B"] == ["label_B" for l in range(0, n_rows)])) # verify complete overwrite on second attempt, AND rotation occurs - fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A1' for l in range(0, n_rows)], dtype='category'), - 'cat_C': pd.Series(['label_C' for l in range(0, n_rows)], dtype='category') - }) + fbs = self.make_fbs( + { + "cat_A": pd.Series(["label_A1" for l in range(0, n_rows)], dtype="category"), + "cat_C": pd.Series(["label_C" for l in range(0, n_rows)], dtype="category"), + } + ) res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) self.assertTrue(path.exists(self.annotations_file)) - df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment='#') - self.assertEqual(set(df.columns), set(['cat_A', 'cat_C'])) - self.assertTrue(np.all(df['cat_A'] == ['label_A1' for l in range(0, n_rows)])) - self.assertTrue(np.all(df['cat_C'] == ['label_C' for l in range(0, n_rows)])) + df = pd.read_csv(self.annotations_file, index_col=0, header=0, comment="#") + self.assertEqual(set(df.columns), set(["cat_A", "cat_C"])) + self.assertTrue(np.all(df["cat_A"] == ["label_A1" for l in range(0, n_rows)])) + self.assertTrue(np.all(df["cat_C"] == ["label_C" for l in range(0, n_rows)])) # rotation name, ext = path.splitext(self.annotations_file) @@ -92,10 +94,12 @@ class WritableAnnotationTest(unittest.TestCase): def test_file_rotation_to_max_9(self): # verify we stop rotation at 9 n_rows = self.data.data.obs.shape[0] - fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), - 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') - }) + fbs = self.make_fbs( + { + "cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"), + "cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"), + } + ) for i in range(0, 11): res = self.data.annotation_put_fbs("obs", fbs) self.assertEqual(res, json.dumps({"status": "OK"})) @@ -111,10 +115,12 @@ class WritableAnnotationTest(unittest.TestCase): # GET (annotation_to_fbs_matrix) n_rows = self.data.data.obs.shape[0] - fbs = self.make_fbs({ - 'cat_A': pd.Series(['label_A' for l in range(0, n_rows)], dtype='category'), - 'cat_B': pd.Series(['label_B' for l in range(0, n_rows)], dtype='category') - }) + fbs = self.make_fbs( + { + "cat_A": pd.Series(["label_A" for l in range(0, n_rows)], dtype="category"), + "cat_B": pd.Series(["label_B" for l in range(0, n_rows)], dtype="category"), + } + ) # put res = self.data.annotation_put_fbs("obs", fbs) @@ -128,28 +134,21 @@ class WritableAnnotationTest(unittest.TestCase): self.assertEqual(annotations["n_rows"], n_rows) self.assertEqual(annotations["n_cols"], 7) self.assertIsNone(annotations["row_idx"]) - self.assertEqual(annotations["col_idx"], [ - obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B" - ]) + self.assertEqual( + annotations["col_idx"], + [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"], + ) col_idx = annotations["col_idx"] - self.assertEqual(annotations["columns"][col_idx.index('cat_A')], [ - 'label_A' for l in range(0, n_rows) - ]) - self.assertEqual(annotations["columns"][col_idx.index('cat_B')], [ - 'label_B' for l in range(0, n_rows) - ]) + self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A" for l in range(0, n_rows)]) + self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B" for l in range(0, n_rows)]) # verify the schema was updated all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} - self.assertEqual(all_col_schema["cat_A"], { - "name": "cat_A", - "type": "categorical", - "categories": ["label_A"], - "writable": True - }) - self.assertEqual(all_col_schema["cat_B"], { - "name": "cat_B", - "type": "categorical", - "categories": ["label_B"], - "writable": True - }) + self.assertEqual( + all_col_schema["cat_A"], + {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True}, + ) + self.assertEqual( + all_col_schema["cat_B"], + {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True}, + ) diff --git a/travis-build.sh b/travis-build.sh deleted file mode 100755 index 9f043d0e..00000000 --- a/travis-build.sh +++ /dev/null @@ -1,5 +0,0 @@ -set -eo pipefail -flake8 server -npm run --prefix client/ build -npm run --prefix client/ unit-test -pytest -s server/test