diff --git a/.github/PULL_REQUEST_TEMPLATE/pull request.md b/.github/PULL_REQUEST_TEMPLATE/pull request.md new file mode 100644 index 00000000..d50c2d6c --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pull request.md @@ -0,0 +1,11 @@ +#### Reviewers +**Functional:** + +**Readability:** + +--- + +## Changes +- add +- remove +- modify diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..0163eec2 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,67 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL Scan" + +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + schedule: + - cron: '0 8 * * *' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + language: [ 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] + # Learn more: + # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml index bd3c41e8..fff70efa 100644 --- a/.github/workflows/compatibility_tests.yml +++ b/.github/workflows/compatibility_tests.yml @@ -25,10 +25,11 @@ jobs: cellxgene-main-with-python-and-anndata-versions: name: python versions x anndata versions runs-on: ubuntu-latest + continue-on-error: true strategy: matrix: - python-version: [3.6, 3.7, 3.8] - anndata-version: [0.6.22.post1, 0.7.1] + python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8 + anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4] test-suite: [smoke-test, smoke-test-annotations] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index b4cfe599..70fcf947 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -31,9 +31,10 @@ jobs: - name: Install dependencies run: | pip install flake8 + pip install black cd client npm install - - name: Lint with flake8 + - name: Format with black and lint with flake8 run: | make lint-server - name: Lint src with eslint diff --git a/.github/workflows/scale-test.yml b/.github/workflows/scale-test.yml new file mode 100644 index 00000000..6e86de0e --- /dev/null +++ b/.github/workflows/scale-test.yml @@ -0,0 +1,30 @@ +name: "Scale test cellxgene APIs for initial loading" + +on: + schedule: + - cron: "0 0 * * Sun" + +jobs: + locust-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install dependencies + run: | + pip install -r server/test/locust/requirements-locust.txt + - name: Dev Scale Test + run: | + locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt + - name: Slack success webhook + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + run: | + DEV_STATS=$(tail -n 61 locust_dev_stats.txt) + DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`" + curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK + + diff --git a/LICENSE.txt b/LICENSE.txt index e0bb8c7d..a34ab341 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013 +Copyright (c) 2017-2020 Chan Zuckerberg Initiative Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in @@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in index 7d059a83..3a3b5b2b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,5 @@ recursive-include server/common/web/static * include server/requirements.txt include server/requirements-prepare.txt +include server/converters/schema/hgnc_complete_set.txt.gz +include server/converters/schema/schema_definitions/* diff --git a/Makefile b/Makefile index 5b588b77..8f9a4083 100644 --- a/Makefile +++ b/Makefile @@ -82,8 +82,9 @@ fmt-py: lint: lint-server lint-client .PHONY: lint-server -lint-server: - flake8 server +lint-server: fmt-py + flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501' + .PHONY: lint-client lint-client: diff --git a/README.md b/README.md index e2cb3215..669d2675 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg ### Contributing -We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. +We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com. @@ -81,7 +81,7 @@ If you believe you have found a security issue, we would appreciate notification # Inspiration -We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data. +We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data. We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation. diff --git a/client/Makefile b/client/Makefile index 3b36337a..1a8d3a81 100644 --- a/client/Makefile +++ b/client/Makefile @@ -3,6 +3,8 @@ include ../common.mk ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv) ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS)) +CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml) + # Packaging .PHONY: clean clean: @@ -31,9 +33,9 @@ start-frontend: .PHONY: smoke-test smoke-test: start_server_and_test \ - 'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \ + 'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \ $(CXG_SERVER_PORT) \ - 'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false' + 'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false' # start an instance of cellxgene and run the end-to-end annotations tests .PHONY: smoke-test-annotations diff --git a/client/__tests__/e2e/__snapshots__/e2e.test.js.snap b/client/__tests__/e2e/__snapshots__/e2e.test.js.snap index e7207fb7..d33b3262 100644 --- a/client/__tests__/e2e/__snapshots__/e2e.test.js.snap +++ b/client/__tests__/e2e/__snapshots__/e2e.test.js.snap @@ -1,5 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`did launch page launched 1`] = `"pbmc3kc3k"`; +exports[`did launch page launched 1`] = `"pbmc3kc3k"`; -exports[`metadata loads categories and values from dataset appear 1`] = `"
louvainvain
tint
"`; +exports[`metadata loads categories and values from dataset appear 1`] = `"
louvainvain
tint
"`; diff --git a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap index 67b743ce..56a349b2 100644 --- a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap +++ b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap @@ -2,22 +2,22 @@ exports[`annotations stacked bar graph renders 1`] = ` Array [ - "
TEST-LABELLABEL
0
", - "
unassignedigned
2133
", + "
TEST-LABELLABEL
0
", + "
unassignedigned
2133
", ] `; exports[`annotations stacked bar graph renders 2`] = ` Array [ - "
TEST-LABELLABEL
0
", - "
unassignedigned
2638
", + "
TEST-LABELLABEL
0
", + "
unassignedigned
2638
", ] `; -exports[`annotations truncate midpoint whitespace 1`] = `"123 456 456"`; +exports[`annotations truncate midpoint whitespace 1`] = `"123 456 456"`; -exports[`annotations truncate midpoint whitespace 2`] = `"123 456 456"`; +exports[`annotations truncate midpoint whitespace 2`] = `"123 456 456"`; -exports[`annotations truncate single character 1`] = `"T"`; +exports[`annotations truncate single character 1`] = `"T"`; -exports[`annotations truncate single character 2`] = `"T"`; +exports[`annotations truncate single character 2`] = `"T"`; diff --git a/client/__tests__/e2e/cellxgeneActions.js b/client/__tests__/e2e/cellxgeneActions.js index eee4714c..7ffcc8cf 100644 --- a/client/__tests__/e2e/cellxgeneActions.js +++ b/client/__tests__/e2e/cellxgeneActions.js @@ -13,8 +13,11 @@ import { getTestClass, getTestId, isElementPresent, + goToPage, } from "./puppeteerUtils"; +import { appUrlBase } from "./config"; + export async function drag(testId, start, end, lasso = false) { const layout = await waitByID(testId); const elBox = await layout.boxModel(); @@ -312,4 +315,74 @@ export async function assertCategoryDoesNotExist(categoryName) { await expect(result).toBe(false); } + +export async function login() { + const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`; + const password = "Test1111"; + + await goToPage(appUrlBase); + + await clickOn("log-in"); + + // (thuang): Auth0 form is unstable and unsafe for input until verified + await waitUntilFormFieldStable('[name="email"]'); + + await expect(page).toFillForm("form", { + email, + password, + }); + + await Promise.all([ + page.waitForNavigation({ waitUntil: "networkidle0" }), + expect(page).toClick('[name="submit"]'), + ]); + + expect(page.url()).toContain(appUrlBase); +} + +export async function logout() { + await clickOnUntil("user-info", async () => { + await waitByID("log-out"); + await Promise.all([ + page.waitForNavigation({ waitUntil: "networkidle0" }), + clickOn("log-out"), + ]); + }); + + await waitByID("log-in"); +} + +async function waitUntilFormFieldStable(selector) { + const MAX_RETRY = 10; + const WAIT_FOR_MS = 200; + + const EXPECTED_VALUE = "aaa"; + + let retry = 0; + + while (retry < MAX_RETRY) { + try { + await expect(page).toFill(selector, EXPECTED_VALUE); + + const fieldHandle = await expect(page).toMatchElement(selector); + + const fieldValue = await page.evaluate( + (input) => input.value, + fieldHandle + ); + + expect(fieldValue).toBe(EXPECTED_VALUE); + + break; + } catch (error) { + retry += 1; + + await page.waitFor(WAIT_FOR_MS); + } + } + + if (retry === MAX_RETRY) { + throw Error("clickOnUntil() assertion failed!"); + } +} /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */ diff --git a/client/__tests__/e2e/e2e.test.js b/client/__tests__/e2e/e2e.test.js index 982a076a..7e044c40 100644 --- a/client/__tests__/e2e/e2e.test.js +++ b/client/__tests__/e2e/e2e.test.js @@ -17,6 +17,7 @@ import { goToPage, typeInto, waitByID, + clickOnUntil, } from "./puppeteerUtils"; import { @@ -31,6 +32,8 @@ import { runDiffExp, selectCategory, subset, + login, + logout, } from "./cellxgeneActions"; const data = datasets[DATASET]; @@ -518,4 +521,34 @@ test("lasso moves after pan", async () => { expect(panCount).toBe(initialCount); }); + +const describeIfCalledByMakeFileTarget = + process.env.CXG_AUTH_TYPE?.toLowerCase() === "test" + ? describe + : describe.skip; + +describeIfCalledByMakeFileTarget("auth buttons", () => { + test("login then logout", async () => { + await goToPage(appUrlBase); + await clickOnUntil("log-in", async () => { + await page.waitForNavigation({ waitUntil: "networkidle0" }); + await waitByID("user-info"); + }); + await logout(); + }); +}); + +const conditionalDescribe = + process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip; + +conditionalDescribe("AuthN Integration", () => { + it("logs in", async () => { + await login(); + }); + + it("logs out", async () => { + await login(); + await logout(); + }); +}); /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */ diff --git a/client/__tests__/e2e/puppeteer.setup.js b/client/__tests__/e2e/puppeteer.setup.js index 40453777..6c593b93 100644 --- a/client/__tests__/e2e/puppeteer.setup.js +++ b/client/__tests__/e2e/puppeteer.setup.js @@ -17,7 +17,9 @@ setDefaultOptions({ timeout: 20 * 1000 }); jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS); -(async () => { +beforeEach(async () => { + await jestPuppeteer.resetBrowser(); + const userAgent = await browser.userAgent(); await page.setUserAgent(`${userAgent}bot`); @@ -53,6 +55,4 @@ jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS); } } }); -})().catch((error) => { - console.error("puppeteer.setup.js error", error); }); diff --git a/client/__tests__/e2e/test_config.yaml b/client/__tests__/e2e/test_config.yaml new file mode 100644 index 00000000..11576878 --- /dev/null +++ b/client/__tests__/e2e/test_config.yaml @@ -0,0 +1,47 @@ +server: + app: + force_https: true + + # By default, cellxgene will serve api requests from the same base url as the webpage. + # In general api_base_url and web_base_url will not need to be set. + # There are two reasons to set these parameters: + # 1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login, + # which then redirects back to the web_base_url. If the web_base_url is not set, it will default to + # the api_base_url. If oauth authentication is used, the api_base_url must be set. + # For a local test (where the server runs on "http://localhost:"), then the api_base_url may be + # set to the string "local". + # 2. The cellxgene deploymnent is in an environment where the webpage and api have + # different base urls. In this case both api_base_url and web_base_url must be set. + # It is up to the server admin to ensure that the networking is setup correctly for this environment. + api_base_url: http://localhost:5005 + web_base_url: http://localhost:3000 + + authentication: + # The authentication types may be "none", "session", "oauth" + # none: No authentication support, features like user_annotations must not be enabled. + # session: A session based userid is automatically generated. (no params needed) + # oauth: oauth2 is used for authentication; parameters are defined in params_oauth. + type: test + +dataset: + app: + about_legal_tos: null + about_legal_privacy: null + + presentation: + max_categories: 1000 + custom_colors: true + + user_annotations: + enable: false + type: local_file_csv + local_file_csv: + directory: null + file: null + ontology: + enable: false + obo_location: null + + embeddings: + names: [] + enable_reembedding: false diff --git a/client/configuration/eslint/eslint.js b/client/configuration/eslint/eslint.js index bafb7d31..51b619db 100644 --- a/client/configuration/eslint/eslint.js +++ b/client/configuration/eslint/eslint.js @@ -4,6 +4,7 @@ module.exports = { extends: [ "airbnb", "plugin:eslint-comments/recommended", + "plugin:@blueprintjs/recommended", "plugin:compat/recommended", "plugin:prettier/recommended", "prettier/react", diff --git a/client/package-lock.json b/client/package-lock.json index 205d3600..e9a19bb0 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -4156,6 +4156,216 @@ "tslib": "~1.10.0" } }, + "@blueprintjs/eslint-plugin": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@blueprintjs/eslint-plugin/-/eslint-plugin-0.3.0.tgz", + "integrity": "sha512-bQEdE4ApEHxCDV8hT9uIxeRbDFKOtRLBT3/Zy3Ku+nowDAYl/8jwZKp6lJuR/nqvsfuIXTnVef6ivwdBEieQfA==", + "dev": true, + "requires": { + "@typescript-eslint/experimental-utils": "^4.2.0", + "eslint": "^7.9.0" + }, + "dependencies": { + "@typescript-eslint/experimental-utils": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.3.0.tgz", + "integrity": "sha512-cmmIK8shn3mxmhpKfzMMywqiEheyfXLV/+yPDnOTvQX/ztngx7Lg/OD26J8gTZfkLKUmaEBxO2jYP3keV7h2OQ==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/scope-manager": "4.3.0", + "@typescript-eslint/types": "4.3.0", + "@typescript-eslint/typescript-estree": "4.3.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^2.0.0" + } + }, + "@typescript-eslint/typescript-estree": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.3.0.tgz", + "integrity": "sha512-ZAI7xjkl+oFdLV/COEz2tAbQbR3XfgqHEGy0rlUXzfGQic6EBCR4s2+WS3cmTPG69aaZckEucBoTxW9PhzHxxw==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.3.0", + "@typescript-eslint/visitor-keys": "4.3.0", + "debug": "^4.1.1", + "globby": "^11.0.1", + "is-glob": "^4.0.1", + "lodash": "^4.17.15", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + } + }, + "acorn": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz", + "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==", + "dev": true + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "eslint": { + "version": "7.10.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz", + "integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@eslint/eslintrc": "^0.1.3", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "enquirer": "^2.3.5", + "eslint-scope": "^5.1.1", + "eslint-utils": "^2.1.0", + "eslint-visitor-keys": "^1.3.0", + "espree": "^7.3.0", + "esquery": "^1.2.0", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "progress": "^2.0.0", + "regexpp": "^3.1.0", + "semver": "^7.2.1", + "strip-ansi": "^6.0.0", + "strip-json-comments": "^3.1.0", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "dependencies": { + "eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + } + } + }, + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + }, + "espree": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz", + "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==", + "dev": true, + "requires": { + "acorn": "^7.4.0", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.3.0" + } + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "requires": { + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } + }, + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, "@blueprintjs/icons": { "version": "3.19.0", "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.19.0.tgz", @@ -4184,6 +4394,76 @@ "minimist": "^1.2.0" } }, + "@eslint/eslintrc": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz", + "integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==", + "dev": true, + "requires": { + "ajv": "^6.12.4", + "debug": "^4.1.1", + "espree": "^7.3.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.2.1", + "js-yaml": "^3.13.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "acorn": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz", + "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==", + "dev": true + }, + "ajv": { + "version": "6.12.5", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz", + "integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + }, + "espree": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz", + "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==", + "dev": true, + "requires": { + "acorn": "^7.4.0", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.3.0" + } + }, + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + } + } + }, "@hapi/address": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", @@ -5137,6 +5417,32 @@ } } }, + "@nodelib/fs.scandir": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", + "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "2.0.3", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz", + "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==", + "dev": true + }, + "@nodelib/fs.walk": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz", + "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==", + "dev": true, + "requires": { + "@nodelib/fs.scandir": "2.1.3", + "fastq": "^1.6.0" + } + }, "@npmcli/move-file": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.0.1.tgz", @@ -5489,6 +5795,22 @@ "eslint-utils": "^2.0.0" } }, + "@typescript-eslint/scope-manager": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.3.0.tgz", + "integrity": "sha512-cTeyP5SCNE8QBRfc+Lgh4Xpzje46kNUhXYfc3pQWmJif92sjrFuHT9hH4rtOkDTo/si9Klw53yIr+djqGZS1ig==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.3.0", + "@typescript-eslint/visitor-keys": "4.3.0" + } + }, + "@typescript-eslint/types": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.3.0.tgz", + "integrity": "sha512-Cx9TpRvlRjOppGsU6Y6KcJnUDOelja2NNCX6AZwtVHRzaJkdytJWMuYiqi8mS35MRNA3cJSwDzXePfmhU6TANw==", + "dev": true + }, "@typescript-eslint/typescript-estree": { "version": "2.34.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz", @@ -5512,6 +5834,24 @@ } } }, + "@typescript-eslint/visitor-keys": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.3.0.tgz", + "integrity": "sha512-xZxkuR7XLM6RhvLkgv9yYlTcBHnTULzfnw4i6+z2TGBLy9yljAypQaZl9c3zFvy7PNI7fYWyvKYtohyF8au3cw==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.3.0", + "eslint-visitor-keys": "^2.0.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz", + "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==", + "dev": true + } + } + }, "@webassemblyjs/ast": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz", @@ -6387,9 +6727,9 @@ } }, "bl": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", - "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", + "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", "dev": true, "requires": { "buffer": "^5.5.0", @@ -6397,16 +6737,6 @@ "readable-stream": "^3.4.0" }, "dependencies": { - "buffer": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz", - "integrity": "sha512-9FTEDjLjwoAkEwyMGDjYJQN2gfRgOKBKRfiglhvibGbpeeU/pQn1bJxQqm32OD/AIeEuHxU9roxXxg34Byp/Ww==", - "dev": true, - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, "readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -7633,6 +7963,15 @@ "xdg-basedir": "^3.0.0" }, "dependencies": { + "dot-prop": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.1.tgz", + "integrity": "sha512-l0p4+mIuJIua0mhxGoh4a+iNL9bmeK5DvnSVQa6T0OhrVmaEa1XScX5Etc673FePCJOArq/4Pa2cLGODUWTPOQ==", + "dev": true, + "requires": { + "is-obj": "^1.0.0" + } + }, "make-dir": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", @@ -8796,6 +9135,15 @@ } } }, + "dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "requires": { + "path-type": "^4.0.0" + } + }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -8935,12 +9283,20 @@ } }, "dot-prop": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", - "integrity": "sha512-tUMXrxlExSW6U2EXiiKGSBVdYgtV8qlHL+C10TsW4PURY/ic+eaysnSkwB4kA/mBlCyy/IKDJ+Lc3wbWeaXtuQ==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz", + "integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==", "dev": true, "requires": { - "is-obj": "^1.0.0" + "is-obj": "^2.0.0" + }, + "dependencies": { + "is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "dev": true + } } }, "duplexer3": { @@ -10187,6 +10543,31 @@ "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", "dev": true }, + "fast-glob": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz", + "integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.0", + "merge2": "^1.3.0", + "micromatch": "^4.0.2", + "picomatch": "^2.2.1" + }, + "dependencies": { + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + } + } + }, "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -10202,6 +10583,15 @@ "resolved": "https://registry.npmjs.org/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz", "integrity": "sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw==" }, + "fastq": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz", + "integrity": "sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==", + "dev": true, + "requires": { + "reusify": "^1.0.4" + } + }, "favicons": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/favicons/-/favicons-5.5.0.tgz", @@ -11159,6 +11549,28 @@ "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==" }, + "globby": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz", + "integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==", + "dev": true, + "requires": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.1.1", + "ignore": "^5.1.4", + "merge2": "^1.3.0", + "slash": "^3.0.0" + }, + "dependencies": { + "array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true + } + } + }, "got": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", @@ -11691,6 +12103,12 @@ "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=", "dev": true }, + "ignore": { + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", + "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", + "dev": true + }, "ignore-walk": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", @@ -14203,6 +14621,12 @@ "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true + }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -15073,8 +15497,7 @@ "pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", - "dev": true + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" }, "parallel-transform": { "version": "1.2.0", @@ -15265,6 +15688,12 @@ "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=", "dev": true }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true + }, "pbkdf2": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz", @@ -15596,20 +16025,10 @@ "vendors": "^1.0.0" }, "dependencies": { - "dot-prop": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", - "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==", - "dev": true, - "requires": { - "is-obj": "^2.0.0" - } - }, "is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", - "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", - "dev": true + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" }, "postcss-selector-parser": { "version": "3.1.2", @@ -15696,20 +16115,10 @@ "postcss-selector-parser": "^3.0.0" }, "dependencies": { - "dot-prop": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", - "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==", - "dev": true, - "requires": { - "is-obj": "^2.0.0" - } - }, "is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", - "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", - "dev": true + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" }, "postcss-selector-parser": { "version": "3.1.2", @@ -17088,6 +17497,12 @@ "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true + }, "rgb-regex": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/rgb-regex/-/rgb-regex-1.0.1.tgz", @@ -17123,6 +17538,12 @@ "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz", "integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==" }, + "run-parallel": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz", + "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==", + "dev": true + }, "run-queue": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz", @@ -18324,20 +18745,10 @@ "postcss-selector-parser": "^3.0.0" }, "dependencies": { - "dot-prop": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", - "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==", - "dev": true, - "requires": { - "is-obj": "^2.0.0" - } - }, "is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", - "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", - "dev": true + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==" }, "postcss-selector-parser": { "version": "3.1.2", diff --git a/client/package.json b/client/package.json index bc7a88b9..0c244c39 100644 --- a/client/package.json +++ b/client/package.json @@ -54,6 +54,7 @@ "is-number": "^7.0.0", "lodash": "^4.17.20", "memoize-one": "^5.1.1", + "pako": "^1.0.11", "react": "^16.13.1", "react-async": "^10.0.1", "react-dom": "^16.13.1", @@ -85,6 +86,7 @@ "@babel/preset-react": "^7.10.4", "@babel/register": "^7.10.5", "@babel/runtime": "^7.10.5", + "@blueprintjs/eslint-plugin": "^0.3.0", "@sentry/webpack-plugin": "^1.12.0", "babel-eslint": "^10.1.0", "babel-jest": "^26.1.0", diff --git a/client/server/development.js b/client/server/development.js index 0b3e3811..1530856e 100644 --- a/client/server/development.js +++ b/client/server/development.js @@ -1,5 +1,3 @@ -const path = require("path"); -const historyApiFallback = require("connect-history-api-fallback"); const chalk = require("chalk"); const express = require("express"); const favicon = require("serve-favicon"); @@ -11,35 +9,51 @@ const utils = require("./utils"); process.env.NODE_ENV = "development"; const CLIENT_PORT = process.env.CXG_CLIENT_PORT; +const { CXG_SERVER_PORT } = process.env; + +const API = { + prefix: `http://localhost:${CXG_SERVER_PORT}/`, +}; // Set up compiler const compiler = webpack(config); -compiler.plugin("invalid", () => { +compiler.hooks.invalid.tap("invalid", () => { utils.clearConsole(); console.log("Compiling..."); }); -compiler.plugin("done", (stats) => { +compiler.hooks.done.tap("done", (stats) => { utils.formatStats(stats, CLIENT_PORT); }); // Launch server const app = express(); -app.use(historyApiFallback({ verbose: false })); - app.use( devMiddleware(compiler, { logLevel: "warn", publicPath: config.output.publicPath, + index: true, }) ); app.use(favicon("./favicon.png")); -app.get("*", (req, res) => { - res.sendFile(path.resolve("index.html")); +app.get("/login", async (req, res) => { + try { + res.redirect(`${API.prefix}login?dataset=http://localhost:${CLIENT_PORT}`); + } catch (err) { + console.error(err); + } +}); + +app.get("/logout", async (req, res) => { + try { + res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`); + } catch (err) { + console.error(err); + } }); app.listen(CLIENT_PORT, (err) => { diff --git a/client/src/actions/annotation.js b/client/src/actions/annotation.js index 8a7a154b..136fe9b3 100644 --- a/client/src/actions/annotation.js +++ b/client/src/actions/annotation.js @@ -2,6 +2,7 @@ Action creators for user annotation */ import _ from "lodash"; +import pako from "pako"; import * as globals from "../globals"; import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager"; @@ -153,7 +154,7 @@ export const annotationCreateLabelInCategory = ( assignSelected ) => async (dispatch, getState) => { /* - Add a new label to a user-defined category. If assignSelected is true, assign + Add a new label to a user-defined category. If assignSelected is true, assign the label to all currently selected cells. */ const { @@ -347,6 +348,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => { const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix)); const matrix = MatrixFBS.encodeMatrixFBS(df); + const compressedMatrix = pako.deflate(matrix); try { const queryString = !dataCollectionNameIsReadOnly && !!dataCollectionName @@ -358,7 +360,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => { `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`, { method: "PUT", - body: matrix, + body: compressedMatrix, headers: new Headers({ "Content-Type": "application/octet-stream", }), diff --git a/client/src/actions/embedding.js b/client/src/actions/embedding.js index 615bcd68..7b781a0d 100644 --- a/client/src/actions/embedding.js +++ b/client/src/actions/embedding.js @@ -5,20 +5,23 @@ action creators related to embeddings choice import { AnnoMatrixObsCrossfilter } from "../annoMatrix"; import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers"; -export async function _switchEmbedding(prevAnnoMatrix, prevCrossfilter, newEmbeddingName) { +export async function _switchEmbedding( + prevAnnoMatrix, + prevCrossfilter, + newEmbeddingName +) { /* DRY helper used by this and reembedding action creators */ const base = prevAnnoMatrix.base(); const embeddingDf = await base.fetch("emb", newEmbeddingName); const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf); - const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix, prevCrossfilter.obsCrossfilter).select( - "emb", - newEmbeddingName, - { - mode: "all", - } - ); + const obsCrossfilter = await new AnnoMatrixObsCrossfilter( + annoMatrix, + prevCrossfilter.obsCrossfilter + ).select("emb", newEmbeddingName, { + mode: "all", + }); return [annoMatrix, obsCrossfilter]; } @@ -30,7 +33,10 @@ export const layoutChoiceAction = (newLayoutChoice) => async ( On layout choice, make sure we have selected all on the previous layout, AND the new layout. */ - const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState(); + const { + annoMatrix: prevAnnoMatrix, + obsCrossfilter: prevCrossfilter, + } = getState(); const [annoMatrix, obsCrossfilter] = await _switchEmbedding( prevAnnoMatrix, prevCrossfilter, diff --git a/client/src/actions/index.js b/client/src/actions/index.js index eb0638b2..5de720b0 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -43,12 +43,12 @@ async function configFetch(dispatch) { async function userInfoFetch(dispatch) { return fetchJson("userinfo").then((response) => { - const { userinfo } = response || {}; + const { userinfo: userInfo } = response || {}; dispatch({ - type: "userinfo load complete", - userinfo, + type: "userInfo load complete", + userInfo, }); - return userinfo; + return userInfo; }); } diff --git a/client/src/actions/reembed.js b/client/src/actions/reembed.js index 2212f380..bfcd4f5f 100644 --- a/client/src/actions/reembed.js +++ b/client/src/actions/reembed.js @@ -79,10 +79,14 @@ export function requestReembed() { type: "reembed: request completed", }); - const { annoMatrix: prevAnnoMatrix } = getState(); + const { + annoMatrix: prevAnnoMatrix, + obsCrossfilter: prevCrossfilter, + } = getState(); const base = prevAnnoMatrix.base().addEmbedding(schema); const [annoMatrix, obsCrossfilter] = await _switchEmbedding( base, + prevCrossfilter, schema.name ); dispatch({ diff --git a/client/src/components/autosave/filenameDialog.js b/client/src/components/autosave/filenameDialog.js index 3937ecee..3ff7e500 100644 --- a/client/src/components/autosave/filenameDialog.js +++ b/client/src/components/autosave/filenameDialog.js @@ -3,18 +3,19 @@ import { connect } from "react-redux"; import { Button, - Tooltip, - InputGroup, - Dialog, Classes, + Code, Colors, + Dialog, + InputGroup, + Tooltip, } from "@blueprintjs/core"; @connect((state) => ({ idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null, annotations: state.annotations, auth: state.config?.authentication, - userinfo: state.userinfo, + userInfo: state.userInfo, writableCategoriesEnabled: state.config?.parameters?.annotations ?? false, })) class FilenameDialog extends React.Component { @@ -96,7 +97,7 @@ class FilenameDialog extends React.Component { writableCategoriesEnabled, annotations, idhash, - userinfo, + userInfo, } = this.props; const { filenameText } = this.state; @@ -104,7 +105,7 @@ class FilenameDialog extends React.Component { annotations.promptForFilename && !annotations.dataCollectionNameIsReadOnly && !annotations.dataCollectionName && - userinfo.is_authenticated ? ( + userInfo.is_authenticated ? (

Your annotations are stored in this file: - + {filenameText}-{idhash}.csv - +

(We added a unique ID to your filename) diff --git a/client/src/components/categorical/category/index.js b/client/src/components/categorical/category/index.js index 6fc628f1..a9649660 100644 --- a/client/src/components/categorical/category/index.js +++ b/client/src/components/categorical/category/index.js @@ -1,7 +1,13 @@ import React, { useRef, useEffect } from "react"; import { connect, shallowEqual } from "react-redux"; import { FaChevronRight, FaChevronDown } from "react-icons/fa"; -import { AnchorButton, Button, Tooltip, Position } from "@blueprintjs/core"; +import { + AnchorButton, + Button, + Classes, + Position, + Tooltip, +} from "@blueprintjs/core"; import { Flipper, Flipped } from "react-flip-toolkit"; import Async from "react-async"; import memoize from "memoize-one"; @@ -301,9 +307,12 @@ const StillLoading = ({ metadataField, checkboxID }) => { alignItems: "flex-start", }} > -

{ - this.ref = ref; - }} style={{ - display: colorAccessor ? "inherit" : "none", position: "absolute", left: 8, top: 35, zIndex: 1, + pointerEvents: "none", }} /> ); diff --git a/client/src/components/embedding/index.js b/client/src/components/embedding/index.js index d4c7562c..e60d93e0 100644 --- a/client/src/components/embedding/index.js +++ b/client/src/components/embedding/index.js @@ -2,13 +2,14 @@ import React from "react"; import { connect } from "react-redux"; import { useAsync } from "react-async"; import { - ButtonGroup, - Popover, Button, + ButtonGroup, + H4, + Popover, + Position, Radio, RadioGroup, Tooltip, - Position, } from "@blueprintjs/core"; import * as globals from "../../globals"; import actions from "../../actions"; @@ -80,7 +81,7 @@ class Embedding extends React.PureComponent { width: 400, }} > -

Embedding Choice

+

Embedding Choice

There are {schema?.dataframe?.nObs} cells in the entire dataset.

diff --git a/client/src/components/geneExpression/menus/addGenes.js b/client/src/components/geneExpression/menus/addGenes.js index 73a920ce..6fec3ec2 100644 --- a/client/src/components/geneExpression/menus/addGenes.js +++ b/client/src/components/geneExpression/menus/addGenes.js @@ -6,11 +6,12 @@ import fuzzysort from "fuzzysort"; import { connect } from "react-redux"; import { Suggest } from "@blueprintjs/select"; import { - MenuItem, Button, + ControlGroup, FormGroup, InputGroup, - ControlGroup, + Intent, + MenuItem, } from "@blueprintjs/core"; import * as globals from "../../../globals"; import actions from "../../../actions"; @@ -278,7 +279,7 @@ class AddGenes extends React.Component { popoverProps={{ minimal: true }} /> - + gene + +
+
+ + + +
); } diff --git a/client/src/components/menubar/authButtons.js b/client/src/components/menubar/authButtons.js index 7eb4c4c0..91af0b42 100644 --- a/client/src/components/menubar/authButtons.js +++ b/client/src/components/menubar/authButtons.js @@ -1,32 +1,175 @@ -import React from "react"; -import { AnchorButton, Tooltip } from "@blueprintjs/core"; +import React, { useState } from "react"; + +import { + AnchorButton, + Button, + MenuItem, + Tooltip, + Popover, + Menu, + Elevation, + PopoverPosition, + Checkbox, + Card, +} from "@blueprintjs/core"; + +import { IconNames } from "@blueprintjs/icons"; + import * as globals from "../../globals"; + import styles from "./menubar.css"; +import { storageGet, storageSet, KEYS } from "../util/localStorage"; + +const BASE_EMOJI = [0x1f9d1, 0x1f468, 0x1f469]; +const SKIN_TONES = [0x1f3fb, 0x1f3fc, 0x1f3fd, 0x1f3fe, 0x1f3ff]; +const MICROSCOPE = 0x1f52c; +const ZERO_WIDTH_JOINER = 0x0200d; + +const LOGIN_PROMPT_OFF = "off"; + const Auth = React.memo((props) => { - const { auth, userinfo } = props; + const [isPromptOpen, setIsPromptOpen] = useState(shouldShowPrompt()); - if (!auth || (auth && !auth.requires_client_login)) return null; + const { auth, userInfo } = props; - return ( -
- - - {!userinfo.is_authenticated ? "Log In" : "Log Out"} - - -
+ const isAuthenticated = userInfo && userInfo.is_authenticated; + + window.userInfo = userInfo; + + const randomInt = Math.random() * 15; + const sexIndex = Math.floor(randomInt / 5); + const skinToneIndex = Math.floor(randomInt % 5); + + const scientist = String.fromCodePoint( + BASE_EMOJI[sexIndex], + SKIN_TONES[skinToneIndex], + ZERO_WIDTH_JOINER, + MICROSCOPE ); + + if (!shouldShowAuth()) return null; + + if (isAuthenticated) { + const PopoverContent = ( + + + + + ); + + return ( + + + + ); + } + + const LoginButton = ( + + + Log In + + + ); + + if (isPromptOpen) { + return ( + } + onInteraction={setIsPromptOpen} + > + {LoginButton} + + ); + } + + return LoginButton; + + function shouldShowAuth() { + return auth && auth.requires_client_login; + } + + function shouldShowPrompt() { + if (storageGet(KEYS.LOGIN_PROMPT) === LOGIN_PROMPT_OFF) return false; + + return shouldShowAuth && !isAuthenticated; + } }); +function PromptContent({ setIsPromptOpen }) { + const [isChecked, setIsChecked] = useState(false); + + function handleOKClick() { + if (isChecked) { + storageSet(KEYS.LOGIN_PROMPT, LOGIN_PROMPT_OFF); + } + + setIsPromptOpen(false); + } + + function handleCheckboxChange() { + setIsChecked(!isChecked); + } + + return ( + +

+ Logging in will enable you to create your own categories and labels. + Logging in later will reset cellxgene to the default view and cause you + to lose progress. +

+ + Do not show me this message again + +
+ +
+ + ); +} + export default Auth; diff --git a/client/src/components/menubar/clip.js b/client/src/components/menubar/clip.js index 5b866bd0..5bc1d18d 100644 --- a/client/src/components/menubar/clip.js +++ b/client/src/components/menubar/clip.js @@ -1,12 +1,16 @@ import React from "react"; import { - Position, Button, - Popover, - NumericInput, + ButtonGroup, Icon, + Intent, + NumericInput, + Popover, + Position, Tooltip, } from "@blueprintjs/core"; +import { IconNames } from "@blueprintjs/icons"; + import { tooltipHoverOpenDelay } from "../../globals"; import styles from "./menubar.css"; @@ -28,13 +32,13 @@ const Clip = React.memo((props) => { pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin; const clipMax = pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax; - const activeClipClass = + const intent = clipPercentileMin > 0 || clipPercentileMax < 100 - ? " bp3-intent-warning" - : ""; + ? Intent.INTENT_WARNING + : Intent.NONE; return ( -
+ {
} /> - + ); }); diff --git a/client/src/components/menubar/index.js b/client/src/components/menubar/index.js index 3fd840c0..cbfeec5e 100644 --- a/client/src/components/menubar/index.js +++ b/client/src/components/menubar/index.js @@ -6,8 +6,8 @@ import * as globals from "../../globals"; import styles from "./menubar.css"; import actions from "../../actions"; import Clip from "./clip"; + import AuthButtons from "./authButtons"; -import InformationMenu from "./infoMenu"; import Subset from "./subset"; import UndoRedoReset from "./undoRedo"; import DiffexpButtons from "./diffexpButtons"; @@ -42,7 +42,7 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers"; celllist2: state.differential.celllist2, libraryVersions: state.config?.["library_versions"], auth: state.config?.authentication, - userinfo: state.userinfo, + userInfo: state.userInfo, undoDisabled: state["@@undoable/past"].length === 0, redoDisabled: state["@@undoable/future"].length === 0, aboutLink: state.config?.links?.["about-dataset"], @@ -204,7 +204,6 @@ class MenuBar extends React.PureComponent { render() { const { dispatch, - libraryVersions, disableDiffexp, undoDisabled, redoDisabled, @@ -212,17 +211,14 @@ class MenuBar extends React.PureComponent { clipPercentileMin, clipPercentileMax, graphInteractionMode, - aboutLink, showCentroidLabels, - privacyURL, - tosURL, categoricalSelection, colorAccessor, subsetPossible, subsetResetPossible, enableReembedding, + userInfo, auth, - userinfo, } = this.props; const { pendingClipPercentiles } = this.state; @@ -248,10 +244,7 @@ class MenuBar extends React.PureComponent { zIndex: 3, }} > - - + { - dispatch({ type: "toggle dataset drawer" }); -}; - -const InformationMenu = React.memo((props) => { - const { libraryVersions, tosURL, privacyURL, dispatch } = props; - return ( -
- - handleClick(dispatch)} - icon={IconNames.BOOK} - text="Dataset Overview" - /> - - - - - - - {tosURL ? ( - - ) : null} - {privacyURL ? ( - - ) : null} - - } - position={Position.BOTTOM_RIGHT} - > -
- ); -}); - -export default InformationMenu; diff --git a/client/src/components/menubar/undoRedo.js b/client/src/components/menubar/undoRedo.js index 601dba32..8591505c 100644 --- a/client/src/components/menubar/undoRedo.js +++ b/client/src/components/menubar/undoRedo.js @@ -1,12 +1,13 @@ import React from "react"; -import { AnchorButton, Tooltip } from "@blueprintjs/core"; +import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core"; +import { IconNames } from "@blueprintjs/icons"; import { tooltipHoverOpenDelay } from "../../globals"; import styles from "./menubar.css"; const UndoRedo = React.memo((props) => { const { undoDisabled, redoDisabled, dispatch } = props; return ( -
+ { > { dispatch({ type: "@@undoable/undo" }); @@ -32,7 +33,7 @@ const UndoRedo = React.memo((props) => { > { dispatch({ type: "@@undoable/redo" }); @@ -43,7 +44,7 @@ const UndoRedo = React.memo((props) => { data-testid="redo" /> -
+ ); }); diff --git a/client/src/components/miniHistogram/index.js b/client/src/components/miniHistogram/index.js index eb57f832..59153d91 100644 --- a/client/src/components/miniHistogram/index.js +++ b/client/src/components/miniHistogram/index.js @@ -72,7 +72,6 @@ export default class MiniHistogram extends React.PureComponent { popoverClassName={Classes.POPOVER_CONTENT_SIZING} > ({ tosURL: state.config?.parameters?.["about_legal_tos"], @@ -37,7 +18,7 @@ class TermsPrompt extends React.PureComponent { constructor(props) { super(props); const { tosURL, privacyURL } = this.props; - const cookieDecision = storageGet(CookieDecision, null); + const cookieDecision = storageGet(KEYS.COOKIE_DECISION, null); const hasDecided = cookieDecision !== null; this.state = { hasDecided, @@ -55,7 +36,7 @@ class TermsPrompt extends React.PureComponent { handleOK = () => { this.setState({ isOpen: false }); - storageSet(CookieDecision, "yes"); + storageSet(KEYS.COOKIE_DECISION, "yes"); if (window.cookieDecisionCallback instanceof Function) { try { window.cookieDecisionCallback(); @@ -67,7 +48,7 @@ class TermsPrompt extends React.PureComponent { handleNo = () => { this.setState({ isOpen: false }); - storageSet(CookieDecision, "no"); + storageSet(KEYS.COOKIE_DECISION, "no"); }; renderTos() { diff --git a/client/src/components/util/localStorage.js b/client/src/components/util/localStorage.js new file mode 100644 index 00000000..5766d9f9 --- /dev/null +++ b/client/src/components/util/localStorage.js @@ -0,0 +1,22 @@ +export const KEYS = { + COOKIE_DECISION: "cxg.cookieDecision", + LOGIN_PROMPT: "cxg.LOGIN_PROMPT", +}; + +export function storageGet(key, defaultValue = null) { + try { + const val = window.localStorage.getItem(key); + if (val === null) return defaultValue; + return val; + } catch (e) { + return defaultValue; + } +} + +export function storageSet(key, value) { + try { + window.localStorage.setItem(key, value); + } catch { + // continue + } +} diff --git a/client/src/components/util/truncate.js b/client/src/components/util/truncate.js index da06a58f..33e64dcc 100644 --- a/client/src/components/util/truncate.js +++ b/client/src/components/util/truncate.js @@ -7,6 +7,8 @@ const SPLIT_STYLE = { display: "flex", overflow: "hidden", justifyContent: "flex-start", + width: "100%", // There are probably additional styles that we don't want to stack + padding: 0, }; const FIRST_HALF_STYLE = { @@ -40,7 +42,7 @@ export default (props) => { ) { throw Error("Only pass a single child with text to Truncate"); } - const originalString = children.props.children; + const originalString = String(children.props.children); let firstString; let secondString; @@ -58,7 +60,7 @@ export default (props) => { } } - const inheritedColor = children.props.style.color; + const inheritedColor = children.props.style?.color; const splitStyle = { ...children.props.style, ...SPLIT_STYLE }; const secondHalfContentStyle = { @@ -93,6 +95,7 @@ export default (props) => { preventOverflow: { enabled: false }, hide: { enabled: false }, }} + targetProps={{ style: children.props.style }} > {newChildren} diff --git a/client/src/reducers/index.js b/client/src/reducers/index.js index b5c34b2a..a16af6f4 100644 --- a/client/src/reducers/index.js +++ b/client/src/reducers/index.js @@ -4,7 +4,7 @@ import thunk from "redux-thunk"; import cascadeReducers from "./cascade"; import undoable from "./undoable"; import config from "./config"; -import userinfo from "./userinfo"; +import userInfo from "./userinfo"; import annoMatrix from "./annoMatrix"; import obsCrossfilter from "./obsCrossfilter"; import categoricalSelection from "./categoricalSelection"; @@ -44,7 +44,7 @@ const Reducer = undoable( ["pointDilation", pointDialation], ["reembedController", reembedController], ["autosave", autosave], - ["userinfo", userinfo], + ["userInfo", userInfo], ]), [ "annoMatrix", diff --git a/client/src/reducers/userinfo.js b/client/src/reducers/userInfo.js similarity index 83% rename from client/src/reducers/userinfo.js rename to client/src/reducers/userInfo.js index 4765c822..6939e949 100644 --- a/client/src/reducers/userinfo.js +++ b/client/src/reducers/userInfo.js @@ -1,4 +1,3 @@ -// jshint esversion: 6 const UserInfo = (state = {}, action) => { switch (action.type) { case "initial data load start": @@ -7,12 +6,12 @@ const UserInfo = (state = {}, action) => { loading: true, error: null, }; - case "userinfo load complete": + case "userInfo load complete": return { ...state, loading: false, error: null, - ...action.userinfo, + ...action.userInfo, }; case "initial data load error": return { diff --git a/client/src/util/stateManager/colorHelpers.js b/client/src/util/stateManager/colorHelpers.js index 4eaea2b7..d324dd37 100644 --- a/client/src/util/stateManager/colorHelpers.js +++ b/client/src/util/stateManager/colorHelpers.js @@ -106,15 +106,19 @@ export function loadUserColorConfig(userColors) { return -1; }) .reduce( - (acc, label, i) => { + (acc, label) => { const color = parseRGB(userColors[category][label]); acc[0][label] = color; - acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]); + acc[1][label] = d3.rgb( + 255 * color[0], + 255 * color[1], + 255 * color[2] + ); return acc; }, [{}, {}] ); - const scale = (i) => scaleMap[i]; + const scale = (label) => scaleMap[label]; convertedUserColors[category] = { colors, scale }; }); return convertedUserColors; diff --git a/client/src/util/stateManager/matrix.js b/client/src/util/stateManager/matrix.js index e9a1e445..52717157 100644 --- a/client/src/util/stateManager/matrix.js +++ b/client/src/util/stateManager/matrix.js @@ -178,26 +178,26 @@ function promoteTypedArray(o) { */ if (isFpTypedArray(o) || Array.isArray(o)) return o; - let TyepdArrayCtor; + let TypedArrayCtor; switch (o.constructor) { case Int8Array: case Uint8Array: case Uint8ClampedArray: case Int16Array: case Uint16Array: - TyepdArrayCtor = Float32Array; + TypedArrayCtor = Float32Array; break; case Int32Array: case Uint32Array: - TyepdArrayCtor = Float64Array; + TypedArrayCtor = Float64Array; break; default: throw new Error("Unexpected data type returned from server."); } - if (o.constructor === TyepdArrayCtor) return o; - return new TyepdArrayCtor(o); + if (o.constructor === TypedArrayCtor) return o; + return new TypedArrayCtor(o); } export function matrixFBSToDataframe(arrayBuffers) { diff --git a/dev_docs/cxg.md b/dev_docs/cxg.md index e851692f..05bf465f 100644 --- a/dev_docs/cxg.md +++ b/dev_docs/cxg.md @@ -1,3 +1,5 @@ +## UPDATE (9/30/2020): Starting today, the name Corpora will only be used as the internal project name, with cellxgene Data Portal being the official product name + # CXG Data Format Specification Document Status: _draft_ diff --git a/dev_docs/schema_guide.md b/dev_docs/schema_guide.md new file mode 100644 index 00000000..cd70611a --- /dev/null +++ b/dev_docs/schema_guide.md @@ -0,0 +1,175 @@ +# Cellxgene Schema Guide + +Datasets included in the [data portal](https://cellxgene.cziscience.com/) and hosted cellxgene need to follow the schema +described [here](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md). That +schema defines some required fields, requirements about feature labels, and some optional fields that mostly help with +presentation. + +The number of fields is rather low, and we expect that information needed to populate those fields should either already +be present in datasets prepared by a submitter or be easy to obtain. However, this still leaves the task of actually +manipulating the dataset so that it follows the schema: adjusting field names, ensuring proper ontologies are used, +converting gene symbols to a common set, etc. This can be tedious and error-prone, and at the beginning of the hosted +cellxgene project, this was always done with engineering support. As we increase the rate at which we add data, we want +to eliminate the need for engineering support so that ultimately submitters themselves can create files that follow the +schema. + +## `cellxgene schema apply` + +To enable this, we have a new cellxgene subcommand, `cellxgene schema`, that handles applying and verifying the schema. +Its first subcommand, `cellxgene schema apply`, takes three inputs: + +1. A source h5ad file. The input needs to be an AnnData file, so if a submitter has, say, a serialized Seurat or + SingleCellExperiment object, it needs to be converted to AnnData first. This can be done with + [sceasy](https://github.com/cellgeni/sceasy) or via + [Seurat](https://satijalab.org/seurat/v3.1/conversion_vignette.html). +2. A configuration yaml file that describes the fields to add and conversions to apply (see below). +3. A name for the new h5ad file that should follow the schema. + +### Configuration yaml + +The configuration yaml file describes how to apply the schema. This is an example of a "skeleton" yaml that has all the +fields required for the 1.0.0 schema but is not yet filled in with any logic: + +``` +uns: + version: + corpora_schema_version: 1.0.0 + corpora_encoding_version: 0.1.0 + contributors: + title: + layer_descriptions: + preprint_doi: + publication_doi: + organism_ontology_term_id: +obs: + tissue_ontology_term_id: + assay_ontology_term_id: + disease_ontology_term_id: + cell_type_ontology_term_id: + sex: + ethnicity_ontology_term_id: + development_stage_ontology_term_id: +fixup_gene_symbols: +``` + +#### Unstructured metadata +The first section is `uns`, which includes metadata fields that describe the whole dataset (see +[here](https://anndata.readthedocs.io/en/latest/) for further description of `uns` and `obs`.). + +The first line is `version`, which is required for most of our tooling to work. The schema version is set at +1.0.0 in the example above, but of course for future versions that should be changed. + +Next is `contributors` which describes who is adding the dataset to the portal. If you consult the schema, you see that +contributors is a list where each element can have `name`, `email`, and `institution`. So when filled out, the +`contributors` field should look like this: + +``` +contributors: + - name: Mary B. Scientist + email: mbs@singlecell.edu + institution: Single-Cell University + - name: Robert J. Scientist + email: rjs@usingle.edu + institution: University of Single Cell +``` + +`title` is the name of the dataset, and is just a string that gets displayed in the portal and cellxgene to identify the +dataset. + +`layer_descriptions` is free text descriptions of the different +[layers](https://anndata.readthedocs.io/en/latest/anndata.AnnData.layers.html) of the AnnData file. It should look like +this when complete, depending on what layers are present: +``` +layer_descriptions: + X: CPM and logged + raw.X: raw +``` +Note that one of the layers needs to be "raw", that is, the AnnData file must contain raw counts. + +The two DOI fields are optional but can be included if the dataset is associated with a publication or preprint. Note +that the DOI should be a full url: +``` +publication_doi: https://doi.org/10.1073%2Fpnas.83.15.5372 +``` + +Finally, the `organism_ontology_term_id` field is the species of the donor organism from the NCBITaxon ontology. The +value for _Homo sapiens_ is `NCBITaxon:9606`: +``` +organism_ontology_term_id: NCBITaxon:9606 +``` +Note that the schema also requires a human-readable `organism` field, but this doesn't need to be included in the yaml. +When the `cellxgene schema apply` script encounters an ontology field, it looks up the label for the term(s) and inserts it +into the appropriate field. + + +#### Observation metadata +The next section is `obs`, which is metadata than can vary for each observation (and "observation" usually means cell). +These fields are all ontology fields except for `sex`, which has its own enumerated set of permitted values. + +There are two ways to fill in the `obs` fields. The first is useful when there is only one value for all the +observations in the dataset. This is not uncommon, for example all cells often come from the same assay. In that case +just insert the ontology term: +``` +assay_ontology_term_id: EFO:0009922 +``` + +The second is for when there is an existing field in the dataset that needs to be mapped to the schema field. For +example, the submitter may have included cell type annotations in a field called `CellType`, and those annotations may +just be free text. This doesn't follow the schema because it needs to be in `cell_type_ontology_term_id` and +`cell_type`, and it needs ontology terms and labels, not just any text. In that case the field can be a dictionary: + +``` +cell_type_ontology_term_id: + CellType: + t-cell: CL:0000084 + b-cell: CL:0000236 +``` + +This will look at the `obs.CellType` field in the dataset, and where it has the value "t-cell", it will insert +`CL:0000084` into `cell_type_ontology_term_id` and its label `T cell` into `cell_type`. + +Now there are often situations where there is no valid ontology term for some field. For example, the dataset may have +been produced via an assay not present in `EFO`. Or, a particular cell type may have no entry in `CL`. In that case, a +free text description can be used in the `ontology_term_id` field: + +``` +assay_ontology_term_id: Sci-Plex +cell_type_ontology_term_id: + CellType: + t-cell: CL:0000084 + b-cell: CL:0000236 + new cell type: new cell type +``` + +In these cases, the `cellxgene schema apply` script will leave the ontology field blank and move the free text +description into the label field. So the `assay_ontology_term_id` in the new dataset would be `""` but `assay` would be +`Sci-Plex`. + + +#### Gene symbol harmonization + +The last section describes how gene symbol conversion should be applied to each of the layers. This is similar to the +`layer_descriptions` field above, but there are only three permitted values: `raw`, `log1p`, and `sqrt`: + +``` +fixup_gene_symbols: + X: log1p + raw.X: raw +``` + +This tells the script how each each layer was transformed from raw values that can be directly summed. `raw` means that +the layer contains raw counts or some linear tranformation of raw counts. `log1p` means that the layer has `log(X + 1)` +for each the raw `X` values. `sqrt` means `sqrt(X)` (this is not common). For layers produced by Seurat's normalization +or SCTransform functions, the correct choice is usually `log1p`. + + +### `cellxgene schema validate` + +The next `cellxgene schema` subcommand is `cellxgene schema validate`, and it validates that a given h5ad follows a +version of the schema. It accepts two parameters: + +1. The h5ad file to check +2. The version of the schema to check against. + +If the validation succeeds, the command will have a zero exit code. If it does not, it will have a non-zero exit code +and will print validation failure messages. diff --git a/docs/_config.yml b/docs/_config.yml index 8ac27944..6e29ea64 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -13,6 +13,8 @@ nav: url: posts/install - title: Gallery url: posts/gallery + - title: Cellxgene data portal + url: https://cellxgene.cziscience.com/ - title: Demo datasets url: posts/demo-data - title: Preparing your data @@ -33,5 +35,3 @@ nav: url: posts/contribute - title: Contact & finding help url: posts/contact - - title: cellxgene.cziscience.com - url: posts/cellxgene_cziscience_com diff --git a/docs/_site/index.html b/docs/_site/index.html index 2a5ed4f1..49e763a0 100644 --- a/docs/_site/index.html +++ b/docs/_site/index.html @@ -7,7 +7,7 @@ Index | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/annotations.html b/docs/_site/posts/annotations.html index 2a76ec9f..09941877 100644 --- a/docs/_site/posts/annotations.html +++ b/docs/_site/posts/annotations.html @@ -7,7 +7,7 @@ annotations | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"annotations","description":"Creating annotations","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/contact.html b/docs/_site/posts/contact.html index 33c31b51..8040eed7 100644 --- a/docs/_site/posts/contact.html +++ b/docs/_site/posts/contact.html @@ -7,7 +7,7 @@ Contact | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Contact","description":"Contact","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/contribute.html b/docs/_site/posts/contribute.html index 494d2ca3..2af2e3e9 100644 --- a/docs/_site/posts/contribute.html +++ b/docs/_site/posts/contribute.html @@ -7,7 +7,7 @@ Code of conduct | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Code of conduct","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/demo-data.html b/docs/_site/posts/demo-data.html index 479e6249..cc0c5958 100644 --- a/docs/_site/posts/demo-data.html +++ b/docs/_site/posts/demo-data.html @@ -7,7 +7,7 @@ demo-data | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/gallery.html b/docs/_site/posts/gallery.html index 2321d81e..ff0f5a81 100644 --- a/docs/_site/posts/gallery.html +++ b/docs/_site/posts/gallery.html @@ -7,7 +7,7 @@ Gallery | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Gallery","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

@@ -130,7 +130,7 @@ Check out the cool data that our users are using cellxgene to explore!

Melanoma

-

CZI’s own cellxgene site

+

CZI’s own cellxgene site

Want us to link to your dataset here? Just send us a note!

diff --git a/docs/_site/posts/hosted.html b/docs/_site/posts/hosted.html index 0e2d2b6a..214b5b8a 100644 --- a/docs/_site/posts/hosted.html +++ b/docs/_site/posts/hosted.html @@ -7,7 +7,7 @@ Hosting cellxgene on the web | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Hosting cellxgene on the web","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

@@ -139,35 +139,36 @@

Deploying cellxgene with Heroku

-

Quickstart

+

Heroku Support

-

Clicking on the following button will forward you to Heroku to begin the deployment process:

+

The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.

-

- Deploy -

+

While we no longer directly support Heroku, it is still possible to create a Heroku app via our provided Dockerfile here and Heroku’s documentation.

-

If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.

+

You may have to tweak the Dockerfile like so:

-

Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:

+
FROM ubuntu:bionic
 
-
Default settings

+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 
-
    
-  
  • App name: the unique name for your deployment
    • 
-  
  • This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
    • 
-  
  • App owner: Who will own this app. Either you personally or an organization/team
    • 
-  
  • Region: Location of the server where the app will be deployed (EU or US)
    • 
-

+RUN apt-get update && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    pip3 install cellxgene
 
-
Configuration

+# ENTRYPOINT ["cellxgene"]  # Heroku doesn't work well with ENTRYPOINT
+
-
    -
  • DATASET: A publicly accessible URL pointing to a .h5ad file to view
    • -
  • This defaults to pbm3k.h5ad
    • -
+

and provide a heroku.yml file similar to this:

-

After filling out the settings and pressing the Deploy app button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!

+
build:
+  docker:
+    web: Dockerfile
+run:
+  web:
+    command:
+      - cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings.
+

What is Heroku?

diff --git a/docs/_site/posts/hosted.md b/docs/_site/posts/hosted.md index 1168bdf0..073ede26 100644 --- a/docs/_site/posts/hosted.md +++ b/docs/_site/posts/hosted.md @@ -38,31 +38,38 @@ If you know of other solutions, drop us a note and we'll add to this list. # Deploying cellxgene with Heroku -## Quickstart +## Heroku Support -Clicking on the following button will forward you to Heroku to begin the deployment process: +The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene. - - Deploy - +While we no longer directly support Heroku, it is still possible to create a Heroku app via [our provided Dockerfile here](https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile) and [Heroku's documentation](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml). -If not already logged in to Heroku, there you will be prompted to log in or sign up for an account. +You may have to tweak the `Dockerfile` like so: -Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app: +```Dockerfile +FROM ubuntu:bionic -### Default settings +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 -- `App name`: the unique name for your deployment -- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/) -- `App owner`: Who will own this app. Either you personally or an organization/team -- `Region`: Location of the server where the app will be deployed (EU or US) +RUN apt-get update && \ + apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \ + pip3 install cellxgene -### Configuration +# ENTRYPOINT ["cellxgene"] # Heroku doesn't work well with ENTRYPOINT +``` -- `DATASET`: A _publicly_ accessible URL pointing to a .h5ad file to view -- This defaults to pbm3k.h5ad +and provide a `heroku.yml` file similar to this: -After filling out the settings and pressing the `Deploy app` button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene! +```yml +build: + docker: + web: Dockerfile +run: + web: + command: + - cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings. +``` ## What is Heroku? diff --git a/docs/_site/posts/install.html b/docs/_site/posts/install.html index 2eb470d8..c6e0daf5 100644 --- a/docs/_site/posts/install.html +++ b/docs/_site/posts/install.html @@ -7,7 +7,7 @@ Install | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Install","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/launch.html b/docs/_site/posts/launch.html index 4aed95b3..62ccbaf8 100644 --- a/docs/_site/posts/launch.html +++ b/docs/_site/posts/launch.html @@ -7,7 +7,7 @@ demo-data | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/methods.html b/docs/_site/posts/methods.html index 8c97a84c..265b5263 100644 --- a/docs/_site/posts/methods.html +++ b/docs/_site/posts/methods.html @@ -7,7 +7,7 @@ Methods | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Methods","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/prepare.html b/docs/_site/posts/prepare.html index 56dce0c2..b1d6d619 100644 --- a/docs/_site/posts/prepare.html +++ b/docs/_site/posts/prepare.html @@ -7,7 +7,7 @@ prepare | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"prepare","description":"Preparing your data","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/roadmap.html b/docs/_site/posts/roadmap.html index 34dc26b9..8a3b177b 100644 --- a/docs/_site/posts/roadmap.html +++ b/docs/_site/posts/roadmap.html @@ -7,7 +7,7 @@ roadmap | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"roadmap","description":"Roadmap","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/troubleshooting.html b/docs/_site/posts/troubleshooting.html index a936237a..8a7a082a 100644 --- a/docs/_site/posts/troubleshooting.html +++ b/docs/_site/posts/troubleshooting.html @@ -7,7 +7,7 @@ Troubleshooting | cellxgene - + @@ -16,10 +16,10 @@ +{"url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Troubleshooting","description":"Troubleshooting","@type":"WebPage","@context":"https://schema.org"} - + @@ -46,6 +46,10 @@ + Cellxgene data portal
+ + + Demo datasets
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/posts/cellxgene_cziscience_com.md b/docs/deprecated/cellxgene_cziscience_com.md similarity index 100% rename from docs/posts/cellxgene_cziscience_com.md rename to docs/deprecated/cellxgene_cziscience_com.md diff --git a/docs/posts/gallery.md b/docs/posts/gallery.md index 13a7fc24..01f45301 100644 --- a/docs/posts/gallery.md +++ b/docs/posts/gallery.md @@ -39,6 +39,6 @@ Check out the cool data that our users are using cellxgene to explore! ### [Melanoma](https://melanoma.cellgeni.sanger.ac.uk/) -### [CZI's own cellxgene site](cellxgene_cziscience_com) +### [CZI's own cellxgene site](https://cellxgene.cziscience.com/) _Want us to link to your dataset here? [Just send us a note!](contact)_ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..f4e80e25 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1191 @@ +{ + "name": "cellxgene", + "requires": true, + "lockfileVersion": 1, + "dependencies": { + "@babel/code-frame": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + "dev": true, + "requires": { + "@babel/highlight": "^7.10.4" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz", + "integrity": "sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==", + "dev": true + }, + "@babel/highlight": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.4.tgz", + "integrity": "sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==", + "dev": true, + "requires": { + "@babel/helper-validator-identifier": "^7.10.4", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "dependencies": { + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + } + } + }, + "@blueprintjs/eslint-plugin": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@blueprintjs/eslint-plugin/-/eslint-plugin-0.3.1.tgz", + "integrity": "sha512-6dMFdRDcRpCtJz1+JhSkv3evvYfRFJpwy68Y6HAJDcsYRi7lf+A+BUC37k7EKfV9302Djih3oAKyJyFo4btkbQ==", + "dev": true, + "requires": { + "@typescript-eslint/experimental-utils": "^4.2.0", + "eslint": "^7.9.0" + } + }, + "@eslint/eslintrc": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.1.tgz", + "integrity": "sha512-XRUeBZ5zBWLYgSANMpThFddrZZkEbGHgUdt5UJjZfnlN9BGCiUBrf+nvbRupSjMvqzwnQN0qwCmOxITt1cfywA==", + "dev": true, + "requires": { + "ajv": "^6.12.4", + "debug": "^4.1.1", + "espree": "^7.3.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.2.1", + "js-yaml": "^3.13.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + } + } + }, + "@nodelib/fs.scandir": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", + "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "2.0.3", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz", + "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==", + "dev": true + }, + "@nodelib/fs.walk": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz", + "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==", + "dev": true, + "requires": { + "@nodelib/fs.scandir": "2.1.3", + "fastq": "^1.6.0" + } + }, + "@types/json-schema": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.6.tgz", + "integrity": "sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw==", + "dev": true + }, + "@typescript-eslint/experimental-utils": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.9.0.tgz", + "integrity": "sha512-0p8GnDWB3R2oGhmRXlEnCvYOtaBCijtA5uBfH5GxQKsukdSQyI4opC4NGTUb88CagsoNQ4rb/hId2JuMbzWKFQ==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/scope-manager": "4.9.0", + "@typescript-eslint/types": "4.9.0", + "@typescript-eslint/typescript-estree": "4.9.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^2.0.0" + } + }, + "@typescript-eslint/scope-manager": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.9.0.tgz", + "integrity": "sha512-q/81jtmcDtMRE+nfFt5pWqO0R41k46gpVLnuefqVOXl4QV1GdQoBWfk5REcipoJNQH9+F5l+dwa9Li5fbALjzg==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.9.0", + "@typescript-eslint/visitor-keys": "4.9.0" + } + }, + "@typescript-eslint/types": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.9.0.tgz", + "integrity": "sha512-luzLKmowfiM/IoJL/rus1K9iZpSJK6GlOS/1ezKplb7MkORt2dDcfi8g9B0bsF6JoRGhqn0D3Va55b+vredFHA==", + "dev": true + }, + "@typescript-eslint/typescript-estree": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.9.0.tgz", + "integrity": "sha512-rmDR++PGrIyQzAtt3pPcmKWLr7MA+u/Cmq9b/rON3//t5WofNR4m/Ybft2vOLj0WtUzjn018ekHjTsnIyBsQug==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.9.0", + "@typescript-eslint/visitor-keys": "4.9.0", + "debug": "^4.1.1", + "globby": "^11.0.1", + "is-glob": "^4.0.1", + "lodash": "^4.17.15", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + } + }, + "@typescript-eslint/visitor-keys": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.9.0.tgz", + "integrity": "sha512-sV45zfdRqQo1A97pOSx3fsjR+3blmwtdCt8LDrXgCX36v4Vmz4KHrhpV6Fo2cRdXmyumxx11AHw0pNJqCNpDyg==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.9.0", + "eslint-visitor-keys": "^2.0.0" + } + }, + "acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true + }, + "acorn-jsx": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz", + "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", + "dev": true + }, + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ansi-colors": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", + "integrity": "sha512-JoX0apGbHaUJBNl6yF+p6JAFYZ666/hhCGKN5t9QFjbJQKUU/g8MNbFDbvfrgKXvI1QpZplPOnwIo99lX/AAmA==", + "dev": true + }, + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, + "array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true + }, + "astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "dev": true + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true + }, + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "dev": true, + "requires": { + "ms": "2.1.2" + } + }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true + }, + "dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "requires": { + "path-type": "^4.0.0" + } + }, + "doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "enquirer": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", + "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", + "dev": true, + "requires": { + "ansi-colors": "^4.1.1" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "eslint": { + "version": "7.14.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.14.0.tgz", + "integrity": "sha512-5YubdnPXrlrYAFCKybPuHIAH++PINe1pmKNc5wQRB9HSbqIK1ywAnntE3Wwua4giKu0bjligf1gLF6qxMGOYRA==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@eslint/eslintrc": "^0.2.1", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "enquirer": "^2.3.5", + "eslint-scope": "^5.1.1", + "eslint-utils": "^2.1.0", + "eslint-visitor-keys": "^2.0.0", + "espree": "^7.3.0", + "esquery": "^1.2.0", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "progress": "^2.0.0", + "regexpp": "^3.1.0", + "semver": "^7.2.1", + "strip-ansi": "^6.0.0", + "strip-json-comments": "^3.1.0", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "dependencies": { + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + } + } + }, + "eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + }, + "eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } + } + }, + "eslint-visitor-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz", + "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==", + "dev": true + }, + "espree": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz", + "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==", + "dev": true, + "requires": { + "acorn": "^7.4.0", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.3.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esquery": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.3.1.tgz", + "integrity": "sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ==", + "dev": true, + "requires": { + "estraverse": "^5.1.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "requires": { + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true + }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "fast-glob": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz", + "integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.0", + "merge2": "^1.3.0", + "micromatch": "^4.0.2", + "picomatch": "^2.2.1" + } + }, + "fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true + }, + "fastq": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.9.0.tgz", + "integrity": "sha512-i7FVWL8HhVY+CTkwFxkN2mk3h+787ixS5S63eb78diVRc1MCssarHq3W5cj0av7YDSwmaV928RNag+U1etRQ7w==", + "dev": true, + "requires": { + "reusify": "^1.0.4" + } + }, + "file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "dev": true, + "requires": { + "flat-cache": "^2.0.1" + } + }, + "fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dev": true, + "requires": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + } + }, + "flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "dev": true + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true + }, + "glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, + "globby": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz", + "integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==", + "dev": true, + "requires": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.1.1", + "ignore": "^5.1.4", + "merge2": "^1.3.0", + "slash": "^3.0.0" + } + }, + "has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "ignore": { + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", + "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", + "dev": true + }, + "import-fresh": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.2.tgz", + "integrity": "sha512-cTPNrlvJT6twpYy+YmKUKrTSjWFs3bjYjAhCwm+z4EOCubZxAuO+hHpRN64TqjEaYSHs7tJAE0w1CKMGmsG/lw==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true + }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true + }, + "levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "requires": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + } + }, + "lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "dev": true + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dev": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true + }, + "micromatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz", + "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==", + "dev": true, + "requires": { + "braces": "^3.0.1", + "picomatch": "^2.0.5" + } + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "optionator": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", + "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", + "dev": true, + "requires": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.3" + } + }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "requires": { + "callsites": "^3.0.0" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true + }, + "picomatch": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", + "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", + "dev": true + }, + "prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true + }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true + }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true + }, + "regexpp": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", + "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==", + "dev": true + }, + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + }, + "run-parallel": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.10.tgz", + "integrity": "sha512-zb/1OuZ6flOlH6tQyMPUrE3x3Ulxjlo9WIVXR4yVYi4H9UXQaeIsPbLn2R3O3vQCnDKkAl2qHiuocKKX4Tz/Sw==", + "dev": true + }, + "semver": { + "version": "7.3.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz", + "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true + }, + "slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + } + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "table": { + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "dev": true, + "requires": { + "ajv": "^6.10.2", + "lodash": "^4.17.14", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + } + }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "dev": true + }, + "to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "requires": { + "is-number": "^7.0.0" + } + }, + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "dev": true + }, + "tsutils": { + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.17.1.tgz", + "integrity": "sha512-kzeQ5B8H3w60nFY2g8cJIuH7JDpsALXySGtwGJ0p2LSjLgay3NdIpqq5SoOBe46bKDW2iq25irHCr8wjomUS2g==", + "dev": true, + "requires": { + "tslib": "^1.8.1" + } + }, + "type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "requires": { + "prelude-ls": "^1.2.1" + } + }, + "type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true + }, + "uri-js": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.0.tgz", + "integrity": "sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g==", + "dev": true, + "requires": { + "punycode": "^2.1.0" + } + }, + "v8-compile-cache": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz", + "integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==", + "dev": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dev": true, + "requires": { + "mkdirp": "^0.5.1" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "dev": true + } + } +} diff --git a/package.json b/package.json index e3722f75..ab2f09d7 100644 --- a/package.json +++ b/package.json @@ -2,5 +2,8 @@ "name": "cellxgene", "scripts": { "postinstall": "npm install --prefix client && npm run build --prefix client && make copy-client-assets" + }, + "devDependencies": { + "@blueprintjs/eslint-plugin": "^0.3.1" } } diff --git a/server/Makefile b/server/Makefile index 700b9a27..016f89cc 100644 --- a/server/Makefile +++ b/server/Makefile @@ -39,3 +39,11 @@ create-test-db: clean-test-db: -docker stop test_db -docker rm test_db + +.PHONY: test-annotations-performance +test-annotations-performance: + python test/performance/performance_test_annotations_backend.py + +.PHONY: test-annotations-scale +test-annotations-scale: + locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt diff --git a/server/__init__.py b/server/__init__.py index 94238d9a..100b1874 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -1,6 +1,5 @@ import logging import sys - from server.common.utils.utils import import_plugins __version__ = "0.16.0" diff --git a/server/app/app.py b/server/app/app.py index d926f515..5f924703 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -6,8 +6,17 @@ from urllib.parse import urlparse import hashlib import os -from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \ - send_from_directory +from flask import ( + Flask, + redirect, + current_app, + make_response, + render_template, + abort, + Blueprint, + request, + send_from_directory, +) from flask_restful import Api, Resource from server_timing import Timing as ServerTiming @@ -87,10 +96,7 @@ def dataset_index(url_dataroot=None, dataset=None): cache_manager = current_app.matrix_data_cache_manager with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor: data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}") - args = { - "SCRIPTS" : scripts, - "INLINE_SCRIPTS" : inline_scripts - } + args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts} return render_template("index.html", **args) except DatasetAccessError as e: @@ -99,13 +105,6 @@ def dataset_index(url_dataroot=None, dataset=None): ) -@webbp.route("/health", methods=["GET"]) -@cache_control_always(no_store=True) -def health(): - config = current_app.app_config - return health_check(config) - - @webbp.errorhandler(RequestException) def handle_request_exception(error): return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True) @@ -224,6 +223,13 @@ def dataroot_index(): return redirect(config.server_config.multi_dataset__index) +class HealthAPI(Resource): + @cache_control(no_store=True) + def get(self): + config = current_app.app_config + return health_check(config) + + class DatasetResource(Resource): """Base class for all Resources that act on datasets.""" @@ -312,8 +318,18 @@ class LayoutObsAPI(DatasetResource): return common_rest.layout_obs_put(request, data_adaptor) -def get_api_resources(bp_api, url_dataroot=None): - api = Api(bp_api) +def get_api_base_resources(bp_base): + """Add resources that are accessed from the api_base_url""" + api = Api(bp_base) + + # Diagnostics routes + api.add_resource(HealthAPI, "/health") + return api + + +def get_api_dataroot_resources(bp_dataroot, url_dataroot=None): + """Add resources that refer to a dataset""" + api = Api(bp_dataroot) def add_resource(resource, url): """convenience function to make the outer function less verbose""" @@ -385,18 +401,24 @@ class Server: parse = urlparse(api_base_url) api_path = parse.path + bp_base = Blueprint("bp_base", __name__, url_prefix=api_path) + base_resources = get_api_base_resources(bp_base) + self.app.register_blueprint(base_resources.blueprint) + if app_config.is_multi_dataset(): # NOTE: These routes only allow the dataset to be in the directory # of the dataroot, and not a subdirectory. We may want to change # the route format at some point for dataroot_dict in server_config.multi_dataset__dataroot.values(): url_dataroot = dataroot_dict["base_url"] - bp_api = Blueprint( - f"api_dataset_{url_dataroot}", __name__, - url_prefix=f"{api_path}/{url_dataroot}/" + api_version + bp_dataroot = Blueprint( + f"api_dataset_{url_dataroot}", + __name__, + url_prefix=f"{api_path}/{url_dataroot}/" + api_version, ) - resources = get_api_resources(bp_api, url_dataroot) - self.app.register_blueprint(resources.blueprint) + dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot) + self.app.register_blueprint(dataroot_resources.blueprint) + self.app.add_url_rule( f"/{url_dataroot}//", f"dataset_index_{url_dataroot}", @@ -407,18 +429,18 @@ class Server: f"/{url_dataroot}//static/", f"static_assets_{url_dataroot}", view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename), - methods=["GET"] + methods=["GET"], ) else: bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}") - resources = get_api_resources(bp_api) + resources = get_api_dataroot_resources(bp_api) self.app.register_blueprint(resources.blueprint) self.app.add_url_rule( "/static/", "static_assets", view_func=lambda filename: send_from_directory("../common/web/static", filename), - methods=["GET"] + methods=["GET"], ) self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager diff --git a/server/auth/__init__.py b/server/auth/__init__.py index 1b33bebc..dfadadde 100644 --- a/server/auth/__init__.py +++ b/server/auth/__init__.py @@ -1,4 +1,3 @@ - # import the built in auth types so they can be registered import server.auth.auth_none # noqa: F401 diff --git a/server/auth/auth.py b/server/auth/auth.py index bb86ea64..03184e8d 100644 --- a/server/auth/auth.py +++ b/server/auth/auth.py @@ -43,6 +43,10 @@ class AuthTypeBase(ABC): """Return the name of the user (string)""" pass + def get_user_picture(self): + """Return the location to the user's picture""" + return None + class AuthTypeClientBase(AuthTypeBase): """Base type for all authentication types that require the client to login""" @@ -76,7 +80,7 @@ class AuthTypeFactory: @staticmethod def register(name, auth_type): - assert(issubclass(auth_type, AuthTypeBase)) + assert issubclass(auth_type, AuthTypeBase) AuthTypeFactory.auth_types[name] = auth_type @staticmethod diff --git a/server/auth/auth_none.py b/server/auth/auth_none.py index c482e3c4..61f82400 100644 --- a/server/auth/auth_none.py +++ b/server/auth/auth_none.py @@ -2,7 +2,6 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory class AuthTypeNone(AuthTypeBase): - def __init__(self, app_config): super().__init__() diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py index 2b4ddf45..f1e50504 100644 --- a/server/auth/auth_oauth.py +++ b/server/auth/auth_oauth.py @@ -24,7 +24,7 @@ except ModuleNotFoundError: class Tokens: """Simple class to represent the tokens that are saved/restored from the cookie""" - def __init__(self, access_token, id_token, refresh_token, expires_at): + def __init__(self, access_token, id_token, refresh_token, expires_at, **kwargs): self.access_token = access_token self.id_token = id_token self.refresh_token = refresh_token @@ -97,8 +97,17 @@ class AuthTypeOAuth(AuthTypeClientBase): return valid_keys = { - "verify_signature", "verify_aud", "verify_iat", "verify_exp", "verify_nbf", "verify_iss", - "verify_sub", "verify_jti", "verify_at_hash", "leeway"} + "verify_signature", + "verify_aud", + "verify_iat", + "verify_exp", + "verify_nbf", + "verify_iss", + "verify_sub", + "verify_jti", + "verify_at_hash", + "leeway", + } keys = set(self.jwt_decode_options.keys()) unknown = keys - valid_keys if unknown: @@ -114,6 +123,7 @@ class AuthTypeOAuth(AuthTypeClientBase): parse = urlparse(self.api_base_url) app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"]) app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"]) + app.add_url_rule(f"{parse.path}/logout_redirect", "logout_redirect", self.logout_redirect, methods=["GET"]) app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"]) def complete_setup(self, flask_app): @@ -136,21 +146,19 @@ class AuthTypeOAuth(AuthTypeClientBase): def get_user_id(self): payload = self.get_userinfo() - if payload and payload.get("sub"): - return payload.get("sub") - return None + return payload.get("sub") if payload else None def get_user_name(self): payload = self.get_userinfo() - if payload and payload.get("name"): - return payload.get("name") - return None + return payload.get("name") if payload else None def get_user_email(self): payload = self.get_userinfo() - if payload and payload.get("email"): - return payload.get("email") - return None + return payload.get("email") if payload else None + + def get_user_picture(self): + payload = self.get_userinfo() + return payload.get("picture") if payload else None def update_response(self, response): response.cache_control.update(dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True)) @@ -158,7 +166,7 @@ class AuthTypeOAuth(AuthTypeClientBase): def login(self): callbackurl = f"{self.api_base_url}/oauth2/callback" return_path = request.args.get("dataset", "") - return_to = f"{self.web_base_url}/{return_path}/" + return_to = f"{self.web_base_url}/{return_path}" # save the return path in the session cookie, accessed in the callback function session["oauth_callback_redirect"] = return_to response = self.client.authorize_redirect(redirect_uri=callbackurl) @@ -166,12 +174,29 @@ class AuthTypeOAuth(AuthTypeClientBase): return response def logout(self): + """ + We would like for the user to remain on the same dataset after logout. oauth requires that + the redirect `returnTo` path be whitelisted by the oauth server, therefore a level of + indirection is used. We first redirect to a single path "logout_redirect", and logout_redirect + will redirect the user's browser back to the current page. + """ self.remove_tokens() - params = {"returnTo": self.web_base_url, "client_id": self.client_id} + redirect_path = request.args.get("dataset", "") + redirect_to = f"{self.web_base_url}/{redirect_path}" + session["oauth_logout_redirect"] = redirect_to + + return_to = f"{self.api_base_url}/logout_redirect" + params = {"returnTo": return_to, "client_id": self.client_id} response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params)) self.update_response(response) return response + def logout_redirect(self): + oauth_logout_redirect = session.pop("oauth_logout_redirect", "/") + response = redirect(oauth_logout_redirect) + self.update_response(response) + return response + def callback(self): data = self.client.authorize_access_token() tokens = Tokens( @@ -193,22 +218,24 @@ class AuthTypeOAuth(AuthTypeClientBase): try: if self.session_cookie: - tokensdict = session.get(self.CXG_TOKENS) - if tokensdict: - g.tokens = Tokens(**tokensdict) + value = session.get(self.CXG_TOKENS) + if value: + g.tokens = Tokens(**value) else: return None else: value = request.cookies.get(self.cookie_params["key"]) - value = base64.b64decode(value) - try: - tokensdict = json.loads(value) - g.tokens = Tokens(**tokensdict) - except (TypeError, KeyError, json.decoder.JSONDecodeError): - g.pop("tokens", None) + if value is None: return None + value = base64.b64decode(value) + value = json.loads(value) + g.tokens = Tokens(**value) - except (TypeError, KeyError): + except Exception: + # there are many types of exceptions that can be raise in the above section. + # It is impractical to list all the exceptions here, since that would be brittle. + # If an exception occurs, then return None, meaning that no token could be retrieved. + current_app.logger.warning(f"auth cookie is in the wrong format: {str(value)}") g.pop("tokens", None) return None @@ -253,7 +280,10 @@ class AuthTypeOAuth(AuthTypeClientBase): def get_logout_url(self, data_adaptor): """Return the url for the logout route""" - return f"{self.api_base_url}/logout" + if data_adaptor and current_app.app_config.is_multi_dataset(): + return f"{self.api_base_url}/logout?dataset={data_adaptor.uri_path}/" + else: + return f"{self.api_base_url}/logout" def check_jwt_payload(self, id_token): try: @@ -303,6 +333,7 @@ class AuthTypeOAuth(AuthTypeClientBase): # if there is no id_token, return None (user is not authenticated) tokens = self.get_tokens() + if tokens is None or tokens.id_token is None: return None diff --git a/server/auth/auth_test.py b/server/auth/auth_test.py index e6b0b8e0..06b92cc7 100644 --- a/server/auth/auth_test.py +++ b/server/auth/auth_test.py @@ -10,12 +10,14 @@ class AuthTypeTest(AuthTypeClientBase): CXGUID = "cxguid_test" CXGUNAME = "cxguname_test" CXGUEMAIL = "cxguemail_test" + CXGUPICTURE = "cxgupicture_test" def __init__(self, app_config): super().__init__() self.user_name = "test_account" self.user_id = "id0001" self.user_email = "test_account@test.com" + self.user_picture = None def is_valid_authentication_type(self): return True @@ -42,11 +44,16 @@ class AuthTypeTest(AuthTypeClientBase): def get_user_email(self): return session.get(self.CXGUEMAIL) + def get_user_picture(self): + return session.get(self.CXGUPICTURE) + def login(self): args = request.args return_to = args.get("dataset", "/") session[self.CXGUID] = args.get("userid", self.user_id) session[self.CXGUNAME] = args.get("username", self.user_name) + session[self.CXGUEMAIL] = args.get("email", self.user_email) + session[self.CXGUPICTURE] = args.get("picture", self.user_picture) return redirect(return_to) def logout(self): diff --git a/server/cli/cli.py b/server/cli/cli.py index dc3e5837..f8f7fde6 100644 --- a/server/cli/cli.py +++ b/server/cli/cli.py @@ -4,6 +4,7 @@ from .convert_to_cxg import convert_to_cxg from .launch import launch from .prepare import prepare from .upgrade import log_upgrade_check +from .schema import schema_cli from .. import __version__ @@ -31,3 +32,4 @@ def cli(upgrade_check): cli.add_command(launch) cli.add_command(prepare) cli.add_command(convert_to_cxg) +cli.add_command(schema_cli) diff --git a/server/cli/convert_to_cxg.py b/server/cli/convert_to_cxg.py index 4e456c08..0cd67e3c 100644 --- a/server/cli/convert_to_cxg.py +++ b/server/cli/convert_to_cxg.py @@ -9,26 +9,24 @@ from server.converters.h5ad_data_file import H5ADDataFile name="convert", short_help="Converts an H5AD dataset to the CXG format.", help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format " - "that has performance and access characteristics amenable to a multi-dataset, multi-user serving " - "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as " - "usually with the generated CXG file.", + "that has performance and access characteristics amenable to a multi-dataset, multi-user serving " + "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as " + "usually with the generated CXG file.", ) @click.argument( - "input-file", - nargs=1, - type=click.Path(exists=True, dir_okay=False), + "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False), ) @click.option( "-o", "--output-directory", help="Name of the output CXG directory. If not provided, will default to be the input filename with a " - "CXG extension.", + "CXG extension.", ) @click.option( "-b", "--backed", help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, " - "but will use less memory.", + "but will use less memory.", default=False, show_default=True, is_flag=True, @@ -37,29 +35,33 @@ from server.converters.h5ad_data_file import H5ADDataFile "-t", "--title", help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, " - "the dataset title will be the filename.", + "the dataset title will be the filename.", ) @click.option( "-a", "--about", help="A fully qualified URL that provides more information about the dataset and will be included as " - "metadata about the CXG file.", + "metadata about the CXG file.", ) @click.option( "-s", "--sparse-threshold", help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X " - "array of the dataset will be sparse. Since the default value is 0.0, the default will be to " - "convert to dense array.", + "array of the dataset will be sparse. Since the default value is 0.0, the default will be to " + "convert to dense array.", default=0.0, show_default=True, ) -@click.option("--obs-names", - help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of " - "the one designated by the dataframe generated-index.") -@click.option("--var-names", - help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of " - "the one designated by the dataframe generated-index.") +@click.option( + "--obs-names", + help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of " + "the one designated by the dataframe generated-index.", +) +@click.option( + "--var-names", + help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of " + "the one designated by the dataframe generated-index.", +) @click.option( "--disable-custom-colors", help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.", @@ -70,8 +72,8 @@ from server.converters.h5ad_data_file import H5ADDataFile @click.option( "--disable-corpora-schema", help="When set, conversion process will neither extract nor store Corpora schema information. See " - "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more " - "information.", + "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more " + "information.", default=False, show_default=True, is_flag=True, @@ -85,30 +87,32 @@ from server.converters.h5ad_data_file import H5ADDataFile ) @click.help_option("--help", "-h", help="Show this message and exit.") def convert_to_cxg( - input_file, - output_directory, - backed, - title, - about, - sparse_threshold, - obs_names, - var_names, - disable_custom_colors, - disable_corpora_schema, - overwrite, + input_file, + output_directory, + backed, + title, + about, + sparse_threshold, + obs_names, + var_names, + disable_custom_colors, + disable_corpora_schema, + overwrite, ): """ Convert a dataset file into CXG. """ - h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names, - use_corpora_schema=not disable_corpora_schema) + h5ad_data_file = H5ADDataFile( + input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema + ) # Get the directory that will hold all the CXG files cxg_output_container = get_output_directory(input_file, output_directory, overwrite) - h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold, - convert_anndata_colors_to_cxg_colors=not disable_custom_colors) + h5ad_data_file.to_cxg( + cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors + ) def get_output_directory(input_filename, output_directory, should_overwrite): diff --git a/server/cli/launch.py b/server/cli/launch.py index d46161d4..a05fd6a1 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -3,15 +3,14 @@ import functools import logging import sys import webbrowser -from os import devnull - +import os import click from flask_compress import Compress from flask_cors import CORS +from server.default_config import default_config from server.app.app import Server -from server.common.app_config import AppConfig -from server.common.default_config import default_config +from server.common.config.app_config import AppConfig from server.common.errors import DatasetAccessError, ConfigurationError from server.common.utils.utils import sort_options @@ -33,7 +32,7 @@ def annotation_args(func): multiple=False, metavar="", help="CSV file to initialize editing of existing annotations; will be altered in-place. " - "Incompatible with --annotations-dir.", + "Incompatible with --annotations-dir.", ) @click.option( "--annotations-dir", @@ -42,7 +41,7 @@ def annotation_args(func): multiple=False, metavar="", help="Directory of where to save output annotations; filename will be specified in the application. " - "Incompatible with --annotations-file.", + "Incompatible with --annotations-file.", ) @click.option( "--experimental-annotations-ontology", @@ -170,7 +169,7 @@ def server_args(func): default=DEFAULT_CONFIG.server_config.app__debug, show_default=True, help="Run in debug mode. This is helpful for cellxgene developers, " - "or when you want more information about an error condition.", + "or when you want more information about an error condition.", ) @click.option( "--verbose", @@ -203,7 +202,7 @@ def server_args(func): multiple=True, metavar="", help="Additional script files to include in HTML page. If not specified, " - "no additional script files will be included.", + "no additional script files will be included.", show_default=False, ) @functools.wraps(func) @@ -223,7 +222,7 @@ def launch_args(func): default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot, metavar="", help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)" - " to folder containing H5AD and/or CXG datasets.", + " to folder containing H5AD and/or CXG datasets.", hidden=True, ) # TODO, unhide when dataroot is supported) @click.argument("datapath", required=False, metavar="") @@ -307,32 +306,32 @@ class CliLaunchServer(Server): ) @launch_args def launch( - datapath, - dataroot, - verbose, - debug, - open_browser, - port, - host, - embedding, - obs_names, - var_names, - max_category_items, - disable_custom_colors, - diffexp_lfc_cutoff, - title, - scripts, - about, - disable_annotations, - annotations_file, - annotations_dir, - backed, - disable_diffexp, - experimental_annotations_ontology, - experimental_annotations_ontology_obo, - experimental_enable_reembedding, - config_file, - dump_default_config, + datapath, + dataroot, + verbose, + debug, + open_browser, + port, + host, + embedding, + obs_names, + var_names, + max_category_items, + disable_custom_colors, + diffexp_lfc_cutoff, + title, + scripts, + about, + disable_annotations, + annotations_file, + annotations_dir, + backed, + disable_diffexp, + experimental_annotations_ontology, + experimental_annotations_ontology_obo, + experimental_enable_reembedding, + config_file, + dump_default_config, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. @@ -443,7 +442,7 @@ def launch( click.echo("[cellxgene] Type CTRL-C at any time to exit.") if not server_config.app__verbose: - f = open(devnull, "w") + f = open(os.devnull, "w") sys.stdout = f try: diff --git a/server/cli/prepare.py b/server/cli/prepare.py index df535db0..67735172 100644 --- a/server/cli/prepare.py +++ b/server/cli/prepare.py @@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options default=False, is_flag=True, help="Do not run quality control metrics. By default cellxgene runs them " - "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", + "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", ) @click.option( "--make-obs-names-unique/--no-make-obs-names-unique", @@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options ) @click.help_option("--help", "-h", help="Show this message and exit.") def prepare( - data, - embedding, - recipe, - output, - plotting, - sparse, - overwrite, - set_obs_names, - set_var_names, - skip_qc, - make_obs_names_unique, - make_var_names_unique, + data, + embedding, + recipe, + output, + plotting, + sparse, + overwrite, + set_obs_names, + set_var_names, + skip_qc, + make_obs_names_unique, + make_var_names_unique, ): """ Preprocess data for use with cellxgene. diff --git a/server/cli/schema.py b/server/cli/schema.py new file mode 100644 index 00000000..5f16ec64 --- /dev/null +++ b/server/cli/schema.py @@ -0,0 +1,72 @@ +import click + +from server.converters.schema import remix, validate + + +@click.group( + name="schema", + subcommand_metavar="COMMAND ", + short_help="Apply and validate the cellxgene data integration schema to an h5ad file.", + context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]), +) +def schema_cli(): + try: + import scanpy # noqa: F401 + except ImportError: + raise click.ClickException( + "[cellxgene] cellxgene schema requires scanpy" + ) + + +@click.command( + name="apply", + short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.", + help="(experimental) Using a yaml file that describes schema values to insert or convert and in input " + "h5ad file, apply the schema changes and create a new, conforming h5ad.", +) +@click.option( + "--source-h5ad", + help="Input h5ad file.", + nargs=1, + required=True, + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--remix-config", + help="Config yaml with information on how to apply the schema.", + nargs=1, + required=True, + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--output-filename", + help="Filename for the new, schema-conforming h5ad file.", + required=True, + nargs=1 +) +def schema_apply(source_h5ad, remix_config, output_filename): + remix.apply_schema(source_h5ad, remix_config, output_filename) + + +@click.command( + name="validate", + short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.", +) +@click.argument( + "h5ad", + nargs=1, + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--shallow", + help="When true, just check that the correct version information is present.", + default=False, + show_default=True, + is_flag=True, +) +def schema_validate(h5ad, shallow): + validate.validate(h5ad, shallow) + + +schema_cli.add_command(schema_apply) +schema_cli.add_command(schema_validate) diff --git a/server/cli/upgrade.py b/server/cli/upgrade.py index 953f92ba..222d7e81 100644 --- a/server/cli/upgrade.py +++ b/server/cli/upgrade.py @@ -10,7 +10,8 @@ from .. import __version__ SEMVER_FORMAT = re.compile( r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[" r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(" - r"?:\.[0-9a-zA-Z-]+)*))?$") + r"?:\.[0-9a-zA-Z-]+)*))?$" +) def log_upgrade_check(): diff --git a/server/common/annotations/hosted_tiledb.py b/server/common/annotations/hosted_tiledb.py index f3df75ed..c096d568 100644 --- a/server/common/annotations/hosted_tiledb.py +++ b/server/common/annotations/hosted_tiledb.py @@ -31,7 +31,14 @@ class AnnotationsHostedTileDB(Annotations): unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names) if unsanitary_original_category_names: raise AnnotationCategoryNameError( - f"{unsanitary_original_category_names} are not valid category names, please resubmit") + f"{unsanitary_original_category_names} are not valid category names, please resubmit" + ) + + def get_user_name(self): + return current_app.auth.get_user_name() + + def get_user_id(self): + return current_app.auth.get_user_id() def is_safe_collection_name(self, name): """ @@ -47,7 +54,7 @@ class AnnotationsHostedTileDB(Annotations): self.CXG_ANNO_COLLECTION = name def read_labels(self, data_adaptor): - user_id = current_app.auth.get_user_id() + user_id = self.get_user_id() if user_id is None: return dataset_name = data_adaptor.get_location() @@ -57,7 +64,15 @@ class AnnotationsHostedTileDB(Annotations): Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id] ) if annotation_object: - df = tiledb.open(annotation_object.tiledb_uri) + if annotation_object.tiledb_uri == "": + # this mean the user has removed all the categories. + return None + try: + df = tiledb.open(annotation_object.tiledb_uri) + except tiledb.TileDBError: + # don't crash if the annotations file is missing or can't be read. + current_app.logger.warning(f"Cannot read annotation file: {annotation_object.tiledb_uri}") + return None pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints) return pandas_df else: @@ -68,11 +83,11 @@ class AnnotationsHostedTileDB(Annotations): index_dims = None schema_hints = json.loads(schema_hints) - if '__pandas_attribute_repr' in tileDBArray.meta: + if "__pandas_attribute_repr" in tileDBArray.meta: # backwards compatibility... unsure if necessary at this point - repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr']) - if '__pandas_index_dims' in tileDBArray.meta: - index_dims = json.loads(tileDBArray.meta['__pandas_index_dims']) + repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"]) + if "__pandas_index_dims" in tileDBArray.meta: + index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"]) data = tileDBArray[:] indexes = list() @@ -80,12 +95,12 @@ class AnnotationsHostedTileDB(Annotations): for col_name, col_val in data.items(): # If the column values are byte literals, decode them if isinstance(col_val[0], bytes): - col_val = [value.decode('utf-8') for value in col_val] + col_val = [value.decode("utf-8") for value in col_val] if schema_hints and col_name in schema_hints: type = schema_hints.get(col_name).get("type") if type and type == "categorical": - new_col = pd.Series(col_val, dtype='category') + new_col = pd.Series(col_val, dtype="category") data[col_name] = new_col elif repr_meta and col_name in repr_meta: new_col = pd.Series(col_val, dtype=repr_meta[col_name]) @@ -102,8 +117,8 @@ class AnnotationsHostedTileDB(Annotations): return new_df def write_labels(self, df, data_adaptor): - auth_user_id = current_app.auth.get_user_id() - user_name = current_app.auth.get_user_name() + auth_user_id = self.get_user_id() + user_name = self.get_user_name() timestamp = time.time() dataset_location = data_adaptor.get_location() dataset_id = self.db.get_or_create_dataset(dataset_location) @@ -123,20 +138,22 @@ class AnnotationsHostedTileDB(Annotations): else: os.makedirs(uri, exist_ok=True) _, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df) - annotation = Annotation( - tiledb_uri=uri, - user_id=user_id, - dataset_id=str(dataset_id), - schema_hints=json.dumps(dataframe_schema_type_hints) - ) if not df.empty: self.check_category_names(df) # convert to tiledb datatypes for col in df: df[col] = df[col].astype(get_dtype_of_array(df[col])) - tiledb.from_pandas(uri, df) + tiledb.from_pandas(uri, df, sparse=True) + else: + uri = "" + annotation = Annotation( + tiledb_uri=uri, + user_id=user_id, + dataset_id=str(dataset_id), + schema_hints=json.dumps(dataframe_schema_type_hints), + ) self.db.session.add(annotation) self.db.session.commit() diff --git a/server/common/app_config.py b/server/common/app_config.py deleted file mode 100644 index e0a49e46..00000000 --- a/server/common/app_config.py +++ /dev/null @@ -1,960 +0,0 @@ -import copy -import os -import sys -import warnings -from os.path import splitext, basename, isdir -from urllib.parse import urlparse, quote_plus - -import yaml -from flatten_dict import flatten, unflatten - -import server.compute.diffexp_cxg as diffexp_tiledb -import server.compute.scanpy -from server import display_version as cellxgene_display_version -from server.auth.auth import AuthTypeFactory -from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB -from server.common.annotations.local_file_csv import AnnotationsLocalFile -from server.common.data_locator import discover_s3_region_name -from server.common.default_config import get_default_config -from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure -from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available -from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType -from server.db.db_utils import DbUtils - -DEFAULT_SERVER_PORT = 5005 -# anything bigger than this will generate a special message -BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB - - -class AppFeature(object): - def __init__(self, path, available=False, method="POST", extra={}): - self.path = path - self.available = available - self.method = method - self.extra = extra - for k, v in extra.items(): - setattr(self, k, v) - - def todict(self): - d = dict(available=self.available, method=self.method, path=self.path) - d.update(self.extra) - return d - - -class AppConfig(object): - """AppConfig stores all the configuration for cellxgene. The configuration is divided into two main parts: - server attributes, and dataset attributes. The server_config contains attributes that refer to the server process - as a whole. The default_dataset_config referes to attributes that are associated with the features and - presentations of a dataset. The dataset config attributes can be overridden depending on the url by which the - dataset was accessed. These are stored in dataroot_config. - AppConfig has methods to initialize, modify, and access the configuration. - """ - - def __init__(self): - - # the default configuration (see default_config.py) - self.default_config = get_default_config() - # the server configuration - self.server_config = ServerConfig(self, self.default_config["server"]) - # the dataset config, unless overridden by an entry in dataroot_config - self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"]) - # a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot - # attribute of the server_config. - self.dataroot_config = {} - - # Set to true when config_completed is called - self.is_completed = False - - def get_dataset_config(self, dataroot_key): - if self.server_config.single_dataset__datapath: - return self.default_dataset_config - else: - return self.dataroot_config.get(dataroot_key, self.default_dataset_config) - - def check_config(self): - """Verify all the attributes have been checked""" - if not self.is_completed: - raise ConfigurationError("The configuration has not been completed") - self.server_config.check_config() - self.default_dataset_config.check_config() - for dataset_config in self.dataroot_config.values(): - dataset_config.check_config() - - def update_server_config(self, **kw): - self.server_config.update(**kw) - self.is_complete = False - - def update_default_dataset_config(self, **kw): - self.default_dataset_config.update(**kw) - # update all the other dataset configs, if any - for value in self.dataroot_config.values(): - value.update(**kw) - self.is_complete = False - - def update_from_config_file(self, config_file): - with open(config_file) as fyaml: - config = yaml.load(fyaml, Loader=yaml.FullLoader) - - self.server_config.update_from_config(config["server"], "server") - self.default_dataset_config.update_from_config(config["dataset"], "dataset") - - per_dataset_config = config.get("per_dataset_config", {}) - for key, dataroot_config in per_dataset_config.items(): - # first create and initialize the dataroot with the default config - self.add_dataroot_config(key, **config["dataset"]) - # then apply the per dataset configuration - self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}") - - self.is_complete = False - - def write_config(self, config_file): - """output the config to a yaml file""" - server = self.server_config.create_mapping(self.server_config.default_config) - dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) - config = dict(server={}, dataset={}) - for attrname in server.keys(): - config["server__" + attrname] = getattr(self.server_config, attrname) - for attrname in dataset.keys(): - config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname) - if self.dataroot_config: - config["per_dataset_config"] = {} - for dataroot_tag, dataroot_config in self.dataroot_config.items(): - dataset = dataroot_config.create_mapping(dataroot_config.default_config) - for attrname in dataset.keys(): - config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname) - - config = unflatten(config, splitter=lambda key: key.split("__")) - yaml.dump(config, open(config_file, "w")) - - def changes_from_default(self): - """Return all the attribute that are different from the default""" - diff_server = self.server_config.changes_from_default() - diff_dataset = self.default_dataset_config.changes_from_default() - diff = dict(server=diff_server, dataset=diff_dataset) - return diff - - def add_dataroot_config(self, dataroot_tag, **kw): - """Create a new dataset config object based on the default dataset config, and kw parameters""" - if dataroot_tag in self.dataroot_config: - raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}") - if type(self.server_config.multi_dataset__dataroot) != dict: - raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary") - if dataroot_tag not in self.server_config.multi_dataset__dataroot: - raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot") - - self.is_completed = False - self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"]) - flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) - config = {key: value[1] for key, value in flat_config.items()} - self.dataroot_config[dataroot_tag].update(**config) - self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag) - - def complete_config(self, messagefn=None): - """The configure options are checked, and any additional setup based on the config - parameters is done""" - - if messagefn is None: - def noop(message): - pass - - messagefn = noop - - # TODO: to give better error messages we can add a mapping between where each config - # attribute originated (e.g. command line argument or config file), then in the error - # messages we can give correct context for attributes with bad value. - context = dict(messagefn=messagefn) - - self.server_config.complete_config(context) - self.default_dataset_config.complete_config(context) - for dataroot_config in self.dataroot_config.values(): - dataroot_config.complete_config(context) - - self.is_completed = True - self.check_config() - - def get_matrix_data_cache_manager(self): - return self.server_config.matrix_data_cache_manager - - def is_multi_dataset(self): - return self.server_config.multi_dataset__dataroot is not None - - def get_title(self, data_adaptor): - return ( - self.server_config.single_dataset__title - if self.server_config.single_dataset__title - else data_adaptor.get_title() - ) - - def get_about(self, data_adaptor): - return ( - self.server_config.single_dataset__about - if self.server_config.single_dataset__about - else data_adaptor.get_about() - ) - - def get_client_config(self, data_adaptor): - """ - Return the configuration as required by the /config REST route - """ - - server_config = self.server_config - dataset_config = data_adaptor.dataset_config - annotation = dataset_config.user_annotations - auth = server_config.auth - - # FIXME The current set of config is not consistently presented: - # we have camalCase, hyphen-text, and underscore_text - - # make sure the configuration has been checked. - self.check_config() - - # features - features = [f.todict() for f in data_adaptor.get_features(annotation)] - - # display_names - title = self.get_title(data_adaptor) - about = self.get_about(data_adaptor) - - display_names = dict(engine=data_adaptor.get_name(), dataset=title) - - # library_versions - library_versions = {} - library_versions.update(data_adaptor.get_library_versions()) - library_versions["cellxgene"] = cellxgene_display_version - - # links - links = {"about-dataset": about} - - # parameters - parameters = { - "layout": dataset_config.embeddings__names, - "max-category-items": dataset_config.presentation__max_categories, - "obs_names": server_config.single_dataset__obs_names, - "var_names": server_config.single_dataset__var_names, - "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff, - "backed": server_config.adaptor__anndata_adaptor__backed, - "disable-diffexp": not dataset_config.diffexp__enable, - "enable-reembedding": dataset_config.embeddings__enable_reembedding, - "annotations": False, - "annotations_file": None, - "annotations_dir": None, - "annotations_cell_ontology_enabled": False, - "annotations_cell_ontology_obopath": None, - "annotations_cell_ontology_terms": None, - "custom_colors": dataset_config.presentation__custom_colors, - "diffexp-may-be-slow": False, - "about_legal_tos": dataset_config.app__about_legal_tos, - "about_legal_privacy": dataset_config.app__about_legal_privacy, - } - - # corpora dataset_props - # TODO/Note: putting info from the dataset into the /config is not ideal. - # However, it is definitely not part of /schema, and we do not have a top-level - # route for data properties. Consider creating one at some point. - corpora_props = data_adaptor.get_corpora_props() - if corpora_props and "default_embedding" in corpora_props: - default_embedding = corpora_props["default_embedding"] - if isinstance(default_embedding, str) and default_embedding.startswith("X_"): - default_embedding = default_embedding[2:] # drop X_ prefix - if default_embedding in data_adaptor.get_embedding_names(): - parameters["default_embedding"] = default_embedding - - data_adaptor.update_parameters(parameters) - if annotation: - annotation.update_parameters(parameters, data_adaptor) - - # gather it all together - c = {} - config = c["config"] = {} - config["features"] = features - config["displayNames"] = display_names - config["library_versions"] = library_versions - config["links"] = links - config["parameters"] = parameters - config["corpora_props"] = corpora_props - config["limits"] = { - "column_request_max": server_config.limits__column_request_max, - "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, - } - - if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): - config["authentication"] = { - "requires_client_login": auth.requires_client_login(), - } - if auth.requires_client_login(): - config["authentication"].update({ - "login": auth.get_login_url(data_adaptor), - "logout": auth.get_logout_url(data_adaptor), - }) - - return c - - def get_client_userinfo(self, data_adaptor): - """ - Return the userinfo as required by the /userinfo REST route - """ - - server_config = self.server_config - dataset_config = data_adaptor.dataset_config - auth = server_config.auth - - # make sure the configuration has been checked. - self.check_config() - - if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): - userinfo = {} - userinfo["userinfo"] = { - "is_authenticated": auth.is_user_authenticated(), - "username": auth.get_user_name(), - "user_id": auth.get_user_id() - } - return userinfo - else: - return None - - -class BaseConfig(object): - """This class handles the mechanics of updating and checking attributes. - Derived classes are expected to store the actual attributes""" - - def __init__(self, app_config, default_config, dictval_cases={}): - # reference back to the app_config - self.app_config = app_config - # the complete set of attribute and their default values (unflattened) - self.default_config = default_config - # attributes where the value may be a dict (and therefore are not flattened) - self.dictval_cases = dictval_cases - # used to make sure every attribute value is checked - self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()} - - def create_mapping(self, config): - """Create a mapping from attribute names to (location in the config tree, value)""" - dc = copy.deepcopy(config) - mapping = {} - - # special cases where the value could be a dict. - # If its value is not None, the entry is added to the mapping, and not included - # in the flattening below. - for dictval_case in self.dictval_cases: - cur = dc - for part in dictval_case[:-1]: - cur = cur.get(part, {}) - val = cur.get(dictval_case[-1]) - if val is not None: - key = "__".join(dictval_case) - mapping[key] = (dictval_case, val) - del cur[dictval_case[-1]] - - flat_config = flatten(dc) - for key, value in flat_config.items(): - # name of the attribute - attr = "__".join(key) - mapping[attr] = (key, value) - - return mapping - - def check_attr(self, attrname, vtype): - val = getattr(self, attrname) - if type(vtype) in (list, tuple): - if type(val) not in vtype: - tnames = ",".join([x.__name__ for x in vtype]) - raise ConfigurationError( - f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}" - ) - else: - if type(val) != vtype: - raise ConfigurationError( - f"Invalid type for attribute: {attrname}, " - f"expected type {vtype.__name__}, got {type(val).__name__}" - ) - - self.attr_checked[attrname] = True - - def check_config(self): - mapping = self.create_mapping(self.default_config) - for key in mapping.keys(): - if not self.attr_checked[key]: - raise ConfigurationError(f"The attr '{key}' has not been checked") - - def update(self, **kw): - for key, value in kw.items(): - if not hasattr(self, key): - raise ConfigurationError(f"unknown config parameter {key}.") - try: - if type(value) == tuple: - # convert tuple values to list values - value = list(value) - setattr(self, key, value) - except KeyError: - raise ConfigurationError(f"Unable to set config parameter {key}.") - - self.attr_checked[key] = False - - def update_from_config(self, config, prefix): - mapping = self.create_mapping(config) - for attr, (key, value) in mapping.items(): - if not hasattr(self, attr): - raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}") - try: - setattr(self, attr, value) - except KeyError: - raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}") - - self.attr_checked[attr] = False - - def changes_from_default(self): - """Return all the attribute that are different from the default""" - mapping = self.create_mapping(self.default_config) - diff = [] - for attrname, (key, defval) in mapping.items(): - curval = getattr(self, attrname) - if curval != defval: - diff.append((attrname, curval, defval)) - return diff - - -class ServerConfig(BaseConfig): - """Manages the config attribute associated with the server.""" - - def __init__(self, app_config, default_config): - dictval_cases = [ - ("app", "csp_directives"), - ("authentication", "params_oauth", "cookie"), - ("authentication", "params_oauth", "jwt_decode_options"), - ("adaptor", "cxg_adaptor", "tiledb_ctx"), - ("multi_dataset", "dataroot"), - ] - super().__init__(app_config, default_config, dictval_cases) - - dc = default_config - try: - self.app__verbose = dc["app"]["verbose"] - self.app__debug = dc["app"]["debug"] - self.app__host = dc["app"]["host"] - self.app__port = dc["app"]["port"] - self.app__open_browser = dc["app"]["open_browser"] - self.app__force_https = dc["app"]["force_https"] - self.app__flask_secret_key = dc["app"]["flask_secret_key"] - self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"] - self.app__server_timing_headers = dc["app"]["server_timing_headers"] - self.app__csp_directives = dc["app"]["csp_directives"] - self.app__api_base_url = dc["app"]["api_base_url"] - self.app__web_base_url = dc["app"]["web_base_url"] - - self.authentication__type = dc["authentication"]["type"] - self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][ - "oauth_api_base_url" - ] - self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"] - self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"] - self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][ - "jwt_decode_options"] - self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"] - self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"] - - self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"] - self.multi_dataset__index = dc["multi_dataset"]["index"] - self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"] - self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"] - self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"] - - self.single_dataset__datapath = dc["single_dataset"]["datapath"] - self.single_dataset__obs_names = dc["single_dataset"]["obs_names"] - self.single_dataset__var_names = dc["single_dataset"]["var_names"] - self.single_dataset__about = dc["single_dataset"]["about"] - self.single_dataset__title = dc["single_dataset"]["title"] - - self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"] - self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"] - self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"] - - self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"] - - self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"] - self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"] - - self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"] - self.limits__column_request_max = dc["limits"]["column_request_max"] - - except KeyError as e: - raise ConfigurationError(f"Unexpected config: {str(e)}") - - # The matrix data cache manager is created during the complete_config and stored here. - self.matrix_data_cache_manager = None - - # The authentication object - self.auth = None - - def complete_config(self, context): - self.handle_app(context) - self.handle_data_source(context) - self.handle_authentication(context) - self.handle_data_locator(context) - self.handle_adaptor(context) # may depend on data_locator - self.handle_single_dataset(context) # may depend on adaptor - self.handle_multi_dataset(context) # may depend on adaptor - self.handle_diffexp(context) - self.handle_limits(context) - - self.check_config() - - def handle_app(self, context): - self.check_attr("app__verbose", bool) - self.check_attr("app__debug", bool) - self.check_attr("app__host", str) - self.check_attr("app__port", (type(None), int)) - self.check_attr("app__open_browser", bool) - self.check_attr("app__force_https", bool) - self.check_attr("app__flask_secret_key", (type(None), str)) - self.check_attr("app__generate_cache_control_headers", bool) - self.check_attr("app__server_timing_headers", bool) - self.check_attr("app__csp_directives", (type(None), dict)) - self.check_attr("app__api_base_url", (type(None), str)) - self.check_attr("app__web_base_url", (type(None), str)) - - if self.app__port: - try: - if not is_port_available(self.app__host, self.app__port): - raise ConfigurationError( - f"The port selected {self.app__port} is in use, please configure an open port." - ) - except OverflowError: - raise ConfigurationError(f"Invalid port: {self.app__port}") - else: - try: - default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT)) - except ValueError: - raise ConfigurationError( - "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT") - ) - try: - self.app__port = find_available_port(self.app__host, default_server_port) - except OverflowError: - raise ConfigurationError(f"Invalid port: {default_server_port}") - - if self.app__debug: - context["messagefn"]("in debug mode, setting verbose=True and open_browser=False") - self.app__verbose = True - self.app__open_browser = False - else: - warnings.formatwarning = custom_format_warning - - if not self.app__verbose: - sys.tracebacklimit = 0 - - # secret key: - # first, from CXG_SECRET_KEY environment variable - # second, from config file - self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key) - - # CSP Directives are a dict of string: list(string) or string: string - if self.app__csp_directives is not None: - for k, v in self.app__csp_directives.items(): - if not isinstance(k, str): - raise ConfigurationError("CSP directive names must be a string.") - if isinstance(v, list): - for policy in v: - if not isinstance(policy, str): - raise ConfigurationError("CSP directive value must be a string or list of strings.") - elif not isinstance(v, str): - raise ConfigurationError("CSP directive value must be a string or list of strings.") - - if self.app__web_base_url is None: - self.app__web_base_url = self.app__api_base_url - - def handle_authentication(self, context): - self.check_attr("authentication__type", (type(None), str)) - - # oauth - ptypes = str if self.authentication__type == "oauth" else (type(None), str) - self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes) - self.check_attr("authentication__params_oauth__client_id", ptypes) - self.check_attr("authentication__params_oauth__client_secret", ptypes) - self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict)) - self.check_attr("authentication__params_oauth__session_cookie", bool) - - if self.authentication__params_oauth__session_cookie: - self.check_attr("authentication__params_oauth__cookie", (type(None), dict)) - else: - self.check_attr("authentication__params_oauth__cookie", dict) - # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable - # second, from config file - self.authentication__params__oauth__client_secret = os.environ.get( - "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret) - - self.auth = AuthTypeFactory.create(self.authentication__type, self) - if self.auth is None: - raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") - - def handle_data_locator(self, context): - self.check_attr("data_locator__s3__region_name", (type(None), bool, str)) - if self.data_locator__s3__region_name is True: - path = self.single_dataset__datapath or self.multi_dataset__dataroot - if type(path) == dict: - # if multi_dataset__dataroot is a dict, then use the first key - # that is in s3. NOTE: it is not supported to have dataroots - # in different regions. - paths = [val.get("dataroot") for val in path.values()] - for path in paths: - if path.startswith("s3://"): - break - if path.startswith("s3://"): - region_name = discover_s3_region_name(path) - if region_name is None: - raise ConfigurationError(f"Unable to discover s3 region name from {path}") - else: - region_name = None - self.data_locator__s3__region_name = region_name - - def handle_data_source(self, context): - self.check_attr("single_dataset__datapath", (str, type(None))) - self.check_attr("multi_dataset__dataroot", (type(None), dict, str)) - - if self.single_dataset__datapath is None: - if self.multi_dataset__dataroot is None: - # TODO: change the error message once dataroot is fully supported - raise ConfigurationError("missing datapath") - return - else: - if self.multi_dataset__dataroot is not None: - raise ConfigurationError("must supply only one of datapath or dataroot") - - def handle_single_dataset(self, context): - self.check_attr("single_dataset__datapath", (str, type(None))) - self.check_attr("single_dataset__title", (str, type(None))) - self.check_attr("single_dataset__about", (str, type(None))) - self.check_attr("single_dataset__obs_names", (str, type(None))) - self.check_attr("single_dataset__var_names", (str, type(None))) - - if self.single_dataset__datapath is None: - return - - # create the matrix data cache manager: - if self.matrix_data_cache_manager is None: - self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None) - - # preload this data set - matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config) - try: - matrix_data_loader.pre_load_validation() - except DatasetAccessError as e: - raise ConfigurationError(str(e)) - - file_size = matrix_data_loader.file_size() - file_basename = basename(self.single_dataset__datapath) - if file_size > BIG_FILE_SIZE_THRESHOLD: - context["messagefn"](f"Loading data from {file_basename}, this may take a while...") - else: - context["messagefn"](f"Loading data from {file_basename}.") - - if self.single_dataset__about: - - def url_check(url): - try: - result = urlparse(url) - if all([result.scheme, result.netloc]): - return True - else: - return False - except ValueError: - return False - - if not url_check(self.single_dataset__about): - raise ConfigurationError( - "Must provide an absolute URL for --about. (Example format: http://example.com)" - ) - - def handle_multi_dataset(self, context): - self.check_attr("multi_dataset__dataroot", (type(None), dict, str)) - self.check_attr("multi_dataset__index", (type(None), bool, str)) - self.check_attr("multi_dataset__allowed_matrix_types", list) - self.check_attr("multi_dataset__matrix_cache__max_datasets", int) - self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)) - - if self.multi_dataset__dataroot is None: - return - - if type(self.multi_dataset__dataroot) == str: - default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot) - self.multi_dataset__dataroot = dict(d=default_dict) - - for tag, dataroot_dict in self.multi_dataset__dataroot.items(): - if "base_url" not in dataroot_dict: - raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}") - if "dataroot" not in dataroot_dict: - raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}") - - base_url = dataroot_dict["base_url"] - - # sanity check for well formed base urls - bad = False - if type(base_url) != str: - bad = True - elif os.path.normpath(base_url) != base_url: - bad = True - else: - base_url_parts = base_url.split("/") - if [quote_plus(part) for part in base_url_parts] != base_url_parts: - bad = True - if ".." in base_url_parts: - bad = True - if bad: - raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}") - - # verify all the base_urls are unique - base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()] - if len(base_urls) > len(set(base_urls)): - raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique") - - # error checking - for mtype in self.multi_dataset__allowed_matrix_types: - try: - MatrixDataType(mtype) - except ValueError: - raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}') - - # create the matrix data cache manager: - if self.matrix_data_cache_manager is None: - self.matrix_data_cache_manager = MatrixDataCacheManager( - max_cached=self.multi_dataset__matrix_cache__max_datasets, - timelimit_s=self.multi_dataset__matrix_cache__timelimit_s, - ) - - def handle_diffexp(self, context): - self.check_attr("diffexp__alg_cxg__max_workers", (str, int)) - self.check_attr("diffexp__alg_cxg__cpu_multiplier", int) - self.check_attr("diffexp__alg_cxg__target_workunit", int) - - max_workers = self.diffexp__alg_cxg__max_workers - cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier - cpu_count = os.cpu_count() - max_workers = min(max_workers, cpu_multiplier * cpu_count) - diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit) - - def handle_adaptor(self, context): - # cxg - self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict) - regionkey = "vfs.s3.region" - if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx: - if type(self.data_locator__s3__region_name) == str: - self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name - - from server.data_cxg.cxg_adaptor import CxgAdaptor - - CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx) - - # anndata - self.check_attr("adaptor__anndata_adaptor__backed", bool) - - def handle_limits(self, context): - self.check_attr("limits__diffexp_cellcount_max", (type(None), int)) - self.check_attr("limits__column_request_max", (type(None), int)) - - def exceeds_limit(self, limit_name, value): - limit_value = getattr(self, "limits__" + limit_name, None) - if limit_value is None: # disabled - return False - return value > limit_value - - def get_api_base_url(self): - if self.app__api_base_url == "local": - return f"http://{self.app__host}:{self.app__port}" - if self.app__api_base_url and self.app__api_base_url.endswith("/"): - return self.app__api_base_url[:-1] - return self.app__api_base_url - - def get_web_base_url(self): - if self.app__web_base_url == "local": - return f"http://{self.app__host}:{self.app__port}" - if self.app__web_base_url is None: - return self.get_api_base_url() - if self.app__web_base_url.endswith("/"): - return self.app__web_base_url[:-1] - return self.api__web_base_url - - -class DatasetConfig(BaseConfig): - """Manages the config attribute associated with a dataset.""" - - def __init__(self, tag, app_config, default_config): - super().__init__(app_config, default_config) - self.tag = tag - dc = default_config - try: - self.app__scripts = dc["app"]["scripts"] - self.app__inline_scripts = dc["app"]["inline_scripts"] - self.app__about_legal_tos = dc["app"]["about_legal_tos"] - self.app__about_legal_privacy = dc["app"]["about_legal_privacy"] - self.app__authentication_enable = dc["app"]["authentication_enable"] - - self.presentation__max_categories = dc["presentation"]["max_categories"] - self.presentation__custom_colors = dc["presentation"]["custom_colors"] - - self.user_annotations__enable = dc["user_annotations"]["enable"] - self.user_annotations__type = dc["user_annotations"]["type"] - self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"] - self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"] - self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"] - self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"] - self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"] - self.user_annotations__hosted_tiledb_array__hosted_file_directory = \ - dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ] # noqa E501 - - self.embeddings__names = dc["embeddings"]["names"] - self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"] - - self.diffexp__enable = dc["diffexp"]["enable"] - self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"] - self.diffexp__top_n = dc["diffexp"]["top_n"] - - except KeyError as e: - raise ConfigurationError(f"Unexpected config: {str(e)}") - - # The annotation object is created during complete_config and stored here. - self.user_annotations = None - - def complete_config(self, context): - self.handle_app(context) - self.handle_presentation(context) - self.handle_user_annotations(context) - self.handle_embeddings(context) - self.handle_diffexp(context) - - def handle_app(self, context): - self.check_attr("app__scripts", list) - self.check_attr("app__inline_scripts", list) - self.check_attr("app__about_legal_tos", (type(None), str)) - self.check_attr("app__about_legal_privacy", (type(None), str)) - self.check_attr("app__authentication_enable", bool) - - # scripts can be string (filename) or dict (attributes). Convert string to dict. - scripts = [] - for s in self.app__scripts: - if isinstance(s, str): - scripts.append({"src": s}) - elif isinstance(s, dict) and isinstance(s["src"], str): - scripts.append(s) - else: - raise ConfigurationError("Scripts must be string or dict") - self.app__scripts = scripts - - def handle_presentation(self, context): - self.check_attr("presentation__max_categories", int) - self.check_attr("presentation__custom_colors", bool) - - def handle_user_annotations(self, context): - self.check_attr("user_annotations__enable", bool) - self.check_attr("user_annotations__type", str) - self.check_attr("user_annotations__local_file_csv__directory", (type(None), str)) - self.check_attr("user_annotations__local_file_csv__file", (type(None), str)) - self.check_attr("user_annotations__ontology__enable", bool) - self.check_attr("user_annotations__ontology__obo_location", (type(None), str)) - self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str)) - self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)) - - if self.user_annotations__enable: - server_config = self.app_config.server_config - if not self.app__authentication_enable: - raise ConfigurationError("user annotations requires authentication to be enabled") - if not server_config.auth.is_valid_authentication_type(): - auth_type = server_config.authentication__type - raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") - - # TODO, replace this with a factory pattern once we have more than one way - # to do annotations. currently only local_file_csv - if self.user_annotations__type == "local_file_csv": - dirname = self.user_annotations__local_file_csv__directory - filename = self.user_annotations__local_file_csv__file - - if filename is not None and dirname is not None: - raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.") - - if filename is not None: - lf_name, lf_ext = splitext(filename) - if lf_ext and lf_ext != ".csv": - raise ConfigurationError(f"annotation file type must be .csv: {filename}") - - if dirname is not None and not isdir(dirname): - try: - os.mkdir(dirname) - except OSError: - raise ConfigurationError("Unable to create directory specified by --annotations-dir") - - self.user_annotations = AnnotationsLocalFile(dirname, filename) - - # if the user has specified a fixed label file, go ahead and validate it - # so that we can remove errors early in the process. - server_config = self.app_config.server_config - if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file: - with server_config.matrix_data_cache_manager.data_adaptor( - self.tag, server_config.single_dataset__datapath, self.app_config - ) as data_adaptor: - data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) - - if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: - try: - self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) - except OntologyLoadFailure as e: - raise ConfigurationError("Unable to load ontology terms\n" + str(e)) - elif self.user_annotations__type == "hosted_tiledb_array": - self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str) - self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str) - self.user_annotations = AnnotationsHostedTileDB( - directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory, - db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri), - ) - else: - raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array') - else: - if self.user_annotations__type == "local_file_csv": - dirname = self.user_annotations__local_file_csv__directory - filename = self.user_annotations__local_file_csv__file - if filename is not None: - context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.") - if dirname is not None: - context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.") - - if self.user_annotations__ontology__enable: - context["messagefn"]( - "Warning: --experimental-annotations-ontology" " ignored as annotations are disabled." - ) - if self.user_annotations__ontology__obo_location is not None: - context["messagefn"]( - "Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled." - ) - - def handle_embeddings(self, context): - self.check_attr("embeddings__names", list) - self.check_attr("embeddings__enable_reembedding", bool) - - server_config = self.app_config.server_config - if self.embeddings__enable_reembedding: - if server_config.single_dataset__datapath: - matrix_data_loader = MatrixDataLoader( - server_config.single_dataset__datapath, app_config=self.app_config - ) - if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD: - raise ConfigurationError("enable-reembedding is only supported with H5AD files.") - if server_config.adaptor__anndata_adaptor__backed: - raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.") - - try: - server.compute.scanpy.get_scanpy_module() - except NotImplementedError: - raise ConfigurationError("Please install scanpy to enable UMAP re-embedding") - - def handle_diffexp(self, context): - self.check_attr("diffexp__enable", bool) - self.check_attr("diffexp__lfc_cutoff", float) - self.check_attr("diffexp__top_n", int) - - server_config = self.app_config.server_config - if server_config.single_dataset__datapath: - with server_config.matrix_data_cache_manager.data_adaptor( - self.tag, server_config.single_dataset__datapath, self.app_config - ) as data_adaptor: - if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): - context["messagefn"]( - "CAUTION: due to the size of your dataset, " - "running differential expression may take longer or fail." - ) diff --git a/server/common/aws_secret_utils.py b/server/common/aws_secret_utils.py index 47f7690b..ce40794d 100644 --- a/server/common/aws_secret_utils.py +++ b/server/common/aws_secret_utils.py @@ -1,72 +1,11 @@ import logging -import os -import sys import boto3 from flask import json -from server.common.data_locator import discover_s3_region_name from server.common.errors import SecretKeyRetrievalError -def handle_config_from_secret(app_config): - """Update configuration from the secret manager""" - secret_name = os.getenv("CXG_AWS_SECRET_NAME") - if not secret_name: - return - - # need to find the secret manager region. - # 1. from CXG_AWS_SECRET_REGION_NAME - # 2. discover from dataroot location (if on s3) - # 3. discover from config file location (if on s3) - secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME") - if secret_region_name is None: - secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) - if not secret_region_name: - from server.eb.app import config_file - secret_region_name = discover_s3_region_name(config_file) - if not secret_region_name: - logging.error("Could not determine the AWS Secret Manager region") - sys.exit(1) - - secrets = get_secret_key(secret_region_name, secret_name) - - if not secrets: - return - - server_attrs = ( - ("flask_secret_key", "app__flask_secret_key"), - ("oauth_client_secret", "authentication__params_oauth__client_secret"), - ) - default_dataset_attrs = ( - ("db_uri", "user_annotations__hosted_tiledb_array__db_uri"), - ) - - # update server configuration attributes - for key, attr in server_attrs: - cur_val = getattr(app_config.server_config, attr) - if cur_val: - continue - - # replace the attr with the secret if it is not set - val = secrets.get(key) - if val: - logging.info(f"set {attr} from secret") - app_config.update_server_config(**{attr : val}) - - # update default dataset configuration attributes - for key, attr in default_dataset_attrs: - cur_val = getattr(app_config.default_dataset_config, attr) - if cur_val: - continue - - # replace the attr with the secret if it is not set - val = secrets.get(key) - if val: - logging.info(f"set {attr} from secret") - app_config.update_default_dataset_config(**{attr : val}) - - def get_secret_key(region_name, secret_name): session = boto3.session.Session() client = session.client(service_name="secretsmanager", region_name=region_name) @@ -79,6 +18,6 @@ def get_secret_key(region_name, secret_name): return secret except Exception as e: logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True) - raise SecretKeyRetrievalError + raise SecretKeyRetrievalError(str(e)) return None diff --git a/server/common/config/__init__.py b/server/common/config/__init__.py new file mode 100644 index 00000000..8a74427b --- /dev/null +++ b/server/common/config/__init__.py @@ -0,0 +1,4 @@ +from server.common.aws_secret_utils import get_secret_key # noqa F504 + +DEFAULT_SERVER_PORT = 5005 +BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB diff --git a/server/common/config/app_config.py b/server/common/config/app_config.py new file mode 100644 index 00000000..5cd4c4a5 --- /dev/null +++ b/server/common/config/app_config.py @@ -0,0 +1,247 @@ +import yaml +from flatten_dict import unflatten + +from server.default_config import get_default_config +from server.common.config.dataset_config import DatasetConfig +from server.common.config.server_config import ServerConfig +from server.common.config.external_config import ExternalConfig +from server.common.errors import ConfigurationError + + +class AppConfig(object): + """ + AppConfig stores all the configuration for cellxgene. + AppConfig contains one or more DatasetConfig(s) and one ServerConfig. + The server_config contains attributes that refer to the server process as a whole. + The default_dataset_config refers to attributes that are associated with the features and + presentations of a dataset. + The dataset config attributes can be overridden depending on the url by which the + dataset was accessed. These are stored in dataroot_config. + AppConfig has methods to initialize, modify, and access the configuration. + """ + + def __init__(self): + + # the default configuration (see default_config.py) + # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as + # defaults within the config class? + self.default_config = get_default_config() + # the server configuration + self.server_config = ServerConfig(self, self.default_config["server"]) + # the dataset config, unless overridden by an entry in dataroot_config + self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"]) + # a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot + # attribute of the server_config. The default dataset config will apply to all datasets unless a different set + # of config vars was passed for a specific dataset under the multidataset config. For example: + """ + per_dataset_config: + d1: + user_annotations: + enable: false + d2: + user_annotations: + enable: true + """ + # dataroot config + self.dataroot_config = {} + + # external config + self.external_config = ExternalConfig(self, self.default_config["external"]) + + # Set to true when config_completed is called + self.is_completed = False + + def get_dataset_config(self, dataroot_key): + if self.server_config.single_dataset__datapath: + return self.default_dataset_config + else: + return self.dataroot_config.get(dataroot_key, self.default_dataset_config) + + def check_config(self): + """Verify all the attributes in the config have been type checked""" + if not self.is_completed: + raise ConfigurationError("The configuration has not been completed") + self.server_config.check_config() + self.default_dataset_config.check_config() + for dataset_config in self.dataroot_config.values(): + dataset_config.check_config() + self.external_config.check_config() + + def update_server_config(self, **kw): + self.server_config.update(**kw) + self.is_complete = False + + def update_default_dataset_config(self, **kw): + self.default_dataset_config.update(**kw) + # update all the other dataset configs, if any + for value in self.dataroot_config.values(): + value.update(**kw) + self.is_complete = False + + def update_single_config_from_path_and_value(self, path, value): + """Update a single config parameter with the value. + Path is a list of string, that gives a path to the config parameter to be updated. + For example, path may be ["server","app","port"]. + """ + self.is_complete = False + if not isinstance(path, list): + raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'") + for part in path: + if not isinstance(part, str): + raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'") + + if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"): + raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'") + + if path[0] == "server": + attr = "__".join(path[1:]) + try: + self.update_server_config(**{attr: value}) + except ConfigurationError: + raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'") + elif path[0] == "dataset": + attr = "__".join(path[1:]) + try: + self.update_default_dataset_config(**{attr: value}) + except ConfigurationError: + raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'") + + elif path[0] == "per_dataset_config": + if len(path) < 2: + raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'") + dataroot = path[1] + if dataroot not in self.dataroot_config: + dataroots = str(list(self.dataroot_config.keys())) + raise ConfigurationError( + f"unknown dataroot when using per_dataset_config: got '{path}'," + f" dataroots specified in config are {dataroots}" + ) + + attr = "__".join(path[2:]) + try: + self.dataroot_config[dataroot].update(**{attr: value}) + except ConfigurationError: + raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'") + + def update_from_config_file(self, config_file): + try: + with open(config_file) as yml_file: + config = yaml.safe_load(yml_file) + except yaml.YAMLError as e: + raise ConfigurationError(f"The specified config file contained an error: {e}") + except OSError as e: + raise ConfigurationError(f"Issue retrieving the specified config file: {e}") + + if config.get("server"): + self.server_config.update_from_config(config["server"], "server") + if config.get("dataset"): + self.default_dataset_config.update_from_config(config["dataset"], "dataset") + + per_dataset_config = config.get("per_dataset_config", {}) + for key, dataroot_config in per_dataset_config.items(): + # first create and initialize the dataroot with the default config + self.add_dataroot_config(key, **config["dataset"]) + # then apply the per dataset configuration + self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}") + + if config.get("external"): + self.external_config.update_from_config(config["external"], "external") + + self.is_complete = False + + def config_to_dict(self): + """return the configuration as an unflattened dict""" + server = self.server_config.create_mapping(self.server_config.default_config) + dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) + external = self.external_config.create_mapping(self.external_config.default_config) + config = dict(server={}, dataset={}) + for attrname in server.keys(): + config["server__" + attrname] = getattr(self.server_config, attrname) + for attrname in dataset.keys(): + config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname) + if self.dataroot_config: + config["per_dataset_config"] = {} + for dataroot_tag, dataroot_config in self.dataroot_config.items(): + dataset = dataroot_config.create_mapping(dataroot_config.default_config) + for attrname in dataset.keys(): + config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname) + for attrname in external.keys(): + config["external__" + attrname] = getattr(self.external_config, attrname) + + config = unflatten(config, splitter=lambda key: key.split("__")) + return config + + def write_config(self, config_file): + """output the config to a yaml file""" + config = self.config_to_dict() + yaml.dump(config, open(config_file, "w")) + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + diff_server = self.server_config.changes_from_default() + diff_dataset = self.default_dataset_config.changes_from_default() + diff_external = self.external.changes_from_default() + diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external) + return diff + + def add_dataroot_config(self, dataroot_tag, **kw): + """Create a new dataset config object based on the default dataset config, and kw parameters""" + if dataroot_tag in self.dataroot_config: + raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}") + if type(self.server_config.multi_dataset__dataroot) != dict: + raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary") + if dataroot_tag not in self.server_config.multi_dataset__dataroot: + raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot") + + self.is_completed = False + self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"]) + flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) + config = {key: value[1] for key, value in flat_config.items()} + self.dataroot_config[dataroot_tag].update(**config) + self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag) + + def complete_config(self, messagefn=None): + """The configure options are checked, and any additional setup based on the config + parameters is done""" + + if messagefn is None: + + def noop(message): + pass + + messagefn = noop + + # TODO: to give better error messages we can add a mapping between where each config + # attribute originated (e.g. command line argument or config file), then in the error + # messages we can give correct context for attributes with bad value. + context = dict(messagefn=messagefn) + + # complete config for external_config first, since this may update values in the other sections + self.external_config.complete_config(context) + self.server_config.complete_config(context) + self.default_dataset_config.complete_config(context) + for dataroot_config in self.dataroot_config.values(): + dataroot_config.complete_config(context) + + self.is_completed = True + self.check_config() + + def get_matrix_data_cache_manager(self): + return self.server_config.matrix_data_cache_manager + + def is_multi_dataset(self): + return self.server_config.multi_dataset__dataroot is not None + + def get_title(self, data_adaptor): + return ( + self.server_config.single_dataset__title + if self.server_config.single_dataset__title + else data_adaptor.get_title() + ) + + def get_about(self, data_adaptor): + return ( + self.server_config.single_dataset__about + if self.server_config.single_dataset__about + else data_adaptor.get_about() + ) diff --git a/server/common/config/base_config.py b/server/common/config/base_config.py new file mode 100644 index 00000000..6b9087e7 --- /dev/null +++ b/server/common/config/base_config.py @@ -0,0 +1,132 @@ +import copy + +from flatten_dict import flatten +from server.common.errors import ConfigurationError + + +class BaseConfig(object): + """ + This class handles the mechanics of updating and checking attributes. + Derived classes are expected to store the actual attributes + Currently DatasetConfig and ServerConfig both inherit from BaseConfig. + """ + + def __init__(self, app_config, default_config, dictval_cases={}): + # reference back to the app_config + self.app_config = app_config + # the complete set of attributes and their default values (unflattened) + self.default_config = default_config + # attributes where the value may be a dict (and therefore are not flattened) + self.dictval_cases = dictval_cases + # used to make sure every attribute value is checked + self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()} + + def create_mapping(self, config): + """ + Create a dictionary where the keys are the name of attributes (using double underscore convention) + For example: authentication__type + + The values are a tuple, + - the first item of the tuple is a tuple of path elements (location in config 'tree') + - the second item is the value of the config parameter + + For example: (('authentication', 'type'), 'session')) + """ + config_copy = copy.deepcopy(config) + mapping = {} + + # special cases where the value could be a dict. + # If its value is not None, the entry is added to the mapping, and not included + # in the flattening below. + for dictval_case in self.dictval_cases: + cur = config_copy + for part in dictval_case[:-1]: + cur = cur.get(part, {}) + val = cur.get(dictval_case[-1]) + if val is not None: + key = "__".join(dictval_case) + mapping[key] = (dictval_case, val) + del cur[dictval_case[-1]] + + flat_config = flatten(config_copy) + for key, value in flat_config.items(): + # name of the attribute + attr = "__".join(key) + mapping[attr] = (key, value) + + return mapping + + def validate_correct_type_of_configuration_attribute(self, attrname, vtype): + val = getattr(self, attrname) + if type(vtype) in (list, tuple): + if type(val) not in vtype: + tnames = ",".join([x.__name__ for x in vtype]) + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}" + ) + else: + if type(val) != vtype: + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, " + f"expected type {vtype.__name__}, got {type(val).__name__}" + ) + + self.attr_checked[attrname] = True + + def check_config(self): + mapping = self.create_mapping(self.default_config) + for key in mapping.keys(): + if not self.attr_checked[key]: + raise ConfigurationError(f"The attr '{key}' has not been checked") + + def update(self, **kw): + """Update the attributes defined in kw with their new values.""" + for key, value in kw.items(): + if not hasattr(self, key): + + # check if the key is setting into a dictval entry. + found_dictval = False + for dictval in self.dictval_cases: + dictvalname = "__".join(dictval) + if dictvalname + "__" in key: + dictkey = key[len(dictvalname) + 2 :] + curdictval = getattr(self, dictvalname) + if curdictval is None: + setattr(self, dictvalname, dict(dictkey=value)) + else: + curdictval[dictkey] = value + + found_dictval = True + break + + if found_dictval: + continue + raise ConfigurationError(f"unknown config parameter {key}.") + try: + if type(value) == tuple: + # convert tuple values to list values + value = list(value) + setattr(self, key, value) + except KeyError: + raise ConfigurationError(f"Unable to set config parameter {key}.") + + self.attr_checked[key] = False + + def update_from_config(self, config, prefix): + mapping = self.create_mapping(config) + for attr, (key, value) in mapping.items(): + if not hasattr(self, attr): + raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}") + setattr(self, attr, value) + + self.attr_checked[attr] = False + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + mapping = self.create_mapping(self.default_config) + diff = [] + for attrname, (key, defval) in mapping.items(): + curval = getattr(self, attrname) + if curval != defval: + diff.append((attrname, curval, defval)) + return diff diff --git a/server/common/config/client_config.py b/server/common/config/client_config.py new file mode 100644 index 00000000..8a70c0b6 --- /dev/null +++ b/server/common/config/client_config.py @@ -0,0 +1,122 @@ +from server import display_version as cellxgene_display_version + + +def get_client_config(app_config, data_adaptor): + """ + Return the configuration as required by the /config REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + annotation = dataset_config.user_annotations + auth = server_config.auth + + # FIXME The current set of config is not consistently presented: + # we have camalCase, hyphen-text, and underscore_text + + # make sure the configuration has been checked. + app_config.check_config() + + # display_names + title = app_config.get_title(data_adaptor) + about = app_config.get_about(data_adaptor) + + display_names = dict(engine=data_adaptor.get_name(), dataset=title) + + # library_versions + library_versions = {} + library_versions.update(data_adaptor.get_library_versions()) + library_versions["cellxgene"] = cellxgene_display_version + + # links + links = {"about-dataset": about} + + # parameters + parameters = { + "layout": dataset_config.embeddings__names, + "max-category-items": dataset_config.presentation__max_categories, + "obs_names": server_config.single_dataset__obs_names, + "var_names": server_config.single_dataset__var_names, + "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff, + "backed": server_config.adaptor__anndata_adaptor__backed, + "disable-diffexp": not dataset_config.diffexp__enable, + "enable-reembedding": dataset_config.embeddings__enable_reembedding, + "annotations": False, + "annotations_file": None, + "annotations_dir": None, + "annotations_cell_ontology_enabled": False, + "annotations_cell_ontology_obopath": None, + "annotations_cell_ontology_terms": None, + "custom_colors": dataset_config.presentation__custom_colors, + "diffexp-may-be-slow": False, + "about_legal_tos": dataset_config.app__about_legal_tos, + "about_legal_privacy": dataset_config.app__about_legal_privacy, + } + + # corpora dataset_props + # TODO/Note: putting info from the dataset into the /config is not ideal. + # However, it is definitely not part of /schema, and we do not have a top-level + # route for data properties. Consider creating one at some point. + corpora_props = data_adaptor.get_corpora_props() + if corpora_props and "default_embedding" in corpora_props: + default_embedding = corpora_props["default_embedding"] + if isinstance(default_embedding, str) and default_embedding.startswith("X_"): + default_embedding = default_embedding[2:] # drop X_ prefix + if default_embedding in data_adaptor.get_embedding_names(): + parameters["default_embedding"] = default_embedding + + data_adaptor.update_parameters(parameters) + if annotation: + annotation.update_parameters(parameters, data_adaptor) + + # gather it all together + client_config = {} + config = client_config["config"] = {} + config["displayNames"] = display_names + config["library_versions"] = library_versions + config["links"] = links + config["parameters"] = parameters + config["corpora_props"] = corpora_props + config["limits"] = { + "column_request_max": server_config.limits__column_request_max, + "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, + } + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + config["authentication"] = { + "requires_client_login": auth.requires_client_login(), + } + if auth.requires_client_login(): + config["authentication"].update( + { + # Todo why are these stored on the data_adaptor? + "login": auth.get_login_url(data_adaptor), + "logout": auth.get_logout_url(data_adaptor), + } + ) + + return client_config + + +def get_client_userinfo(app_config, data_adaptor): + """ + Return the userinfo as required by the /userinfo REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + auth = server_config.auth + + # make sure the configuration has been checked. + app_config.check_config() + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + userinfo = {} + userinfo["userinfo"] = { + "is_authenticated": auth.is_user_authenticated(), + "username": auth.get_user_name(), + "user_id": auth.get_user_id(), + "email": auth.get_user_email(), + "picture": auth.get_user_picture(), + } + return userinfo diff --git a/server/common/config/dataset_config.py b/server/common/config/dataset_config.py new file mode 100644 index 00000000..7586ec2f --- /dev/null +++ b/server/common/config/dataset_config.py @@ -0,0 +1,230 @@ +import os +from os.path import splitext, isdir + +from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB +from server.common.annotations.local_file_csv import AnnotationsLocalFile +from server.common.config.base_config import BaseConfig +from server.common.errors import ConfigurationError, OntologyLoadFailure +from server.compute.scanpy import get_scanpy_module +from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType +from server.db.db_utils import DbUtils + + +class DatasetConfig(BaseConfig): + """Manages the config attribute associated with a dataset.""" + + def __init__(self, tag, app_config, default_config): + super().__init__(app_config, default_config) + self.tag = tag + try: + self.app__scripts = default_config["app"]["scripts"] + self.app__inline_scripts = default_config["app"]["inline_scripts"] + self.app__about_legal_tos = default_config["app"]["about_legal_tos"] + self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"] + self.app__authentication_enable = default_config["app"]["authentication_enable"] + + self.presentation__max_categories = default_config["presentation"]["max_categories"] + self.presentation__custom_colors = default_config["presentation"]["custom_colors"] + + self.user_annotations__enable = default_config["user_annotations"]["enable"] + self.user_annotations__type = default_config["user_annotations"]["type"] + self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][ + "directory" + ] + self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"] + self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"] + self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][ + "obo_location" + ] + self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][ + "hosted_tiledb_array" + ]["db_uri"] + self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][ + "hosted_tiledb_array" + ]["hosted_file_directory"] + + self.embeddings__names = default_config["embeddings"]["names"] + self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"] + + self.diffexp__enable = default_config["diffexp"]["enable"] + self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"] + self.diffexp__top_n = default_config["diffexp"]["top_n"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + # The annotation object is created during complete_config and stored here. + self.user_annotations = None + + def complete_config(self, context): + self.handle_app() + self.handle_presentation() + self.handle_user_annotations(context) + self.handle_embeddings() + self.handle_diffexp(context) + + def handle_app(self): + self.validate_correct_type_of_configuration_attribute("app__scripts", list) + self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list) + self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool) + + # scripts can be string (filename) or dict (attributes). Convert string to dict. + scripts = [] + for script in self.app__scripts: + try: + if isinstance(script, str): + scripts.append({"src": script}) + elif isinstance(script, dict) and isinstance(script["src"], str): + scripts.append(script) + else: + raise Exception + except Exception as e: + raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}") + + self.app__scripts = scripts + + def handle_presentation(self): + self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int) + self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool) + + def handle_user_annotations(self, context): + self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool) + self.validate_correct_type_of_configuration_attribute("user_annotations__type", str) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__directory", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__file", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__ontology__obo_location", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__hosted_tiledb_array__db_uri", (type(None), str) + ) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str) + ) + if self.user_annotations__enable: + server_config = self.app_config.server_config + if not self.app__authentication_enable: + raise ConfigurationError("user annotations requires authentication to be enabled") + if not server_config.auth.is_valid_authentication_type(): + auth_type = server_config.authentication__type + raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") + + if self.user_annotations__type == "local_file_csv": + self.handle_local_file_csv_annotations() + elif self.user_annotations__type == "hosted_tiledb_array": + self.handle_hosted_tiledb_annotations() + else: + raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array') + if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: + try: + self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) + except OntologyLoadFailure as e: + raise ConfigurationError("Unable to load ontology terms\n" + str(e)) + else: + self.check_annotation_config_vars_not_set(context) + + def handle_local_file_csv_annotations(self): + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + if filename is not None and dirname is not None: + raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.") + + if filename is not None: + lf_name, lf_ext = splitext(filename) + if lf_ext and lf_ext != ".csv": + raise ConfigurationError(f"annotation file type must be .csv: {filename}") + + if dirname is not None and not isdir(dirname): + try: + os.mkdir(dirname) + except OSError: + raise ConfigurationError("Unable to create directory specified by --annotations-dir") + + self.user_annotations = AnnotationsLocalFile(dirname, filename) + + # if the user has specified a fixed label file, go ahead and validate it + # so that we can remove errors early in the process. + server_config = self.app_config.server_config + if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file: + with server_config.matrix_data_cache_manager.data_adaptor( + self.tag, server_config.single_dataset__datapath, self.app_config + ) as data_adaptor: + data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) + + def handle_hosted_tiledb_annotations(self): + self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__hosted_tiledb_array__hosted_file_directory", str + ) + self.user_annotations = AnnotationsHostedTileDB( + directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory, + db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri), + ) + + def check_annotation_config_vars_not_set(self, context): + if self.user_annotations__type is not None: + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + db_uri = self.user_annotations__hosted_tiledb_array__db_uri + hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory + if filename is not None: + context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.") + if dirname is not None: + context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.") + if db_uri is not None: + context["messagefn"]("Warning: db_uri ignored as annotations are disabled.") + if hosted_file_dirname is not None: + context["messagefn"]( + "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled." + ) + + if self.user_annotations__ontology__enable: + context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.") + if self.user_annotations__ontology__obo_location is not None: + context["messagefn"]( + "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled." + ) + + def handle_embeddings(self): + self.validate_correct_type_of_configuration_attribute("embeddings__names", list) + self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool) + + server_config = self.app_config.server_config + if self.embeddings__enable_reembedding: + if server_config.single_dataset__datapath: + matrix_data_loader = MatrixDataLoader( + server_config.single_dataset__datapath, app_config=self.app_config + ) + if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD: + raise ConfigurationError("enable-reembedding is only supported with H5AD files.") + if server_config.adaptor__anndata_adaptor__backed: + raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.") + + try: + get_scanpy_module() + except NotImplementedError: + # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported + raise ConfigurationError("Please install scanpy to enable UMAP re-embedding") + + def handle_diffexp(self, context): + self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool) + self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float) + self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int) + + server_config = self.app_config.server_config + if server_config.single_dataset__datapath: + with server_config.matrix_data_cache_manager.data_adaptor( + self.tag, server_config.single_dataset__datapath, self.app_config + ) as data_adaptor: + if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): + context["messagefn"]( + "CAUTION: due to the size of your dataset, " + "running differential expression may take longer or fail." + ) diff --git a/server/common/config/external_config.py b/server/common/config/external_config.py new file mode 100644 index 00000000..bebfbfa9 --- /dev/null +++ b/server/common/config/external_config.py @@ -0,0 +1,96 @@ +import os + +from server.common.config.base_config import BaseConfig +from server.common.errors import ConfigurationError +from server.common.config import get_secret_key +from server.common.errors import SecretKeyRetrievalError +from server.common.utils.type_conversion_utils import convert_string_to_value + + +class ExternalConfig(BaseConfig): + """Manages the config attribute associated with external configuration sources, such as + environment variables or the AWS Secrets Manager.""" + + def __init__(self, app_config, default_config): + super().__init__(app_config, default_config) + try: + self.environment = default_config["environment"] + self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"] + self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + def complete_config(self, context): + self.handle_environment(context) + self.handle_aws_secrets_manager(context) + + def handle_environment(self, context): + """For each environment variable defined, get the value (if it is set), + and set the specified config parameter""" + self.validate_correct_type_of_configuration_attribute("environment", list) + for envdict in self.environment: + name = envdict.get("name") + if name is None: + raise ConfigurationError("environment: 'name' is missing") + required = envdict.get("required", False) + if type(required) != bool: + raise ConfigurationError("environment: 'required' must be a bool") + path = envdict.get("path") + if path is None: + raise ConfigurationError("environment: 'path' is missing") + + value = os.environ.get(name) + if value is None: + if required: + raise ConfigurationError(f"required environment variable '{name}' not set") + else: + value = convert_string_to_value(value) + self.app_config.update_single_config_from_path_and_value(path, value) + + def handle_aws_secrets_manager(self, context): + """For each aws secret defined, get the key/values, and set the specified config parameter""" + self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list) + + if not self.aws_secrets_manager__secrets: + return + + self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str) + + for secret in self.aws_secrets_manager__secrets: + secret_name = secret.get("name") + if secret_name is None: + raise ConfigurationError("aws_secrets_manager: 'name' is missing") + if not isinstance(secret_name, str): + raise ConfigurationError("aws_secrets_manager: 'name' must be a string") + + try: + secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name) + except SecretKeyRetrievalError as e: + raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}") + + values = secret.get("values") + if values is None: + raise ConfigurationError("aws_secrets_manager: 'values' is missing") + if not isinstance(values, list): + raise ConfigurationError("aws_secrets_manager: 'values' must be a list") + + for value in values: + key = value.get("key") + if key is None: + raise ConfigurationError(f"missing 'key' in secret values: {secret_name}") + path = value.get("path") + if path is None: + raise ConfigurationError(f"missing 'path' in secret values: {secret_name}") + required = value.get("required", False) + if type(required) != bool: + raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}") + + secret_value = secret_dict.get(key) + if secret_value is None: + if required: + raise ConfigurationError(f"required secret '{secret_name}:{key}' not set") + else: + secret_value = convert_string_to_value(secret_value) + self.app_config.update_single_config_from_path_and_value(path, secret_value) diff --git a/server/common/config/server_config.py b/server/common/config/server_config.py new file mode 100644 index 00000000..14eac613 --- /dev/null +++ b/server/common/config/server_config.py @@ -0,0 +1,380 @@ +import os +import sys +import warnings +from os.path import basename +from urllib.parse import urlparse, quote_plus + +from server.auth.auth import AuthTypeFactory +from server.common.config.base_config import BaseConfig +from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD +from server.common.errors import ConfigurationError, DatasetAccessError +from server.common.data_locator import discover_s3_region_name +from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning +from server.compute import diffexp_cxg as diffexp_tiledb +from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType + + +class ServerConfig(BaseConfig): + """Manages the config attribute associated with the server.""" + + def __init__(self, app_config, default_config): + dictval_cases = [ + ("app", "csp_directives"), + ("authentication", "params_oauth", "cookie"), + ("authentication", "params_oauth", "jwt_decode_options"), + ("adaptor", "cxg_adaptor", "tiledb_ctx"), + ("multi_dataset", "dataroot"), + ] + super().__init__(app_config, default_config, dictval_cases) + + try: + self.app__verbose = default_config["app"]["verbose"] + self.app__debug = default_config["app"]["debug"] + self.app__host = default_config["app"]["host"] + self.app__port = default_config["app"]["port"] + self.app__open_browser = default_config["app"]["open_browser"] + self.app__force_https = default_config["app"]["force_https"] + self.app__flask_secret_key = default_config["app"]["flask_secret_key"] + self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"] + self.app__server_timing_headers = default_config["app"]["server_timing_headers"] + self.app__csp_directives = default_config["app"]["csp_directives"] + self.app__api_base_url = default_config["app"]["api_base_url"] + self.app__web_base_url = default_config["app"]["web_base_url"] + + self.authentication__type = default_config["authentication"]["type"] + self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][ + "oauth_api_base_url" + ] + self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"] + self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][ + "client_secret" + ] + self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][ + "jwt_decode_options" + ] + self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][ + "session_cookie" + ] + self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"] + + self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"] + self.multi_dataset__index = default_config["multi_dataset"]["index"] + self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"] + self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][ + "max_datasets" + ] + self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][ + "timelimit_s" + ] + + self.single_dataset__datapath = default_config["single_dataset"]["datapath"] + self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"] + self.single_dataset__var_names = default_config["single_dataset"]["var_names"] + self.single_dataset__about = default_config["single_dataset"]["about"] + self.single_dataset__title = default_config["single_dataset"]["title"] + + self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"] + self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"] + self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"] + + self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"] + + self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"] + self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"] + + self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"] + self.limits__column_request_max = default_config["limits"]["column_request_max"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + # The matrix data cache manager is created during the complete_config and stored here. + self.matrix_data_cache_manager = None + + # The authentication object + self.auth = None + + def complete_config(self, context): + self.handle_app(context) + self.handle_data_source() + self.handle_authentication() + self.handle_data_locator() + self.handle_adaptor() # may depend on data_locator + self.handle_single_dataset(context) # may depend on adaptor + self.handle_multi_dataset() # may depend on adaptor + self.handle_diffexp() + self.handle_limits() + + self.check_config() + + def handle_app(self, context): + self.validate_correct_type_of_configuration_attribute("app__verbose", bool) + self.validate_correct_type_of_configuration_attribute("app__debug", bool) + self.validate_correct_type_of_configuration_attribute("app__host", str) + self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("app__open_browser", bool) + self.validate_correct_type_of_configuration_attribute("app__force_https", bool) + self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str) + self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool) + self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool) + self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict)) + self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str)) + + if self.app__port: + try: + if not is_port_available(self.app__host, self.app__port): + raise ConfigurationError( + f"The port selected {self.app__port} is in use, please configure an open port." + ) + except OverflowError: + raise ConfigurationError(f"Invalid port: {self.app__port}") + else: + try: + default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT)) + except ValueError: + raise ConfigurationError( + "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT") + ) + try: + self.app__port = find_available_port(self.app__host, default_server_port) + except OverflowError: + raise ConfigurationError(f"Invalid port: {default_server_port}") + + if self.app__debug: + context["messagefn"]("in debug mode, setting verbose=True and open_browser=False") + self.app__verbose = True + self.app__open_browser = False + else: + warnings.formatwarning = custom_format_warning + + if not self.app__verbose: + sys.tracebacklimit = 0 + + # CSP Directives are a dict of string: list(string) or string: string + if self.app__csp_directives is not None: + for k, v in self.app__csp_directives.items(): + if not isinstance(k, str): + raise ConfigurationError("CSP directive names must be a string.") + if isinstance(v, list): + for policy in v: + if not isinstance(policy, str): + raise ConfigurationError("CSP directive value must be a string or list of strings.") + elif not isinstance(v, str): + raise ConfigurationError("CSP directive value must be a string or list of strings.") + + if self.app__web_base_url is None: + self.app__web_base_url = self.app__api_base_url + + def handle_authentication(self): + self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str)) + + # oauth + ptypes = str if self.authentication__type == "oauth" else (type(None), str) + self.validate_correct_type_of_configuration_attribute( + "authentication__params_oauth__oauth_api_base_url", ptypes + ) + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes) + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes) + self.validate_correct_type_of_configuration_attribute( + "authentication__params_oauth__jwt_decode_options", (type(None), dict) + ) + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool) + + if self.authentication__params_oauth__session_cookie: + self.validate_correct_type_of_configuration_attribute( + "authentication__params_oauth__cookie", (type(None), dict) + ) + else: + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict) + + self.auth = AuthTypeFactory.create(self.authentication__type, self) + if self.auth is None: + raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") + + def handle_data_locator(self): + self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str)) + if self.data_locator__s3__region_name is True: + path = self.single_dataset__datapath or self.multi_dataset__dataroot + + if type(path) == dict: + # if multi_dataset__dataroot is a dict, then use the first key + # that is in s3. NOTE: it is not supported to have dataroots + # in different regions. + paths = [val.get("dataroot") for val in path.values()] + for path in paths: + if path.startswith("s3://"): + break + if path.startswith("s3://"): + region_name = discover_s3_region_name(path) + if region_name is None: + raise ConfigurationError(f"Unable to discover s3 region name from {path}") + else: + region_name = None + self.data_locator__s3__region_name = region_name + + def handle_data_source(self): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str)) + + if self.single_dataset__datapath and self.multi_dataset__dataroot: + raise ConfigurationError( + "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both" + ) + if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None: + raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets") + + def handle_single_dataset(self, context): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None))) + + if self.single_dataset__datapath is None: + return + + # create the matrix data cache manager: + if self.matrix_data_cache_manager is None: + self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None) + + # preload this data set + matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config) + try: + matrix_data_loader.pre_load_validation() + except DatasetAccessError as e: + raise ConfigurationError(str(e)) + + file_size = matrix_data_loader.file_size() + file_basename = basename(self.single_dataset__datapath) + if file_size > BIG_FILE_SIZE_THRESHOLD: + context["messagefn"](f"Loading data from {file_basename}, this may take a while...") + else: + context["messagefn"](f"Loading data from {file_basename}.") + + if self.single_dataset__about: + + def url_check(url): + try: + result = urlparse(url) + if all([result.scheme, result.netloc]): + return True + else: + return False + except ValueError: + return False + + if not url_check(self.single_dataset__about): + raise ConfigurationError( + "Must provide an absolute URL for --about. (Example format: http://example.com)" + ) + + def handle_multi_dataset(self): + self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str)) + self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str)) + self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list) + self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int) + self.validate_correct_type_of_configuration_attribute( + "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float) + ) + + if self.multi_dataset__dataroot is None: + return + + if type(self.multi_dataset__dataroot) == str: + default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot) + self.multi_dataset__dataroot = dict(d=default_dict) + + for tag, dataroot_dict in self.multi_dataset__dataroot.items(): + if "base_url" not in dataroot_dict: + raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}") + if "dataroot" not in dataroot_dict: + raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}") + + base_url = dataroot_dict["base_url"] + + # sanity check for well formed base urls + bad = False + if type(base_url) != str: + bad = True + elif os.path.normpath(base_url) != base_url: + bad = True + else: + base_url_parts = base_url.split("/") + if [quote_plus(part) for part in base_url_parts] != base_url_parts: + bad = True + if ".." in base_url_parts: + bad = True + if bad: + raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}") + + # verify all the base_urls are unique + base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()] + if len(base_urls) > len(set(base_urls)): + raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique") + + # error checking + for mtype in self.multi_dataset__allowed_matrix_types: + try: + MatrixDataType(mtype) + except ValueError: + raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}') + + # create the matrix data cache manager: + if self.matrix_data_cache_manager is None: + self.matrix_data_cache_manager = MatrixDataCacheManager( + max_cached=self.multi_dataset__matrix_cache__max_datasets, + timelimit_s=self.multi_dataset__matrix_cache__timelimit_s, + ) + + def handle_diffexp(self): + self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int)) + self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int) + self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int) + + max_workers = self.diffexp__alg_cxg__max_workers + cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier + cpu_count = os.cpu_count() + max_workers = min(max_workers, cpu_multiplier * cpu_count) + diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit) + + def handle_adaptor(self): + # cxg + self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict) + regionkey = "vfs.s3.region" + if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx: + if type(self.data_locator__s3__region_name) == str: + self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name + + from server.data_cxg.cxg_adaptor import CxgAdaptor + + CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx) + + # anndata + self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool) + + def handle_limits(self): + self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int)) + + def exceeds_limit(self, limit_name, value): + limit_value = getattr(self, "limits__" + limit_name, None) + if limit_value is None: # disabled + return False + return value > limit_value + + def get_api_base_url(self): + if self.app__api_base_url == "local": + return f"http://{self.app__host}:{self.app__port}" + if self.app__api_base_url and self.app__api_base_url.endswith("/"): + return self.app__api_base_url[:-1] + return self.app__api_base_url + + def get_web_base_url(self): + if self.app__web_base_url == "local": + return f"http://{self.app__host}:{self.app__port}" + if self.app__web_base_url is None: + return self.get_api_base_url() + if self.app__web_base_url.endswith("/"): + return self.app__web_base_url[:-1] + return self.app__web_base_url diff --git a/server/common/errors.py b/server/common/errors.py index 5e8281e6..ca339bee 100644 --- a/server/common/errors.py +++ b/server/common/errors.py @@ -42,14 +42,14 @@ define_request_exception( define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota") define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format") define_request_exception( - "AuthenticationError", - "Raised when there is an authentication error", - default_status_code=HTTPStatus.UNAUTHORIZED) + "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED +) define_request_exception( "AnnotationCategoryNameError", "Raised when an annotation category name cant be saved", - default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY) + default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY, +) define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails") define_exception("ConfigurationError", "Raised when checking configuration errors") diff --git a/server/common/rest.py b/server/common/rest.py index be099f94..ca140709 100644 --- a/server/common/rest.py +++ b/server/common/rest.py @@ -2,10 +2,12 @@ import copy import logging import sys from http import HTTPStatus +import zlib from flask import make_response, jsonify, current_app, abort from werkzeug.urls import url_unquote +from server.common.config.client_config import get_client_config, get_client_userinfo from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg from server.common.errors import ( FilterError, @@ -117,12 +119,12 @@ def schema_get(data_adaptor): def config_get(app_config, data_adaptor): - config = app_config.get_client_config(data_adaptor) + config = get_client_config(app_config, data_adaptor) return make_response(jsonify(config), HTTPStatus.OK) def userinfo_get(app_config, data_adaptor): - config = app_config.get_client_userinfo(data_adaptor) + config = get_client_userinfo(app_config, data_adaptor) return make_response(jsonify(config), HTTPStatus.OK) @@ -154,17 +156,21 @@ def annotations_put_fbs_helper(data_adaptor, fbs): new_label_df = decode_matrix_fbs(fbs) if not new_label_df.empty: - data_adaptor.check_new_labels(new_label_df) + new_label_df = data_adaptor.check_new_labels(new_label_df) annotations.write_labels(new_label_df, data_adaptor) +def inflate(data): + return zlib.decompress(data) + + def annotations_obs_put(request, data_adaptor): annotations = data_adaptor.dataset_config.user_annotations if annotations is None: return abort(HTTPStatus.NOT_IMPLEMENTED) anno_collection = request.args.get("annotation-collection-name", default=None) - fbs = request.get_data() + fbs = inflate(request.get_data()) if anno_collection is not None: if not annotations.is_safe_collection_name(anno_collection): diff --git a/server/common/utils/cxg_generation_utils.py b/server/common/utils/cxg_generation_utils.py index f29f4bd0..d3ec2b40 100644 --- a/server/common/utils/cxg_generation_utils.py +++ b/server/common/utils/cxg_generation_utils.py @@ -111,7 +111,7 @@ def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx): def convert_matrix_to_cxg_array( - matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None + matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None ): """ Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array` diff --git a/server/common/utils/matrix_utils.py b/server/common/utils/matrix_utils.py index 3eeddc10..60dfb19b 100644 --- a/server/common/utils/matrix_utils.py +++ b/server/common/utils/matrix_utils.py @@ -41,16 +41,19 @@ def is_matrix_sparse(matrix: np.ndarray, sparse_threshold): number_of_non_zero_elements += np.count_nonzero(matrix_subset) if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix: if end_row_index != total_number_of_rows: - percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / ( - end_row_index * total_number_of_columns) + percentage_of_non_zero_elements = ( + 100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns) + ) logging.info( f"Matrix is not sparse. Percentage of non-zero elements (estimate): " - f"{percentage_of_non_zero_elements:6.2f}") + f"{percentage_of_non_zero_elements:6.2f}" + ) else: percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements logging.info( f"Matrix is not sparse. Percentage of non-zero elements (exact): " - f"{percentage_of_non_zero_elements:6.2f}") + f"{percentage_of_non_zero_elements:6.2f}" + ) return False is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold diff --git a/server/common/utils/type_conversion_utils.py b/server/common/utils/type_conversion_utils.py index ac6b3fe4..90ceabbb 100644 --- a/server/common/utils/type_conversion_utils.py +++ b/server/common/utils/type_conversion_utils.py @@ -9,8 +9,10 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame): schema_type_hints_by_column_name = {} for column_name, column_values in dataframe.items(): - dtypes_by_column_name[column_name], schema_type_hints_by_column_name[column_name] = \ - get_dtype_and_schema_of_array(column_values) + ( + dtypes_by_column_name[column_name], + schema_type_hints_by_column_name[column_name], + ) = get_dtype_and_schema_of_array(column_values) return dtypes_by_column_name, schema_type_hints_by_column_name @@ -24,8 +26,10 @@ def get_schema_type_hint_of_array(array: pd.Series): def get_dtype_and_schema_of_array(array: pd.Series): - return (get_dtype_from_dtype(array.dtype, array_values=array), - get_schema_type_hint_from_dtype(array.dtype, array_values=array)) + return ( + get_dtype_from_dtype(array.dtype, array_values=array), + get_schema_type_hint_from_dtype(array.dtype, array_values=array), + ) def get_dtype_from_dtype(dtype, array_values=None): @@ -84,24 +88,15 @@ def get_schema_type_hint_from_dtype(dtype, array_values=None): def can_cast_to_float32(dtype, array_values): """ - A dtype can be cast to float32 if it is a float type and converting it to float32 presents the same output as the - original values. Note that NaNs fail equality (i.e. np.NaN != np.NaN) so we use np.testing.assert_equal to ensure - that the arrays are equal minus NaNs. + Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is + a float. We also handle a special case here where the array is a Series object with integer categorical values AND NaNs. - Since NaNs are floating points in numpy, we upcast the integer array to float32. + Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True. """ if dtype.kind == "f": - # Try to convert the array to float32 - converted_float32_values = array_values.to_numpy(np.float32) - original_values = array_values.to_numpy() - - # Verify that the two arrays are equal except for NaNs (which will equate to be unequal). - if not ((converted_float32_values != original_values) == np.isnan(original_values)).all(): - return False - - if dtype != np.float32: + if not np.can_cast(dtype, np.float32): logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.") return True @@ -133,9 +128,11 @@ def can_cast_to_int32(dtype, array_values=None): if np.can_cast(dtype, np.int32): return True ii32 = np.iinfo(np.int32) - if not ordered_array_values.empty and ( - ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \ - ordered_array_values.empty: + if ( + not ordered_array_values.empty + and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) + or ordered_array_values.empty + ): return True return False @@ -145,3 +142,17 @@ def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype): logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.") return series_to_convert.to_numpy(dtype) + + +def convert_string_to_value(value: str): + """convert a string to value with the most appropriate type""" + if value.lower() == "true": + return True + if value.lower() == "false": + return False + if value == "null": + return None + try: + return eval(value) + except: # noqa E722 + return value diff --git a/server/converters/h5ad_data_file.py b/server/converters/h5ad_data_file.py index c8223822..c79dd785 100644 --- a/server/converters/h5ad_data_file.py +++ b/server/converters/h5ad_data_file.py @@ -24,14 +24,14 @@ class H5ADDataFile: another format (currently just CXG is supported). """ def __init__( - self, - input_filename, - backed=False, - dataset_title=None, - dataset_about=None, - obs_index_column_name=None, - vars_index_column_name=None, - use_corpora_schema=True, + self, + input_filename, + backed=False, + dataset_title=None, + dataset_about=None, + obs_index_column_name=None, + vars_index_column_name=None, + use_corpora_schema=True, ): self.input_filename = input_filename self.backed = backed diff --git a/server/converters/schema/__init__.py b/server/converters/schema/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/converters/schema/gene_symbol.py b/server/converters/schema/gene_symbol.py new file mode 100644 index 00000000..2d0de5a7 --- /dev/null +++ b/server/converters/schema/gene_symbol.py @@ -0,0 +1,211 @@ +"""Helpers for converting and checking HGNC gene symbols.""" + +import argparse +import enum +import logging +import os +import re +import numpy as np +import pandas as pd + + +def get_upgraded_var_index(var, hgnc_path=None): + """Given an anndata var dataframe, return a new index for the dataframe + where human gene symbols have been upgraded to the current HGNC set. + """ + + if not hgnc_path: + hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz") + + hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path) + + return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index]) + + +class SymbolStatus(enum.Enum): + """The status of a symbol in the HGNC database. + + APPROVED: Currently a valid symbol + WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown + not to exist _unless_ that symbol is also approved + AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for + multiple approved symbols + UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved + symbol + UNKNOWN: A symbol that does not appear in HGNC + """ + + APPROVED = 1 + WITHDRAWN = 2 + AMBIGUOUS = 3 + UPGRADABLE = 4 + UNKNOWN = 5 + + +class HGNCSymbolChecker: + """Handle checking and correcting HGNC symbols.""" + + def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map): + self.approved_symbols = approved_symbols + self.withdrawn_symbols = withdrawn_symbols + self.ambiguous_symbols = ambiguous_symbols + self.symbol_map = symbol_map + + def print_symbol_map(self): + """Print out a map from old symbol to new symbol.""" + + for symbol_pair in self.symbol_map.items(): + print("\t".join(symbol_pair)) + + def check_symbol(self, symbol): + """See if a symbol if approved or something else.""" + if symbol in self.approved_symbols: + return SymbolStatus.APPROVED + + if symbol in self.withdrawn_symbols: + return SymbolStatus.WITHDRAWN + + if symbol in self.ambiguous_symbols: + return SymbolStatus.AMBIGUOUS + + if symbol in self.symbol_map: + return SymbolStatus.UPGRADABLE + + return SymbolStatus.UNKNOWN + + def upgrade_symbol(self, symbol): + """Return the approved symbol for the given symbol. + + If the symbol cannot be upgraded, just return the original symbol. + """ + + fixed_symbol, stripped_symbol = format_symbol(symbol) + + if fixed_symbol in self.approved_symbols: + return fixed_symbol + elif fixed_symbol in self.symbol_map: + return self.symbol_map[fixed_symbol] + elif stripped_symbol in self.approved_symbols: + return stripped_symbol + elif stripped_symbol in self.symbol_map: + return self.symbol_map[stripped_symbol] + + return symbol + + @classmethod + def from_hgnc_records(cls, hgnc_dataset_path): + """Parse a hgnc database download into a HGNCSymbolChecker object.""" + + def all_symbols(record): + """Get all the symbols associated with an HGNC record including previous, alias, + and approved.""" + yield format_symbol(record["symbol"])[0] + for symbol in alias_and_previous_symbols(record): + yield symbol + + def alias_and_previous_symbols(record): + """Get alias and previous symbols from an HGNC record.""" + for field in ("alias_symbol", "prev_symbol"): + if record[field] is not np.nan: + for symbol in record[field].split("|"): + yield format_symbol(symbol)[0] + # Sometimes something like HGNC:1234 appears in datasets, which we + # want to fix as well. + yield record["hgnc_id"] + + hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records") + + # Get all symbols that are currently approved. + approved_symbols = set() + for record in hgnc_records: + if record["status"] == "Approved": + approved_symbols.add(format_symbol(record["symbol"])[0]) + + # Get all symbols that have been withdrawn + withdrawn_symbols = set() + for record in hgnc_records: + if record["status"] == "Entry Withdrawn": + for symbol in all_symbols(record): + withdrawn_symbols.add(symbol) + + # If a symbol is both approved and withdrawn, be optimistic and call it approved + logging.warning( + f"Some symbols are simulaneously withdrawn and approved\n" + f"We will treat them at approved:\n" + f"{withdrawn_symbols.intersection(approved_symbols)}" + ) + withdrawn_symbols = withdrawn_symbols.difference(approved_symbols) + + # Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol + alias_previous_to_approved = {} + ambiguous_symbols = set() + + for record in hgnc_records: + if record["status"] == "Approved": + + # The approved symbol is what we'll map to + approved_symbol = format_symbol(record["symbol"])[0] + + for symbol in alias_and_previous_symbols(record): + + # If the alias or previous symbol is also an approved symbol, + # we'll just leave it alone + if symbol in approved_symbols: + continue + + # If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous + if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol: + ambiguous_symbols.add(symbol) + else: + alias_previous_to_approved[symbol] = approved_symbol + + # Remove all the ambiguous symbols from the map + for ambiguous_symbol in ambiguous_symbols: + alias_previous_to_approved.pop(ambiguous_symbol) + + return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved) + + +def format_symbol(symbol): + """HGNC rules say symbols should all be upper case except for C#orf#. However, case is + variable in both alias and previous symbols as well as in the symbols we get in + submissions. So, upper case everything except for the one situation where mixed-case + is allowed, which are the genes like C2orf157. + + Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered + names persist throughout the life of the object. They won't match against the HGNC database + and we want to merge them, so we need to strip off the suffix and try matching again. + + This function takes a symbol and returns the symbol with the fixed case and also with the + seurat/scanpy suffix stripped off. + """ + + match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE) + + if match: + fixed_case = f"C{match.group(2)}orf{match.group(4)}" + else: + fixed_case = symbol.upper() + + suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case) + + return fixed_case, suffix_stripped + + +def main(): + """When called as main, parse a given hgnc download and print out a map from old to new + symbol. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/" + ) + args = parser.parse_args() + + hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset) + + hgnc_symbol_checker.print_symbol_map() + + +if __name__ == "__main__": + main() diff --git a/server/converters/schema/hgnc_complete_set.txt.gz b/server/converters/schema/hgnc_complete_set.txt.gz new file mode 100644 index 00000000..29c3c7a9 Binary files /dev/null and b/server/converters/schema/hgnc_complete_set.txt.gz differ diff --git a/server/converters/schema/ontology.py b/server/converters/schema/ontology.py new file mode 100644 index 00000000..8a524402 --- /dev/null +++ b/server/converters/schema/ontology.py @@ -0,0 +1,86 @@ +"""Methods for working with ontologies and the OLS.""" +from urllib.parse import quote_plus + +import requests + +OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api" + +# Curie means something like CL:0000001 + + +def _ontology_name(curie): + """Get the name of the ontology from the curie, CL or UBERON for example.""" + return curie.split(":")[0] + + +def _ontology_value(curie): + """Get the id component of the curie, 0000001 from CL:0000001 for example.""" + return curie.split(":")[1] + + +def _double_encode(url): + """Double url encode a url. This is required by the OLS API.""" + return quote_plus(quote_plus(url)) + + +def _iri(curie): + """Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org""" + if _ontology_name(curie) == "EFO": + return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}" + return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}" + + +class OntologyLookupError(Exception): + """Exception for some problem with looking up ontology information.""" + + +def _ontology_info_url(curie): + """Get the to make a GET to to get information about an ontology term.""" + + # If the curie is empty, just return an empty string. This happens when there is no + # valid ontology value. + if not curie: + return "" + else: + return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}" + + +def get_ontology_label(curie): + """For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'""" + + url = _ontology_info_url(curie) + + if not url: + return "" + + response = requests.get(url) + + if not response.ok: + raise OntologyLookupError( + f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}" + ) + return response.json()["label"] + + +def lookup_candidate_term(label, ontology="cl", method="select"): + """Lookup candidate terms for a label. This is useful when there is an existing label in a + submitted dataset, and you want to find an appropriate ontology term. + + Args: + label: the label to find ontology terms for + ontology: the ontology to search in, cl or uberon or efo for example + method: select or search. search provides much broader results + + Returns: + list of (curie, label) tuples returned by OLS + """ + # using OLS REST API [https://www.ebi.ac.uk/ols/docs/api] + url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}" + response = requests.get(url) + + if not response.ok: + raise OntologyLookupError( + f"Label {label} lookup failed, got status code {response.status_code}: {response.text}" + ) + + return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]] diff --git a/server/converters/schema/remix.py b/server/converters/schema/remix.py new file mode 100644 index 00000000..5cc60746 --- /dev/null +++ b/server/converters/schema/remix.py @@ -0,0 +1,264 @@ +import argparse +import collections +import json +import logging +import math +import string + +import anndata +import numpy as np +import pandas as pd +import yaml + +from . import gene_symbol +from . import ontology +from . import validate + +REPLACE_SUFFIX = "_original" +ONTOLOGY_SUFFIX = "_ontology_term_id" + + +def is_curie(value): + """Return True iff the value is an OBO-id CURIE like EFO:000001""" + return (value.count(":") + and all(len(part) > 0 for part in value.split(":")) + and all(c in string.digits for c in value.split(":")[1])) + + +def is_ontology_field(field_name): + """Return True iff the field_name is an ontology field like tissue_ontology_term_id""" + return field_name.endswith(ONTOLOGY_SUFFIX) + + +def get_label_field_name(field_name): + """Get the associated label field from an ontology field, assay_ontology_term_id --> assay""" + return field_name[: -len(ONTOLOGY_SUFFIX)] + + +def split_suffix(maybe_curie): + """Split off the (cell culture) or (organoid) suffix.""" + + suffixes = [" (cell culture)", " (organoid)"] + for suffix in suffixes: + if maybe_curie.endswith(suffix): + return maybe_curie[:-len(suffix)], suffix + return maybe_curie, "" + + +def get_curie_and_label(maybe_curie): + """Given a string that might be a curie, return a (curie, label) pair""" + + maybe_curie, suffix = split_suffix(maybe_curie) + if not is_curie(maybe_curie): + return ("", maybe_curie + suffix) + return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix) + + +def safe_add_field(adata_attr, field_name, field_value): + """Add a field and value to an AnnData, but don't clobber an exising value.""" + + if ( + isinstance(field_value, list) + and field_value + and isinstance(field_value[0], dict) + ): + field_value = json.dumps(field_value) + if field_name in adata_attr: + adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name] + adata_attr[field_name] = field_value + + +def remix_uns(adata, uns_config): + """Add fields from the config to adata.uns""" + for field_name, field_value in uns_config.items(): + + if is_ontology_field(field_name): + # If it's an ontology field, look it up + label_field_name = get_label_field_name(field_name) + ontology_term, ontology_label = get_curie_and_label(field_value) + safe_add_field(adata.uns, field_name, ontology_term) + safe_add_field(adata.uns, label_field_name, ontology_label) + else: + safe_add_field(adata.uns, field_name, field_value) + + +def remix_obs(adata, obs_config): + """Add fields from the config to adata.obs""" + + for field_name, field_value in obs_config.items(): + + if isinstance(field_value, dict): + # If the value is a dict, that means we are supposed to map from an + # existing column to the new one + source_column, column_map = next(iter(field_value.items())) + nan_value = None + for key in column_map: + if isinstance(key, float) and math.isnan(key): + nan_value = column_map[key] + if nan_value is not None: + column_map["nan"] = nan_value + + for key in column_map: + if key not in adata.obs[source_column].unique(): + logging.warning(f'Key {key} not in adata.obs["{source_column}"]') + + for value in adata.obs[source_column].unique(): + if value not in column_map: + logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict') + + if is_ontology_field(field_name): + ontology_term_map, ontology_label_map = {}, {} + logging.info(f"Looking up labels for {field_name}") + for original_value, maybe_curie in column_map.items(): + curie, label = get_curie_and_label(maybe_curie) + ontology_term_map[original_value] = curie + ontology_label_map[original_value] = label + logging.info(f"Mapping {original_value} -> {curie} -> {label}") + + ontology_column = adata.obs[source_column].replace( + ontology_term_map, inplace=False + ) + label_column = adata.obs[source_column].replace( + ontology_label_map, inplace=False + ) + + safe_add_field(adata.obs, field_name, ontology_column) + safe_add_field( + adata.obs, get_label_field_name(field_name), label_column + ) + else: + label_column = adata.obs[source_column].replace( + column_map, inplace=False + ) + safe_add_field(adata.obs, field_name, label_column) + + else: + if is_ontology_field(field_name): + # If it's an ontology field, look it up + label_field_name = get_label_field_name(field_name) + ontology_term, ontology_label = get_curie_and_label(field_value) + safe_add_field(adata.obs, field_name, ontology_term) + safe_add_field(adata.obs, label_field_name, ontology_label) + else: + safe_add_field(adata.obs, field_name, field_value) + + +def merge_df(df, domain, index, columns): + """ + Given a dataframe with duplicate column labels, merge and return a dataframe where + the duplicates have been merged together, resulting in a dataframe with unique column + labels. + + "merge" depends on the value of domain. If the domain is "raw", then duplicate columns + can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then + summed, and then logged or sqrt'd again. + """ + + if not isinstance(df, np.ndarray): + to_merge = df.toarray() + else: + to_merge = df + if domain == "raw": + merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum( + axis=1, level=0, skipna=False + ) + elif domain == "log1p": + merged_df = ( + pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns) + .sum(axis=1, level=0, skipna=False) + ) + merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns) + elif domain == "sqrt": + merged_df = ( + pd.DataFrame(np.square(to_merge), index=index, columns=columns) + .sum(axis=1, level=0, skipna=False) + ) + merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns) + + return merged_df + + +def fixup_gene_symbols(adata, fixup_config): + """Update the var index to hold a consistent set of HGNC gene symbols.""" + + upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var) + + merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index) + fixup_adata = anndata.AnnData( + X=merged_X, + obs=adata.obs, + var=merged_X.columns.to_frame(name="hgnc_gene_symbol"), + uns=adata.uns, + obsm=adata.obsm, + ) + + for layer, domain in fixup_config.items(): + if layer == "X": + continue + if layer == "raw.X": + df = adata.raw.X + else: + df = adata.layers[layer] + + merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index) + assert merged_df.index.equals(merged_X.index) + assert merged_df.columns.equals(merged_X.columns) + + if domain == "raw": + fixup_raw = anndata.AnnData( + X=merged_df, + obs=adata.obs, + var=merged_X.columns.to_frame(name="hgnc_gene_symbol"), + ) + fixup_adata.raw = fixup_raw + else: + fixup_adata.layers[layer] = merged_df + + return fixup_adata + +def _strip_version(adata): + """Remove version information from the AnnData object.""" + + if "version" in adata.uns_keys(): + del adata.uns["version"] + +def apply_schema(source_h5ad, remix_config, output_filename): + + try: + import scanpy + except ImportError: + raise ImportError("scanpy must be installed for cellxgene schema") + adata = scanpy.read_h5ad(source_h5ad) + config = yaml.load(open(remix_config), Loader=yaml.FullLoader) + remix_uns(adata, config["uns"]) + remix_obs(adata, config["obs"]) + + if config.get("fixup_gene_symbols"): + adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"]) + + if ("version" in adata.uns_keys() + and isinstance(adata.uns["version"], collections.Mapping) + and "corpora_schema_version" in adata.uns["version"]): + schema_version = adata.uns["version"]["corpora_schema_version"] + try: + validate.get_schema_definition(schema_version) + except ValueError: + logging.warning(f"Stripping version information out of AnnData because schema " + f"version {schema_version} is unknown.") + _strip_version(adata) + + if not validate.validate_adata(adata, shallow=False): + logging.warning(f"Stripping version information out of AnnData because it does not " + f"follow schema version {schema_version} .") + _strip_version(adata) + + adata.write_h5ad(output_filename, compression="gzip") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--source-h5ad", required=True) + parser.add_argument("--remix-config", required=True) + parser.add_argument("--output-filename", required=True) + args = parser.parse_args() + apply_schema(args.source_h5ad, args.remix_config, args.output_filename) diff --git a/server/converters/schema/schema_definitions/1_0_0.yaml b/server/converters/schema/schema_definitions/1_0_0.yaml new file mode 100644 index 00000000..acff2238 --- /dev/null +++ b/server/converters/schema/schema_definitions/1_0_0.yaml @@ -0,0 +1,95 @@ +title: Corpora schema version 1.0.0 +type: anndata +components: + uns: + type: dict + keys: + version: + type: dict + keys: + corpora_schema_version: null + corpora_encoding_version: null + title: + type: string + contributors: + type: stringified list of dicts + layer_descriptions: + type: dict + keys: + X: null + organism: + type: string + nullable: false + organism_ontology_term_id: + type: curie + prefixes: + - NCBITaxon + var: + type: dataframe + index: + type: human-readable string + unique: true + obs: + type: dataframe + index: + unique: true + columns: + tissue: + type: human-readable string + nullable: false + tissue_ontology_term_id: + type: suffixed curie + nullable: true + prefixes: + - UBERON + assay: + type: human-readable string + nullable: false + assay_ontology_term_id: + type: curie + nullable: true + prefixes: + - EFO + disease: + type: human-readable string + nullable: false + disease_ontology_term_id: + type: curie + nullable: true + prefixes: + - MONDO + - PATO + cell_type: + type: human-readable string + nullable: false + cell_type_ontology_term_id: + type: curie + nullable: true + prefixes: + - CL + - UBERON + sex: + type: string + enum: + - male + - female + - mixed + - unknown + - other + ethnicity: + type: human-readable string + nullable: false + ethnicity_ontology_term_id: + type: curie + nullable: true + prefixes: + - HANCESTRO + development_stage: + type: human-readable string + nullable: false + development_stage_ontology_term_id: + type: curie + nullable: true + prefixes: + - HsapDv + - EFO diff --git a/server/converters/schema/validate.py b/server/converters/schema/validate.py new file mode 100644 index 00000000..ec463dd5 --- /dev/null +++ b/server/converters/schema/validate.py @@ -0,0 +1,236 @@ +import json +import re +import os +import sys + +import pandas as pd +import yaml + + +def _is_null(v): + """Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad.""" + return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0) + + +def _validate_stringified_list_of_dicts(s): + """Verify that a string can be parsed into a list. + + We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to + json.dumps them. This verifies that we can load them back. + """ + + try: + list_ = json.loads(s) + if not isinstance(list_, list): + return False + for el in list_: + if not isinstance(el, dict): + return False + return True + except (json.JSONDecodeError, TypeError): + pass + return False + + +def _validate_human_readable_string(s): + """Verify that a string is human-readable. + + There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind + of vague and subjective. I feel like I can read many strings. So here we just check for the main ways + that fails: someone puts in an ontology term id or and ensembl gene/transcript id. + + Returns False if s is not a string or is one of those bad string types. + """ + + return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s)) + + +def _validate_curie(c, prefixes): + """Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the + prefix of the curies is in prefixes. + """ + + if not c: + return True + + match = re.match(r"([A-Z]\w+):\d+$", c) + + if prefixes: + return match and match.group(1) in prefixes + else: + return match + + +def _validate_suffixed_curie(c, prefixes): + """Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'""" + + # Pull off the suffix + suffix = re.findall(r"\ \(.*\)$", c) + if suffix: + c = c[: -len(suffix[0])] + return _validate_curie(c, prefixes) + + +def _validate_column(column, column_name, df_name, schema_def): + """Given a schema definition and the column of a dataframe, verify that the column satifies + the schema. + """ + + errors = [] + + if schema_def.get("unique"): + if column.nunique() != len(column): + errors.append(f"Column {column_name} in dataframe {df_name} is not unique.") + + if "nullable" in schema_def and not schema_def["nullable"]: + if any(_is_null(v) for v in column): + errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.") + + if schema_def.get("type") == "human-readable string": + non_readables = [v for v in column if not _validate_human_readable_string(v)] + if non_readables: + errors.append( + f"Column {column_name} in dataframe {df_name} contains non-human-readable " + f"values like {non_readables[0]}" + ) + + if schema_def.get("type") in ("curie", "suffixed curie"): + validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie + non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))] + if non_valid_curies: + errors.append( + f"Column {column_name} in dataframe {df_name} contains invalid ontology values like " + f"{non_valid_curies[0]}." + ) + if "prefixes" in schema_def: + errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}." + + if "enum" in schema_def: + bad_enums = [v for v in column if v not in schema_def["enum"]] + if bad_enums: + errors.append( + f"Column {column_name} in dataframe {df_name} contains unpermitted values like " + f"{bad_enums[0]}. Values must be one of {schema_def['enum']}." + ) + + return errors + + +def _validate_dict(dict_, dict_name, schema_def): + """Given a schema definition and dict, verify that the dict satifies the schema.""" + + errors = [] + + for key in schema_def.get("keys", []): + if key not in dict_: + errors.append(f"{dict_name} is missing key {key}.") + elif schema_def["keys"][key]: + if schema_def["keys"][key]["type"] == "stringified list of dicts": + if not _validate_stringified_list_of_dicts(dict_[key]): + errors.append( + f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}" + ) + elif schema_def["keys"][key]["type"] == "dict": + errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key])) + elif schema_def["keys"][key]["type"] == "curie": + if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]): + errors.append(f"Key {key} in {dict_name} contains invalid ontology value.") + if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]: + if _is_null(dict_[key]): + errors.append(f"Key {key} in dict {dict_name} is an empty value.") + + return errors + + +def _validate_dataframe(df, df_name, schema_def): + """Given a dataframe and schema definition, verify that the dataframe follows the schema.""" + + errors = [] + + if "index" in schema_def: + errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"])) + + for column in schema_def.get("columns", []): + if column not in df.columns: + errors.append(f"Dataframe {df_name} is missing column {column}.") + else: + errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column])) + + return errors + + +def get_schema_definition(version): + """Look up and read a schema definition based on a version number like "1.0.0".""" + + path = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml" + ) + + if not os.path.isfile(path): + raise ValueError(f"No definition for version {version} found.") + + return yaml.load(open(path), Loader=yaml.FullLoader) + + +def deep_check(adata, schema_def): + """Perform a "deep" check of the AnnData object using the schema definition. + + This checks all the columns and unstructured metadata rather than just the version. + + Returns a list of error messages. If that list is empty, the object passed validation. + """ + + errors = [] + + for component, component_def in schema_def["components"].items(): + if component_def["type"] == "dataframe": + errors.extend(_validate_dataframe(getattr(adata, component), component, component_def)) + elif component_def["type"] == "dict": + errors.extend(_validate_dict(getattr(adata, component), component, component_def)) + else: + raise ValueError(f"Unexpected component type {component['type']}") + + return errors + + +def validate_adata(adata, shallow): + """Validate an AnnData object. If shallow, just check that the required version information is + present. + """ + + # Does it have the version information written into uns? + if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]: + print("AnnData file is missing corpora version information") + return False + + # We can stop here if it's a "shallow" check, that is, if we're just + # checking that version is present. + if shallow: + return True + + schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"]) + + errors = deep_check(adata, schema_def) + + for error in errors: + print(error) + + return not errors + + +def validate(h5ad_path, shallow=False): + """Entry point for validation.""" + + try: + import scanpy + except ImportError: + raise ImportError("scanpy must be installed for cellxgene schema") + + try: + adata = scanpy.read_h5ad(h5ad_path, backed="r") + except (OSError, TypeError): + print(f"Unable to open {h5ad_path} with scanpy.") + sys.exit(1) + + if not validate_adata(adata, shallow): + sys.exit(1) diff --git a/server/data_anndata/anndata_adaptor.py b/server/data_anndata/anndata_adaptor.py index 3145a833..1312a8f6 100644 --- a/server/data_anndata/anndata_adaptor.py +++ b/server/data_anndata/anndata_adaptor.py @@ -177,10 +177,10 @@ class AnndataAdaptor(DataAdaptor): ) def _validate_and_initialize(self): - if anndata_version_is_pre_070() and self.server_config.adaptor__anndata_adaptor__backed: + if anndata_version_is_pre_070(): warnings.warn( - "Use of --backed mode with anndata versions older than 0.7 will have serious " - "performance issues. Please update to at least anndata 0.7 or later." + "Use of anndata versions older than 0.7 will have serious issues. Please update to at " + "least anndata 0.7 or later." ) # var and obs column names must be unique diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index 20dafdd0..cb1e3c57 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd from server_timing import Timing as ServerTiming -from server.common.app_config import AppFeature, AppConfig +from server.common.config.app_config import AppConfig from server.common.constants import Axis from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError from server.common.utils.utils import jsonify_numpy @@ -155,17 +155,6 @@ class DataAdaptor(metaclass=ABCMeta): """ pass - def get_features(self, annotations=None): - """Return list of features, to return as part of the config route""" - features = [ - AppFeature("/cluster/", method="POST", available=False), - AppFeature("/layout/obs", method="GET", available=self.get_embedding_names() is not None), - AppFeature("/layout/obs", method="PUT", available=self.dataset_config.embeddings__enable_reembedding), - AppFeature("/diffexp/", method="POST", available=self.dataset_config.diffexp__enable), - AppFeature("/annotations/obs", method="PUT", available=annotations is not None), - ] - return features - def update_parameters(self, parameters): parameters.update(self.parameters) @@ -173,7 +162,7 @@ class DataAdaptor(metaclass=ABCMeta): mask = np.zeros((count,), dtype=np.bool) for i in filter: if type(i) == list: - mask[i[0]: i[1]] = True + mask[i[0] : i[1]] = True else: mask[i] = True return mask @@ -260,6 +249,23 @@ class DataAdaptor(metaclass=ABCMeta): if labels_df.shape[0] != shape[0]: raise ValueError("Labels file must have same number of rows as data file.") + # This will convert a float column that contains integer data into an integer type. + # This case can occur when a user makes a copy of a category that originally contained integer data. + # The client always copies array data to floats, therefore the copy will contain floats instead of integers. + # float data is not allowed as a categorical type. + if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]): + labels_df = labels_df.convert_dtypes() + for col, dtype in zip(labels_df, labels_df.dtypes): + if isinstance(dtype, pd.Int32Dtype): + labels_df[col] = labels_df[col].astype("int32") + if isinstance(dtype, pd.Int64Dtype): + labels_df[col] = labels_df[col].astype("int64") + + if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]): + raise ValueError("Columns may not have floating point types") + + return labels_df + def data_frame_to_fbs_matrix(self, filter, axis): """ Retrieves data 'X' and returns in a flatbuffer Matrix. @@ -314,7 +320,7 @@ class DataAdaptor(metaclass=ABCMeta): top_n = self.dataset_config.diffexp__top_n if self.server_config.exceeds_limit( - "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) + "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) ): raise ExceedsLimitError("Diffexp request exceeds max cell count limit") diff --git a/server/db/cellxgene_orm.py b/server/db/cellxgene_orm.py index f2209b63..2860f6b1 100644 --- a/server/db/cellxgene_orm.py +++ b/server/db/cellxgene_orm.py @@ -1,11 +1,6 @@ import uuid -from sqlalchemy import ( - Column, - DateTime, - ForeignKey, - String, - func, JSON) +from sqlalchemy import Column, DateTime, ForeignKey, String, func, JSON from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship diff --git a/server/db/db_utils.py b/server/db/db_utils.py index 1664303f..b3030bf2 100644 --- a/server/db/db_utils.py +++ b/server/db/db_utils.py @@ -42,9 +42,9 @@ class DbUtils: def get_or_create_dataset(self, dataset_name): try: - dataset_id = self.query( - table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name] - )[0].id + dataset_id = self.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])[ + 0 + ].id except IndexError: dataset_id = uuid.uuid4() dataset = CellxGeneDataset(id=dataset_id, name=dataset_name) @@ -54,9 +54,7 @@ class DbUtils: def get_or_create_user(self, user_id): try: - user_id = self.query( - table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id] - )[0].id + user_id = self.query(table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id])[0].id except IndexError: user = CellxGeneUser(id=user_id) self.session.add(user) diff --git a/server/common/default_config.py b/server/default_config.py similarity index 77% rename from server/common/default_config.py rename to server/default_config.py index 16e0c27d..20922ef9 100644 --- a/server/common/default_config.py +++ b/server/default_config.py @@ -205,6 +205,60 @@ dataset: lfc_cutoff: 0.01 top_n: 10 +external: + # You can retrieve configuration parameters from this config file, the environment, + # the AWS secrets manager, or from the "cellxgene launch" command line arguments. + # They are applied in that order, meaning that if a parameter is defined in more + # than one location, the last one applied takes effect. + + # environment variables: + # This section describes how to map environment variables to configuration parameters. + # The format is a list defining an environment variable. + # Each entry in the list is a dictionary with three entries: + # name: the name of the environment variable + # path: the path within the cellxgene configuration to update. + # required: (default=False) a boolean. If true, then it is an error if the environment variable is not set. + + environment: + - name: CXG_SECRET_KEY + path: [server, app, flask_secret_key] + required: false + - name: CXG_OAUTH_CLIENT_SECRET + path: [server, authentication, params_oauth, client_secret] + required: false + + # AWS Secrets Manager + # This section describes how to map aws secrets to configuration parameters. + # The format is the region for the secrets manager, then a list of secrets. + # each secret has a name, and a list of values. + # Each entry in the list of values is a dictionary with three entries: + # key: the key of the aws secret. + # path: the path within the cellxgene configuration to update. + # required: (default=False) a boolean. If true, then it is an error if the key does not exist in the secret. + # + # example: + # aws_secrets_manager: + # region: us-west-2 + # - name: my_first_secret + # values: + # - key: flask_secret_key + # path: [server, app, flask_secret_key] + # required: true + # - key: db_uri + # path: [dataset, user_annotations, hosted_tiledb_array, db_uri] + # required: true + # - name: my_auth_secret + # values: + # - key: client_secret + # path: [server, authentication, params_oauth, client_secret] + # required: true + # - key: client_id + # path: [server, authentication, params_oauth, client_id] + # required: true + + aws_secrets_manager: + region: null + secrets: [] """ diff --git a/server/eb/Makefile b/server/eb/Makefile index 3c9c5b4a..05b24f83 100644 --- a/server/eb/Makefile +++ b/server/eb/Makefile @@ -27,6 +27,9 @@ build: clean if [ -f customize/config.yaml ] ; then \ cp customize/config.yaml artifact.dir; \ fi ; \ + if [ -f customize/Dockerfile ] ; then \ + cp customize/Dockerfile artifact.dir; \ + fi ; \ if [ -f customize/requirements.txt ] ; then \ pip install requirements-parser ; \ pip install packaging ; \ diff --git a/server/eb/README.md b/server/eb/README.md index f4209f15..306eda00 100644 --- a/server/eb/README.md +++ b/server/eb/README.md @@ -1,12 +1,12 @@ # AWS Elastic Beanstalk -This directory contains script to aid in creating and deploying cellxgene on -an AWS Elastic Beanstalk instance. +This directory contains scripts to aid in creating and deploying cellxgene on +AWS Elastic Beanstalk. This will result in a variant of cellxgene, running on AWS EC2 instances, serving data from S3. -All datasets must be in the new CXG (tiledb) format - see the converter script cxgtool.py -in server/converters - and located in a single S3 prefix, which is accessible to the instance. -In the current incarnation, no access control or authentication support is available +All datasets must be in the CXG (tiledb) format (see `cellxene convert --help`), +and located under a single S3 prefix, which is accessible to the instance. +In the current incarnation, no access control is available (outside of anything you configure yourself), so this is most appropriate for public datasets. This is early development work, and will change significantly in the near future. @@ -17,10 +17,10 @@ We would love feedback on it, but please assume it will change. 1. Some familiarity with AWS EB, S3, and IAM are needed. 2. Install the awsebcli. -Instruction are here: -https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html + Instruction are here: + https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html -3. In the top level directory, run ```make build-client``` to create the client static assets. +3. In the top level directory, run `make build-client` to create the client static assets. ## Steps @@ -31,20 +31,21 @@ There are many more options to these commands that may be important or necessary The following choices are known to work. -* S3 Bucket. -* POSIX filesystem (such as Lustre) -* Lustre filesystem backed by S3 +- S3 Bucket. +- POSIX filesystem (such as Lustre) +- Lustre filesystem backed by S3 S3 is convenient and the relatively inexpensive option. Lustre is higher performance, but more expensive, and slightly more complex to setup and manage. -AWS supports a feature to back the Lustre filesystem with S3, which give an easy to manage and high +AWS supports a feature to back the Lustre filesystem with S3, which gives an easy to manage, high performance option. -Once the storage is in place, the next step is to copy your matrix files to that location. -Currently cellxgene supports a flat file organization. Each matrix file is located from -the same s3 prefix or filesystem directory. This location is specified in the configuration as the dataroot. +Once the storage is in place, the next step is to copy your data files to that location. +Currently cellxgene supports a flat file organization. Each matrix file is located under +the same s3 prefix or filesystem directory. This location is specified in the configuration +as the dataroot. -### 2. Create an elastic beanstalk application. For example: +### 2. Create an elastic beanstalk application. For example: ``` EB_APP=cellxgene-app @@ -54,9 +55,9 @@ eb init -p python-3.6 $EB_APP ### 3. Configuring cellxgene All the cellxgene configuration options can be set from a configuration file. -This file can be generated like this: +A yaml config file containing all of the default configuration options can be generated like this: -```cellxgene launch --dump-default-config > myconfig.yaml``` +`cellxgene launch --dump-default-config > myconfig.yaml` The config file may then be customized before the app is deployed. @@ -66,18 +67,14 @@ First, if your config file is named "config.yaml" and exists in `customize/confi then it will be bundled with the application zip file and installed along side the app on the EB servers. -Second, a potentially more flexible approach is to place your config file in a location accessible to the EB -servers, such as in S3. For example: s3://my-bucket/my-datasets/config.yaml. +Second, a potentially more flexible approach is to place your config file in a location accessible +to the EB servers, such as in S3. For example: s3://my-bucket/my-datasets/config.yaml. Set the CXG_CONFIG_FILE environment variable to specify this location. -Another option is to set the CXG_DATAROOT environment variable. The dataroot +Another option is to set the CXG_DATAROOT environment variable. The dataroot is the location where the matrix files are located. This environment variable will override the dataroot in the config file (if specified). -- Note: Certain features, such as user annotations, are automatically disabled by the EB app, -and cannot be enabled using configuration. They may be enabled manually by modifying app.py, however -this is not supported or recommended at this time. - ### 4. Customization The deployment can be customized in several ways, by adding files to a directory called @@ -93,15 +90,15 @@ The cellxgene server can serve additional static webpages that will be associate These include the about_legal_tos (terms of service), and about_legal_privacy, for example. To use this feature, do the following: -* In this directory, create a sub directory called "customize/deploy/". -* Copy the files you want to serve into this directory -* modify your configuration file to set the location to these file: /static/cellxgene/deploy/ +- In this directory, create a sub directory called "customize/deploy/". +- Copy the files you want to serve into this directory +- modify your configuration file to set the location to these file: /static/cellxgene/deploy/ -Example: you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene. +Example: you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene. Assume files called "tos.html" and "privacy.html" exist. ``` -$ mkdir static +$ mkdir -p customize/deploy $ cp /tos.html customize/deploy/tos.html $ cp /privacy.html customize/deploy/privacy.html @@ -116,14 +113,14 @@ about_legal_privacy: /static/cellxgene/deploy/privacy.html Additional scripts can be added using the server/inline_scripts config parameters. To include these scripts in the deployment, use the following steps: -* In this directory, create a sub directory called "customize/inline_scripts". -* Copy the script files into this directory -* modify your configuration file to set the location to these file (leaving off customize/inline_scripts) +- In this directory, create a sub directory called "customize/inline_scripts". +- Copy the script files into this directory +- Modify your configuration file to set the location to these file (leaving off customize/inline_scripts) For example, to add an inline script called "myscript.js": ``` -$ mkdir scripts +$ mkdir -p customize/inline_scripts $ cp /myscript.js customize/inline_scripts/myscript.js # edit the config.yaml $ grep inline_scripts config.yaml @@ -135,14 +132,14 @@ $ grep inline_scripts config.yaml Optionally, you can add plugins to the server python code. To include a plugin in the deployment use the following steps: ``` -$ mkdir plugins +$ mkdir -p customize/plugins $ cp /.py customize/plugins/.py ``` #### ebextensions Any additional config files intended for the `.ebextensions` directory of the artifact can be added -to the `customize/ebextensions` directory. Any file found here will be copied over. +to the `customize/ebextensions` directory. Any file found here will be copied over. #### requirements.txt @@ -152,6 +149,7 @@ This is useful to ensure that the dependencies do not change from one deployment Therefore the custom/requirements.txt must all have exact versions specified (e.g. anndata==0.7.1). This file can be generated the first time using a process like this: + ``` # assume you are running in this directory $ virtualenv temp @@ -168,6 +166,20 @@ If a future cellxgene version updates its requirements by modifying a module ver or adding a new dependency, then the `make build` process will detect any incompatibilities and raise an error. +#### File structure for customizations + +The following diagram shows the file structure for the customization directory. + +``` +customization ++-- config.yaml ++-- deploy/ ++-- inline_scripts/ ++-- plugins/ ++-- ebextensions/ ++-- requirements.txt +``` + ### 5. Create the artifact.zip file for the application ``` @@ -176,19 +188,13 @@ $ make build ### 6. Flask secret key -The application requires as secret key to be provided to flask, the web framework used by cellxgene. +The application requires a secret key to be provided to flask, the web framework used by cellxgene. There are three ways to provide the secret key: -- In the configuration file: update the server/flask_secret_key attribute. +- In the configuration file, update the server/flask_secret_key attribute. +- In the configuration file, update the external/aws_secrets_manager section to set the + secret name and key that defines the flask secret key. - An environment variable: `CXG_SECRET_KEY` -- Managed by the AWS Secret Manager - -If using the AWS Secret Manager, then the secret name is passed as an environment variable: CXG_AWS_SECRET_NAME. -The secret must contain a key with the name "flask_secret_key". -The region name for the AWS Secret Manager must be specified (e.g. us-east-1). -The most straightforward way is to specified it with the CXG_AWS_SECRET_REGION_NAME environment variable. -If this environment variable is not defined, then the app attempts to determine the region from the -dataroot (if in s3), or the config file location (if in s3). ### 7. Create an environment @@ -203,7 +209,8 @@ $ EB_INSTANCE=m5.large $ CXG_DATAROOT= $ CXG_CONFIG_FILE= -# Potentially also set envvars for the secret key. +# Potentially also set an environment variable for the flask secret key, +# and other environemet variable described in the configuration file. $ eb create $EB_ENV --instance-type $EB_INSTANCE \ --envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE @@ -227,3 +234,67 @@ $ eb deploy $EB_ENV ``` $ eb open $EB_ENV ``` + +## Advanced Features + +### Authentication + +Authentication can be configured in the configuration file. Authentication is required +for User Annotations (see below). User Annotations is a feature where annotations can be +created by the user +, and +then associated with the user's id. +When the user revisits the site, their annotations will be available. + +There are three main authentication modes: null, session, or oauth. +In the configuration file specify the authentication mode by setting +`server / authentication / type`. + +#### null + +Authentication is disabled: user annotations cannot be enabled. + +#### session + +The user is associated with their client browser session. This approach is +simple to setup, but not recommended for hosted cellxgene, since the user will not have access to +their annotations when running from a different browser, or if their cookies get cleared. + +#### oauth + +A user logs into cellxgene using an identity provider (like Google), or logs in using +an email/password. This is the best option, but requires making use of an oauth service and +additional configuration of the cellxgene server. + +To see what this looks like, please look at https://cellxgene.cziscience.com/, +and view one of the cellxgene datasets. +For this server, Auth0 (auth0.com) is used for authentication, but there are other options. +There are good sources of documentation online that describe how to use one of these +services. + +The `params_oauth` section in the configuration file describes characteristics of the +authentication service, like "client_id" and "client_secret". +For security, the client_secret needs to be protected. One option is to +store it in the AWS Secrets Manager. + +### User Annotations + +User annotations can be configured in the configuration file both generally and for a specific data route. The annotations feature is only available when Authorization is enabled. +To enable Annotations, it is necessary to create a relational database and add the database uri (typically `postgresql://[user[:password]@][netloc][:port][/dbname]`) to the secrets manager under `DB_URI`. +The hosted version of cellxgene runs on AWS's [Aurora PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.AuroraPostgreSQL.html) but any sqlalchemy compatible relational database should work. +Once the database is set up apply the cellxgene schema to your database by running the following inside the cellxgene repo +`PROJECT_ROOT=$(git rev-parse --show-toplevel)` +`python3` +Inside the python console +`from sqlalchemy import create_engine` +`from server.db.cellxgene_orm import Base` +`uri = "[DB_URI]”` +`engine = create_engine(uri)` + + Base.metadata.create_all(engine)` + +To check the schema was properly applied (or just to check what is in the database at any point) +ssh into your database. For a postgres database this entails running: +`psql [DB_URI]` + +You'll also need to update your IAM policies to allow the instance to write to the s3 bucket. diff --git a/server/eb/app.py b/server/eb/app.py index 7a5e834c..7de161b7 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -9,8 +9,6 @@ from flask import json import logging from flask_talisman import Talisman from flask_cors import CORS -from server.common.aws_secret_utils import handle_config_from_secret -from server.common.errors import SecretKeyRetrievalError if os.path.isdir("/opt/python/log"): @@ -26,7 +24,7 @@ SERVERDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.append(SERVERDIR) try: - from server.common.app_config import AppConfig + from server.common.config.app_config import AppConfig from server.app.app import Server from server.common.data_locator import DataLocator, discover_s3_region_name except Exception: @@ -61,8 +59,7 @@ class WSGIServer(Server): csp = { "default-src": ["'self'"], "connect-src": ["'self'"] + extra_connect_src, - "script-src": ["'self'", "'unsafe-eval'"] - + obsolete_browser_script_hash + script_hashes, + "script-src": ["'self'", "'unsafe-eval'"] + obsolete_browser_script_hash + script_hashes, "style-src": ["'self'", "'unsafe-inline'"], "img-src": ["'self'", "https://cellxgene.cziscience.com", "data:"], "object-src": ["'none'"], @@ -104,7 +101,7 @@ class WSGIServer(Server): if len(script_hashes) == 0: logging.error("Content security policy hashes are missing, falling back to unsafe-inline policy") - return (script_hashes) + return script_hashes @staticmethod def compute_inline_csp_hashes(app, app_config): @@ -166,28 +163,14 @@ try: logging.info("Configuration from CXG_DATAROOT") app_config.update_server_config(multi_dataset__dataroot=dataroot) - # update from secret manager - try: - handle_config_from_secret(app_config) - except SecretKeyRetrievalError: - sys.exit(1) - - # features are unsupported in the current hosted server - app_config.update_default_dataset_config( - embeddings__enable_reembedding=False, - ) + # overwrite configuration for the eb app + app_config.update_default_dataset_config(embeddings__enable_reembedding=False,) app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],) + + # complete config app_config.complete_config(logging.info) - if not app_config.server_config.app__flask_secret_key: - logging.critical( - "flask_secret_key is not provided. Either set in config file, CXG_SECRET_KEY environment variable, " - "or in AWS Secret Manager" - ) - sys.exit(1) - server = WSGIServer(app_config) - debug = False application = server.app @@ -202,7 +185,7 @@ else: if __name__ == "__main__": try: - application.run(debug=debug, threaded=not debug, use_debugger=False) + application.run(host=app_config.server_config.app__host, debug=debug, threaded=not debug, use_debugger=False) except Exception: logging.critical("Caught exception during initialization", exc_info=True) sys.exit(1) diff --git a/server/eb/check_config.py b/server/eb/check_config.py new file mode 100644 index 00000000..2ea1c25e --- /dev/null +++ b/server/eb/check_config.py @@ -0,0 +1,39 @@ +import sys +import argparse +import yaml + +from server.common.config.app_config import AppConfig + + +def main(): + parser = argparse.ArgumentParser("A script to check hosted configuration files") + parser.add_argument("config_file", help="the configuration file") + parser.add_argument( + "-s", + "--show", + default=False, + action="store_true", + help="print the configuration. NOTE: this may print secret values to stdout", + ) + + args = parser.parse_args() + + app_config = AppConfig() + try: + app_config.update_from_config_file(args.config_file) + app_config.complete_config() + except Exception as e: + print(f"Error: {str(e)}") + print("FAIL:", args.config_file) + sys.exit(1) + + if args.show: + yaml_config = app_config.config_to_dict() + yaml.dump(yaml_config, sys.stdout) + + print("PASS:", args.config_file) + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/server/requirements-dev.txt b/server/requirements-dev.txt index 0fb57ce8..9f4fd724 100644 --- a/server/requirements-dev.txt +++ b/server/requirements-dev.txt @@ -3,9 +3,8 @@ black bumpversion>=0.5 codecov>=2.0.15 parameterized>=0.7.0 -psycopg2==2.7.7 +psycopg2-binary>=2.8.5 pytest>=3.6.3 python-jose>=3.2.0 -scanpy>=1.4.6 twine>=1.12.1 -r requirements.txt diff --git a/server/requirements-prepare.txt b/server/requirements-prepare.txt index 630015df..254f827b 100644 --- a/server/requirements-prepare.txt +++ b/server/requirements-prepare.txt @@ -1,3 +1,2 @@ -scanpy>=1.3.7 python-igraph louvain>=0.6 diff --git a/server/requirements.txt b/server/requirements.txt index d188764b..18561771 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,4 +1,4 @@ -anndata>=0.6.20 +anndata>=0.7.0 boto3>=1.12.18 click>=7.1.2 fastobo>=0.6.1 @@ -11,14 +11,16 @@ flask-talisman>=0.7.0 flatbuffers>=1.11.0 flatten-dict>=0.2.0 fsspec>=0.4.4,<0.8.0 -numba>=0.49.1 -numpy>=1.16.0 -packaging>=20.0 -pandas>=0.24.2 -PyYAML>=5.3 -scipy>=1.3.0 -requests>=2.22.0 -sqlalchemy>=1.3.18 -tiledb>=0.5.9,>=0.6.2 -s3fs==0.4.2 gunicorn>=20.0.4 +h5py<3.0.0 # h5py returns bytes instead of str, which breaks many assumptions +numba>=0.49.1 +numpy>=1.15.0 +packaging>=20.0 +pandas>=1.0,!=1.1 # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446 +PyYAML>=5.3 +scipy>=1.0 +requests>=2.22.0 +tiledb>=0.5.9,>=0.6.2,!=0.7.2 +s3fs==0.4.2 +scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here +sqlalchemy>=1.3.18 diff --git a/server/test/__init__.py b/server/test/__init__.py index 8ed6e3e7..1b96f7c1 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -13,7 +13,8 @@ import requests from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB from server.common.annotations.local_file_csv import AnnotationsLocalFile -from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT +from server.common.config.app_config import AppConfig +from server.common.config import DEFAULT_SERVER_PORT from server.common.data_locator import DataLocator from server.common.utils.utils import find_available_port from server.data_common.fbs.matrix import encode_matrix_fbs @@ -33,8 +34,7 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType): data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( - multi_dataset__dataroot=data_locator.path, - authentication__type="test", + app__flask_secret_key="secret", multi_dataset__dataroot=data_locator.path, authentication__type="test", ) config.update_default_dataset_config( embeddings__names=["umap"], @@ -42,16 +42,13 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType): diffexp__lfc_cutoff=0.01, user_annotations__type="hosted_tiledb_array", user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432", - user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir + user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir, ) config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) - annotations = AnnotationsHostedTileDB( - tmp_dir, - DbUtils("postgresql://postgres:test_pw@localhost:5432"), - ) + annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),) return data, tmp_dir, annotations @@ -67,7 +64,10 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( - single_dataset__obs_names=None, single_dataset__var_names=None, single_dataset__datapath=data_locator.path + app__flask_secret_key="secret", + single_dataset__obs_names=None, + single_dataset__var_names=None, + single_dataset__datapath=data_locator.path, ) config.update_default_dataset_config( embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, @@ -100,6 +100,7 @@ def skip_if(condition, reason: str): def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}): config = AppConfig() config.update_server_config( + app__flask_secret_key="secret", single_dataset__obs_names=None, single_dataset__var_names=None, adaptor__anndata_adaptor__backed=backed, @@ -120,7 +121,7 @@ def random_string(n): return "".join(random.choice(string.ascii_letters) for _ in range(n)) -def start_test_server(command_line_args=[], app_config=None): +def start_test_server(command_line_args=[], app_config=None, env=None): """ Command line arguments can be passed in, as well as an app_config. This function is meant to be used like this, for example: @@ -158,7 +159,7 @@ def start_test_server(command_line_args=[], app_config=None): command.extend(["-c", config_file]) server = f"http://localhost:{port}" - ps = Popen(command) + ps = Popen(command, env=env) for _ in range(10): try: @@ -181,10 +182,10 @@ def stop_test_server(ps): @contextmanager -def test_server(command_line_args=[], app_config=None): +def test_server(command_line_args=[], app_config=None, env=None): """A context to run the cellxgene server.""" - ps, server = start_test_server(command_line_args, app_config) + ps, server = start_test_server(command_line_args, app_config, env) try: yield server finally: diff --git a/server/test/fixtures/database/__init__.py b/server/test/fixtures/database/__init__.py index 5a7fa984..0088f6d4 100644 --- a/server/test/fixtures/database/__init__.py +++ b/server/test/fixtures/database/__init__.py @@ -29,34 +29,26 @@ class TestDatabase: def _create_test_user(self): user = CellxGeneUser(id="test_user_id") - user2 = CellxGeneUser(id='1234') + user2 = CellxGeneUser(id="1234") self.db.session.add(user) self.db.session.add(user2) self.db.session.commit() def _create_test_dataset(self): - dataset = CellxGeneDataset( - name="test_dataset", - ) + dataset = CellxGeneDataset(name="test_dataset",) self.db.session.add(dataset) self.db.session.commit() def _create_test_annotation(self): - dataset = self.db.query([CellxGeneDataset], - [CellxGeneDataset.name == "test_dataset"], - )[0] - annotation = Annotation( - tiledb_uri="tiledb_uri", - user_id="test_user_id", - dataset_id=str(dataset.id) - ) + dataset = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == "test_dataset"],)[0] + annotation = Annotation(tiledb_uri="tiledb_uri", user_id="test_user_id", dataset_id=str(dataset.id)) self.db.session.add(annotation) self.db.session.commit() @staticmethod def get_random_string(): letters = string.ascii_lowercase - return ''.join(random.choice(letters) for i in range(12)) + return "".join(random.choice(letters) for i in range(12)) def _create_test_users(self, user_count: int = 10): users = [] @@ -80,10 +72,8 @@ class TestDatabase: for i in range(annotation_count): dataset = self.order_by_random(CellxGeneDataset) user = self.order_by_random(CellxGeneUser) - annotations.append(Annotation( - tiledb_uri=self.get_random_string(), - user_id=user.id, - dataset_id=str(dataset.id) - )) + annotations.append( + Annotation(tiledb_uri=self.get_random_string(), user_id=user.id, dataset_id=str(dataset.id)) + ) self.db.session.add_all(annotations) self.db.session.commit() diff --git a/server/test/fixtures/dataset_config_outline.py b/server/test/fixtures/dataset_config_outline.py new file mode 100644 index 00000000..a3d99d78 --- /dev/null +++ b/server/test/fixtures/dataset_config_outline.py @@ -0,0 +1,37 @@ +f""" +dataset: + app: + scripts: {scripts} #list of strs (filenames) or dicts containing keys + inline_scripts: {inline_scripts} #list of strs (filenames) + + about_legal_tos: {about_legal_tos} + about_legal_privacy: {about_legal_privacy} + + authentication_enable: {authentication_enable} + + presentation: + max_categories: {max_categories} + custom_colors: {custom_colors} + + user_annotations: + enable: {enable_users_annotations} + type: {annotation_type} + hosted_tiledb_array: + db_uri: {db_uri} + hosted_file_directory: {hosted_file_directory} + local_file_csv: + directory: {local_file_csv_directory} + file: {local_file_csv_file} + ontology: + enable: {ontology_enabled} + obo_location: {obo_location} + + embeddings: + names: {embedding_names} + enable_reembedding: {enable_reembedding} + + diffexp: + enable: {enable_difexp} + lfc_cutoff: {lfc_cutoff} + top_n: {top_n} +""" diff --git a/server/test/fixtures/hgnc_example.txt.gz b/server/test/fixtures/hgnc_example.txt.gz new file mode 100644 index 00000000..4c7704c4 Binary files /dev/null and b/server/test/fixtures/hgnc_example.txt.gz differ diff --git a/server/test/fixtures/schema_test_data/generate_test_data.sh b/server/test/fixtures/schema_test_data/generate_test_data.sh new file mode 100755 index 00000000..d0f7d699 --- /dev/null +++ b/server/test/fixtures/schema_test_data/generate_test_data.sh @@ -0,0 +1,139 @@ +#!/bin/bash +wget "https://s3-us-west-2.amazonaws.com/10x.files/samples/cell/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz" +tar xf "pbmc3k_filtered_gene_bc_matrices.tar.gz" + +python3 - < genes_tmp.tsv; mv genes_tmp.tsv merged/genes.tsv + +echo -e "\n\n\nRunning tutorial on original\n\n\n" +Rscript - < - -- does intitial render -- - 4. concurrently load all /annotations/obs - -- fully initialized -- + 1. Load index.html, etc. + 2. Concurrently load /config, /schema + 3. Concurrently load /layout/obs, /annotations/var?annotation-name= + -- Does initial render -- + 4. Concurrently load all /annotations/obs and all /layouts/obs + -- Fully initialized -- """ - # users hit all of the init routes as fast as they can, subject to the ordering constraints - # and network latency + # Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency. wait_time = between(0.01, 0.1) def on_start(self): self.dataset = self.parent.dataset self.client.verify = False + self.api_less_client = HttpSession( + base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""), + request_success=self.client.request_success, + request_failure=self.client.request_failure, + ) - @seq_task(1) + @task def index(self): - self.client.get(f"{self.dataset}/", stream=True).close() + self.api_less_client.get(f"{self.dataset}", stream=True) - @seq_task(2) - def loadConfigSchema(self): - def config(): - self.client.get(f"{self.dataset}{API}/config", stream=True).close() + @task + def loadConfigAndSchema(self): + self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True) + self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True) - def schema(): - self.client.get(f"{self.dataset}{API}/schema", stream=True).close() - - group = Group() - group.spawn(config) - group.spawn(schema) - group.join() - - @seq_task(3) + @task def loadBootstrapData(self): - def layout(): - self.client.get( - f"{self.dataset}{API}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True - ).close() - - def varAnnotationIndex(): - self.client.get( - f"{self.dataset}{API}/annotations/var?annotation-name={self.parent.var_index_name()}", - headers={"Accept": "application/octet-stream"}, - stream=True, - ).close() - - group = Group() - group.spawn(layout) - group.spawn(varAnnotationIndex) - group.join() - - @seq_task(4) - def loadObsAnnotations(self): - def obs_annotation(name): - self.client.get( - f"{self.dataset}{API}/annotations/obs?annotation-name={name}", - headers={"Accept": "application/octet-stream"}, - stream=True, - ).close() + self.client.get( + f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True + ) + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}", + headers={"Accept": "application/octet-stream"}, + catch_response=True, + ) + @task + def loadObsAnnotationsAndLayouts(self): obs_names = self.parent.obs_annotation_names() - group = Group() for name in obs_names: - group.spawn(obs_annotation, name) - group.join() + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}", + headers={"Accept": "application/octet-stream"}, + stream=True, + ) - @seq_task(5) + layouts = self.parent.layout_names() + for name in layouts: + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}", + headers={"Accept": "application/octet-stream"}, + stream=True, + ) + + @task def done(self): self.interrupt() @@ -146,19 +147,19 @@ class ViewDataset(TaskSet): """ Simulate user occasionally loading some expression data for a gene """ + gene_name = random.choice(self.gene_names) filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}} self.client.put( - f"{self.dataset}{API}/data/var", + f"{self.dataset}/{API_SUFFIX}/data/var", data=json.dumps(filter), headers={"Content-Type": "application/json", "Accept": "application/octet-stream"}, stream=True, ).close() -class CellxgeneUser(HttpLocust): - task_set = ViewDataset +class CellxgeneUser(HttpUser): + tasks = [CellXGeneTasks] - # most ops do not require back-end interaction, so slow cadence - # for users + # Most ops do not require back-end interaction, so slow cadence for users wait_time = between(10, 60) diff --git a/server/test/locust/requirements-locust.txt b/server/test/locust/requirements-locust.txt index 282a9af7..8eaebfd6 100644 --- a/server/test/locust/requirements-locust.txt +++ b/server/test/locust/requirements-locust.txt @@ -1 +1 @@ -locustio +locust diff --git a/server/test/performance/performance_test_annotations_backend.py b/server/test/performance/performance_test_annotations_backend.py new file mode 100644 index 00000000..d0548523 --- /dev/null +++ b/server/test/performance/performance_test_annotations_backend.py @@ -0,0 +1,215 @@ +import json +import string +from contextlib import contextmanager +from timeit import default_timer +import concurrent.futures +import numpy as np +import requests +import sys +from server.data_common.fbs.matrix import encode_matrix_fbs +import pandas as pd +import random + +""" +Before running, sign into the dataportal, copy the cookie and paste it below. To test in staging or prod update the +url base below. It is also possible to configure the number of categories created and the number of unique labels per +category. +""" + +cookie = "" + +test_datasets = { + "smallest": { + "dataset_url": "kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg", + "name": "smallest", + "num_cells": 5270, + }, + "10k": { + "dataset_url": "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg", + "name": "10k", + "num_cells": 9409, + }, + "80k": { + "dataset_url": "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg", # noqa E501 + "name": "80k", + "num_cells": 81736, + }, + "140k": {"dataset_url": "Single_cell_drug_screening_a549-42-remixed.cxg", "name": "140k", "num_cells": 143015}, + "largest": {"dataset_url": "human_cell_landscape.cxg", "name": "largest", "num_cells": 599926}, + "1million": {"dataset_url": None, "name": "1million", "num_cells": 1000000}, + "4million": {"dataset_url": None, "name": "4million", "num_cells": 4000000}, +} + +url_base = "https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/" +annotations_category_count = [1, 10, 50] +max_labels = [5, 50, 100] + + +class PerformanceTestingAnnotations: + def __init__( + self, + datasets=test_datasets, + annotations_category_count=annotations_category_count, + max_labels=max_labels, + url_base=url_base, + ): + self.test_datasets = datasets + self.annotations_category_count = annotations_category_count + self.max_labels = max_labels + self.url_base = url_base + self.test_notes = self.create_info_dict() + + def set_cell_count(self, dataset_name): + dataset_url = self.test_datasets[dataset_name]["dataset_url"] + headers = {"Content-Type": "application/octet-stream", "Cookie": cookie} + response = self.client.get(f"{self.url_base}{dataset_url}/api/v0.2/schema", headers=headers) + cell_count = json.loads(response._content)["schema"]["dataframe"]["nObs"] + self.test_datasets[dataset_name]["cell_count"] = cell_count + + def create_info_dict(self): + request_info = {} + for dataset in self.test_datasets.keys(): + request_info[dataset] = {} + for cat_count in self.annotations_category_count: + request_info[dataset][f"num_categories_{cat_count}"] = {} + for unique_labels in self.max_labels: + request_info[dataset][f"num_categories_{cat_count}"][f"max_label_{unique_labels}"] = {} + return request_info + + def create_annotations_dict_multi_process(self, dataset_name, category_count, label_max): + annotation_dict = {} + futures = [] + categories = [f"Category{i}" for i in range(category_count)] + if not self.test_datasets[dataset_name]["num_cells"]: + self.set_cell_count(dataset_name) + with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor: + for category in categories: + futures.append( + executor.submit( + self.build_array_for_category, + category, + self.test_datasets[dataset_name]["num_cells"], + label_max, + ) + ) + for future in concurrent.futures.as_completed(futures): + try: + result = future.result() + category_name, cells = result + annotation_dict[category_name] = pd.Series(cells, dtype="category") + except Exception as e: + print(f"Issue creating the annotations dict: {e}") + return annotation_dict + + def build_array_for_category(self, category_name, cell_count, label_max): + unique_label_count = label_max + labels = self.generate_labels(unique_label_count) + cells_per_label = int(cell_count / len(labels)) + extra = cell_count % len(labels) + cells = [] + for label in labels: + cells.extend([label] * cells_per_label) + cells.extend(["extra"] * extra) + rng = np.random.default_rng() + rng.shuffle(cells) + return category_name, cells + + @staticmethod + def convert_to_fbs(annotation_dict): + df = pd.DataFrame(annotation_dict) + return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) + + @staticmethod + def generate_labels(unique_label_count): + labels = ["undefined"] + for i in range(unique_label_count): + length = random.randrange(10, 20) + labels.append(f"{i}__" + "".join(random.choice(string.ascii_letters) for z in range(length))) + return labels + + @contextmanager + def elapsed_timer(self): + start = default_timer() + elapser = lambda: default_timer() - start # noqa E731 + yield lambda: elapser() + end = default_timer() + elapser = lambda: end - start # noqa E731 + + def create_matrix(self, dataset_name, num_cat, max_labels): + with self.elapsed_timer() as elapsed: + annon_dict = self.create_annotations_dict_multi_process(dataset_name, num_cat, max_labels) + dict_size = sum(sys.getsizeof(value) for value in annon_dict.values()) / 1024 ** 2 + self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["annotation_dict"] = { + "creation_time": str(elapsed()), + "size": f"{dict_size} mb", + } + df = pd.DataFrame(annon_dict) + df_size = sys.getsizeof(df) / 1024 ** 2 + self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["data_frame"] = { + "creation_time": str(elapsed()), + "size": f"{df_size} mb", + } + try: + matrix = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) + matrix_size = sys.getsizeof(matrix) / 1024 ** 2 + self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["fbs_matrix"] = { + "creation_time": str(elapsed()), + "size": f"{matrix_size} mb", + } + return matrix + except Exception as e: + print(f"Issue creating fbs matrix: {e}, for {dataset_name}") + return [] + + def send_put_request(self, dataset_url, data): + url = self.url_base + f"{dataset_url}/api/v0.2/annotations/obs" + with self.elapsed_timer() as elapsed: + try: + headers = {"Content-Type": "application/octet-stream", "Cookie": cookie} + response = requests.put(url=url, data=data, headers=headers) + except Exception as e: + print(f"Issue with put request: {e}") + return None, elapsed() + return response, elapsed() + + def test_categories_max_label_matrix(self, dataset_name): + for unique_labels in self.max_labels: + for category_count in self.annotations_category_count: + print(f"Starting dataset: {dataset_name}, categories: {category_count}, labels: {unique_labels}") + fbs_matrix = self.create_matrix(dataset_name, category_count, unique_labels) + if self.test_datasets[dataset_name]["dataset_url"] and fbs_matrix: + response, response_time = self.send_put_request( + self.test_datasets[dataset_name]["dataset_url"], fbs_matrix + ) + if response is None: + self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][ + "put_request" + ] = {"response_status": "failed", "request_time": str(response_time)} + else: + self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][ + "put_request" + ] = {"response_status": response.status_code, "request_time": str(response_time)} + + +def test_all_datasets(): + """ + Run time is dependent on number of datasets, dataset size, number of categories/number being tested and number of + unique label counts being tested. However it generally takes a long time. I recommend running this in tmux + """ + perf_test = PerformanceTestingAnnotations() + for dataset_name in perf_test.test_datasets.keys(): + print(f"Testing annotation creation for: {dataset_name}") + try: + perf_test.test_categories_max_label_matrix(dataset_name) + except Exception as e: + print(f"something went wrong with {dataset_name}: {e}") + return perf_test.test_notes + + +def main(): + notes = test_all_datasets() + print(notes) + + +if __name__ == "__main__": + main() diff --git a/server/test/performance/run_diffexp.py b/server/test/performance/run_diffexp.py index bbbb3094..5cd1fbef 100644 --- a/server/test/performance/run_diffexp.py +++ b/server/test/performance/run_diffexp.py @@ -7,7 +7,7 @@ import numpy as np import server.compute.diffexp_cxg as diffexp_cxg import server.compute.diffexp_generic as diffexp_generic -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.data_common.matrix_loader import MatrixDataLoader from server.data_cxg.cxg_adaptor import CxgAdaptor diff --git a/server/test/performance/scale_test_annotations.py b/server/test/performance/scale_test_annotations.py new file mode 100644 index 00000000..a26a16c0 --- /dev/null +++ b/server/test/performance/scale_test_annotations.py @@ -0,0 +1,45 @@ +import time +import random + +from locust import HttpUser, between, task + +random.seed(time.time()) +""" +To run this script sign into cellxgene in the desired environment and grab the returned cookie, update the cookie +variable below with your cookie and run the following command to see results in the terminal: +locust -f server/test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt + +Or if you want to use the locust gui run: +locust -f server/test/performance/scale_test_annotations.py -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ + +If you want to test staging you'll need to substitute staging for dev in the host url +To test prod you'll need to replace dev.single-cell.czi.technology with cziscience.com +If you'd like to test additional datasets you'll need to add them to the dataset_urls array + +Todo @mdunitz update script to retrieve different annotation categories -- may need to create them to ensure the +categories are shared across datasets for a given user. +""" +cookie = "" + + +class WebsiteUser(HttpUser): + wait_time = between(1, 2) + dataset_urls = [ + "human_cell_landscape.cxg", + "Single_cell_drug_screening_a549-42-remixed.cxg", + "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg", + "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg", + ] + + @task + def get_annotations(self): + dataset_url = random.choice(self.dataset_urls) + url = f"{dataset_url}/api/v0.2/annotations/obs?annotation-name=cell_type" + headers = {"Content-Type": "application/octet-stream", "Cookie": cookie} + self.client.get(url, headers=headers) + + @task + def get_schema(self): + dataset_url = random.choice(self.dataset_urls) + headers = {"Content-Type": "application/octet-stream", "Cookie": cookie} + self.client.get(f"{dataset_url}/api/v0.2/schema", headers=headers) diff --git a/server/test/test_database/test_database.py b/server/test/test_database/test_database.py index 4ecb4f27..91798fdb 100644 --- a/server/test/test_database/test_database.py +++ b/server/test/test_database/test_database.py @@ -16,45 +16,48 @@ class DatabaseTest(unittest.TestCase): del cls.db def test_user_creation(self): - one_user = self.db.get(table=CellxGeneUser, entity_id='test_user_id') - self.assertEqual(one_user.id, 'test_user_id') + one_user = self.db.get(table=CellxGeneUser, entity_id="test_user_id") + self.assertEqual(one_user.id, "test_user_id") user_count = self.db.session.query(CellxGeneUser).count() self.assertGreater(user_count, 10) def test_dataset_creation(self): - one_dataset = self.db.query(table_args=[CellxGeneDataset], - filter_args=[CellxGeneDataset.name == 'test_dataset']) - self.assertEqual(one_dataset[0].name, 'test_dataset') + one_dataset = self.db.query( + table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"] + ) + self.assertEqual(one_dataset[0].name, "test_dataset") dataset_count = self.db.session.query(CellxGeneDataset).count() self.assertGreater(dataset_count, 10) def test_annotation_creation(self): - one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0] - self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri') + one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == "tiledb_uri"])[0] + self.assertEqual(one_annotation.tiledb_uri, "tiledb_uri") annotation_count = self.db.session.query(Annotation).count() self.assertGreater(annotation_count, 10) def test_get_most_recent_annotation_for_user_dataset(self): - dataset_id = str(self.db.query(table_args=[CellxGeneDataset], - filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id) + dataset_id = str( + self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"])[0].id + ) # have to commit separately because created_at time written on the db server - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_0")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_1")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_2")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_3")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_4")) self.db.session.commit() - most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id, - Annotation.user_id == 'test_user_id']) + most_recent_annotation = self.db.query_for_most_recent( + Annotation, [Annotation.dataset_id == dataset_id, Annotation.user_id == "test_user_id"] + ) - self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4') + self.assertEqual(most_recent_annotation.tiledb_uri, "tiledb_uri_4") diff --git a/server/test/unit/auth/test_auth.py b/server/test/unit/auth/test_auth.py index b07e38d3..2caba6b1 100644 --- a/server/test/unit/auth/test_auth.py +++ b/server/test/unit/auth/test_auth.py @@ -2,7 +2,7 @@ import unittest import requests -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.test import FIXTURES_ROOT, test_server @@ -11,15 +11,14 @@ class AuthTest(unittest.TestCase): self.dataset_dataroot = FIXTURES_ROOT def test_auth_none(self): - c = AppConfig() - c.update_server_config( - authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot - ) - c.update_default_dataset_config(user_annotations__enable=False) + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot) + app_config.update_default_dataset_config(user_annotations__enable=False) - c.complete_config() + app_config.complete_config() - with test_server(app_config=c) as server: + with test_server(app_config=app_config) as server: session = requests.Session() config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() @@ -27,14 +26,13 @@ class AuthTest(unittest.TestCase): self.assertIsNone(userinfo) def test_auth_session(self): - c = AppConfig() - c.update_server_config( - authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot - ) - c.update_default_dataset_config(user_annotations__enable=True) - c.complete_config() + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot) + app_config.update_default_dataset_config(user_annotations__enable=True) + app_config.complete_config() - with test_server(app_config=c) as server: + with test_server(app_config=app_config) as server: session = requests.Session() config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() @@ -44,9 +42,10 @@ class AuthTest(unittest.TestCase): self.assertEqual(userinfo["userinfo"]["username"], "anonymous") def test_auth_test(self): - c = AppConfig() - c.update_server_config(authentication__type="test") - c.update_server_config( + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config(authentication__type="test") + app_config.update_server_config( multi_dataset__dataroot=dict( a1=dict(dataroot=self.dataset_dataroot, base_url="auth"), a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"), @@ -54,12 +53,12 @@ class AuthTest(unittest.TestCase): ) # specialize the configs - c.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True) - c.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False) + app_config.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True) + app_config.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False) - c.complete_config() + app_config.complete_config() - with test_server(app_config=c) as server: + with test_server(app_config=app_config) as server: session = requests.Session() # auth datasets @@ -86,6 +85,7 @@ class AuthTest(unittest.TestCase): userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertTrue(userinfo["userinfo"]["is_authenticated"]) self.assertEqual(userinfo["userinfo"]["username"], "test_account") + self.assertEqual(userinfo["userinfo"]["picture"], None) self.assertTrue(config["config"]["parameters"]["annotations"]) r = session.get(f"{server}/{logout_uri}") @@ -104,15 +104,22 @@ class AuthTest(unittest.TestCase): self.assertIsNone(userinfo) self.assertFalse(config["config"]["parameters"]["annotations"]) + # login with a picture + session.get(f"{server}/{login_uri}&picture=myimage.png") + userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json() + self.assertTrue(userinfo["userinfo"]["is_authenticated"]) + self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png") + def test_auth_test_single(self): - c = AppConfig() - c.update_server_config( - authentication__type="test", - single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg") + app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config( + authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg" + ) - c.complete_config() + app_config.complete_config() - with test_server(app_config=c) as server: + with test_server(app_config=app_config) as server: session = requests.Session() config = session.get(f"{server}/api/v0.2/config").json() userinfo = session.get(f"{server}/api/v0.2/userinfo").json() @@ -127,10 +134,10 @@ class AuthTest(unittest.TestCase): self.assertEqual(login_uri, "/login") self.assertEqual(logout_uri, "/logout") - r = session.get(f"{server}/{login_uri}") + response = session.get(f"{server}/{login_uri}") # check that the login redirect worked - self.assertEqual(r.history[0].status_code, 302) - self.assertEqual(r.url, f"{server}/") + self.assertEqual(response.history[0].status_code, 302) + self.assertEqual(response.url, f"{server}/") config = session.get(f"{server}/api/v0.2/config").json() userinfo = session.get(f"{server}/api/v0.2/userinfo").json() @@ -138,10 +145,10 @@ class AuthTest(unittest.TestCase): self.assertEqual(userinfo["userinfo"]["username"], "test_account") self.assertTrue(config["config"]["parameters"]["annotations"]) - r = session.get(f"{server}/{logout_uri}") + response = session.get(f"{server}/{logout_uri}") # check that the logout redirect worked - self.assertEqual(r.history[0].status_code, 302) - self.assertEqual(r.url, f"{server}/") + self.assertEqual(response.history[0].status_code, 302) + self.assertEqual(response.url, f"{server}/") config = session.get(f"{server}/api/v0.2/config").json() userinfo = session.get(f"{server}/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py index 42734a7b..caa13968 100644 --- a/server/test/unit/auth/test_oauth.py +++ b/server/test/unit/auth/test_oauth.py @@ -9,7 +9,7 @@ from flask import Flask, jsonify, make_response, request, redirect from multiprocessing import Process import jose -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.test import FIXTURES_ROOT, test_server # This tests the oauth authentication type. @@ -46,7 +46,7 @@ def token(): "scope": "openid profile email", "expires_in": TOKEN_EXPIRES, "token_type": "Bearer", - "expires_at": expires_at + "expires_at": expires_at, } return make_response(jsonify(r)) @@ -63,35 +63,57 @@ def jwks(): return make_response(jsonify(dict(keys=[data]))) -# The port that the mock oauth server will listen on -PORT = random.randint(10000, 12000) - - # function to launch the mock oauth server -def launch_mock_oauth(): - mock_oauth_app.run(port=PORT) +def launch_mock_oauth(mock_port): + mock_oauth_app.run(port=mock_port) class AuthTest(unittest.TestCase): - def setUp(self): - self.dataset_dataroot = FIXTURES_ROOT - self.mock_oauth_process = Process(target=launch_mock_oauth) - self.mock_oauth_process.start() + @classmethod + def setUpClass(cls): + # The port that the mock oauth server will listen on + cls.mock_port = random.randint(10000, 12000) + cls.dataset_dataroot = FIXTURES_ROOT + cls.mock_oauth_process = Process(target=launch_mock_oauth, args=(cls.mock_port,)) + cls.mock_oauth_process.start() - def tearDown(self): - self.mock_oauth_process.terminate() + # Verify that the mock oauth server is ready (accepting requests) before starting the tests. + + # The following lines are polling until the mock server is ready. + # The issue is we are starting a mock oauth server, then we are starting a cellxgene server, + # which will start making requests to the mock oauth server. + # So there is a race condition because the mock oauth server needs to be ready before it gets requests. + # We check to see if it is ready, and if not we wait 1 second, then try again. + # If it gets to 5 seconds, which is shouldn't, we assume something has gone wrong and fail the test. + server_okay = False + for _ in range(5): + try: + response = requests.get(f"http://localhost:{cls.mock_port}/.well-known/jwks.json") + if response.status_code == 200: + server_okay = True + break + except: # noqa: E722 + pass + + # wait one second and try again + time.sleep(1) + + assert(server_okay) + + @classmethod + def tearDownClass(cls): + cls.mock_oauth_process.terminate() def auth_flow(self, app_config, cookie_key=None): app_config.update_server_config( app__api_base_url="local", authentication__type="oauth", - authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}", + authentication__params_oauth__oauth_api_base_url=f"http://localhost:{self.mock_port}", authentication__params_oauth__client_id="mock_client_id", authentication__params_oauth__client_secret="mock_client_secret", - authentication__params_oauth__jwt_decode_options={ - "verify_signature": False, "verify_iss": False - }) + authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False}, + ) app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot) app_config.complete_config() @@ -112,7 +134,7 @@ class AuthTest(unittest.TestCase): logout_uri = config["config"]["authentication"]["logout"] self.assertEqual(login_uri, f"{server}/login?dataset=d/pbmc3k.cxg/") - self.assertEqual(logout_uri, f"{server}/logout") + self.assertEqual(logout_uri, f"{server}/logout?dataset=d/pbmc3k.cxg/") r = session.get(login_uri) # check that the login redirect worked @@ -122,6 +144,7 @@ class AuthTest(unittest.TestCase): userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertTrue(userinfo["userinfo"]["is_authenticated"]) self.assertEqual(userinfo["userinfo"]["username"], "fake_user") + self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com") self.assertTrue(config["config"]["parameters"]["annotations"]) if cookie_key: @@ -147,10 +170,32 @@ class AuthTest(unittest.TestCase): self.assertNotEqual(access_token_before, access_token_after) self.assertNotEqual(id_token_before, id_token_after) + # invalid cookie is rejected + session.cookies.set(cookie_key, "TEST_" + cookie) + self.assertTrue(cookie_key in session.cookies) + response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo") + # this is not an error, the invalid cookie is just ignored. + self.assertEqual(response.status_code, 200) + userinfo = response.json() + self.assertFalse(userinfo["userinfo"]["is_authenticated"]) + self.assertIsNone(userinfo["userinfo"]["username"]) + + # invalid id_token is rejected + test_token = token + test_token["id_token"] = "TEST_" + id_token_after + encoded_cookie = base64.b64encode(json.dumps(test_token).encode()).decode() + session.cookies.set(cookie_key, encoded_cookie) + response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo") + # this is not an error, the invalid id_token is just ignored. + self.assertEqual(response.status_code, 200) + userinfo = response.json() + self.assertFalse(userinfo["userinfo"]["is_authenticated"]) + self.assertIsNone(userinfo["userinfo"]["username"]) + r = session.get(logout_uri) # check that the logout redirect worked self.assertEqual(r.history[0].status_code, 302) - self.assertEqual(r.url, f"{server}") + self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/") config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) @@ -160,14 +205,14 @@ class AuthTest(unittest.TestCase): def test_auth_oauth_session(self): # test with session cookies app_config = AppConfig() - app_config.update_server_config( - authentication__params_oauth__session_cookie=True, - ) + app_config.update_server_config(app__flask_secret_key="secret") + app_config.update_server_config(authentication__params_oauth__session_cookie=True,) self.auth_flow(app_config) def test_auth_oauth_cookie(self): # test with specified cookie app_config = AppConfig() + app_config.update_server_config(app__flask_secret_key="secret") app_config.update_server_config( authentication__params_oauth__session_cookie=False, authentication__params_oauth__cookie=dict(key="test_cxguser", httponly=True, max_age=60), diff --git a/server/test/unit/cli/test_launch.py b/server/test/unit/cli/test_launch.py new file mode 100644 index 00000000..09906abe --- /dev/null +++ b/server/test/unit/cli/test_launch.py @@ -0,0 +1,27 @@ +import filecmp +import os +import shutil +import unittest + +import yaml + +from server.default_config import default_config +from server.test import FIXTURES_ROOT + + +class CLIPLaunchTests(unittest.TestCase): + tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs") + + @classmethod + def setUpClass(cls) -> None: + os.mkdir(cls.tmp_dir) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_dir) + + def test_dump_default_config(self): + os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt") + with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config: + expected_config.write(yaml.dump(default_config)) + filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt") diff --git a/server/test/unit/common/config/__init__.py b/server/test/unit/common/config/__init__.py new file mode 100644 index 00000000..8a48be5e --- /dev/null +++ b/server/test/unit/common/config/__init__.py @@ -0,0 +1,283 @@ +import os +import shutil +import unittest +import random +from unittest import mock +import yaml + +from server.test import FIXTURES_ROOT + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class ConfigTests(unittest.TestCase): + tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir") + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_fixtures_directory) + + @classmethod + def setUpClass(cls) -> None: + os.makedirs(cls.tmp_fixtures_directory) + + def custom_server_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="secret", + generate_cache_control_headers="false", + server_timing_headers="false", + csp_directives="null", + api_base_url="null", + web_base_url="null", + auth_type="session", + oauth_api_base_url="null", + client_id="null", + client_secret="null", + jwt_decode_options="null", + session_cookie="true", + cookie="null", + dataroot="null", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + diffexp_max_workers=64, + cpu_multiplier=4, + target_workunit="16_000_000", + data_locater_region_name="us-east-1", + cxg_tile_cache_size=8589934592, + cxg_num_reader_threads=32, + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + config_file_name="server_config.yaml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py") + with open(server_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + server_config = eval(config) + with open(configfile, "w") as server_config_file: + server_config_file.write(server_config) + return configfile + + def custom_app_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="secret", + generate_cache_control_headers="false", + server_timing_headers="false", + csp_directives="null", + api_base_url="null", + web_base_url="null", + auth_type="session", + oauth_api_base_url="null", + client_id="null", + client_secret="null", + jwt_decode_options="null", + session_cookie="true", + cookie="null", + dataroot="null", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + diffexp_max_workers=64, + cpu_multiplier=4, + target_workunit="16_000_000", + data_locater_region_name="us-east-1", + cxg_tile_cache_size=8589934592, + cxg_num_reader_threads=32, + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + scripts=[], + inline_scripts=[], + about_legal_tos="null", + about_legal_privacy="null", + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + ontology_enabled="false", + obo_location="null", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + environment=None, + aws_secrets_manager_region=None, + aws_secrets_manager_secrets=[], + config_file_name="app_config.yml", + ): + random_num = random.randrange(999999) + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config = self.custom_server_config( + verbose=verbose, + debug=debug, + host=host, + port=port, + open_browser=open_browser, + force_https=force_https, + flask_secret_key=flask_secret_key, + generate_cache_control_headers=generate_cache_control_headers, + server_timing_headers=server_timing_headers, + csp_directives=csp_directives, + api_base_url=api_base_url, + web_base_url=web_base_url, + auth_type=auth_type, + oauth_api_base_url=oauth_api_base_url, + client_id=client_id, + client_secret=client_secret, + jwt_decode_options=jwt_decode_options, + session_cookie=session_cookie, + cookie=cookie, + dataroot=dataroot, + index=index, + allowed_matrix_types=allowed_matrix_types, + max_cached_datasets=max_cached_datasets, + timelimit_s=timelimit_s, + dataset_datapath=dataset_datapath, + obs_names=obs_names, + var_names=var_names, + about=about, + title=title, + diffexp_max_workers=diffexp_max_workers, + cpu_multiplier=cpu_multiplier, + target_workunit=target_workunit, + data_locater_region_name=data_locater_region_name, + cxg_tile_cache_size=cxg_tile_cache_size, + cxg_num_reader_threads=cxg_num_reader_threads, + anndata_backed=anndata_backed, + column_request_max=column_request_max, + diffexp_cellcount_max=diffexp_cellcount_max, + config_file_name=f"temp_server_config_{random_num}.yml", + ) + dataset_config = self.custom_dataset_config( + scripts=scripts, + inline_scripts=inline_scripts, + about_legal_tos=about_legal_tos, + about_legal_privacy=about_legal_privacy, + authentication_enable=authentication_enable, + max_categories=max_categories, + custom_colors=custom_colors, + enable_users_annotations=enable_users_annotations, + annotation_type=annotation_type, + db_uri=db_uri, + hosted_file_directory=hosted_file_directory, + local_file_csv_directory=local_file_csv_directory, + local_file_csv_file=local_file_csv_file, + ontology_enabled=ontology_enabled, + obo_location=obo_location, + embedding_names=embedding_names, + enable_reembedding=enable_reembedding, + enable_difexp=enable_difexp, + lfc_cutoff=lfc_cutoff, + top_n=top_n, + config_file_name=f"temp_dataset_config_{random_num}.yml", + ) + external_config = self.custom_external_config( + environment=environment, + aws_secrets_manager_region=aws_secrets_manager_region, + aws_secrets_manager_secrets=aws_secrets_manager_secrets, + config_file_name=f"temp_external_config_{random_num}.yml", + ) + + with open(configfile, "w") as app_config_file: + app_config_file.write(open(server_config).read()) + app_config_file.write(open(dataset_config).read()) + app_config_file.write(open(external_config).read()) + + return configfile + + def custom_dataset_config( + self, + scripts=[], + inline_scripts=[], + about_legal_tos="null", + about_legal_privacy="null", + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + ontology_enabled="false", + obo_location="null", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + config_file_name="dataset_config.yml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py") + with open(dataset_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + dataset_config = eval(config) + with open(configfile, "w") as dataset_config_file: + dataset_config_file.write(dataset_config) + + return configfile + + def custom_external_config( + self, + environment=None, + aws_secrets_manager_region=None, + aws_secrets_manager_secrets=[], + config_file_name="external_config.yaml", + ): + # set to the default if environment is None + if environment is None: + environment = [ + dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False), + dict( + name="CXG_OAUTH_CLIENT_SECRET", + path=["server", "authentication", "params_oauth", "client_secret"], + required=False, + ), + ] + external_config = { + "external": { + "environment": environment, + "aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets}, + } + } + + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + with open(configfile, "w") as external_config_file: + yaml.dump(external_config, external_config_file) + return configfile diff --git a/server/test/unit/common/config/test_app_config.py b/server/test/unit/common/config/test_app_config.py new file mode 100644 index 00000000..ea8fe047 --- /dev/null +++ b/server/test/unit/common/config/test_app_config.py @@ -0,0 +1,220 @@ +import os +import tempfile +import unittest + +import yaml + +from server.default_config import default_config +from server.common.config.app_config import AppConfig +from server.test.unit.common.config import ConfigTests +from server.common.errors import ConfigurationError +from server.test import FIXTURES_ROOT + + +class AppConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_get_default_config_correctly_reads_default_config_file(self): + app_default_config = AppConfig().default_config + + expected_config = yaml.load(default_config, Loader=yaml.Loader) + + server_config = app_default_config["server"] + dataset_config = app_default_config["dataset"] + + expected_server_config = expected_config["server"] + expected_dataset_config = expected_config["dataset"] + + self.assertDictEqual(app_default_config, expected_config) + self.assertDictEqual(server_config, expected_server_config) + self.assertDictEqual(dataset_config, expected_dataset_config) + + def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self): + datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad" + file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name) + config = AppConfig() + config.update_from_config_file(file_name) + + self.assertEqual(config.get_dataset_config(""), config.default_dataset_config) + + def test_update_server_config_updates_server_config_and_config_status(self): + config = self.get_config() + config.complete_config() + config.check_config() + config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + with self.assertRaises(ConfigurationError): + config.server_config.check_config() + + def test_write_config_outputs_yaml_with_all_config_vars(self): + config = self.get_config() + config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml") + with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config: + default_config_yml = yaml.safe_load(default_config) + + with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config: + output_config_yml = yaml.safe_load(output_config) + self.maxDiff = None + self.assertEqual(default_config_yml, output_config_yml) + + def test_update_app_config(self): + config = AppConfig() + config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir") + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)]) + + config = AppConfig() + config.update_default_dataset_config(app__scripts=(), app__inline_scripts=()) + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[]) + vars = config.default_dataset_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) + vars = config.default_dataset_config.changes_from_default() + self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) + + def test_configfile_no_dataset_section(self): + # test a config file without a dataset section + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + server: + app: + flask_secret_key: secret + multi_dataset: + dataroot: test_dataroot + + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + server_changes = app_config.server_config.changes_from_default() + dataset_changes = app_config.default_dataset_config.changes_from_default() + self.assertEqual( + server_changes, + [("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)], + ) + self.assertEqual(dataset_changes, []) + + def test_configfile_no_server_section(self): + # test a config file without a dataset section + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + dataset: + user_annotations: + enable: false + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + server_changes = app_config.server_config.changes_from_default() + dataset_changes = app_config.default_dataset_config.changes_from_default() + self.assertEqual(server_changes, []) + self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)]) + + def test_simple_update_single_config_from_path_and_value(self): + """Update a simple config parameter""" + + config = AppConfig() + config.server_config.multi_dataset__dataroot = dict( + s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"), + s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"), + ) + config.add_dataroot_config("s1") + config.add_dataroot_config("s2") + + # test simple value in server + config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret") + self.assertEqual(config.server_config.app__flask_secret_key, "mysecret") + + # test simple value in default dataset + config.update_single_config_from_path_and_value( + ["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"], "mydburi", + ) + self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi") + self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "mydburi") + self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi") + + # test simple value in specific dataset + config.update_single_config_from_path_and_value( + ["per_dataset_config", "s1", "user_annotations", "hosted_tiledb_array", "db_uri"], "s1dburi" + ) + self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi") + self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "s1dburi") + self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi") + + # error checking + bad_paths = [ + ( + ["dataset", "does", "not", "exist"], + "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'", + ), + (["does", "not", "exist"], "path must start with 'server', 'dataset', or 'per_dataset_config'"), + ([], "path must start with 'server', 'dataset', or 'per_dataset_config'"), + (["per_dataset_config"], "missing dataroot when using per_dataset_config: got '['per_dataset_config']'"), + ( + ["per_dataset_config", "unknown"], + "unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']'," + " dataroots specified in config are ['s1', 's2']", + ), + ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"), + ("string", "path must be a list of strings, got 'string'"), + ] + for bad_path, error_message in bad_paths: + with self.assertRaises(ConfigurationError) as config_error: + config.update_single_config_from_path_and_value(bad_path, "value") + + self.assertEqual(config_error.exception.message, error_message) + + def test_dict_update_single_config_from_path_and_value(self): + """Update a config parameter that has a value of dict""" + + # the path leads to a dict config param, set the config parameter to the new value + config = AppConfig() + config.update_single_config_from_path_and_value( + ["server", "authentication", "params_oauth", "cookie"], dict(key="mykey1", max_age=100) + ) + self.assertEqual(config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100)) + + # the path leads to an entry within a dict config param, the value is simple + config = AppConfig() + config.server_config.authentication__params_oauth__cookie = dict(key="mykey1", max_age=100) + config.update_single_config_from_path_and_value( + ["server", "authentication", "params_oauth", "cookie", "httponly"], True, + ) + self.assertEqual( + config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100, httponly=True) + ) diff --git a/server/test/unit/common/config/test_base_config.py b/server/test/unit/common/config/test_base_config.py new file mode 100644 index 00000000..d5d538cd --- /dev/null +++ b/server/test/unit/common/config/test_base_config.py @@ -0,0 +1,65 @@ +import unittest + +from server.common.config.app_config import AppConfig +from server.test import FIXTURES_ROOT +from server.test.unit.common.config import ConfigTests +from server.common.errors import ConfigurationError + + +class BaseConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_mapping_creation_returns_map_of_server_and_dataset_config(self): + config = AppConfig() + mapping = config.default_dataset_config.create_mapping(config.default_config) + self.assertIsNotNone(mapping["server__app__verbose"]) + self.assertIsNotNone(mapping["dataset__presentation__max_categories"]) + self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"]) + self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"]) + + def test_changes_from_default_returns_list_of_nondefault_config_values(self): + config = self.get_config(verbose="true", lfc_cutoff=0.05) + server_changes = config.server_config.changes_from_default() + dataset_changes = config.default_dataset_config.changes_from_default() + + self.assertEqual( + server_changes, + [ + ("app__verbose", True, False), + ("app__flask_secret_key", "secret", None), + ("multi_dataset__dataroot", FIXTURES_ROOT, None), + ("multi_dataset__matrix_cache__timelimit_s", 5, 30), + ("data_locator__s3__region_name", "us-east-1", True), + ], + ) + self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)]) + + def test_check_config_throws_error_if_attr_has_not_been_checked(self): + config = self.get_config(verbose="true") + config.complete_config() + config.check_config() + config.update_server_config(app__verbose=False) + with self.assertRaises(ConfigurationError): + config.check_config() diff --git a/server/test/unit/common/config/test_dataset_config.py b/server/test/unit/common/config/test_dataset_config.py new file mode 100644 index 00000000..b65c7d83 --- /dev/null +++ b/server/test/unit/common/config/test_dataset_config.py @@ -0,0 +1,264 @@ +import os +import tempfile + +import requests +import unittest +from unittest.mock import patch + +from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB +from server.common.annotations.local_file_csv import AnnotationsLocalFile +from server.common.config.app_config import AppConfig +from server.common.config.base_config import BaseConfig +from server.test import test_server, PROJECT_ROOT, FIXTURES_ROOT + +from server.common.errors import ConfigurationError +from server.test.unit.common.config import ConfigTests + + +class TestDatasetConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.dataset_config = self.config.default_dataset_config + self.config.complete_config() + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_datatset_config_sets_vars_from_default_config(self): + config = AppConfig() + self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000) + self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv") + self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01) + self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location) + + @patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.dataset_config.complete_config(self.context) + self.assertEqual(mock_check_attrs.call_count, 21) + + def test_app_sets_script_vars(self): + config = self.get_config(scripts=["path/to/script"]) + config.default_dataset_config.handle_app() + + self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}]) + + config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}]) + config.default_dataset_config.handle_app() + self.assertEqual( + config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}] + ) + + config = self.get_config(scripts=["path/to/script", "different/script/path"]) + config.default_dataset_config.handle_app() + # TODO @madison -- is this the desired functionality? + self.assertEqual( + config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}] + ) + + config = self.get_config(scripts=[{"more": "different/script/path"}]) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_app() + + def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self): + config = self.get_config(enable_users_annotations="true", authentication_enable="false") + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_user_annotations(self.context) + + config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend") + with self.assertRaises(ConfigurationError): + config.server_config.complete_config(self.context) + + def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self): + config = self.get_config( + enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set" + ) + config.default_dataset_config.handle_user_annotations(self.context) + + self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."]) + + @patch("server.common.config.dataset_config.DbUtils") + def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils): + mock_db_utils.return_value = "123" + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_user_annotations(self.context) + self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile) + + config = self.get_config( + enable_users_annotations="true", + authentication_enable="true", + annotation_type="hosted_tiledb_array", + db_uri="gotta/set/this", + hosted_file_directory="and/this", + ) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_user_annotations(self.context) + self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB) + + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL" + ) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_user_annotations(self.context) + + def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self): + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_local_file_csv_annotations() + self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile) + cwd = os.getcwd() + self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd) + + def test_handle_embeddings__checks_data_file_types(self): + file_name = self.custom_app_config( + embedding_names=["name1", "name2"], + enable_reembedding="true", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", + anndata_backed="true", + config_file_name=self.config_file_name, + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_embeddings() + + def test_handle_diffexp__raises_warning_for_large_datasets(self): + config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_diffexp(self.context) + self.assertEqual(len(self.context["messages"]), 0) + + def test_multi_dataset(self): + config = AppConfig() + # test for illegal url_dataroots + for illegal in ("../b", "!$*", "\\n", "", "(bad)"): + config.update_server_config( + app__flask_secret_key="secret", + multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}, + ) + with self.assertRaises(ConfigurationError): + config.complete_config() + + # test for legal url_dataroots + for legal in ("d", "this.is-okay_", "a/b"): + config.update_server_config( + app__flask_secret_key="secret", + multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}, + ) + config.complete_config() + + # test that multi dataroots work end to end + config.update_server_config( + app__flask_secret_key="secret", + multi_dataset__dataroot=dict( + s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), + s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), + s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), + ), + ) + + # Change this default to test if the dataroot overrides below work. + config.update_default_dataset_config(app__about_legal_tos="tos_default.html") + + # specialize the configs for set1 + config.add_dataroot_config( + "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html" + ) + + # specialize the configs for set2 + config.add_dataroot_config( + "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html" + ) + + # no specializations for set3 (they get the default dataset config) + config.complete_config() + + with test_server(app_config=config) as server: + session = requests.Session() + + response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is False + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html" + + response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html" + + response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html" + + response = session.get(f"{server}/health") + assert response.json()["status"] == "pass" + + def test_configfile_with_specialization(self): + # test that per_dataset_config config load the default config, then the specialized config + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + server: + multi_dataset: + dataroot: + test: + base_url: test + dataroot: fake_dataroot + + dataset: + user_annotations: + enable: false + type: hosted_tiledb_array + hosted_tiledb_array: + db_uri: fake_db_uri + hosted_file_directory: fake_dir + + per_dataset_config: + test: + user_annotations: + enable: true + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + + test_config = app_config.dataroot_config["test"] + + # test config from default + self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array") + self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri") + + # test config from specialization + self.assertTrue(test_config.user_annotations__enable) diff --git a/server/test/unit/common/config/test_external_config.py b/server/test/unit/common/config/test_external_config.py new file mode 100644 index 00000000..5e3825e6 --- /dev/null +++ b/server/test/unit/common/config/test_external_config.py @@ -0,0 +1,231 @@ +import os +from unittest.mock import patch + +import requests + +from server.common.errors import ConfigurationError +from server.common.config.app_config import AppConfig +from server.test import test_server, FIXTURES_ROOT +from server.common.utils.type_conversion_utils import convert_string_to_value +from server.test.unit.common.config import ConfigTests + + +class TestExternalConfig(ConfigTests): + def test_type_convert(self): + # The values from environment variables and aws secrets are returned as strings. + # These values need to be converted to the proper types. + + self.assertEqual(convert_string_to_value("1"), int(1)) + self.assertEqual(convert_string_to_value("1.1"), float(1.1)) + self.assertEqual(convert_string_to_value("string"), "string") + self.assertEqual(convert_string_to_value("true"), True) + self.assertEqual(convert_string_to_value("True"), True) + self.assertEqual(convert_string_to_value("false"), False) + self.assertEqual(convert_string_to_value("False"), False) + self.assertEqual(convert_string_to_value("null"), None) + self.assertEqual(convert_string_to_value("None"), None) + self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string")) + + def test_environment_variable(self): + configfile = self.custom_external_config( + environment=[ + dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True), + dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True), + ], + config_file_name="environment_external_config.yaml", + ) + + env = os.environ + env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k.cxg" + env["DIFFEXP"] = "False" + with test_server(command_line_args=["-c", configfile], env=env) as server: + session = requests.Session() + response = session.get(f"{server}/api/v0.2/config") + data_config = response.json() + self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k") + self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"]) + + env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad" + env["DIFFEXP"] = "True" + with test_server(command_line_args=["-c", configfile], env=env) as server: + session = requests.Session() + response = session.get(f"{server}/api/v0.2/config") + data_config = response.json() + self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b") + self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"]) + + def test_environment_variable_errors(self): + + # no name + app_config = AppConfig() + app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "environment: 'name' is missing") + + # required has wrong type + app_config = AppConfig() + app_config.external_config.environment = [ + dict(name="myenvar", required="optional", path=["this", "is", "a", "path"]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool") + + # no path + app_config = AppConfig() + app_config.external_config.environment = [dict(name="myenvar", required=True)] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "environment: 'path' is missing") + + # required environment variable is not set + app_config = AppConfig() + app_config.external_config.environment = [ + dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set") + + @patch("server.common.config.external_config.get_secret_key") + def test_aws_secrets_manager(self, mock_get_secret_key): + mock_get_secret_key.return_value = { + "oauth_client_secret": "mock_oauth_secret", + "db_uri": "mock_db_uri", + } + configfile = self.custom_external_config( + aws_secrets_manager_region="us-west-2", + aws_secrets_manager_secrets=[ + dict( + name="my_secret", + values=[ + dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=False), + dict( + key="db_uri", + path=["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"], + required=True, + ), + dict( + key="oauth_client_secret", + path=["server", "authentication", "params_oauth", "client_secret"], + required=True, + ), + ], + ) + ], + config_file_name="secret_external_config.yaml", + ) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg" + app_config.server_config.app__flask_secret_key = "original" + app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg" + + app_config.complete_config() + + self.assertEqual(app_config.server_config.app__flask_secret_key, "original") + self.assertEqual(app_config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret") + self.assertEqual(app_config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri") + + @patch("server.common.config.external_config.get_secret_key") + def test_aws_secrets_manager_error(self, mock_get_secret_key): + mock_get_secret_key.return_value = { + "oauth_client_secret": "mock_oauth_secret", + "db_uri": "mock_db_uri", + } + + # no region + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = None + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual( + config_error.exception.message, + "Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType", + ) + + # missing secret name + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing") + + # secret name wrong type + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string") + + # missing values name + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing") + + # values wrong type + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"])) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list") + + # entry missing key + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret") + + # entry required is wrong type + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret") + + # entry missing path + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict(name="mysecret", values=[dict(key="db_uri", required=True)]) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret") + + # secret missing required key + app_config = AppConfig() + app_config.external_config.aws_secrets_manager__region = "us-west-2" + app_config.external_config.aws_secrets_manager__secrets = [ + dict( + name="mysecret", + values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])], + ) + ] + with self.assertRaises(ConfigurationError) as config_error: + app_config.complete_config() + self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set") diff --git a/server/test/unit/common/config/test_server_config.py b/server/test/unit/common/config/test_server_config.py new file mode 100644 index 00000000..e7598b7f --- /dev/null +++ b/server/test/unit/common/config/test_server_config.py @@ -0,0 +1,306 @@ +import os +import unittest +from unittest import mock +from unittest.mock import patch + +from server.common.config.base_config import BaseConfig +from server.common.utils.utils import find_available_port +from server.test import PROJECT_ROOT, FIXTURES_ROOT + +import requests + +from server.common.config.app_config import AppConfig +from server.common.errors import ConfigurationError +from server.test import test_server +from server.test.unit.common.config import ConfigTests + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class TestServerConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_raises_error_if_default_config_is_invalid(self): + invalid_config = self.get_config(port="not_valid") + with self.assertRaises(ConfigurationError): + invalid_config.complete_config() + + @patch("server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.server_config.complete_config(self.context) + self.assertEqual(mock_check_attrs.call_count, 40) + + def test_handle_app__throws_error_if_port_doesnt_exist(self): + config = self.get_config(port=99999999) + with self.assertRaises(ConfigurationError): + config.server_config.handle_app(self.context) + + @patch("server.common.config.server_config.discover_s3_region_name") + def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name): + mock_discover_region_name.return_value = None + # Default config + self.assertEqual(self.config.server_config.data_locator__s3__region_name, None) + # hard coded + config = self.get_config() + self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") + # incorrectly formatted + dataroot = { + "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"}, + "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"}, + } + file_name = self.custom_app_config( + dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true" + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_locator() + + @patch("server.common.config.server_config.discover_s3_region_name") + def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name): + mock_discover_region_name.return_value = "us-west-2" + dataroot = { + "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"}, + "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"}, + } + file_name = self.custom_app_config( + dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true" + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.handle_data_locator() + self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2") + mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev") + + def test_handle_app___can_use_envar_port(self): + config = self.get_config(port=24) + self.assertEqual(config.server_config.app__port, 24) + + # Note if the port is set in the config file it will NOT be overwritten by a different envvar + os.environ["CXG_SERVER_PORT"] = "4008" + self.config = AppConfig() + self.config.update_server_config(app__flask_secret_key="secret") + self.config.server_config.handle_app(self.context) + self.assertEqual(self.config.server_config.app__port, 4008) + del os.environ["CXG_SERVER_PORT"] + + def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self): + config = self.get_config(flask_secret_key="KEY_FROM_FILE") + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE") + + os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV" + config.external_config.handle_environment(self.context) + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV") + + def test_handle_app__sets_web_base_url(self): + config = self.get_config(web_base_url="anything.com") + self.assertEqual(config.server_config.app__web_base_url, "anything.com") + + def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self): + config = self.get_config(client_secret="KEY_FROM_FILE") + config.server_config.handle_authentication() + self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE") + + os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV" + config.external_config.handle_environment(self.context) + + self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV") + + def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", + config_file_name="two_data_roots.yml", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_source() + + file_name = self.custom_app_config(config_file_name="zero_roots.yml") + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_source() + + def test_get_api_base_url_works(self): + + # test the api_base_url feature, and that it can contain a path + config = AppConfig() + backend_port = find_available_port("localhost", 10000) + config.update_server_config( + app__flask_secret_key="secret", + app__api_base_url=f"http://localhost:{backend_port}/additional/path", + multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset", + ) + + config.complete_config() + + with test_server(["-p", str(backend_port)], app_config=config) as server: + session = requests.Session() + self.assertEqual(server, f"http://localhost:{backend_port}") + response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config") + self.assertEqual(response.status_code, 200) + data_config = response.json() + self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k") + + # test the health check at the correct url + response = session.get(f"{server}/additional/path/health") + assert response.json()["status"] == "pass" + + def test_get_web_base_url_works(self): + config = self.get_config(web_base_url="www.thisisawebsite.com") + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "www.thisisawebsite.com") + + config = self.get_config(web_base_url="local", port=12) + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "http://localhost:12") + + config = self.get_config(web_base_url="www.thisisawebsite.com/") + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "www.thisisawebsite.com") + + config = self.get_config(api_base_url="www.api_base.com/") + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "www.api_base.com") + + def test_config_for_single_dataset(self): + file_name = self.custom_app_config( + config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg" + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.handle_single_dataset(self.context) + self.assertIsNotNone(config.server_config.matrix_data_cache_manager) + + file_name = self.custom_app_config( + config_file_name="single_dataset_with_about.yml", + about="www.cziscience.com", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg", + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_single_dataset(self.context) + + def test_multi_dataset_raises_error_for_illegal_routes(self): + # test for illegal url_dataroots + for illegal in ("../b", "!$*", "\\n", "", "(bad)"): + self.config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}} + ) + with self.assertRaises(ConfigurationError): + self.config.complete_config() + + def test_multidataset_works_for_legal_routes(self): + # test for legal url_dataroots + for legal in ("d", "this.is-okay_", "a/b"): + self.config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}} + ) + self.config.complete_config() + + def test_mulitdatasets_work_e2e(self): + # test that multi dataroots work end to end + self.config.update_server_config( + multi_dataset__dataroot=dict( + s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), + s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), + s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), + ) + ) + + # Change this default to test if the dataroot overrides below work. + self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html") + + # specialize the configs for set1 + self.config.add_dataroot_config( + "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html" + ) + + # specialize the configs for set2 + self.config.add_dataroot_config( + "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html" + ) + + # no specializations for set3 (they get the default dataset config) + self.config.complete_config() + + with test_server(app_config=self.config) as server: + session = requests.Session() + + response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is False + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html" + + response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html" + + response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html" + + response = session.get(f"{server}/health") + assert response.json()["status"] == "pass" + + @patch("server.common.config.server_config.diffexp_tiledb.set_config") + def test_handle_diffexp(self, mock_tiledb_config): + custom_config_file = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", + cpu_multiplier=3, + diffexp_max_workers=1, + target_workunit=4, + config_file_name=self.config_file_name, + ) + config = AppConfig() + config.update_from_config_file(custom_config_file) + config.server_config.handle_diffexp() + # called with the min of diffexp_max_workers and cpus*cpu_multiplier + mock_tiledb_config.assert_called_once_with(1, 4) + + @patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context") + def test_handle_adaptor(self, mock_tiledb_context): + custom_config = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2 + ) + config = AppConfig() + config.update_from_config_file(custom_config) + config.server_config.handle_adaptor() + mock_tiledb_context.assert_called_once_with( + {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"} + ) diff --git a/server/test/unit/common/test_api.py b/server/test/unit/common/test_api.py index ee979b1f..54a3dc82 100644 --- a/server/test/unit/common/test_api.py +++ b/server/test/unit/common/test_api.py @@ -1,6 +1,7 @@ import shutil import time import unittest +import zlib from http import HTTPStatus import pandas as pd @@ -8,8 +9,14 @@ import requests import server.test.unit.decode_fbs as decode_fbs from server.data_common.matrix_loader import MatrixDataType -from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server, - stop_test_server) +from server.test import ( + data_with_tmp_annotations, + make_fbs, + PROJECT_ROOT, + FIXTURES_ROOT, + start_test_server, + stop_test_server, +) from server.test.fixtures.fixtures import pbmc3k_colors BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} @@ -43,7 +50,6 @@ class EndPoints(object): result_data = result.json() self.assertIn("library_versions", result_data["config"]) self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") - self.assertEqual(len(result_data["config"]["features"]), 5) def test_get_layout_fbs(self): endpoint = "layout/obs" @@ -338,7 +344,7 @@ class EndPointsAnnotations(EndPoints): url = f"{self.URL_BASE}{endpoint}?{query}" n_rows = self.data.get_shape()[0] fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")}) - result = self.session.put(url, data=fbs) + result = self.session.put(url, data=zlib.compress(fbs)) self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/json") self.assertEqual(result.json(), {"status": "OK"}) @@ -381,11 +387,14 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): @classmethod def setUpClass(cls): - cls._setupClass(cls, [ - f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", - "--disable-annotations", - "--experimental-enable-reembedding", - ]) + cls._setupClass( + cls, + [ + f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + "--disable-annotations", + "--experimental-enable-reembedding", + ], + ) @classmethod def tearDownClass(cls): @@ -403,10 +412,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints): @classmethod def setUpClass(cls): - cls._setupClass(cls, [ - f"{FIXTURES_ROOT}/pbmc3k.cxg", - "--disable-annotations", - ]) + cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"]) @classmethod def tearDownClass(cls): @@ -423,7 +429,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations): cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations( MatrixDataType.H5AD, annotations_fixture=True ) - cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ]) + cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()]) @classmethod def tearDownClass(cls): @@ -439,11 +445,7 @@ class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations): @classmethod def setUpClass(cls): cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True) - cls._setupClass(cls, [ - "--annotations-file", - cls.annotations.output_file, - cls.data.get_location(), - ]) + cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()]) @classmethod def tearDownClass(cls): diff --git a/server/test/unit/common/test_app_config.py b/server/test/unit/common/test_app_config.py deleted file mode 100644 index f8a20093..00000000 --- a/server/test/unit/common/test_app_config.py +++ /dev/null @@ -1,198 +0,0 @@ -import os -import unittest -from unittest import mock -from unittest.mock import patch -import tempfile - -import requests - -from server.common.app_config import AppConfig -from server.common.errors import ConfigurationError -from server.common.utils.utils import find_available_port -from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT - - -# NOTE, there are more tests that should be written for AppConfig. -# this is just a start. - -def mockenv(**envvars): - return mock.patch.dict(os.environ, envvars) - - -class AppConfigTest(unittest.TestCase): - def test_update(self): - config = AppConfig() - config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir") - vars = config.server_config.changes_from_default() - self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)]) - - config = AppConfig() - config.update_default_dataset_config(app__scripts=(), app__inline_scripts=()) - vars = config.server_config.changes_from_default() - self.assertCountEqual(vars, []) - - config = AppConfig() - config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[]) - vars = config.default_dataset_config.changes_from_default() - self.assertCountEqual(vars, []) - - config = AppConfig() - config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) - vars = config.default_dataset_config.changes_from_default() - self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) - - def test_multi_dataset(self): - - config = AppConfig() - # test for illegal url_dataroots - for illegal in ("../b", "!$*", "\\n", "", "(bad)"): - config.update_server_config( - multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} - ) - with self.assertRaises(ConfigurationError): - config.complete_config() - - # test for legal url_dataroots - for legal in ("d", "this.is-okay_", "a/b"): - config.update_server_config( - multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} - ) - config.complete_config() - - # test that multi dataroots work end to end - config.update_server_config( - multi_dataset__dataroot=dict( - s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), - s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), - s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), - ) - ) - - # Change this default to test if the dataroot overrides below work. - config.update_default_dataset_config(app__about_legal_tos="tos_default.html") - - # specialize the configs for set1 - config.add_dataroot_config( - "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html" - ) - - # specialize the configs for set2 - config.add_dataroot_config( - "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html" - ) - - # no specializations for set3 (they get the default dataset config) - config.complete_config() - - with test_server(app_config=config) as server: - session = requests.Session() - - response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") - data_config = response.json() - assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" - assert data_config["config"]["parameters"]["annotations"] is False - assert data_config["config"]["parameters"]["disable-diffexp"] is False - assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html" - - response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") - data_config = response.json() - assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" - assert data_config["config"]["parameters"]["annotations"] is True - assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html" - - response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") - data_config = response.json() - assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" - assert data_config["config"]["parameters"]["annotations"] is True - assert data_config["config"]["parameters"]["disable-diffexp"] is False - assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html" - - response = session.get(f"{server}/health") - assert response.json()["status"] == "pass" - - @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION") - @patch('server.common.aws_secret_utils.get_secret_key') - def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key): - mock_get_secret_key.return_value = { - "flask_secret_key": "mock_flask_secret", - "oauth_client_secret": "mock_oauth_secret", - "db_uri": "mock_db_uri" - } - - config = AppConfig() - - with self.assertLogs(level="INFO") as logger: - from server.common.aws_secret_utils import handle_config_from_secret - # should not throw error - # "AttributeError: 'XConfig' object has no attribute 'x'" - handle_config_from_secret(config) - - # should log 3 lines (one for each var set from a secret) - self.assertEqual(len(logger.output), 3) - self.assertIn('INFO:root:set app__flask_secret_key from secret', logger.output[0]) - self.assertIn('INFO:root:set authentication__params_oauth__client_secret from secret', logger.output[1]) - self.assertIn('INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret', logger.output[2]) - self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret") - self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret") - self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri") - - def test_api_base_url(self): - - # test the api_base_url feature, and that it can contain a path - config = AppConfig() - backend_port = find_available_port("localhost", 10000) - config.update_server_config( - app__api_base_url=f"http://localhost:{backend_port}/additional/path/before/dataroot", - multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset" - ) - - config.complete_config() - - with test_server(["-p", str(backend_port)], app_config=config) as server: - session = requests.Session() - self.assertEqual(server, f"http://localhost:{backend_port}") - response = session.get(f"{server}/additional/path/before/dataroot/d/pbmc3k.h5ad/api/v0.2/config") - self.assertEqual(response.status_code, 200) - data_config = response.json() - self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k") - - def test_configfile_with_specialization(self): - # test that per_dataset_config config load the default config, then the specialized config - - with tempfile.TemporaryDirectory() as tempdir: - configfile = os.path.join(tempdir, "config.yaml") - with open(configfile, "w") as fconfig: - config = """ - server: - multi_dataset: - dataroot: - test: - base_url: test - dataroot: fake_dataroot - - dataset: - user_annotations: - enable: false - type: hosted_tiledb_array - hosted_tiledb_array: - db_uri: fake_db_uri - hosted_file_directory: fake_dir - - per_dataset_config: - test: - user_annotations: - enable: true - """ - fconfig.write(config) - - app_config = AppConfig() - app_config.update_from_config_file(configfile) - - test_config = app_config.dataroot_config["test"] - - # test config from default - self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array") - self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri") - - # test config from specialization - self.assertTrue(test_config.user_annotations__enable) diff --git a/server/test/unit/common/test_corpora.py b/server/test/unit/common/test_corpora.py index 1a0222dc..5f26dae9 100644 --- a/server/test/unit/common/test_corpora.py +++ b/server/test/unit/common/test_corpora.py @@ -87,24 +87,17 @@ class CorporaRESTAPITest(unittest.TestCase): def setCorporaFields(cls, path): adata = anndata.read_h5ad(path) corpora_props = { - "version": { - "corpora_schema_version": "1.0.0", - "corpora_encoding_version": "0.1.0" - }, + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, "title": "PBMC3K", - "contributors": json.dumps([ - {"name": "name"} - ]), - "layer_descriptions": { - "X": "raw counts" - }, + "contributors": json.dumps([{"name": "name"}]), + "layer_descriptions": {"X": "raw counts"}, "organism": "human", "organism_ontology_term_id": "unknown", "project_name": "test project", "project_description": "test description", - "project_links": json.dumps([ - {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"} - ]), + "project_links": json.dumps( + [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}] + ), "default_embedding": "X_tsne", } adata.uns.update(corpora_props) diff --git a/server/test/unit/common/test_writable_annotation.py b/server/test/unit/common/test_writable_annotation.py index f650fecf..00cbd0f3 100644 --- a/server/test/unit/common/test_writable_annotation.py +++ b/server/test/unit/common/test_writable_annotation.py @@ -2,7 +2,7 @@ import json import shutil import unittest from os import path, listdir -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock import numpy as np import pandas as pd @@ -27,7 +27,7 @@ class auth(object): class WritableTileDBStoredAnnotationTest(unittest.TestCase): def setUp(self): - self.user_id = '1234' + self.user_id = "1234" self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD) self.data.dataset_config.user_annotations = self.annotations self.db = self.annotations.db @@ -38,7 +38,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): } self.fbs = make_fbs(self.test_dict) self.df = pd.DataFrame(self.test_dict) - self.app = Flask('fake_app') + self.app = Flask("fake_app") self.app.__setattr__("auth", auth) def tearDown(self): @@ -65,8 +65,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): self.annotations.write_labels(self.df, self.data) dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id annotation = self.db.query_for_most_recent( - Annotation, - [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)] + Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)] ) # retrieve tiledb array df = tiledb.open(annotation.tiledb_uri) @@ -78,7 +77,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self): with self.app.test_request_context(): - new_name = 'new_dataset/location' + new_name = "new_dataset/location" self.data.get_location = MagicMock(return_value=new_name) num_datasets = len(self.db.query([CellxGeneDataset])) self.annotation_put_fbs(self.fbs) @@ -130,19 +129,34 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): with self.assertRaises(KeyError): self.annotation_put_fbs(fbs_bad) - @patch('server.common.annotations.hosted_tiledb.current_app') - def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id): - mock_user_id.auth.get_user_id.return_value = '1234' - self.annotations.write_labels(self.df, self.data) - # get uri - dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id - annotation = self.db.query_for_most_recent( - Annotation, - [Annotation.user_id == '1234', Annotation.dataset_id == str(dataset_id)] - ) + def test_write_labels_stores_df_as_tiledb_array(self): + with self.app.test_request_context(): + self.annotations.write_labels(self.df, self.data) + # get uri + dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id + annotation = self.db.query_for_most_recent( + Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)] + ) - df = tiledb.open(annotation.tiledb_uri) - self.assertEqual(type(df), tiledb.array.SparseArray) + df = tiledb.open(annotation.tiledb_uri) + self.assertEqual(type(df), tiledb.array.SparseArray) + + def test_remove_categories(self): + with self.app.test_request_context(): + # update empty category data, which is how annotations are removed + empty = make_fbs({}) + self.annotation_put_fbs(empty) + + # verify that the tiledb uri is an empty string. + dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id + annotation = self.db.query_for_most_recent( + Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)] + ) + self.assertEqual(annotation.tiledb_uri, "") + + # verify that read_labels returns None + df = self.annotations.read_labels(self.data) + self.assertIsNone(df) class WritableAnnotationTest(unittest.TestCase): @@ -271,19 +285,31 @@ class WritableAnnotationTest(unittest.TestCase): {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True}, ) - def test_config(self): - features = self.data.get_features(self.annotations) + def test_put_float_data(self): + # verify that OBS PUTs (annotation_put_fbs) are accessible via + # GET (annotation_to_fbs_matrix) - # test each for singular presence and accuracy of available flag - def check_feature(method, path, available): - feature = list( - filter(lambda f: f.method == method and f.path == path and f.available == available, features) - ) - self.assertIsNotNone(feature) - self.assertEqual(len(feature), 1) + n_rows = self.data.get_shape()[0] - check_feature("POST", "/cluster/", False) - check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable) - check_feature("GET", "/layout/obs", True) - check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding) - check_feature("PUT", "/annotations/obs", True) + # verifies that floating point with decimals fail. + fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))}) + with self.assertRaises(ValueError) as exception_context: + res = self.annotation_put_fbs(fbs) + self.assertEqual(str(exception_context.exception), "Columns may not have floating point types") + + # verifies that floating point that can be converted to int passes + fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")}) + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + + # check read_labels + labels = self.annotations.read_labels(None) + fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels) + schema = schema_get_helper(self.data) + annotations = decode_fbs.decode_matrix_FBS(fbsAll) + self.assertEqual(annotations["n_rows"], n_rows) + all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} + self.assertEqual( + all_col_schema["cat_F_PASS"], + {"name": "cat_F_PASS", "type": "int32", "writable": True}, + ) diff --git a/server/test/unit/common/utils/test_cxg_generation_utils.py b/server/test/unit/common/utils/test_cxg_generation_utils.py index 57893913..b9043c33 100644 --- a/server/test/unit/common/utils/test_cxg_generation_utils.py +++ b/server/test/unit/common/utils/test_cxg_generation_utils.py @@ -8,8 +8,12 @@ import numpy as np import tiledb from pandas import Series, DataFrame -from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array, - convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array) +from server.common.utils.cxg_generation_utils import ( + convert_dictionary_to_cxg_group, + convert_dataframe_to_cxg_array, + convert_ndarray_to_cxg_dense_array, + convert_matrix_to_cxg_array, +) PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() @@ -28,8 +32,9 @@ class TestCxgGenerationUtils(unittest.TestCase): dictionary_name = "favorite_desserts" expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}" - convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary, - group_metadata_name=dictionary_name) + convert_dictionary_to_cxg_group( + self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name + ) array = tiledb.open(expected_array_directory) actual_stored_metadata = dict(array.meta.items()) @@ -44,13 +49,16 @@ class TestCxgGenerationUtils(unittest.TestCase): random_dataframe_name = f"random_dataframe_{uuid4()}" random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category}) - convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, - "int_category", tiledb.Ctx()) + convert_dataframe_to_cxg_array( + self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx() + ) expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}" expected_array_metadata = { - "cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, - "index": "int_category"})} + "cxg_schema": json.dumps( + {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"} + ) + } actual_stored_dataframe_array = tiledb.open(expected_array_directory) actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items()) @@ -95,7 +103,7 @@ class TestCxgGenerationUtils(unittest.TestCase): self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[:, :][''].size == 0) + self.assertTrue(actual_stored_array[:, :][""].size == 0) def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self): matrix = np.zeros([3, 3]) @@ -110,10 +118,10 @@ class TestCxgGenerationUtils(unittest.TestCase): self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[0, 0][''] == 1) - self.assertTrue(actual_stored_array[1, 1][''] == 1) - self.assertTrue(actual_stored_array[2, 2][''] == 2) - self.assertTrue(actual_stored_array[:, :][''].size == 3) + self.assertTrue(actual_stored_array[0, 0][""] == 1) + self.assertTrue(actual_stored_array[1, 1][""] == 1) + self.assertTrue(actual_stored_array[2, 2][""] == 2) + self.assertTrue(actual_stored_array[:, :][""].size == 3) def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self): matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}" @@ -122,14 +130,15 @@ class TestCxgGenerationUtils(unittest.TestCase): # a matrix of zeros which is sparse. column_shift = np.ones((3, 2)) - convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(), - column_shift_for_sparse_encoding=column_shift) + convert_matrix_to_cxg_array( + matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift + ) actual_stored_array = tiledb.open(matrix_name) self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[:, :][''].size == 0) + self.assertTrue(actual_stored_array[:, :][""].size == 0) def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self): matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}" @@ -137,13 +146,14 @@ class TestCxgGenerationUtils(unittest.TestCase): # Only column shift the first column of ones. column_shift = np.array([[1, 0], [1, 0]]) - convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(), - column_shift_for_sparse_encoding=column_shift) + convert_matrix_to_cxg_array( + matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift + ) actual_stored_array = tiledb.open(matrix_name) self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[0, 1][''] == 1) - self.assertTrue(actual_stored_array[1, 1][''] == 1) - self.assertTrue(actual_stored_array[:, :][''].size == 2) + self.assertTrue(actual_stored_array[0, 1][""] == 1) + self.assertTrue(actual_stored_array[1, 1][""] == 1) + self.assertTrue(actual_stored_array[:, :][""].size == 2) diff --git a/server/test/unit/common/utils/test_matrix_utils.py b/server/test/unit/common/utils/test_matrix_utils.py index ffda1045..9cc9daf5 100644 --- a/server/test/unit/common/utils/test_matrix_utils.py +++ b/server/test/unit/common/utils/test_matrix_utils.py @@ -6,7 +6,6 @@ from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_ class TestMatrixUtils(unittest.TestCase): - def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self): matrix = np.array([1, 2, 3]) diff --git a/server/test/unit/common/utils/test_sanitization_utils.py b/server/test/unit/common/utils/test_sanitization_utils.py index 8ef04218..e209be95 100644 --- a/server/test/unit/common/utils/test_sanitization_utils.py +++ b/server/test/unit/common/utils/test_sanitization_utils.py @@ -4,7 +4,6 @@ from server.common.utils.sanitization_utils import sanitize_values_in_list, sani class TestSanitizationUtils(unittest.TestCase): - def test__sanitize_values_in_list__not_strings_raises_exception(self): keys_to_sanitize = [1, 2, 3] diff --git a/server/test/unit/common/utils/test_type_conversion_utils.py b/server/test/unit/common/utils/test_type_conversion_utils.py index f1653697..ade2d999 100644 --- a/server/test/unit/common/utils/test_type_conversion_utils.py +++ b/server/test/unit/common/utils/test_type_conversion_utils.py @@ -5,12 +5,17 @@ from unittest.mock import patch import numpy as np from pandas import Series, DataFrame -from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \ - get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe, convert_pandas_series_to_numpy +from server.common.utils.type_conversion_utils import ( + can_cast_to_float32, + can_cast_to_int32, + get_dtype_of_array, + get_schema_type_hint_of_array, + get_dtypes_and_schemas_of_dataframe, + convert_pandas_series_to_numpy, +) class TestTypeConversionUtils(unittest.TestCase): - def test__can_cast_to_float32__string_is_false(self): array_to_convert = Series(data=["1", "2", "3"], dtype=str) @@ -97,8 +102,9 @@ class TestTypeConversionUtils(unittest.TestCase): expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode] for test_type_index in range(len(types)): - with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", - i=test_type_index): + with self.subTest( + f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) @@ -123,8 +129,9 @@ class TestTypeConversionUtils(unittest.TestCase): expected_dtypes = [np.float32, np.int32] for test_type_index in range(len(types)): - with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", - i=test_type_index): + with self.subTest( + f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) @@ -141,8 +148,9 @@ class TestTypeConversionUtils(unittest.TestCase): expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}] for test_type_index in range(len(types)): - with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", - i=test_type_index): + with self.subTest( + f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) @@ -160,8 +168,9 @@ class TestTypeConversionUtils(unittest.TestCase): for test_type_index in range(len(types)): with self.subTest( - f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", - i=test_type_index): + f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", + i=test_type_index, + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) @@ -171,8 +180,10 @@ class TestTypeConversionUtils(unittest.TestCase): dataframe = DataFrame({"float_array": float_array, "category_array": category_array}) expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode} - expected_schema_type_hints_dict = {"float_array": {"type": "float32"}, - "category_array": {"type": "categorical", "categories": ["a", "b"]}} + expected_schema_type_hints_dict = { + "float_array": {"type": "float32"}, + "category_array": {"type": "categorical", "categories": ["a", "b"]}, + } actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe) @@ -201,5 +212,6 @@ class TestTypeConversionUtils(unittest.TestCase): with self.assertLogs(level="ERROR") as logger: convert_pandas_series_to_numpy(int_series, np.int32) - self.assertIn("Cannot convert a pandas Series object to an integer dtype if it contains NaNs", - logger.output[0]) + self.assertIn( + "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0] + ) diff --git a/server/test/unit/converters/schema/__init__.py b/server/test/unit/converters/schema/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/unit/converters/schema/test_gene_symbol.py b/server/test/unit/converters/schema/test_gene_symbol.py new file mode 100644 index 00000000..dceca312 --- /dev/null +++ b/server/test/unit/converters/schema/test_gene_symbol.py @@ -0,0 +1,61 @@ +import os +import unittest + +import pandas as pd + +from server.test import FIXTURES_ROOT +from server.converters.schema import gene_symbol + + +class TestHGNCSymbolChecker(unittest.TestCase): + + def setUp(self): + self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz") + self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path) + + def test_symbol_upgrade(self): + self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2") + + # Strip off seurat endings when appropriate + self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2") + + # DIFF6 is ambiguous so don't upgrade it + self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6") + self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6") + + # ARG1 is approved + self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1") + + # HAP1 is both approved and withdrawn + self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1") + + # Leave unknown symbols alone + self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL") + self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol") + + # Upgrade HGNC ids unless you can't find it + self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2") + self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1") + self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456") + + def test_check_symbol(self): + self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE) + self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS) + self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN) + + # HAP1 is one of the approved and withdrawn symbols + self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED) + + def test_upgrade_index(self): + index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"]) + var_df = pd.DataFrame([[0] * len(index)], index=index) + upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path) + self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"]) diff --git a/server/test/unit/converters/schema/test_ontology.py b/server/test/unit/converters/schema/test_ontology.py new file mode 100644 index 00000000..f6504da6 --- /dev/null +++ b/server/test/unit/converters/schema/test_ontology.py @@ -0,0 +1,129 @@ +import json + +import unittest +import unittest.mock + +from server.converters.schema import ontology + + +class TestOntologyParsing(unittest.TestCase): + def setUp(self): + + self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"] + + self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"] + + self.values = ["0002048", "0000174", "9606", "0008995"] + + self.iris = [ + "http://purl.obolibrary.org/obo/UBERON_0002048", + "http://purl.obolibrary.org/obo/HsapDv_0000174", + "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "http://www.ebi.ac.uk/efo/EFO_0008995", + ] + + URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/" + self.urls = [ + URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048", + URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174", + URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606", + URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995", + ] + + self.responses = { + "UBERON:0002048": { + "iri": "http://purl.obolibrary.org/obo/UBERON_0002048", + "description": ["Respiration organ that develops as an outpocketing of the esophagus."], + "label": "lung", + }, + "HsapDv:0000174": { + "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174", + "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."], + "label": "1-month-old human stage", + }, + "NCBITaxon:9606": { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "description": None, + "label": "Homo sapiens", + }, + "EFO:0008995": { + "iri": "http://www.ebi.ac.uk/efo/EFO_0008995", + "description": [ + ( + '10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated ' + "gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent " + "of 1 million pipetting steps. Successive versions of the 10x chemistry use different " + "barcode locations to improve the sequencing yield and quality of 10x experiments." + ) + ], + "label": "10X sequencing", + }, + } + + def test_ontololgy_name(self): + for curie, expected_name in zip(self.curies, self.names): + self.assertEqual(ontology._ontology_name(curie), expected_name) + + def test_ontololgy_value(self): + for curie, expected_value in zip(self.curies, self.values): + self.assertEqual(ontology._ontology_value(curie), expected_value) + + def test_iri(self): + for curie, expected_iri in zip(self.curies, self.iris): + self.assertEqual(ontology._iri(curie), expected_iri) + + def test_ontology_info_url(self): + for curie, expected_url in zip(self.curies, self.urls): + self.assertEqual(ontology._ontology_info_url(curie), expected_url) + + def test_empty_ontology_info_url(self): + self.assertEqual(ontology._ontology_info_url(""), "") + + +class TestOntologyLookup(unittest.TestCase): + def setUp(self): + self.responses = { + "UBERON:0002048": { + "iri": "http://purl.obolibrary.org/obo/UBERON_0002048", + "description": ["Respiration organ that develops as an outpocketing of the esophagus."], + "label": "lung", + }, + "HsapDv:0000174": { + "iri": "http://purl.obolibrary.org/obo/HsapDv_0000174", + "description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."], + "label": "1-month-old human stage", + }, + "NCBITaxon:9606": { + "iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606", + "description": None, + "label": "Homo sapiens", + }, + "EFO:0008995": { + "iri": "http://www.ebi.ac.uk/efo/EFO_0008995", + "description": [ + ('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated ' + 'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent ' + 'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode ' + 'locations to improve the sequencing yield and quality of 10x experiments.') + ], + "label": "10X sequencing", + }, + } + + self.labels = { + "UBERON:0002048": "lung", + "HsapDv:0000174": "1-month-old human stage", + "NCBITaxon:9606": "Homo sapiens", + "EFO:0008995": "10X sequencing", + } + + @unittest.mock.patch("requests.get") + def test_lookup_label(self, mock_get): + + for curie, response in self.responses.items(): + mock_get.return_value.content = json.dumps(response) + mock_get.return_value.json.return_value = response + mock_get.return_value.status_code = 200 + + label = ontology.get_ontology_label(curie) + self.assertEqual(label, self.labels[curie]) diff --git a/server/test/unit/converters/schema/test_remix.py b/server/test/unit/converters/schema/test_remix.py new file mode 100644 index 00000000..3e99f879 --- /dev/null +++ b/server/test/unit/converters/schema/test_remix.py @@ -0,0 +1,257 @@ +import json +import os +import unittest +import unittest.mock + +import anndata +import numpy +import pandas as pd +import scanpy as sc + +from server.converters.schema import remix + +PROJECT_ROOT = os.popen("git rev-parse --show-toplevel").read().strip() + + +class TestApplySchema(unittest.TestCase): + + def setUp(self): + self.source_h5ad_path = f"{PROJECT_ROOT}/server/test/fixtures/pbmc3k-CSC-gz.h5ad" + self.output_h5ad_path = f"{PROJECT_ROOT}/server/test/fixtures/test_remix.h5ad" + self.config_path = f"{PROJECT_ROOT}/server/test/fixtures/test_config.yaml" + self.bad_config_path = f"{PROJECT_ROOT}/server/test/fixtures/test_bad_config.yaml" + + def tearDown(self): + try: + os.remove(self.output_h5ad_path) + except OSError: + pass + + @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label") + def test_apply_schema(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "test label" + remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path) + new_adata = sc.read_h5ad(self.output_h5ad_path) + + self.assertIn("cell_type", new_adata.obs.columns) + self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist()) + self.assertListEqual( + ["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"], + sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist()) + ) + + self.assertIn("version", new_adata.uns_keys()) + + @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label") + def test_apply_bad_schema(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "test label" + remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path) + new_adata = sc.read_h5ad(self.output_h5ad_path) + + # Should refuse to write the version + self.assertNotIn("version", new_adata.uns_keys()) + +class TestFieldParsing(unittest.TestCase): + + def test_is_curie(self): + self.assertTrue(remix.is_curie("EFO:00001")) + self.assertTrue(remix.is_curie("UBERON:123456")) + self.assertTrue(remix.is_curie("HsapDv:0001")) + self.assertFalse(remix.is_curie("UBERON")) + self.assertFalse(remix.is_curie("UBERON:")) + self.assertFalse(remix.is_curie("123456")) + + def test_is_ontology_field(self): + self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id")) + self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id")) + self.assertFalse(remix.is_ontology_field("cell_ontology")) + self.assertFalse(remix.is_ontology_field("method")) + + def test_get_label_field_name(self): + self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id")) + self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id")) + + def test_split_suffix(self): + self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)")) + self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)")) + self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234")) + self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)")) + + @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label") + def test_get_curie_and_label(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "test label" + self.assertEqual( + remix.get_curie_and_label("UBERON:1234"), + ("UBERON:1234", "test label") + ) + self.assertEqual( + remix.get_curie_and_label("UBERON:1234 (cell culture)"), + ("UBERON:1234 (cell culture)", "test label (cell culture)") + ) + self.assertEqual( + remix.get_curie_and_label("whatever"), + ("", "whatever") + ) + + +class TestManipulateAnndata(unittest.TestCase): + + def setUp(self): + + self.cell_count = 20 + self.gene_count = 200 + X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count)) + uns = {"organism": "monkey", "experiment": "monkey experiment"} + obs = pd.DataFrame( + index=[f"Cell{d}" for d in range(self.cell_count)], + columns=["tissue", "CellType"], + data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2) + ) + var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)]) + + self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns) + + def test_safe_add_field(self): + + remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count) + self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count) + self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count) + + remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}]) + self.assertEqual( + self.adata.uns["contributors"], + json.dumps([{"name": "contributor1"}, {"name": "contributor2"}]) + ) + + @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label") + def test_remix_uns(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "Pan troglodytes" + uns_config = { + "version": { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + }, + "organism_ontology_term_id": "NCBITaxon:9598", + "contributors": [ + { + "name": "scientist", + "email": "scientist@science.com" + } + ] + } + + remix.remix_uns(self.adata, uns_config) + + self.assertEqual( + sorted(self.adata.uns_keys()), + sorted(["organism_original", "organism", "organism_ontology_term_id", + "contributors", "version", "experiment"]) + ) + + self.assertEqual(self.adata.uns['organism'], "Pan troglodytes") + self.assertEqual(self.adata.uns['organism_original'], "monkey") + self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598") + self.assertEqual(self.adata.uns['contributors'], + json.dumps([{"name": "scientist", "email": "scientist@science.com"}])) + + @unittest.mock.patch("server.converters.schema.ontology.get_ontology_label") + def test_remix_obs(self, mock_get_ontology_label): + mock_get_ontology_label.return_value = "lung (in a monkey)" + obs_config = { + "tissue_ontology_term_id": { + "tissue": { + "lung": "UBERON:00000" + } + }, + "cell_color": { + "CellType": { + "epithelial": "fuschia", + "endothelial": "khaki" + } + }, + "sex": "male" + } + + remix.remix_obs(self.adata, obs_config) + self.assertEqual( + sorted(self.adata.obs_keys()), + sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"]) + ) + + self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original)) + self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id)) + self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue)) + self.assertTrue(all(v == "male" for v in self.adata.obs.sex)) + self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki")) + for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color))) + + +class TestFixupGeneSymbols(unittest.TestCase): + + def setUp(self): + self.seurat_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/seurat_tutorial.h5ad" + self.seurat_merged_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad" + self.sctransform_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/sctransform.h5ad" + self.sctransform_merged_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/sctransform_merged.h5ad" + + # There's lots of MALAT1, but it doesn't collide with any other names, + # so it shouldn't change during merging. + self.stable_gene = "MALAT1" + + def test_fixup_gene_symbols_seurat(self): + + if not os.path.isfile(self.seurat_path): + return unittest.skip( + "Skipping gene symbol conversion tests because test h5ads are not present. To create them, " + "run server/test/fixtures/schema_test_data/generate_test_data.sh" + ) + + original_adata = sc.read_h5ad(self.seurat_path) + merged_adata = sc.read_h5ad(self.seurat_merged_path) + + fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"} + + fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config) + + self.assertEqual( + merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum() + ) + self.assertAlmostEqual( + merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum() + ) + + self.assertAlmostEqual( + merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum() + ) + + def test_fixup_gene_symbols_sctransform(self): + + if not os.path.isfile(self.sctransform_path): + return unittest.skip( + "Skipping gene symbol conversion tests because test h5ads are not present. To create them, " + "run server/test/fixtures/schema_test_data/generate_test_data.sh" + ) + + original_adata = sc.read_h5ad(self.sctransform_path) + merged_adata = sc.read_h5ad(self.sctransform_merged_path) + + fixup_config = {"X": "log1p", "counts": "raw"} + + fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config) + + # sctransform does a bunch of stuff, including slightly modifying the + # raw counts. So we can't assert for exact equality the way we do with + # the vanilla seurat tutorial. But, the results should still be very + # close. + merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum() + fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum() + self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable) + + self.assertAlmostEqual( + merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(), + fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(), + 0 + ) diff --git a/server/test/unit/converters/schema/test_validate.py b/server/test/unit/converters/schema/test_validate.py new file mode 100644 index 00000000..fba885a6 --- /dev/null +++ b/server/test/unit/converters/schema/test_validate.py @@ -0,0 +1,435 @@ +import json +import os +import unittest + +import pandas as pd +import scanpy as sc + +from server.converters.schema import validate + +PROJECT_ROOT = os.popen("git rev-parse --show-toplevel").read().strip() + + +class TestFieldValidation(unittest.TestCase): + + def test_validate_stringified_list_of_dicts(self): + + good = json.dumps([{"a": 1}, {2: "x", "z": "y"}]) + not_stringified = [{"a": 1}, {2: "x", "z": "y"}] + not_a_list = json.dumps({"bad": "dict"}) + not_json = "oh hey!" + + self.assertTrue(validate._validate_stringified_list_of_dicts(good)) + + self.assertFalse(validate._validate_stringified_list_of_dicts(not_stringified)) + self.assertFalse(validate._validate_stringified_list_of_dicts(not_a_list)) + self.assertFalse(validate._validate_stringified_list_of_dicts(not_json)) + + def test_validate_human_readable_string(self): + + good = "oh hey!" + curie = "EFO:0001" + ensg = "ENSG000001234" + enst = "ENST000005678" + + self.assertTrue(validate._validate_human_readable_string(good)) + + self.assertFalse(validate._validate_human_readable_string(curie)) + self.assertFalse(validate._validate_human_readable_string(ensg)) + self.assertFalse(validate._validate_human_readable_string(enst)) + + def test_validate_curie(self): + + self.assertTrue(validate._validate_curie("UBERON:00001", ["UBERON", "EFO"])) + self.assertTrue(validate._validate_curie("HsapDv:00002", ["HsapDv"])) + + self.assertFalse(validate._validate_curie("HsapDv:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("EFO:00002 (organoid)", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("EFO:00002 extra", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("UBERON:ABCD", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("Uberon:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("UBERON:", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_curie("UBERON", ["UBERON", "EFO"])) + + def test_validate_suffixed_curie(self): + + self.assertTrue(validate._validate_suffixed_curie("EFO:00001", ["UBERON", "EFO"])) + self.assertTrue(validate._validate_suffixed_curie("UBERON:00001 (cell culture)", ["UBERON", "EFO"])) + + self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002 (organoid)", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002(organoid)", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("EFO:00002 extra", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("UBERON:ABCD", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("Uberon:00002", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("UBERON:", ["UBERON", "EFO"])) + self.assertFalse(validate._validate_suffixed_curie("UBERON", ["UBERON", "EFO"])) + + +class TestColumnValidation(unittest.TestCase): + + def test_validate_unique(self): + unique = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]], + index=["X", "Y", "Z"], columns=["col1", "col2"]) + duped = pd.DataFrame([["abc", "def"], ["ghi", "qrs"], ["abc", "qrs"]], + index=["X", "Y", "X"], columns=["col1", "col2"]) + + schema_def = {"unique": True} + + errors = validate._validate_column(unique.index, "index", "unique_df", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(duped.index, "index", "duped_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("is not unique", errors[0]) + + errors = validate._validate_column(unique["col1"], "col1", "unique_df", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("is not unique", errors[0]) + + schema_def = {"unique": False} + errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def) + self.assertFalse(errors) + + def test_validate_nullable(self): + non_null = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]], + index=["X", "Y", "Z"], columns=["col1", "col2"]) + has_null = pd.DataFrame([["abc", "", None], ["ghi", "jkl", 1], ["mnop", "qrs", 2]], + index=["X", "Y", "Z"], columns=["col1", "col2", "col3"]) + + schema_def = {"nullable": False} + errors = validate._validate_column(non_null["col1"], "col1", "nonnull_df", schema_def) + self.assertFalse(errors) + errors = validate._validate_column(has_null["col1"], "col1", "hasnull_df", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("contains empty values", errors[0]) + errors = validate._validate_column(has_null["col3"], "col3", "hasnull_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("contains empty values", errors[0]) + + schema_def = {"nullable": True} + errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def) + self.assertFalse(errors) + + def test_human_readable(self): + hr_df = pd.DataFrame( + [["for you, a human", "UBERON:12345", "UBERON:1234 (thundercat)"], + ["hope you're well", "bit of lungs", "brain"]], + index=["ENSG00001", "ENSG00002"], + columns=["good", "curie", "suffixed_curie"]) + + schema_def = {"type": "human-readable string"} + errors = validate._validate_column(hr_df["good"], "good", "hr", schema_def) + self.assertFalse(errors) + + errors = validate._validate_column(hr_df["curie"], "curie", "hr", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + errors = validate._validate_column(hr_df["suffixed_curie"], "suffixed_curie", "hr", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + errors = validate._validate_column(hr_df.index, "ensg", "hr", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + def test_curie(self): + + curie_df = pd.DataFrame( + [["EFO:00001", "HsapDv:00001 (cell culture)", "EFO:", "MONDO:0001 cell culture"], + ["UBERON:00002", "HsapDv:00002 (organoid)", "EFO:12345", "MONDO:0002 (baba yaga)"], + ["EFO:0000000005", "HsapDv:000004 (humanzee)", "EFO:000002", "MONDO:0004 (TMNT)"]], + index=["X", "Y", "Z"], + columns=["good", "good_suffix", "bad", "bad_suffix"]) + + # Good + schema_def = {"type": "curie", "prefixes": ["EFO", "UBERON"]} + errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def) + self.assertFalse(errors) + + # Good suffix + schema_def = {"type": "suffixed curie", "prefixes": ["HsapDv", "WHATEVER"]} + errors = validate._validate_column(curie_df["good_suffix"], "good_suffix", "curie_df", schema_def) + self.assertFalse(errors) + + # Bad prefix + schema_def = {"type": "curie", "prefixes": ["EFO"]} + errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + self.assertIn("must be curies from one of these", errors[0]) + + # Bad curies + schema_def = {"type": "curie", "prefixes": ["EFO"]} + errors = validate._validate_column(curie_df["bad"], "bad", "curie_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + + # Bad suffixes + schema_def = {"type": "suffixed curie", "prefixes": ["EFO"]} + errors = validate._validate_column(curie_df["bad_suffix"], "bad_suffix", "curie_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + + def test_enum(self): + enum_df = pd.DataFrame( + [["abc", "ghi"], + ["def", "jkl"]], + index=["X", "Y"], + columns=["col1", "col2"]) + + # All match + schema_def = {"type": "string", "enum": ["abc", "def", "xyz"]} + errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def) + self.assertFalse(errors) + + # Missing value + schema_def = {"type": "string", "enum": ["abc", "xyz"]} + errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("unpermitted values", errors[0]) + + +class TestDictValidations(unittest.TestCase): + + + def test_key_presence(self): + + schema_def = {"keys": {"abc": None, "def": None}} + + dict_ = {"abc": "123", "def": "456"} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + # Missing keys are bad + dict_ = {"abc": "123"} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("missing key", errors[0]) + + # Extra keys are okay + dict_ = {"abc": "123", "def": "456", "xyz": "789"} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + # Better not be empty come on + dict_ = {} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 2) + + def test_nullable(self): + + schema_def = {"keys": {"abc": {"type": "string", "nullable": False}, + "def": {"type": "string", "nullable": True}}} + + dict_ = {"abc": "xyz", "def": ""} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + dict_ = {"abc": "", "def": ""} + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("empty value", errors[0]) + + def test_recurse(self): + + schema_def = { + "keys": { + "subdict": { + "type": "dict", + "keys": { + "subdict_key1": None, + "subdict_key2": None + } + }, + "ontology": { + "type": "curie", + "prefixes": ["ONTOLOGY"] + }, + "blob": { + "type": "stringified list of dicts" + } + } + } + + dict_ = { + "subdict": {"subdict_key1": "any", "subdict_key2": "any"}, + "ontology": "ONTOLOGY:123456", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertFalse(errors) + + dict_ = { + "subdict": {"subdict_key1": "any"}, + "ontology": "ONTOLOGY:123456", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("missing key", errors[0]) + + dict_ = { + "subdict": {"subdict_key1": "any", "subdict_key2": "any"}, + "ontology": "oh no not an ontology term", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("invalid ontology", errors[0]) + + dict_ = { + "subdict": {"subdict_key1": "any", "subdict_key2": "any"}, + "ontology": "ONTOLOGY:123456", + "blob": [{"abc": 123}, {"def": 456}] + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("JSON-encoded list of dicts", errors[0]) + + # Multiple errors + dict_ = { + "subdict": {"subdict_key1": "any"}, + "ontology": "oh no not an ontology term", + "blob": json.dumps([{"abc": 123}, {"def": 456}]) + } + errors = validate._validate_dict(dict_, "d", schema_def) + self.assertEqual(len(errors), 2) + + +class TestDataframeValidation(unittest.TestCase): + + def test_column_presence(self): + df = pd.DataFrame( + [["abc", "EFO:123"], + ["def", "UBERON:456"]], + columns=["hr_string", "ontology"], + index=["X", "Y"] + ) + + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertFalse(errors) + + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "another_hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("missing column", errors[0]) + + # Extra is okay + df = pd.DataFrame( + [["abc", "EFO:123", "extra"], + ["def", "UBERON:456", "extra"]], + columns=["hr_string", "ontology", "extra"], + index=["X", "Y"] + ) + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertFalse(errors) + + + def test_index(self): + df = pd.DataFrame( + [["abc", "123"], + ["def", "456"]], + columns=["col1", "col2"], + index=["ENSG0001", "ENSG0002"] + ) + + schema_def = {"index": {"unique": True}} + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertFalse(errors) + + schema_def = {"index": {"type": "human-readable string"}} + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("non-human-readable", errors[0]) + + df = pd.DataFrame( + [["abc", "123"], + ["def", "456"]], + columns=["col1", "col2"], + index=["ENSG0001", "ENSG0001"] + ) + schema_def = {"index": {"unique": True}} + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 1) + self.assertIn("is not unique", errors[0]) + + def test_recurse(self): + + df = pd.DataFrame( + [["abc", "HsapDv:0001"], + ["EFO:123", "UBERON:456"]], + columns=["hr_string", "ontology"], + index=["X", "Y"] + ) + schema_def = { + "columns": { + "hr_string": {"type": "human-readable string"}, + "ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]} + } + } + errors = validate._validate_dataframe(df, "df", schema_def) + self.assertEqual(len(errors), 2) + self.assertEqual(len([e for e in errors if "non-human-readable" in e]), 1) + self.assertEqual(len([e for e in errors if "invalid ontology" in e]), 1) + + +class TestGetSchema(unittest.TestCase): + + def test_get_schema(self): + self.assertIsInstance(validate.get_schema_definition("1.0.0"), dict) + + with self.assertRaises(ValueError): + validate.get_schema_definition("10.1.5") + + +class TestValidate(unittest.TestCase): + + def setUp(self): + self.source_h5ad_path = f"{PROJECT_ROOT}/server/test/fixtures/pbmc3k-CSC-gz.h5ad" + + def test_shallow(self): + + adata = sc.read_h5ad(self.source_h5ad_path) + self.assertFalse(validate.validate_adata(adata, True)) + + adata.uns["version"] = { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + } + self.assertTrue(validate.validate_adata(adata, True)) + + def test_deep(self): + adata = sc.read_h5ad(self.source_h5ad_path) + self.assertFalse(validate.validate_adata(adata, False)) + + adata.uns["version"] = { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + } + self.assertFalse(validate.validate_adata(adata, False)) diff --git a/server/test/unit/converters/test_h5ad_data_file.py b/server/test/unit/converters/test_h5ad_data_file.py index f8587ebd..99ad40af 100644 --- a/server/test/unit/converters/test_h5ad_data_file.py +++ b/server/test/unit/converters/test_h5ad_data_file.py @@ -16,7 +16,6 @@ PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() class TestH5ADDataFile(unittest.TestCase): - def setUp(self): self.sample_anndata = self._create_sample_anndata_dataset() self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata) @@ -40,8 +39,12 @@ class TestH5ADDataFile(unittest.TestCase): def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self): with self.assertLogs(level="WARN") as logger: - H5ADDataFile(self.sample_h5ad_filename, dataset_title="My Awesome Dataset", - dataset_about="http://www.awesomedataset.com", use_corpora_schema=False) + H5ADDataFile( + self.sample_h5ad_filename, + dataset_title="My Awesome Dataset", + dataset_about="http://www.awesomedataset.com", + use_corpora_schema=False, + ) self.assertIn("will override any metadata that is extracted", logger.output[0]) @@ -49,10 +52,12 @@ class TestH5ADDataFile(unittest.TestCase): h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False) self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all()) - self.assertEqual(h5ad_file.anndata.obs.sort_index(inplace=True), - self.sample_anndata.obs.sort_index(inplace=True)) - self.assertEqual(h5ad_file.anndata.var.sort_index(inplace=True), - self.sample_anndata.var.sort_index(inplace=True)) + self.assertEqual( + h5ad_file.anndata.obs.sort_index(inplace=True), self.sample_anndata.obs.sort_index(inplace=True) + ) + self.assertEqual( + h5ad_file.anndata.var.sort_index(inplace=True), self.sample_anndata.var.sort_index(inplace=True) + ) for key in h5ad_file.anndata.obsm.keys(): self.assertIn(key, self.sample_anndata.obsm.keys()) @@ -73,8 +78,12 @@ class TestH5ADDataFile(unittest.TestCase): self.assertIn("name_0", h5ad_file.var.columns) def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self): - h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False, - obs_index_column_name="float_category", vars_index_column_name="int_category") + h5ad_file = H5ADDataFile( + self.sample_h5ad_filename, + use_corpora_schema=False, + obs_index_column_name="float_category", + vars_index_column_name="int_category", + ) self.assertNotIn("name_0", h5ad_file.obs.columns) self.assertNotIn("name_0", h5ad_file.var.columns) @@ -82,15 +91,23 @@ class TestH5ADDataFile(unittest.TestCase): def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self): with self.assertRaises(Exception) as exception_context: - H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False, - obs_index_column_name="float_category", vars_index_column_name="bool_category") + H5ADDataFile( + self.sample_h5ad_filename, + use_corpora_schema=False, + obs_index_column_name="float_category", + vars_index_column_name="bool_category", + ) self.assertIn("Please prepare data to contain unique values", str(exception_context.exception)) def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self): with self.assertRaises(Exception) as exception_context: - H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False, - obs_index_column_name="unknown_category", vars_index_column_name="i_dont_exist") + H5ADDataFile( + self.sample_h5ad_filename, + use_corpora_schema=False, + obs_index_column_name="unknown_category", + vars_index_column_name="i_dont_exist", + ) self.assertIn("does not exist", str(exception_context.exception)) @@ -101,8 +118,9 @@ class TestH5ADDataFile(unittest.TestCase): self.assertEqual(h5ad_file.dataset_about, "www.link.com") def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self): - h5ad_file = H5ADDataFile(self.sample_h5ad_filename, dataset_about="override_about", - dataset_title="override_title") + h5ad_file = H5ADDataFile( + self.sample_h5ad_filename, dataset_about="override_about", dataset_title="override_title" + ) self.assertEqual(h5ad_file.dataset_title, "override_title") self.assertEqual(h5ad_file.dataset_about, "override_about") @@ -145,8 +163,11 @@ class TestH5ADDataFile(unittest.TestCase): remove(sparse_with_column_shift_filename) def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False): - expected_directories, expected_obs_files, expected_var_files = \ - self._get_expected_generated_list_of_tiledb_files() + ( + expected_directories, + expected_obs_files, + expected_var_files, + ) = self._get_expected_generated_list_of_tiledb_files() for directory in expected_directories: self.assertTrue(path.isdir(directory)) @@ -187,8 +208,18 @@ class TestH5ADDataFile(unittest.TestCase): var_files.append("bool_category.tdb") var_files.append("int_category.tdb") - return [metadata_directory, main_x_directory, overall_embedding_directory, specific_embedding_directory, - obs_directory, var_directory], obs_files, var_files + return ( + [ + metadata_directory, + main_x_directory, + overall_embedding_directory, + specific_embedding_directory, + obs_directory, + var_directory, + ], + obs_files, + var_files, + ) def _write_anndata_to_file(self, anndata): temporary_filename = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}.h5ad" @@ -204,7 +235,8 @@ class TestH5ADDataFile(unittest.TestCase): random_string_category = Series(data=["a", "b", "b"], dtype="category") random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32) obs_dataframe = DataFrame( - data={"string_category": random_string_category, "float_category": random_float_category}) + data={"string_category": random_string_category, "float_category": random_float_category} + ) obs = obs_dataframe # Create vars @@ -230,6 +262,7 @@ class TestH5ADDataFile(unittest.TestCase): # Set project links to be a dictionary uns["project_links"] = json.dumps( - [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}]) + [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}] + ) return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns) diff --git a/server/test/unit/data_anndata/test_anndata_adaptor.py b/server/test/unit/data_anndata/test_anndata_adaptor.py index d4a3a778..d0df5db1 100644 --- a/server/test/unit/data_anndata/test_anndata_adaptor.py +++ b/server/test/unit/data_anndata/test_anndata_adaptor.py @@ -94,23 +94,6 @@ class AdaptorTest(unittest.TestCase): with pytest.raises(TypeError): self.data._create_schema() - def test_config(self): - features = self.data.get_features(annotations=None) - - # test each for singular presence and accuracy of available flag - def check_feature(method, path, available): - feature = list( - filter(lambda f: f.method == method and f.path == path and f.available == available, features) - ) - self.assertIsNotNone(feature) - self.assertEqual(len(feature), 1) - - check_feature("POST", "/cluster/", False) - check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable) - check_feature("GET", "/layout/obs", True) - check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding) - check_feature("PUT", "/annotations/obs", False) - def test_layout(self): fbs = self.data.layout_to_fbs_matrix(fields=None) layout = decode_fbs.decode_matrix_FBS(fbs) diff --git a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py index 35ccb886..9724eadb 100644 --- a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py +++ b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py @@ -3,7 +3,7 @@ import json from server.data_anndata.anndata_adaptor import AnndataAdaptor from server.common.data_locator import DataLocator -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.test import PROJECT_ROOT @@ -16,6 +16,7 @@ class DataLoadAdaptorTest(unittest.TestCase): self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") config = AppConfig() config.update_server_config(single_dataset__datapath=self.data_file.path) + config.update_server_config(app__flask_secret_key="secret") config.complete_config() self.data = AnndataAdaptor(self.data_file, config) @@ -45,6 +46,7 @@ class DataLocatorAdaptorTest(unittest.TestCase): config.update_server_config( single_dataset__obs_names=None, single_dataset__var_names=None, ) + config.update_server_config(app__flask_secret_key="secret") config.update_default_dataset_config( embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, ) diff --git a/server/test/unit/data_common/test_matrix_loader.py b/server/test/unit/data_common/test_matrix_loader.py index d365b001..9631de32 100644 --- a/server/test/unit/data_common/test_matrix_loader.py +++ b/server/test/unit/data_common/test_matrix_loader.py @@ -4,7 +4,7 @@ import tempfile import time import unittest -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.common.errors import DatasetAccessError from server.data_common.matrix_loader import MatrixDataCacheManager from server.test import FIXTURES_ROOT @@ -38,7 +38,7 @@ class MatrixCacheTest(unittest.TestCase): result = {} for k, v in datasets.items(): # filter out the dirname and the .cxg from the name - newk = int(k[1][len(dirname) + 1: -4]) + newk = int(k[1][len(dirname) + 1 : -4]) result[newk] = v return result diff --git a/server/test/unit/eb/test_eb.py b/server/test/unit/eb/test_eb.py index b43fda24..61cf1867 100644 --- a/server/test/unit/eb/test_eb.py +++ b/server/test/unit/eb/test_eb.py @@ -3,9 +3,10 @@ import tempfile import requests import subprocess from server.test import PROJECT_ROOT, FIXTURES_ROOT -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from contextlib import contextmanager import time +import os @contextmanager @@ -34,14 +35,12 @@ class Elastic_Beanstalk_Test(unittest.TestCase): tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/server") tempdirname = tempdir.name - c = AppConfig() + config = AppConfig() # test that eb works - c.update_server_config( - multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame" - ) + config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame") - c.complete_config() - c.write_config(f"{tempdirname}/config.yaml") + config.complete_config() + config.write_config(f"{tempdirname}/config.yaml") subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/server/eb", shell=True) subprocess.check_call(["make", "build"], cwd=tempdirname) @@ -52,3 +51,33 @@ class Elastic_Beanstalk_Test(unittest.TestCase): r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config") data_config = r.json() assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + + def test_config(self): + check_config_script = os.path.join(PROJECT_ROOT, "server", "eb", "check_config.py") + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + app_config = AppConfig() + app_config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}") + app_config.write_config(configfile) + + command = ["python", check_config_script, configfile] + + # test failure mode (flask_secret_key not set) + env = os.environ.copy() + env.pop("CXG_SECRET_KEY", None) + with self.assertRaises(subprocess.CalledProcessError) as exception_context: + subprocess.check_output(command, env=env) + output = str(exception_context.exception.stdout, "utf-8") + self.assertTrue( + output.startswith( + "Error: Invalid type for attribute: app__flask_secret_key, expected type str, got NoneType" + ) + ) + self.assertEqual(exception_context.exception.returncode, 1) + + # test passing case + env = os.environ.copy() + env["CXG_SECRET_KEY"] = "secret" + output = subprocess.check_output(command, env=env) + output = str(output, "utf-8") + self.assertTrue(output.startswith("PASS"))