diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index b4cfe599..70fcf947 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -31,9 +31,10 @@ jobs: - name: Install dependencies run: | pip install flake8 + pip install black cd client npm install - - name: Lint with flake8 + - name: Format with black and lint with flake8 run: | make lint-server - name: Lint src with eslint diff --git a/LICENSE.txt b/LICENSE.txt index e0bb8c7d..a34ab341 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2013 +Copyright (c) 2017-2020 Chan Zuckerberg Initiative Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in @@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile index 5b588b77..ca8754f7 100644 --- a/Makefile +++ b/Makefile @@ -82,8 +82,9 @@ fmt-py: lint: lint-server lint-client .PHONY: lint-server -lint-server: - flake8 server +lint-server: fmt-py + flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821' + .PHONY: lint-client lint-client: diff --git a/README.md b/README.md index e2cb3215..ef313e53 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ If you believe you have found a security issue, we would appreciate notification # Inspiration -We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data. +We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data. We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation. diff --git a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap index 01ed89a8..56a349b2 100644 --- a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap +++ b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap @@ -3,14 +3,14 @@ exports[`annotations stacked bar graph renders 1`] = ` Array [ "
TEST-LABELLABEL
0
", - "
unassignedigned
2133
", + "
unassignedigned
2133
", ] `; exports[`annotations stacked bar graph renders 2`] = ` Array [ "
TEST-LABELLABEL
0
", - "
unassignedigned
2638
", + "
unassignedigned
2638
", ] `; diff --git a/client/__tests__/e2e/cellxgeneActions.js b/client/__tests__/e2e/cellxgeneActions.js index eee4714c..9fda69b1 100644 --- a/client/__tests__/e2e/cellxgeneActions.js +++ b/client/__tests__/e2e/cellxgeneActions.js @@ -13,8 +13,11 @@ import { getTestClass, getTestId, isElementPresent, + goToPage, } from "./puppeteerUtils"; +import { appUrlBase } from "./config"; + export async function drag(testId, start, end, lasso = false) { const layout = await waitByID(testId); const elBox = await layout.boxModel(); @@ -312,4 +315,75 @@ export async function assertCategoryDoesNotExist(categoryName) { await expect(result).toBe(false); } + +export async function login() { + const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`; + const password = "Test1111"; + + await goToPage(appUrlBase); + + await clickOn("auth-button"); + + // (thuang): Auth0 form is unstable and unsafe for input until verified + await waitUntilFormFieldStable('[name="email"]'); + + await expect(page).toFillForm("form", { + email, + password, + }); + + await Promise.all([ + page.waitForNavigation({ waitUntil: "networkidle0" }), + expect(page).toClick('[name="submit"]'), + ]); + + expect(page.url()).toContain(appUrlBase); +} + +export async function logout() { + await clickOnUntil("menu", async () => { + await expect(page).toMatch("Log Out"); + + await Promise.all([ + page.waitForNavigation({ waitUntil: "networkidle0" }), + expect(page).toClick("a", { text: "Log Out" }), + ]); + }); + + await expect(page).toMatch("Log In"); +} + +async function waitUntilFormFieldStable(selector) { + const MAX_RETRY = 10; + const WAIT_FOR_MS = 200; + + const EXPECTED_VALUE = "aaa"; + + let retry = 0; + + while (retry < MAX_RETRY) { + try { + await expect(page).toFill(selector, EXPECTED_VALUE); + + const fieldHandle = await expect(page).toMatchElement(selector); + + const fieldValue = await page.evaluate( + (input) => input.value, + fieldHandle + ); + + expect(fieldValue).toBe(EXPECTED_VALUE); + + break; + } catch (error) { + retry += 1; + + await page.waitFor(WAIT_FOR_MS); + } + } + + if (retry === MAX_RETRY) { + throw Error("clickOnUntil() assertion failed!"); + } +} /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */ diff --git a/client/__tests__/e2e/e2e.test.js b/client/__tests__/e2e/e2e.test.js index 982a076a..05f52314 100644 --- a/client/__tests__/e2e/e2e.test.js +++ b/client/__tests__/e2e/e2e.test.js @@ -31,6 +31,8 @@ import { runDiffExp, selectCategory, subset, + login, + logout, } from "./cellxgeneActions"; const data = datasets[DATASET]; @@ -518,4 +520,18 @@ test("lasso moves after pan", async () => { expect(panCount).toBe(initialCount); }); + +const conditionalDescribe = + process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip; + +conditionalDescribe("AuthN Integration", () => { + it("logs in", async () => { + await login(); + }); + + it("logs out", async () => { + await login(); + await logout(); + }); +}); /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */ diff --git a/client/__tests__/e2e/puppeteer.setup.js b/client/__tests__/e2e/puppeteer.setup.js index 40453777..6c593b93 100644 --- a/client/__tests__/e2e/puppeteer.setup.js +++ b/client/__tests__/e2e/puppeteer.setup.js @@ -17,7 +17,9 @@ setDefaultOptions({ timeout: 20 * 1000 }); jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS); -(async () => { +beforeEach(async () => { + await jestPuppeteer.resetBrowser(); + const userAgent = await browser.userAgent(); await page.setUserAgent(`${userAgent}bot`); @@ -53,6 +55,4 @@ jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS); } } }); -})().catch((error) => { - console.error("puppeteer.setup.js error", error); }); diff --git a/client/configuration/eslint/eslint.js b/client/configuration/eslint/eslint.js index bafb7d31..51b619db 100644 --- a/client/configuration/eslint/eslint.js +++ b/client/configuration/eslint/eslint.js @@ -4,6 +4,7 @@ module.exports = { extends: [ "airbnb", "plugin:eslint-comments/recommended", + "plugin:@blueprintjs/recommended", "plugin:compat/recommended", "plugin:prettier/recommended", "prettier/react", diff --git a/client/package-lock.json b/client/package-lock.json index 7fe2f456..5da239b5 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -4156,6 +4156,216 @@ "tslib": "~1.10.0" } }, + "@blueprintjs/eslint-plugin": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@blueprintjs/eslint-plugin/-/eslint-plugin-0.3.0.tgz", + "integrity": "sha512-bQEdE4ApEHxCDV8hT9uIxeRbDFKOtRLBT3/Zy3Ku+nowDAYl/8jwZKp6lJuR/nqvsfuIXTnVef6ivwdBEieQfA==", + "dev": true, + "requires": { + "@typescript-eslint/experimental-utils": "^4.2.0", + "eslint": "^7.9.0" + }, + "dependencies": { + "@typescript-eslint/experimental-utils": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.3.0.tgz", + "integrity": "sha512-cmmIK8shn3mxmhpKfzMMywqiEheyfXLV/+yPDnOTvQX/ztngx7Lg/OD26J8gTZfkLKUmaEBxO2jYP3keV7h2OQ==", + "dev": true, + "requires": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/scope-manager": "4.3.0", + "@typescript-eslint/types": "4.3.0", + "@typescript-eslint/typescript-estree": "4.3.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^2.0.0" + } + }, + "@typescript-eslint/typescript-estree": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.3.0.tgz", + "integrity": "sha512-ZAI7xjkl+oFdLV/COEz2tAbQbR3XfgqHEGy0rlUXzfGQic6EBCR4s2+WS3cmTPG69aaZckEucBoTxW9PhzHxxw==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.3.0", + "@typescript-eslint/visitor-keys": "4.3.0", + "debug": "^4.1.1", + "globby": "^11.0.1", + "is-glob": "^4.0.1", + "lodash": "^4.17.15", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + } + }, + "acorn": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz", + "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==", + "dev": true + }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "eslint": { + "version": "7.10.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz", + "integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@eslint/eslintrc": "^0.1.3", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "enquirer": "^2.3.5", + "eslint-scope": "^5.1.1", + "eslint-utils": "^2.1.0", + "eslint-visitor-keys": "^1.3.0", + "espree": "^7.3.0", + "esquery": "^1.2.0", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "progress": "^2.0.0", + "regexpp": "^3.1.0", + "semver": "^7.2.1", + "strip-ansi": "^6.0.0", + "strip-json-comments": "^3.1.0", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "dependencies": { + "eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + } + } + }, + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + }, + "espree": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz", + "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==", + "dev": true, + "requires": { + "acorn": "^7.4.0", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.3.0" + } + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "requires": { + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } + }, + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true + }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, "@blueprintjs/icons": { "version": "3.19.0", "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.19.0.tgz", @@ -4184,6 +4394,76 @@ "minimist": "^1.2.0" } }, + "@eslint/eslintrc": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz", + "integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==", + "dev": true, + "requires": { + "ajv": "^6.12.4", + "debug": "^4.1.1", + "espree": "^7.3.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.2.1", + "js-yaml": "^3.13.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "acorn": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz", + "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==", + "dev": true + }, + "ajv": { + "version": "6.12.5", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz", + "integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + }, + "espree": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz", + "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==", + "dev": true, + "requires": { + "acorn": "^7.4.0", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.3.0" + } + }, + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + } + } + }, "@hapi/address": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz", @@ -5137,6 +5417,32 @@ } } }, + "@nodelib/fs.scandir": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", + "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "2.0.3", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz", + "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==", + "dev": true + }, + "@nodelib/fs.walk": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz", + "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==", + "dev": true, + "requires": { + "@nodelib/fs.scandir": "2.1.3", + "fastq": "^1.6.0" + } + }, "@npmcli/move-file": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.0.1.tgz", @@ -5489,6 +5795,22 @@ "eslint-utils": "^2.0.0" } }, + "@typescript-eslint/scope-manager": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.3.0.tgz", + "integrity": "sha512-cTeyP5SCNE8QBRfc+Lgh4Xpzje46kNUhXYfc3pQWmJif92sjrFuHT9hH4rtOkDTo/si9Klw53yIr+djqGZS1ig==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.3.0", + "@typescript-eslint/visitor-keys": "4.3.0" + } + }, + "@typescript-eslint/types": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.3.0.tgz", + "integrity": "sha512-Cx9TpRvlRjOppGsU6Y6KcJnUDOelja2NNCX6AZwtVHRzaJkdytJWMuYiqi8mS35MRNA3cJSwDzXePfmhU6TANw==", + "dev": true + }, "@typescript-eslint/typescript-estree": { "version": "2.34.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz", @@ -5512,6 +5834,24 @@ } } }, + "@typescript-eslint/visitor-keys": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.3.0.tgz", + "integrity": "sha512-xZxkuR7XLM6RhvLkgv9yYlTcBHnTULzfnw4i6+z2TGBLy9yljAypQaZl9c3zFvy7PNI7fYWyvKYtohyF8au3cw==", + "dev": true, + "requires": { + "@typescript-eslint/types": "4.3.0", + "eslint-visitor-keys": "^2.0.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz", + "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==", + "dev": true + } + } + }, "@webassemblyjs/ast": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz", @@ -6387,9 +6727,9 @@ } }, "bl": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz", - "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", + "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", "dev": true, "requires": { "buffer": "^5.5.0", @@ -6397,16 +6737,6 @@ "readable-stream": "^3.4.0" }, "dependencies": { - "buffer": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz", - "integrity": "sha512-9FTEDjLjwoAkEwyMGDjYJQN2gfRgOKBKRfiglhvibGbpeeU/pQn1bJxQqm32OD/AIeEuHxU9roxXxg34Byp/Ww==", - "dev": true, - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, "readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -8796,6 +9126,15 @@ } } }, + "dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "requires": { + "path-type": "^4.0.0" + } + }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -10187,6 +10526,31 @@ "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", "dev": true }, + "fast-glob": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz", + "integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==", + "dev": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.0", + "merge2": "^1.3.0", + "micromatch": "^4.0.2", + "picomatch": "^2.2.1" + }, + "dependencies": { + "glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "requires": { + "is-glob": "^4.0.1" + } + } + } + }, "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -10202,6 +10566,15 @@ "resolved": "https://registry.npmjs.org/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz", "integrity": "sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw==" }, + "fastq": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz", + "integrity": "sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==", + "dev": true, + "requires": { + "reusify": "^1.0.4" + } + }, "favicons": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/favicons/-/favicons-5.5.0.tgz", @@ -11159,6 +11532,28 @@ "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==" }, + "globby": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz", + "integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==", + "dev": true, + "requires": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.1.1", + "ignore": "^5.1.4", + "merge2": "^1.3.0", + "slash": "^3.0.0" + }, + "dependencies": { + "array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true + } + } + }, "got": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", @@ -11691,6 +12086,12 @@ "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=", "dev": true }, + "ignore": { + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", + "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", + "dev": true + }, "ignore-walk": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", @@ -14189,6 +14590,12 @@ "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" }, + "merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true + }, "methods": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", @@ -15251,6 +15658,12 @@ "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=", "dev": true }, + "path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true + }, "pbkdf2": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz", @@ -17065,6 +17478,12 @@ "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" }, + "reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true + }, "rgb-regex": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/rgb-regex/-/rgb-regex-1.0.1.tgz", @@ -17100,6 +17519,12 @@ "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz", "integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA==" }, + "run-parallel": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz", + "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==", + "dev": true + }, "run-queue": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz", diff --git a/client/package.json b/client/package.json index d5fd7294..85a48aa8 100644 --- a/client/package.json +++ b/client/package.json @@ -84,6 +84,7 @@ "@babel/preset-react": "^7.10.4", "@babel/register": "^7.10.5", "@babel/runtime": "^7.10.5", + "@blueprintjs/eslint-plugin": "^0.3.0", "@sentry/webpack-plugin": "^1.12.0", "babel-eslint": "^10.1.0", "babel-jest": "^26.1.0", diff --git a/client/src/actions/embedding.js b/client/src/actions/embedding.js index 615bcd68..7b781a0d 100644 --- a/client/src/actions/embedding.js +++ b/client/src/actions/embedding.js @@ -5,20 +5,23 @@ action creators related to embeddings choice import { AnnoMatrixObsCrossfilter } from "../annoMatrix"; import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers"; -export async function _switchEmbedding(prevAnnoMatrix, prevCrossfilter, newEmbeddingName) { +export async function _switchEmbedding( + prevAnnoMatrix, + prevCrossfilter, + newEmbeddingName +) { /* DRY helper used by this and reembedding action creators */ const base = prevAnnoMatrix.base(); const embeddingDf = await base.fetch("emb", newEmbeddingName); const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf); - const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix, prevCrossfilter.obsCrossfilter).select( - "emb", - newEmbeddingName, - { - mode: "all", - } - ); + const obsCrossfilter = await new AnnoMatrixObsCrossfilter( + annoMatrix, + prevCrossfilter.obsCrossfilter + ).select("emb", newEmbeddingName, { + mode: "all", + }); return [annoMatrix, obsCrossfilter]; } @@ -30,7 +33,10 @@ export const layoutChoiceAction = (newLayoutChoice) => async ( On layout choice, make sure we have selected all on the previous layout, AND the new layout. */ - const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState(); + const { + annoMatrix: prevAnnoMatrix, + obsCrossfilter: prevCrossfilter, + } = getState(); const [annoMatrix, obsCrossfilter] = await _switchEmbedding( prevAnnoMatrix, prevCrossfilter, diff --git a/client/src/components/autosave/filenameDialog.js b/client/src/components/autosave/filenameDialog.js index 3937ecee..08fe1462 100644 --- a/client/src/components/autosave/filenameDialog.js +++ b/client/src/components/autosave/filenameDialog.js @@ -3,11 +3,12 @@ import { connect } from "react-redux"; import { Button, - Tooltip, - InputGroup, - Dialog, Classes, + Code, Colors, + Dialog, + InputGroup, + Tooltip, } from "@blueprintjs/core"; @connect((state) => ({ @@ -145,9 +146,9 @@ class FilenameDialog extends React.Component {

Your annotations are stored in this file: - + {filenameText}-{idhash}.csv - +

(We added a unique ID to your filename) diff --git a/client/src/components/brushableHistogram/index.js b/client/src/components/brushableHistogram/index.js index 71175b2d..a18608b2 100644 --- a/client/src/components/brushableHistogram/index.js +++ b/client/src/components/brushableHistogram/index.js @@ -5,12 +5,13 @@ https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771 https://bl.ocks.org/mbostock/3019563 */ import React, { useEffect, useRef, useState, useCallback } from "react"; -import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core"; +import { Button, ButtonGroup, Icon, Tooltip } from "@blueprintjs/core"; import { connect } from "react-redux"; import * as d3 from "d3"; import { interpolateCool } from "d3-scale-chromatic"; import Async from "react-async"; import memoize from "memoize-one"; +import { IconNames } from "@blueprintjs/icons"; import * as globals from "../../globals"; import actions from "../../actions"; import { histogramContinuous } from "../../util/dataframe/histogram"; @@ -26,7 +27,7 @@ function maybeScientific(x) { const _ticks = x.ticks(4); if (x.domain().some((n) => Math.abs(n) >= 10000)) { - /* + /* heuristic: if the last tick d3 wants to render has one significant digit ie., 2000, render 2e+3, but if it's anything else ie., 42000000 render 4.20e+n @@ -99,7 +100,7 @@ const HistogramFooter = React.memo( pvalAdj, }) => { /* - Footer of each histogram. Will render range, title, and optionally + Footer of each histogram. Will render range, title, and optionally differential expression info. Required props: @@ -214,10 +215,7 @@ const HistogramHeader = React.memo( > {onScatterPlotXClick && onScatterPlotYClick ? ( - + - + gene + +

+
+ + + + {!userinfo.is_authenticated ? ( + + ) : null} +
); } diff --git a/client/src/components/menubar/authButtons.js b/client/src/components/menubar/authButtons.js index 7eb4c4c0..985b3da3 100644 --- a/client/src/components/menubar/authButtons.js +++ b/client/src/components/menubar/authButtons.js @@ -1,5 +1,5 @@ import React from "react"; -import { AnchorButton, Tooltip } from "@blueprintjs/core"; +import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core"; import * as globals from "../../globals"; import styles from "./menubar.css"; @@ -9,9 +9,9 @@ const Auth = React.memo((props) => { if (!auth || (auth && !auth.requires_client_login)) return null; return ( -
+ @@ -19,13 +19,12 @@ const Auth = React.memo((props) => { type="button" data-testid="auth-button" disabled={false} - icon={!userinfo.is_authenticated ? "log-in" : "log-out"} href={!userinfo.is_authenticated ? auth.login : auth.logout} > {!userinfo.is_authenticated ? "Log In" : "Log Out"} -
+ ); }); diff --git a/client/src/components/menubar/clip.js b/client/src/components/menubar/clip.js index 5b866bd0..5bc1d18d 100644 --- a/client/src/components/menubar/clip.js +++ b/client/src/components/menubar/clip.js @@ -1,12 +1,16 @@ import React from "react"; import { - Position, Button, - Popover, - NumericInput, + ButtonGroup, Icon, + Intent, + NumericInput, + Popover, + Position, Tooltip, } from "@blueprintjs/core"; +import { IconNames } from "@blueprintjs/icons"; + import { tooltipHoverOpenDelay } from "../../globals"; import styles from "./menubar.css"; @@ -28,13 +32,13 @@ const Clip = React.memo((props) => { pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin; const clipMax = pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax; - const activeClipClass = + const intent = clipPercentileMin > 0 || clipPercentileMax < 100 - ? " bp3-intent-warning" - : ""; + ? Intent.INTENT_WARNING + : Intent.NONE; return ( -
+ {
} /> - + ); }); diff --git a/client/src/components/menubar/index.js b/client/src/components/menubar/index.js index 3fd840c0..b4271302 100644 --- a/client/src/components/menubar/index.js +++ b/client/src/components/menubar/index.js @@ -6,8 +6,7 @@ import * as globals from "../../globals"; import styles from "./menubar.css"; import actions from "../../actions"; import Clip from "./clip"; -import AuthButtons from "./authButtons"; -import InformationMenu from "./infoMenu"; + import Subset from "./subset"; import UndoRedoReset from "./undoRedo"; import DiffexpButtons from "./diffexpButtons"; @@ -204,7 +203,6 @@ class MenuBar extends React.PureComponent { render() { const { dispatch, - libraryVersions, disableDiffexp, undoDisabled, redoDisabled, @@ -212,17 +210,12 @@ class MenuBar extends React.PureComponent { clipPercentileMin, clipPercentileMax, graphInteractionMode, - aboutLink, showCentroidLabels, - privacyURL, - tosURL, categoricalSelection, colorAccessor, subsetPossible, subsetResetPossible, enableReembedding, - auth, - userinfo, } = this.props; const { pendingClipPercentiles } = this.state; @@ -248,10 +241,6 @@ class MenuBar extends React.PureComponent { zIndex: 3, }} > - - { @@ -9,35 +16,45 @@ const handleClick = (dispatch) => { }; const InformationMenu = React.memo((props) => { - const { libraryVersions, tosURL, privacyURL, dispatch } = props; + const { + libraryVersions, + tosURL, + privacyURL, + auth, + userinfo, + dispatch, + } = props; return ( -
+ handleClick(dispatch)} - icon={IconNames.INFO_SIGN} + icon="info-sign" text="Dataset Overview" /> { href={privacyURL} target="_blank" text="Privacy Policy" + rel="noopener" /> ) : null} + + {auth?.["requires_client_login"] && + userinfo?.["is_authenticated"] ? ( + <> + + + + ) : null} } position={Position.BOTTOM_RIGHT} + modifiers={{ + preventOverflow: { enabled: false }, + hide: { enabled: false }, + }} >
+ ); }); diff --git a/client/src/components/menubar/undoRedo.js b/client/src/components/menubar/undoRedo.js index 601dba32..8591505c 100644 --- a/client/src/components/menubar/undoRedo.js +++ b/client/src/components/menubar/undoRedo.js @@ -1,12 +1,13 @@ import React from "react"; -import { AnchorButton, Tooltip } from "@blueprintjs/core"; +import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core"; +import { IconNames } from "@blueprintjs/icons"; import { tooltipHoverOpenDelay } from "../../globals"; import styles from "./menubar.css"; const UndoRedo = React.memo((props) => { const { undoDisabled, redoDisabled, dispatch } = props; return ( -
+ { > { dispatch({ type: "@@undoable/undo" }); @@ -32,7 +33,7 @@ const UndoRedo = React.memo((props) => { > { dispatch({ type: "@@undoable/redo" }); @@ -43,7 +44,7 @@ const UndoRedo = React.memo((props) => { data-testid="redo" /> -
+ ); }); diff --git a/client/src/components/miniHistogram/index.js b/client/src/components/miniHistogram/index.js index eb57f832..59153d91 100644 --- a/client/src/components/miniHistogram/index.js +++ b/client/src/components/miniHistogram/index.js @@ -72,7 +72,6 @@ export default class MiniHistogram extends React.PureComponent { popoverClassName={Classes.POPOVER_CONTENT_SIZING} > Index | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/annotations.html b/docs/_site/posts/annotations.html index 2a76ec9f..725ab7db 100644 --- a/docs/_site/posts/annotations.html +++ b/docs/_site/posts/annotations.html @@ -7,19 +7,19 @@ annotations | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/annotations.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"annotations","description":"Creating annotations","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/contact.html b/docs/_site/posts/contact.html index 33c31b51..2bbefffa 100644 --- a/docs/_site/posts/contact.html +++ b/docs/_site/posts/contact.html @@ -7,19 +7,19 @@ Contact | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/contact.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Contact","description":"Contact","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/contribute.html b/docs/_site/posts/contribute.html index 494d2ca3..9727cffb 100644 --- a/docs/_site/posts/contribute.html +++ b/docs/_site/posts/contribute.html @@ -7,19 +7,19 @@ Code of conduct | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/contribute.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Code of conduct","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/demo-data.html b/docs/_site/posts/demo-data.html index 479e6249..4b05dfbe 100644 --- a/docs/_site/posts/demo-data.html +++ b/docs/_site/posts/demo-data.html @@ -7,19 +7,19 @@ demo-data | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/demo-data.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/gallery.html b/docs/_site/posts/gallery.html index 2321d81e..5b19ba47 100644 --- a/docs/_site/posts/gallery.html +++ b/docs/_site/posts/gallery.html @@ -7,19 +7,19 @@ Gallery | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/gallery.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Gallery","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

@@ -130,7 +130,7 @@ Check out the cool data that our users are using cellxgene to explore!

Melanoma

-

CZI’s own cellxgene site

+

CZI’s own cellxgene site

Want us to link to your dataset here? Just send us a note!

diff --git a/docs/_site/posts/hosted.html b/docs/_site/posts/hosted.html index 0e2d2b6a..b85e5d96 100644 --- a/docs/_site/posts/hosted.html +++ b/docs/_site/posts/hosted.html @@ -7,19 +7,19 @@ Hosting cellxgene on the web | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/hosted.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Hosting cellxgene on the web","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

@@ -139,35 +139,36 @@

Deploying cellxgene with Heroku

-

Quickstart

+

Heroku Support

-

Clicking on the following button will forward you to Heroku to begin the deployment process:

+

The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.

-

- Deploy -

+

While we no longer directly support Heroku, it is still possible to create a Heroku app via our provided Dockerfile here and Heroku’s documentation.

-

If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.

+

You may have to tweak the Dockerfile like so:

-

Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:

+
FROM ubuntu:bionic
 
-
Default settings

+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 
-
    
-  
  • App name: the unique name for your deployment
    • 
-  
  • This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
    • 
-  
  • App owner: Who will own this app. Either you personally or an organization/team
    • 
-  
  • Region: Location of the server where the app will be deployed (EU or US)
    • 
-

+RUN apt-get update && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    pip3 install cellxgene
 
-
Configuration

+# ENTRYPOINT ["cellxgene"]  # Heroku doesn't work well with ENTRYPOINT
+
-
    -
  • DATASET: A publicly accessible URL pointing to a .h5ad file to view
    • -
  • This defaults to pbm3k.h5ad
    • -
+

and provide a heroku.yml file similar to this:

-

After filling out the settings and pressing the Deploy app button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!

+
build:
+  docker:
+    web: Dockerfile
+run:
+  web:
+    command:
+      - cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings.
+

What is Heroku?

diff --git a/docs/_site/posts/hosted.md b/docs/_site/posts/hosted.md index 1168bdf0..073ede26 100644 --- a/docs/_site/posts/hosted.md +++ b/docs/_site/posts/hosted.md @@ -38,31 +38,38 @@ If you know of other solutions, drop us a note and we'll add to this list. # Deploying cellxgene with Heroku -## Quickstart +## Heroku Support -Clicking on the following button will forward you to Heroku to begin the deployment process: +The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene. - - Deploy - +While we no longer directly support Heroku, it is still possible to create a Heroku app via [our provided Dockerfile here](https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile) and [Heroku's documentation](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml). -If not already logged in to Heroku, there you will be prompted to log in or sign up for an account. +You may have to tweak the `Dockerfile` like so: -Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app: +```Dockerfile +FROM ubuntu:bionic -### Default settings +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 -- `App name`: the unique name for your deployment -- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/) -- `App owner`: Who will own this app. Either you personally or an organization/team -- `Region`: Location of the server where the app will be deployed (EU or US) +RUN apt-get update && \ + apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \ + pip3 install cellxgene -### Configuration +# ENTRYPOINT ["cellxgene"] # Heroku doesn't work well with ENTRYPOINT +``` -- `DATASET`: A _publicly_ accessible URL pointing to a .h5ad file to view -- This defaults to pbm3k.h5ad +and provide a `heroku.yml` file similar to this: -After filling out the settings and pressing the `Deploy app` button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene! +```yml +build: + docker: + web: Dockerfile +run: + web: + command: + - cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings. +``` ## What is Heroku? diff --git a/docs/_site/posts/install.html b/docs/_site/posts/install.html index 2eb470d8..649209dc 100644 --- a/docs/_site/posts/install.html +++ b/docs/_site/posts/install.html @@ -7,19 +7,19 @@ Install | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/install.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Install","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/launch.html b/docs/_site/posts/launch.html index 4aed95b3..86c8dd3b 100644 --- a/docs/_site/posts/launch.html +++ b/docs/_site/posts/launch.html @@ -7,19 +7,19 @@ demo-data | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/launch.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/methods.html b/docs/_site/posts/methods.html index 8c97a84c..f27e270e 100644 --- a/docs/_site/posts/methods.html +++ b/docs/_site/posts/methods.html @@ -7,19 +7,19 @@ Methods | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/methods.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Methods","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/prepare.html b/docs/_site/posts/prepare.html index 56dce0c2..2b3b0f05 100644 --- a/docs/_site/posts/prepare.html +++ b/docs/_site/posts/prepare.html @@ -7,19 +7,19 @@ prepare | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/prepare.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"prepare","description":"Preparing your data","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/roadmap.html b/docs/_site/posts/roadmap.html index 34dc26b9..837a3bb7 100644 --- a/docs/_site/posts/roadmap.html +++ b/docs/_site/posts/roadmap.html @@ -7,19 +7,19 @@ roadmap | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/roadmap.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"roadmap","description":"Roadmap","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/_site/posts/troubleshooting.html b/docs/_site/posts/troubleshooting.html index a936237a..c43ee9af 100644 --- a/docs/_site/posts/troubleshooting.html +++ b/docs/_site/posts/troubleshooting.html @@ -7,19 +7,19 @@ Troubleshooting | cellxgene - + - - + + +{"url":"http://localhost:4000/cellxgene/posts/troubleshooting.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Troubleshooting","description":"Troubleshooting","@type":"WebPage","@context":"https://schema.org"} - + @@ -50,6 +50,10 @@ + All other datasets
+ + + Preparing your data
@@ -85,10 +89,6 @@ Contact & finding help
- - cellxgene.cziscience.com
- - Code

diff --git a/docs/posts/cellxgene_cziscience_com.md b/docs/deprecated/cellxgene_cziscience_com.md similarity index 100% rename from docs/posts/cellxgene_cziscience_com.md rename to docs/deprecated/cellxgene_cziscience_com.md diff --git a/docs/posts/gallery.md b/docs/posts/gallery.md index 13a7fc24..01f45301 100644 --- a/docs/posts/gallery.md +++ b/docs/posts/gallery.md @@ -39,6 +39,6 @@ Check out the cool data that our users are using cellxgene to explore! ### [Melanoma](https://melanoma.cellgeni.sanger.ac.uk/) -### [CZI's own cellxgene site](cellxgene_cziscience_com) +### [CZI's own cellxgene site](https://cellxgene.cziscience.com/) _Want us to link to your dataset here? [Just send us a note!](contact)_ diff --git a/server/__init__.py b/server/__init__.py index e076acd8..b7efe9b5 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -1,6 +1,5 @@ import logging import sys - from server.common.utils.utils import import_plugins __version__ = "0.16.5-rc.0" diff --git a/server/app/app.py b/server/app/app.py index 7e02b4e1..da167453 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -6,8 +6,17 @@ from urllib.parse import urlparse import hashlib import os -from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \ - send_from_directory +from flask import ( + Flask, + redirect, + current_app, + make_response, + render_template, + abort, + Blueprint, + request, + send_from_directory, +) from flask_restful import Api, Resource from server_timing import Timing as ServerTiming @@ -87,10 +96,7 @@ def dataset_index(url_dataroot=None, dataset=None): cache_manager = current_app.matrix_data_cache_manager with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor: data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}") - args = { - "SCRIPTS" : scripts, - "INLINE_SCRIPTS" : inline_scripts - } + args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts} return render_template("index.html", **args) except DatasetAccessError as e: @@ -417,8 +423,9 @@ class Server: for dataroot_dict in server_config.multi_dataset__dataroot.values(): url_dataroot = dataroot_dict["base_url"] bp_dataroot = Blueprint( - f"api_dataset_{url_dataroot}", __name__, - url_prefix=f"{api_path}/{url_dataroot}/" + api_version + f"api_dataset_{url_dataroot}", + __name__, + url_prefix=f"{api_path}/{url_dataroot}/" + api_version, ) dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot) self.app.register_blueprint(dataroot_resources.blueprint) @@ -433,7 +440,7 @@ class Server: f"/{url_dataroot}//static/", f"static_assets_{url_dataroot}", view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename), - methods=["GET"] + methods=["GET"], ) else: @@ -444,7 +451,7 @@ class Server: "/static/", "static_assets", view_func=lambda filename: send_from_directory("../common/web/static", filename), - methods=["GET"] + methods=["GET"], ) self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager diff --git a/server/auth/__init__.py b/server/auth/__init__.py index 1b33bebc..dfadadde 100644 --- a/server/auth/__init__.py +++ b/server/auth/__init__.py @@ -1,4 +1,3 @@ - # import the built in auth types so they can be registered import server.auth.auth_none # noqa: F401 diff --git a/server/auth/auth.py b/server/auth/auth.py index bb86ea64..145616cc 100644 --- a/server/auth/auth.py +++ b/server/auth/auth.py @@ -76,7 +76,7 @@ class AuthTypeFactory: @staticmethod def register(name, auth_type): - assert(issubclass(auth_type, AuthTypeBase)) + assert issubclass(auth_type, AuthTypeBase) AuthTypeFactory.auth_types[name] = auth_type @staticmethod diff --git a/server/auth/auth_none.py b/server/auth/auth_none.py index c482e3c4..61f82400 100644 --- a/server/auth/auth_none.py +++ b/server/auth/auth_none.py @@ -2,7 +2,6 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory class AuthTypeNone(AuthTypeBase): - def __init__(self, app_config): super().__init__() diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py index a5107934..d7162318 100644 --- a/server/auth/auth_oauth.py +++ b/server/auth/auth_oauth.py @@ -97,8 +97,17 @@ class AuthTypeOAuth(AuthTypeClientBase): return valid_keys = { - "verify_signature", "verify_aud", "verify_iat", "verify_exp", "verify_nbf", "verify_iss", - "verify_sub", "verify_jti", "verify_at_hash", "leeway"} + "verify_signature", + "verify_aud", + "verify_iat", + "verify_exp", + "verify_nbf", + "verify_iss", + "verify_sub", + "verify_jti", + "verify_at_hash", + "leeway", + } keys = set(self.jwt_decode_options.keys()) unknown = keys - valid_keys if unknown: @@ -114,6 +123,7 @@ class AuthTypeOAuth(AuthTypeClientBase): parse = urlparse(self.api_base_url) app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"]) app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"]) + app.add_url_rule(f"{parse.path}/logout_redirect", "logout_redirect", self.logout_redirect, methods=["GET"]) app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"]) def complete_setup(self, flask_app): @@ -166,12 +176,29 @@ class AuthTypeOAuth(AuthTypeClientBase): return response def logout(self): + """ + We would like for the user to remain on the same dataset after logout. oauth requires that + the redirect `returnTo` path be whitelisted by the oauth server, therefore a level of + indirection is used. We first redirect to a single path "logout_redirect", and logout_redirect + will redirect the user's browser back to the current page. + """ self.remove_tokens() - params = {"returnTo": self.web_base_url, "client_id": self.client_id} + redirect_path = request.args.get("dataset", "") + redirect_to = f"{self.web_base_url}/{redirect_path}" + session["oauth_logout_redirect"] = redirect_to + + return_to = f"{self.api_base_url}/logout_redirect" + params = {"returnTo": return_to, "client_id": self.client_id} response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params)) self.update_response(response) return response + def logout_redirect(self): + oauth_logout_redirect = session.pop("oauth_logout_redirect", "/") + response = redirect(oauth_logout_redirect) + self.update_response(response) + return response + def callback(self): data = self.client.authorize_access_token() tokens = Tokens( @@ -253,7 +280,10 @@ class AuthTypeOAuth(AuthTypeClientBase): def get_logout_url(self, data_adaptor): """Return the url for the logout route""" - return f"{self.api_base_url}/logout" + if data_adaptor and current_app.app_config.is_multi_dataset(): + return f"{self.api_base_url}/logout?dataset={data_adaptor.uri_path}/" + else: + return f"{self.api_base_url}/logout" def check_jwt_payload(self, id_token): try: diff --git a/server/auth/auth_test.py b/server/auth/auth_test.py index e6b0b8e0..4594dae0 100644 --- a/server/auth/auth_test.py +++ b/server/auth/auth_test.py @@ -47,6 +47,7 @@ class AuthTypeTest(AuthTypeClientBase): return_to = args.get("dataset", "/") session[self.CXGUID] = args.get("userid", self.user_id) session[self.CXGUNAME] = args.get("username", self.user_name) + session[self.CXGUEMAIL] = args.get("email", self.user_email) return redirect(return_to) def logout(self): diff --git a/server/cli/convert_to_cxg.py b/server/cli/convert_to_cxg.py index 4e456c08..0cd67e3c 100644 --- a/server/cli/convert_to_cxg.py +++ b/server/cli/convert_to_cxg.py @@ -9,26 +9,24 @@ from server.converters.h5ad_data_file import H5ADDataFile name="convert", short_help="Converts an H5AD dataset to the CXG format.", help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format " - "that has performance and access characteristics amenable to a multi-dataset, multi-user serving " - "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as " - "usually with the generated CXG file.", + "that has performance and access characteristics amenable to a multi-dataset, multi-user serving " + "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as " + "usually with the generated CXG file.", ) @click.argument( - "input-file", - nargs=1, - type=click.Path(exists=True, dir_okay=False), + "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False), ) @click.option( "-o", "--output-directory", help="Name of the output CXG directory. If not provided, will default to be the input filename with a " - "CXG extension.", + "CXG extension.", ) @click.option( "-b", "--backed", help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, " - "but will use less memory.", + "but will use less memory.", default=False, show_default=True, is_flag=True, @@ -37,29 +35,33 @@ from server.converters.h5ad_data_file import H5ADDataFile "-t", "--title", help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, " - "the dataset title will be the filename.", + "the dataset title will be the filename.", ) @click.option( "-a", "--about", help="A fully qualified URL that provides more information about the dataset and will be included as " - "metadata about the CXG file.", + "metadata about the CXG file.", ) @click.option( "-s", "--sparse-threshold", help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X " - "array of the dataset will be sparse. Since the default value is 0.0, the default will be to " - "convert to dense array.", + "array of the dataset will be sparse. Since the default value is 0.0, the default will be to " + "convert to dense array.", default=0.0, show_default=True, ) -@click.option("--obs-names", - help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of " - "the one designated by the dataframe generated-index.") -@click.option("--var-names", - help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of " - "the one designated by the dataframe generated-index.") +@click.option( + "--obs-names", + help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of " + "the one designated by the dataframe generated-index.", +) +@click.option( + "--var-names", + help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of " + "the one designated by the dataframe generated-index.", +) @click.option( "--disable-custom-colors", help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.", @@ -70,8 +72,8 @@ from server.converters.h5ad_data_file import H5ADDataFile @click.option( "--disable-corpora-schema", help="When set, conversion process will neither extract nor store Corpora schema information. See " - "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more " - "information.", + "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more " + "information.", default=False, show_default=True, is_flag=True, @@ -85,30 +87,32 @@ from server.converters.h5ad_data_file import H5ADDataFile ) @click.help_option("--help", "-h", help="Show this message and exit.") def convert_to_cxg( - input_file, - output_directory, - backed, - title, - about, - sparse_threshold, - obs_names, - var_names, - disable_custom_colors, - disable_corpora_schema, - overwrite, + input_file, + output_directory, + backed, + title, + about, + sparse_threshold, + obs_names, + var_names, + disable_custom_colors, + disable_corpora_schema, + overwrite, ): """ Convert a dataset file into CXG. """ - h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names, - use_corpora_schema=not disable_corpora_schema) + h5ad_data_file = H5ADDataFile( + input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema + ) # Get the directory that will hold all the CXG files cxg_output_container = get_output_directory(input_file, output_directory, overwrite) - h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold, - convert_anndata_colors_to_cxg_colors=not disable_custom_colors) + h5ad_data_file.to_cxg( + cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors + ) def get_output_directory(input_filename, output_directory, should_overwrite): diff --git a/server/cli/launch.py b/server/cli/launch.py index d46161d4..a05fd6a1 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -3,15 +3,14 @@ import functools import logging import sys import webbrowser -from os import devnull - +import os import click from flask_compress import Compress from flask_cors import CORS +from server.default_config import default_config from server.app.app import Server -from server.common.app_config import AppConfig -from server.common.default_config import default_config +from server.common.config.app_config import AppConfig from server.common.errors import DatasetAccessError, ConfigurationError from server.common.utils.utils import sort_options @@ -33,7 +32,7 @@ def annotation_args(func): multiple=False, metavar="", help="CSV file to initialize editing of existing annotations; will be altered in-place. " - "Incompatible with --annotations-dir.", + "Incompatible with --annotations-dir.", ) @click.option( "--annotations-dir", @@ -42,7 +41,7 @@ def annotation_args(func): multiple=False, metavar="", help="Directory of where to save output annotations; filename will be specified in the application. " - "Incompatible with --annotations-file.", + "Incompatible with --annotations-file.", ) @click.option( "--experimental-annotations-ontology", @@ -170,7 +169,7 @@ def server_args(func): default=DEFAULT_CONFIG.server_config.app__debug, show_default=True, help="Run in debug mode. This is helpful for cellxgene developers, " - "or when you want more information about an error condition.", + "or when you want more information about an error condition.", ) @click.option( "--verbose", @@ -203,7 +202,7 @@ def server_args(func): multiple=True, metavar="", help="Additional script files to include in HTML page. If not specified, " - "no additional script files will be included.", + "no additional script files will be included.", show_default=False, ) @functools.wraps(func) @@ -223,7 +222,7 @@ def launch_args(func): default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot, metavar="", help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)" - " to folder containing H5AD and/or CXG datasets.", + " to folder containing H5AD and/or CXG datasets.", hidden=True, ) # TODO, unhide when dataroot is supported) @click.argument("datapath", required=False, metavar="") @@ -307,32 +306,32 @@ class CliLaunchServer(Server): ) @launch_args def launch( - datapath, - dataroot, - verbose, - debug, - open_browser, - port, - host, - embedding, - obs_names, - var_names, - max_category_items, - disable_custom_colors, - diffexp_lfc_cutoff, - title, - scripts, - about, - disable_annotations, - annotations_file, - annotations_dir, - backed, - disable_diffexp, - experimental_annotations_ontology, - experimental_annotations_ontology_obo, - experimental_enable_reembedding, - config_file, - dump_default_config, + datapath, + dataroot, + verbose, + debug, + open_browser, + port, + host, + embedding, + obs_names, + var_names, + max_category_items, + disable_custom_colors, + diffexp_lfc_cutoff, + title, + scripts, + about, + disable_annotations, + annotations_file, + annotations_dir, + backed, + disable_diffexp, + experimental_annotations_ontology, + experimental_annotations_ontology_obo, + experimental_enable_reembedding, + config_file, + dump_default_config, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. @@ -443,7 +442,7 @@ def launch( click.echo("[cellxgene] Type CTRL-C at any time to exit.") if not server_config.app__verbose: - f = open(devnull, "w") + f = open(os.devnull, "w") sys.stdout = f try: diff --git a/server/cli/prepare.py b/server/cli/prepare.py index df535db0..67735172 100644 --- a/server/cli/prepare.py +++ b/server/cli/prepare.py @@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options default=False, is_flag=True, help="Do not run quality control metrics. By default cellxgene runs them " - "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", + "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", ) @click.option( "--make-obs-names-unique/--no-make-obs-names-unique", @@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options ) @click.help_option("--help", "-h", help="Show this message and exit.") def prepare( - data, - embedding, - recipe, - output, - plotting, - sparse, - overwrite, - set_obs_names, - set_var_names, - skip_qc, - make_obs_names_unique, - make_var_names_unique, + data, + embedding, + recipe, + output, + plotting, + sparse, + overwrite, + set_obs_names, + set_var_names, + skip_qc, + make_obs_names_unique, + make_var_names_unique, ): """ Preprocess data for use with cellxgene. diff --git a/server/cli/upgrade.py b/server/cli/upgrade.py index 953f92ba..222d7e81 100644 --- a/server/cli/upgrade.py +++ b/server/cli/upgrade.py @@ -10,7 +10,8 @@ from .. import __version__ SEMVER_FORMAT = re.compile( r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[" r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(" - r"?:\.[0-9a-zA-Z-]+)*))?$") + r"?:\.[0-9a-zA-Z-]+)*))?$" +) def log_upgrade_check(): diff --git a/server/common/annotations/hosted_tiledb.py b/server/common/annotations/hosted_tiledb.py index f3df75ed..54f7dac2 100644 --- a/server/common/annotations/hosted_tiledb.py +++ b/server/common/annotations/hosted_tiledb.py @@ -31,7 +31,8 @@ class AnnotationsHostedTileDB(Annotations): unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names) if unsanitary_original_category_names: raise AnnotationCategoryNameError( - f"{unsanitary_original_category_names} are not valid category names, please resubmit") + f"{unsanitary_original_category_names} are not valid category names, please resubmit" + ) def is_safe_collection_name(self, name): """ @@ -68,11 +69,11 @@ class AnnotationsHostedTileDB(Annotations): index_dims = None schema_hints = json.loads(schema_hints) - if '__pandas_attribute_repr' in tileDBArray.meta: + if "__pandas_attribute_repr" in tileDBArray.meta: # backwards compatibility... unsure if necessary at this point - repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr']) - if '__pandas_index_dims' in tileDBArray.meta: - index_dims = json.loads(tileDBArray.meta['__pandas_index_dims']) + repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"]) + if "__pandas_index_dims" in tileDBArray.meta: + index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"]) data = tileDBArray[:] indexes = list() @@ -80,12 +81,12 @@ class AnnotationsHostedTileDB(Annotations): for col_name, col_val in data.items(): # If the column values are byte literals, decode them if isinstance(col_val[0], bytes): - col_val = [value.decode('utf-8') for value in col_val] + col_val = [value.decode("utf-8") for value in col_val] if schema_hints and col_name in schema_hints: type = schema_hints.get(col_name).get("type") if type and type == "categorical": - new_col = pd.Series(col_val, dtype='category') + new_col = pd.Series(col_val, dtype="category") data[col_name] = new_col elif repr_meta and col_name in repr_meta: new_col = pd.Series(col_val, dtype=repr_meta[col_name]) @@ -127,7 +128,7 @@ class AnnotationsHostedTileDB(Annotations): tiledb_uri=uri, user_id=user_id, dataset_id=str(dataset_id), - schema_hints=json.dumps(dataframe_schema_type_hints) + schema_hints=json.dumps(dataframe_schema_type_hints), ) if not df.empty: self.check_category_names(df) diff --git a/server/common/app_config.py b/server/common/app_config.py deleted file mode 100644 index 3102bae0..00000000 --- a/server/common/app_config.py +++ /dev/null @@ -1,962 +0,0 @@ -import copy -import os -import sys -import warnings -from os.path import splitext, basename, isdir -from urllib.parse import urlparse, quote_plus - -import yaml -from flatten_dict import flatten, unflatten - -import server.compute.diffexp_cxg as diffexp_tiledb -import server.compute.scanpy -from server import display_version as cellxgene_display_version -from server.auth.auth import AuthTypeFactory -from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB -from server.common.annotations.local_file_csv import AnnotationsLocalFile -from server.common.data_locator import discover_s3_region_name -from server.common.default_config import get_default_config -from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure -from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available -from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType -from server.db.db_utils import DbUtils - -DEFAULT_SERVER_PORT = 5005 -# anything bigger than this will generate a special message -BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB - - -class AppFeature(object): - def __init__(self, path, available=False, method="POST", extra={}): - self.path = path - self.available = available - self.method = method - self.extra = extra - for k, v in extra.items(): - setattr(self, k, v) - - def todict(self): - d = dict(available=self.available, method=self.method, path=self.path) - d.update(self.extra) - return d - - -class AppConfig(object): - """AppConfig stores all the configuration for cellxgene. The configuration is divided into two main parts: - server attributes, and dataset attributes. The server_config contains attributes that refer to the server process - as a whole. The default_dataset_config referes to attributes that are associated with the features and - presentations of a dataset. The dataset config attributes can be overridden depending on the url by which the - dataset was accessed. These are stored in dataroot_config. - AppConfig has methods to initialize, modify, and access the configuration. - """ - - def __init__(self): - - # the default configuration (see default_config.py) - self.default_config = get_default_config() - # the server configuration - self.server_config = ServerConfig(self, self.default_config["server"]) - # the dataset config, unless overridden by an entry in dataroot_config - self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"]) - # a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot - # attribute of the server_config. - self.dataroot_config = {} - - # Set to true when config_completed is called - self.is_completed = False - - def get_dataset_config(self, dataroot_key): - if self.server_config.single_dataset__datapath: - return self.default_dataset_config - else: - return self.dataroot_config.get(dataroot_key, self.default_dataset_config) - - def check_config(self): - """Verify all the attributes have been checked""" - if not self.is_completed: - raise ConfigurationError("The configuration has not been completed") - self.server_config.check_config() - self.default_dataset_config.check_config() - for dataset_config in self.dataroot_config.values(): - dataset_config.check_config() - - def update_server_config(self, **kw): - self.server_config.update(**kw) - self.is_complete = False - - def update_default_dataset_config(self, **kw): - self.default_dataset_config.update(**kw) - # update all the other dataset configs, if any - for value in self.dataroot_config.values(): - value.update(**kw) - self.is_complete = False - - def update_from_config_file(self, config_file): - with open(config_file) as fyaml: - config = yaml.load(fyaml, Loader=yaml.FullLoader) - - if config.get("server"): - self.server_config.update_from_config(config["server"], "server") - if config.get("dataset"): - self.default_dataset_config.update_from_config(config["dataset"], "dataset") - - per_dataset_config = config.get("per_dataset_config", {}) - for key, dataroot_config in per_dataset_config.items(): - # first create and initialize the dataroot with the default config - self.add_dataroot_config(key, **config["dataset"]) - # then apply the per dataset configuration - self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}") - - self.is_complete = False - - def write_config(self, config_file): - """output the config to a yaml file""" - server = self.server_config.create_mapping(self.server_config.default_config) - dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) - config = dict(server={}, dataset={}) - for attrname in server.keys(): - config["server__" + attrname] = getattr(self.server_config, attrname) - for attrname in dataset.keys(): - config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname) - if self.dataroot_config: - config["per_dataset_config"] = {} - for dataroot_tag, dataroot_config in self.dataroot_config.items(): - dataset = dataroot_config.create_mapping(dataroot_config.default_config) - for attrname in dataset.keys(): - config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname) - - config = unflatten(config, splitter=lambda key: key.split("__")) - yaml.dump(config, open(config_file, "w")) - - def changes_from_default(self): - """Return all the attribute that are different from the default""" - diff_server = self.server_config.changes_from_default() - diff_dataset = self.default_dataset_config.changes_from_default() - diff = dict(server=diff_server, dataset=diff_dataset) - return diff - - def add_dataroot_config(self, dataroot_tag, **kw): - """Create a new dataset config object based on the default dataset config, and kw parameters""" - if dataroot_tag in self.dataroot_config: - raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}") - if type(self.server_config.multi_dataset__dataroot) != dict: - raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary") - if dataroot_tag not in self.server_config.multi_dataset__dataroot: - raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot") - - self.is_completed = False - self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"]) - flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) - config = {key: value[1] for key, value in flat_config.items()} - self.dataroot_config[dataroot_tag].update(**config) - self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag) - - def complete_config(self, messagefn=None): - """The configure options are checked, and any additional setup based on the config - parameters is done""" - - if messagefn is None: - def noop(message): - pass - - messagefn = noop - - # TODO: to give better error messages we can add a mapping between where each config - # attribute originated (e.g. command line argument or config file), then in the error - # messages we can give correct context for attributes with bad value. - context = dict(messagefn=messagefn) - - self.server_config.complete_config(context) - self.default_dataset_config.complete_config(context) - for dataroot_config in self.dataroot_config.values(): - dataroot_config.complete_config(context) - - self.is_completed = True - self.check_config() - - def get_matrix_data_cache_manager(self): - return self.server_config.matrix_data_cache_manager - - def is_multi_dataset(self): - return self.server_config.multi_dataset__dataroot is not None - - def get_title(self, data_adaptor): - return ( - self.server_config.single_dataset__title - if self.server_config.single_dataset__title - else data_adaptor.get_title() - ) - - def get_about(self, data_adaptor): - return ( - self.server_config.single_dataset__about - if self.server_config.single_dataset__about - else data_adaptor.get_about() - ) - - def get_client_config(self, data_adaptor): - """ - Return the configuration as required by the /config REST route - """ - - server_config = self.server_config - dataset_config = data_adaptor.dataset_config - annotation = dataset_config.user_annotations - auth = server_config.auth - - # FIXME The current set of config is not consistently presented: - # we have camalCase, hyphen-text, and underscore_text - - # make sure the configuration has been checked. - self.check_config() - - # features - features = [f.todict() for f in data_adaptor.get_features(annotation)] - - # display_names - title = self.get_title(data_adaptor) - about = self.get_about(data_adaptor) - - display_names = dict(engine=data_adaptor.get_name(), dataset=title) - - # library_versions - library_versions = {} - library_versions.update(data_adaptor.get_library_versions()) - library_versions["cellxgene"] = cellxgene_display_version - - # links - links = {"about-dataset": about} - - # parameters - parameters = { - "layout": dataset_config.embeddings__names, - "max-category-items": dataset_config.presentation__max_categories, - "obs_names": server_config.single_dataset__obs_names, - "var_names": server_config.single_dataset__var_names, - "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff, - "backed": server_config.adaptor__anndata_adaptor__backed, - "disable-diffexp": not dataset_config.diffexp__enable, - "enable-reembedding": dataset_config.embeddings__enable_reembedding, - "annotations": False, - "annotations_file": None, - "annotations_dir": None, - "annotations_cell_ontology_enabled": False, - "annotations_cell_ontology_obopath": None, - "annotations_cell_ontology_terms": None, - "custom_colors": dataset_config.presentation__custom_colors, - "diffexp-may-be-slow": False, - "about_legal_tos": dataset_config.app__about_legal_tos, - "about_legal_privacy": dataset_config.app__about_legal_privacy, - } - - # corpora dataset_props - # TODO/Note: putting info from the dataset into the /config is not ideal. - # However, it is definitely not part of /schema, and we do not have a top-level - # route for data properties. Consider creating one at some point. - corpora_props = data_adaptor.get_corpora_props() - if corpora_props and "default_embedding" in corpora_props: - default_embedding = corpora_props["default_embedding"] - if isinstance(default_embedding, str) and default_embedding.startswith("X_"): - default_embedding = default_embedding[2:] # drop X_ prefix - if default_embedding in data_adaptor.get_embedding_names(): - parameters["default_embedding"] = default_embedding - - data_adaptor.update_parameters(parameters) - if annotation: - annotation.update_parameters(parameters, data_adaptor) - - # gather it all together - c = {} - config = c["config"] = {} - config["features"] = features - config["displayNames"] = display_names - config["library_versions"] = library_versions - config["links"] = links - config["parameters"] = parameters - config["corpora_props"] = corpora_props - config["limits"] = { - "column_request_max": server_config.limits__column_request_max, - "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, - } - - if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): - config["authentication"] = { - "requires_client_login": auth.requires_client_login(), - } - if auth.requires_client_login(): - config["authentication"].update({ - "login": auth.get_login_url(data_adaptor), - "logout": auth.get_logout_url(data_adaptor), - }) - - return c - - def get_client_userinfo(self, data_adaptor): - """ - Return the userinfo as required by the /userinfo REST route - """ - - server_config = self.server_config - dataset_config = data_adaptor.dataset_config - auth = server_config.auth - - # make sure the configuration has been checked. - self.check_config() - - if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): - userinfo = {} - userinfo["userinfo"] = { - "is_authenticated": auth.is_user_authenticated(), - "username": auth.get_user_name(), - "user_id": auth.get_user_id() - } - return userinfo - else: - return None - - -class BaseConfig(object): - """This class handles the mechanics of updating and checking attributes. - Derived classes are expected to store the actual attributes""" - - def __init__(self, app_config, default_config, dictval_cases={}): - # reference back to the app_config - self.app_config = app_config - # the complete set of attribute and their default values (unflattened) - self.default_config = default_config - # attributes where the value may be a dict (and therefore are not flattened) - self.dictval_cases = dictval_cases - # used to make sure every attribute value is checked - self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()} - - def create_mapping(self, config): - """Create a mapping from attribute names to (location in the config tree, value)""" - dc = copy.deepcopy(config) - mapping = {} - - # special cases where the value could be a dict. - # If its value is not None, the entry is added to the mapping, and not included - # in the flattening below. - for dictval_case in self.dictval_cases: - cur = dc - for part in dictval_case[:-1]: - cur = cur.get(part, {}) - val = cur.get(dictval_case[-1]) - if val is not None: - key = "__".join(dictval_case) - mapping[key] = (dictval_case, val) - del cur[dictval_case[-1]] - - flat_config = flatten(dc) - for key, value in flat_config.items(): - # name of the attribute - attr = "__".join(key) - mapping[attr] = (key, value) - - return mapping - - def check_attr(self, attrname, vtype): - val = getattr(self, attrname) - if type(vtype) in (list, tuple): - if type(val) not in vtype: - tnames = ",".join([x.__name__ for x in vtype]) - raise ConfigurationError( - f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}" - ) - else: - if type(val) != vtype: - raise ConfigurationError( - f"Invalid type for attribute: {attrname}, " - f"expected type {vtype.__name__}, got {type(val).__name__}" - ) - - self.attr_checked[attrname] = True - - def check_config(self): - mapping = self.create_mapping(self.default_config) - for key in mapping.keys(): - if not self.attr_checked[key]: - raise ConfigurationError(f"The attr '{key}' has not been checked") - - def update(self, **kw): - for key, value in kw.items(): - if not hasattr(self, key): - raise ConfigurationError(f"unknown config parameter {key}.") - try: - if type(value) == tuple: - # convert tuple values to list values - value = list(value) - setattr(self, key, value) - except KeyError: - raise ConfigurationError(f"Unable to set config parameter {key}.") - - self.attr_checked[key] = False - - def update_from_config(self, config, prefix): - mapping = self.create_mapping(config) - for attr, (key, value) in mapping.items(): - if not hasattr(self, attr): - raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}") - try: - setattr(self, attr, value) - except KeyError: - raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}") - - self.attr_checked[attr] = False - - def changes_from_default(self): - """Return all the attribute that are different from the default""" - mapping = self.create_mapping(self.default_config) - diff = [] - for attrname, (key, defval) in mapping.items(): - curval = getattr(self, attrname) - if curval != defval: - diff.append((attrname, curval, defval)) - return diff - - -class ServerConfig(BaseConfig): - """Manages the config attribute associated with the server.""" - - def __init__(self, app_config, default_config): - dictval_cases = [ - ("app", "csp_directives"), - ("authentication", "params_oauth", "cookie"), - ("authentication", "params_oauth", "jwt_decode_options"), - ("adaptor", "cxg_adaptor", "tiledb_ctx"), - ("multi_dataset", "dataroot"), - ] - super().__init__(app_config, default_config, dictval_cases) - - dc = default_config - try: - self.app__verbose = dc["app"]["verbose"] - self.app__debug = dc["app"]["debug"] - self.app__host = dc["app"]["host"] - self.app__port = dc["app"]["port"] - self.app__open_browser = dc["app"]["open_browser"] - self.app__force_https = dc["app"]["force_https"] - self.app__flask_secret_key = dc["app"]["flask_secret_key"] - self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"] - self.app__server_timing_headers = dc["app"]["server_timing_headers"] - self.app__csp_directives = dc["app"]["csp_directives"] - self.app__api_base_url = dc["app"]["api_base_url"] - self.app__web_base_url = dc["app"]["web_base_url"] - - self.authentication__type = dc["authentication"]["type"] - self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][ - "oauth_api_base_url" - ] - self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"] - self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"] - self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][ - "jwt_decode_options"] - self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"] - self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"] - - self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"] - self.multi_dataset__index = dc["multi_dataset"]["index"] - self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"] - self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"] - self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"] - - self.single_dataset__datapath = dc["single_dataset"]["datapath"] - self.single_dataset__obs_names = dc["single_dataset"]["obs_names"] - self.single_dataset__var_names = dc["single_dataset"]["var_names"] - self.single_dataset__about = dc["single_dataset"]["about"] - self.single_dataset__title = dc["single_dataset"]["title"] - - self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"] - self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"] - self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"] - - self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"] - - self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"] - self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"] - - self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"] - self.limits__column_request_max = dc["limits"]["column_request_max"] - - except KeyError as e: - raise ConfigurationError(f"Unexpected config: {str(e)}") - - # The matrix data cache manager is created during the complete_config and stored here. - self.matrix_data_cache_manager = None - - # The authentication object - self.auth = None - - def complete_config(self, context): - self.handle_app(context) - self.handle_data_source(context) - self.handle_authentication(context) - self.handle_data_locator(context) - self.handle_adaptor(context) # may depend on data_locator - self.handle_single_dataset(context) # may depend on adaptor - self.handle_multi_dataset(context) # may depend on adaptor - self.handle_diffexp(context) - self.handle_limits(context) - - self.check_config() - - def handle_app(self, context): - self.check_attr("app__verbose", bool) - self.check_attr("app__debug", bool) - self.check_attr("app__host", str) - self.check_attr("app__port", (type(None), int)) - self.check_attr("app__open_browser", bool) - self.check_attr("app__force_https", bool) - self.check_attr("app__flask_secret_key", (type(None), str)) - self.check_attr("app__generate_cache_control_headers", bool) - self.check_attr("app__server_timing_headers", bool) - self.check_attr("app__csp_directives", (type(None), dict)) - self.check_attr("app__api_base_url", (type(None), str)) - self.check_attr("app__web_base_url", (type(None), str)) - - if self.app__port: - try: - if not is_port_available(self.app__host, self.app__port): - raise ConfigurationError( - f"The port selected {self.app__port} is in use, please configure an open port." - ) - except OverflowError: - raise ConfigurationError(f"Invalid port: {self.app__port}") - else: - try: - default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT)) - except ValueError: - raise ConfigurationError( - "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT") - ) - try: - self.app__port = find_available_port(self.app__host, default_server_port) - except OverflowError: - raise ConfigurationError(f"Invalid port: {default_server_port}") - - if self.app__debug: - context["messagefn"]("in debug mode, setting verbose=True and open_browser=False") - self.app__verbose = True - self.app__open_browser = False - else: - warnings.formatwarning = custom_format_warning - - if not self.app__verbose: - sys.tracebacklimit = 0 - - # secret key: - # first, from CXG_SECRET_KEY environment variable - # second, from config file - self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key) - - # CSP Directives are a dict of string: list(string) or string: string - if self.app__csp_directives is not None: - for k, v in self.app__csp_directives.items(): - if not isinstance(k, str): - raise ConfigurationError("CSP directive names must be a string.") - if isinstance(v, list): - for policy in v: - if not isinstance(policy, str): - raise ConfigurationError("CSP directive value must be a string or list of strings.") - elif not isinstance(v, str): - raise ConfigurationError("CSP directive value must be a string or list of strings.") - - if self.app__web_base_url is None: - self.app__web_base_url = self.app__api_base_url - - def handle_authentication(self, context): - self.check_attr("authentication__type", (type(None), str)) - - # oauth - ptypes = str if self.authentication__type == "oauth" else (type(None), str) - self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes) - self.check_attr("authentication__params_oauth__client_id", ptypes) - self.check_attr("authentication__params_oauth__client_secret", ptypes) - self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict)) - self.check_attr("authentication__params_oauth__session_cookie", bool) - - if self.authentication__params_oauth__session_cookie: - self.check_attr("authentication__params_oauth__cookie", (type(None), dict)) - else: - self.check_attr("authentication__params_oauth__cookie", dict) - # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable - # second, from config file - self.authentication__params__oauth__client_secret = os.environ.get( - "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret) - - self.auth = AuthTypeFactory.create(self.authentication__type, self) - if self.auth is None: - raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") - - def handle_data_locator(self, context): - self.check_attr("data_locator__s3__region_name", (type(None), bool, str)) - if self.data_locator__s3__region_name is True: - path = self.single_dataset__datapath or self.multi_dataset__dataroot - if type(path) == dict: - # if multi_dataset__dataroot is a dict, then use the first key - # that is in s3. NOTE: it is not supported to have dataroots - # in different regions. - paths = [val.get("dataroot") for val in path.values()] - for path in paths: - if path.startswith("s3://"): - break - if path.startswith("s3://"): - region_name = discover_s3_region_name(path) - if region_name is None: - raise ConfigurationError(f"Unable to discover s3 region name from {path}") - else: - region_name = None - self.data_locator__s3__region_name = region_name - - def handle_data_source(self, context): - self.check_attr("single_dataset__datapath", (str, type(None))) - self.check_attr("multi_dataset__dataroot", (type(None), dict, str)) - - if self.single_dataset__datapath is None: - if self.multi_dataset__dataroot is None: - # TODO: change the error message once dataroot is fully supported - raise ConfigurationError("missing datapath") - return - else: - if self.multi_dataset__dataroot is not None: - raise ConfigurationError("must supply only one of datapath or dataroot") - - def handle_single_dataset(self, context): - self.check_attr("single_dataset__datapath", (str, type(None))) - self.check_attr("single_dataset__title", (str, type(None))) - self.check_attr("single_dataset__about", (str, type(None))) - self.check_attr("single_dataset__obs_names", (str, type(None))) - self.check_attr("single_dataset__var_names", (str, type(None))) - - if self.single_dataset__datapath is None: - return - - # create the matrix data cache manager: - if self.matrix_data_cache_manager is None: - self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None) - - # preload this data set - matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config) - try: - matrix_data_loader.pre_load_validation() - except DatasetAccessError as e: - raise ConfigurationError(str(e)) - - file_size = matrix_data_loader.file_size() - file_basename = basename(self.single_dataset__datapath) - if file_size > BIG_FILE_SIZE_THRESHOLD: - context["messagefn"](f"Loading data from {file_basename}, this may take a while...") - else: - context["messagefn"](f"Loading data from {file_basename}.") - - if self.single_dataset__about: - - def url_check(url): - try: - result = urlparse(url) - if all([result.scheme, result.netloc]): - return True - else: - return False - except ValueError: - return False - - if not url_check(self.single_dataset__about): - raise ConfigurationError( - "Must provide an absolute URL for --about. (Example format: http://example.com)" - ) - - def handle_multi_dataset(self, context): - self.check_attr("multi_dataset__dataroot", (type(None), dict, str)) - self.check_attr("multi_dataset__index", (type(None), bool, str)) - self.check_attr("multi_dataset__allowed_matrix_types", list) - self.check_attr("multi_dataset__matrix_cache__max_datasets", int) - self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)) - - if self.multi_dataset__dataroot is None: - return - - if type(self.multi_dataset__dataroot) == str: - default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot) - self.multi_dataset__dataroot = dict(d=default_dict) - - for tag, dataroot_dict in self.multi_dataset__dataroot.items(): - if "base_url" not in dataroot_dict: - raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}") - if "dataroot" not in dataroot_dict: - raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}") - - base_url = dataroot_dict["base_url"] - - # sanity check for well formed base urls - bad = False - if type(base_url) != str: - bad = True - elif os.path.normpath(base_url) != base_url: - bad = True - else: - base_url_parts = base_url.split("/") - if [quote_plus(part) for part in base_url_parts] != base_url_parts: - bad = True - if ".." in base_url_parts: - bad = True - if bad: - raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}") - - # verify all the base_urls are unique - base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()] - if len(base_urls) > len(set(base_urls)): - raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique") - - # error checking - for mtype in self.multi_dataset__allowed_matrix_types: - try: - MatrixDataType(mtype) - except ValueError: - raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}') - - # create the matrix data cache manager: - if self.matrix_data_cache_manager is None: - self.matrix_data_cache_manager = MatrixDataCacheManager( - max_cached=self.multi_dataset__matrix_cache__max_datasets, - timelimit_s=self.multi_dataset__matrix_cache__timelimit_s, - ) - - def handle_diffexp(self, context): - self.check_attr("diffexp__alg_cxg__max_workers", (str, int)) - self.check_attr("diffexp__alg_cxg__cpu_multiplier", int) - self.check_attr("diffexp__alg_cxg__target_workunit", int) - - max_workers = self.diffexp__alg_cxg__max_workers - cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier - cpu_count = os.cpu_count() - max_workers = min(max_workers, cpu_multiplier * cpu_count) - diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit) - - def handle_adaptor(self, context): - # cxg - self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict) - regionkey = "vfs.s3.region" - if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx: - if type(self.data_locator__s3__region_name) == str: - self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name - - from server.data_cxg.cxg_adaptor import CxgAdaptor - - CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx) - - # anndata - self.check_attr("adaptor__anndata_adaptor__backed", bool) - - def handle_limits(self, context): - self.check_attr("limits__diffexp_cellcount_max", (type(None), int)) - self.check_attr("limits__column_request_max", (type(None), int)) - - def exceeds_limit(self, limit_name, value): - limit_value = getattr(self, "limits__" + limit_name, None) - if limit_value is None: # disabled - return False - return value > limit_value - - def get_api_base_url(self): - if self.app__api_base_url == "local": - return f"http://{self.app__host}:{self.app__port}" - if self.app__api_base_url and self.app__api_base_url.endswith("/"): - return self.app__api_base_url[:-1] - return self.app__api_base_url - - def get_web_base_url(self): - if self.app__web_base_url == "local": - return f"http://{self.app__host}:{self.app__port}" - if self.app__web_base_url is None: - return self.get_api_base_url() - if self.app__web_base_url.endswith("/"): - return self.app__web_base_url[:-1] - return self.api__web_base_url - - -class DatasetConfig(BaseConfig): - """Manages the config attribute associated with a dataset.""" - - def __init__(self, tag, app_config, default_config): - super().__init__(app_config, default_config) - self.tag = tag - dc = default_config - try: - self.app__scripts = dc["app"]["scripts"] - self.app__inline_scripts = dc["app"]["inline_scripts"] - self.app__about_legal_tos = dc["app"]["about_legal_tos"] - self.app__about_legal_privacy = dc["app"]["about_legal_privacy"] - self.app__authentication_enable = dc["app"]["authentication_enable"] - - self.presentation__max_categories = dc["presentation"]["max_categories"] - self.presentation__custom_colors = dc["presentation"]["custom_colors"] - - self.user_annotations__enable = dc["user_annotations"]["enable"] - self.user_annotations__type = dc["user_annotations"]["type"] - self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"] - self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"] - self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"] - self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"] - self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"] - self.user_annotations__hosted_tiledb_array__hosted_file_directory = \ - dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ] # noqa E501 - - self.embeddings__names = dc["embeddings"]["names"] - self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"] - - self.diffexp__enable = dc["diffexp"]["enable"] - self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"] - self.diffexp__top_n = dc["diffexp"]["top_n"] - - except KeyError as e: - raise ConfigurationError(f"Unexpected config: {str(e)}") - - # The annotation object is created during complete_config and stored here. - self.user_annotations = None - - def complete_config(self, context): - self.handle_app(context) - self.handle_presentation(context) - self.handle_user_annotations(context) - self.handle_embeddings(context) - self.handle_diffexp(context) - - def handle_app(self, context): - self.check_attr("app__scripts", list) - self.check_attr("app__inline_scripts", list) - self.check_attr("app__about_legal_tos", (type(None), str)) - self.check_attr("app__about_legal_privacy", (type(None), str)) - self.check_attr("app__authentication_enable", bool) - - # scripts can be string (filename) or dict (attributes). Convert string to dict. - scripts = [] - for s in self.app__scripts: - if isinstance(s, str): - scripts.append({"src": s}) - elif isinstance(s, dict) and isinstance(s["src"], str): - scripts.append(s) - else: - raise ConfigurationError("Scripts must be string or dict") - self.app__scripts = scripts - - def handle_presentation(self, context): - self.check_attr("presentation__max_categories", int) - self.check_attr("presentation__custom_colors", bool) - - def handle_user_annotations(self, context): - self.check_attr("user_annotations__enable", bool) - self.check_attr("user_annotations__type", str) - self.check_attr("user_annotations__local_file_csv__directory", (type(None), str)) - self.check_attr("user_annotations__local_file_csv__file", (type(None), str)) - self.check_attr("user_annotations__ontology__enable", bool) - self.check_attr("user_annotations__ontology__obo_location", (type(None), str)) - self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str)) - self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)) - - if self.user_annotations__enable: - server_config = self.app_config.server_config - if not self.app__authentication_enable: - raise ConfigurationError("user annotations requires authentication to be enabled") - if not server_config.auth.is_valid_authentication_type(): - auth_type = server_config.authentication__type - raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") - - # TODO, replace this with a factory pattern once we have more than one way - # to do annotations. currently only local_file_csv - if self.user_annotations__type == "local_file_csv": - dirname = self.user_annotations__local_file_csv__directory - filename = self.user_annotations__local_file_csv__file - - if filename is not None and dirname is not None: - raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.") - - if filename is not None: - lf_name, lf_ext = splitext(filename) - if lf_ext and lf_ext != ".csv": - raise ConfigurationError(f"annotation file type must be .csv: {filename}") - - if dirname is not None and not isdir(dirname): - try: - os.mkdir(dirname) - except OSError: - raise ConfigurationError("Unable to create directory specified by --annotations-dir") - - self.user_annotations = AnnotationsLocalFile(dirname, filename) - - # if the user has specified a fixed label file, go ahead and validate it - # so that we can remove errors early in the process. - server_config = self.app_config.server_config - if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file: - with server_config.matrix_data_cache_manager.data_adaptor( - self.tag, server_config.single_dataset__datapath, self.app_config - ) as data_adaptor: - data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) - - if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: - try: - self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) - except OntologyLoadFailure as e: - raise ConfigurationError("Unable to load ontology terms\n" + str(e)) - elif self.user_annotations__type == "hosted_tiledb_array": - self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str) - self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str) - self.user_annotations = AnnotationsHostedTileDB( - directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory, - db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri), - ) - else: - raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array') - else: - if self.user_annotations__type == "local_file_csv": - dirname = self.user_annotations__local_file_csv__directory - filename = self.user_annotations__local_file_csv__file - if filename is not None: - context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.") - if dirname is not None: - context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.") - - if self.user_annotations__ontology__enable: - context["messagefn"]( - "Warning: --experimental-annotations-ontology" " ignored as annotations are disabled." - ) - if self.user_annotations__ontology__obo_location is not None: - context["messagefn"]( - "Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled." - ) - - def handle_embeddings(self, context): - self.check_attr("embeddings__names", list) - self.check_attr("embeddings__enable_reembedding", bool) - - server_config = self.app_config.server_config - if self.embeddings__enable_reembedding: - if server_config.single_dataset__datapath: - matrix_data_loader = MatrixDataLoader( - server_config.single_dataset__datapath, app_config=self.app_config - ) - if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD: - raise ConfigurationError("enable-reembedding is only supported with H5AD files.") - if server_config.adaptor__anndata_adaptor__backed: - raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.") - - try: - server.compute.scanpy.get_scanpy_module() - except NotImplementedError: - raise ConfigurationError("Please install scanpy to enable UMAP re-embedding") - - def handle_diffexp(self, context): - self.check_attr("diffexp__enable", bool) - self.check_attr("diffexp__lfc_cutoff", float) - self.check_attr("diffexp__top_n", int) - - server_config = self.app_config.server_config - if server_config.single_dataset__datapath: - with server_config.matrix_data_cache_manager.data_adaptor( - self.tag, server_config.single_dataset__datapath, self.app_config - ) as data_adaptor: - if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): - context["messagefn"]( - "CAUTION: due to the size of your dataset, " - "running differential expression may take longer or fail." - ) diff --git a/server/common/aws_secret_utils.py b/server/common/aws_secret_utils.py index 47f7690b..070ed160 100644 --- a/server/common/aws_secret_utils.py +++ b/server/common/aws_secret_utils.py @@ -1,72 +1,11 @@ import logging -import os -import sys import boto3 from flask import json -from server.common.data_locator import discover_s3_region_name from server.common.errors import SecretKeyRetrievalError -def handle_config_from_secret(app_config): - """Update configuration from the secret manager""" - secret_name = os.getenv("CXG_AWS_SECRET_NAME") - if not secret_name: - return - - # need to find the secret manager region. - # 1. from CXG_AWS_SECRET_REGION_NAME - # 2. discover from dataroot location (if on s3) - # 3. discover from config file location (if on s3) - secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME") - if secret_region_name is None: - secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) - if not secret_region_name: - from server.eb.app import config_file - secret_region_name = discover_s3_region_name(config_file) - if not secret_region_name: - logging.error("Could not determine the AWS Secret Manager region") - sys.exit(1) - - secrets = get_secret_key(secret_region_name, secret_name) - - if not secrets: - return - - server_attrs = ( - ("flask_secret_key", "app__flask_secret_key"), - ("oauth_client_secret", "authentication__params_oauth__client_secret"), - ) - default_dataset_attrs = ( - ("db_uri", "user_annotations__hosted_tiledb_array__db_uri"), - ) - - # update server configuration attributes - for key, attr in server_attrs: - cur_val = getattr(app_config.server_config, attr) - if cur_val: - continue - - # replace the attr with the secret if it is not set - val = secrets.get(key) - if val: - logging.info(f"set {attr} from secret") - app_config.update_server_config(**{attr : val}) - - # update default dataset configuration attributes - for key, attr in default_dataset_attrs: - cur_val = getattr(app_config.default_dataset_config, attr) - if cur_val: - continue - - # replace the attr with the secret if it is not set - val = secrets.get(key) - if val: - logging.info(f"set {attr} from secret") - app_config.update_default_dataset_config(**{attr : val}) - - def get_secret_key(region_name, secret_name): session = boto3.session.Session() client = session.client(service_name="secretsmanager", region_name=region_name) diff --git a/server/common/config/__init__.py b/server/common/config/__init__.py new file mode 100644 index 00000000..fb2439e7 --- /dev/null +++ b/server/common/config/__init__.py @@ -0,0 +1,66 @@ +import logging +import os +import sys + +from server.common.aws_secret_utils import get_secret_key +from server.common.data_locator import discover_s3_region_name + +DEFAULT_SERVER_PORT = 5005 +BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB + + +def handle_config_from_secret(app_config): + """Update configuration from the secret manager""" + secret_name = os.getenv("CXG_AWS_SECRET_NAME") + if not secret_name: + return + + # need to find the secret manager region. + # 1. from CXG_AWS_SECRET_REGION_NAME + # 2. discover from dataroot location (if on s3) + # 3. discover from config file location (if on s3) + secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME") + if secret_region_name is None: + secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) + if not secret_region_name: + from server.eb.app import config_file + + secret_region_name = discover_s3_region_name(config_file) + if not secret_region_name: + logging.error("Could not determine the AWS Secret Manager region") + sys.exit(1) + + secrets = get_secret_key(secret_region_name, secret_name) + + if not secrets: + return + + server_attrs = ( + ("flask_secret_key", "app__flask_secret_key"), + ("oauth_client_secret", "authentication__params_oauth__client_secret"), + ) + default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),) + + # update server configuration attributes + for key, attr in server_attrs: + cur_val = getattr(app_config.server_config, attr) + if cur_val: + continue + + # replace the attr with the secret if it is not set + val = secrets.get(key) + if val: + logging.info(f"set {attr} from secret") + app_config.update_server_config(**{attr: val}) + + # update default dataset configuration attributes + for key, attr in default_dataset_attrs: + cur_val = getattr(app_config.default_dataset_config, attr) + if cur_val: + continue + + # replace the attr with the secret if it is not set + val = secrets.get(key) + if val: + logging.info(f"set {attr} from secret") + app_config.update_default_dataset_config(**{attr: val}) diff --git a/server/common/config/app_config.py b/server/common/config/app_config.py new file mode 100644 index 00000000..422dcb8e --- /dev/null +++ b/server/common/config/app_config.py @@ -0,0 +1,183 @@ +import yaml +from flatten_dict import unflatten + +from server.default_config import get_default_config +from server.common.config.dataset_config import DatasetConfig +from server.common.config.server_config import ServerConfig +from server.common.errors import ConfigurationError + + +class AppConfig(object): + """ + AppConfig stores all the configuration for cellxgene. + AppConfig contains one or more DatasetConfig(s) and one ServerConfig. + The server_config contains attributes that refer to the server process as a whole. + The default_dataset_config refers to attributes that are associated with the features and + presentations of a dataset. + The dataset config attributes can be overridden depending on the url by which the + dataset was accessed. These are stored in dataroot_config. + AppConfig has methods to initialize, modify, and access the configuration. + """ + + def __init__(self): + + # the default configuration (see default_config.py) + # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as + # defaults within the config class? + self.default_config = get_default_config() + # the server configuration + self.server_config = ServerConfig(self, self.default_config["server"]) + # the dataset config, unless overridden by an entry in dataroot_config + self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"]) + # a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot + # attribute of the server_config. The default dataset config will apply to all datasets unless a different set + # of config vars was passed for a specific dataset under the multidataset config. For example: + """ + per_dataset_config: + d1: + user_annotations: + enable: false + d2: + user_annotations: + enable: true + """ + # dataroot config + self.dataroot_config = {} + + # Set to true when config_completed is called + self.is_completed = False + + def get_dataset_config(self, dataroot_key): + if self.server_config.single_dataset__datapath: + return self.default_dataset_config + else: + return self.dataroot_config.get(dataroot_key, self.default_dataset_config) + + def check_config(self): + """Verify all the attributes in the config have been type checked""" + if not self.is_completed: + raise ConfigurationError("The configuration has not been completed") + self.server_config.check_config() + self.default_dataset_config.check_config() + for dataset_config in self.dataroot_config.values(): + dataset_config.check_config() + + def update_server_config(self, **kw): + self.server_config.update(**kw) + self.is_complete = False + + def update_default_dataset_config(self, **kw): + self.default_dataset_config.update(**kw) + # update all the other dataset configs, if any + for value in self.dataroot_config.values(): + value.update(**kw) + self.is_complete = False + + def update_from_config_file(self, config_file): + try: + with open(config_file) as yml_file: + config = yaml.safe_load(yml_file) + except yaml.YAMLError as e: + raise ConfigurationError(f"The specified config file contained an error: {e}") + except OSError as e: + raise ConfigurationError(f"Issue retrieving the specified config file: {e}") + + if config.get("server"): + self.server_config.update_from_config(config["server"], "server") + if config.get("dataset"): + self.default_dataset_config.update_from_config(config["dataset"], "dataset") + + per_dataset_config = config.get("per_dataset_config", {}) + for key, dataroot_config in per_dataset_config.items(): + # first create and initialize the dataroot with the default config + self.add_dataroot_config(key, **config["dataset"]) + # then apply the per dataset configuration + self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}") + + self.is_complete = False + + def write_config(self, config_file): + """output the config to a yaml file""" + server = self.server_config.create_mapping(self.server_config.default_config) + dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) + config = dict(server={}, dataset={}) + for attrname in server.keys(): + config["server__" + attrname] = getattr(self.server_config, attrname) + for attrname in dataset.keys(): + config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname) + if self.dataroot_config: + config["per_dataset_config"] = {} + for dataroot_tag, dataroot_config in self.dataroot_config.items(): + dataset = dataroot_config.create_mapping(dataroot_config.default_config) + for attrname in dataset.keys(): + config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname) + + config = unflatten(config, splitter=lambda key: key.split("__")) + yaml.dump(config, open(config_file, "w")) + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + diff_server = self.server_config.changes_from_default() + diff_dataset = self.default_dataset_config.changes_from_default() + diff = dict(server=diff_server, dataset=diff_dataset) + return diff + + def add_dataroot_config(self, dataroot_tag, **kw): + """Create a new dataset config object based on the default dataset config, and kw parameters""" + if dataroot_tag in self.dataroot_config: + raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}") + if type(self.server_config.multi_dataset__dataroot) != dict: + raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary") + if dataroot_tag not in self.server_config.multi_dataset__dataroot: + raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot") + + self.is_completed = False + self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"]) + flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) + config = {key: value[1] for key, value in flat_config.items()} + self.dataroot_config[dataroot_tag].update(**config) + self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag) + + def complete_config(self, messagefn=None): + """The configure options are checked, and any additional setup based on the config + parameters is done""" + + if messagefn is None: + + def noop(message): + pass + + messagefn = noop + + # TODO: to give better error messages we can add a mapping between where each config + # attribute originated (e.g. command line argument or config file), then in the error + # messages we can give correct context for attributes with bad value. + context = dict(messagefn=messagefn) + + self.server_config.complete_config(context) + self.default_dataset_config.complete_config(context) + for dataroot_config in self.dataroot_config.values(): + dataroot_config.complete_config(context) + + self.is_completed = True + self.check_config() + + def get_matrix_data_cache_manager(self): + return self.server_config.matrix_data_cache_manager + + def is_multi_dataset(self): + return self.server_config.multi_dataset__dataroot is not None + + def get_title(self, data_adaptor): + return ( + self.server_config.single_dataset__title + if self.server_config.single_dataset__title + else data_adaptor.get_title() + ) + + def get_about(self, data_adaptor): + return ( + self.server_config.single_dataset__about + if self.server_config.single_dataset__about + else data_adaptor.get_about() + ) diff --git a/server/common/config/base_config.py b/server/common/config/base_config.py new file mode 100644 index 00000000..a0b46f1b --- /dev/null +++ b/server/common/config/base_config.py @@ -0,0 +1,113 @@ +import copy + +from flatten_dict import flatten +from server.common.errors import ConfigurationError + + +class BaseConfig(object): + """ + This class handles the mechanics of updating and checking attributes. + Derived classes are expected to store the actual attributes + Currently DatasetConfig and ServerConfig both inherit from BaseConfig. + """ + + def __init__(self, app_config, default_config, dictval_cases={}): + # reference back to the app_config + self.app_config = app_config + # the complete set of attributes and their default values (unflattened) + self.default_config = default_config + # attributes where the value may be a dict (and therefore are not flattened) + self.dictval_cases = dictval_cases + # used to make sure every attribute value is checked + self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()} + + def create_mapping(self, config): + """ + Create a dictionary where the keys are the name of attributes (using double underscore convention) + For example: authentication__type + + The values are a tuple, + - the first item of the tuple is a tuple of path elements (location in config 'tree') + - the second item is the value of the config parameter + + For example: (('authentication', 'type'), 'session')) + """ + config_copy = copy.deepcopy(config) + mapping = {} + + # special cases where the value could be a dict. + # If its value is not None, the entry is added to the mapping, and not included + # in the flattening below. + for dictval_case in self.dictval_cases: + cur = config_copy + for part in dictval_case[:-1]: + cur = cur.get(part, {}) + val = cur.get(dictval_case[-1]) + if val is not None: + key = "__".join(dictval_case) + mapping[key] = (dictval_case, val) + del cur[dictval_case[-1]] + + flat_config = flatten(config_copy) + for key, value in flat_config.items(): + # name of the attribute + attr = "__".join(key) + mapping[attr] = (key, value) + + return mapping + + def validate_correct_type_of_configuration_attribute(self, attrname, vtype): + val = getattr(self, attrname) + if type(vtype) in (list, tuple): + if type(val) not in vtype: + tnames = ",".join([x.__name__ for x in vtype]) + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}" + ) + else: + if type(val) != vtype: + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, " + f"expected type {vtype.__name__}, got {type(val).__name__}" + ) + + self.attr_checked[attrname] = True + + def check_config(self): + mapping = self.create_mapping(self.default_config) + for key in mapping.keys(): + if not self.attr_checked[key]: + raise ConfigurationError(f"The attr '{key}' has not been checked") + + def update(self, **kw): + for key, value in kw.items(): + if not hasattr(self, key): + raise ConfigurationError(f"unknown config parameter {key}.") + try: + if type(value) == tuple: + # convert tuple values to list values + value = list(value) + setattr(self, key, value) + except KeyError: + raise ConfigurationError(f"Unable to set config parameter {key}.") + + self.attr_checked[key] = False + + def update_from_config(self, config, prefix): + mapping = self.create_mapping(config) + for attr, (key, value) in mapping.items(): + if not hasattr(self, attr): + raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}") + setattr(self, attr, value) + + self.attr_checked[attr] = False + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + mapping = self.create_mapping(self.default_config) + diff = [] + for attrname, (key, defval) in mapping.items(): + curval = getattr(self, attrname) + if curval != defval: + diff.append((attrname, curval, defval)) + return diff diff --git a/server/common/config/client_config.py b/server/common/config/client_config.py new file mode 100644 index 00000000..ccddf2c1 --- /dev/null +++ b/server/common/config/client_config.py @@ -0,0 +1,121 @@ +from server import display_version as cellxgene_display_version + + +def get_client_config(app_config, data_adaptor): + """ + Return the configuration as required by the /config REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + annotation = dataset_config.user_annotations + auth = server_config.auth + + # FIXME The current set of config is not consistently presented: + # we have camalCase, hyphen-text, and underscore_text + + # make sure the configuration has been checked. + app_config.check_config() + + # display_names + title = app_config.get_title(data_adaptor) + about = app_config.get_about(data_adaptor) + + display_names = dict(engine=data_adaptor.get_name(), dataset=title) + + # library_versions + library_versions = {} + library_versions.update(data_adaptor.get_library_versions()) + library_versions["cellxgene"] = cellxgene_display_version + + # links + links = {"about-dataset": about} + + # parameters + parameters = { + "layout": dataset_config.embeddings__names, + "max-category-items": dataset_config.presentation__max_categories, + "obs_names": server_config.single_dataset__obs_names, + "var_names": server_config.single_dataset__var_names, + "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff, + "backed": server_config.adaptor__anndata_adaptor__backed, + "disable-diffexp": not dataset_config.diffexp__enable, + "enable-reembedding": dataset_config.embeddings__enable_reembedding, + "annotations": False, + "annotations_file": None, + "annotations_dir": None, + "annotations_cell_ontology_enabled": False, + "annotations_cell_ontology_obopath": None, + "annotations_cell_ontology_terms": None, + "custom_colors": dataset_config.presentation__custom_colors, + "diffexp-may-be-slow": False, + "about_legal_tos": dataset_config.app__about_legal_tos, + "about_legal_privacy": dataset_config.app__about_legal_privacy, + } + + # corpora dataset_props + # TODO/Note: putting info from the dataset into the /config is not ideal. + # However, it is definitely not part of /schema, and we do not have a top-level + # route for data properties. Consider creating one at some point. + corpora_props = data_adaptor.get_corpora_props() + if corpora_props and "default_embedding" in corpora_props: + default_embedding = corpora_props["default_embedding"] + if isinstance(default_embedding, str) and default_embedding.startswith("X_"): + default_embedding = default_embedding[2:] # drop X_ prefix + if default_embedding in data_adaptor.get_embedding_names(): + parameters["default_embedding"] = default_embedding + + data_adaptor.update_parameters(parameters) + if annotation: + annotation.update_parameters(parameters, data_adaptor) + + # gather it all together + client_config = {} + config = client_config["config"] = {} + config["displayNames"] = display_names + config["library_versions"] = library_versions + config["links"] = links + config["parameters"] = parameters + config["corpora_props"] = corpora_props + config["limits"] = { + "column_request_max": server_config.limits__column_request_max, + "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, + } + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + config["authentication"] = { + "requires_client_login": auth.requires_client_login(), + } + if auth.requires_client_login(): + config["authentication"].update( + { + # Todo why are these stored on the data_adaptor? + "login": auth.get_login_url(data_adaptor), + "logout": auth.get_logout_url(data_adaptor), + } + ) + + return client_config + + +def get_client_userinfo(app_config, data_adaptor): + """ + Return the userinfo as required by the /userinfo REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + auth = server_config.auth + + # make sure the configuration has been checked. + app_config.check_config() + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + userinfo = {} + userinfo["userinfo"] = { + "is_authenticated": auth.is_user_authenticated(), + "username": auth.get_user_name(), + "user_id": auth.get_user_id(), + "email": auth.get_user_email(), + } + return userinfo diff --git a/server/common/config/dataset_config.py b/server/common/config/dataset_config.py new file mode 100644 index 00000000..8c8231ac --- /dev/null +++ b/server/common/config/dataset_config.py @@ -0,0 +1,234 @@ +import os +from os.path import splitext, isdir + +from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB +from server.common.annotations.local_file_csv import AnnotationsLocalFile +from server.common.config.base_config import BaseConfig +from server.common.errors import ConfigurationError, OntologyLoadFailure +from server.compute.scanpy import get_scanpy_module +from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType +from server.db.db_utils import DbUtils + + +class DatasetConfig(BaseConfig): + """Manages the config attribute associated with a dataset.""" + + def __init__(self, tag, app_config, default_config): + super().__init__(app_config, default_config) + self.tag = tag + try: + self.app__scripts = default_config["app"]["scripts"] + self.app__inline_scripts = default_config["app"]["inline_scripts"] + self.app__about_legal_tos = default_config["app"]["about_legal_tos"] + self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"] + self.app__authentication_enable = default_config["app"]["authentication_enable"] + + self.presentation__max_categories = default_config["presentation"]["max_categories"] + self.presentation__custom_colors = default_config["presentation"]["custom_colors"] + + self.user_annotations__enable = default_config["user_annotations"]["enable"] + self.user_annotations__type = default_config["user_annotations"]["type"] + self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][ + "directory" + ] # noqa E501 + self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"] + self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"] + self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][ + "obo_location" + ] # noqa E501 + self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][ + "hosted_tiledb_array" + ][ + "db_uri" + ] # noqa E501 + self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][ + "hosted_tiledb_array" + ][ + "hosted_file_directory" + ] # noqa E501 + + self.embeddings__names = default_config["embeddings"]["names"] + self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"] + + self.diffexp__enable = default_config["diffexp"]["enable"] + self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"] + self.diffexp__top_n = default_config["diffexp"]["top_n"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + # The annotation object is created during complete_config and stored here. + self.user_annotations = None + + def complete_config(self, context): + self.handle_app() + self.handle_presentation() + self.handle_user_annotations(context) + self.handle_embeddings() + self.handle_diffexp(context) + + def handle_app(self): + self.validate_correct_type_of_configuration_attribute("app__scripts", list) + self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list) + self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool) + + # scripts can be string (filename) or dict (attributes). Convert string to dict. + scripts = [] + for script in self.app__scripts: + try: + if isinstance(script, str): + scripts.append({"src": script}) + elif isinstance(script, dict) and isinstance(script["src"], str): + scripts.append(script) + else: + raise Exception + except Exception as e: + raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}") + + self.app__scripts = scripts + + def handle_presentation(self): + self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int) + self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool) + + def handle_user_annotations(self, context): + self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool) + self.validate_correct_type_of_configuration_attribute("user_annotations__type", str) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__directory", (type(None), str) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__file", (type(None), str) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__ontology__obo_location", (type(None), str) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute( + "user_annotations__hosted_tiledb_array__db_uri", (type(None), str) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute( + "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str) + ) # noqa E501 + if self.user_annotations__enable: + server_config = self.app_config.server_config + if not self.app__authentication_enable: + raise ConfigurationError("user annotations requires authentication to be enabled") + if not server_config.auth.is_valid_authentication_type(): + auth_type = server_config.authentication__type + raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") + + if self.user_annotations__type == "local_file_csv": + self.handle_local_file_csv_annotations() + elif self.user_annotations__type == "hosted_tiledb_array": + self.handle_hosted_tiledb_annotations() + else: + raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array') + if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: + try: + self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) + except OntologyLoadFailure as e: + raise ConfigurationError("Unable to load ontology terms\n" + str(e)) + else: + self.check_annotation_config_vars_not_set(context) + + def handle_local_file_csv_annotations(self): + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + if filename is not None and dirname is not None: + raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.") + + if filename is not None: + lf_name, lf_ext = splitext(filename) + if lf_ext and lf_ext != ".csv": + raise ConfigurationError(f"annotation file type must be .csv: {filename}") + + if dirname is not None and not isdir(dirname): + try: + os.mkdir(dirname) + except OSError: + raise ConfigurationError("Unable to create directory specified by --annotations-dir") + + self.user_annotations = AnnotationsLocalFile(dirname, filename) + + # if the user has specified a fixed label file, go ahead and validate it + # so that we can remove errors early in the process. + server_config = self.app_config.server_config + if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file: + with server_config.matrix_data_cache_manager.data_adaptor( + self.tag, server_config.single_dataset__datapath, self.app_config + ) as data_adaptor: + data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) + + def handle_hosted_tiledb_annotations(self): + self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__hosted_tiledb_array__hosted_file_directory", str + ) # noqa E501 + self.user_annotations = AnnotationsHostedTileDB( + directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory, + db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri), + ) + + def check_annotation_config_vars_not_set(self, context): + if self.user_annotations__type is not None: + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + db_uri = self.user_annotations__hosted_tiledb_array__db_uri + hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory + if filename is not None: + context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.") + if dirname is not None: + context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.") + if db_uri is not None: + context["messagefn"]("Warning: db_uri ignored as annotations are disabled.") + if hosted_file_dirname is not None: + context["messagefn"]( + "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled." + ) + + if self.user_annotations__ontology__enable: + context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.") + if self.user_annotations__ontology__obo_location is not None: + context["messagefn"]( + "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled." + ) + + def handle_embeddings(self): + self.validate_correct_type_of_configuration_attribute("embeddings__names", list) + self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool) + + server_config = self.app_config.server_config + if self.embeddings__enable_reembedding: + if server_config.single_dataset__datapath: + matrix_data_loader = MatrixDataLoader( + server_config.single_dataset__datapath, app_config=self.app_config + ) + if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD: + raise ConfigurationError("enable-reembedding is only supported with H5AD files.") + if server_config.adaptor__anndata_adaptor__backed: + raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.") + + try: + get_scanpy_module() + except NotImplementedError: + # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported + raise ConfigurationError("Please install scanpy to enable UMAP re-embedding") + + def handle_diffexp(self, context): + self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool) + self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float) + self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int) + + server_config = self.app_config.server_config + if server_config.single_dataset__datapath: + with server_config.matrix_data_cache_manager.data_adaptor( + self.tag, server_config.single_dataset__datapath, self.app_config + ) as data_adaptor: + if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): + context["messagefn"]( + "CAUTION: due to the size of your dataset, " + "running differential expression may take longer or fail." + ) diff --git a/server/common/config/server_config.py b/server/common/config/server_config.py new file mode 100644 index 00000000..508b3d4f --- /dev/null +++ b/server/common/config/server_config.py @@ -0,0 +1,390 @@ +import os +import sys +import warnings +from os.path import basename +from urllib.parse import urlparse, quote_plus + +from server.auth.auth import AuthTypeFactory +from server.common.config.base_config import BaseConfig +from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD +from server.common.errors import ConfigurationError, DatasetAccessError +from server.common.data_locator import discover_s3_region_name +from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning +from server.compute import diffexp_cxg as diffexp_tiledb +from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType + + +class ServerConfig(BaseConfig): + """Manages the config attribute associated with the server.""" + + def __init__(self, app_config, default_config): + dictval_cases = [ + ("app", "csp_directives"), + ("authentication", "params_oauth", "cookie"), + ("authentication", "params_oauth", "jwt_decode_options"), + ("adaptor", "cxg_adaptor", "tiledb_ctx"), + ("multi_dataset", "dataroot"), + ] + super().__init__(app_config, default_config, dictval_cases) + + try: + self.app__verbose = default_config["app"]["verbose"] + self.app__debug = default_config["app"]["debug"] + self.app__host = default_config["app"]["host"] + self.app__port = default_config["app"]["port"] + self.app__open_browser = default_config["app"]["open_browser"] + self.app__force_https = default_config["app"]["force_https"] + self.app__flask_secret_key = default_config["app"]["flask_secret_key"] + self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"] + self.app__server_timing_headers = default_config["app"]["server_timing_headers"] + self.app__csp_directives = default_config["app"]["csp_directives"] + self.app__api_base_url = default_config["app"]["api_base_url"] + self.app__web_base_url = default_config["app"]["web_base_url"] + + self.authentication__type = default_config["authentication"]["type"] + self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][ + "oauth_api_base_url" + ] # noqa E501 + self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"] + self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][ + "client_secret" + ] # noqa E501 + self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][ + "jwt_decode_options" + ] # noqa E501 + self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][ + "session_cookie" + ] # noqa E501 + self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"] + + self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"] + self.multi_dataset__index = default_config["multi_dataset"]["index"] + self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"] + self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][ + "max_datasets" + ] # noqa E501 + self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][ + "timelimit_s" + ] # noqa E501 + + self.single_dataset__datapath = default_config["single_dataset"]["datapath"] + self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"] + self.single_dataset__var_names = default_config["single_dataset"]["var_names"] + self.single_dataset__about = default_config["single_dataset"]["about"] + self.single_dataset__title = default_config["single_dataset"]["title"] + + self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"] + self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"] + self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"] + + self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"] + + self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"] + self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"] + + self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"] + self.limits__column_request_max = default_config["limits"]["column_request_max"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + # The matrix data cache manager is created during the complete_config and stored here. + self.matrix_data_cache_manager = None + + # The authentication object + self.auth = None + + def complete_config(self, context): + self.handle_app(context) + self.handle_data_source() + self.handle_authentication() + self.handle_data_locator() + self.handle_adaptor() # may depend on data_locator + self.handle_single_dataset(context) # may depend on adaptor + self.handle_multi_dataset() # may depend on adaptor + self.handle_diffexp() + self.handle_limits() + + self.check_config() + + def handle_app(self, context): + self.validate_correct_type_of_configuration_attribute("app__verbose", bool) + self.validate_correct_type_of_configuration_attribute("app__debug", bool) + self.validate_correct_type_of_configuration_attribute("app__host", str) + self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("app__open_browser", bool) + self.validate_correct_type_of_configuration_attribute("app__force_https", bool) + self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool) + self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool) + self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict)) + self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str)) + + if self.app__port: + try: + if not is_port_available(self.app__host, self.app__port): + raise ConfigurationError( + f"The port selected {self.app__port} is in use, please configure an open port." + ) + except OverflowError: + raise ConfigurationError(f"Invalid port: {self.app__port}") + else: + try: + default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT)) + except ValueError: + raise ConfigurationError( + "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT") + ) + try: + self.app__port = find_available_port(self.app__host, default_server_port) + except OverflowError: + raise ConfigurationError(f"Invalid port: {default_server_port}") + + if self.app__debug: + context["messagefn"]("in debug mode, setting verbose=True and open_browser=False") + self.app__verbose = True + self.app__open_browser = False + else: + warnings.formatwarning = custom_format_warning + + if not self.app__verbose: + sys.tracebacklimit = 0 + + # secret key: + # first, from CXG_SECRET_KEY environment variable + # second, from config file + self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key) + + # CSP Directives are a dict of string: list(string) or string: string + if self.app__csp_directives is not None: + for k, v in self.app__csp_directives.items(): + if not isinstance(k, str): + raise ConfigurationError("CSP directive names must be a string.") + if isinstance(v, list): + for policy in v: + if not isinstance(policy, str): + raise ConfigurationError("CSP directive value must be a string or list of strings.") + elif not isinstance(v, str): + raise ConfigurationError("CSP directive value must be a string or list of strings.") + + if self.app__web_base_url is None: + self.app__web_base_url = self.app__api_base_url + + def handle_authentication(self): + self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str)) + + # oauth + ptypes = str if self.authentication__type == "oauth" else (type(None), str) + self.validate_correct_type_of_configuration_attribute( + "authentication__params_oauth__oauth_api_base_url", ptypes + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes) + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes) + self.validate_correct_type_of_configuration_attribute( + "authentication__params_oauth__jwt_decode_options", (type(None), dict) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool) + + if self.authentication__params_oauth__session_cookie: + self.validate_correct_type_of_configuration_attribute( + "authentication__params_oauth__cookie", (type(None), dict) + ) # noqa E501 + else: + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict) + # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable + # second, from config file + self.authentication__params_oauth__client_secret = os.environ.get( + "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret + ) + + self.auth = AuthTypeFactory.create(self.authentication__type, self) + if self.auth is None: + raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") + + def handle_data_locator(self): + self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str)) + if self.data_locator__s3__region_name is True: + path = self.single_dataset__datapath or self.multi_dataset__dataroot + + if type(path) == dict: + # if multi_dataset__dataroot is a dict, then use the first key + # that is in s3. NOTE: it is not supported to have dataroots + # in different regions. + paths = [val.get("dataroot") for val in path.values()] + for path in paths: + if path.startswith("s3://"): + break + if path.startswith("s3://"): + region_name = discover_s3_region_name(path) + if region_name is None: + raise ConfigurationError(f"Unable to discover s3 region name from {path}") + else: + region_name = None + self.data_locator__s3__region_name = region_name + + def handle_data_source(self): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str)) + + if self.single_dataset__datapath and self.multi_dataset__dataroot: + raise ConfigurationError( + "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both" + ) + if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None: + raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets") + + def handle_single_dataset(self, context): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None))) + + if self.single_dataset__datapath is None: + return + + # create the matrix data cache manager: + if self.matrix_data_cache_manager is None: + self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None) + + # preload this data set + matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config) + try: + matrix_data_loader.pre_load_validation() + except DatasetAccessError as e: + raise ConfigurationError(str(e)) + + file_size = matrix_data_loader.file_size() + file_basename = basename(self.single_dataset__datapath) + if file_size > BIG_FILE_SIZE_THRESHOLD: + context["messagefn"](f"Loading data from {file_basename}, this may take a while...") + else: + context["messagefn"](f"Loading data from {file_basename}.") + + if self.single_dataset__about: + + def url_check(url): + try: + result = urlparse(url) + if all([result.scheme, result.netloc]): + return True + else: + return False + except ValueError: + return False + + if not url_check(self.single_dataset__about): + raise ConfigurationError( + "Must provide an absolute URL for --about. (Example format: http://example.com)" + ) + + def handle_multi_dataset(self): + self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str)) + self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str)) + self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list) + self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int) + self.validate_correct_type_of_configuration_attribute( + "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float) + ) # noqa E501 + + if self.multi_dataset__dataroot is None: + return + + if type(self.multi_dataset__dataroot) == str: + default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot) + self.multi_dataset__dataroot = dict(d=default_dict) + + for tag, dataroot_dict in self.multi_dataset__dataroot.items(): + if "base_url" not in dataroot_dict: + raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}") + if "dataroot" not in dataroot_dict: + raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}") + + base_url = dataroot_dict["base_url"] + + # sanity check for well formed base urls + bad = False + if type(base_url) != str: + bad = True + elif os.path.normpath(base_url) != base_url: + bad = True + else: + base_url_parts = base_url.split("/") + if [quote_plus(part) for part in base_url_parts] != base_url_parts: + bad = True + if ".." in base_url_parts: + bad = True + if bad: + raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}") + + # verify all the base_urls are unique + base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()] + if len(base_urls) > len(set(base_urls)): + raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique") + + # error checking + for mtype in self.multi_dataset__allowed_matrix_types: + try: + MatrixDataType(mtype) + except ValueError: + raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}') + + # create the matrix data cache manager: + if self.matrix_data_cache_manager is None: + self.matrix_data_cache_manager = MatrixDataCacheManager( + max_cached=self.multi_dataset__matrix_cache__max_datasets, + timelimit_s=self.multi_dataset__matrix_cache__timelimit_s, + ) + + def handle_diffexp(self): + self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int)) + self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int) + self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int) + + max_workers = self.diffexp__alg_cxg__max_workers + cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier + cpu_count = os.cpu_count() + max_workers = min(max_workers, cpu_multiplier * cpu_count) + diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit) + + def handle_adaptor(self): + # cxg + self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict) + regionkey = "vfs.s3.region" + if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx: + if type(self.data_locator__s3__region_name) == str: + self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name + + from server.data_cxg.cxg_adaptor import CxgAdaptor + + CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx) + + # anndata + self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool) + + def handle_limits(self): + self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int)) + + def exceeds_limit(self, limit_name, value): + limit_value = getattr(self, "limits__" + limit_name, None) + if limit_value is None: # disabled + return False + return value > limit_value + + def get_api_base_url(self): + if self.app__api_base_url == "local": + return f"http://{self.app__host}:{self.app__port}" + if self.app__api_base_url and self.app__api_base_url.endswith("/"): + return self.app__api_base_url[:-1] + return self.app__api_base_url + + def get_web_base_url(self): + if self.app__web_base_url == "local": + return f"http://{self.app__host}:{self.app__port}" + if self.app__web_base_url is None: + return self.get_api_base_url() + if self.app__web_base_url.endswith("/"): + return self.app__web_base_url[:-1] + return self.app__web_base_url diff --git a/server/common/errors.py b/server/common/errors.py index 5e8281e6..ca339bee 100644 --- a/server/common/errors.py +++ b/server/common/errors.py @@ -42,14 +42,14 @@ define_request_exception( define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota") define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format") define_request_exception( - "AuthenticationError", - "Raised when there is an authentication error", - default_status_code=HTTPStatus.UNAUTHORIZED) + "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED +) define_request_exception( "AnnotationCategoryNameError", "Raised when an annotation category name cant be saved", - default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY) + default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY, +) define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails") define_exception("ConfigurationError", "Raised when checking configuration errors") diff --git a/server/common/rest.py b/server/common/rest.py index be099f94..9baeca4c 100644 --- a/server/common/rest.py +++ b/server/common/rest.py @@ -6,6 +6,7 @@ from http import HTTPStatus from flask import make_response, jsonify, current_app, abort from werkzeug.urls import url_unquote +from server.common.config.client_config import get_client_config, get_client_userinfo from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg from server.common.errors import ( FilterError, @@ -117,12 +118,12 @@ def schema_get(data_adaptor): def config_get(app_config, data_adaptor): - config = app_config.get_client_config(data_adaptor) + config = get_client_config(app_config, data_adaptor) return make_response(jsonify(config), HTTPStatus.OK) def userinfo_get(app_config, data_adaptor): - config = app_config.get_client_userinfo(data_adaptor) + config = get_client_userinfo(app_config, data_adaptor) return make_response(jsonify(config), HTTPStatus.OK) diff --git a/server/common/utils/cxg_generation_utils.py b/server/common/utils/cxg_generation_utils.py index f29f4bd0..d3ec2b40 100644 --- a/server/common/utils/cxg_generation_utils.py +++ b/server/common/utils/cxg_generation_utils.py @@ -111,7 +111,7 @@ def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx): def convert_matrix_to_cxg_array( - matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None + matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None ): """ Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array` diff --git a/server/common/utils/matrix_utils.py b/server/common/utils/matrix_utils.py index 3eeddc10..60dfb19b 100644 --- a/server/common/utils/matrix_utils.py +++ b/server/common/utils/matrix_utils.py @@ -41,16 +41,19 @@ def is_matrix_sparse(matrix: np.ndarray, sparse_threshold): number_of_non_zero_elements += np.count_nonzero(matrix_subset) if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix: if end_row_index != total_number_of_rows: - percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / ( - end_row_index * total_number_of_columns) + percentage_of_non_zero_elements = ( + 100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns) + ) logging.info( f"Matrix is not sparse. Percentage of non-zero elements (estimate): " - f"{percentage_of_non_zero_elements:6.2f}") + f"{percentage_of_non_zero_elements:6.2f}" + ) else: percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements logging.info( f"Matrix is not sparse. Percentage of non-zero elements (exact): " - f"{percentage_of_non_zero_elements:6.2f}") + f"{percentage_of_non_zero_elements:6.2f}" + ) return False is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold diff --git a/server/common/utils/type_conversion_utils.py b/server/common/utils/type_conversion_utils.py index ac6b3fe4..4bc6bf88 100644 --- a/server/common/utils/type_conversion_utils.py +++ b/server/common/utils/type_conversion_utils.py @@ -9,8 +9,10 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame): schema_type_hints_by_column_name = {} for column_name, column_values in dataframe.items(): - dtypes_by_column_name[column_name], schema_type_hints_by_column_name[column_name] = \ - get_dtype_and_schema_of_array(column_values) + ( + dtypes_by_column_name[column_name], + schema_type_hints_by_column_name[column_name], + ) = get_dtype_and_schema_of_array(column_values) return dtypes_by_column_name, schema_type_hints_by_column_name @@ -24,8 +26,10 @@ def get_schema_type_hint_of_array(array: pd.Series): def get_dtype_and_schema_of_array(array: pd.Series): - return (get_dtype_from_dtype(array.dtype, array_values=array), - get_schema_type_hint_from_dtype(array.dtype, array_values=array)) + return ( + get_dtype_from_dtype(array.dtype, array_values=array), + get_schema_type_hint_from_dtype(array.dtype, array_values=array), + ) def get_dtype_from_dtype(dtype, array_values=None): @@ -133,9 +137,11 @@ def can_cast_to_int32(dtype, array_values=None): if np.can_cast(dtype, np.int32): return True ii32 = np.iinfo(np.int32) - if not ordered_array_values.empty and ( - ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \ - ordered_array_values.empty: + if ( + not ordered_array_values.empty + and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) + or ordered_array_values.empty + ): return True return False diff --git a/server/converters/h5ad_data_file.py b/server/converters/h5ad_data_file.py index c8223822..c79dd785 100644 --- a/server/converters/h5ad_data_file.py +++ b/server/converters/h5ad_data_file.py @@ -24,14 +24,14 @@ class H5ADDataFile: another format (currently just CXG is supported). """ def __init__( - self, - input_filename, - backed=False, - dataset_title=None, - dataset_about=None, - obs_index_column_name=None, - vars_index_column_name=None, - use_corpora_schema=True, + self, + input_filename, + backed=False, + dataset_title=None, + dataset_about=None, + obs_index_column_name=None, + vars_index_column_name=None, + use_corpora_schema=True, ): self.input_filename = input_filename self.backed = backed diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index 20dafdd0..36af8339 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd from server_timing import Timing as ServerTiming -from server.common.app_config import AppFeature, AppConfig +from server.common.config.app_config import AppConfig from server.common.constants import Axis from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError from server.common.utils.utils import jsonify_numpy @@ -155,17 +155,6 @@ class DataAdaptor(metaclass=ABCMeta): """ pass - def get_features(self, annotations=None): - """Return list of features, to return as part of the config route""" - features = [ - AppFeature("/cluster/", method="POST", available=False), - AppFeature("/layout/obs", method="GET", available=self.get_embedding_names() is not None), - AppFeature("/layout/obs", method="PUT", available=self.dataset_config.embeddings__enable_reembedding), - AppFeature("/diffexp/", method="POST", available=self.dataset_config.diffexp__enable), - AppFeature("/annotations/obs", method="PUT", available=annotations is not None), - ] - return features - def update_parameters(self, parameters): parameters.update(self.parameters) @@ -173,7 +162,7 @@ class DataAdaptor(metaclass=ABCMeta): mask = np.zeros((count,), dtype=np.bool) for i in filter: if type(i) == list: - mask[i[0]: i[1]] = True + mask[i[0] : i[1]] = True else: mask[i] = True return mask @@ -314,7 +303,7 @@ class DataAdaptor(metaclass=ABCMeta): top_n = self.dataset_config.diffexp__top_n if self.server_config.exceeds_limit( - "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) + "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) ): raise ExceedsLimitError("Diffexp request exceeds max cell count limit") diff --git a/server/db/cellxgene_orm.py b/server/db/cellxgene_orm.py index f2209b63..2860f6b1 100644 --- a/server/db/cellxgene_orm.py +++ b/server/db/cellxgene_orm.py @@ -1,11 +1,6 @@ import uuid -from sqlalchemy import ( - Column, - DateTime, - ForeignKey, - String, - func, JSON) +from sqlalchemy import Column, DateTime, ForeignKey, String, func, JSON from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship diff --git a/server/db/db_utils.py b/server/db/db_utils.py index 1664303f..b3030bf2 100644 --- a/server/db/db_utils.py +++ b/server/db/db_utils.py @@ -42,9 +42,9 @@ class DbUtils: def get_or_create_dataset(self, dataset_name): try: - dataset_id = self.query( - table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name] - )[0].id + dataset_id = self.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])[ + 0 + ].id except IndexError: dataset_id = uuid.uuid4() dataset = CellxGeneDataset(id=dataset_id, name=dataset_name) @@ -54,9 +54,7 @@ class DbUtils: def get_or_create_user(self, user_id): try: - user_id = self.query( - table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id] - )[0].id + user_id = self.query(table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id])[0].id except IndexError: user = CellxGeneUser(id=user_id) self.session.add(user) diff --git a/server/common/default_config.py b/server/default_config.py similarity index 99% rename from server/common/default_config.py rename to server/default_config.py index 16e0c27d..88e96fe5 100644 --- a/server/common/default_config.py +++ b/server/default_config.py @@ -204,7 +204,6 @@ dataset: enable: true lfc_cutoff: 0.01 top_n: 10 - """ diff --git a/server/eb/app.py b/server/eb/app.py index 7a5e834c..0844f599 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -9,7 +9,7 @@ from flask import json import logging from flask_talisman import Talisman from flask_cors import CORS -from server.common.aws_secret_utils import handle_config_from_secret +from server.common.config import handle_config_from_secret from server.common.errors import SecretKeyRetrievalError @@ -26,7 +26,7 @@ SERVERDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.append(SERVERDIR) try: - from server.common.app_config import AppConfig + from server.common.config.app_config import AppConfig from server.app.app import Server from server.common.data_locator import DataLocator, discover_s3_region_name except Exception: @@ -61,8 +61,7 @@ class WSGIServer(Server): csp = { "default-src": ["'self'"], "connect-src": ["'self'"] + extra_connect_src, - "script-src": ["'self'", "'unsafe-eval'"] - + obsolete_browser_script_hash + script_hashes, + "script-src": ["'self'", "'unsafe-eval'"] + obsolete_browser_script_hash + script_hashes, "style-src": ["'self'", "'unsafe-inline'"], "img-src": ["'self'", "https://cellxgene.cziscience.com", "data:"], "object-src": ["'none'"], @@ -104,7 +103,7 @@ class WSGIServer(Server): if len(script_hashes) == 0: logging.error("Content security policy hashes are missing, falling back to unsafe-inline policy") - return (script_hashes) + return script_hashes @staticmethod def compute_inline_csp_hashes(app, app_config): @@ -173,9 +172,7 @@ try: sys.exit(1) # features are unsupported in the current hosted server - app_config.update_default_dataset_config( - embeddings__enable_reembedding=False, - ) + app_config.update_default_dataset_config(embeddings__enable_reembedding=False,) app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],) app_config.complete_config(logging.info) diff --git a/server/test/__init__.py b/server/test/__init__.py index 8ed6e3e7..bbb1de0b 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -13,7 +13,8 @@ import requests from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB from server.common.annotations.local_file_csv import AnnotationsLocalFile -from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT +from server.common.config.app_config import AppConfig +from server.common.config import DEFAULT_SERVER_PORT from server.common.data_locator import DataLocator from server.common.utils.utils import find_available_port from server.data_common.fbs.matrix import encode_matrix_fbs @@ -33,8 +34,7 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType): data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( - multi_dataset__dataroot=data_locator.path, - authentication__type="test", + multi_dataset__dataroot=data_locator.path, authentication__type="test", ) config.update_default_dataset_config( embeddings__names=["umap"], @@ -42,16 +42,13 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType): diffexp__lfc_cutoff=0.01, user_annotations__type="hosted_tiledb_array", user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432", - user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir + user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir, ) config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) - annotations = AnnotationsHostedTileDB( - tmp_dir, - DbUtils("postgresql://postgres:test_pw@localhost:5432"), - ) + annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),) return data, tmp_dir, annotations diff --git a/server/test/fixtures/database/__init__.py b/server/test/fixtures/database/__init__.py index 5a7fa984..0088f6d4 100644 --- a/server/test/fixtures/database/__init__.py +++ b/server/test/fixtures/database/__init__.py @@ -29,34 +29,26 @@ class TestDatabase: def _create_test_user(self): user = CellxGeneUser(id="test_user_id") - user2 = CellxGeneUser(id='1234') + user2 = CellxGeneUser(id="1234") self.db.session.add(user) self.db.session.add(user2) self.db.session.commit() def _create_test_dataset(self): - dataset = CellxGeneDataset( - name="test_dataset", - ) + dataset = CellxGeneDataset(name="test_dataset",) self.db.session.add(dataset) self.db.session.commit() def _create_test_annotation(self): - dataset = self.db.query([CellxGeneDataset], - [CellxGeneDataset.name == "test_dataset"], - )[0] - annotation = Annotation( - tiledb_uri="tiledb_uri", - user_id="test_user_id", - dataset_id=str(dataset.id) - ) + dataset = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == "test_dataset"],)[0] + annotation = Annotation(tiledb_uri="tiledb_uri", user_id="test_user_id", dataset_id=str(dataset.id)) self.db.session.add(annotation) self.db.session.commit() @staticmethod def get_random_string(): letters = string.ascii_lowercase - return ''.join(random.choice(letters) for i in range(12)) + return "".join(random.choice(letters) for i in range(12)) def _create_test_users(self, user_count: int = 10): users = [] @@ -80,10 +72,8 @@ class TestDatabase: for i in range(annotation_count): dataset = self.order_by_random(CellxGeneDataset) user = self.order_by_random(CellxGeneUser) - annotations.append(Annotation( - tiledb_uri=self.get_random_string(), - user_id=user.id, - dataset_id=str(dataset.id) - )) + annotations.append( + Annotation(tiledb_uri=self.get_random_string(), user_id=user.id, dataset_id=str(dataset.id)) + ) self.db.session.add_all(annotations) self.db.session.commit() diff --git a/server/test/fixtures/dataset_config_outline.py b/server/test/fixtures/dataset_config_outline.py new file mode 100644 index 00000000..a3d99d78 --- /dev/null +++ b/server/test/fixtures/dataset_config_outline.py @@ -0,0 +1,37 @@ +f""" +dataset: + app: + scripts: {scripts} #list of strs (filenames) or dicts containing keys + inline_scripts: {inline_scripts} #list of strs (filenames) + + about_legal_tos: {about_legal_tos} + about_legal_privacy: {about_legal_privacy} + + authentication_enable: {authentication_enable} + + presentation: + max_categories: {max_categories} + custom_colors: {custom_colors} + + user_annotations: + enable: {enable_users_annotations} + type: {annotation_type} + hosted_tiledb_array: + db_uri: {db_uri} + hosted_file_directory: {hosted_file_directory} + local_file_csv: + directory: {local_file_csv_directory} + file: {local_file_csv_file} + ontology: + enable: {ontology_enabled} + obo_location: {obo_location} + + embeddings: + names: {embedding_names} + enable_reembedding: {enable_reembedding} + + diffexp: + enable: {enable_difexp} + lfc_cutoff: {lfc_cutoff} + top_n: {top_n} +""" diff --git a/server/test/fixtures/server_config_outline.py b/server/test/fixtures/server_config_outline.py new file mode 100644 index 00000000..4bbb4675 --- /dev/null +++ b/server/test/fixtures/server_config_outline.py @@ -0,0 +1,62 @@ +f"""server: + app: + verbose: {verbose} + debug: {debug} + host: {host} + port: {port} + open_browser: {open_browser} + force_https: {force_https} + flask_secret_key: {flask_secret_key} + generate_cache_control_headers: {generate_cache_control_headers} + server_timing_headers: {server_timing_headers} + csp_directives: {csp_directives} + api_base_url: {api_base_url} + web_base_url: {web_base_url} + authentication: + type: {auth_type} + params_oauth: + oauth_api_base_url: {oauth_api_base_url} + client_id: {client_id} + client_secret: {client_secret} + jwt_decode_options: {jwt_decode_options} + session_cookie: {session_cookie} + cookie: {cookie} + + multi_dataset: + dataroot: {dataroot} + index: {index} + allowed_matrix_types: {allowed_matrix_types} + matrix_cache: + max_datasets: {max_cached_datasets} + timelimit_s: {timelimit_s} + + single_dataset: + datapath: {dataset_datapath} + obs_names: {obs_names} + var_names: {var_names} + about: {about} + title: {title} + + diffexp: + alg_cxg: # number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count) + max_workers: {diffexp_max_workers} + cpu_multiplier: {cpu_multiplier} + target_workunit: {target_workunit} # The target number of matrix elements that are evaluated in one thread. + + data_locator: + s3: + region_name: {data_locater_region_name} + + adaptor: + cxg_adaptor: + tiledb_ctx: + sm.tile_cache_size: {cxg_tile_cache_size} + sm.num_reader_threads: {cxg_num_reader_threads} + + anndata_adaptor: + backed: {anndata_backed} + + limits: + column_request_max: {column_request_max} + diffexp_cellcount_max: {diffexp_cellcount_max} +""" diff --git a/server/test/performance/run_diffexp.py b/server/test/performance/run_diffexp.py index bbbb3094..5cd1fbef 100644 --- a/server/test/performance/run_diffexp.py +++ b/server/test/performance/run_diffexp.py @@ -7,7 +7,7 @@ import numpy as np import server.compute.diffexp_cxg as diffexp_cxg import server.compute.diffexp_generic as diffexp_generic -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.data_common.matrix_loader import MatrixDataLoader from server.data_cxg.cxg_adaptor import CxgAdaptor diff --git a/server/test/test_database/test_database.py b/server/test/test_database/test_database.py index 4ecb4f27..91798fdb 100644 --- a/server/test/test_database/test_database.py +++ b/server/test/test_database/test_database.py @@ -16,45 +16,48 @@ class DatabaseTest(unittest.TestCase): del cls.db def test_user_creation(self): - one_user = self.db.get(table=CellxGeneUser, entity_id='test_user_id') - self.assertEqual(one_user.id, 'test_user_id') + one_user = self.db.get(table=CellxGeneUser, entity_id="test_user_id") + self.assertEqual(one_user.id, "test_user_id") user_count = self.db.session.query(CellxGeneUser).count() self.assertGreater(user_count, 10) def test_dataset_creation(self): - one_dataset = self.db.query(table_args=[CellxGeneDataset], - filter_args=[CellxGeneDataset.name == 'test_dataset']) - self.assertEqual(one_dataset[0].name, 'test_dataset') + one_dataset = self.db.query( + table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"] + ) + self.assertEqual(one_dataset[0].name, "test_dataset") dataset_count = self.db.session.query(CellxGeneDataset).count() self.assertGreater(dataset_count, 10) def test_annotation_creation(self): - one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0] - self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri') + one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == "tiledb_uri"])[0] + self.assertEqual(one_annotation.tiledb_uri, "tiledb_uri") annotation_count = self.db.session.query(Annotation).count() self.assertGreater(annotation_count, 10) def test_get_most_recent_annotation_for_user_dataset(self): - dataset_id = str(self.db.query(table_args=[CellxGeneDataset], - filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id) + dataset_id = str( + self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"])[0].id + ) # have to commit separately because created_at time written on the db server - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_0")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_1")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_2")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_3")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_4")) self.db.session.commit() - most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id, - Annotation.user_id == 'test_user_id']) + most_recent_annotation = self.db.query_for_most_recent( + Annotation, [Annotation.dataset_id == dataset_id, Annotation.user_id == "test_user_id"] + ) - self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4') + self.assertEqual(most_recent_annotation.tiledb_uri, "tiledb_uri_4") diff --git a/server/test/unit/auth/test_auth.py b/server/test/unit/auth/test_auth.py index b07e38d3..f7f6f358 100644 --- a/server/test/unit/auth/test_auth.py +++ b/server/test/unit/auth/test_auth.py @@ -2,7 +2,7 @@ import unittest import requests -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.test import FIXTURES_ROOT, test_server @@ -12,9 +12,7 @@ class AuthTest(unittest.TestCase): def test_auth_none(self): c = AppConfig() - c.update_server_config( - authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot - ) + c.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot) c.update_default_dataset_config(user_annotations__enable=False) c.complete_config() @@ -28,9 +26,7 @@ class AuthTest(unittest.TestCase): def test_auth_session(self): c = AppConfig() - c.update_server_config( - authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot - ) + c.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot) c.update_default_dataset_config(user_annotations__enable=True) c.complete_config() @@ -107,8 +103,8 @@ class AuthTest(unittest.TestCase): def test_auth_test_single(self): c = AppConfig() c.update_server_config( - authentication__type="test", - single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg") + authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg" + ) c.complete_config() diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py index 42734a7b..8b250ed6 100644 --- a/server/test/unit/auth/test_oauth.py +++ b/server/test/unit/auth/test_oauth.py @@ -9,7 +9,7 @@ from flask import Flask, jsonify, make_response, request, redirect from multiprocessing import Process import jose -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.test import FIXTURES_ROOT, test_server # This tests the oauth authentication type. @@ -46,7 +46,7 @@ def token(): "scope": "openid profile email", "expires_in": TOKEN_EXPIRES, "token_type": "Bearer", - "expires_at": expires_at + "expires_at": expires_at, } return make_response(jsonify(r)) @@ -89,9 +89,8 @@ class AuthTest(unittest.TestCase): authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}", authentication__params_oauth__client_id="mock_client_id", authentication__params_oauth__client_secret="mock_client_secret", - authentication__params_oauth__jwt_decode_options={ - "verify_signature": False, "verify_iss": False - }) + authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False}, + ) app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot) app_config.complete_config() @@ -112,7 +111,7 @@ class AuthTest(unittest.TestCase): logout_uri = config["config"]["authentication"]["logout"] self.assertEqual(login_uri, f"{server}/login?dataset=d/pbmc3k.cxg/") - self.assertEqual(logout_uri, f"{server}/logout") + self.assertEqual(logout_uri, f"{server}/logout?dataset=d/pbmc3k.cxg/") r = session.get(login_uri) # check that the login redirect worked @@ -122,6 +121,7 @@ class AuthTest(unittest.TestCase): userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertTrue(userinfo["userinfo"]["is_authenticated"]) self.assertEqual(userinfo["userinfo"]["username"], "fake_user") + self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com") self.assertTrue(config["config"]["parameters"]["annotations"]) if cookie_key: @@ -150,7 +150,7 @@ class AuthTest(unittest.TestCase): r = session.get(logout_uri) # check that the logout redirect worked self.assertEqual(r.history[0].status_code, 302) - self.assertEqual(r.url, f"{server}") + self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/") config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json() userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json() self.assertFalse(userinfo["userinfo"]["is_authenticated"]) @@ -160,9 +160,7 @@ class AuthTest(unittest.TestCase): def test_auth_oauth_session(self): # test with session cookies app_config = AppConfig() - app_config.update_server_config( - authentication__params_oauth__session_cookie=True, - ) + app_config.update_server_config(authentication__params_oauth__session_cookie=True,) self.auth_flow(app_config) def test_auth_oauth_cookie(self): diff --git a/server/test/unit/cli/test_launch.py b/server/test/unit/cli/test_launch.py new file mode 100644 index 00000000..ac388cc0 --- /dev/null +++ b/server/test/unit/cli/test_launch.py @@ -0,0 +1,28 @@ +import filecmp +import os +import shutil +import unittest + +import yaml + +from server.default_config import default_config +from server.test import FIXTURES_ROOT + + +class CLIPLaunchTests(unittest.TestCase): + tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs") + + @classmethod + def setUpClass(cls) -> None: + os.mkdir(cls.tmp_dir) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_dir) + + +def test_dump_default_config(self): + os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt") + with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config: + expected_config.write(yaml.dump(default_config)) + filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt") diff --git a/server/test/unit/common/config/__init__.py b/server/test/unit/common/config/__init__.py new file mode 100644 index 00000000..fa730df2 --- /dev/null +++ b/server/test/unit/common/config/__init__.py @@ -0,0 +1,246 @@ +import os +import shutil +import unittest +import random +from unittest import mock + +from server.test import FIXTURES_ROOT + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class ConfigTests(unittest.TestCase): + tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir") + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_fixtures_directory) + + @classmethod + def setUpClass(cls) -> None: + os.makedirs(cls.tmp_fixtures_directory) + + def custom_server_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="null", + generate_cache_control_headers="false", + server_timing_headers="false", + csp_directives="null", + api_base_url="null", + web_base_url="null", + auth_type="session", + oauth_api_base_url="null", + client_id="null", + client_secret="null", + jwt_decode_options="null", + session_cookie="true", + cookie="null", + dataroot="null", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + diffexp_max_workers=64, + cpu_multiplier=4, + target_workunit="16_000_000", + data_locater_region_name="us-east-1", + cxg_tile_cache_size=8589934592, + cxg_num_reader_threads=32, + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + config_file_name="server_config.yaml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py") + with open(server_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + server_config = eval(config) + with open(configfile, "w") as server_config_file: + server_config_file.write(server_config) + return configfile + + def custom_app_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="null", + generate_cache_control_headers="false", + server_timing_headers="false", + csp_directives="null", + api_base_url="null", + web_base_url="null", + auth_type="session", + oauth_api_base_url="null", + client_id="null", + client_secret="null", + jwt_decode_options="null", + session_cookie="true", + cookie="null", + dataroot="null", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + diffexp_max_workers=64, + cpu_multiplier=4, + target_workunit="16_000_000", + data_locater_region_name="us-east-1", + cxg_tile_cache_size=8589934592, + cxg_num_reader_threads=32, + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + scripts=[], + inline_scripts=[], + about_legal_tos="null", + about_legal_privacy="null", + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + ontology_enabled="false", + obo_location="null", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + config_file_name="app_config.yml", + ): + random_num = random.randrange(999999) + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config = self.custom_server_config( + verbose=verbose, + debug=debug, + host=host, + port=port, + open_browser=open_browser, + force_https=force_https, + flask_secret_key=flask_secret_key, + generate_cache_control_headers=generate_cache_control_headers, + server_timing_headers=server_timing_headers, + csp_directives=csp_directives, + api_base_url=api_base_url, + web_base_url=web_base_url, + auth_type=auth_type, + oauth_api_base_url=oauth_api_base_url, + client_id=client_id, + client_secret=client_secret, + jwt_decode_options=jwt_decode_options, + session_cookie=session_cookie, + cookie=cookie, + dataroot=dataroot, + index=index, + allowed_matrix_types=allowed_matrix_types, + max_cached_datasets=max_cached_datasets, + timelimit_s=timelimit_s, + dataset_datapath=dataset_datapath, + obs_names=obs_names, + var_names=var_names, + about=about, + title=title, + diffexp_max_workers=diffexp_max_workers, + cpu_multiplier=cpu_multiplier, + target_workunit=target_workunit, + data_locater_region_name=data_locater_region_name, + cxg_tile_cache_size=cxg_tile_cache_size, + cxg_num_reader_threads=cxg_num_reader_threads, + anndata_backed=anndata_backed, + column_request_max=column_request_max, + diffexp_cellcount_max=diffexp_cellcount_max, + config_file_name=f"temp_server_config_{random_num}.yml", + ) + dataset_config = self.custom_dataset_config( + scripts=scripts, + inline_scripts=inline_scripts, + about_legal_tos=about_legal_tos, + about_legal_privacy=about_legal_privacy, + authentication_enable=authentication_enable, + max_categories=max_categories, + custom_colors=custom_colors, + enable_users_annotations=enable_users_annotations, + annotation_type=annotation_type, + db_uri=db_uri, + hosted_file_directory=hosted_file_directory, + local_file_csv_directory=local_file_csv_directory, + local_file_csv_file=local_file_csv_file, + ontology_enabled=ontology_enabled, + obo_location=obo_location, + embedding_names=embedding_names, + enable_reembedding=enable_reembedding, + enable_difexp=enable_difexp, + lfc_cutoff=lfc_cutoff, + top_n=top_n, + config_file_name=f"temp_dataset_config_{random_num}.yml", + ) + with open(server_config) as server_config: + with open(dataset_config) as dataset_config: + with open(configfile, "w") as app_config_file: + for line in server_config: + app_config_file.write(line) + for line in dataset_config: + app_config_file.write(line) + + return configfile + + def custom_dataset_config( + self, + scripts=[], + inline_scripts=[], + about_legal_tos="null", + about_legal_privacy="null", + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + ontology_enabled="false", + obo_location="null", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + config_file_name="dataset_config.yml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py") + with open(dataset_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + dataset_config = eval(config) + with open(configfile, "w") as dataset_config_file: + dataset_config_file.write(dataset_config) + + return configfile diff --git a/server/test/unit/common/config/test_app_config.py b/server/test/unit/common/config/test_app_config.py new file mode 100644 index 00000000..2dc3273d --- /dev/null +++ b/server/test/unit/common/config/test_app_config.py @@ -0,0 +1,140 @@ +import os +import tempfile +import unittest + +import yaml + +from server.default_config import default_config +from server.common.config.app_config import AppConfig +from server.test.unit.common.config import ConfigTests +from server.common.errors import ConfigurationError +from server.test import FIXTURES_ROOT + + +class AppConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_get_default_config_correctly_reads_default_config_file(self): + app_default_config = AppConfig().default_config + + expected_config = yaml.load(default_config, Loader=yaml.Loader) + + server_config = app_default_config["server"] + dataset_config = app_default_config["dataset"] + + expected_server_config = expected_config["server"] + expected_dataset_config = expected_config["dataset"] + + self.assertDictEqual(app_default_config, expected_config) + self.assertDictEqual(server_config, expected_server_config) + self.assertDictEqual(dataset_config, expected_dataset_config) + + def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self): + datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad" + file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name) + config = AppConfig() + config.update_from_config_file(file_name) + + self.assertEqual(config.get_dataset_config(""), config.default_dataset_config) + + def test_update_server_config_updates_server_config_and_config_status(self): + config = self.get_config() + config.complete_config() + config.check_config() + config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + with self.assertRaises(ConfigurationError): + config.server_config.check_config() + + def test_write_config_outputs_yaml_with_all_config_vars(self): + config = self.get_config() + config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml") + with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config: + default_config_yml = yaml.safe_load(default_config) + + with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config: + output_config_yml = yaml.safe_load(output_config) + self.maxDiff = None + self.assertEqual(default_config_yml, output_config_yml) + + def test_update_app_config(self): + config = AppConfig() + config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir") + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)]) + + config = AppConfig() + config.update_default_dataset_config(app__scripts=(), app__inline_scripts=()) + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[]) + vars = config.default_dataset_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) + vars = config.default_dataset_config.changes_from_default() + self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) + + def test_configfile_no_dataset_section(self): + # test a config file without a dataset section + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + server: + multi_dataset: + dataroot: test_dataroot + + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + server_changes = app_config.server_config.changes_from_default() + dataset_changes = app_config.default_dataset_config.changes_from_default() + self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)]) + self.assertEqual(dataset_changes, []) + + def test_configfile_no_server_section(self): + # test a config file without a dataset section + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + dataset: + user_annotations: + enable: false + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + server_changes = app_config.server_config.changes_from_default() + dataset_changes = app_config.default_dataset_config.changes_from_default() + self.assertEqual(server_changes, []) + self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)]) diff --git a/server/test/unit/common/config/test_base_config.py b/server/test/unit/common/config/test_base_config.py new file mode 100644 index 00000000..d41082cf --- /dev/null +++ b/server/test/unit/common/config/test_base_config.py @@ -0,0 +1,63 @@ +import unittest + +from server.common.config.app_config import AppConfig +from server.test import FIXTURES_ROOT +from server.test.unit.common.config import ConfigTests +from server.common.errors import ConfigurationError + + +class BaseConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_mapping_creation_returns_map_of_server_and_dataset_config(self): + config = AppConfig() + mapping = config.default_dataset_config.create_mapping(config.default_config) + self.assertIsNotNone(mapping["server__app__verbose"]) + self.assertIsNotNone(mapping["dataset__presentation__max_categories"]) + self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"]) + self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"]) + + def test_changes_from_default_returns_list_of_nondefault_config_values(self): + config = self.get_config(verbose="true", lfc_cutoff=0.05) + server_changes = config.server_config.changes_from_default() + dataset_changes = config.default_dataset_config.changes_from_default() + + self.assertEqual( + server_changes, + [ + ("app__verbose", True, False), + ("multi_dataset__dataroot", FIXTURES_ROOT, None), + ("multi_dataset__matrix_cache__timelimit_s", 5, 30), + ("data_locator__s3__region_name", "us-east-1", True), + ], + ) + self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)]) + + def test_check_config_throws_error_if_attr_has_not_been_checked(self): + config = self.get_config(verbose="true") + config.complete_config() + config.check_config() + config.update_server_config(app__verbose=False) + with self.assertRaises(ConfigurationError): + config.check_config() diff --git a/server/test/unit/common/config/test_dataset_config.py b/server/test/unit/common/config/test_dataset_config.py new file mode 100644 index 00000000..a32d1f87 --- /dev/null +++ b/server/test/unit/common/config/test_dataset_config.py @@ -0,0 +1,260 @@ +import os +import tempfile + +import requests +import unittest +from unittest.mock import patch + +from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB +from server.common.annotations.local_file_csv import AnnotationsLocalFile +from server.common.config.app_config import AppConfig +from server.common.config.base_config import BaseConfig +from server.test import test_server, PROJECT_ROOT, FIXTURES_ROOT + +from server.common.errors import ConfigurationError +from server.test.unit.common.config import ConfigTests + + +class TestDatasetConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.dataset_config = self.config.default_dataset_config + self.config.complete_config() + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_datatset_config_sets_vars_from_default_config(self): + config = AppConfig() + self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000) + self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv") + self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01) + self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location) + + @patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.dataset_config.complete_config(self.context) + self.assertEqual(mock_check_attrs.call_count, 21) + + def test_app_sets_script_vars(self): + config = self.get_config(scripts=["path/to/script"]) + config.default_dataset_config.handle_app() + + self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}]) + + config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}]) + config.default_dataset_config.handle_app() + self.assertEqual( + config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}] + ) + + config = self.get_config(scripts=["path/to/script", "different/script/path"]) + config.default_dataset_config.handle_app() + # TODO @madison -- is this the desired functionality? + self.assertEqual( + config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}] + ) + + config = self.get_config(scripts=[{"more": "different/script/path"}]) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_app() + + def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self): + config = self.get_config(enable_users_annotations="true", authentication_enable="false") + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_user_annotations(self.context) + + config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend") + with self.assertRaises(ConfigurationError): + config.server_config.complete_config(self.context) + + def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self): + config = self.get_config( + enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set" + ) + config.default_dataset_config.handle_user_annotations(self.context) + + self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."]) + + @patch("server.common.config.dataset_config.DbUtils") + def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils): + mock_db_utils.return_value = "123" + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_user_annotations(self.context) + self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile) + + config = self.get_config( + enable_users_annotations="true", + authentication_enable="true", + annotation_type="hosted_tiledb_array", + db_uri="gotta/set/this", + hosted_file_directory="and/this", + ) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_user_annotations(self.context) + self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB) + + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL" + ) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_user_annotations(self.context) + + def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self): + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_local_file_csv_annotations() + self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile) + cwd = os.getcwd() + self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd) + + def test_handle_embeddings__checks_data_file_types(self): + file_name = self.custom_app_config( + embedding_names=["name1", "name2"], + enable_reembedding="true", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", + anndata_backed="true", + config_file_name=self.config_file_name, + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_embeddings() + + def test_handle_diffexp__raises_warning_for_large_datasets(self): + config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_diffexp(self.context) + self.assertEqual(len(self.context["messages"]), 0) + + def test_multi_dataset(self): + config = AppConfig() + # test for illegal url_dataroots + for illegal in ("../b", "!$*", "\\n", "", "(bad)"): + config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} + ) + with self.assertRaises(ConfigurationError): + config.complete_config() + + # test for legal url_dataroots + for legal in ("d", "this.is-okay_", "a/b"): + config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} + ) + config.complete_config() + + # test that multi dataroots work end to end + config.update_server_config( + multi_dataset__dataroot=dict( + s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), + s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), + s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), + ) + ) + + # Change this default to test if the dataroot overrides below work. + config.update_default_dataset_config(app__about_legal_tos="tos_default.html") + + # specialize the configs for set1 + config.add_dataroot_config( + "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html" + ) + + # specialize the configs for set2 + config.add_dataroot_config( + "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html" + ) + + # no specializations for set3 (they get the default dataset config) + config.complete_config() + + with test_server(app_config=config) as server: + session = requests.Session() + + response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is False + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html" + + response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html" + + response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html" + + response = session.get(f"{server}/health") + assert response.json()["status"] == "pass" + + def test_configfile_with_specialization(self): + # test that per_dataset_config config load the default config, then the specialized config + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + server: + multi_dataset: + dataroot: + test: + base_url: test + dataroot: fake_dataroot + + dataset: + user_annotations: + enable: false + type: hosted_tiledb_array + hosted_tiledb_array: + db_uri: fake_db_uri + hosted_file_directory: fake_dir + + per_dataset_config: + test: + user_annotations: + enable: true + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + + test_config = app_config.dataroot_config["test"] + + # test config from default + self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array") + self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri") + + # test config from specialization + self.assertTrue(test_config.user_annotations__enable) diff --git a/server/test/unit/common/config/test_server_config.py b/server/test/unit/common/config/test_server_config.py new file mode 100644 index 00000000..78ee9570 --- /dev/null +++ b/server/test/unit/common/config/test_server_config.py @@ -0,0 +1,335 @@ +import os +import unittest +from unittest import mock +from unittest.mock import patch + +from server.common.config.base_config import BaseConfig +from server.common.utils.utils import find_available_port +from server.test import PROJECT_ROOT, FIXTURES_ROOT + +import requests + +from server.common.config.app_config import AppConfig +from server.common.errors import ConfigurationError +from server.test import test_server +from server.test.unit.common.config import ConfigTests + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class TestServerConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_raises_error_if_default_config_is_invalid(self): + invalid_config = self.get_config(port="not_valid") + with self.assertRaises(ConfigurationError): + invalid_config.complete_config() + + @patch("server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.server_config.complete_config(self.context) + self.assertEqual(mock_check_attrs.call_count, 40) + + def test_handle_app__throws_error_if_port_doesnt_exist(self): + config = self.get_config(port=99999999) + with self.assertRaises(ConfigurationError): + config.server_config.handle_app(self.context) + + @patch("server.common.config.server_config.discover_s3_region_name") + def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name): + mock_discover_region_name.return_value = None + # Default config + self.assertEqual(self.config.server_config.data_locator__s3__region_name, None) + # hard coded + config = self.get_config() + self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") + # incorrectly formatted + dataroot = { + "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"}, + "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"}, + } + file_name = self.custom_app_config( + dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true" + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_locator() + + @patch("server.common.config.server_config.discover_s3_region_name") + def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name): + mock_discover_region_name.return_value = "us-west-2" + dataroot = { + "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"}, + "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"}, + } + file_name = self.custom_app_config( + dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true" + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.handle_data_locator() + self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2") + mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev") + + def test_handle_app___can_use_envar_port(self): + config = self.get_config(port=24) + self.assertEqual(config.server_config.app__port, 24) + + # Note if the port is set in the config file it will NOT be overwritten by a different envvar + os.environ["CXG_SERVER_PORT"] = "4008" + self.config = AppConfig() + self.config.server_config.handle_app(self.context) + self.assertEqual(self.config.server_config.app__port, 4008) + + def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self): + config = self.get_config(flask_secret_key="KEY_FROM_FILE") + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE") + + os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV" + config.server_config.handle_app(self.context) + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV") + + def test_handle_app__sets_web_base_url(self): + config = self.get_config(web_base_url="anything.com") + self.assertEqual(config.server_config.app__web_base_url, "anything.com") + + def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self): + config = self.get_config(client_secret="KEY_FROM_FILE") + config.server_config.handle_authentication() + self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE") + + os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV" + config.server_config.handle_authentication() + + self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV") + + def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", + config_file_name="two_data_roots.yml", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_source() + + file_name = self.custom_app_config(config_file_name="zero_roots.yml") + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_source() + + def test_get_api_base_url_works(self): + + # test the api_base_url feature, and that it can contain a path + config = AppConfig() + backend_port = find_available_port("localhost", 10000) + config.update_server_config( + app__api_base_url=f"http://localhost:{backend_port}/additional/path", + multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset", + ) + + config.complete_config() + + with test_server(["-p", str(backend_port)], app_config=config) as server: + session = requests.Session() + self.assertEqual(server, f"http://localhost:{backend_port}") + response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config") + self.assertEqual(response.status_code, 200) + data_config = response.json() + self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k") + + # test the health check at the correct url + response = session.get(f"{server}/additional/path/health") + assert response.json()["status"] == "pass" + + # also check that the old URL still works. + # NOTE: this old URL location will soon be deprecated, and when that happens + # this check can be removed. + response = session.get(f"{server}/health") + assert response.json()["status"] == "pass" + + def test_get_web_base_url_works(self): + config = self.get_config(web_base_url="www.thisisawebsite.com") + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "www.thisisawebsite.com") + + config = self.get_config(web_base_url="local", port=12) + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "http://localhost:12") + + config = self.get_config(web_base_url="www.thisisawebsite.com/") + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "www.thisisawebsite.com") + + config = self.get_config(api_base_url="www.api_base.com/") + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "www.api_base.com") + + def test_config_for_single_dataset(self): + file_name = self.custom_app_config( + config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg" + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.handle_single_dataset(self.context) + self.assertIsNotNone(config.server_config.matrix_data_cache_manager) + + file_name = self.custom_app_config( + config_file_name="single_dataset_with_about.yml", + about="www.cziscience.com", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg", + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_single_dataset(self.context) + + def test_multi_dataset_raises_error_for_illegal_routes(self): + # test for illegal url_dataroots + for illegal in ("../b", "!$*", "\\n", "", "(bad)"): + self.config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} + ) + with self.assertRaises(ConfigurationError): + self.config.complete_config() + + def test_multidataset_works_for_legal_routes(self): + # test for legal url_dataroots + for legal in ("d", "this.is-okay_", "a/b"): + self.config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} + ) + self.config.complete_config() + + def test_mulitdatasets_work_e2e(self): + # test that multi dataroots work end to end + self.config.update_server_config( + multi_dataset__dataroot=dict( + s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), + s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), + s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), + ) + ) + + # Change this default to test if the dataroot overrides below work. + self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html") + + # specialize the configs for set1 + self.config.add_dataroot_config( + "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html" + ) + + # specialize the configs for set2 + self.config.add_dataroot_config( + "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html" + ) + + # no specializations for set3 (they get the default dataset config) + self.config.complete_config() + + with test_server(app_config=self.config) as server: + session = requests.Session() + + response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is False + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html" + + response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html" + + response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html" + + response = session.get(f"{server}/health") + assert response.json()["status"] == "pass" + + @patch("server.common.config.server_config.diffexp_tiledb.set_config") + def test_handle_diffexp(self, mock_tiledb_config): + custom_config_file = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", + cpu_multiplier=3, + diffexp_max_workers=1, + target_workunit=4, + config_file_name=self.config_file_name, + ) + config = AppConfig() + config.update_from_config_file(custom_config_file) + config.server_config.handle_diffexp() + # called with the min of diffexp_max_workers and cpus*cpu_multiplier + mock_tiledb_config.assert_called_once_with(1, 4) + + @patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context") + def test_handle_adaptor(self, mock_tiledb_context): + custom_config = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2 + ) + config = AppConfig() + config.update_from_config_file(custom_config) + config.server_config.handle_adaptor() + mock_tiledb_context.assert_called_once_with( + {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"} + ) + + @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION") + @patch("server.common.config.get_secret_key") + def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key): + mock_get_secret_key.return_value = { + "flask_secret_key": "mock_flask_secret", + "oauth_client_secret": "mock_oauth_secret", + "db_uri": "mock_db_uri", + } + + config = AppConfig() + + with self.assertLogs(level="INFO") as logger: + from server.common.config import handle_config_from_secret + + # should not throw error + # "AttributeError: 'XConfig' object has no attribute 'x'" + handle_config_from_secret(config) + + # should log 3 lines (one for each var set from a secret) + self.assertEqual(len(logger.output), 3) + self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0]) + self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1]) + self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2]) + self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret") + self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret") + self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri") diff --git a/server/test/unit/common/test_api.py b/server/test/unit/common/test_api.py index ee979b1f..b7dca258 100644 --- a/server/test/unit/common/test_api.py +++ b/server/test/unit/common/test_api.py @@ -8,8 +8,14 @@ import requests import server.test.unit.decode_fbs as decode_fbs from server.data_common.matrix_loader import MatrixDataType -from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server, - stop_test_server) +from server.test import ( + data_with_tmp_annotations, + make_fbs, + PROJECT_ROOT, + FIXTURES_ROOT, + start_test_server, + stop_test_server, +) from server.test.fixtures.fixtures import pbmc3k_colors BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} @@ -43,7 +49,6 @@ class EndPoints(object): result_data = result.json() self.assertIn("library_versions", result_data["config"]) self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") - self.assertEqual(len(result_data["config"]["features"]), 5) def test_get_layout_fbs(self): endpoint = "layout/obs" @@ -381,11 +386,14 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): @classmethod def setUpClass(cls): - cls._setupClass(cls, [ - f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", - "--disable-annotations", - "--experimental-enable-reembedding", - ]) + cls._setupClass( + cls, + [ + f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + "--disable-annotations", + "--experimental-enable-reembedding", + ], + ) @classmethod def tearDownClass(cls): @@ -403,10 +411,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints): @classmethod def setUpClass(cls): - cls._setupClass(cls, [ - f"{FIXTURES_ROOT}/pbmc3k.cxg", - "--disable-annotations", - ]) + cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"]) @classmethod def tearDownClass(cls): @@ -423,7 +428,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations): cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations( MatrixDataType.H5AD, annotations_fixture=True ) - cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ]) + cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()]) @classmethod def tearDownClass(cls): @@ -439,11 +444,7 @@ class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations): @classmethod def setUpClass(cls): cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True) - cls._setupClass(cls, [ - "--annotations-file", - cls.annotations.output_file, - cls.data.get_location(), - ]) + cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()]) @classmethod def tearDownClass(cls): diff --git a/server/test/unit/common/test_app_config.py b/server/test/unit/common/test_app_config.py deleted file mode 100644 index c3dd1436..00000000 --- a/server/test/unit/common/test_app_config.py +++ /dev/null @@ -1,249 +0,0 @@ -import os -import unittest -from unittest import mock -from unittest.mock import patch -import tempfile - -import requests - -from server.common.app_config import AppConfig -from server.common.errors import ConfigurationError -from server.common.utils.utils import find_available_port -from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT - - -# NOTE, there are more tests that should be written for AppConfig. -# this is just a start. - -def mockenv(**envvars): - return mock.patch.dict(os.environ, envvars) - - -class AppConfigTest(unittest.TestCase): - def test_update(self): - config = AppConfig() - config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir") - vars = config.server_config.changes_from_default() - self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)]) - - config = AppConfig() - config.update_default_dataset_config(app__scripts=(), app__inline_scripts=()) - vars = config.server_config.changes_from_default() - self.assertCountEqual(vars, []) - - config = AppConfig() - config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[]) - vars = config.default_dataset_config.changes_from_default() - self.assertCountEqual(vars, []) - - config = AppConfig() - config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) - vars = config.default_dataset_config.changes_from_default() - self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) - - def test_multi_dataset(self): - - config = AppConfig() - # test for illegal url_dataroots - for illegal in ("../b", "!$*", "\\n", "", "(bad)"): - config.update_server_config( - multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} - ) - with self.assertRaises(ConfigurationError): - config.complete_config() - - # test for legal url_dataroots - for legal in ("d", "this.is-okay_", "a/b"): - config.update_server_config( - multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} - ) - config.complete_config() - - # test that multi dataroots work end to end - config.update_server_config( - multi_dataset__dataroot=dict( - s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), - s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), - s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), - ) - ) - - # Change this default to test if the dataroot overrides below work. - config.update_default_dataset_config(app__about_legal_tos="tos_default.html") - - # specialize the configs for set1 - config.add_dataroot_config( - "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html" - ) - - # specialize the configs for set2 - config.add_dataroot_config( - "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html" - ) - - # no specializations for set3 (they get the default dataset config) - config.complete_config() - - with test_server(app_config=config) as server: - session = requests.Session() - - response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") - data_config = response.json() - assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" - assert data_config["config"]["parameters"]["annotations"] is False - assert data_config["config"]["parameters"]["disable-diffexp"] is False - assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html" - - response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") - data_config = response.json() - assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" - assert data_config["config"]["parameters"]["annotations"] is True - assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html" - - response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") - data_config = response.json() - assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" - assert data_config["config"]["parameters"]["annotations"] is True - assert data_config["config"]["parameters"]["disable-diffexp"] is False - assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html" - - response = session.get(f"{server}/health") - assert response.json()["status"] == "pass" - - @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION") - @patch('server.common.aws_secret_utils.get_secret_key') - def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key): - mock_get_secret_key.return_value = { - "flask_secret_key": "mock_flask_secret", - "oauth_client_secret": "mock_oauth_secret", - "db_uri": "mock_db_uri" - } - - config = AppConfig() - - with self.assertLogs(level="INFO") as logger: - from server.common.aws_secret_utils import handle_config_from_secret - # should not throw error - # "AttributeError: 'XConfig' object has no attribute 'x'" - handle_config_from_secret(config) - - # should log 3 lines (one for each var set from a secret) - self.assertEqual(len(logger.output), 3) - self.assertIn('INFO:root:set app__flask_secret_key from secret', logger.output[0]) - self.assertIn('INFO:root:set authentication__params_oauth__client_secret from secret', logger.output[1]) - self.assertIn('INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret', logger.output[2]) - self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret") - self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret") - self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri") - - def test_api_base_url(self): - - # test the api_base_url feature, and that it can contain a path - config = AppConfig() - backend_port = find_available_port("localhost", 10000) - config.update_server_config( - app__api_base_url=f"http://localhost:{backend_port}/additional/path", - multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset" - ) - - config.complete_config() - - with test_server(["-p", str(backend_port)], app_config=config) as server: - session = requests.Session() - self.assertEqual(server, f"http://localhost:{backend_port}") - response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config") - self.assertEqual(response.status_code, 200) - data_config = response.json() - self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k") - - # test the health check at the correct url - response = session.get(f"{server}/additional/path/health") - assert response.json()["status"] == "pass" - - # also check that the old URL still works. - # NOTE: this old URL location will soon be deprecated, and when that happens - # this check can be removed. - response = session.get(f"{server}/health") - assert response.json()["status"] == "pass" - - def test_configfile_with_specialization(self): - # test that per_dataset_config config load the default config, then the specialized config - - with tempfile.TemporaryDirectory() as tempdir: - configfile = os.path.join(tempdir, "config.yaml") - with open(configfile, "w") as fconfig: - config = """ - server: - multi_dataset: - dataroot: - test: - base_url: test - dataroot: fake_dataroot - - dataset: - user_annotations: - enable: false - type: hosted_tiledb_array - hosted_tiledb_array: - db_uri: fake_db_uri - hosted_file_directory: fake_dir - - per_dataset_config: - test: - user_annotations: - enable: true - """ - fconfig.write(config) - - app_config = AppConfig() - app_config.update_from_config_file(configfile) - - test_config = app_config.dataroot_config["test"] - - # test config from default - self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array") - self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri") - - # test config from specialization - self.assertTrue(test_config.user_annotations__enable) - - def test_configfile_no_dataset_section(self): - # test a config file without a dataset section - - with tempfile.TemporaryDirectory() as tempdir: - configfile = os.path.join(tempdir, "config.yaml") - with open(configfile, "w") as fconfig: - config = """ - server: - multi_dataset: - dataroot: test_dataroot - - """ - fconfig.write(config) - - app_config = AppConfig() - app_config.update_from_config_file(configfile) - server_changes = app_config.server_config.changes_from_default() - dataset_changes = app_config.default_dataset_config.changes_from_default() - self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)]) - self.assertEqual(dataset_changes, []) - - def test_configfile_no_server_section(self): - # test a config file without a dataset section - - with tempfile.TemporaryDirectory() as tempdir: - configfile = os.path.join(tempdir, "config.yaml") - with open(configfile, "w") as fconfig: - config = """ - dataset: - user_annotations: - enable: false - """ - fconfig.write(config) - - app_config = AppConfig() - app_config.update_from_config_file(configfile) - server_changes = app_config.server_config.changes_from_default() - dataset_changes = app_config.default_dataset_config.changes_from_default() - self.assertEqual(server_changes, []) - self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)]) diff --git a/server/test/unit/common/test_corpora.py b/server/test/unit/common/test_corpora.py index 1a0222dc..5f26dae9 100644 --- a/server/test/unit/common/test_corpora.py +++ b/server/test/unit/common/test_corpora.py @@ -87,24 +87,17 @@ class CorporaRESTAPITest(unittest.TestCase): def setCorporaFields(cls, path): adata = anndata.read_h5ad(path) corpora_props = { - "version": { - "corpora_schema_version": "1.0.0", - "corpora_encoding_version": "0.1.0" - }, + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, "title": "PBMC3K", - "contributors": json.dumps([ - {"name": "name"} - ]), - "layer_descriptions": { - "X": "raw counts" - }, + "contributors": json.dumps([{"name": "name"}]), + "layer_descriptions": {"X": "raw counts"}, "organism": "human", "organism_ontology_term_id": "unknown", "project_name": "test project", "project_description": "test description", - "project_links": json.dumps([ - {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"} - ]), + "project_links": json.dumps( + [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}] + ), "default_embedding": "X_tsne", } adata.uns.update(corpora_props) diff --git a/server/test/unit/common/test_writable_annotation.py b/server/test/unit/common/test_writable_annotation.py index f650fecf..0e369e56 100644 --- a/server/test/unit/common/test_writable_annotation.py +++ b/server/test/unit/common/test_writable_annotation.py @@ -27,7 +27,7 @@ class auth(object): class WritableTileDBStoredAnnotationTest(unittest.TestCase): def setUp(self): - self.user_id = '1234' + self.user_id = "1234" self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD) self.data.dataset_config.user_annotations = self.annotations self.db = self.annotations.db @@ -38,7 +38,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): } self.fbs = make_fbs(self.test_dict) self.df = pd.DataFrame(self.test_dict) - self.app = Flask('fake_app') + self.app = Flask("fake_app") self.app.__setattr__("auth", auth) def tearDown(self): @@ -65,8 +65,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): self.annotations.write_labels(self.df, self.data) dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id annotation = self.db.query_for_most_recent( - Annotation, - [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)] + Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)] ) # retrieve tiledb array df = tiledb.open(annotation.tiledb_uri) @@ -78,7 +77,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self): with self.app.test_request_context(): - new_name = 'new_dataset/location' + new_name = "new_dataset/location" self.data.get_location = MagicMock(return_value=new_name) num_datasets = len(self.db.query([CellxGeneDataset])) self.annotation_put_fbs(self.fbs) @@ -130,15 +129,14 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): with self.assertRaises(KeyError): self.annotation_put_fbs(fbs_bad) - @patch('server.common.annotations.hosted_tiledb.current_app') + @patch("server.common.annotations.hosted_tiledb.current_app") def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id): - mock_user_id.auth.get_user_id.return_value = '1234' + mock_user_id.auth.get_user_id.return_value = "1234" self.annotations.write_labels(self.df, self.data) # get uri dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id annotation = self.db.query_for_most_recent( - Annotation, - [Annotation.user_id == '1234', Annotation.dataset_id == str(dataset_id)] + Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)] ) df = tiledb.open(annotation.tiledb_uri) @@ -270,20 +268,3 @@ class WritableAnnotationTest(unittest.TestCase): all_col_schema["cat_B"], {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True}, ) - - def test_config(self): - features = self.data.get_features(self.annotations) - - # test each for singular presence and accuracy of available flag - def check_feature(method, path, available): - feature = list( - filter(lambda f: f.method == method and f.path == path and f.available == available, features) - ) - self.assertIsNotNone(feature) - self.assertEqual(len(feature), 1) - - check_feature("POST", "/cluster/", False) - check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable) - check_feature("GET", "/layout/obs", True) - check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding) - check_feature("PUT", "/annotations/obs", True) diff --git a/server/test/unit/common/utils/test_cxg_generation_utils.py b/server/test/unit/common/utils/test_cxg_generation_utils.py index 57893913..b9043c33 100644 --- a/server/test/unit/common/utils/test_cxg_generation_utils.py +++ b/server/test/unit/common/utils/test_cxg_generation_utils.py @@ -8,8 +8,12 @@ import numpy as np import tiledb from pandas import Series, DataFrame -from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array, - convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array) +from server.common.utils.cxg_generation_utils import ( + convert_dictionary_to_cxg_group, + convert_dataframe_to_cxg_array, + convert_ndarray_to_cxg_dense_array, + convert_matrix_to_cxg_array, +) PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() @@ -28,8 +32,9 @@ class TestCxgGenerationUtils(unittest.TestCase): dictionary_name = "favorite_desserts" expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}" - convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary, - group_metadata_name=dictionary_name) + convert_dictionary_to_cxg_group( + self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name + ) array = tiledb.open(expected_array_directory) actual_stored_metadata = dict(array.meta.items()) @@ -44,13 +49,16 @@ class TestCxgGenerationUtils(unittest.TestCase): random_dataframe_name = f"random_dataframe_{uuid4()}" random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category}) - convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, - "int_category", tiledb.Ctx()) + convert_dataframe_to_cxg_array( + self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx() + ) expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}" expected_array_metadata = { - "cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, - "index": "int_category"})} + "cxg_schema": json.dumps( + {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"} + ) + } actual_stored_dataframe_array = tiledb.open(expected_array_directory) actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items()) @@ -95,7 +103,7 @@ class TestCxgGenerationUtils(unittest.TestCase): self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[:, :][''].size == 0) + self.assertTrue(actual_stored_array[:, :][""].size == 0) def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self): matrix = np.zeros([3, 3]) @@ -110,10 +118,10 @@ class TestCxgGenerationUtils(unittest.TestCase): self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[0, 0][''] == 1) - self.assertTrue(actual_stored_array[1, 1][''] == 1) - self.assertTrue(actual_stored_array[2, 2][''] == 2) - self.assertTrue(actual_stored_array[:, :][''].size == 3) + self.assertTrue(actual_stored_array[0, 0][""] == 1) + self.assertTrue(actual_stored_array[1, 1][""] == 1) + self.assertTrue(actual_stored_array[2, 2][""] == 2) + self.assertTrue(actual_stored_array[:, :][""].size == 3) def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self): matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}" @@ -122,14 +130,15 @@ class TestCxgGenerationUtils(unittest.TestCase): # a matrix of zeros which is sparse. column_shift = np.ones((3, 2)) - convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(), - column_shift_for_sparse_encoding=column_shift) + convert_matrix_to_cxg_array( + matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift + ) actual_stored_array = tiledb.open(matrix_name) self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[:, :][''].size == 0) + self.assertTrue(actual_stored_array[:, :][""].size == 0) def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self): matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}" @@ -137,13 +146,14 @@ class TestCxgGenerationUtils(unittest.TestCase): # Only column shift the first column of ones. column_shift = np.array([[1, 0], [1, 0]]) - convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(), - column_shift_for_sparse_encoding=column_shift) + convert_matrix_to_cxg_array( + matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift + ) actual_stored_array = tiledb.open(matrix_name) self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[0, 1][''] == 1) - self.assertTrue(actual_stored_array[1, 1][''] == 1) - self.assertTrue(actual_stored_array[:, :][''].size == 2) + self.assertTrue(actual_stored_array[0, 1][""] == 1) + self.assertTrue(actual_stored_array[1, 1][""] == 1) + self.assertTrue(actual_stored_array[:, :][""].size == 2) diff --git a/server/test/unit/common/utils/test_matrix_utils.py b/server/test/unit/common/utils/test_matrix_utils.py index ffda1045..9cc9daf5 100644 --- a/server/test/unit/common/utils/test_matrix_utils.py +++ b/server/test/unit/common/utils/test_matrix_utils.py @@ -6,7 +6,6 @@ from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_ class TestMatrixUtils(unittest.TestCase): - def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self): matrix = np.array([1, 2, 3]) diff --git a/server/test/unit/common/utils/test_sanitization_utils.py b/server/test/unit/common/utils/test_sanitization_utils.py index 8ef04218..e209be95 100644 --- a/server/test/unit/common/utils/test_sanitization_utils.py +++ b/server/test/unit/common/utils/test_sanitization_utils.py @@ -4,7 +4,6 @@ from server.common.utils.sanitization_utils import sanitize_values_in_list, sani class TestSanitizationUtils(unittest.TestCase): - def test__sanitize_values_in_list__not_strings_raises_exception(self): keys_to_sanitize = [1, 2, 3] diff --git a/server/test/unit/common/utils/test_type_conversion_utils.py b/server/test/unit/common/utils/test_type_conversion_utils.py index f1653697..ade2d999 100644 --- a/server/test/unit/common/utils/test_type_conversion_utils.py +++ b/server/test/unit/common/utils/test_type_conversion_utils.py @@ -5,12 +5,17 @@ from unittest.mock import patch import numpy as np from pandas import Series, DataFrame -from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \ - get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe, convert_pandas_series_to_numpy +from server.common.utils.type_conversion_utils import ( + can_cast_to_float32, + can_cast_to_int32, + get_dtype_of_array, + get_schema_type_hint_of_array, + get_dtypes_and_schemas_of_dataframe, + convert_pandas_series_to_numpy, +) class TestTypeConversionUtils(unittest.TestCase): - def test__can_cast_to_float32__string_is_false(self): array_to_convert = Series(data=["1", "2", "3"], dtype=str) @@ -97,8 +102,9 @@ class TestTypeConversionUtils(unittest.TestCase): expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode] for test_type_index in range(len(types)): - with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", - i=test_type_index): + with self.subTest( + f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) @@ -123,8 +129,9 @@ class TestTypeConversionUtils(unittest.TestCase): expected_dtypes = [np.float32, np.int32] for test_type_index in range(len(types)): - with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", - i=test_type_index): + with self.subTest( + f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) @@ -141,8 +148,9 @@ class TestTypeConversionUtils(unittest.TestCase): expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}] for test_type_index in range(len(types)): - with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", - i=test_type_index): + with self.subTest( + f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) @@ -160,8 +168,9 @@ class TestTypeConversionUtils(unittest.TestCase): for test_type_index in range(len(types)): with self.subTest( - f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", - i=test_type_index): + f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", + i=test_type_index, + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) @@ -171,8 +180,10 @@ class TestTypeConversionUtils(unittest.TestCase): dataframe = DataFrame({"float_array": float_array, "category_array": category_array}) expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode} - expected_schema_type_hints_dict = {"float_array": {"type": "float32"}, - "category_array": {"type": "categorical", "categories": ["a", "b"]}} + expected_schema_type_hints_dict = { + "float_array": {"type": "float32"}, + "category_array": {"type": "categorical", "categories": ["a", "b"]}, + } actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe) @@ -201,5 +212,6 @@ class TestTypeConversionUtils(unittest.TestCase): with self.assertLogs(level="ERROR") as logger: convert_pandas_series_to_numpy(int_series, np.int32) - self.assertIn("Cannot convert a pandas Series object to an integer dtype if it contains NaNs", - logger.output[0]) + self.assertIn( + "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0] + ) diff --git a/server/test/unit/converters/test_h5ad_data_file.py b/server/test/unit/converters/test_h5ad_data_file.py index f8587ebd..99ad40af 100644 --- a/server/test/unit/converters/test_h5ad_data_file.py +++ b/server/test/unit/converters/test_h5ad_data_file.py @@ -16,7 +16,6 @@ PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() class TestH5ADDataFile(unittest.TestCase): - def setUp(self): self.sample_anndata = self._create_sample_anndata_dataset() self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata) @@ -40,8 +39,12 @@ class TestH5ADDataFile(unittest.TestCase): def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self): with self.assertLogs(level="WARN") as logger: - H5ADDataFile(self.sample_h5ad_filename, dataset_title="My Awesome Dataset", - dataset_about="http://www.awesomedataset.com", use_corpora_schema=False) + H5ADDataFile( + self.sample_h5ad_filename, + dataset_title="My Awesome Dataset", + dataset_about="http://www.awesomedataset.com", + use_corpora_schema=False, + ) self.assertIn("will override any metadata that is extracted", logger.output[0]) @@ -49,10 +52,12 @@ class TestH5ADDataFile(unittest.TestCase): h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False) self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all()) - self.assertEqual(h5ad_file.anndata.obs.sort_index(inplace=True), - self.sample_anndata.obs.sort_index(inplace=True)) - self.assertEqual(h5ad_file.anndata.var.sort_index(inplace=True), - self.sample_anndata.var.sort_index(inplace=True)) + self.assertEqual( + h5ad_file.anndata.obs.sort_index(inplace=True), self.sample_anndata.obs.sort_index(inplace=True) + ) + self.assertEqual( + h5ad_file.anndata.var.sort_index(inplace=True), self.sample_anndata.var.sort_index(inplace=True) + ) for key in h5ad_file.anndata.obsm.keys(): self.assertIn(key, self.sample_anndata.obsm.keys()) @@ -73,8 +78,12 @@ class TestH5ADDataFile(unittest.TestCase): self.assertIn("name_0", h5ad_file.var.columns) def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self): - h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False, - obs_index_column_name="float_category", vars_index_column_name="int_category") + h5ad_file = H5ADDataFile( + self.sample_h5ad_filename, + use_corpora_schema=False, + obs_index_column_name="float_category", + vars_index_column_name="int_category", + ) self.assertNotIn("name_0", h5ad_file.obs.columns) self.assertNotIn("name_0", h5ad_file.var.columns) @@ -82,15 +91,23 @@ class TestH5ADDataFile(unittest.TestCase): def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self): with self.assertRaises(Exception) as exception_context: - H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False, - obs_index_column_name="float_category", vars_index_column_name="bool_category") + H5ADDataFile( + self.sample_h5ad_filename, + use_corpora_schema=False, + obs_index_column_name="float_category", + vars_index_column_name="bool_category", + ) self.assertIn("Please prepare data to contain unique values", str(exception_context.exception)) def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self): with self.assertRaises(Exception) as exception_context: - H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False, - obs_index_column_name="unknown_category", vars_index_column_name="i_dont_exist") + H5ADDataFile( + self.sample_h5ad_filename, + use_corpora_schema=False, + obs_index_column_name="unknown_category", + vars_index_column_name="i_dont_exist", + ) self.assertIn("does not exist", str(exception_context.exception)) @@ -101,8 +118,9 @@ class TestH5ADDataFile(unittest.TestCase): self.assertEqual(h5ad_file.dataset_about, "www.link.com") def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self): - h5ad_file = H5ADDataFile(self.sample_h5ad_filename, dataset_about="override_about", - dataset_title="override_title") + h5ad_file = H5ADDataFile( + self.sample_h5ad_filename, dataset_about="override_about", dataset_title="override_title" + ) self.assertEqual(h5ad_file.dataset_title, "override_title") self.assertEqual(h5ad_file.dataset_about, "override_about") @@ -145,8 +163,11 @@ class TestH5ADDataFile(unittest.TestCase): remove(sparse_with_column_shift_filename) def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False): - expected_directories, expected_obs_files, expected_var_files = \ - self._get_expected_generated_list_of_tiledb_files() + ( + expected_directories, + expected_obs_files, + expected_var_files, + ) = self._get_expected_generated_list_of_tiledb_files() for directory in expected_directories: self.assertTrue(path.isdir(directory)) @@ -187,8 +208,18 @@ class TestH5ADDataFile(unittest.TestCase): var_files.append("bool_category.tdb") var_files.append("int_category.tdb") - return [metadata_directory, main_x_directory, overall_embedding_directory, specific_embedding_directory, - obs_directory, var_directory], obs_files, var_files + return ( + [ + metadata_directory, + main_x_directory, + overall_embedding_directory, + specific_embedding_directory, + obs_directory, + var_directory, + ], + obs_files, + var_files, + ) def _write_anndata_to_file(self, anndata): temporary_filename = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}.h5ad" @@ -204,7 +235,8 @@ class TestH5ADDataFile(unittest.TestCase): random_string_category = Series(data=["a", "b", "b"], dtype="category") random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32) obs_dataframe = DataFrame( - data={"string_category": random_string_category, "float_category": random_float_category}) + data={"string_category": random_string_category, "float_category": random_float_category} + ) obs = obs_dataframe # Create vars @@ -230,6 +262,7 @@ class TestH5ADDataFile(unittest.TestCase): # Set project links to be a dictionary uns["project_links"] = json.dumps( - [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}]) + [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}] + ) return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns) diff --git a/server/test/unit/data_anndata/test_anndata_adaptor.py b/server/test/unit/data_anndata/test_anndata_adaptor.py index d4a3a778..d0df5db1 100644 --- a/server/test/unit/data_anndata/test_anndata_adaptor.py +++ b/server/test/unit/data_anndata/test_anndata_adaptor.py @@ -94,23 +94,6 @@ class AdaptorTest(unittest.TestCase): with pytest.raises(TypeError): self.data._create_schema() - def test_config(self): - features = self.data.get_features(annotations=None) - - # test each for singular presence and accuracy of available flag - def check_feature(method, path, available): - feature = list( - filter(lambda f: f.method == method and f.path == path and f.available == available, features) - ) - self.assertIsNotNone(feature) - self.assertEqual(len(feature), 1) - - check_feature("POST", "/cluster/", False) - check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable) - check_feature("GET", "/layout/obs", True) - check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding) - check_feature("PUT", "/annotations/obs", False) - def test_layout(self): fbs = self.data.layout_to_fbs_matrix(fields=None) layout = decode_fbs.decode_matrix_FBS(fbs) diff --git a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py index 35ccb886..b3db3cb3 100644 --- a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py +++ b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py @@ -3,7 +3,7 @@ import json from server.data_anndata.anndata_adaptor import AnndataAdaptor from server.common.data_locator import DataLocator -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.test import PROJECT_ROOT diff --git a/server/test/unit/data_common/test_matrix_loader.py b/server/test/unit/data_common/test_matrix_loader.py index d365b001..9631de32 100644 --- a/server/test/unit/data_common/test_matrix_loader.py +++ b/server/test/unit/data_common/test_matrix_loader.py @@ -4,7 +4,7 @@ import tempfile import time import unittest -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.common.errors import DatasetAccessError from server.data_common.matrix_loader import MatrixDataCacheManager from server.test import FIXTURES_ROOT @@ -38,7 +38,7 @@ class MatrixCacheTest(unittest.TestCase): result = {} for k, v in datasets.items(): # filter out the dirname and the .cxg from the name - newk = int(k[1][len(dirname) + 1: -4]) + newk = int(k[1][len(dirname) + 1 : -4]) result[newk] = v return result diff --git a/server/test/unit/eb/test_eb.py b/server/test/unit/eb/test_eb.py index b43fda24..c148f6bc 100644 --- a/server/test/unit/eb/test_eb.py +++ b/server/test/unit/eb/test_eb.py @@ -3,7 +3,7 @@ import tempfile import requests import subprocess from server.test import PROJECT_ROOT, FIXTURES_ROOT -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from contextlib import contextmanager import time @@ -36,9 +36,7 @@ class Elastic_Beanstalk_Test(unittest.TestCase): c = AppConfig() # test that eb works - c.update_server_config( - multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame" - ) + c.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame") c.complete_config() c.write_config(f"{tempdirname}/config.yaml")