From 009fa2ff02a85519d5d37ba7886553ea6216e112 Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Tue, 21 Jul 2020 12:51:39 -0700 Subject: [PATCH 01/55] clear selection state upon subset (#1655) --- client/src/reducers/continuousSelection.js | 1 + client/src/reducers/graphSelection.js | 1 + 2 files changed, 2 insertions(+) diff --git a/client/src/reducers/continuousSelection.js b/client/src/reducers/continuousSelection.js index 6f823ca5..2060d3bf 100644 --- a/client/src/reducers/continuousSelection.js +++ b/client/src/reducers/continuousSelection.js @@ -3,6 +3,7 @@ import { makeContinuousDimensionName } from "../util/nameCreators"; const ContinuousSelection = (state = {}, action) => { switch (action.type) { case "reset subset": + case "subset to selection": case "set clip quantiles": { return {}; } diff --git a/client/src/reducers/graphSelection.js b/client/src/reducers/graphSelection.js index 7b5b47c6..15589462 100644 --- a/client/src/reducers/graphSelection.js +++ b/client/src/reducers/graphSelection.js @@ -7,6 +7,7 @@ const GraphSelection = ( ) => { switch (action.type) { case "set clip quantiles": + case "subset to selection": case "reset subset": case "set layout choice": { return { From a44da11f3ff9ab4b9fcf7c3b5aba065207d1837e Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Wed, 22 Jul 2020 12:14:31 -0700 Subject: [PATCH 02/55] correctly handle non-string categoricals (#1660) --- client/src/annoMatrix/schema.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/annoMatrix/schema.js b/client/src/annoMatrix/schema.js index 9ce03765..1c66febe 100644 --- a/client/src/annoMatrix/schema.js +++ b/client/src/annoMatrix/schema.js @@ -68,7 +68,7 @@ export function _normalizeCategoricalSchema(colSchema, col) { const { type, writable } = colSchema; if (type === "string" || type === "boolean" || type === "categorical") { const categorySet = new Set( - col.summarize().categories.concat(colSchema.categories ?? []) + col.summarizeCategorical().categories.concat(colSchema.categories ?? []) ); if (writable && !categorySet.has(unassignedCategoryLabel)) { categorySet.add(unassignedCategoryLabel); From 03bad044362a0bda40c049e88e452074ec0bf45e Mon Sep 17 00:00:00 2001 From: Colin Megill Date: Wed, 22 Jul 2020 18:48:21 -0400 Subject: [PATCH 03/55] Embedding button to lower left, cell selection (#1658) * embedding * menu bottom left * button * change gutters to support lower toolbar * fix scatterplot layout * fix tests to match new layout * fix smoke tests to match new layout * better sentence, dataset.nObs to top * scatterplot position Co-authored-by: bkmartinjr --- .../__snapshots__/e2eAnnotations.test.js.snap | 2 +- client/__tests__/e2e/data.js | 6 +- client/src/components/app.js | 2 + client/src/components/embedding/index.js | 105 ++++++++++++++++ client/src/components/graph/graph.js | 8 +- client/src/components/menubar/embedding.js | 118 ------------------ client/src/components/menubar/index.js | 2 - .../src/components/scatterplot/scatterplot.js | 27 ++-- client/src/globals.js | 2 +- 9 files changed, 136 insertions(+), 136 deletions(-) create mode 100644 client/src/components/embedding/index.js delete mode 100644 client/src/components/menubar/embedding.js diff --git a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap index 3ad14bca..f74f258b 100644 --- a/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap +++ b/client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap @@ -3,7 +3,7 @@ exports[`annotations stacked bar graph renders 1`] = ` Array [ "
TEST-LABELLABEL
0
", - "
unassignedigned
2132
", + "
unassignedigned
2133
", ] `; diff --git a/client/__tests__/e2e/data.js b/client/__tests__/e2e/data.js index b7f8db77..9796b96b 100644 --- a/client/__tests__/e2e/data.js +++ b/client/__tests__/e2e/data.js @@ -27,7 +27,7 @@ export const datasets = { lasso: [ { "coordinates-as-percent": { x1: 0.1, y1: 0.25, x2: 0.7, y2: 0.75 }, - count: "1173", + count: "1131", }, ], categorical: [ @@ -121,8 +121,8 @@ export const datasets = { }, newCount: { bySubsetConfig: { - false: "599", - true: "594", + false: "668", + true: "659", }, }, }, diff --git a/client/src/components/app.js b/client/src/components/app.js index fe1de5e7..4de58f1c 100644 --- a/client/src/components/app.js +++ b/client/src/components/app.js @@ -11,6 +11,7 @@ import Legend from "./continuousLegend"; import Graph from "./graph/graph"; import MenuBar from "./menubar"; import Autosave from "./autosave"; +import Embedding from "./embedding"; import TermsOfServicePrompt from "./termsPrompt"; import actions from "../actions"; @@ -73,6 +74,7 @@ class App extends React.Component { {(viewportRef) => ( <> + diff --git a/client/src/components/embedding/index.js b/client/src/components/embedding/index.js new file mode 100644 index 00000000..868b6ae1 --- /dev/null +++ b/client/src/components/embedding/index.js @@ -0,0 +1,105 @@ +import React from "react"; +import { connect } from "react-redux"; +import { + ButtonGroup, + Popover, + Button, + Radio, + RadioGroup, + Tooltip, + Position, +} from "@blueprintjs/core"; +import * as globals from "../../globals"; +import actions from "../../actions"; + +@connect((state) => { + return { + layoutChoice: state.layoutChoice, + schema: state.annoMatrix?.schema, + crossfilter: state.obsCrossfilter, + }; +}) +class Embedding extends React.PureComponent { + constructor(props) { + super(props); + this.state = {}; + } + + handleLayoutChoiceChange = (e) => { + const { dispatch } = this.props; + dispatch(actions.layoutChoiceAction(e.currentTarget.value)); + }; + + render() { + const { layoutChoice, schema, crossfilter } = this.props; + return ( + + + + + } + // minimal /* removes arrow */ + position={Position.TOP_LEFT} + content={ +
+

Embedding Choice

+

+ There are {schema?.dataframe?.nObs} cells in the entire dataset. +

+ + {layoutChoice.available.map((name) => ( + + ))} + +
+ } + /> + + ); + } +} + +export default Embedding; diff --git a/client/src/components/graph/graph.js b/client/src/components/graph/graph.js index c91f6e0a..7ec11976 100644 --- a/client/src/components/graph/graph.js +++ b/client/src/components/graph/graph.js @@ -34,8 +34,10 @@ function createProjectionTF(viewportWidth, viewportHeight) { the projection transform accounts for the screen size & other layout */ const fractionToUse = 0.95; // fraction of min dimension to use - const topGutterSizePx = 32; // toolbar box height - const heightMinusGutter = viewportHeight - topGutterSizePx; + const topGutterSizePx = 32; // top gutter for tools + const bottomGutterSizePx = 32; // bottom gutter for tools + const heightMinusGutter = + viewportHeight - topGutterSizePx - bottomGutterSizePx; const minDim = Math.min(viewportWidth, heightMinusGutter); const aspectScale = [ (fractionToUse * minDim) / viewportWidth, @@ -44,7 +46,7 @@ function createProjectionTF(viewportWidth, viewportHeight) { const m = mat3.create(); mat3.fromTranslation(m, [ 0, - -topGutterSizePx / viewportHeight / aspectScale[1], + (bottomGutterSizePx - topGutterSizePx) / viewportHeight / aspectScale[1], ]); mat3.scale(m, m, aspectScale); return m; diff --git a/client/src/components/menubar/embedding.js b/client/src/components/menubar/embedding.js deleted file mode 100644 index d0bfc260..00000000 --- a/client/src/components/menubar/embedding.js +++ /dev/null @@ -1,118 +0,0 @@ -import React from "react"; -import { - ButtonGroup, - Popover, - Button, - Radio, - RadioGroup, - Tooltip, - Position, -} from "@blueprintjs/core"; -import { connect } from "react-redux"; -import * as globals from "../../globals"; -import styles from "./menubar.css"; -import actions from "../../actions"; - -@connect((state) => ({ - layoutChoice: state.layoutChoice, - // disabled temporarily. TODO - issue #1606 - // reembedController: state.reembedController, - // enableReembedding: state.config?.parameters?.["enable-reembedding"] ?? false, - enableReembedding: false, -})) -class Embedding extends React.PureComponent { - handleLayoutChoiceChange = (e) => { - const { dispatch } = this.props; - dispatch(actions.layoutChoiceAction(e.currentTarget.value)); - }; - - // eslint-disable-next-line class-methods-use-this -- temporary disable - renderReembedding() { - return null; - /* disabled pending rewrite. TODO - issue #1606 - const { - enableReembedding, - world, - universe, - dispatch, - reembedController, - } = this.props; - - if (!enableReembedding) return null; - - const loading = !!reembedController?.pendingFetch; - const disabled = World.worldEqUniverse(world, universe); - const tipContent = disabled - ? "Subset cells first, then click to recompute UMAP embedding." - : "Click to recompute UMAP embedding on the current cell subset."; - - return ( - - dispatch(actions.requestReembed())} - loading={loading} - /> - - ); -*/ - } - - render() { - const { layoutChoice } = this.props; - - return ( - - - } @@ -87,9 +89,9 @@ class Embedding extends React.PureComponent { selectedValue={layoutChoice.current} > {layoutChoice.available.map((name) => ( - ))} @@ -103,3 +105,41 @@ class Embedding extends React.PureComponent { } export default Embedding; + +const loadEmbeddingCounts = async ({ annoMatrix, layoutName }) => { + const embedding = await annoMatrix.fetch("emb", layoutName); + const discreteCellIndex = getDiscreteCellEmbeddingRowIndex(embedding); + return { embedding, discreteCellIndex }; +}; + +const LayoutChoice = ({ annoMatrix, layoutName }) => { + return ( + + {({ data, error, isPending }) => { + if (error) { + /* log, as this is unexpected */ + console.error(error); + } + if (error || isPending) { + /* still loading, or errored out - just omit counts (TODO: spinner?) */ + return ; + } + if (data) { + const { embedding, discreteCellIndex } = data; + const isAllCells = discreteCellIndex.size() === embedding.length; + const sizeHint = `${discreteCellIndex.size()} ${ + isAllCells ? "(all) " : "" + }cells`; + return ( + + ); + } + return null; + }} + + ); +}; diff --git a/client/src/components/menubar/index.js b/client/src/components/menubar/index.js index bbfd00fc..63d30b90 100644 --- a/client/src/components/menubar/index.js +++ b/client/src/components/menubar/index.js @@ -10,6 +10,7 @@ import InformationMenu from "./infoMenu"; import Subset from "./subset"; import UndoRedoReset from "./undoRedo"; import DiffexpButtons from "./diffexpButtons"; +import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers"; @connect((state) => { const { annoMatrix } = state; @@ -17,9 +18,11 @@ import DiffexpButtons from "./diffexpButtons"; const selectedCount = crossfilter.countSelected(); const subsetPossible = - selectedCount !== 0 && selectedCount !== crossfilter.size(); // ie, not all are selected - const subsetResetPossible = - annoMatrix.nObs !== annoMatrix.schema.dataframe.nObs; + selectedCount !== 0 && selectedCount !== crossfilter.size(); // ie, not all and not none are selected + const embSubsetView = getEmbSubsetView(annoMatrix); + const subsetResetPossible = !embSubsetView + ? annoMatrix.nObs !== annoMatrix.schema.dataframe.nObs + : annoMatrix.nObs !== embSubsetView.nObs; return { subsetPossible, diff --git a/client/src/util/stateManager/viewStackHelpers.js b/client/src/util/stateManager/viewStackHelpers.js new file mode 100644 index 00000000..c1f3bac8 --- /dev/null +++ b/client/src/util/stateManager/viewStackHelpers.js @@ -0,0 +1,175 @@ +/* +The annoMatrix view stack has a set of conventions which are assumed elsewhere in the +application. These helper functions make it simple for action creators to manage +the stack. + +The annoMatrix module does not care about this order, but we maintain it as +a convention to make it simpler to manipulate the views. + +Terminology: +- clip view: AnnoMatrixClipView +- subset view: AnnoMatrixRowSubsetView +- user subset view: create by the user explicitly subsetting by selection +- embedding subset view: implicitly created by switching the current embedding +- loader, or base annoMatrix: the root, which loads data + +Rules: +1. there will be zero or one clip view +2. there will be zero or more subset views +3. there will be zero or one embedding view +4. there will be one loader/base, which is always the bottom view +5. the view ordering MUST be (top to bottom): + + [clip] -> [user subset] -> [embedding subset] -> loader + +There is code elsewhere in the app (eg, menubar/clip.js) which assumes this order. + +Views can be interogated for their type with the following: + +* is a view: annoMatrix.isView +* is the loader: !anonMatrix.isView (or annoMatrix === annoMatrix.base()) +* is a clip view: annoMatrix.isClipped (or annoMatrix.clipRange) +* is a subset view: (annoMatrix.isView && !annoMatrix.isClipped) +* is a user subset view: annoMatrix.userFlags?.isUserSubsetView +* is an embedding subset view: annomatrix.userFlags?.isEmbSubsetView + +*/ + +import { clip, isubsetMask, isubset } from "../../annoMatrix"; +import { memoize } from "../dataframe/util"; + +export function _clipAnnoMatrix(annoMatrix, min, max) { + /* + clip the annoMatrix. + */ + return annoMatrix.isClipped + ? clip(annoMatrix.viewOf, min, max) + : clip(annoMatrix, min, max); +} + +export function _userSubsetAnnoMatrix(annoMatrix, mask) { + /* + user-requested row subset of annoMatrix, to be added on top of any + other previous row subsets. + */ + const { clipRange } = annoMatrix; + if (clipRange) { + annoMatrix = annoMatrix.viewOf; + } + + annoMatrix = isubsetMask(annoMatrix, mask); + annoMatrix.userFlags.isUserSubsetView = true; + + if (clipRange) { + annoMatrix = clip(annoMatrix, ...clipRange); + } + + return annoMatrix; +} + +export function _userResetSubsetAnnoMatrix(annoMatrix) { + /* + Reset/remove all user-requested subsets. Do not remove clip or embedding subset. + */ + + /* stash clipping info, if any */ + const { clipRange } = annoMatrix; + if (clipRange) { + annoMatrix = annoMatrix.viewOf; + } + + /* pop all views except embedding subset and loader */ + while (annoMatrix.isView && annoMatrix.userFlags.isUserSubsetView) { + annoMatrix = annoMatrix.viewOf; + } + + /* re-apply the clip, if any */ + if (clipRange) { + annoMatrix = clip(annoMatrix, ...clipRange); + } + + return annoMatrix; +} + +export function _setEmbeddingSubset(annoMatrix, embeddingDf) { + /* + Set the embedding subset view. Only create a subset view for the embedding + when it is needed, ie, there are NaN values in the embeddings. + */ + const embRowOffsets = _getEmbeddingRowOffsets( + annoMatrix.rowIndex, + embeddingDf + ); + + const curEmbSubsetView = getEmbSubsetView(annoMatrix); + + /* if no current embedding subset, and no new embedding subset, just noop */ + if (!embRowOffsets && !curEmbSubsetView) return annoMatrix; + + // ... otherwise, do the work + + /* stash clipping info, if any */ + const clipRange = annoMatrix.isClipped ? annoMatrix.clipRange : null; + + /* pop all subsets, user or embedding */ + while (annoMatrix.isView) { + annoMatrix = annoMatrix.viewOf; + } + + /* apply new embedding row index, if needed */ + if (embRowOffsets) { + annoMatrix = isubset(annoMatrix, embRowOffsets); + annoMatrix.userFlags.isEmbSubsetView = true; + } + + /* re-apply clip, if needed */ + if (clipRange) { + annoMatrix = clip(annoMatrix, ...clipRange); + } + + return annoMatrix; +} + +function _getEmbeddingRowOffsets(baseRowIndex, embeddingDf) { + /* + given a dataframe containing an embedding: + - if the embedding contains no NaN coordinates, return null + - if the embedding contains NaN coordinates, return a rowIndex + that contains only the rows with discrete valued coordinates. + + Currently assumes that there will be onl two dimensions in the embedding. + */ + const X = embeddingDf.icol(0).asArray(); + const Y = embeddingDf.icol(1).asArray(); + const offsets = new Int32Array(X.length); + let numOffsets = 0; + + for (let i = 0, l = X.length; i < l; i += 1) { + if (!Number.isNaN(X[i]) && !Number.isNaN(Y[i])) { + offsets[numOffsets] = i; + numOffsets += 1; + } + } + + if (numOffsets === X.length) return null; + return offsets.subarray(0, numOffsets); +} + +export function _getDiscreteCellEmbeddingRowIndex(embeddingDf) { + const idx = _getEmbeddingRowOffsets(embeddingDf.rowIndex, embeddingDf); + if (idx === null) return embeddingDf.rowIndex; + return embeddingDf.rowIndex.isubset(idx); +} +export const getDiscreteCellEmbeddingRowIndex = memoize( + _getDiscreteCellEmbeddingRowIndex, + (df) => df.__id +); + +export function getEmbSubsetView(annoMatrix) { + /* if there is an embedding subset in the view stack, return it. Falsish if not. */ + while (annoMatrix.isView) { + if (annoMatrix.userFlags.isEmbSubsetView) return annoMatrix; + annoMatrix = annoMatrix.viewOf; + } + return undefined; +} From 14cd1b9f0b59fdf4ec5faa12a1ab3024beb0f38b Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Mon, 27 Jul 2020 12:52:22 -0700 Subject: [PATCH 10/55] work around blueprint restriction (#1677) --- client/src/components/embedding/index.js | 91 +++++++++++++----------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/client/src/components/embedding/index.js b/client/src/components/embedding/index.js index 586218d8..337843ab 100644 --- a/client/src/components/embedding/index.js +++ b/client/src/components/embedding/index.js @@ -1,6 +1,6 @@ import React from "react"; import { connect } from "react-redux"; -import Async from "react-async"; +import { useAsync } from "react-async"; import { ButtonGroup, Popover, @@ -84,18 +84,11 @@ class Embedding extends React.PureComponent {

There are {schema?.dataframe?.nObs} cells in the entire dataset.

- - {layoutChoice.available.map((name) => ( - - ))} - + annoMatrix={annoMatrix} + layoutChoice={layoutChoice} + /> } /> @@ -106,40 +99,58 @@ class Embedding extends React.PureComponent { export default Embedding; -const loadEmbeddingCounts = async ({ annoMatrix, layoutName }) => { - const embedding = await annoMatrix.fetch("emb", layoutName); - const discreteCellIndex = getDiscreteCellEmbeddingRowIndex(embedding); - return { embedding, discreteCellIndex }; +const loadAllEmbeddingCounts = async ({ annoMatrix, available }) => { + const embeddings = await Promise.all( + available.map((name) => annoMatrix.fetch("emb", name)) + ); + return available.map((name, idx) => ({ + embeddingName: name, + embedding: embeddings[idx], + discreteCellIndex: getDiscreteCellEmbeddingRowIndex(embeddings[idx]), + })); }; -const LayoutChoice = ({ annoMatrix, layoutName }) => { - return ( - - {({ data, error, isPending }) => { - if (error) { - /* log, as this is unexpected */ - console.error(error); - } - if (error || isPending) { - /* still loading, or errored out - just omit counts (TODO: spinner?) */ - return ; - } - if (data) { - const { embedding, discreteCellIndex } = data; +const EmbeddingChoices = ({ onChange, annoMatrix, layoutChoice }) => { + const { available } = layoutChoice; + const { data, error, isPending } = useAsync({ + promiseFn: loadAllEmbeddingCounts, + annoMatrix, + available, + }); + + if (error) { + /* log, as this is unexpected */ + console.error(error); + } + if (error || isPending) { + /* still loading, or errored out - just omit counts (TODO: spinner?) */ + return ( + + {layoutChoice.available.map((name) => ( + + ))} + + ); + } + if (data) { + return ( + + {data.map((summary) => { + const { discreteCellIndex, embedding, embeddingName } = summary; const isAllCells = discreteCellIndex.size() === embedding.length; const sizeHint = `${discreteCellIndex.size()} ${ isAllCells ? "(all) " : "" }cells`; return ( - + ); - } - return null; - }} - - ); + })} + + ); + } + return null; }; From 98c5cae9f4b57ab23512e48339895422bf057c4d Mon Sep 17 00:00:00 2001 From: maniarathi Date: Mon, 27 Jul 2020 12:55:44 -0700 Subject: [PATCH 11/55] Update issue templates --- .github/ISSUE_TEMPLATE/---bug-report.md | 32 ++++++++++++++++++++ .github/ISSUE_TEMPLATE/---feature-request.md | 20 ++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/---bug-report.md create mode 100644 .github/ISSUE_TEMPLATE/---feature-request.md diff --git a/.github/ISSUE_TEMPLATE/---bug-report.md b/.github/ISSUE_TEMPLATE/---bug-report.md new file mode 100644 index 00000000..099be093 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/---bug-report.md @@ -0,0 +1,32 @@ +--- +name: "\U0001F41E Bug report" +about: Create a report to identify a bug in cellxgene +title: "[BUG]" +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Version (please complete the following information):** + - Desktop or hosted?: + - Browser (if hosted) [e.g. chrome, safari]: + - Version [e.g. 0.13.0]: + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/---feature-request.md b/.github/ISSUE_TEMPLATE/---feature-request.md new file mode 100644 index 00000000..4983c5a8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/---feature-request.md @@ -0,0 +1,20 @@ +--- +name: "\U0001F4A1 Feature request" +about: Suggest an idea for this project +title: "[FEATURE REQUEST]" +labels: user request +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. From 38ce1f90fb3428269d38fe2ccc260c55662e263e Mon Sep 17 00:00:00 2001 From: maniarathi Date: Mon, 27 Jul 2020 12:57:59 -0700 Subject: [PATCH 12/55] Update issue templates --- .github/ISSUE_TEMPLATE/---question-clarification.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/---question-clarification.md diff --git a/.github/ISSUE_TEMPLATE/---question-clarification.md b/.github/ISSUE_TEMPLATE/---question-clarification.md new file mode 100644 index 00000000..aefca3a7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/---question-clarification.md @@ -0,0 +1,10 @@ +--- +name: "\U0001F9D0 Question/Clarification" +about: Ask a question or for a clarification +title: "[QUESTION]" +labels: question +assignees: '' + +--- + + From bbef27b8c9c0dc97e0fb70506c783e93683dae95 Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Mon, 27 Jul 2020 19:34:04 -0700 Subject: [PATCH 13/55] minor prose change on embedding chooser (#1678) --- client/src/components/embedding/index.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/client/src/components/embedding/index.js b/client/src/components/embedding/index.js index 337843ab..b71ae1a0 100644 --- a/client/src/components/embedding/index.js +++ b/client/src/components/embedding/index.js @@ -136,11 +136,8 @@ const EmbeddingChoices = ({ onChange, annoMatrix, layoutChoice }) => { return ( {data.map((summary) => { - const { discreteCellIndex, embedding, embeddingName } = summary; - const isAllCells = discreteCellIndex.size() === embedding.length; - const sizeHint = `${discreteCellIndex.size()} ${ - isAllCells ? "(all) " : "" - }cells`; + const { discreteCellIndex, embeddingName } = summary; + const sizeHint = `${discreteCellIndex.size()} cells`; return ( Date: Tue, 28 Jul 2020 13:28:30 -0700 Subject: [PATCH 14/55] Add basic authentication in the server (#1670) * Add basic authentication in the server A pattern for creating authentication methods is introduced, with three authentication types defined: none - no authentication session - like the current session based auth used for user annotations test - used to test the login/logout process end to end The config endpoint now returns informations about the authentication, like if the user is authenticated and their username. The redirect uri's for login and logout are also returned if the authentication type requires login This is the first a several PRs for authentication. *. Update server tests to avoid hardcoded ports test_api and test_nan_rest now use a common function for starting a test server, than will initially choose a random port. --- server/app/app.py | 18 ++++ server/auth/__init__.py | 6 ++ server/auth/auth.py | 80 ++++++++++++++++ server/auth/auth_none.py | 27 ++++++ server/auth/auth_session.py | 36 ++++++++ server/auth/auth_test.py | 69 ++++++++++++++ server/common/annotations.py | 19 ++-- server/common/app_config.py | 39 ++++++++ server/common/default_config.py | 12 +++ server/data_common/data_adaptor.py | 5 + server/test/__init__.py | 29 ++++-- server/test/test_api.py | 120 ++++++------------------ server/test/test_auth.py | 142 +++++++++++++++++++++++++++++ server/test/test_nan_rest.py | 28 ++---- 14 files changed, 498 insertions(+), 132 deletions(-) create mode 100644 server/auth/__init__.py create mode 100644 server/auth/auth.py create mode 100644 server/auth/auth_none.py create mode 100644 server/auth/auth_session.py create mode 100644 server/auth/auth_test.py create mode 100644 server/test/test_auth.py diff --git a/server/app/app.py b/server/app/app.py index e290ca35..1f59513c 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -85,6 +85,7 @@ def dataset_index(url_dataroot=None, dataset=None): try: cache_manager = current_app.matrix_data_cache_manager with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor: + data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}") dataset_title = app_config.get_title(data_adaptor) return render_template( "index.html", datasetTitle=dataset_title, SCRIPTS=scripts, INLINE_SCRIPTS=inline_scripts @@ -143,6 +144,7 @@ def rest_get_data_adaptor(func): def wrapped_function(self, dataset=None): try: with get_data_adaptor(self.url_dataroot, dataset) as data_adaptor: + data_adaptor.set_uri_path(f"{self.url_dataroot}/{dataset}") return func(self, data_adaptor) except DatasetAccessError as e: return common_rest.abort_and_log( @@ -160,6 +162,17 @@ def dataroot_test_index(): config = current_app.app_config server_config = config.server_config + + auth = server_config.auth + if auth.is_valid(): + if server_config.auth.is_authenticated(): + data += f"

Logged in as {auth.get_userid()} / {auth.get_username()}

" + if auth.requires_client_login(): + if server_config.auth.is_authenticated(): + data += "

Logout

" + else: + data += "

Login

" + datasets = [] for dataroot_dict in server_config.multi_dataset__dataroot.values(): dataroot = dataroot_dict["dataroot"] @@ -338,10 +351,15 @@ class Server: lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset), methods=["GET"], ) + else: bp_api = Blueprint("api", __name__, url_prefix=api_version) resources = get_api_resources(bp_api) self.app.register_blueprint(resources.blueprint) + self.app.auth = server_config.auth + if self.app.auth.requires_client_login(): + self.app.auth.add_url_rules(self.app) + self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager self.app.app_config = app_config diff --git a/server/auth/__init__.py b/server/auth/__init__.py new file mode 100644 index 00000000..2af0d1ef --- /dev/null +++ b/server/auth/__init__.py @@ -0,0 +1,6 @@ + +# import the built in auth types so they can be registered + +import server.auth.auth_none # noqa: F401 +import server.auth.auth_test # noqa: F401 +import server.auth.auth_session # noqa: F401 diff --git a/server/auth/auth.py b/server/auth/auth.py new file mode 100644 index 00000000..3262a9c1 --- /dev/null +++ b/server/auth/auth.py @@ -0,0 +1,80 @@ +from abc import ABC, abstractmethod + + +class AuthTypeBase(ABC): + """Base type for all authentication types.""" + + def __init__(self): + super().__init__() + + @abstractmethod + def set_params(self, params): + """Set the parameters from app config. raise ConfigurationError if any params are invalid""" + pass + + @abstractmethod + def is_valid(self): + """Return True if the auth type can return user info (AuthTypeNone is the only one that cannot)""" + pass + + def requires_client_login(self): + """Return True if the user needs to login from the client (e.g. Login button is shown)""" + return False + + @abstractmethod + def is_authenticated(self): + """Return True if the user is authenticated""" + pass + + @abstractmethod + def get_userid(self): + """Return the id for this user (string)""" + pass + + @abstractmethod + def get_username(self): + """Return the name of the user (string)""" + pass + + +class AuthTypeClientBase(AuthTypeBase): + """Base type for all authentication types that require the client to login""" + + def __init__(self): + super().__init__() + + def requires_client_login(self): + return True + + @abstractmethod + def add_url_rules(self, selfapp): + """Add url rules to the app (like /login, /logout, etc)""" + pass + + @abstractmethod + def get_login_url(self, data_adaptor): + """Return the url for the login route""" + pass + + @abstractmethod + def get_logout_url(self, data_adaptor): + """Return the url for the logout route""" + pass + + +class AuthTypeFactory: + """Factory class to create an authentication type""" + + auth_types = {} + + @staticmethod + def register(name, auth_type): + assert(issubclass(auth_type, AuthTypeBase)) + AuthTypeFactory.auth_types[name] = auth_type + + @staticmethod + def create(name): + auth_type = AuthTypeFactory.auth_types.get(name) + if auth_type is None: + return None + return auth_type() diff --git a/server/auth/auth_none.py b/server/auth/auth_none.py new file mode 100644 index 00000000..9b2b8c0a --- /dev/null +++ b/server/auth/auth_none.py @@ -0,0 +1,27 @@ +from server.auth.auth import AuthTypeBase, AuthTypeFactory +from server.common.errors import ConfigurationError + + +class AuthTypeNone(AuthTypeBase): + + def __init__(self): + super().__init__() + + def is_valid(self): + return False + + def set_params(self, params): + if params: + raise ConfigurationError("not expecting authentication parameters") + + def is_authenticated(self): + return True + + def get_userid(self): + return None + + def get_username(self): + return None + + +AuthTypeFactory.register(None, AuthTypeNone) diff --git a/server/auth/auth_session.py b/server/auth/auth_session.py new file mode 100644 index 00000000..d5754bc4 --- /dev/null +++ b/server/auth/auth_session.py @@ -0,0 +1,36 @@ +from server.auth.auth import AuthTypeBase, AuthTypeFactory +from flask import session +from uuid import uuid4 + + +class AuthTypeSession(AuthTypeBase): + """Session based authentication. The user is always logged. The user id is a random number + associated with the session. This is a good choice for desktop servers.""" + + # key in the session token for userid + CXGUID = "cxguid" + + def __init__(self): + super().__init__() + + def is_valid(self): + return True + + def set_params(self, params): + return + + def is_authenticated(self): + # always authenticated + return True + + def get_userid(self): + if self.CXGUID not in session: + session[self.CXGUID] = uuid4().hex + session.permanent = True + return session[self.CXGUID] + + def get_username(self): + return "anonymous" + + +AuthTypeFactory.register("session", AuthTypeSession) diff --git a/server/auth/auth_test.py b/server/auth/auth_test.py new file mode 100644 index 00000000..d2222c12 --- /dev/null +++ b/server/auth/auth_test.py @@ -0,0 +1,69 @@ +from server.auth.auth import AuthTypeClientBase, AuthTypeFactory +from flask import session, request, redirect, current_app + + +class AuthTypeTest(AuthTypeClientBase): + """An authentication type for testing client based logins. When the login route is accessed + the user is automatically logged in with a default or configured username""" + + # key in session token with userid and username + CXGUID = "cxguid_test" + CXGUNAME = "cxguname_test" + + def __init__(self): + super().__init__() + self.username = "test_account" + self.userid = "id0001" + + def is_valid(self): + return True + + def requires_client_login(self): + return True + + def add_url_rules(self, app): + app.add_url_rule("/login", "login", self.login, methods=["GET"]) + app.add_url_rule("/logout", "logout", self.logout, methods=["GET"]) + + def set_params(self, params): + if params: + self.username = params.get("username", self.username) + self.userid = params.get("userid", self.userid) + + def is_authenticated(self): + return self.CXGUID in session + + def get_userid(self): + return session.get(self.CXGUID) + + def get_username(self): + return session.get(self.CXGUNAME) + + def login(self): + args = request.args + return_to = args.get("dataset", "/") + session[self.CXGUID] = args.get("userid", self.userid) + session[self.CXGUNAME] = args.get("username", self.username) + return redirect(return_to) + + def logout(self): + session.clear() + return_to = request.args.get("dataset", "/") + return redirect(return_to) + + def get_login_url(self, data_adaptor): + """Return the url for the login route""" + if current_app.app_config.is_multi_dataset(): + return f"/login?dataset={data_adaptor.uri_path}" + else: + return "/login" + + def get_logout_url(self, data_adaptor): + """Return the url for the logout route""" + if current_app.app_config.is_multi_dataset(): + return f"/logout?dataset={data_adaptor.uri_path}" + else: + return "/logout" + + +AuthTypeFactory.register("test", AuthTypeTest) diff --git a/server/common/annotations.py b/server/common/annotations.py index 45e53122..25176ccb 100644 --- a/server/common/annotations.py +++ b/server/common/annotations.py @@ -1,6 +1,5 @@ from datetime import datetime import re -from uuid import uuid4 import os import pandas as pd from hashlib import blake2b @@ -11,7 +10,7 @@ from server.common.errors import AnnotationsError, OntologyLoadFailure from server.common.utils import series_to_schema import fsspec import fastobo -from flask import session +from flask import session, current_app from abc import ABCMeta, abstractmethod @@ -80,7 +79,6 @@ class Annotations(metaclass=ABCMeta): class AnnotationsLocalFile(Annotations): - CXGUID = "cxguid" CXG_ANNO_COLLECTION = "cxg_anno_collection" def __init__(self, output_dir, output_file): @@ -159,18 +157,12 @@ class AnnotationsLocalFile(Annotations): self.last_fname = fname self.last_labels = df - def _get_userid(self): - if self.CXGUID not in session: - session[self.CXGUID] = uuid4().hex - session.permanent = True - return session[self.CXGUID] - def _get_userdata_idhash(self, data_adaptor): """ Return a short hash that weakly identifies the user and dataset. Used to create safe annotations output file names. """ - uid = self._get_userid() + uid = current_app.auth.get_userid() id = (uid + data_adaptor.get_location()).encode() idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8") return idhash @@ -257,8 +249,9 @@ class AnnotationsLocalFile(Annotations): elif session is not None: collection = self.get_collection() - params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor) - params["annotations-data-collection-is-read-only"] = False - params["annotations-data-collection-name"] = collection + if current_app.auth.is_authenticated(): + params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor) + params["annotations-data-collection-is-read-only"] = False + params["annotations-data-collection-name"] = collection parameters.update(params) diff --git a/server/common/app_config.py b/server/common/app_config.py index 3731f9b4..dab9d6a7 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -16,6 +16,7 @@ from server.common.annotations import AnnotationsLocalFile from server.common.utils import custom_format_warning import server.compute.diffexp_cxg as diffexp_tiledb from server.common.data_locator import discover_s3_region_name +from server.auth.auth import AuthTypeFactory DEFAULT_SERVER_PORT = 5005 # anything bigger than this will generate a special message @@ -194,6 +195,7 @@ class AppConfig(object): server_config = self.server_config dataset_config = data_adaptor.dataset_config annotation = dataset_config.user_annotations + auth = server_config.auth # FIXME The current set of config is not consistently presented: # we have camalCase, hyphen-text, and underscore_text @@ -257,6 +259,18 @@ class AppConfig(object): "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, } + if dataset_config.app__authentication_enable and auth.is_valid(): + config["authentication"] = { + "is_authenticated": auth.is_authenticated(), + "requires_client_login": auth.requires_client_login(), + "username": auth.get_username(), + } + if auth.requires_client_login(): + config["authentication"].update({ + "login": auth.get_login_url(data_adaptor), + "logout" : auth.get_logout_url(data_adaptor), + }) + return c @@ -366,6 +380,7 @@ class ServerConfig(BaseConfig): def __init__(self, app_config, default_config): dictval_cases = [ ("app", "csp_directives"), + ("authentication", "params"), ("adaptor", "cxg_adaptor", "tiledb_ctx"), ("multi_dataset", "dataroot"), ] @@ -384,6 +399,9 @@ class ServerConfig(BaseConfig): self.app__server_timing_headers = dc["app"]["server_timing_headers"] self.app__csp_directives = dc["app"]["csp_directives"] + self.authentication__type = dc["authentication"]["type"] + self.authentication__params = dc["authentication"]["params"] + self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"] self.multi_dataset__index = dc["multi_dataset"]["index"] self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"] @@ -414,8 +432,12 @@ class ServerConfig(BaseConfig): # The matrix data cache manager is created during the complete_config and stored here. self.matrix_data_cache_manager = None + # The authentication object (BCM -- better name) + self.auth = None + def complete_config(self, context): self.handle_app(context) + self.handle_authentication(context) self.handle_data_locator(context) self.handle_adaptor(context) # may depend on data_locator self.handle_single_dataset(context) # may depend on adaptor @@ -484,6 +506,14 @@ class ServerConfig(BaseConfig): elif not isinstance(v, str): raise ConfigurationError("CSP directive value must be a string or list of strings.") + def handle_authentication(self, context): + self.check_attr("authentication__type", (type(None), str)) + self.check_attr("authentication__params", (type(None), dict)) + self.auth = AuthTypeFactory.create(self.authentication__type) + if self.auth is None: + raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") + self.auth.set_params(self.authentication__params) + def handle_data_locator(self, context): self.check_attr("data_locator__s3__region_name", (type(None), bool, str)) if self.data_locator__s3__region_name is True: @@ -660,6 +690,7 @@ class DatasetConfig(BaseConfig): self.app__inline_scripts = dc["app"]["inline_scripts"] self.app__about_legal_tos = dc["app"]["about_legal_tos"] self.app__about_legal_privacy = dc["app"]["about_legal_privacy"] + self.app__authentication_enable = dc["app"]["authentication_enable"] self.presentation__max_categories = dc["presentation"]["max_categories"] self.presentation__custom_colors = dc["presentation"]["custom_colors"] @@ -696,6 +727,7 @@ class DatasetConfig(BaseConfig): self.check_attr("app__inline_scripts", list) self.check_attr("app__about_legal_tos", (type(None), str)) self.check_attr("app__about_legal_privacy", (type(None), str)) + self.check_attr("app__authentication_enable", bool) # scripts can be string (filename) or dict (attributes). Convert string to dict. scripts = [] @@ -721,6 +753,13 @@ class DatasetConfig(BaseConfig): self.check_attr("user_annotations__ontology__obo_location", (type(None), str)) if self.user_annotations__enable: + server_config = self.app_config.server_config + if not self.app__authentication_enable: + raise ConfigurationError("user annotations requires authentication to be enabled") + if not server_config.auth.is_valid(): + auth_type = server_config.authentication__type + raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") + # TODO, replace this with a factory pattern once we have more than one way # to do annotations. currently only local_file_csv if self.user_annotations__type != "local_file_csv": diff --git a/server/common/default_config.py b/server/common/default_config.py index 0b0c5118..bc585391 100644 --- a/server/common/default_config.py +++ b/server/common/default_config.py @@ -14,6 +14,15 @@ server: server_timing_headers: false csp_directives: null + authentication: + # The authentication types may be "none" or "session" + # none: No authentication support, features like user_annotations must not be enabled. + # session: A session based userid is automatically generated. + type: session + + # a dictionary of parameters that may be required for an authentication type + params: null + multi_dataset: # If dataroot is set, then cellxgene may serve multiple datasets. This parameter is not # compatible with single_dataset/datapath. @@ -132,6 +141,9 @@ dataset: about_legal_tos: null about_legal_privacy: null + # allow authentication support + authentication_enable: true + presentation: max_categories: 1000 custom_colors: true diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index eabec8d1..14a07b5e 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -28,6 +28,11 @@ class DataAdaptor(metaclass=ABCMeta): # parameters set by this data adaptor based on the data. self.parameters = {} + self.uri_path = None + + def set_uri_path(self, path): + # uri path to the dataset, e.g. /d/ + self.uri_path = path @staticmethod @abstractmethod diff --git a/server/test/__init__.py b/server/test/__init__.py index bc82be90..44103190 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -86,15 +86,14 @@ def random_string(n): return "".join(random.choice(string.ascii_letters) for _ in range(n)) -@contextmanager -def test_server(command_line_args=[], app_config=None): - """A context to run the cellxgene server. +def start_test_server(command_line_args=[], app_config=None): + """ Command line arguments can be passed in, as well as an app_config. This function is meant to be used like this, for example: with test_server(...) as server: - r = requests.get(f"{server}/...") - // check r + r = requests.get(f"{server}/...") + // check r where the server can be accessed within the context, and is terminated when the context is exited. @@ -104,7 +103,8 @@ def test_server(command_line_args=[], app_config=None): yaml config file, which this server will read and parse. """ - port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT)) + start = random.randint(DEFAULT_SERVER_PORT, 2**16 - 1) + port = int(os.environ.get("CXG_SERVER_PORT", start)) port = find_available_port("localhost", port) command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose", "--port=%d" % port] + command_line_args @@ -128,10 +128,25 @@ def test_server(command_line_args=[], app_config=None): if tempdir: tempdir.cleanup() + return ps, server + + +def stop_test_server(ps): + try: + ps.terminate() + except ProcessLookupError: + pass + + +@contextmanager +def test_server(command_line_args=[], app_config=None): + """A context to run the cellxgene server.""" + + ps, server = start_test_server(command_line_args, app_config) try: yield server finally: try: - ps.terminate() + stop_test_server(ps) except ProcessLookupError: pass diff --git a/server/test/test_api.py b/server/test/test_api.py index dad7ea5e..d3ecb315 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -2,7 +2,6 @@ import shutil import time import unittest from http import HTTPStatus -from subprocess import Popen import pandas as pd import requests @@ -11,6 +10,7 @@ import server.test.decode_fbs as decode_fbs from server.data_common.matrix_loader import MatrixDataType from server.test import data_with_tmp_annotations, make_fbs, PROJECT_ROOT from server.test.test_datasets.fixtures import pbmc3k_colors +from server.test import start_test_server, stop_test_server BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} @@ -21,9 +21,6 @@ BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} class EndPoints(object): ANNOTATIONS_ENABLED = True - def setUp(self): - self.session = requests.Session() - def test_initialize(self): endpoint = "schema" url = f"{self.URL_BASE}{endpoint}" @@ -308,13 +305,13 @@ class EndPoints(object): def test_static(self): endpoint = "static" file = "assets/favicon.ico" - url = f"{self.LOCAL_URL}{endpoint}/{file}" + url = f"{self.server}/{endpoint}/{file}" result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) - @staticmethod - def _setUpClass(child_class, start_command): - child_class.ps = Popen(start_command) + def _setupClass(child_class, command_line): + child_class.ps, child_class.server = start_test_server(command_line) + child_class.URL_BASE = f"{child_class.server}/api/v0.2/" child_class.session = requests.Session() for i in range(90): try: @@ -323,13 +320,6 @@ class EndPoints(object): except requests.exceptions.ConnectionError: time.sleep(1) - @staticmethod - def _tearDownClass(child_class): - try: - child_class.ps.terminate() - except ProcessLookupError: - pass - class EndPointsAnnotations(EndPoints): def test_get_schema_existing_writable(self): @@ -385,32 +375,19 @@ class EndPointsAnnotations(EndPoints): class EndPointsAnndata(unittest.TestCase, EndPoints): """Test Case for endpoints""" - PORT = 5010 - LOCAL_URL = f"http://127.0.0.1:{PORT}/" - VERSION = "v0.2" - URL_BASE = f"{LOCAL_URL}api/{VERSION}/" ANNOTATIONS_ENABLED = False @classmethod def setUpClass(cls): - cls._setUpClass( - cls, - [ - "cellxgene", - "--no-upgrade-check", - "launch", - f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", - "--disable-annotations", - "--verbose", - "--experimental-enable-reembedding", - "--port", - str(cls.PORT), - ], - ) + cls._setupClass(cls, [ + f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + "--disable-annotations", + "--experimental-enable-reembedding", + ]) @classmethod def tearDownClass(cls): - cls._tearDownClass(cls) + stop_test_server(cls.ps) @property def annotations_enabled(self): @@ -420,98 +397,57 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): class EndPointsCxg(unittest.TestCase, EndPoints): """Test Case for endpoints""" - PORT = 5011 - LOCAL_URL = f"http://127.0.0.1:{PORT}/" - VERSION = "v0.2" - URL_BASE = f"{LOCAL_URL}api/{VERSION}/" ANNOTATIONS_ENABLED = False @classmethod def setUpClass(cls): - cls._setUpClass( - cls, - [ - "cellxgene", - "--no-upgrade-check", - "launch", - f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg", - "--disable-annotations", - "--verbose", - "--port", - str(cls.PORT), - ], - ) + cls._setupClass(cls, [ + f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg", + "--disable-annotations", + ]) @classmethod def tearDownClass(cls): - cls._tearDownClass(cls) + stop_test_server(cls.ps) class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations): """Test Case for endpoints""" - PORT = 5012 - LOCAL_URL = f"http://127.0.0.1:{PORT}/" - VERSION = "v0.2" - URL_BASE = f"{LOCAL_URL}api/{VERSION}/" ANNOTATIONS_ENABLED = True - MATRIX_DATA_TYPE = MatrixDataType.H5AD @classmethod def setUpClass(cls): cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations( MatrixDataType.H5AD, annotations_fixture=True ) - cls._setUpClass( - cls, - [ - "cellxgene", - "--no-upgrade-check", - "launch", - "--annotations-file", - cls.annotations.output_file, - "--verbose", - "--port", - str(cls.PORT), - cls.data.get_location(), - ], - ) + cls._setupClass(cls, [ + "--annotations-file", + cls.annotations.output_file, + cls.data.get_location(), + ]) @classmethod def tearDownClass(cls): shutil.rmtree(cls.tmp_dir) - cls._tearDownClass(cls) + stop_test_server(cls.ps) class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations): """Test Case for endpoints""" - PORT = 5013 - LOCAL_URL = f"http://127.0.0.1:{PORT}/" - VERSION = "v0.2" - URL_BASE = f"{LOCAL_URL}api/{VERSION}/" ANNOTATIONS_ENABLED = True - MATRIX_DATA_TYPE = MatrixDataType.CXG @classmethod def setUpClass(cls): cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True) - cls._setUpClass( - cls, - [ - "cellxgene", - "--no-upgrade-check", - "launch", - "--annotations-file", - cls.annotations.output_file, - "--verbose", - "--port", - str(cls.PORT), - cls.data.get_location(), - ], - ) + cls._setupClass(cls, [ + "--annotations-file", + cls.annotations.output_file, + cls.data.get_location(), + ]) @classmethod def tearDownClass(cls): shutil.rmtree(cls.tmp_dir) - cls._tearDownClass(cls) + stop_test_server(cls.ps) diff --git a/server/test/test_auth.py b/server/test/test_auth.py new file mode 100644 index 00000000..bd4c776e --- /dev/null +++ b/server/test/test_auth.py @@ -0,0 +1,142 @@ +import unittest +from server.common.app_config import AppConfig +from server.test import PROJECT_ROOT, test_server +import requests + + +class AuthTest(unittest.TestCase): + def test_auth_none(self): + c = AppConfig() + c.update_server_config( + authentication__type=None, multi_dataset__dataroot=f"{PROJECT_ROOT}/server/test/test_datasets" + ) + c.update_default_dataset_config(user_annotations__enable=False) + + c.complete_config() + + with test_server(app_config=c) as server: + session = requests.Session() + r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config") + data_config = r.json() + assert "authentication" not in data_config["config"] + + def test_auth_session(self): + c = AppConfig() + c.update_server_config( + authentication__type="session", multi_dataset__dataroot=f"{PROJECT_ROOT}/server/test/test_datasets" + ) + c.update_default_dataset_config(user_annotations__enable=True) + c.complete_config() + + with test_server(app_config=c) as server: + session = requests.Session() + r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config") + data_config = r.json() + assert data_config["config"]["authentication"]["is_authenticated"] + assert not data_config["config"]["authentication"]["requires_client_login"] + assert data_config["config"]["authentication"]["username"] == "anonymous" + + def test_auth_test(self): + c = AppConfig() + c.update_server_config(authentication__type="test") + c.update_server_config( + multi_dataset__dataroot=dict( + a1=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="auth"), + a2=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="no-auth"), + ) + ) + + # specialize the configs + c.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True) + c.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False) + + c.complete_config() + + with test_server(app_config=c) as server: + session = requests.Session() + + # auth datasets + r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config") + data_config = r.json() + assert not data_config["config"]["authentication"]["is_authenticated"] + assert data_config["config"]["authentication"]["requires_client_login"] + assert data_config["config"]["authentication"]["username"] is None + assert data_config["config"]["parameters"]["annotations"] + + login_uri = data_config["config"]["authentication"]["login"] + logout_uri = data_config["config"]["authentication"]["logout"] + + assert login_uri == "/login?dataset=auth/pbmc3k.cxg" + assert logout_uri == "/logout?dataset=auth/pbmc3k.cxg" + + r = session.get(f"{server}/{login_uri}") + # check that the login redirect worked + assert r.history[0].status_code == 302 + assert r.url == f"{server}/auth/pbmc3k.cxg/" + + r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config") + data_config = r.json() + assert data_config["config"]["authentication"]["is_authenticated"] + assert data_config["config"]["authentication"]["username"] == "test_account" + assert data_config["config"]["parameters"]["annotations"] + + r = session.get(f"{server}/{logout_uri}") + # check that the logout redirect worked + assert r.history[0].status_code == 302 + assert r.url == f"{server}/auth/pbmc3k.cxg/" + r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config") + data_config = r.json() + assert not data_config["config"]["authentication"]["is_authenticated"] + assert data_config["config"]["authentication"]["username"] is None + assert data_config["config"]["parameters"]["annotations"] + + # no-auth datasets + r = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config") + data_config = r.json() + assert "authentication" not in data_config["config"] + assert not data_config["config"]["parameters"]["annotations"] + + def test_auth_test_single(self): + c = AppConfig() + c.update_server_config( + authentication__type="test", + single_dataset__datapath=f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg") + + c.complete_config() + + with test_server(app_config=c) as server: + session = requests.Session() + + r = session.get(f"{server}/api/v0.2/config") + data_config = r.json() + assert not data_config["config"]["authentication"]["is_authenticated"] + assert data_config["config"]["authentication"]["requires_client_login"] + assert data_config["config"]["authentication"]["username"] is None + assert data_config["config"]["parameters"]["annotations"] + + login_uri = data_config["config"]["authentication"]["login"] + logout_uri = data_config["config"]["authentication"]["logout"] + + assert login_uri == "/login" + assert logout_uri == "/logout" + + r = session.get(f"{server}/{login_uri}") + # check that the login redirect worked + assert r.history[0].status_code == 302 + assert r.url == f"{server}/" + + r = session.get(f"{server}/api/v0.2/config") + data_config = r.json() + assert data_config["config"]["authentication"]["is_authenticated"] + assert data_config["config"]["authentication"]["username"] == "test_account" + assert data_config["config"]["parameters"]["annotations"] + + r = session.get(f"{server}/{logout_uri}") + # check that the logout redirect worked + assert r.history[0].status_code == 302 + assert r.url == f"{server}/" + r = session.get(f"{server}/api/v0.2/config") + data_config = r.json() + assert not data_config["config"]["authentication"]["is_authenticated"] + assert data_config["config"]["authentication"]["username"] is None + assert data_config["config"]["parameters"]["annotations"] diff --git a/server/test/test_nan_rest.py b/server/test/test_nan_rest.py index 8e958eeb..00c072e5 100644 --- a/server/test/test_nan_rest.py +++ b/server/test/test_nan_rest.py @@ -1,17 +1,13 @@ from http import HTTPStatus -from subprocess import Popen import unittest -import time import math +from server.test import start_test_server, stop_test_server import server.test.decode_fbs as decode_fbs import requests -LOCAL_URL = "http://127.0.0.1:5006/" VERSION = "v0.2" -URL_BASE = f"{LOCAL_URL}api/{VERSION}/" - BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} @@ -20,33 +16,25 @@ class WithNaNs(unittest.TestCase): @classmethod def setUpClass(cls): - cls.ps = Popen(["cellxgene", "launch", "test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]) - session = requests.Session() - for i in range(90): - try: - session.get(f"{URL_BASE}schema") - except requests.exceptions.ConnectionError: - time.sleep(1) + cls.ps, cls.server = start_test_server(["test/test_datasets/nan.h5ad"]) @classmethod def tearDownClass(cls): - try: - cls.ps.terminate() - except ProcessLookupError: - pass + stop_test_server(cls.ps) def setUp(self): self.session = requests.Session() + self.url_base = f"{self.server}/api/{VERSION}/" def test_initialize(self): endpoint = "schema" - url = f"{URL_BASE}{endpoint}" + url = f"{self.url_base}{endpoint}" result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) def test_data(self): endpoint = "data/var" - url = f"{URL_BASE}{endpoint}" + url = f"{self.url_base}{endpoint}" filter = {"filter": {"var": {"index": [[0, 20]]}}} result = self.session.put(url, json=filter) self.assertEqual(result.status_code, HTTPStatus.OK) @@ -56,7 +44,7 @@ class WithNaNs(unittest.TestCase): def test_annotation_obs(self): endpoint = "annotations/obs" - url = f"{URL_BASE}{endpoint}" + url = f"{self.url_base}{endpoint}" result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") @@ -65,7 +53,7 @@ class WithNaNs(unittest.TestCase): def test_annotation_var(self): endpoint = "annotations/var" - url = f"{URL_BASE}{endpoint}" + url = f"{self.url_base}{endpoint}" result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/octet-stream") From 59f989d26f1227554b137113dcde3324b000997e Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Tue, 28 Jul 2020 17:32:27 -0700 Subject: [PATCH 15/55] initial support for corpora schema conventions (#1676) * initial support for corpora schema conventions * remove debugging print * add corpora util module * tests * lint * PR review edits * PR changes * more PR changes * more PR chnages * PR fixes * formatting * PR updates * lint * PR review --- dev_docs/cxg.md | 162 +++++++++++++++++ server/app/app.py | 2 +- server/common/app_config.py | 10 +- server/common/corpora.py | 90 ++++++++++ server/converters/cxgtool.py | 169 +++++++++--------- server/data_anndata/anndata_adaptor.py | 4 + server/data_common/data_adaptor.py | 3 + server/data_cxg/cxg_adaptor.py | 16 +- server/test/test_anndata_adaptor_data_load.py | 7 +- server/test/test_corpora.py | 73 ++++++++ server/test/test_cxgtool.py | 7 +- server/test/test_diffexp.py | 11 +- 12 files changed, 450 insertions(+), 104 deletions(-) create mode 100644 dev_docs/cxg.md create mode 100644 server/common/corpora.py create mode 100644 server/test/test_corpora.py diff --git a/dev_docs/cxg.md b/dev_docs/cxg.md new file mode 100644 index 00000000..e851692f --- /dev/null +++ b/dev_docs/cxg.md @@ -0,0 +1,162 @@ +# CXG Data Format Specification + +Document Status: _draft_ + +Version: 0.2.0 (_DRAFT, not yet approved_) + +Date Last Modified: 2020-07-23 + +## Introduction + +CXG is a cellxgene-private data format, used for at-rest storage of annotated matrix data. It is similar to [AnnData](https://anndata.readthedocs.io/en/stable/), but with performance and access characteristics amenable to a multi-dataset, multi-user serving environment. + +CXG is built upon the [TileDB](https://tiledb.com/) embedded database. Each CXG is a TileDB [group](https://docs.tiledb.com/main/api-usage/object-management), which in turn includes one or more TileDB multi-dimensional arrays. + +This document presumes familiarity with [TileDB terminology and concepts](https://docs.tiledb.com/main/), the [Corpora schema](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md) and its [H5AD encoding](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md), and the AnnData/H5AD data model. + +This document also leverages the current cellxgene schema, which is documented in the [REST API spec](./REST_API.md). + +### Terminology + +Unless explicitly noted, the AnnData conventions and terminology are adopted when referring to general annotated matrix characteristics (eg, `n_obs` is the number of observations/rows/cells in the annotated matrix). Where implied by context, eg, "TileDB array attribute", domain-specific terms are used. + +Where capitalized, [IETF RFC 2119](https://www.ietf.org/rfc/rfc2119.txt) conventions are followed (ie, conventions MUST be followed). + +_Author's note:_ if you see any ambiguous terms, please call them out for clarification. + +### Reserved + +The `cxg` prefix is used for CXG-specific names. + +### Encoding Data With TileDB Arrays + +The TileDB array schema authoritatively defines the characteristics of each array (eg, the type of `X` is defined by [`X.schema`](https://tiledb-inc-tiledb-py.readthedocs-hosted.com/en/stable/python-api.html#tiledb.libtiledb.Array.schema)). In some cases, additional metadata is required for the CXG, and is attched to the array using the TileDB [array metadata](https://docs.tiledb.com/main/basic-concepts/array-metadata) capability. + +All TileDB arrays MUST have a uint32 domain, zero based. All X counts and embedding coordinates SHOULD be coerced to float32, which is ample precision for visualization purposes, and MUST be a numeric type. Dataframe (metadata) types are generally preserved, or where that is not possible, converted to something with equal representative value in the cellxgene application (eg, categorical types are converted to string, bools to uint8, etc). + +CXG consumers (readers) MUST be prepared to handle any legal TileDB compression, global layout and tile size. CXG writers SHOULD attempt to encode data using best-effort heuristics for time and space considerations (eg, dense/sparse encoding tradeoffs). + +## Entities + +### CXG + +The CXG is a TileDB group containing all data and metadata for a single annotated matrix. The following objects MUST be present in a CXG, except where noted as optional: +* __obs__: a TileDB array, of shape (n_obs,), containing obs annotations, each annotation stored in a separate TileDB array attribute. +* __var__: a TileDB array, of shape (n_var,), containing var annotations, each annotation stored in a separate TileDB array attribute. +* __X__: a TileDB array, of shape (n_obs, n_var), with a single TileDB attribute of numeric type. +* __X_col_shift__: (optional) TilebDB Array used in column shift encoding, shape (n_var,), dtype = X.dtype. Single unnamed numeric attribute. +* __emb__: a TileDB group, which in turn contains all (zero or more) embeddings. +* __emb__/\__: a TileDB array, with a single anonymous attribute, of numeric type, and shape (n_obs, N>=2). +* __cxg_group_metadata__: an empty TileDB array, used to store CXG-wide metadata + +### obs and var + +All per-observation (obs) and per-feature (var) data is encoded in a TileDB array named `obs` and `var` respectively, with shape (n_obs,) and (n_var,). Each TileDB array has an array attribute for each obs/var column. All TileDB array attributes will have the same type and value as the original data, eg, float32, with the following exceptions: +* bool is encoded as uint8 (1/0) +* categorical is encoded as string +* Numeric types are cast to 32-bit equivalents + +In addition to the obs/var data, both TileDB arrays contain an optional 'cxg_schema' metadata field that is a JSON string containing per-column (attribute) schema hinting. This is used where the TileDB native typing information is insufficient to reconstruct useful information such as categorical typing from Pandas DataFrames, and to communicate which column is the preferred human-readable index for obs & var. + +The `cxg_schema` JSON string is attached to the TileDB array metadata, and is a dictionary containing the following top-level names: +* "index": string, containing the name of the index column +* \: optional, a JSON dict, contain a schema definition using the same format as the cellxgene REST API /schema route + +For example: +``` +{ + "index": "obs_index", + "louvain": { "type": "categorical", "categories": [ "0", "1", "2", "3", "4" ]} + "is_useful": { "type": "boolean" } +} +``` + +### X + +TileDB array, with a single anonymous attribute, shape (n_obs, n_var), containing the count matrix (equivalent to the AnnData `X` array). MUST have numeric type, and SHOULD be float32. The TileDB schema defines type and sparsity, and both dense and sparse encoding are supported. + +### X_col_shift + +Optional TileDB array, used to encode-per column offsets for column-shift sparse encoding. The TileDB array will have a single anonymous attribute, of the same type as the X array, and shape (n_var,). + +If the X array is sparse, and X_col_shift exists, then all values in the i'th column were subtracted by X_col_shift[i]. + +### emb and embedding arrays + +A CXG must have a group named `emb`, which will contain all embeddings. Embeddings are encoded as TileDB arrays, of numeric type and shape (n_obs, >=2). The arrays SHOULD be coerced to float32, and MUST be a numeric type. The TileDB array name will be assumed to be the embedding name (conventionally, embedding names in CXG are _not_ prefixed with an `X_` as they are in AnnData). + +CXG supports zero or more embeddings. Note that cellxgene currently _requires_ at least one embedding. + +### cxg_group_metadata + +Required, but empty TileDB array, used to store CXG-wide metadata. The following fields are defined: +* __cxg_version__: (required) a semver string identifying the specification version used to encode the CXG. +* __cxg_properties__: (optional) a dictionary containing dataset wide properties, defined below. +* __cxg_category_colors__: (optional) a categorical color table, defined below. + +#### cxg_properties + +The properties metadata dictionary contains dataset-wide properties, encoded as a JSON dictionary. Currently, the following fields are defined: +* title: string, dataset human name (eg, "Lung Tissue") +* about: string, fully-qualified http/https URL, linking to more information on the dataset. + +All implementions MUST ignore unrecognized fields. + +#### cxg_category_colors + +This optional field contains a copy of the category color table, which MAY be used to display category-specific color labels. This is a JSON dictionary, containing a per-category color-table. Each color table is named `{category_name}_colors`, and is itself a dictionary mapping label name to RGB color. For example: + +``` +{ + "louvain_colors": { + "0": "#FFFFFF", + "1": "#000000" + } +} +``` + +## Corpora Schema Encoding + +The [Corpora schema](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md) and [Corpora AnnData encoding](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md) define a set of metadata and encoding conventions for annotated matrices. When a Corpora dataset is encoded as a CXG, the following shall apply. + +### Corpora metadata property + +A CXG containing a Corpora dataset will contain a property in the __cxg_group_metadata__ field named `corpora`. The value will be a JSON encoded string, which in turn contains all properties defined in the [Corpora AnnData uns](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md#uns) container. For example: + +``` +{ + "corpora": { + "version": { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0", + } + } +} +``` + +The `corpora` metadata, if present, MUST contain the version information. Optionality of other values in this object will follow the specifications set forth in the relevant Corpora schema specification (ie, optional fields are optional, required are present, etc), with the following changes: +* the contents of `corpora_encoding_version` MUST be identical to the `cxg_version`, as this field is defined as the current object encoding version, *NOT* the source data encoding version. +* the entire encoding will be JSON, rather than a hybrid Python/JSON encoding, but will otherwise follow the data structure defined by the AnnData Corpora encoding. +* the `_colors` will be omitted in favor of `cxg_category_colors` + +### Other Corpora fields + +All other Corpora schema fields will be encoded into a CXG using the conventions defined in the [Corpora Schema AnnData Implementation](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md). For example, fields in `AnnData.obs` will be encoded in the CXG `obs`array as defined [above](#obs-and-var). + +### Compatibility with CXG 0.1.0 + +For backwards compatibility and continuity with CXG version 0.1.0, the following MUST be implemented. + +#### Presentation Hints +* The [Corpora `title`](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md#presentation-metadata) value MUST be saved in the `cxg_properties.title` field. +* The [Corpora `color_map`](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md#presentation-hints), when present in the dataset, MUST be saved in the `cxg_category_colors` field and NOT in the `corpora` field. +* The [Corpora SUMMARY `project_link`](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md#presentation-hints), if present, MUST be saved in the `cxg_properties.about` field. + +Where these values differ in the final CXG, the `cxg_properties` values WILL take precedence. + +## CXG Version History + +There were several ad hoc version of CXG created prior to this spec. This describes the _proposed_ next version of CXG, which incoporates support for Corpora schema semantics. Prior verisons: +* _unnamed_ - an unnamed development version. Did not include explicit versioning support in the data model, but can be detected by the absence of __cxg_group_metadata__ and any version property. Created in early 2020, and not actively used in production +* 0.1 - the first and current version, defined to support the capabilities of the mid-2020 cellxgene. Created in early 2020, and in active use. Includes everything in this spec, excluding Corpora schema support. __NOTE:__ this version is encoded with a short-hand (malformed) semver version number. +* 0.2.0 - this specification. diff --git a/server/app/app.py b/server/app/app.py index 1f59513c..e082e978 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -130,7 +130,7 @@ def get_data_adaptor(url_dataroot=None, dataset=None): # sufficient to check that the datapath starts with the # dataroot to determine that the datapath is under the dataroot. if not datapath.startswith(dataroot): - raise DatasetAccessError("Invalid dataset {url_dataroot}/{dataset}") + raise DatasetAccessError(f"Invalid dataset {url_dataroot}/{dataset}") if datapath is None: return common_rest.abort_and_log(HTTPStatus.BAD_REQUEST, "Invalid dataset NONE", loglevel=logging.INFO) diff --git a/server/common/app_config.py b/server/common/app_config.py index dab9d6a7..da2f6463 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -242,6 +242,12 @@ class AppConfig(object): "about_legal_privacy": dataset_config.app__about_legal_privacy, } + # dataset_props + # TODO/Note: putting info from the dataset into the /config is not ideal. + # However, it is definitely not part of /schema, and we do not have a top-level + # route for data properties. Consider creating one at some point. + corpora_props = data_adaptor.get_corpora_props() + data_adaptor.update_parameters(parameters) if annotation: annotation.update_parameters(parameters, data_adaptor) @@ -254,6 +260,7 @@ class AppConfig(object): config["library_versions"] = library_versions config["links"] = links config["parameters"] = parameters + config["corpora_props"] = corpora_props config["limits"] = { "column_request_max": server_config.limits__column_request_max, "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, @@ -825,7 +832,8 @@ class DatasetConfig(BaseConfig): if server_config.single_dataset__datapath: if self.embeddings__enable_reembedding: matrix_data_loader = MatrixDataLoader( - server_config.single_dataset__datapath, app_config=self.app_config) + server_config.single_dataset__datapath, app_config=self.app_config + ) if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD: raise ConfigurationError("'enable-reembedding is only supported with H5AD files.") if server_config.adaptor__anndata_adaptor__backed: diff --git a/server/common/corpora.py b/server/common/corpora.py new file mode 100644 index 00000000..9f48873b --- /dev/null +++ b/server/common/corpora.py @@ -0,0 +1,90 @@ +""" +Corpora schema conventions support. Helper functions for reading. + +https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md + +https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md +""" +import collections +import json + +from server.cli.upgrade import validate_version_str + + +def corpora_get_versions_from_anndata(adata): + """ + Given an AnnData object, return: + * None - if not a Corpora object + * [ corpora_schema_version, corpora_encoding_version ] - if a Corpora object + + Implements the identification protocol defined in the specification. + """ + + # per Corpora AnnData spec, this is a corpora file if the following is true + if "version" not in adata.uns_keys(): + return None + version = adata.uns["version"] + if not isinstance(version, collections.abc.Mapping) or "corpora_schema_version" not in version: + return None + + corpora_schema_version = version.get("corpora_schema_version") + corpora_encoding_version = version.get("corpora_encoding_version") + + # TODO: spec says these must be SEMVER values, so check. + if validate_version_str(corpora_schema_version) and validate_version_str(corpora_encoding_version): + return [corpora_schema_version, corpora_encoding_version] + + +def corpora_is_version_supported(corpora_schema_version, corpora_encoding_version): + return ( + corpora_schema_version + and corpora_encoding_version + and corpora_schema_version.startswith("1.") + and corpora_encoding_version.startswith("0.1.") + ) + + +def corpora_get_props_from_anndata(adata): + """ + Get Corpora dataset properties from an AnnData + """ + versions = corpora_get_versions_from_anndata(adata) + if versions is None: + return None + [corpora_schema_version, corpora_encoding_version] = versions + version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version) + if not version_is_supported: + raise ValueError("Unsupported Corpora schema version") + + required_simple_fields = [ + "version", + "title", + "layer_descriptions", + "organism", + "organism_ontology_term_id", + "project_name", + "project_description", + ] + # Spec says some values encoded as JSON due to the inability of AnnData to store complex types. + required_json_fields = ["contributors", "project_links"] + optional_simple_fields = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"] + + corpora_props = {} + for key in required_simple_fields: + if key not in adata.uns: + raise KeyError(f"missing Corpora schema field {key}") + corpora_props[key] = adata.uns[key] + + for key in required_json_fields: + if key not in adata.uns: + raise KeyError(f"missing Corpora schema field {key}") + try: + corpora_props[key] = json.loads(adata.uns[key]) + except json.JSONDecodeError: + raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string") + + for key in optional_simple_fields: + if key in adata.uns: + corpora_props[key] = adata.uns[key] + + return corpora_props diff --git a/server/converters/cxgtool.py b/server/converters/cxgtool.py index 0b8fb3a4..0d7f8453 100644 --- a/server/converters/cxgtool.py +++ b/server/converters/cxgtool.py @@ -1,70 +1,9 @@ """ This program converts an [AnnData H5AD](https://anndata.readthedocs.io/en/stable/) -into a cellxgene TileDB structure, aka a 'CXG'. - -The organization of the TileDB structure is: - - the.cxg TileDB Group - ├─ obs TileDB array containing cell (row) attributes, one attribute per - │ dataframe column, shape (n_obs,) - ├─ var TileDB array containing gene (column) attributes, with one attribute per - │ dataframe column, shape (n_var,) - ├─ X Main count matrix as a 2D TileDB array, single unnamed numeric attribute - ├─ X_col_shift TilebDB Array used in column shift encoding, shape (n_var,), dtype = X.dtype. - │ Single unnamed numeric attribute. If this array is sparse, and X_col_shift exists, - │ then all values in the i'th column were subtracted by X_col_shift[i]. - ├─ emb TileDB group, storing optional embeddings (group may be empty) - │ └─ TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N) - └─ cxg_group_metadata Empty array used only to stash metadata about the overall object. - └─ cxg_category_colors CXG colors object as described below: - { - "": { - "": "", - ... - }, - ... - } - ... - -All arrays are defined to have a uint32 domain, zero based. All X counts and embedding -coordinates are coerced to float32, which is ample precision for visualization purposes. -Dataframe (metadata) types are generally preserved, or where that is not possible, -converted to something with equal representative value in the cellxgene application -(eg, categorical types are converted to string, bools to uint8, etc). - -The following objects are also decorated with auxiliary metadata using TileDB -array metadata: - -* cxg_group_metadata: minimally, will contain a 'cxg_version' field, which - is a semver string identifying the version number of the CXG layout. - It may also contain 'cxg_parameters', a JSON-encoded parameter list - describing CXG-wide dataset parameters. - -* obs, var: both contain an optional 'cxg_schema' field that is a json string, - containing per-column (attribute) schema hinting. This is used where the TileDB - native typing information is insufficient to reconstruct useful information - such as categorical typing from Pandas DataFrames, and to communicate which column - is the preferred human-readable index for obs & var. - -This file also embodies a number of empirically derived tiledb schema parameters, -including the global data layout, spatial tile size, and the like. The CXG is -self-describing in these areas, and the actual values (eg, tile size) are empirically -derived from benchmarking. They may change in the future. - -cxgtool.py will extract color information stored in arrays in the 'uns' anndata -property with the key "{category_name}_colors". For this to work, the following -command must result in a mapping from category names to matplotlib-compatible colors: - -``` -dict(zip(adata.obs[cat].cat.categories, adata.uns[f"{cat}_colors"])) -``` - ---- - -TODO/ISSUES: -* add sub-command structure to argparse, for future sub-commands -* Possible future work: accept Loom files +into a cellxgene TileDB structure, aka a [CXG](../../dev_docs/cxg.md). +IF YOU UPDATE THIS FILE, IN ANY WAY THAT MODIFIES THE CXG FORMAT or CONTENTS, +YOU MUST UPDATE THE CXG SPECIFICATION and VERSION NUMBER. """ import re import anndata @@ -77,10 +16,16 @@ from scipy.stats import mode from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors from server.common.errors import ColorFormatException +from server.common.corpora import ( + corpora_get_props_from_anndata, + corpora_get_versions_from_anndata, + corpora_is_version_supported, +) -# the CXG container version number. Must be a semver string. -CXG_VERSION = "0.1" +# the CXG container version number. Must be a semver string (major.minor.patch) +# DO NOT UPDATE THIS WITHOUT ALSO UPDATING THE CXG SPECIFICATION. +CXG_VERSION = "0.2.0" # log_level must have a default log_level = 3 @@ -125,6 +70,12 @@ def main(): default=0.0, # force dense by default help="The X array will be sparse if the percent of non-zeros falls below this value", ) + parser.add_argument( + "--disable-corpora", + action="store_true", + default=False, + help="Disable extraction and storing of Corpora schema information.", + ) args = parser.parse_args() global log_level @@ -136,25 +87,30 @@ def main(): basefname = splitext(basename(args.h5ad))[0] out = args.out if args.out is not None else basefname container = out if splitext(out)[1] == ".cxg" else out + ".cxg" - title = args.title if args.title is not None else basefname + + corpora_props = load_corpora_props(args, adata) if not args.disable_corpora else None + cxg_group_metadata = create_cxg_group_metadata( + adata, + basefname, + title=args.title, + about=args.about, + corpora_props=corpora_props, + extract_colors=not args.disable_custom_colors, + ) write_cxg( adata, container, - title, + cxg_group_metadata=cxg_group_metadata, var_names=args.var_names, obs_names=args.obs_names, - about=args.about, - extract_colors=not args.disable_custom_colors, sparse_threshold=args.sparse_threshold, ) log(1, "done") -def write_cxg( - adata, container, title, var_names=None, obs_names=None, about=None, extract_colors=False, sparse_threshold=5.0 -): +def write_cxg(adata, container, cxg_group_metadata, var_names=None, obs_names=None, sparse_threshold=5.0): if not adata.var.index.is_unique: raise ValueError("Variable index is not unique - unable to convert.") if not adata.obs.index.is_unique: @@ -179,19 +135,7 @@ def write_cxg( log(1, f"\t...group created, with name {container}") # dataset metadata - metadata_dict = dict(cxg_version=CXG_VERSION, cxg_properties=json.dumps({"title": title, "about": about})) - if extract_colors: - try: - metadata_dict["cxg_category_colors"] = json.dumps( - convert_anndata_category_colors_to_cxg_category_colors(adata) - ) - except ColorFormatException: - log( - 0, - "Warning: failed to extract colors from h5ad file! " - "Fix the h5ad file or rerun with --disable-custom-colors. See help for details.", - ) - save_metadata(container, metadata_dict) + save_metadata(container, cxg_group_metadata) log(1, "\t...dataset metadata saved") # var/gene dataframe @@ -601,6 +545,59 @@ def save_metadata(container, metadata_dict): A.meta[k] = v +def load_corpora_props(args, adata): + versions = corpora_get_versions_from_anndata(adata) + if versions is None: + return None + + [corpora_schema_version, corpora_encoding_version] = versions + corpora_props = corpora_get_props_from_anndata(adata) + version_is_supported = corpora_is_version_supported(corpora_schema_version, corpora_encoding_version) + if not version_is_supported or not corpora_props: + log(0, "ERROR: Unknown source file schema version is unsupported") + raise ValueError("Unsupported Corpora schema version") + + log(1, "FYI, file appears to be encoded using Corpora schema standards...") + if args.title is not None or args.about is not None: + log(0, "Warning: explicit specification of --title or --about will override Corpora schema fields.") + + return corpora_props + + +def create_cxg_group_metadata(adata, basefname, title=None, about=None, corpora_props=None, extract_colors=True): + + if corpora_props is not None: + # clobber encoding version to be OUR version, not the source H5AD encoding + corpora_props["version"].update({"corpora_encoding_version": CXG_VERSION}) + corpora_project_links = corpora_props.get("project_links", []) + corpora_about_link = next( + (link for link in corpora_project_links if (link.get("link_type", None) == "SUMMARY")), {} + ) + else: + corpora_about_link = {} + + title = title or corpora_about_link.get("link_name", basefname) + about = about or corpora_about_link.get("link_url") + + cxg_group_metadata = {"cxg_version": CXG_VERSION, "cxg_properties": json.dumps({"title": title, "about": about})} + if corpora_props is not None: + cxg_group_metadata.update({"corpora": json.dumps(corpora_props)}) + + if extract_colors: + try: + cxg_group_metadata["cxg_category_colors"] = json.dumps( + convert_anndata_category_colors_to_cxg_category_colors(adata) + ) + except ColorFormatException: + log( + 0, + "Warning: failed to extract colors from h5ad file! " + "Fix the h5ad file or rerun with --disable-custom-colors. See help for details.", + ) + + return cxg_group_metadata + + def sanitize_keys(keys): """ We need names to be safe to use as attribute names in tiledb. See: diff --git a/server/data_anndata/anndata_adaptor.py b/server/data_anndata/anndata_adaptor.py index 6022928e..69496614 100644 --- a/server/data_anndata/anndata_adaptor.py +++ b/server/data_anndata/anndata_adaptor.py @@ -17,6 +17,7 @@ from server.common.constants import Axis, MAX_LAYOUTS from server.common.errors import PrepareError, DatasetAccessError, FilterError from server.compute.scanpy import scanpy_umap import server.compute.diffexp_generic as diffexp_generic +from server.common.corpora import corpora_get_props_from_anndata anndata_version = version.parse(str(anndata.__version__)).release @@ -57,6 +58,9 @@ class AnndataAdaptor(DataAdaptor): def open(data_locator, app_config, dataset_config=None): return AnndataAdaptor(data_locator, app_config, dataset_config) + def get_corpora_props(self): + return corpora_get_props_from_anndata(self.data) + def get_name(self): return "cellxgene anndata adaptor version" diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index 14a07b5e..20f8b601 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -135,6 +135,9 @@ class DataAdaptor(metaclass=ABCMeta): location = location[:-1] return splitext(basename(location))[0] + def get_corpora_props(self): + return None + @abstractmethod def get_schema(self): """ diff --git a/server/data_cxg/cxg_adaptor.py b/server/data_cxg/cxg_adaptor.py index 2705f43b..41822c29 100644 --- a/server/data_cxg/cxg_adaptor.py +++ b/server/data_cxg/cxg_adaptor.py @@ -74,6 +74,9 @@ class CxgAdaptor(DataAdaptor): def get_title(self): return self.title if self.title else super().get_title() + def get_corpora_props(self): + return self.corpora_props if self.corpora_props else super().get_corpora_props() + def get_name(self): return "cellxgene cxg adaptor version" @@ -144,26 +147,31 @@ class CxgAdaptor(DataAdaptor): * version 0.1 -- metadata attache to cxg_group_metadata array. Same as 0, except it adds group metadata. """ + title = None + about = None + corpora_props = None if self.has_array("cxg_group_metadata"): # version >0 gmd = self.open_array("cxg_group_metadata") cxg_version = gmd.meta["cxg_version"] - if cxg_version == "0.1": + # version 0.1 used a malformed/shorthand semver string. + if cxg_version == "0.1" or cxg_version == "0.2.0": cxg_properties = json.loads(gmd.meta["cxg_properties"]) title = cxg_properties.get("title", None) about = cxg_properties.get("about", None) + if cxg_version == "0.2.0": + corpora_props = json.loads(gmd.meta["corpora"]) if "corpora" in gmd.meta else None else: # version 0 cxg_version = "0.0" - title = None - about = None - if cxg_version not in ["0.0", "0.1"]: + if cxg_version not in ["0.0", "0.1", "0.2.0"]: raise DatasetAccessError(f"cxg matrix is not valid: {self.url}") self.title = title self.about = about self.cxg_version = cxg_version + self.corpora_props = corpora_props @staticmethod def _open_array(uri, tiledb_ctx): diff --git a/server/test/test_anndata_adaptor_data_load.py b/server/test/test_anndata_adaptor_data_load.py index e5c01a03..35ccb886 100644 --- a/server/test/test_anndata_adaptor_data_load.py +++ b/server/test/test_anndata_adaptor_data_load.py @@ -43,13 +43,10 @@ class DataLocatorAdaptorTest(unittest.TestCase): def get_basic_config(self): config = AppConfig() config.update_server_config( - single_dataset__obs_names=None, - single_dataset__var_names=None, + single_dataset__obs_names=None, single_dataset__var_names=None, ) config.update_default_dataset_config( - embeddings__names=["umap"], - presentation__max_categories=100, - diffexp__lfc_cutoff=0.01, + embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01, ) return config diff --git a/server/test/test_corpora.py b/server/test/test_corpora.py new file mode 100644 index 00000000..b0637706 --- /dev/null +++ b/server/test/test_corpora.py @@ -0,0 +1,73 @@ +import unittest +import anndata +import json + +from server.common.corpora import ( + corpora_get_versions_from_anndata, + corpora_is_version_supported, + corpora_get_props_from_anndata, +) +from server.test import PROJECT_ROOT + + +class CorporaAPITest(unittest.TestCase): + def test_corpora_get_versions_from_anndata(self): + adata = self._get_h5ad() + + if "version" in adata.uns: + del adata.uns["version"] + self.assertIsNone(corpora_get_versions_from_anndata(adata)) + + # something bogus + adata.uns["version"] = 99 + self.assertIsNone(corpora_get_versions_from_anndata(adata)) + + # something legit + adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"} + self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"]) + + def test_corpora_is_version_supported(self): + self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0")) + self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0")) + self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0")) + + def test_corpora_get_props_from_anndata(self): + adata = self._get_h5ad() + + if "version" in adata.uns: + del adata.uns["version"] + self.assertIsNone(corpora_get_props_from_anndata(adata)) + + # something bogus + adata.uns["version"] = 99 + self.assertIsNone(corpora_get_props_from_anndata(adata)) + + # unsupported version, but missing required values + adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"} + with self.assertRaises(ValueError): + corpora_get_props_from_anndata(adata) + + # legit version, but missing required values + adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"} + with self.assertRaises(KeyError): + corpora_get_props_from_anndata(adata) + + some_fields = { + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, + "title": "title", + "layer_descriptions": "layer_descriptions", + "organism": "organism", + "organism_ontology_term_id": "organism_ontology_term_id", + "project_name": "project_name", + "project_description": "project_description", + "contributors": json.dumps([{"contributors": "contributors"}]), + "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]), + } + for k in some_fields: + adata.uns[k] = some_fields[k] + some_fields["contributors"] = json.loads(some_fields["contributors"]) + some_fields["project_links"] = json.loads(some_fields["project_links"]) + self.assertEqual(corpora_get_props_from_anndata(adata), some_fields) + + def _get_h5ad(self): + return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") diff --git a/server/test/test_cxgtool.py b/server/test/test_cxgtool.py index f7daf5db..91ab8605 100644 --- a/server/test/test_cxgtool.py +++ b/server/test/test_cxgtool.py @@ -4,7 +4,7 @@ import unittest import anndata from server.common.data_locator import DataLocator -from server.converters.cxgtool import write_cxg +from server.converters.cxgtool import write_cxg, create_cxg_group_metadata from server.data_cxg.cxg_adaptor import CxgAdaptor from server.test import PROJECT_ROOT, app_config, random_string from server.test.test_datasets.fixtures import pbmc3k_colors @@ -33,6 +33,9 @@ class TestCxgAdaptor(unittest.TestCase): data_locator = f"/tmp/test_{rand_str}.cxg" self.fixtures.append(data_locator) source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") - write_cxg(adata=source_h5ad, container=data_locator, title="pbmc3k", **kwargs) + cxg_group_metadata = create_cxg_group_metadata( + adata=source_h5ad, basefname="pbmc3k.h5ad", title="pbmc3k", **kwargs + ) + write_cxg(adata=source_h5ad, container=data_locator, cxg_group_metadata=cxg_group_metadata) config = app_config(data_locator) return CxgAdaptor(DataLocator(data_locator), config) diff --git a/server/test/test_diffexp.py b/server/test/test_diffexp.py index abe1fc57..ee49681f 100644 --- a/server/test/test_diffexp.py +++ b/server/test/test_diffexp.py @@ -3,7 +3,7 @@ from server.data_common.matrix_loader import MatrixDataLoader from server.test import PROJECT_ROOT, app_config import server.compute.diffexp_cxg as diffexp_cxg import server.compute.diffexp_generic as diffexp_generic -from server.converters.cxgtool import write_cxg +from server.converters.cxgtool import write_cxg, create_cxg_group_metadata from server.test.create_test_matrix import create_test_h5ad from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs import numpy as np @@ -16,8 +16,7 @@ class DiffExpTest(unittest.TestCase): adaptor types and different algorithms.""" def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}): - config = app_config(path, extra_server_config=extra_server_config, - extra_dataset_config=extra_dataset_config) + config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config) loader = MatrixDataLoader(path) adaptor = loader.open(config) return adaptor @@ -105,13 +104,15 @@ class DiffExpTest(unittest.TestCase): adata = adaptor_anndata.data sparsename = os.path.join(dirname, "sparse.cxg") - write_cxg(adata=adata, container=sparsename, title="sparse", sparse_threshold=11) + cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="sparse.h5ad", title="sparse",) + write_cxg(adata=adata, container=sparsename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=11) adaptor_sparse = self.load_dataset(sparsename) assert adaptor_sparse.open_array("X").schema.sparse assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift densename = os.path.join(dirname, "dense.cxg") - write_cxg(adata=adata, container=densename, title="dense", sparse_threshold=0) + cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="dense.h5ad", title="dense",) + write_cxg(adata=adata, container=densename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=0) adaptor_dense = self.load_dataset(densename) assert not adaptor_dense.open_array("X").schema.sparse assert not adaptor_dense.has_array("X_col_shift") From af3a76c354d58861bc04053ddac1cf7ccff4198e Mon Sep 17 00:00:00 2001 From: maniarathi Date: Wed, 29 Jul 2020 12:38:46 -0700 Subject: [PATCH 16/55] Adding relative links support to jekyll (#1680) --- docs/Gemfile | 1 + docs/_config.yml | 3 +++ docs/_site/index.html | 2 +- docs/_site/posts/annotations.html | 2 +- docs/_site/posts/contact.html | 2 +- docs/_site/posts/contribute.html | 2 +- docs/_site/posts/demo-data.html | 2 +- docs/_site/posts/gallery.html | 2 +- docs/_site/posts/hosted.html | 2 +- docs/_site/posts/install.html | 2 +- docs/_site/posts/launch.html | 2 +- docs/_site/posts/methods.html | 2 +- docs/_site/posts/prepare.html | 2 +- docs/_site/posts/roadmap.html | 2 +- docs/_site/posts/troubleshooting.html | 2 +- 15 files changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/Gemfile b/docs/Gemfile index e44e449c..a6011a22 100644 --- a/docs/Gemfile +++ b/docs/Gemfile @@ -1,2 +1,3 @@ source 'https://rubygems.org' gem "github-pages", group: :jekyll_plugins +gem 'jekyll-relative-links' diff --git a/docs/_config.yml b/docs/_config.yml index dc80f90a..8ac27944 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -3,6 +3,9 @@ show_downloads: false url: "https://chanzuckerberg.github.io" baseurl: "/cellxgene" +plugins: + - jekyll-relative-links + logo: cellxgene-logo.png nav: diff --git a/docs/_site/index.html b/docs/_site/index.html index 0c2d6eb2..71d7482e 100644 --- a/docs/_site/index.html +++ b/docs/_site/index.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","headline":"Index","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/annotations.html b/docs/_site/posts/annotations.html index cbf5b5e2..776b59fb 100644 --- a/docs/_site/posts/annotations.html +++ b/docs/_site/posts/annotations.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","headline":"annotations","description":"Creating annotations","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/contact.html b/docs/_site/posts/contact.html index 121d0056..9aadb3a4 100644 --- a/docs/_site/posts/contact.html +++ b/docs/_site/posts/contact.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","headline":"Contact","description":"Contact","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/contribute.html b/docs/_site/posts/contribute.html index e96084ab..8fa3fe30 100644 --- a/docs/_site/posts/contribute.html +++ b/docs/_site/posts/contribute.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","headline":"Code of conduct","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/demo-data.html b/docs/_site/posts/demo-data.html index ec6b9380..ac710163 100644 --- a/docs/_site/posts/demo-data.html +++ b/docs/_site/posts/demo-data.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/gallery.html b/docs/_site/posts/gallery.html index 5322f334..d1ead529 100644 --- a/docs/_site/posts/gallery.html +++ b/docs/_site/posts/gallery.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","headline":"Gallery","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/hosted.html b/docs/_site/posts/hosted.html index 30f0a0c4..1bec5484 100644 --- a/docs/_site/posts/hosted.html +++ b/docs/_site/posts/hosted.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","headline":"Hosting cellxgene on the web","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/install.html b/docs/_site/posts/install.html index ecfd6c08..8b80a6ba 100644 --- a/docs/_site/posts/install.html +++ b/docs/_site/posts/install.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","headline":"Install","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/launch.html b/docs/_site/posts/launch.html index e0332840..606320c6 100644 --- a/docs/_site/posts/launch.html +++ b/docs/_site/posts/launch.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/methods.html b/docs/_site/posts/methods.html index d7c20ecf..bebb0e3c 100644 --- a/docs/_site/posts/methods.html +++ b/docs/_site/posts/methods.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","headline":"Methods","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/prepare.html b/docs/_site/posts/prepare.html index 3cb05e6a..edfc1426 100644 --- a/docs/_site/posts/prepare.html +++ b/docs/_site/posts/prepare.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","headline":"prepare","description":"Preparing your data","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/roadmap.html b/docs/_site/posts/roadmap.html index 83c295aa..3ce34e68 100644 --- a/docs/_site/posts/roadmap.html +++ b/docs/_site/posts/roadmap.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","headline":"roadmap","description":"Roadmap","@type":"WebPage","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/troubleshooting.html b/docs/_site/posts/troubleshooting.html index f6476753..a86940ff 100644 --- a/docs/_site/posts/troubleshooting.html +++ b/docs/_site/posts/troubleshooting.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","headline":"Troubleshooting","description":"Troubleshooting","@type":"WebPage","@context":"https://schema.org"} - + From 5633d7c7612a406a741525ec8d9dc1e1263761c5 Mon Sep 17 00:00:00 2001 From: bmccandless Date: Wed, 29 Jul 2020 13:05:59 -0700 Subject: [PATCH 17/55] Fix server exception classes (#1683) str(e) and e.message will both show the error message. refactored the error.py file to simplify our exception class definitions --- server/common/errors.py | 98 +++++++++++++---------------------------- 1 file changed, 30 insertions(+), 68 deletions(-) diff --git a/server/common/errors.py b/server/common/errors.py index d1dda2cf..ca927084 100644 --- a/server/common/errors.py +++ b/server/common/errors.py @@ -1,85 +1,47 @@ from http import HTTPStatus -class RequestException(Exception): +class CellxgeneException(Exception): + """Base class for cellxgene exceptions""" + + def __init__(self, message): + self.message = message + super().__init__(message) + + +class RequestException(CellxgeneException): """Baseclass for exceptions that can be raised from a request.""" # The default status code is 400 (Bad Request) default_status_code = HTTPStatus.BAD_REQUEST def __init__(self, message, status_code=None): - Exception.__init__(self) - self.message = message + super().__init__(message) self.status_code = status_code or self.default_status_code -class FilterError(RequestException): - """Raised when filter is malformed""" - - pass +def define_exception(name, doc): + globals()[name] = type(name, (CellxgeneException,), dict(__doc__=doc)) -class JSONEncodingValueError(RequestException): - """Raised when data cannot be encoded into json""" - - pass +def define_request_exception(name, doc, default_status_code=HTTPStatus.BAD_REQUEST): + globals()[name] = type(name, (RequestException,), dict(__doc__=doc, default_status_code=default_status_code)) -class MimeTypeError(RequestException): - """Raised when incompatible MIME type selected""" +define_request_exception("FilterError", "Raised when filter is malformed") +define_request_exception("JSONEncodingValueError", "Raised when data cannot be encoded into json") +define_request_exception("MimeTypeError", "Raised when incompatible MIME type selected") +define_request_exception("DatasetAccessError", "Raised when file loaded into a DataAdaptor is misformatted") +define_request_exception("DisabledFeatureError", "Raised when an attempt to use a disabled feature occurs") +define_request_exception("AnnotationsError", "Raised when an attempt to use the annotations feature fails") +define_request_exception( + "ComputeError", + "Raised when an error occurs during a compute algorithm (such as diffexp)", + HTTPStatus.INTERNAL_SERVER_ERROR, +) +define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota") +define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format") - pass - - -class DatasetAccessError(RequestException): - """Raised when file loaded into a DataAdaptor is misformatted""" - - pass - - -class DisabledFeatureError(RequestException): - """Raised when an attempt to use a disabled feature occurs""" - - pass - - -class AnnotationsError(RequestException): - """Raised when an attempt to use the annotations feature fails""" - - pass - - -class ComputeError(RequestException): - """Raised when an error occurs during a compute algorithm (such as diffexp)""" - - default_status_code = HTTPStatus.INTERNAL_SERVER_ERROR - - -class ExceedsLimitError(RequestException): - """Raised when an HTTP request exceeds a limit/quota""" - - pass - - -class ColorFormatException(RequestException): - """Raised when color helper functions encounter an unknown color format""" - - pass - - -class OntologyLoadFailure(Exception): - """Raised when reading the ontology file fails""" - - pass - - -class ConfigurationError(Exception): - """Raised when checking configuration errors""" - - pass - - -class PrepareError(Exception): - """Raised when data is misprepared""" - - pass +define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails") +define_exception("ConfigurationError", "Raised when checking configuration errors") +define_exception("PrepareError", "Raised when data is misprepared") From bd147abb3f0e8ecf7f1fbb4e50403f5fec23a780 Mon Sep 17 00:00:00 2001 From: bmccandless Date: Wed, 29 Jul 2020 16:03:47 -0700 Subject: [PATCH 18/55] Fix eb logging. (#1692) It now logs the requests to the file Fixes #1611 --- server/eb/app.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/eb/app.py b/server/eb/app.py index 438cd9bb..ce3a77b6 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -19,9 +19,6 @@ if os.path.isdir("/opt/python/log"): datefmt="%Y-%m-%d %H:%M:%S", ) -# echo the logs to stdout. Useful for local testing -logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) - SERVERDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.append(SERVERDIR) From 75cb513dd95e3ec95cbacfbd456f8525e741ec1f Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Thu, 30 Jul 2020 12:31:36 -0700 Subject: [PATCH 19/55] re-implement re-embeddings (#1679) * fix mispelling * re-implement re-embedding * always load base embedding to fetch counts * format * lint * fix tests * lint * fix accept handling * test log * more debug * more * more * more * more * remove logging * logging * jsonify * remove debugging logs * lint * clean up errors a bit * fix issue found in PR review * PR review changes --- client/src/actions/embedding.js | 29 +++++--- client/src/actions/reembed.js | 35 ++++----- client/src/annoMatrix/annoMatrix.js | 13 ++++ client/src/annoMatrix/crossfilter.js | 5 ++ client/src/annoMatrix/loader.js | 68 +++++++++++------ client/src/annoMatrix/views.js | 77 ++++++++++++-------- client/src/components/embedding/index.js | 2 +- client/src/components/menubar/index.js | 5 ++ client/src/components/menubar/reembedding.js | 40 ++++++++++ client/src/reducers/index.js | 4 +- client/src/reducers/layoutChoice.js | 20 +---- client/src/reducers/reembed.js | 35 --------- server/common/rest.py | 18 +---- server/compute/scanpy.py | 10 ++- server/data_anndata/anndata_adaptor.py | 12 ++- server/data_common/data_adaptor.py | 3 +- server/test/test_anndata_adaptor.py | 10 +-- server/test/test_api.py | 18 +++-- 18 files changed, 225 insertions(+), 179 deletions(-) create mode 100644 client/src/components/menubar/reembedding.js diff --git a/client/src/actions/embedding.js b/client/src/actions/embedding.js index 555c69ab..5e882160 100644 --- a/client/src/actions/embedding.js +++ b/client/src/actions/embedding.js @@ -5,6 +5,23 @@ action creators related to embeddings choice import { AnnoMatrixObsCrossfilter } from "../annoMatrix"; import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers"; +export async function _switchEmbedding(prevAnnoMatrix, newEmbeddingName) { + /* + DRY helper used by this and reembedding action creators + */ + const base = prevAnnoMatrix.base(); + const embeddingDf = await base.fetch("emb", newEmbeddingName); + const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf); + const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix).select( + "emb", + newEmbeddingName, + { + mode: "all", + } + ); + return [annoMatrix, obsCrossfilter]; +} + export const layoutChoiceAction = (newLayoutChoice) => async ( dispatch, getState @@ -14,15 +31,9 @@ export const layoutChoiceAction = (newLayoutChoice) => async ( layout. */ const { annoMatrix: prevAnnoMatrix } = getState(); - - const embeddingDf = await prevAnnoMatrix.base().fetch("emb", newLayoutChoice); - const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf); - const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix).select( - "emb", - newLayoutChoice, - { - mode: "all", - } + const [annoMatrix, obsCrossfilter] = await _switchEmbedding( + prevAnnoMatrix, + newLayoutChoice ); dispatch({ type: "set layout choice", diff --git a/client/src/actions/reembed.js b/client/src/actions/reembed.js index 43b38e1f..2212f380 100644 --- a/client/src/actions/reembed.js +++ b/client/src/actions/reembed.js @@ -1,10 +1,10 @@ import { API } from "../globals"; -import { MatrixFBS } from "../util/stateManager"; import { postNetworkErrorToast, postAsyncSuccessToast, postAsyncFailureToast, } from "../components/framework/toasters"; +import { _switchEmbedding } from "./embedding"; function abortableFetch(request, opts, timeout = 0) { const controller = new AbortController(); @@ -24,7 +24,7 @@ function abortableFetch(request, opts, timeout = 0) { async function doReembedFetch(dispatch, getState) { const state = getState(); - let cells = state.world.obsAnnotations.rowIndex.labels(); + let cells = state.annoMatrix.rowIndex.labels(); // These lines ensure that we convert any TypedArray to an Array. // This is necessary because JSON.stringify() does some very strange @@ -54,10 +54,7 @@ async function doReembedFetch(dispatch, getState) { }); const res = await af.ready(); - if ( - res.ok && - res.headers.get("Content-Type").includes("application/octet-stream") - ) { + if (res.ok && res.headers.get("Content-Type").includes("application/json")) { return res; } @@ -67,7 +64,6 @@ async function doReembedFetch(dispatch, getState) { if (body && body.length > 0) { msg = `${msg} -- ${body}`; } - postNetworkErrorToast(msg); throw new Error(msg); } @@ -78,17 +74,24 @@ export function requestReembed() { return async (dispatch, getState) => { try { const res = await doReembedFetch(dispatch, getState); - const schema = JSON.parse(res.headers.get("CxG-Schema")); - const buffer = await res.arrayBuffer(); - const df = MatrixFBS.matrixFBSToDataframe(buffer); + const schema = await res.json(); dispatch({ type: "reembed: request completed", }); + + const { annoMatrix: prevAnnoMatrix } = getState(); + const base = prevAnnoMatrix.base().addEmbedding(schema); + const [annoMatrix, obsCrossfilter] = await _switchEmbedding( + base, + schema.name + ); dispatch({ type: "reembed: add reembedding", - embedding: df, schema, + annoMatrix, + obsCrossfilter, }); + postAsyncSuccessToast("Re-embedding has completed."); } catch (error) { dispatch({ @@ -103,13 +106,3 @@ export function requestReembed() { } }; } - -/* disabled until reimplementation occurs -export function reembedResetWorldToUniverse(dispatch, getState) { - const { reembedController } = getState(); - if (reembedController.pendingFetch) reembedController.pendingFetch.abort(); - dispatch({ - type: "reembed: clear all reembeddings", - }); -} -*/ diff --git a/client/src/annoMatrix/annoMatrix.js b/client/src/annoMatrix/annoMatrix.js index 15ddb008..092d9007 100644 --- a/client/src/annoMatrix/annoMatrix.js +++ b/client/src/annoMatrix/annoMatrix.js @@ -397,6 +397,19 @@ export default class AnnoMatrix { _subclassResponsibility(); } + // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements + addEmbedding(colSchema) { + /* + Add a new obs embedding to the AnnoMatrix, with provided schema. + Returns a new annomatrix. + + Typical use will be to add a re-embedding that the server has calculated. + + Will throw if the column schema is invalid (eg, duplicate name). + */ + _subclassResponsibility(); + } + /** ** Private interfaces below. **/ diff --git a/client/src/annoMatrix/crossfilter.js b/client/src/annoMatrix/crossfilter.js index b3972978..f9f5bc53 100644 --- a/client/src/annoMatrix/crossfilter.js +++ b/client/src/annoMatrix/crossfilter.js @@ -118,6 +118,11 @@ export default class AnnoMatrixObsCrossfilter { return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter); } + addEmbedding(colSchema) { + const annoMatrix = this.annoMatrix.addEmbedding(colSchema); + return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter); + } + /** Selection state - API is identical to ImmutableTypedCrossfilter, as these are just wrappers to lazy create indices. diff --git a/client/src/annoMatrix/loader.js b/client/src/annoMatrix/loader.js index e10e3ee1..0b5fedcf 100644 --- a/client/src/annoMatrix/loader.js +++ b/client/src/annoMatrix/loader.js @@ -6,6 +6,7 @@ import { removeObsAnnoColumn, addObsAnnoCategory, removeObsAnnoCategory, + addObsLayout, } from "../util/stateManager/schemaHelpers"; import { isArrayOrTypedArray } from "../util/typeHelpers"; import { _whereCacheCreate } from "./whereCache"; @@ -47,9 +48,9 @@ export default class AnnoMatrixLoader extends AnnoMatrix { const colSchema = _getColumnSchema(this.schema, "obs", col); _writableCategoryTypeCheck(colSchema); // throws on error - const o = this._clone(); - o.schema = addObsAnnoCategory(this.schema, col, category); - return o; + const newAnnoMatrix = this._clone(); + newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, category); + return newAnnoMatrix; } async removeObsAnnoCategory(col, category, unassignedCategory) { @@ -59,13 +60,17 @@ export default class AnnoMatrixLoader extends AnnoMatrix { const colSchema = _getColumnSchema(this.schema, "obs", col); _writableCategoryTypeCheck(colSchema); // throws on error - const o = await this.resetObsColumnValues( + const newAnnoMatrix = await this.resetObsColumnValues( col, category, unassignedCategory ); - o.schema = removeObsAnnoCategory(o.schema, col, category); - return o; + newAnnoMatrix.schema = removeObsAnnoCategory( + newAnnoMatrix.schema, + col, + category + ); + return newAnnoMatrix; } dropObsColumn(col) { @@ -75,10 +80,10 @@ export default class AnnoMatrixLoader extends AnnoMatrix { const colSchema = _getColumnSchema(this.schema, "obs", col); _writableCheck(colSchema); // throws on error - const o = this._clone(); - o._cache.obs = this._cache.obs.dropCol(col); - o.schema = removeObsAnnoColumn(this.schema, col); - return o; + const newAnnoMatrix = this._clone(); + newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col); + newAnnoMatrix.schema = removeObsAnnoColumn(this.schema, col); + return newAnnoMatrix; } addObsColumn(colSchema, Ctor, value) { @@ -98,7 +103,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix { throw new Error("column already exists"); } - const o = this._clone(); + const newAnnoMatrix = this._clone(); let data; if (isArrayOrTypedArray(value)) { if (value.constructor !== Ctor) @@ -109,10 +114,13 @@ export default class AnnoMatrixLoader extends AnnoMatrix { } else { data = new Ctor(this.nObs).fill(value); } - o._cache.obs = this._cache.obs.withCol(colName, data); - _normalizeCategoricalSchema(colSchema, o._cache.obs.col(colName)); - o.schema = addObsAnnoColumn(this.schema, colName, colSchema); - return o; + newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data); + _normalizeCategoricalSchema( + colSchema, + newAnnoMatrix._cache.obs.col(colName) + ); + newAnnoMatrix.schema = addObsAnnoColumn(this.schema, colName, colSchema); + return newAnnoMatrix; } renameObsColumn(oldCol, newCol) { @@ -155,13 +163,13 @@ export default class AnnoMatrixLoader extends AnnoMatrix { data[idx] = value; } - const o = this._clone(); - o._cache.obs = this._cache.obs.replaceColData(col, data); + const newAnnoMatrix = this._clone(); + newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data); const { categories } = colSchema; if (!categories?.includes(value)) { - o.schema = addObsAnnoCategory(this.schema, col, value); + newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, value); } - return o; + return newAnnoMatrix; } async resetObsColumnValues(col, oldValue, newValue) { @@ -185,13 +193,27 @@ export default class AnnoMatrixLoader extends AnnoMatrix { if (data[i] === oldValue) data[i] = newValue; } - const o = this._clone(); - o._cache.obs = this._cache.obs.replaceColData(col, data); + const newAnnoMatrix = this._clone(); + newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data); const { categories } = colSchema; if (!categories?.includes(newValue)) { - o.schema = addObsAnnoCategory(this.schema, col, newValue); + newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, newValue); } - return o; + return newAnnoMatrix; + } + + addEmbedding(colSchema) { + /* + add new layout to the obs embeddings + */ + const { name: colName } = colSchema; + if (_getColumnSchema(this.schema, "emb", colName)) { + throw new Error("column already exists"); + } + + const newAnnoMatrix = this._clone(); + newAnnoMatrix.schema = addObsLayout(this.schema, colSchema); + return newAnnoMatrix; } /** diff --git a/client/src/annoMatrix/views.js b/client/src/annoMatrix/views.js index f97e05a0..5b9b9c26 100644 --- a/client/src/annoMatrix/views.js +++ b/client/src/annoMatrix/views.js @@ -17,59 +17,74 @@ class AnnoMatrixView extends AnnoMatrix { } addObsAnnoCategory(col, category) { - const o = this._clone(); - o.viewOf = this.viewOf.addObsAnnoCategory(col, category); - o.schema = o.viewOf.schema; - return o; + const newAnnoMatrix = this._clone(); + newAnnoMatrix.viewOf = this.viewOf.addObsAnnoCategory(col, category); + newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema; + return newAnnoMatrix; } async removeObsAnnoCategory(col, category, unassignedCategory) { - const o = this._clone(); - o.viewOf = await this.viewOf.removeObsAnnoCategory( + const newAnnoMatrix = this._clone(); + newAnnoMatrix.viewOf = await this.viewOf.removeObsAnnoCategory( col, category, unassignedCategory ); - o.schema = o.viewOf.schema; - return o; + newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema; + return newAnnoMatrix; } dropObsColumn(col) { - const o = this._clone(); - o.viewOf = this.viewOf.dropObsColumn(col); - o._cache.obs = this._cache.obs.dropCol(col); - o.schema = o.viewOf.schema; - return o; + const newAnnoMatrix = this._clone(); + newAnnoMatrix.viewOf = this.viewOf.dropObsColumn(col); + newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col); + newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema; + return newAnnoMatrix; } addObsColumn(colSchema, Ctor, value) { - const o = this._clone(); - o.viewOf = this.viewOf.addObsColumn(colSchema, Ctor, value); - o.schema = o.viewOf.schema; - return o; + const newAnnoMatrix = this._clone(); + newAnnoMatrix.viewOf = this.viewOf.addObsColumn(colSchema, Ctor, value); + newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema; + return newAnnoMatrix; } renameObsColumn(oldCol, newCol) { - const o = this._clone(); - o.viewOf = this.viewOf.renameObsColumn(oldCol, newCol); - o.schema = o.viewOf.schema; - return o; + const newAnnoMatrix = this._clone(); + newAnnoMatrix.viewOf = this.viewOf.renameObsColumn(oldCol, newCol); + newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema; + return newAnnoMatrix; } async setObsColumnValues(col, rowLabels, value) { - const o = this._clone(); - o.viewOf = await this.viewOf.setObsColumnValues(col, rowLabels, value); - o._cache.obs = this._cache.obs.dropCol(col); - o.schema = o.viewOf.schema; - return o; + const newAnnoMatrix = this._clone(); + newAnnoMatrix.viewOf = await this.viewOf.setObsColumnValues( + col, + rowLabels, + value + ); + newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col); + newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema; + return newAnnoMatrix; } async resetObsColumnValues(col, oldValue, newValue) { - const o = this._clone(); - o.viewOf = await this.viewOf.resetObsColumnValues(col, oldValue, newValue); - o._cache.obs = this._cache.obs.dropCol(col); - o.schema = o.viewOf.schema; - return o; + const newAnnoMatrix = this._clone(); + newAnnoMatrix.viewOf = await this.viewOf.resetObsColumnValues( + col, + oldValue, + newValue + ); + newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col); + newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema; + return newAnnoMatrix; + } + + addEmbedding(colSchema) { + const newAnnoMatrix = this._clone(); + newAnnoMatrix.viewOf = this.viewOf.addEmbedding(colSchema); + newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema; + return newAnnoMatrix; } } diff --git a/client/src/components/embedding/index.js b/client/src/components/embedding/index.js index b71ae1a0..d4c7562c 100644 --- a/client/src/components/embedding/index.js +++ b/client/src/components/embedding/index.js @@ -101,7 +101,7 @@ export default Embedding; const loadAllEmbeddingCounts = async ({ annoMatrix, available }) => { const embeddings = await Promise.all( - available.map((name) => annoMatrix.fetch("emb", name)) + available.map((name) => annoMatrix.base().fetch("emb", name)) ); return available.map((name, idx) => ({ embeddingName: name, diff --git a/client/src/components/menubar/index.js b/client/src/components/menubar/index.js index 63d30b90..27e6ba7f 100644 --- a/client/src/components/menubar/index.js +++ b/client/src/components/menubar/index.js @@ -10,6 +10,7 @@ import InformationMenu from "./infoMenu"; import Subset from "./subset"; import UndoRedoReset from "./undoRedo"; import DiffexpButtons from "./diffexpButtons"; +import Reembedding from "./reembedding"; import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers"; @connect((state) => { @@ -49,6 +50,8 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers"; tosURL: state.config?.parameters?.["about_legal_tos"], privacyURL: state.config?.parameters?.["about_legal_privacy"], categoricalSelection: state.categoricalSelection, + enableReembedding: + state.config?.parameters?.["enable-reembedding"] ?? false, }; }) class MenuBar extends React.PureComponent { @@ -214,6 +217,7 @@ class MenuBar extends React.PureComponent { colorAccessor, subsetPossible, subsetResetPossible, + enableReembedding, } = this.props; const { pendingClipPercentiles } = this.state; @@ -266,6 +270,7 @@ class MenuBar extends React.PureComponent { this.handleClipPercentileMinValueChange } /> + {enableReembedding ? : null} ({ + reembedController: state.reembedController, + annoMatrix: state.annoMatrix, +})) +class Reembedding extends React.PureComponent { + render() { + const { dispatch, annoMatrix, reembedController } = this.props; + const loading = !!reembedController?.pendingFetch; + const disabled = annoMatrix.nObs === annoMatrix.schema.dataframe.nObs; + const tipContent = disabled + ? "Subset cells first, then click to recompute UMAP embedding." + : "Click to recompute UMAP embedding on the current cell subset."; + + return ( + + + dispatch(actions.requestReembed())} + loading={loading} + /> + + + ); + } +} + +export default Reembedding; diff --git a/client/src/reducers/index.js b/client/src/reducers/index.js index d910cef0..76d60f0c 100644 --- a/client/src/reducers/index.js +++ b/client/src/reducers/index.js @@ -18,7 +18,7 @@ import autosave from "./autosave"; import ontology from "./ontology"; import centroidLabels from "./centroidLabels"; import pointDialation from "./pointDilation"; -import { reembedController, reembedding } from "./reembed"; +import { reembedController } from "./reembed"; import { gcMiddleware as annoMatrixGC } from "../annoMatrix"; import undoableConfig from "./undoableConfig"; @@ -30,7 +30,6 @@ const Reducer = undoable( ["obsCrossfilter", obsCrossfilter], ["ontology", ontology], ["annotations", annotations], - ["reembedding", reembedding], ["layoutChoice", layoutChoice], ["categoricalSelection", categoricalSelection], ["continuousSelection", continuousSelection], @@ -55,7 +54,6 @@ const Reducer = undoable( "layoutChoice", "centroidLabels", "annotations", - "reembedding", ], undoableConfig ); diff --git a/client/src/reducers/layoutChoice.js b/client/src/reducers/layoutChoice.js index 48cff07e..14aa3f73 100644 --- a/client/src/reducers/layoutChoice.js +++ b/client/src/reducers/layoutChoice.js @@ -48,27 +48,15 @@ const LayoutChoice = ( } case "reembed: add reembedding": { + const { schema } = nextSharedState.annoMatrix; const { name } = action.schema; const available = Array.from(new Set(state.available).add(name)); + const currentDimNames = schema.layout.obsByName[name].dims; return { ...state, available, - }; - } - - case "reembed: clear all reembeddings": { - const { annoMatrix } = nextSharedState; - const { current } = state; - const dflt = setToDefaultLayout(annoMatrix.schema); - if (dflt.available.includes(current)) { - return { - ...state, - available: dflt.available, - }; - } - return { - ...state, - ...dflt, + current: name, + currentDimNames, }; } diff --git a/client/src/reducers/reembed.js b/client/src/reducers/reembed.js index a12a1e9b..02f649dc 100644 --- a/client/src/reducers/reembed.js +++ b/client/src/reducers/reembed.js @@ -27,38 +27,3 @@ export const reembedController = ( } } }; - -/* -actual reembedding data is part of the undo/redo history -*/ -export const reembedding = ( - state = { - reembeddings: new Map(), - }, - action -) => { - switch (action.type) { - case "reembed: add reembedding": { - const { schema, embedding } = action; - const { name } = schema.name; - const { reembeddings } = state; - return { - ...state, - reembeddings: new Map(reembeddings).set(name, { - name, - schema, - embedding, - }), - }; - } - case "reembed: clear all reembeddings": { - return { - ...state, - reembeddings: new Map(), - }; - } - default: { - return state; - } - } -}; diff --git a/server/common/rest.py b/server/common/rest.py index 0a0c2b50..b2e30305 100644 --- a/server/common/rest.py +++ b/server/common/rest.py @@ -304,10 +304,6 @@ def layout_obs_put(request, data_adaptor): if not data_adaptor.dataset_config.embeddings__enable_reembedding: return abort(HTTPStatus.NOT_IMPLEMENTED) - preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) - if preferred_mimetype != "application/octet-stream": - return abort(HTTPStatus.NOT_ACCEPTABLE) - args = request.get_json() filter = args["filter"] if args else None if not filter: @@ -315,17 +311,9 @@ def layout_obs_put(request, data_adaptor): method = args["method"] if args else "umap" try: - schema, fbs = data_adaptor.compute_embedding(method, filter) - return make_response( - fbs, - HTTPStatus.OK, - { - "Content-Type": "application/octet-stream", - "CxG-Schema": json.dumps(schema), - "Access-Control-Expose-Headers": "CxG-Schema", - }, - ) + schema = data_adaptor.compute_embedding(method, filter) + return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"}) except NotImplementedError as e: - return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e), include_exc_info=True) + return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e)) except (ValueError, DisabledFeatureError, FilterError) as e: return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) diff --git a/server/compute/scanpy.py b/server/compute/scanpy.py index cebafabd..36c2e1a9 100644 --- a/server/compute/scanpy.py +++ b/server/compute/scanpy.py @@ -1,4 +1,5 @@ import importlib +import numpy as np """ Wrapper for various scanpy modules. Will raise NotImplementedError if the scanpy @@ -11,8 +12,8 @@ def get_scanpy_module(): sc = importlib.import_module("scanpy") # Future: we could enforce versions here, eg, lookat sc.__version__ return sc - except ModuleNotFoundError: - raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") + except ModuleNotFoundError as e: + raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e except Exception as e: # will capture other ImportError corner cases raise NotImplementedError() from e @@ -46,4 +47,7 @@ def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap sc.pp.neighbors(adata, **neighbors_options) sc.tl.umap(adata, **umap_options) - return adata.obsm["X_umap"] + umap = adata.obsm["X_umap"] + result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN) + result[obs_mask] = umap + return result diff --git a/server/data_anndata/anndata_adaptor.py b/server/data_anndata/anndata_adaptor.py index 69496614..f9c781ad 100644 --- a/server/data_anndata/anndata_adaptor.py +++ b/server/data_anndata/anndata_adaptor.py @@ -1,7 +1,6 @@ import warnings import numpy as np -import pandas as pd from pandas.core.dtypes.dtypes import CategoricalDtype import anndata from scipy import sparse @@ -314,16 +313,15 @@ class AnndataAdaptor(DataAdaptor): raise FilterError("Error parsing filter") with ServerTiming.time("layout.compute"): X_umap = scanpy_umap(self.data, obs_mask) - normalized_layout = DataAdaptor.normalize_embedding(X_umap) # Server picks reemedding name, which must not collide with any other - # embedding name generated by this backed. + # embedding name generated by this backend. name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}" dims = [f"{name}_0", f"{name}_1"] - df = pd.DataFrame(normalized_layout, columns=dims) - fbs = encode_matrix_fbs(df, col_idx=df.columns, row_idx=None) - schema = {"name": name, "type": "float32", "dims": dims} - return (schema, fbs) + layout_schema = {"name": name, "type": "float32", "dims": dims} + self.schema["layout"]["obs"].append(layout_schema) + self.data.obsm[f"X_{name}"] = X_umap + return layout_schema def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None): if top_n is None: diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index 20f8b601..84ac5e64 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -71,8 +71,7 @@ class DataAdaptor(metaclass=ABCMeta): @abstractmethod def compute_embedding(self, method, filter): - """compute a new embedding on the specified obs subset, and return a - tuple of (schema, fbs).""" + """compute a new embedding on the specified obs subset, and return the embedding schema. """ pass @abstractmethod diff --git a/server/test/test_anndata_adaptor.py b/server/test/test_anndata_adaptor.py index 3afe2ea3..440e9b82 100644 --- a/server/test/test_anndata_adaptor.py +++ b/server/test/test_anndata_adaptor.py @@ -237,14 +237,14 @@ class AdaptorTest(unittest.TestCase): self.data.compute_embedding("umap", filter) return - (schema, fbs) = self.data.compute_embedding("umap", filter) + schema = self.data.compute_embedding("umap", filter) self.assertIsInstance(schema["name"], str) name = schema["name"] self.assertEqual(schema["type"], "float32") self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"]) - emb = decode_fbs.decode_matrix_FBS(fbs) - self.assertEqual(emb["n_rows"], 100) - self.assertEqual(emb["n_cols"], 2) - self.assertEqual(emb["col_idx"], [f"{name}_0", f"{name}_1"]) + emb = self.data.data.obsm[f"X_{name}"] + self.assertEqual(emb.shape, (2638, 2)) + self.assertTrue(np.isfinite(emb[0:100]).all()) + self.assertTrue(np.isnan(emb[100:]).all()) diff --git a/server/test/test_api.py b/server/test/test_api.py index d3ecb315..7b50c426 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -73,21 +73,23 @@ class EndPoints(object): # attempt to reembed with umap over 100 cells. endpoint = "layout/obs" url = f"{self.URL_BASE}{endpoint}" - header = {"Accept": "application/octet-stream"} data = {} data["filter"] = {} data["filter"]["obs"] = {} data["filter"]["obs"]["index"] = list(range(100)) data["method"] = "umap" - result = self.session.put(url, headers=header, json=data) + result = self.session.put(url, json=data) self.assertEqual(result.status_code, HTTPStatus.OK) - df = decode_fbs.decode_matrix_FBS(result.content) - self.assertEqual(df["n_rows"], 100) - self.assertEqual(df["n_cols"], 2) - cols = list(df["col_idx"]) - self.assertTrue(cols[0].startswith("reembed:umap_") and cols[0].endswith("_0")) - self.assertTrue(cols[1].startswith("reembed:umap_") and cols[1].endswith("_1")) + result_data = result.json() + self.assertIsInstance(result_data, dict) + self.assertEqual(result_data["type"], "float32") + self.assertTrue(result_data["name"].startswith("reembed:umap_")) + self.assertIsInstance(result_data["dims"], list) + self.assertEqual(len(result_data["dims"]), 2) + dims = result_data["dims"] + self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0")) + self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1")) def test_bad_filter(self): endpoint = "data/var" From d748b9f6915b8e6306527d9d03ef39a600bf85c5 Mon Sep 17 00:00:00 2001 From: Madison Dunitz Date: Thu, 30 Jul 2020 15:54:57 -0500 Subject: [PATCH 20/55] use czi-sci-single-cell-eng github user/access token (#1698) --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 48d1923a..aafcabd5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,4 +10,4 @@ jobs: steps: - name: repository dispatch run: | - curl -XPOST -u mdunitz:${{secrets.SCINFRA_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}' + curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}' From 055511fe60cc0c518d59c2503b538c15484ff209 Mon Sep 17 00:00:00 2001 From: Severiano Badajoz Date: Thu, 30 Jul 2020 15:33:48 -0700 Subject: [PATCH 21/55] fix colorby popup settings (#1694) --- client/src/components/categorical/category/index.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/client/src/components/categorical/category/index.js b/client/src/components/categorical/category/index.js index 77208b8e..90999deb 100644 --- a/client/src/components/categorical/category/index.js +++ b/client/src/components/categorical/category/index.js @@ -1,7 +1,7 @@ import React, { useRef, useEffect } from "react"; import { connect, shallowEqual } from "react-redux"; import { FaChevronRight, FaChevronDown } from "react-icons/fa"; -import { AnchorButton, Button, Tooltip } from "@blueprintjs/core"; +import { AnchorButton, Button, Tooltip, Position } from "@blueprintjs/core"; import { Flipper, Flipped } from "react-flip-toolkit"; import Async from "react-async"; import memoize from "memoize-one"; @@ -438,9 +438,13 @@ const CategoryHeader = React.memo( ? `Coloring by ${metadataField} is disabled, as it exceeds the limit of ${globals.maxCategoricalOptionsToDisplay} labels` : "Use as color scale" } - position="bottom" - usePortal={false} + position={Position.LEFT} + usePortal hoverOpenDelay={globals.tooltipHoverOpenDelay} + modifiers={{ + preventOverflow: { enabled: false }, + hide: { enabled: false }, + }} > Date: Fri, 31 Jul 2020 07:36:48 -0700 Subject: [PATCH 22/55] add support for corpora default_embedding field (#1696) * fix mispelling * re-implement re-embedding * always load base embedding to fetch counts * format * lint * fix tests * lint * fix accept handling * test log * more debug * more * more * more * more * remove logging * logging * jsonify * remove debugging logs * lint * clean up errors a bit * fix issue found in PR review * add support for corpora default_embedding * fix botched merge * PR review * PR review --- client/src/actions/index.js | 11 +++++- server/common/app_config.py | 8 +++- server/test/test_corpora.py | 75 ++++++++++++++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/client/src/actions/index.js b/client/src/actions/index.js index d9a77420..a829b45b 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -58,7 +58,7 @@ const doInitialDataLoad = () => dispatch({ type: "initial data load start" }); try { - const [, schema] = await Promise.all([ + const [config, schema] = await Promise.all([ configFetch(dispatch), schemaFetch(dispatch), userColorsFetchAndLoad(dispatch), @@ -75,6 +75,15 @@ const doInitialDataLoad = () => obsCrossfilter, }); dispatch({ type: "initial data load complete" }); + + const defaultEmbedding = config?.parameters?.["default_embedding"]; + const layoutSchema = schema?.schema?.layout?.obs ?? []; + if ( + defaultEmbedding && + layoutSchema.some((s) => s.name === defaultEmbedding) + ) { + dispatch(embActions.layoutChoiceAction(defaultEmbedding)); + } } catch (error) { dispatch({ type: "initial data load error", error }); } diff --git a/server/common/app_config.py b/server/common/app_config.py index da2f6463..ebca5f7f 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -242,11 +242,17 @@ class AppConfig(object): "about_legal_privacy": dataset_config.app__about_legal_privacy, } - # dataset_props + # corpora dataset_props # TODO/Note: putting info from the dataset into the /config is not ideal. # However, it is definitely not part of /schema, and we do not have a top-level # route for data properties. Consider creating one at some point. corpora_props = data_adaptor.get_corpora_props() + if corpora_props and "default_embedding" in corpora_props: + default_embedding = corpora_props["default_embedding"] + if isinstance(default_embedding, str) and default_embedding.startswith("X_"): + default_embedding = default_embedding[2:] # drop X_ prefix + if default_embedding in data_adaptor.get_embedding_names(): + parameters["default_embedding"] = default_embedding data_adaptor.update_parameters(parameters) if annotation: diff --git a/server/test/test_corpora.py b/server/test/test_corpora.py index b0637706..f502244c 100644 --- a/server/test/test_corpora.py +++ b/server/test/test_corpora.py @@ -1,13 +1,19 @@ import unittest import anndata import json +import tempfile +import shutil +from http import HTTPStatus +import requests from server.common.corpora import ( corpora_get_versions_from_anndata, corpora_is_version_supported, corpora_get_props_from_anndata, ) -from server.test import PROJECT_ROOT +from server.test import PROJECT_ROOT, start_test_server, stop_test_server + +VERSION = "v0.2" class CorporaAPITest(unittest.TestCase): @@ -71,3 +77,70 @@ class CorporaAPITest(unittest.TestCase): def _get_h5ad(self): return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + + +class CorporaRESTAPITest(unittest.TestCase): + """ Confirm endpoints reflect Corpora-specific features """ + + @classmethod + def setCorporaFields(cls, path): + adata = anndata.read_h5ad(path) + corpora_props = { + "version": { + "corpora_schema_version": "1.0.0", + "corpora_encoding_version": "0.1.0" + }, + "title": "PBMC3K", + "contributors": json.dumps([ + {"name": "name"} + ]), + "layer_descriptions": { + "X": "raw counts" + }, + "organism": "human", + "organism_ontology_term_id": "unknown", + "project_name": "test project", + "project_description": "test description", + "project_links": json.dumps([ + {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"} + ]), + "default_embedding": "X_tsne" + } + adata.uns.update(corpora_props) + adata.write(path) + + @classmethod + def setUpClass(cls): + cls.tmp_dir = tempfile.TemporaryDirectory() + src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad" + dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad" + shutil.copyfile(src, dst) + cls.setCorporaFields(dst) + cls.ps, cls.server = start_test_server([dst]) + + @classmethod + def tearDownClass(cls): + stop_test_server(cls.ps) + cls.tmp_dir.cleanup() + + def setUp(self): + self.session = requests.Session() + self.url_base = f"{self.server}/api/{VERSION}/" + + def test_config(self): + endpoint = "config" + url = f"{self.url_base}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + + result_data = result.json() + self.assertIsInstance(result_data["config"]["corpora_props"], dict) + self.assertIsInstance(result_data["config"]["parameters"], dict) + + corpora_props = result_data["config"]["corpora_props"] + parameters = result_data["config"]["parameters"] + + self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0") + self.assertEqual(corpora_props["organism"], "human") + self.assertEqual(parameters["default_embedding"], "tsne") From bb2326525efcc4ec2273c3c75c3512f0d05f8d84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Jul 2020 13:54:56 -0400 Subject: [PATCH 23/55] Bump elliptic from 6.5.2 to 6.5.3 in /client (#1697) Bumps [elliptic](https://github.com/indutny/elliptic) from 6.5.2 to 6.5.3. - [Release notes](https://github.com/indutny/elliptic/releases) - [Commits](https://github.com/indutny/elliptic/compare/v6.5.2...v6.5.3) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- client/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index 3b22dcf3..545957cd 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -8946,9 +8946,9 @@ "dev": true }, "elliptic": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.2.tgz", - "integrity": "sha512-f4x70okzZbIQl/NSRLkI/+tteV/9WqL98zx+SQ69KbXxmVrmjwsNUPn/gYJJ0sHvEak24cZgHIPegRePAtA/xw==", + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", + "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", "dev": true, "requires": { "bn.js": "^4.4.0", From 2afa48cf119379924d473cf7f7002f53900c8574 Mon Sep 17 00:00:00 2001 From: bmccandless Date: Fri, 31 Jul 2020 18:16:57 -0700 Subject: [PATCH 24/55] add oauth authentication (#1681) * add oauth authentication Add support for OAuth2. Change the interface to AuthTypeBase - better handling of config parameters - add a complete_setup function for additional setup steps Added a function wrapper to enforce authentication for the routes that require authenticaiton. * change fsspec requirement fsspec 0.8.0 breaks our tests it imports a module that is does not require. --- server/app/app.py | 31 ++++-- server/auth/__init__.py | 1 + server/auth/auth.py | 31 ++++-- server/auth/auth_none.py | 19 ++-- server/auth/auth_oauth.py | 188 ++++++++++++++++++++++++++++++++ server/auth/auth_session.py | 17 +-- server/auth/auth_test.py | 29 ++--- server/common/annotations.py | 12 +- server/common/app_config.py | 33 ++++-- server/common/default_config.py | 21 +++- server/common/errors.py | 4 + server/requirements.txt | 2 +- 12 files changed, 319 insertions(+), 69 deletions(-) create mode 100644 server/auth/auth_oauth.py diff --git a/server/app/app.py b/server/app/app.py index e082e978..cbbb6e4b 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -139,6 +139,18 @@ def get_data_adaptor(url_dataroot=None, dataset=None): return cache_manager.data_adaptor(dataset_key, datapath, config) +def requires_authentication(func): + @wraps(func) + def wrapped_function(self, *args, **kwargs): + auth = current_app.auth + if auth.is_user_authenticated(): + return func(self, *args, **kwargs) + else: + return make_response("not authenticated", HTTPStatus.UNAUTHORIZED) + + return wrapped_function + + def rest_get_data_adaptor(func): @wraps(func) def wrapped_function(self, dataset=None): @@ -164,11 +176,11 @@ def dataroot_test_index(): server_config = config.server_config auth = server_config.auth - if auth.is_valid(): - if server_config.auth.is_authenticated(): - data += f"

Logged in as {auth.get_userid()} / {auth.get_username()}

" + if auth.is_valid_authentication_type(): + if server_config.auth.is_user_authenticated(): + data += f"

Logged in as {auth.get_user_id()} / {auth.get_user_name()} / {auth.get_user_email()}

" if auth.requires_client_login(): - if server_config.auth.is_authenticated(): + if server_config.auth.is_user_authenticated(): data += "

Logout

" else: data += "

Login

" @@ -237,6 +249,7 @@ class AnnotationsObsAPI(DatasetResource): def get(self, data_adaptor): return common_rest.annotations_obs_get(request, data_adaptor) + @requires_authentication @cache_control(no_store=True) @rest_get_data_adaptor def put(self, data_adaptor): @@ -357,9 +370,11 @@ class Server: resources = get_api_resources(bp_api) self.app.register_blueprint(resources.blueprint) - self.app.auth = server_config.auth - if self.app.auth.requires_client_login(): - self.app.auth.add_url_rules(self.app) - self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager self.app.app_config = app_config + + auth = server_config.auth + self.app.auth = auth + if auth.requires_client_login(): + auth.add_url_rules(self.app) + auth.complete_setup(self.app) diff --git a/server/auth/__init__.py b/server/auth/__init__.py index 2af0d1ef..1b33bebc 100644 --- a/server/auth/__init__.py +++ b/server/auth/__init__.py @@ -4,3 +4,4 @@ import server.auth.auth_none # noqa: F401 import server.auth.auth_test # noqa: F401 import server.auth.auth_session # noqa: F401 +import server.auth.auth_oauth # noqa: F401 diff --git a/server/auth/auth.py b/server/auth/auth.py index 3262a9c1..bb86ea64 100644 --- a/server/auth/auth.py +++ b/server/auth/auth.py @@ -8,13 +8,9 @@ class AuthTypeBase(ABC): super().__init__() @abstractmethod - def set_params(self, params): - """Set the parameters from app config. raise ConfigurationError if any params are invalid""" - pass - - @abstractmethod - def is_valid(self): - """Return True if the auth type can return user info (AuthTypeNone is the only one that cannot)""" + def is_valid_authentication_type(self): + """Return True if the auth type is valid, e.g. it can return userinfo and username. + (AuthTypeNone is the only one type that returns False)""" pass def requires_client_login(self): @@ -22,17 +18,28 @@ class AuthTypeBase(ABC): return False @abstractmethod - def is_authenticated(self): + def complete_setup(self, app): + """complete any setup that may be needed by this auth type. The Flask app is passed in. + This is the last auth function called before the server starts to run.""" + pass + + @abstractmethod + def is_user_authenticated(self): """Return True if the user is authenticated""" pass @abstractmethod - def get_userid(self): + def get_user_id(self): """Return the id for this user (string)""" pass @abstractmethod - def get_username(self): + def get_user_name(self): + """Return the name of the user (string)""" + pass + + @abstractmethod + def get_user_email(self): """Return the name of the user (string)""" pass @@ -73,8 +80,8 @@ class AuthTypeFactory: AuthTypeFactory.auth_types[name] = auth_type @staticmethod - def create(name): + def create(name, app_config): auth_type = AuthTypeFactory.auth_types.get(name) if auth_type is None: return None - return auth_type() + return auth_type(app_config) diff --git a/server/auth/auth_none.py b/server/auth/auth_none.py index 9b2b8c0a..c482e3c4 100644 --- a/server/auth/auth_none.py +++ b/server/auth/auth_none.py @@ -1,26 +1,27 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory -from server.common.errors import ConfigurationError class AuthTypeNone(AuthTypeBase): - def __init__(self): + def __init__(self, app_config): super().__init__() - def is_valid(self): + def is_valid_authentication_type(self): return False - def set_params(self, params): - if params: - raise ConfigurationError("not expecting authentication parameters") + def complete_setup(self, app): + pass - def is_authenticated(self): + def is_user_authenticated(self): return True - def get_userid(self): + def get_user_id(self): return None - def get_username(self): + def get_user_name(self): + return None + + def get_user_email(self): return None diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py new file mode 100644 index 00000000..88669f88 --- /dev/null +++ b/server/auth/auth_oauth.py @@ -0,0 +1,188 @@ +from flask import session, request, redirect, current_app, has_request_context +from server.auth.auth import AuthTypeClientBase, AuthTypeFactory +from server.common.errors import AuthenticationError, ConfigurationError +from urllib.parse import urlencode +from urllib.request import urlopen +import json + +# It is not required to have authlib or jose. +# However, it is a configuration error to use this auth type if they are not installed. +missingimport = [] +try: + from authlib.integrations.flask_client import OAuth +except ModuleNotFoundError: + missingimport.append("authlib") + +try: + from jose import jwt +except ModuleNotFoundError: + missingimport.append("jose") + + +class AuthTypeOAuth(AuthTypeClientBase): + """An authentication type for oauth2 logins.""" + + CXG_ID_TOKEN = "id_token" + + def __init__(self, app_config): + super().__init__() + if missingimport: + raise ConfigurationError(f"oauth requires these modules: {', '.join(missingimport)}") + self.algorithms = ["RS256"] + self.api_base_url = app_config.authentication__params_oauth__api_base_url + self.client_id = app_config.authentication__params_oauth__client_id + self.client_secret = app_config.authentication__params_oauth__client_secret + self.callback_base_url = app_config.authentication__params_oauth__callback_base_url + self.audience = self.client_id + + # load the jwks (JSON Web Key Set). + # The JSON Web Key Set (JWKS) is a set of keys which contains the public keys used to verify + # any JSON Web Token (JWT) issued by the authorization server and signed using the RS256 + try: + jwksloc = f"{self.api_base_url}/.well-known/jwks.json" + jwksurl = urlopen(jwksloc) + self.jwks = json.loads(jwksurl.read()) + except Exception: + raise ConfigurationError(f"error in oauth, api_url_base: {self.api_base_url}, cannot access {jwksloc}") + + def is_valid_authentication_type(self): + return True + + def requires_client_login(self): + return True + + def add_url_rules(self, app): + app.add_url_rule("/login", "login", self.login, methods=["GET"]) + app.add_url_rule("/logout", "logout", self.logout, methods=["GET"]) + app.add_url_rule("/oauth2/callback", "callback", self.callback, methods=["GET"]) + + def complete_setup(self, flask_app): + self.oauth = OAuth(flask_app) + if self.callback_base_url is None: + # In this case, assume the server is running on the same host as the client, + # and the oauth provider has been configured + # with a callback that understands a localhost callback (e.g. A http://localhost:5005). + server_config = flask_app.app_config.server_config + self.callback_base_url = f"http://{server_config.app__host}:{server_config.app__port}" + + self.client = self.oauth.register( + "oauth", + client_id=self.client_id, + client_secret=self.client_secret, + api_base_url=self.api_base_url, + access_token_url=f"{self.api_base_url}/oauth/token", + authorize_url=f"{self.api_base_url}/authorize", + client_kwargs={ + "scope" : "openid profile email", + } + ) + + def is_user_authenticated(self): + try: + payload = self.get_jwt_payload() + return payload is not None + except AuthenticationError: + return False + + def get_user_id(self): + payload = self.get_jwt_payload() + if payload and payload.get("sub"): + return payload.get("sub") + return None + + def get_user_name(self): + payload = self.get_jwt_payload() + if payload and payload.get("name"): + return payload.get("name") + return None + + def get_user_email(self): + payload = self.get_jwt_payload() + if payload and payload.get("email"): + return payload.get("email") + return None + + def login(self): + callbackurl = f'{self.callback_base_url}/oauth2/callback' + return_path = request.args.get("dataset", "") + return_to = f"{self.callback_base_url}/{return_path}" + # save the return path in the session cookie, accessed in the callback function + session["oauth_callback_redirect"] = return_to + return self.client.authorize_redirect(redirect_uri=callbackurl) + + def logout(self): + if self.CXG_ID_TOKEN in session: + del session[self.CXG_ID_TOKEN] + return_path = request.args.get("dataset", "") + return_to = f"{self.callback_base_url}/{return_path}" + params = {'returnTo' : return_to, 'client_id' : self.client_id} + return redirect(self.client.api_base_url + '/v2/logout?' + urlencode(params)) + + def callback(self): + token = self.client.authorize_access_token() + id_token = token.get("id_token") + session[self.CXG_ID_TOKEN] = id_token + del session["oauth_callback_redirect"] + oauth_callback_redirect = session.get("oauth_callback_redirect", "/") + resp = redirect(oauth_callback_redirect) + return resp + + def get_login_url(self, data_adaptor): + """Return the url for the login route""" + if current_app.app_config.is_multi_dataset(): + return f"/login?dataset={data_adaptor.uri_path}" + else: + return "/login" + + def get_logout_url(self, data_adaptor): + """Return the url for the logout route""" + if current_app.app_config.is_multi_dataset(): + return f"/logout?dataset={data_adaptor.uri_path}" + else: + return "/logout" + + def get_token(self): + """Function to return the token""" + return session.get(self.CXG_ID_TOKEN) + + def get_jwt_payload(self): + if not has_request_context(): + return None + + token = self.get_token() + if token is None: + return None + + unverified_header = jwt.get_unverified_header(token) + rsa_key = {} + for key in self.jwks['keys']: + if key['kid'] == unverified_header['kid']: + rsa_key = { + 'kty': key['kty'], + 'kid': key['kid'], + 'use': key['use'], + 'n': key['n'], + 'e': key['e'] + } + if rsa_key: + try: + payload = jwt.decode( + token, + rsa_key, + algorithms=self.algorithms, + audience=self.audience, + issuer=self.api_base_url + "/" + ) + return payload + + except jwt.JWTError as e: + raise AuthenticationError(f"invalid signature: {str(e)}") + except jwt.ExpiredSignatureError as e: + raise AuthenticationError(f"token expired: {str(e)}") + except jwt.JWTClaimsError as e: + raise AuthenticationError(f"invalid claims {str(e)}") + + raise AuthenticationError("Unable to find the appropriate key") + + +AuthTypeFactory.register("oauth", AuthTypeOAuth) diff --git a/server/auth/auth_session.py b/server/auth/auth_session.py index d5754bc4..95157323 100644 --- a/server/auth/auth_session.py +++ b/server/auth/auth_session.py @@ -10,27 +10,30 @@ class AuthTypeSession(AuthTypeBase): # key in the session token for userid CXGUID = "cxguid" - def __init__(self): + def __init__(self, app_config): super().__init__() - def is_valid(self): + def is_valid_authentication_type(self): return True - def set_params(self, params): - return + def complete_setup(self, app): + pass - def is_authenticated(self): + def is_user_authenticated(self): # always authenticated return True - def get_userid(self): + def get_user_id(self): if self.CXGUID not in session: session[self.CXGUID] = uuid4().hex session.permanent = True return session[self.CXGUID] - def get_username(self): + def get_user_name(self): return "anonymous" + def get_user_email(self): + return None + AuthTypeFactory.register("session", AuthTypeSession) diff --git a/server/auth/auth_test.py b/server/auth/auth_test.py index d2222c12..e6b0b8e0 100644 --- a/server/auth/auth_test.py +++ b/server/auth/auth_test.py @@ -9,13 +9,15 @@ class AuthTypeTest(AuthTypeClientBase): # key in session token with userid and username CXGUID = "cxguid_test" CXGUNAME = "cxguname_test" + CXGUEMAIL = "cxguemail_test" - def __init__(self): + def __init__(self, app_config): super().__init__() - self.username = "test_account" - self.userid = "id0001" + self.user_name = "test_account" + self.user_id = "id0001" + self.user_email = "test_account@test.com" - def is_valid(self): + def is_valid_authentication_type(self): return True def requires_client_login(self): @@ -25,25 +27,26 @@ class AuthTypeTest(AuthTypeClientBase): app.add_url_rule("/login", "login", self.login, methods=["GET"]) app.add_url_rule("/logout", "logout", self.logout, methods=["GET"]) - def set_params(self, params): - if params: - self.username = params.get("username", self.username) - self.userid = params.get("userid", self.userid) + def complete_setup(self, app): + pass - def is_authenticated(self): + def is_user_authenticated(self): return self.CXGUID in session - def get_userid(self): + def get_user_id(self): return session.get(self.CXGUID) - def get_username(self): + def get_user_name(self): return session.get(self.CXGUNAME) + def get_user_email(self): + return session.get(self.CXGUEMAIL) + def login(self): args = request.args return_to = args.get("dataset", "/") - session[self.CXGUID] = args.get("userid", self.userid) - session[self.CXGUNAME] = args.get("username", self.username) + session[self.CXGUID] = args.get("userid", self.user_id) + session[self.CXGUNAME] = args.get("username", self.user_name) return redirect(return_to) def logout(self): diff --git a/server/common/annotations.py b/server/common/annotations.py index 25176ccb..592e4805 100644 --- a/server/common/annotations.py +++ b/server/common/annotations.py @@ -10,7 +10,7 @@ from server.common.errors import AnnotationsError, OntologyLoadFailure from server.common.utils import series_to_schema import fsspec import fastobo -from flask import session, current_app +from flask import session, current_app, has_request_context from abc import ABCMeta, abstractmethod @@ -46,8 +46,8 @@ class Annotations(metaclass=ABCMeta): raise OntologyLoadFailure("Error loading OBO file") from e def get_schema(self, data_adaptor): - labels = self.read_labels(data_adaptor) schema = [] + labels = self.read_labels(data_adaptor) if labels is not None and not labels.empty: for col in labels.columns: col_schema = dict(name=col, writable=True) @@ -113,6 +113,10 @@ class AnnotationsLocalFile(Annotations): return session.get(self.CXG_ANNO_COLLECTION) def read_labels(self, data_adaptor): + if has_request_context(): + if not current_app.auth.is_user_authenticated(): + return pd.DataFrame() + fname = self._get_filename(data_adaptor) with self.label_lock: if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0: @@ -162,7 +166,7 @@ class AnnotationsLocalFile(Annotations): Return a short hash that weakly identifies the user and dataset. Used to create safe annotations output file names. """ - uid = current_app.auth.get_userid() + uid = current_app.auth.get_user_id() id = (uid + data_adaptor.get_location()).encode() idhash = base64.b32encode(blake2b(id, digest_size=5).digest()).decode("utf-8") return idhash @@ -249,7 +253,7 @@ class AnnotationsLocalFile(Annotations): elif session is not None: collection = self.get_collection() - if current_app.auth.is_authenticated(): + if current_app.auth.is_user_authenticated(): params["annotations-user-data-idhash"] = self._get_userdata_idhash(data_adaptor) params["annotations-data-collection-is-read-only"] = False params["annotations-data-collection-name"] = collection diff --git a/server/common/app_config.py b/server/common/app_config.py index ebca5f7f..d54c86b0 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -272,11 +272,11 @@ class AppConfig(object): "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, } - if dataset_config.app__authentication_enable and auth.is_valid(): + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): config["authentication"] = { - "is_authenticated": auth.is_authenticated(), + "is_authenticated": auth.is_user_authenticated(), "requires_client_login": auth.requires_client_login(), - "username": auth.get_username(), + "username": auth.get_user_name(), } if auth.requires_client_login(): config["authentication"].update({ @@ -393,7 +393,6 @@ class ServerConfig(BaseConfig): def __init__(self, app_config, default_config): dictval_cases = [ ("app", "csp_directives"), - ("authentication", "params"), ("adaptor", "cxg_adaptor", "tiledb_ctx"), ("multi_dataset", "dataroot"), ] @@ -413,7 +412,11 @@ class ServerConfig(BaseConfig): self.app__csp_directives = dc["app"]["csp_directives"] self.authentication__type = dc["authentication"]["type"] - self.authentication__params = dc["authentication"]["params"] + self.authentication__params_oauth__api_base_url = dc["authentication"]["params_oauth"]["api_base_url"] + self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"] + self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"] + self.authentication__params_oauth__callback_base_url = \ + dc["authentication"]["params_oauth"]["callback_base_url"] self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"] self.multi_dataset__index = dc["multi_dataset"]["index"] @@ -445,7 +448,7 @@ class ServerConfig(BaseConfig): # The matrix data cache manager is created during the complete_config and stored here. self.matrix_data_cache_manager = None - # The authentication object (BCM -- better name) + # The authentication object self.auth = None def complete_config(self, context): @@ -521,11 +524,21 @@ class ServerConfig(BaseConfig): def handle_authentication(self, context): self.check_attr("authentication__type", (type(None), str)) - self.check_attr("authentication__params", (type(None), dict)) - self.auth = AuthTypeFactory.create(self.authentication__type) + + # oauth + ptypes = str if self.authentication__type == "oauth" else (type(None), str) + self.check_attr("authentication__params_oauth__api_base_url", ptypes) + self.check_attr("authentication__params_oauth__client_id", ptypes) + self.check_attr("authentication__params_oauth__client_secret", ptypes) + self.check_attr("authentication__params_oauth__callback_base_url", (type(None), str)) + # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable + # second, from config file + self.authentication__params__oauth__client_secret = os.environ.get( + "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret) + + self.auth = AuthTypeFactory.create(self.authentication__type, self) if self.auth is None: raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") - self.auth.set_params(self.authentication__params) def handle_data_locator(self, context): self.check_attr("data_locator__s3__region_name", (type(None), bool, str)) @@ -769,7 +782,7 @@ class DatasetConfig(BaseConfig): server_config = self.app_config.server_config if not self.app__authentication_enable: raise ConfigurationError("user annotations requires authentication to be enabled") - if not server_config.auth.is_valid(): + if not server_config.auth.is_valid_authentication_type(): auth_type = server_config.authentication__type raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") diff --git a/server/common/default_config.py b/server/common/default_config.py index bc585391..0fd79620 100644 --- a/server/common/default_config.py +++ b/server/common/default_config.py @@ -5,7 +5,7 @@ server: app: verbose: false debug: false - host: "127.0.0.1" + host: localhost port : null open_browser: false force_https: false @@ -15,13 +15,24 @@ server: csp_directives: null authentication: - # The authentication types may be "none" or "session" + # The authentication types may be "none", "session", "oauth" # none: No authentication support, features like user_annotations must not be enabled. - # session: A session based userid is automatically generated. + # session: A session based userid is automatically generated. (no params needed) + # oauth: oauth2 is used for authentication; parameters are defined in params_oauth. type: session - # a dictionary of parameters that may be required for an authentication type - params: null + params_oauth: + # url to the auth server + api_base_url: null + # client_id of this app + client_id: null + # the client_secret known to the auth server and this app + client_secret: null + # cellxgene server location; + # the browser will be redirected to locations relative to this location during login and logout. + # A value of None, indicates the client and server are on the localhost. http://localhost: will be used. + callback_base_url: null + multi_dataset: # If dataroot is set, then cellxgene may serve multiple datasets. This parameter is not diff --git a/server/common/errors.py b/server/common/errors.py index ca927084..dfdb9e39 100644 --- a/server/common/errors.py +++ b/server/common/errors.py @@ -41,6 +41,10 @@ define_request_exception( ) define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota") define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format") +define_request_exception( + "AuthenticationError", + "Raised when there is an authentication error", + default_status_code=HTTPStatus.UNAUTHORIZED) define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails") define_exception("ConfigurationError", "Raised when checking configuration errors") diff --git a/server/requirements.txt b/server/requirements.txt index d47e383f..3b1909d1 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -10,7 +10,7 @@ flask-server-timing>=0.1.2 flask-talisman>=0.7.0 flatbuffers>=1.10.0 flatten-dict>=0.2.0 -fsspec>=0.4.4 +fsspec>=0.4.4,<0.8.0 numba>=0.49.1 numpy>=1.16.0 packaging>=20.0 From ce13a9c7ca81f204da787750d736bef2a2cb63ff Mon Sep 17 00:00:00 2001 From: bmccandless Date: Mon, 3 Aug 2020 10:45:21 -0700 Subject: [PATCH 25/55] oauth support, add the token in a configuration specified cookie (#1702) * oauth support, add the token in a configuration specified cookie Previously, the id token was stored in the session token. Now, it can be placed in a different cookie with different properties. --- server/auth/auth_oauth.py | 71 +++++++++++++++++++++++++++------ server/common/app_config.py | 9 +++++ server/common/default_config.py | 9 +++++ 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py index 88669f88..ca2adf95 100644 --- a/server/auth/auth_oauth.py +++ b/server/auth/auth_oauth.py @@ -1,4 +1,4 @@ -from flask import session, request, redirect, current_app, has_request_context +from flask import session, request, redirect, current_app, after_this_request, has_request_context, g from server.auth.auth import AuthTypeClientBase, AuthTypeFactory from server.common.errors import AuthenticationError, ConfigurationError from urllib.parse import urlencode @@ -24,15 +24,20 @@ class AuthTypeOAuth(AuthTypeClientBase): CXG_ID_TOKEN = "id_token" - def __init__(self, app_config): + def __init__(self, server_config): super().__init__() if missingimport: raise ConfigurationError(f"oauth requires these modules: {', '.join(missingimport)}") self.algorithms = ["RS256"] - self.api_base_url = app_config.authentication__params_oauth__api_base_url - self.client_id = app_config.authentication__params_oauth__client_id - self.client_secret = app_config.authentication__params_oauth__client_secret - self.callback_base_url = app_config.authentication__params_oauth__callback_base_url + self.api_base_url = server_config.authentication__params_oauth__api_base_url + self.client_id = server_config.authentication__params_oauth__client_id + self.client_secret = server_config.authentication__params_oauth__client_secret + self.callback_base_url = server_config.authentication__params_oauth__callback_base_url + self.session_cookie = server_config.authentication__params_oauth__session_cookie + self.cookie_params = server_config.authentication__params_oauth__cookie + self._validate_cookie_params() + + # set the audience self.audience = self.client_id # load the jwks (JSON Web Key Set). @@ -45,6 +50,21 @@ class AuthTypeOAuth(AuthTypeClientBase): except Exception: raise ConfigurationError(f"error in oauth, api_url_base: {self.api_base_url}, cannot access {jwksloc}") + def _validate_cookie_params(self): + """check the cookie_params, and raise a ConfigurationError if there is something wrong""" + if self.session_cookie: + return + + if not isinstance(self.cookie_params, dict): + raise ConfigurationError("either session_cookie or cookie must be set") + valid_keys = {"key", "max_age", "expires", "path", "domain", "secure", "httponly", "samesite"} + keys = set(self.cookie_params.keys()) + unknown = keys - valid_keys + if unknown: + raise ConfigurationError(f"unexpected key in cookie params: {', '.join(unknown)}") + if "key" not in keys: + raise ConfigurationError("must have a key (name) in the cookie params") + def is_valid_authentication_type(self): return True @@ -111,8 +131,15 @@ class AuthTypeOAuth(AuthTypeClientBase): return self.client.authorize_redirect(redirect_uri=callbackurl) def logout(self): - if self.CXG_ID_TOKEN in session: - del session[self.CXG_ID_TOKEN] + if self.session_cookie: + if self.CXG_ID_TOKEN in session: + del session[self.CXG_ID_TOKEN] + else: + @after_this_request + def remove_cookie(response): + response.set_cookie(self.cookie_params["key"], "", expires=0) + return response + return_path = request.args.get("dataset", "") return_to = f"{self.callback_base_url}/{return_path}" params = {'returnTo' : return_to, 'client_id' : self.client_id} @@ -121,10 +148,23 @@ class AuthTypeOAuth(AuthTypeClientBase): def callback(self): token = self.client.authorize_access_token() id_token = token.get("id_token") - session[self.CXG_ID_TOKEN] = id_token - del session["oauth_callback_redirect"] - oauth_callback_redirect = session.get("oauth_callback_redirect", "/") + oauth_callback_redirect = session.pop("oauth_callback_redirect", "/") resp = redirect(oauth_callback_redirect) + + if self.session_cookie: + session[self.CXG_ID_TOKEN] = id_token + else: + args = self.cookie_params.copy() + del args["key"] + try: + resp.set_cookie( + self.cookie_params["key"], + id_token, + **args) + g.token = id_token + except Exception as e: + raise AuthenticationError(f"unable to set_cookie {self.cookie_params}") from e + return resp def get_login_url(self, data_adaptor): @@ -143,7 +183,14 @@ class AuthTypeOAuth(AuthTypeClientBase): def get_token(self): """Function to return the token""" - return session.get(self.CXG_ID_TOKEN) + if "token" in g: + return g.token + if self.session_cookie: + g.token = session.get(self.CXG_ID_TOKEN) + else: + g.token = request.cookies.get(self.cookie_params["key"]) + + return g.token def get_jwt_payload(self): if not has_request_context(): diff --git a/server/common/app_config.py b/server/common/app_config.py index d54c86b0..aef6af98 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -393,6 +393,7 @@ class ServerConfig(BaseConfig): def __init__(self, app_config, default_config): dictval_cases = [ ("app", "csp_directives"), + ("authentication", "params_oauth", "cookie"), ("adaptor", "cxg_adaptor", "tiledb_ctx"), ("multi_dataset", "dataroot"), ] @@ -417,6 +418,8 @@ class ServerConfig(BaseConfig): self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"] self.authentication__params_oauth__callback_base_url = \ dc["authentication"]["params_oauth"]["callback_base_url"] + self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"] + self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"] self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"] self.multi_dataset__index = dc["multi_dataset"]["index"] @@ -531,6 +534,12 @@ class ServerConfig(BaseConfig): self.check_attr("authentication__params_oauth__client_id", ptypes) self.check_attr("authentication__params_oauth__client_secret", ptypes) self.check_attr("authentication__params_oauth__callback_base_url", (type(None), str)) + self.check_attr("authentication__params_oauth__session_cookie", bool) + + if self.authentication__params_oauth__session_cookie: + self.check_attr("authentication__params_oauth__cookie", (type(None), dict)) + else: + self.check_attr("authentication__params_oauth__cookie", dict) # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable # second, from config file self.authentication__params__oauth__client_secret = os.environ.get( diff --git a/server/common/default_config.py b/server/common/default_config.py index 0fd79620..f77559bc 100644 --- a/server/common/default_config.py +++ b/server/common/default_config.py @@ -33,6 +33,15 @@ server: # A value of None, indicates the client and server are on the localhost. http://localhost: will be used. callback_base_url: null + # if true, the jwt containing the id_token is stored in a session cookie + session_cookie: true + + # if session_cookie is false, then a regular cookie will be used. In that case + # the cookie will be defined by a dictionary of parameters. + # The keys of the dictionary match the parameters of the flask set_cookie api + # (https://flask.palletsprojects.com/en/1.1.x/api/), and with the same meaning. + # legal keys: key, max_age, expires, path, domain, secure, httponly, and samesite. + cookie: null multi_dataset: # If dataroot is set, then cellxgene may serve multiple datasets. This parameter is not From 8bbc183647985805ae3f513d0df39f8964d2ed9f Mon Sep 17 00:00:00 2001 From: Severiano Badajoz Date: Mon, 3 Aug 2020 12:50:09 -0700 Subject: [PATCH 26/55] Explicit Browser Support (#1682) * add FastestSmallestTextEncoderDecoder polyfill * remove nomodule from script import * add browserslist * add obsolete-webpack-plugin * switch out modern-browser for preset-env * add prompt on non target browser * propagate prod changes to dev * add core-js-3 and TextEncoder TextDecoder (#1671) * add script to remove react, style html * propagate changes to prod * add script-ext-html-webpack-plugin for async * more styling * add eslint-plugin-compat * extend compat plugin * add existing polyfills * add github fetch polyfill * add AbortController polyfill * change promptOnNonTargetBrowser to false * propagate * add browser links * prettier * add browser support to readme * move polyfills to webpack * remove CDN encoder polyfill * Add no Explorer support * fix incorrect package name * propagate changes * Update README.md Co-authored-by: Ambrose J Carr * add new deps * create shared config * swap out html-loader for filestream * sanitize template Co-authored-by: Timmy Huang Co-authored-by: Ambrose J Carr --- README.md | 31 ++- client/configuration/babel/babel.dev.js | 9 +- client/configuration/babel/babel.prod.js | 9 +- client/configuration/eslint/eslint.js | 12 + .../webpack/obsoleteHTMLTemplate.html | 81 ++++++ .../webpack/webpack.config.dev.js | 59 ++--- .../webpack/webpack.config.prod.js | 55 +---- .../webpack/webpack.config.shared.js | 74 ++++++ client/package-lock.json | 233 +++++++++++++++++- client/package.json | 19 +- client/src/index.js | 1 - client/src/util/stateManager/matrix.js | 4 +- 12 files changed, 482 insertions(+), 105 deletions(-) create mode 100644 client/configuration/webpack/obsoleteHTMLTemplate.html create mode 100644 client/configuration/webpack/webpack.config.shared.js diff --git a/README.md b/README.md index 2330f20f..18f6fdc3 100644 --- a/README.md +++ b/README.md @@ -14,26 +14,30 @@ Whether you need to visualize one thousand cells or one million, cellxgene helps # Getting started + ### The comprehensive guide to cellxgene + [The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in: -* Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery) -* Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch) -* [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene -* Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development -* [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene + +- Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery) +- Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch) +- [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene +- Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development +- [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene ### Quick start To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install) Install the package. -``` bash + +```bash pip install cellxgene ``` Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file -``` bash +```bash cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad ``` @@ -41,6 +45,17 @@ To explore more datasets already formatted for cellxgene, check out the [Demo da see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own data for cellxgene. +### Supported browsers + +cellxgene currently supports the following browsers: + +- Google Chrome 61+ +- Edge 15+ +- Firefox 60+ +- Safari 10.1+ + +Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser. + ### Finding help We'd love to hear from you! @@ -51,6 +66,7 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg # Developing with cellxgene ### Contributing + We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com. @@ -64,6 +80,7 @@ This project was started with the sole goal of empowering the scientific communi If you believe you have found a security issue, we would appreciate notification. Please send email to . # About + ### Core team The current core team: diff --git a/client/configuration/babel/babel.dev.js b/client/configuration/babel/babel.dev.js index 8f32d635..117f6da5 100644 --- a/client/configuration/babel/babel.dev.js +++ b/client/configuration/babel/babel.dev.js @@ -2,7 +2,14 @@ module.exports = { babelrc: false, cacheDirectory: true, presets: [ - ["modern-browsers", { loose: true, modules: false }], + [ + "@babel/preset-env", + { + useBuiltIns: "entry", + corejs: 3, + modules: false, + }, + ], "@babel/preset-react", ], plugins: [ diff --git a/client/configuration/babel/babel.prod.js b/client/configuration/babel/babel.prod.js index 604e2ef2..620cd563 100644 --- a/client/configuration/babel/babel.prod.js +++ b/client/configuration/babel/babel.prod.js @@ -1,7 +1,14 @@ module.exports = { babelrc: false, presets: [ - ["modern-browsers", { loose: true, modules: false }], + [ + "@babel/preset-env", + { + useBuiltIns: "entry", + corejs: 3, + modules: false, + }, + ], "@babel/preset-react", ], plugins: [ diff --git a/client/configuration/eslint/eslint.js b/client/configuration/eslint/eslint.js index a1189581..66637c26 100644 --- a/client/configuration/eslint/eslint.js +++ b/client/configuration/eslint/eslint.js @@ -4,9 +4,21 @@ module.exports = { extends: [ "airbnb", "plugin:eslint-comments/recommended", + "plugin:compat/recommended", "plugin:prettier/recommended", "prettier/react", ], + settings: { + polyfills: [ + "TextDecoder", + "TextEncoder", + "fetch", + "Request", + "Response", + "Headers", + "AbortController", + ], + }, env: { browser: true, commonjs: true, es6: true }, globals: { expect: true, diff --git a/client/configuration/webpack/obsoleteHTMLTemplate.html b/client/configuration/webpack/obsoleteHTMLTemplate.html new file mode 100644 index 00000000..e5c96849 --- /dev/null +++ b/client/configuration/webpack/obsoleteHTMLTemplate.html @@ -0,0 +1,81 @@ + +
+ +
+
+ Unsupported Browser +
+
+ cellxgene is currently supported on the following browsers +
+ +
+
diff --git a/client/configuration/webpack/webpack.config.dev.js b/client/configuration/webpack/webpack.config.dev.js index 5175a08b..e0881c0a 100644 --- a/client/configuration/webpack/webpack.config.dev.js +++ b/client/configuration/webpack/webpack.config.dev.js @@ -1,65 +1,32 @@ -// jshint esversion: 6 const path = require("path"); const webpack = require("webpack"); const HtmlWebpackPlugin = require("html-webpack-plugin"); const FaviconsWebpackPlugin = require("favicons-webpack-plugin"); +const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin"); +const MiniCssExtractPlugin = require("mini-css-extract-plugin"); + +const { merge } = require("webpack-merge"); + +const sharedConfig = require("./webpack.config.shared"); +const babelOptions = require("../babel/babel.dev"); -const src = path.resolve("src"); const fonts = path.resolve("src/fonts"); const nodeModules = path.resolve("node_modules"); -const babelOptions = require("../babel/babel.dev"); - -module.exports = { +const devConfig = { mode: "development", devtool: "eval", - entry: ["./src/index"], output: { - path: path.resolve("build"), pathinfo: true, filename: "static/js/bundle.js", - publicPath: "/", }, module: { rules: [ { - test: /\.js$/, - include: src, + test: /\.jsx?$/, loader: "babel-loader", options: babelOptions, }, - { - test: /\.css$/, - include: src, - exclude: [path.resolve(src, "index.css")], - loader: [ - { - loader: "style-loader", - }, - { - loader: "css-loader", - options: { - modules: { - localIdentName: "[name]__[local]___[hash:base64:5]", - }, - }, - }, - ], - }, - { - test: /index\.css$/, - include: [path.resolve(src, "index.css")], - loader: [ - { - loader: "style-loader", - }, - { - loader: "css-loader", - }, - ], - }, - - { test: /\.json$/, include: [src, nodeModules], loader: "json-loader" }, { test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i, loader: "file-loader", @@ -88,6 +55,9 @@ module.exports = { }, }, }), + new MiniCssExtractPlugin({ + filename: "static/[name].css", + }), new webpack.NoEmitOnErrorsPlugin(), new webpack.DefinePlugin({ __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })", @@ -97,5 +67,10 @@ module.exports = { process.env.CXG_SERVER_PORT ), }), + new ScriptExtHtmlWebpackPlugin({ + async: "obsolete", + }), ], }; + +module.exports = merge(sharedConfig, devConfig); diff --git a/client/configuration/webpack/webpack.config.prod.js b/client/configuration/webpack/webpack.config.prod.js index 0bcee9ee..a2115e7f 100644 --- a/client/configuration/webpack/webpack.config.prod.js +++ b/client/configuration/webpack/webpack.config.prod.js @@ -1,32 +1,28 @@ -// jshint esversion: 6 const path = require("path"); const HtmlWebpackPlugin = require("html-webpack-plugin"); -const MiniCssExtractPlugin = require("mini-css-extract-plugin"); -const FaviconsWebpackPlugin = require("favicons-webpack-plugin"); const { CleanWebpackPlugin } = require("clean-webpack-plugin"); const TerserJSPlugin = require("terser-webpack-plugin"); const CleanCss = require("clean-css"); const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin"); +const FaviconsWebpackPlugin = require("favicons-webpack-plugin"); +const MiniCssExtractPlugin = require("mini-css-extract-plugin"); -const CspHashPlugin = require("./cspHashPlugin"); - -const src = path.resolve("src"); -const fonts = path.resolve("src/fonts"); -const nodeModules = path.resolve("node_modules"); +const { merge } = require("webpack-merge"); const babelOptions = require("../babel/babel.prod"); -const publicPath = "/"; +const CspHashPlugin = require("./cspHashPlugin"); +const sharedConfig = require("./webpack.config.shared"); -module.exports = { +const fonts = path.resolve("src/fonts"); +const nodeModules = path.resolve("node_modules"); + +const prodConfig = { mode: "production", bail: true, cache: false, - entry: ["./src/index.js"], output: { filename: "static/[name]-[contenthash].js", - path: path.resolve("build"), - publicPath, }, optimization: { minimize: true, @@ -41,39 +37,10 @@ module.exports = { module: { rules: [ { - test: /\.js$/, - include: src, + test: /\.jsx?$/, loader: "babel-loader", options: babelOptions, }, - { - test: /\.css$/, - include: src, - exclude: [path.resolve(src, "index.css")], - use: [ - MiniCssExtractPlugin.loader, - { - loader: "css-loader", - options: { - modules: { - localIdentName: "[name]__[local]___[hash:base64:5]", - }, - importLoaders: 1, - }, - }, - ], - }, - { - test: /index\.css$/, - include: [path.resolve(src, "index.css")], - use: [MiniCssExtractPlugin.loader, "css-loader"], - }, - { - test: /\.json$/, - include: [src, nodeModules], - loader: "json-loader", - exclude: /manifest.json$/, - }, { test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i, loader: "file-loader", @@ -121,3 +88,5 @@ module.exports = { maxAssetSize: 2000000, }, }; + +module.exports = merge(sharedConfig, prodConfig); diff --git a/client/configuration/webpack/webpack.config.shared.js b/client/configuration/webpack/webpack.config.shared.js new file mode 100644 index 00000000..2a8fcfdd --- /dev/null +++ b/client/configuration/webpack/webpack.config.shared.js @@ -0,0 +1,74 @@ +const path = require("path"); +const fs = require("fs"); +const MiniCssExtractPlugin = require("mini-css-extract-plugin"); +const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin"); +const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin"); + +const src = path.resolve("src"); +const nodeModules = path.resolve("node_modules"); + +const publicPath = "/"; + +const rawObsoleteHTMLTemplate = fs.readFileSync( + `${__dirname}/obsoleteHTMLTemplate.html`, + "utf8" +); + +const obsoleteHTMLTemplate = rawObsoleteHTMLTemplate.replace(/'/g, '"'); + +module.exports = { + entry: [ + "core-js", + "regenerator-runtime/runtime", + "fastestsmallesttextencoderdecoder", + "whatwg-fetch", + "abort-controller/polyfill", + "./src/index", + ], + output: { + path: path.resolve("build"), + publicPath, + }, + module: { + rules: [ + { + test: /\.css$/, + include: src, + exclude: [path.resolve(src, "index.css")], + use: [ + MiniCssExtractPlugin.loader, + { + loader: "css-loader", + options: { + modules: { + localIdentName: "[name]__[local]___[hash:base64:5]", + }, + importLoaders: 1, + }, + }, + ], + }, + { + test: /index\.css$/, + include: [path.resolve(src, "index.css")], + use: [MiniCssExtractPlugin.loader, "css-loader"], + }, + { + test: /\.json$/, + include: [src, nodeModules], + loader: "json-loader", + exclude: /manifest.json$/, + }, + ], + }, + plugins: [ + new ObsoleteWebpackPlugin({ + name: "obsolete", + template: obsoleteHTMLTemplate, + promptOnNonTargetBrowser: false, + }), + new ScriptExtHtmlWebpackPlugin({ + async: "obsolete", + }), + ], +}; diff --git a/client/package-lock.json b/client/package-lock.json index 545957cd..bf146854 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -4069,6 +4069,24 @@ "regenerator-runtime": "^0.13.4" } }, + "@babel/runtime-corejs2": { + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs2/-/runtime-corejs2-7.10.5.tgz", + "integrity": "sha512-LJwyb1ac//Jr2zrGTTaNJhrP1wYCgVw9rzHbQPogKXCTLQ60EEWgeNtuqs6cLsq64O557SYzziCrOxNp0rRi8w==", + "dev": true, + "requires": { + "core-js": "^2.6.5", + "regenerator-runtime": "^0.13.4" + }, + "dependencies": { + "core-js": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz", + "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==", + "dev": true + } + } + }, "@babel/runtime-corejs3": { "version": "7.10.5", "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.10.5.tgz", @@ -5686,6 +5704,14 @@ "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.3.tgz", "integrity": "sha512-tsFzPpcttalNjFBCFMqsKYQcWxxen1pgJR56by//QwvJc4/OUS3kPOOttx2tSIfjsylB0pYu7f5D3K1RCxUnUg==" }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "requires": { + "event-target-shim": "^5.0.0" + } + }, "accepts": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz", @@ -6057,6 +6083,12 @@ "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=" }, + "ast-metadata-inferer": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/ast-metadata-inferer/-/ast-metadata-inferer-0.4.0.tgz", + "integrity": "sha512-tKHdBe8N/Vq2nLAm4YPBVREVZjMux6KrqyPfNQgIbDl0t7HaNSmy8w4OyVHYg/cvyn5BW7o7pVwpjPte89Zhcg==", + "dev": true + }, "ast-types-flow": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", @@ -6944,6 +6976,12 @@ "lodash.uniq": "^4.5.0" } }, + "caniuse-db": { + "version": "1.0.30001107", + "resolved": "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30001107.tgz", + "integrity": "sha512-ffbV17yvEamsNm4N4dDDHdj147tWwdKw+mGyeOmvQcnu+gu455xUg8degvUOCB+fIAm7Rv3gXVn7XlTiYKymMQ==", + "dev": true + }, "caniuse-lite": { "version": "1.0.30001039", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001039.tgz", @@ -7725,10 +7763,9 @@ "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=" }, "core-js": { - "version": "2.6.11", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz", - "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==", - "dev": true + "version": "3.6.5", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.6.5.tgz", + "integrity": "sha512-vZVEEwZoIsI+vPEuoF9Iqf5H7/M3eeQqWlQnYa8FSKKePuYTf5MWnxb5SDAzCa60b3JBRS5g9b+Dq7b1y/RCrA==" }, "core-js-compat": { "version": "3.6.5", @@ -9448,6 +9485,94 @@ } } }, + "eslint-plugin-compat": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-compat/-/eslint-plugin-compat-3.8.0.tgz", + "integrity": "sha512-5CuWUSZXZkXLCQJBriEpndn/YWrvggDSHTpRJq++kR8GVcsWbTdp8Eh+nBA7JlrNi7ZJ/+kniOVXmn3bpnxuRA==", + "dev": true, + "requires": { + "ast-metadata-inferer": "^0.4.0", + "browserslist": "^4.12.2", + "caniuse-db": "^1.0.30001090", + "core-js": "^3.6.5", + "find-up": "^4.1.0", + "lodash.memoize": "4.1.2", + "mdn-browser-compat-data": "^1.0.28", + "semver": "7.3.2" + }, + "dependencies": { + "browserslist": { + "version": "4.13.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.13.0.tgz", + "integrity": "sha512-MINatJ5ZNrLnQ6blGvePd/QOz9Xtu+Ne+x29iQSCHfkU5BugKVJwZKn/iiL8UbpIpa3JhviKjz+XxMo0m2caFQ==", + "dev": true, + "requires": { + "caniuse-lite": "^1.0.30001093", + "electron-to-chromium": "^1.3.488", + "escalade": "^3.0.1", + "node-releases": "^1.1.58" + } + }, + "caniuse-lite": { + "version": "1.0.30001107", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001107.tgz", + "integrity": "sha512-86rCH+G8onCmdN4VZzJet5uPELII59cUzDphko3thQFgAQG1RNa+sVLDoALIhRYmflo5iSIzWY3vu1XTWtNMQQ==", + "dev": true + }, + "electron-to-chromium": { + "version": "1.3.510", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.510.tgz", + "integrity": "sha512-sLtGB0znXdmo6lM8hy5wTVo+fLqvIuO8hEpgc0DvPmFZqvBu/WB7AarEwhxVKjf3rVbws/rC8Xf+AlsOb36lJQ==", + "dev": true + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "requires": { + "p-locate": "^4.1.0" + } + }, + "node-releases": { + "version": "1.1.60", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-1.1.60.tgz", + "integrity": "sha512-gsO4vjEdQaTusZAEebUWp2a5d7dF5DYoIpDG7WySnk7BuZDW+GPpHXoXXuYawRBr/9t5q54tirPz79kFIWg4dA==", + "dev": true + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "requires": { + "p-limit": "^2.2.0" + } + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true + } + } + }, "eslint-plugin-eslint-comments": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/eslint-plugin-eslint-comments/-/eslint-plugin-eslint-comments-3.2.0.tgz", @@ -9716,6 +9841,11 @@ "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", "dev": true }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" + }, "events": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/events/-/events-3.1.0.tgz", @@ -10067,6 +10197,11 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" }, + "fastestsmallesttextencoderdecoder": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz", + "integrity": "sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw==" + }, "favicons": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/favicons/-/favicons-5.5.0.tgz", @@ -13886,6 +14021,15 @@ "safe-buffer": "^5.1.2" } }, + "mdn-browser-compat-data": { + "version": "1.0.32", + "resolved": "https://registry.npmjs.org/mdn-browser-compat-data/-/mdn-browser-compat-data-1.0.32.tgz", + "integrity": "sha512-dqIstpk2ysqa6XcI8/fz1yB6bOKrIs61RIEE00Dj7+WHReXlGrCIiol1NBPsLUNE+HC/4y2f8va8vy1WsiCkAQ==", + "dev": true, + "requires": { + "extend": "3.0.2" + } + }, "mdn-data": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.4.tgz", @@ -14769,6 +14913,26 @@ "has": "^1.0.3" } }, + "obsolete-web": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/obsolete-web/-/obsolete-web-0.5.6.tgz", + "integrity": "sha512-rrs7kSJxOVFvvY7wuCfjg/ngXbO6q1m7Llq30RaN9WYIDH1zAoA1xWLIY39D3e5967g/NeCasI/o8ojhMMkaaA==", + "dev": true, + "requires": { + "@babel/runtime-corejs2": "^7.0.0" + } + }, + "obsolete-webpack-plugin": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/obsolete-webpack-plugin/-/obsolete-webpack-plugin-0.5.6.tgz", + "integrity": "sha512-oKlRW4ycxJfF/mojtpGuQwaP+J4JwIgjFuFnMgURB6AaKxAVaRwiO0oWhqYjwwJ5LxhVybOl+CnGAlhHBHBdEQ==", + "dev": true, + "requires": { + "browserslist": "^4.0.0", + "obsolete-web": "^0.5.6", + "webpack-sources": "^1.0.0" + } + }, "omggif": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/omggif/-/omggif-1.0.10.tgz", @@ -16491,9 +16655,9 @@ } }, "regenerator-runtime": { - "version": "0.13.5", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz", - "integrity": "sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==" + "version": "0.13.7", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", + "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==" }, "regenerator-transform": { "version": "0.14.5", @@ -17147,6 +17311,14 @@ "ajv-keywords": "^3.1.0" } }, + "script-ext-html-webpack-plugin": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/script-ext-html-webpack-plugin/-/script-ext-html-webpack-plugin-2.1.4.tgz", + "integrity": "sha512-7MAv3paAMfh9y2Rg+yQKp9jEGC5cEcmdge4EomRqri10qoczmliYEVPVNz0/5e9QQ202e05qDll9B8zZlY9N1g==", + "requires": { + "debug": "^4.1.1" + } + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -19513,6 +19685,14 @@ "minimist": "^1.2.0", "request": "^2.88.0", "rx": "^4.1.0" + }, + "dependencies": { + "core-js": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz", + "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==", + "dev": true + } } }, "wait-port": { @@ -20168,6 +20348,35 @@ "uuid": "^3.3.2" } }, + "webpack-merge": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-5.0.9.tgz", + "integrity": "sha512-P4teh6O26xIDPugOGX61wPxaeP918QOMjmzhu54zTVcLtOS28ffPWtnv+ilt3wscwBUCL2WNMnh97XkrKqt9Fw==", + "requires": { + "clone-deep": "^4.0.1", + "wildcard": "^2.0.0" + }, + "dependencies": { + "clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "requires": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + } + }, + "shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "requires": { + "kind-of": "^6.0.2" + } + } + } + }, "webpack-sources": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", @@ -20194,6 +20403,11 @@ "iconv-lite": "0.4.24" } }, + "whatwg-fetch": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.2.0.tgz", + "integrity": "sha512-SdGPoQMMnzVYThUbSrEvqTlkvC1Ux27NehaJ/GUHBfNrh5Mjg+1/uRyFMwVnxO2MrikMWvWAqUGgQOfVU4hT7w==" + }, "whatwg-mimetype": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz", @@ -20319,6 +20533,11 @@ } } }, + "wildcard": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/wildcard/-/wildcard-2.0.0.tgz", + "integrity": "sha512-JcKqAHLPxcdb9KM49dufGXn2x3ssnfjbcaQdLlfZsL9rH9wgDQjUtDxbo8NE0F6SFvydeu1VhZe7hZuHsB2/pw==" + }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", diff --git a/client/package.json b/client/package.json index 094e62f6..9c247eea 100644 --- a/client/package.json +++ b/client/package.json @@ -29,12 +29,23 @@ "resolutions": { "eslint-scope": "3.7.1" }, + "browserslist": [ + "Chrome > 60", + "Safari >= 10.1", + "iOS >= 10.3", + "Firefox >= 60", + "Edge >= 15", + "not Explorer > 0" + ], "dependencies": { "@blueprintjs/core": "^3.30.0", "@blueprintjs/icons": "^3.19.0", "@blueprintjs/select": "^3.13.5", + "abort-controller": "^3.0.0", + "core-js": "^3.6.5", "d3": "^4.10.0", "d3-scale-chromatic": "^1.5.0", + "fastestsmallesttextencoderdecoder": "^1.0.22", "flatbuffers": "^1.11.0", "fuzzysort": "^1.1.4", "gl-mat4": "^1.2.0", @@ -52,8 +63,12 @@ "react-redux": "^7.2.0", "redux": "^4.0.5", "redux-thunk": "^2.3.0", + "regenerator-runtime": "^0.13.7", "regl": "^1.6.1", - "tinyqueue": "^2.0.3" + "script-ext-html-webpack-plugin": "^2.1.4", + "tinyqueue": "^2.0.3", + "webpack-merge": "^5.0.9", + "whatwg-fetch": "^3.2.0" }, "devDependencies": { "@babel/core": "^7.10.5", @@ -85,6 +100,7 @@ "eslint-config-airbnb": "^18.2.0", "eslint-config-prettier": "^6.11.0", "eslint-loader": "^3.0.4", + "eslint-plugin-compat": "^3.8.0", "eslint-plugin-eslint-comments": "^3.2.0", "eslint-plugin-filenames": "^1.3.2", "eslint-plugin-import": "^2.22.0", @@ -107,6 +123,7 @@ "json-loader": "^0.5.7", "lint-staged": "^10.2.11", "mini-css-extract-plugin": "^0.9.0", + "obsolete-webpack-plugin": "^0.5.6", "optimize-css-assets-webpack-plugin": "^5.0.3", "prettier": "^2.0.5", "puppeteer": "^3.3.0", diff --git a/client/src/index.js b/client/src/index.js index 7f2de221..163f8abc 100644 --- a/client/src/index.js +++ b/client/src/index.js @@ -1,4 +1,3 @@ -// jshint esversion: 6 import React from "react"; import ReactDOM from "react-dom"; import { Provider } from "react-redux"; diff --git a/client/src/util/stateManager/matrix.js b/client/src/util/stateManager/matrix.js index 05d4e61d..e9a1e445 100644 --- a/client/src/util/stateManager/matrix.js +++ b/client/src/util/stateManager/matrix.js @@ -170,11 +170,11 @@ export function encodeMatrixFBS(df) { function promoteTypedArray(o) { /* - Decide what internal data type to use for the data returned from + Decide what internal data type to use for the data returned from the server. TODO - future optimization: not all int32/uint32 data series require - promotion to float64. We COULD simply look at the data to decide. + promotion to float64. We COULD simply look at the data to decide. */ if (isFpTypedArray(o) || Array.isArray(o)) return o; From f632a8db9161f025f839cf685c982abef0aae5eb Mon Sep 17 00:00:00 2001 From: Madison Dunitz Date: Mon, 3 Aug 2020 17:54:06 -0500 Subject: [PATCH 27/55] Dunitz/db setup (#1619) * initial database setup --- Makefile | 5 ++ server/Makefile | 28 ++++++- server/db/__init__.py | 0 server/db/cellxgene_orm.py | 60 +++++++++++++++ server/db/create_db.py | 18 +++++ server/db/db_utils.py | 43 +++++++++++ server/eb/.ebextensions/database.config | 0 server/eb/app.py | 6 +- server/requirements.txt | 2 + server/test/fixtures/__init__.py | 0 server/test/fixtures/database/__init__.py | 87 ++++++++++++++++++++++ server/test/test_database/__init__.py | 0 server/test/test_database/test_database.py | 34 +++++++++ 13 files changed, 278 insertions(+), 5 deletions(-) create mode 100644 server/db/__init__.py create mode 100644 server/db/cellxgene_orm.py create mode 100644 server/db/create_db.py create mode 100644 server/db/db_utils.py create mode 100644 server/eb/.ebextensions/database.config create mode 100644 server/test/fixtures/__init__.py create mode 100644 server/test/fixtures/database/__init__.py create mode 100644 server/test/test_database/__init__.py create mode 100644 server/test/test_database/test_database.py diff --git a/Makefile b/Makefile index 0fe4633f..9c0eb686 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,11 @@ smoke-test: smoke-test-annotations: cd client && $(MAKE) smoke-test-annotations +.PHONY: test-db +test-db: + cd server && $(MAKE) test-db + + # FORMATTING CODE .PHOHY: fmt diff --git a/server/Makefile b/server/Makefile index 74c44f7b..700b9a27 100644 --- a/server/Makefile +++ b/server/Makefile @@ -7,11 +7,35 @@ clean: rm -f common/web/csp-hashes.json .PHONY: unit-test -unit-test: +unit-test: create-test-db PYTHONWARNINGS=ignore:ResourceWarning coverage run \ --source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \ --omit=.coverage,data_common/fbs/NetEncoding,venv \ -m unittest discover \ --start-directory test/ \ --top-level-directory ../ \ - --verbose + --verbose; test_result=$$?; \ + $(MAKE) clean-test-db; \ + exit $$test_result \ + + +.PHONY: test-db +test-db: create-test-db + PYTHONWARNINGS=ignore:ResourceWarning coverage run \ + --source=app,cli,common,compute,converters,data_anndata,data_common,data_cxg \ + --omit=.coverage,data_common/fbs/NetEncoding,venv \ + -m unittest discover \ + --start-directory test/test_database \ + --top-level-directory ../ \ + --verbose; test_result=$$?; \ + $(MAKE) clean-test-db; \ + exit $$test_result + +.PHONY: create-test-db +create-test-db: + -docker run -d -p 5432:5432 --name test_db -e POSTGRES_PASSWORD=test_pw postgres + +.PHONY: clean-test-db +clean-test-db: + -docker stop test_db + -docker rm test_db diff --git a/server/db/__init__.py b/server/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/db/cellxgene_orm.py b/server/db/cellxgene_orm.py new file mode 100644 index 00000000..27823bc2 --- /dev/null +++ b/server/db/cellxgene_orm.py @@ -0,0 +1,60 @@ +from datetime import datetime + +from sqlalchemy import ( + Column, + DateTime, + ForeignKey, + String, +) +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship + +Base = declarative_base() + + +class CellxGeneUser(Base): + """ + A registered CellxGene user. + Links a user to their annotations + """ + + __tablename__ = "cxguser" + + id = Column(String, primary_key=True) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) + + # Relationships + annotations = relationship("Annotation", back_populates="cxguser") + + +class Annotation(Base): + """ + An annotation is a link between a user, a dataset and tiledb dataframe. A user can have multiple annotations for a + dataset, the most recent annotation (based on created_at) will be the default returned when queried + """ + + __tablename__ = "annotation" + + id = Column(String, primary_key=True) + tiledb_uri = Column(String) + user_id = Column(String, ForeignKey("cxguser.id"), nullable=False) + dataset_id = Column(String, ForeignKey("cxgdataset.id"), nullable=False) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + + # Relationships + cxguser = relationship("CellxGeneUser", back_populates="annotations") + dataset = relationship("CellxGeneDataset", back_populates="annotations") + + +class CellxGeneDataset(Base): + """ + Datasets refer to cellxgene datasets stored in tiledb + """ + + __tablename__ = "cxgdataset" + + id = Column(String, primary_key=True) + name = Column(String) + created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + annotations = relationship("Annotation", back_populates="dataset") diff --git a/server/db/create_db.py b/server/db/create_db.py new file mode 100644 index 00000000..c0a7cf98 --- /dev/null +++ b/server/db/create_db.py @@ -0,0 +1,18 @@ +""" +Drops and recreates all tables for local testing according to cellxgene_orm.py +""" +from sqlalchemy import create_engine + +from server.db.cellxgene_orm import Base + + +def create_db(database_uri: str = "postgresql://postgres:test_pw@localhost:5432"): + engine = create_engine(database_uri) + print("Dropping tables") + Base.metadata.drop_all(engine) + print("Recreating tables") + Base.metadata.create_all(engine) + + +if __name__ == "__main__": + create_db() diff --git a/server/db/db_utils.py b/server/db/db_utils.py new file mode 100644 index 00000000..2cf3f59e --- /dev/null +++ b/server/db/db_utils.py @@ -0,0 +1,43 @@ +import typing + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from server.db.cellxgene_orm import Base + + +class DbUtils: + def __init__(self, database_uri: str = "postgresql://postgres:test_pw@localhost:5432"): + self.session = DBSessionMaker(database_uri).session() + self.engine = self.session.get_bind() + + def get(self, table: Base, entity_id: typing.Union[str, typing.Tuple[str]]) -> typing.Union[Base, None]: + """ + Query a table row by its primary key + :param table: SQLAlchemy Table to query + :param entity_id: Primary key of desired row + :return: SQLAlchemy Table object, None if not found + """ + return self.session.query(table).get(entity_id) + + def query(self, table_args: typing.List[Base], filter_args: typing.List[bool] = None) -> typing.List[Base]: + """ + Query the database using the current DB session + :param table_args: List of SQLAlchemy Tables to query/join + :param filter_args: List of SQLAlchemy filter conditions + :return: List of SQLAlchemy query response objects + """ + return ( + self.session.query(*table_args).filter(*filter_args).all() + if filter_args + else self.session.query(*table_args).all() + ) + + +class DBSessionMaker: + def __init__(self, database_uri): + self.engine = create_engine(database_uri, connect_args={"connect_timeout": 5}) + self.session_maker = sessionmaker(bind=self.engine) + + def session(self, **kwargs): + return self.session_maker(**kwargs) diff --git a/server/eb/.ebextensions/database.config b/server/eb/.ebextensions/database.config new file mode 100644 index 00000000..e69de29b diff --git a/server/eb/app.py b/server/eb/app.py index ce3a77b6..64645fc3 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -31,7 +31,7 @@ except Exception: sys.exit(1) -def get_flask_secret_key(region_name, secret_name): +def get_secret_key(region_name, secret_name, secret_key): session = boto3.session.Session() client = session.client(service_name="secretsmanager", region_name=region_name) @@ -40,7 +40,7 @@ def get_flask_secret_key(region_name, secret_name): if "SecretString" in get_secret_value_response: var = get_secret_value_response["SecretString"] secret = json.loads(var) - return secret.get("flask_secret_key") + return secret.get(secret_key) except Exception: logging.critical("Caught exception during get_secret_key", exc_info=True) sys.exit(1) @@ -173,7 +173,7 @@ try: logging.error("Could not determine the AWS Secret Manager region") sys.exit(1) - flask_secret_key = get_flask_secret_key(secret_region_name, secret_name) + flask_secret_key = get_secret_key(secret_region_name, secret_name, 'flask_secret_key') app_config.update_server_config(app__flask_secret_key=flask_secret_key) # features are unsupported in the current hosted server diff --git a/server/requirements.txt b/server/requirements.txt index 3b1909d1..a4948f7d 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -15,9 +15,11 @@ numba>=0.49.1 numpy>=1.16.0 packaging>=20.0 pandas>=0.24.2 +psycopg2==2.7.7 PyYAML>=5.3 scipy>=1.3.0 requests>=2.22.0 +sqlalchemy>=1.3.18 tiledb>=0.5.9,>=0.6.2 s3fs>=0.4.2 gunicorn>=20.0.4 diff --git a/server/test/fixtures/__init__.py b/server/test/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/fixtures/database/__init__.py b/server/test/fixtures/database/__init__.py new file mode 100644 index 00000000..82b17b58 --- /dev/null +++ b/server/test/fixtures/database/__init__.py @@ -0,0 +1,87 @@ +import string +import random + + +from sqlalchemy import func + +from server.db.cellxgene_orm import CellxGeneUser, CellxGeneDataset, Annotation, Base +from server.db.create_db import create_db +from server.db.db_utils import DbUtils + + +class TestDatabase: + def __init__(self): + local_db_uri = "postgresql://postgres:test_pw@localhost:5432" + create_db(local_db_uri) + self.db = DbUtils(local_db_uri) + self._populate_test_data() + self._populate_test_data_many() + + def _populate_test_data(self): + self._create_test_user() + self._create_test_dataset() + self._create_test_annotation() + + def _populate_test_data_many(self): + self._create_test_users() + self._create_test_datasets() + self._create_test_annotations() + + def _create_test_user(self): + user = CellxGeneUser(id="test_user_id") + self.db.session.add(user) + self.db.session.commit() + + def _create_test_dataset(self): + dataset = CellxGeneDataset( + id="test_dataset_id", + name="test_dataset", + ) + self.db.session.add(dataset) + self.db.session.commit() + + def _create_test_annotation(self): + annotation = Annotation( + id="test_annotation_id", + tiledb_uri="tiledb_uri", + user_id="test_user_id", + dataset_id="test_dataset_id" + ) + self.db.session.add(annotation) + self.db.session.commit() + + @staticmethod + def get_random_string(): + letters = string.ascii_lowercase + return ''.join(random.choice(letters) for i in range(12)) + + def _create_test_users(self, user_count: int = 10): + users = [] + for i in range(user_count): + users.append(CellxGeneUser(id=self.get_random_string())) + self.db.session.add_all(users) + self.db.session.commit() + + def _create_test_datasets(self, dataset_count: int = 10): + datasets = [] + for i in range(dataset_count): + datasets.append(CellxGeneDataset(id=self.get_random_string(), name=self.get_random_string())) + self.db.session.add_all(datasets) + self.db.session.commit() + + def order_by_random(self, table: Base): + return self.db.session.query(table).order_by(func.random()).first() + + def _create_test_annotations(self, annotation_count: int = 10): + annotations = [] + for i in range(annotation_count): + dataset = self.order_by_random(CellxGeneDataset) + user = self.order_by_random(CellxGeneUser) + annotations.append(Annotation( + id=self.get_random_string(), + tiledb_uri=self.get_random_string(), + user_id=user.id, + dataset_id=dataset.id + )) + self.db.session.add_all(annotations) + self.db.session.commit() diff --git a/server/test/test_database/__init__.py b/server/test/test_database/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_database/test_database.py b/server/test/test_database/test_database.py new file mode 100644 index 00000000..706d2fdc --- /dev/null +++ b/server/test/test_database/test_database.py @@ -0,0 +1,34 @@ +import unittest +from server.db.cellxgene_orm import CellxGeneUser, CellxGeneDataset, Annotation +from server.db.db_utils import DbUtils +from server.test.fixtures.database import TestDatabase + + +class AppConfigTest(unittest.TestCase): + db = DbUtils("postgresql://postgres:test_pw@localhost:5432") + + @classmethod + def setUpClass(cls) -> None: + TestDatabase() + + @classmethod + def tearDownClass(cls) -> None: + del cls.db + + def test_user_creation(self): + one_user = self.db.get(table=CellxGeneUser, entity_id='test_user_id') + self.assertEqual(one_user.id, 'test_user_id') + user_count = self.db.session.query(CellxGeneUser).count() + self.assertGreater(user_count, 10) + + def test_dataset_creation(self): + one_dataset = self.db.get(table=CellxGeneDataset, entity_id='test_dataset_id') + self.assertEqual(one_dataset.id, 'test_dataset_id') + dataset_count = self.db.session.query(CellxGeneDataset).count() + self.assertGreater(dataset_count, 10) + + def test_annotation_creation(self): + one_annotation = self.db.get(table=Annotation, entity_id='test_annotation_id') + self.assertEqual(one_annotation.id, 'test_annotation_id') + annotation_count = self.db.session.query(Annotation).count() + self.assertGreater(annotation_count, 10) From 6bda27f5542fb7f469425e1cd99f2f37268b095f Mon Sep 17 00:00:00 2001 From: Marcus Kinsella Date: Tue, 4 Aug 2020 11:07:08 -0700 Subject: [PATCH 28/55] Add datasets 45-47 (#1706) --- docs/_site/index.html | 4 ++-- docs/_site/posts/annotations.html | 4 ++-- docs/_site/posts/contact.html | 4 ++-- docs/_site/posts/contribute.html | 4 ++-- docs/_site/posts/demo-data.html | 4 ++-- docs/_site/posts/gallery.html | 4 ++-- docs/_site/posts/hosted.html | 4 ++-- docs/_site/posts/install.html | 4 ++-- docs/_site/posts/launch.html | 4 ++-- docs/_site/posts/methods.html | 4 ++-- docs/_site/posts/prepare.html | 4 ++-- docs/_site/posts/roadmap.html | 4 ++-- docs/_site/posts/troubleshooting.html | 4 ++-- docs/posts/cellxgene_cziscience_com.md | 18 ++++++++++++++++++ 14 files changed, 44 insertions(+), 26 deletions(-) diff --git a/docs/_site/index.html b/docs/_site/index.html index 71d7482e..5e31853b 100644 --- a/docs/_site/index.html +++ b/docs/_site/index.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","headline":"Index","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/annotations.html b/docs/_site/posts/annotations.html index 776b59fb..047cae99 100644 --- a/docs/_site/posts/annotations.html +++ b/docs/_site/posts/annotations.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","headline":"annotations","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/contact.html b/docs/_site/posts/contact.html index 9aadb3a4..62984446 100644 --- a/docs/_site/posts/contact.html +++ b/docs/_site/posts/contact.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","headline":"Contact","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/contribute.html b/docs/_site/posts/contribute.html index 8fa3fe30..63f96724 100644 --- a/docs/_site/posts/contribute.html +++ b/docs/_site/posts/contribute.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Code of conduct","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/demo-data.html b/docs/_site/posts/demo-data.html index ac710163..c334df2d 100644 --- a/docs/_site/posts/demo-data.html +++ b/docs/_site/posts/demo-data.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/gallery.html b/docs/_site/posts/gallery.html index d1ead529..d598b49e 100644 --- a/docs/_site/posts/gallery.html +++ b/docs/_site/posts/gallery.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Gallery","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/hosted.html b/docs/_site/posts/hosted.html index 1bec5484..fdd16e19 100644 --- a/docs/_site/posts/hosted.html +++ b/docs/_site/posts/hosted.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Hosting cellxgene on the web","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/install.html b/docs/_site/posts/install.html index 8b80a6ba..74e0dcaa 100644 --- a/docs/_site/posts/install.html +++ b/docs/_site/posts/install.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Install","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/launch.html b/docs/_site/posts/launch.html index 606320c6..86cc93b8 100644 --- a/docs/_site/posts/launch.html +++ b/docs/_site/posts/launch.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/methods.html b/docs/_site/posts/methods.html index bebb0e3c..d68880d2 100644 --- a/docs/_site/posts/methods.html +++ b/docs/_site/posts/methods.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Methods","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/prepare.html b/docs/_site/posts/prepare.html index edfc1426..84fdee63 100644 --- a/docs/_site/posts/prepare.html +++ b/docs/_site/posts/prepare.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","headline":"prepare","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/roadmap.html b/docs/_site/posts/roadmap.html index 3ce34e68..74bc785b 100644 --- a/docs/_site/posts/roadmap.html +++ b/docs/_site/posts/roadmap.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","headline":"roadmap","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/troubleshooting.html b/docs/_site/posts/troubleshooting.html index a86940ff..7588c003 100644 --- a/docs/_site/posts/troubleshooting.html +++ b/docs/_site/posts/troubleshooting.html @@ -16,10 +16,10 @@ +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","headline":"Troubleshooting","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","@context":"https://schema.org"} - + diff --git a/docs/posts/cellxgene_cziscience_com.md b/docs/posts/cellxgene_cziscience_com.md index 833ae0ce..c9727f6f 100644 --- a/docs/posts/cellxgene_cziscience_com.md +++ b/docs/posts/cellxgene_cziscience_com.md @@ -311,5 +311,23 @@ with a link to embed on your own site, please drop us a note at Nature + + Single Soma Transcriptomics - AT8 + + bioRxiv preprint + + + + Single Soma Transcriptomics - MAP2 + + bioRxiv preprint + + + + Single Soma Transcriptomics - MAP2AT8 + + bioRxiv preprint + + From 550847f7638340597c517a1e1078f9678bfa4525 Mon Sep 17 00:00:00 2001 From: Marcus Kinsella Date: Tue, 4 Aug 2020 16:03:44 -0700 Subject: [PATCH 29/55] Add dataset 29 (#1712) --- docs/_site/index.html | 2 +- docs/_site/posts/annotations.html | 2 +- docs/_site/posts/contact.html | 2 +- docs/_site/posts/contribute.html | 2 +- docs/_site/posts/demo-data.html | 2 +- docs/_site/posts/gallery.html | 2 +- docs/_site/posts/hosted.html | 2 +- docs/_site/posts/install.html | 2 +- docs/_site/posts/launch.html | 2 +- docs/_site/posts/methods.html | 2 +- docs/_site/posts/prepare.html | 2 +- docs/_site/posts/roadmap.html | 2 +- docs/_site/posts/troubleshooting.html | 2 +- docs/posts/cellxgene_cziscience_com.md | 6 ++++++ 14 files changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/_site/index.html b/docs/_site/index.html index 5e31853b..2a5ed4f1 100644 --- a/docs/_site/index.html +++ b/docs/_site/index.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","headline":"Index","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/annotations.html b/docs/_site/posts/annotations.html index 047cae99..2a76ec9f 100644 --- a/docs/_site/posts/annotations.html +++ b/docs/_site/posts/annotations.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","headline":"annotations","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/contact.html b/docs/_site/posts/contact.html index 62984446..33c31b51 100644 --- a/docs/_site/posts/contact.html +++ b/docs/_site/posts/contact.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","headline":"Contact","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/contribute.html b/docs/_site/posts/contribute.html index 63f96724..494d2ca3 100644 --- a/docs/_site/posts/contribute.html +++ b/docs/_site/posts/contribute.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Code of conduct","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/demo-data.html b/docs/_site/posts/demo-data.html index c334df2d..479e6249 100644 --- a/docs/_site/posts/demo-data.html +++ b/docs/_site/posts/demo-data.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/gallery.html b/docs/_site/posts/gallery.html index d598b49e..2321d81e 100644 --- a/docs/_site/posts/gallery.html +++ b/docs/_site/posts/gallery.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Gallery","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/hosted.html b/docs/_site/posts/hosted.html index fdd16e19..0e2d2b6a 100644 --- a/docs/_site/posts/hosted.html +++ b/docs/_site/posts/hosted.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Hosting cellxgene on the web","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/install.html b/docs/_site/posts/install.html index 74e0dcaa..2eb470d8 100644 --- a/docs/_site/posts/install.html +++ b/docs/_site/posts/install.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Install","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/launch.html b/docs/_site/posts/launch.html index 86cc93b8..4aed95b3 100644 --- a/docs/_site/posts/launch.html +++ b/docs/_site/posts/launch.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/methods.html b/docs/_site/posts/methods.html index d68880d2..8c97a84c 100644 --- a/docs/_site/posts/methods.html +++ b/docs/_site/posts/methods.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Methods","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/prepare.html b/docs/_site/posts/prepare.html index 84fdee63..56dce0c2 100644 --- a/docs/_site/posts/prepare.html +++ b/docs/_site/posts/prepare.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","headline":"prepare","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/roadmap.html b/docs/_site/posts/roadmap.html index 74bc785b..34dc26b9 100644 --- a/docs/_site/posts/roadmap.html +++ b/docs/_site/posts/roadmap.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","headline":"roadmap","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/troubleshooting.html b/docs/_site/posts/troubleshooting.html index 7588c003..a936237a 100644 --- a/docs/_site/posts/troubleshooting.html +++ b/docs/_site/posts/troubleshooting.html @@ -19,7 +19,7 @@ {"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","headline":"Troubleshooting","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","@context":"https://schema.org"} - + diff --git a/docs/posts/cellxgene_cziscience_com.md b/docs/posts/cellxgene_cziscience_com.md index c9727f6f..43144045 100644 --- a/docs/posts/cellxgene_cziscience_com.md +++ b/docs/posts/cellxgene_cziscience_com.md @@ -329,5 +329,11 @@ with a link to embed on your own site, please drop us a note at bioRxiv preprint + + Single-cell longitudinal analysis of SARS-CoV-2 infection in human bronchial epithelial cells + + bioRxiv preprint + + From cdae4f9f106d509c05f45e497d7f41a3e55c03da Mon Sep 17 00:00:00 2001 From: maniarathi Date: Wed, 5 Aug 2020 08:31:02 -0700 Subject: [PATCH 30/55] Reorganize the server testing directory (#1705) --- client/Makefile | 2 +- server/test/__init__.py | 5 ++-- .../{test_datasets => fixtures}/fixtures.py | 0 .../test/{test_datasets => fixtures}/nan.h5ad | Bin .../pbmc3k-CSC-gz.h5ad | Bin .../pbmc3k-CSR-gz.h5ad | Bin .../pbmc3k-annotations.csv | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../pbmc3k.cxg/X/__array_schema.tdb | Bin .../pbmc3k.cxg/X/__lock.tdb | 0 .../pbmc3k.cxg/__tiledb_group.tdb | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../cxg_group_metadata/__array_schema.tdb | Bin .../pbmc3k.cxg/cxg_group_metadata/__lock.tdb | 0 ...182255768_fa7617ae99f843929911e4bc7b03e3db | Bin .../pbmc3k.cxg/emb/__tiledb_group.tdb | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../emb/draw_graph_fr/__array_schema.tdb | Bin .../pbmc3k.cxg/emb/draw_graph_fr/__lock.tdb | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../pbmc3k.cxg/emb/pca/__array_schema.tdb | Bin .../pbmc3k.cxg/emb/pca/__lock.tdb | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../pbmc3k.cxg/emb/tsne/__array_schema.tdb | Bin .../pbmc3k.cxg/emb/tsne/__lock.tdb | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../pbmc3k.cxg/emb/umap/__array_schema.tdb | Bin .../pbmc3k.cxg/emb/umap/__lock.tdb | 0 .../__fragment_metadata.tdb | Bin .../louvain.tdb | Bin .../louvain_var.tdb | Bin .../n_counts.tdb | Bin .../n_genes.tdb | Bin .../name_0.tdb | Bin .../name_0_var.tdb | Bin .../percent_mito.tdb | Bin .../pbmc3k.cxg/obs/__array_schema.tdb | Bin .../pbmc3k.cxg/obs/__lock.tdb | 0 ...182255787_e19a340576a340db85c37b61b83dbe57 | Bin .../__fragment_metadata.tdb | Bin .../n_cells.tdb | Bin .../name_0.tdb | Bin .../name_0_var.tdb | Bin .../pbmc3k.cxg/var/__array_schema.tdb | Bin .../pbmc3k.cxg/var/__lock.tdb | 0 ...182255771_5a3a50f0b360403eab0e427c429f574e | Bin .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../pbmc3k_v0.cxg/X/__array_schema.tdb | Bin .../pbmc3k_v0.cxg/X/__lock.tdb | 0 .../pbmc3k_v0.cxg/__tiledb_group.tdb | 0 .../pbmc3k_v0.cxg/emb/__tiledb_group.tdb | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../emb/draw_graph_fr/__array_schema.tdb | Bin .../emb/draw_graph_fr/__lock.tdb | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../pbmc3k_v0.cxg/emb/pca/__array_schema.tdb | Bin .../pbmc3k_v0.cxg/emb/pca/__lock.tdb | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../pbmc3k_v0.cxg/emb/tsne/__array_schema.tdb | Bin .../pbmc3k_v0.cxg/emb/tsne/__lock.tdb | 0 .../__attr.tdb | Bin .../__fragment_metadata.tdb | Bin .../pbmc3k_v0.cxg/emb/umap/__array_schema.tdb | Bin .../pbmc3k_v0.cxg/emb/umap/__lock.tdb | 0 .../__fragment_metadata.tdb | Bin .../louvain.tdb | Bin .../louvain_var.tdb | Bin .../n_counts.tdb | Bin .../n_genes.tdb | Bin .../name_0.tdb | Bin .../name_0_var.tdb | Bin .../percent_mito.tdb | Bin .../pbmc3k_v0.cxg/obs/__array_schema.tdb | Bin .../pbmc3k_v0.cxg/obs/__lock.tdb | 0 ...858534024_e856aef5b0e244b1a4b7dde70cd864d0 | Bin .../__fragment_metadata.tdb | Bin .../n_cells.tdb | Bin .../name_0.tdb | Bin .../name_0_var.tdb | Bin .../pbmc3k_v0.cxg/var/__array_schema.tdb | Bin .../pbmc3k_v0.cxg/var/__lock.tdb | 0 ...858533966_1362646d804b4982b35052a367633436 | Bin server/test/{ => fixtures}/schema.json | 0 server/{ => test}/locust/README.md | 0 server/test/locust/__init__.py | 0 server/{ => test}/locust/config.py | 0 server/{ => test}/locust/locustfile.py | 2 +- .../{ => test}/locust/requirements-locust.txt | 0 server/test/performance/__init__.py | 0 .../{ => performance}/create_test_matrix.py | 0 server/test/{ => performance}/run_diffexp.py | 0 server/test/unit/__init__.py | 0 server/test/unit/auth/__init__.py | 0 server/test/{ => unit/auth}/test_auth.py | 19 +++++++++------ server/test/unit/cli/__init__.py | 0 .../cli/test_prepare.py} | 0 .../cli/test_upgrade.py} | 0 server/test/unit/common/__init__.py | 0 server/test/{ => unit/common}/test_api.py | 12 ++++----- .../test/{ => unit/common}/test_app_config.py | 6 ++--- server/test/{ => unit/common}/test_colors.py | 2 +- server/test/{ => unit/common}/test_corpora.py | 0 .../test/{ => unit/common}/test_nan_rest.py | 4 +-- .../common/test_rest.py} | 0 .../common/test_utils.py} | 0 .../common}/test_writable_annotation.py | 2 +- server/test/unit/compute/__init__.py | 0 .../compute/test_diffexp_cxg.py} | 8 +++--- server/test/unit/converters/__init__.py | 0 .../{ => unit/converters}/test_cxgtool.py | 2 +- server/test/unit/data_anndata/__init__.py | 0 .../data_anndata}/test_anndata_adaptor.py | 23 +++++++++--------- .../test_anndata_adaptor_data_load.py | 0 .../data_anndata}/test_nan_anndata_adaptor.py | 14 +++++------ server/test/unit/data_common/__init__.py | 0 server/test/unit/data_common/fbs/__init__.py | 0 .../data_common/fbs/test_matrix.py} | 2 +- .../data_common/test_matrix_loader.py} | 18 +++++++------- server/test/unit/data_cxg/__init__.py | 0 .../{ => unit/data_cxg}/test_cxg_adaptor.py | 6 ++--- server/test/{ => unit}/decode_fbs.py | 10 +++----- server/test/unit/eb/__init__.py | 0 server/test/{ => unit/eb}/test_eb.py | 4 +-- 133 files changed, 72 insertions(+), 69 deletions(-) rename server/test/{test_datasets => fixtures}/fixtures.py (100%) rename server/test/{test_datasets => fixtures}/nan.h5ad (100%) rename server/test/{test_datasets => fixtures}/pbmc3k-CSC-gz.h5ad (100%) rename server/test/{test_datasets => fixtures}/pbmc3k-CSR-gz.h5ad (100%) rename server/test/{test_datasets => fixtures}/pbmc3k-annotations.csv (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/X/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/X/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/__tiledb_group.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/cxg_group_metadata/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/cxg_group_metadata/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/cxg_group_metadata/__meta/__1587182255768_1587182255768_fa7617ae99f843929911e4bc7b03e3db (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/__tiledb_group.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/draw_graph_fr/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/draw_graph_fr/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/pca/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/pca/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/tsne/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/tsne/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/umap/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/emb/umap/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain_var.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_counts.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_genes.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0_var.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/percent_mito.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/obs/__meta/__1587182255787_1587182255787_e19a340576a340db85c37b61b83dbe57 (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/n_cells.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0_var.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/var/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/var/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k.cxg/var/__meta/__1587182255771_1587182255771_5a3a50f0b360403eab0e427c429f574e (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/X/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/X/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/__tiledb_group.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/__tiledb_group.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/draw_graph_fr/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/draw_graph_fr/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/pca/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/pca/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/tsne/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/tsne/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__attr.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/umap/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/emb/umap/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain_var.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_counts.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_genes.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0_var.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/percent_mito.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/obs/__meta/__1576858534024_1576858534024_e856aef5b0e244b1a4b7dde70cd864d0 (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/__fragment_metadata.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/n_cells.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0_var.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/var/__array_schema.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/var/__lock.tdb (100%) rename server/test/{test_datasets => fixtures}/pbmc3k_v0.cxg/var/__meta/__1576858533966_1576858533966_1362646d804b4982b35052a367633436 (100%) rename server/test/{ => fixtures}/schema.json (100%) rename server/{ => test}/locust/README.md (100%) create mode 100644 server/test/locust/__init__.py rename server/{ => test}/locust/config.py (100%) rename server/{ => test}/locust/locustfile.py (99%) rename server/{ => test}/locust/requirements-locust.txt (100%) create mode 100644 server/test/performance/__init__.py rename server/test/{ => performance}/create_test_matrix.py (100%) rename server/test/{ => performance}/run_diffexp.py (100%) create mode 100644 server/test/unit/__init__.py create mode 100644 server/test/unit/auth/__init__.py rename server/test/{ => unit/auth}/test_auth.py (92%) create mode 100644 server/test/unit/cli/__init__.py rename server/test/{test_cli_prepare.py => unit/cli/test_prepare.py} (100%) rename server/test/{test_cli_upgrade.py => unit/cli/test_upgrade.py} (100%) create mode 100644 server/test/unit/common/__init__.py rename server/test/{ => unit/common}/test_api.py (98%) rename server/test/{ => unit/common}/test_app_config.py (94%) rename server/test/{ => unit/common}/test_colors.py (96%) rename server/test/{ => unit/common}/test_corpora.py (100%) rename server/test/{ => unit/common}/test_nan_rest.py (94%) rename server/test/{test_filter.py => unit/common/test_rest.py} (100%) rename server/test/{test_plugins.py => unit/common/test_utils.py} (100%) rename server/test/{ => unit/common}/test_writable_annotation.py (99%) create mode 100644 server/test/unit/compute/__init__.py rename server/test/{test_diffexp.py => unit/compute/test_diffexp_cxg.py} (96%) create mode 100644 server/test/unit/converters/__init__.py rename server/test/{ => unit/converters}/test_cxgtool.py (96%) create mode 100644 server/test/unit/data_anndata/__init__.py rename server/test/{ => unit/data_anndata}/test_anndata_adaptor.py (94%) rename server/test/{ => unit/data_anndata}/test_anndata_adaptor_data_load.py (100%) rename server/test/{ => unit/data_anndata}/test_nan_anndata_adaptor.py (93%) create mode 100644 server/test/unit/data_common/__init__.py create mode 100644 server/test/unit/data_common/fbs/__init__.py rename server/test/{test_fbs.py => unit/data_common/fbs/test_matrix.py} (98%) rename server/test/{test_matrixcache.py => unit/data_common/test_matrix_loader.py} (96%) create mode 100644 server/test/unit/data_cxg/__init__.py rename server/test/{ => unit/data_cxg}/test_cxg_adaptor.py (74%) rename server/test/{ => unit}/decode_fbs.py (99%) create mode 100644 server/test/unit/eb/__init__.py rename server/test/{ => unit/eb}/test_eb.py (90%) diff --git a/client/Makefile b/client/Makefile index 5033053f..3b36337a 100644 --- a/client/Makefile +++ b/client/Makefile @@ -1,6 +1,6 @@ include ../common.mk -ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/test_datasets/pbmc3k-annotations.csv) +ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv) ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS)) # Packaging diff --git a/server/test/__init__.py b/server/test/__init__.py index 44103190..e4d8d67a 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -20,16 +20,17 @@ from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() +FIXTURES_ROOT = PROJECT_ROOT + "/server/test/fixtures" def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): tmp_dir = tempfile.mkdtemp() annotations_file = path.join(tmp_dir, "test_annotations.csv") if annotations_fixture: - shutil.copyfile(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-annotations.csv", annotations_file) + shutil.copyfile(f"{PROJECT_ROOT}/server/test/fixtures/pbmc3k-annotations.csv", annotations_file) fname = { MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", - MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg", + MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg", }[ext] data_locator = DataLocator(fname) config = AppConfig() diff --git a/server/test/test_datasets/fixtures.py b/server/test/fixtures/fixtures.py similarity index 100% rename from server/test/test_datasets/fixtures.py rename to server/test/fixtures/fixtures.py diff --git a/server/test/test_datasets/nan.h5ad b/server/test/fixtures/nan.h5ad similarity index 100% rename from server/test/test_datasets/nan.h5ad rename to server/test/fixtures/nan.h5ad diff --git a/server/test/test_datasets/pbmc3k-CSC-gz.h5ad b/server/test/fixtures/pbmc3k-CSC-gz.h5ad similarity index 100% rename from server/test/test_datasets/pbmc3k-CSC-gz.h5ad rename to server/test/fixtures/pbmc3k-CSC-gz.h5ad diff --git a/server/test/test_datasets/pbmc3k-CSR-gz.h5ad b/server/test/fixtures/pbmc3k-CSR-gz.h5ad similarity index 100% rename from server/test/test_datasets/pbmc3k-CSR-gz.h5ad rename to server/test/fixtures/pbmc3k-CSR-gz.h5ad diff --git a/server/test/test_datasets/pbmc3k-annotations.csv b/server/test/fixtures/pbmc3k-annotations.csv similarity index 100% rename from server/test/test_datasets/pbmc3k-annotations.csv rename to server/test/fixtures/pbmc3k-annotations.csv diff --git a/server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__attr.tdb b/server/test/fixtures/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__attr.tdb rename to server/test/fixtures/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/X/__array_schema.tdb b/server/test/fixtures/pbmc3k.cxg/X/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/X/__array_schema.tdb rename to server/test/fixtures/pbmc3k.cxg/X/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/X/__lock.tdb b/server/test/fixtures/pbmc3k.cxg/X/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/X/__lock.tdb rename to server/test/fixtures/pbmc3k.cxg/X/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/__tiledb_group.tdb b/server/test/fixtures/pbmc3k.cxg/__tiledb_group.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/__tiledb_group.tdb rename to server/test/fixtures/pbmc3k.cxg/__tiledb_group.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__attr.tdb b/server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__attr.tdb rename to server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__array_schema.tdb b/server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__array_schema.tdb rename to server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__lock.tdb b/server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__lock.tdb rename to server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__meta/__1587182255768_1587182255768_fa7617ae99f843929911e4bc7b03e3db b/server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__meta/__1587182255768_1587182255768_fa7617ae99f843929911e4bc7b03e3db similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__meta/__1587182255768_1587182255768_fa7617ae99f843929911e4bc7b03e3db rename to server/test/fixtures/pbmc3k.cxg/cxg_group_metadata/__meta/__1587182255768_1587182255768_fa7617ae99f843929911e4bc7b03e3db diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/__tiledb_group.tdb b/server/test/fixtures/pbmc3k.cxg/emb/__tiledb_group.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/__tiledb_group.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/__tiledb_group.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__attr.tdb b/server/test/fixtures/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__attr.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__array_schema.tdb b/server/test/fixtures/pbmc3k.cxg/emb/draw_graph_fr/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__array_schema.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/draw_graph_fr/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__lock.tdb b/server/test/fixtures/pbmc3k.cxg/emb/draw_graph_fr/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__lock.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/draw_graph_fr/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__attr.tdb b/server/test/fixtures/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__attr.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/pca/__array_schema.tdb b/server/test/fixtures/pbmc3k.cxg/emb/pca/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/pca/__array_schema.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/pca/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/pca/__lock.tdb b/server/test/fixtures/pbmc3k.cxg/emb/pca/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/pca/__lock.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/pca/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__attr.tdb b/server/test/fixtures/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__attr.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__array_schema.tdb b/server/test/fixtures/pbmc3k.cxg/emb/tsne/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/tsne/__array_schema.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/tsne/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__lock.tdb b/server/test/fixtures/pbmc3k.cxg/emb/tsne/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/tsne/__lock.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/tsne/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__attr.tdb b/server/test/fixtures/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__attr.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/umap/__array_schema.tdb b/server/test/fixtures/pbmc3k.cxg/emb/umap/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/umap/__array_schema.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/umap/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/umap/__lock.tdb b/server/test/fixtures/pbmc3k.cxg/emb/umap/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/umap/__lock.tdb rename to server/test/fixtures/pbmc3k.cxg/emb/umap/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain_var.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain_var.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain_var.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain_var.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_counts.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_counts.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_counts.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_counts.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_genes.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_genes.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_genes.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_genes.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0_var.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0_var.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0_var.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0_var.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/percent_mito.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/percent_mito.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/percent_mito.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/percent_mito.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__array_schema.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__array_schema.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__lock.tdb b/server/test/fixtures/pbmc3k.cxg/obs/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__lock.tdb rename to server/test/fixtures/pbmc3k.cxg/obs/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1587182255787_1587182255787_e19a340576a340db85c37b61b83dbe57 b/server/test/fixtures/pbmc3k.cxg/obs/__meta/__1587182255787_1587182255787_e19a340576a340db85c37b61b83dbe57 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1587182255787_1587182255787_e19a340576a340db85c37b61b83dbe57 rename to server/test/fixtures/pbmc3k.cxg/obs/__meta/__1587182255787_1587182255787_e19a340576a340db85c37b61b83dbe57 diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/n_cells.tdb b/server/test/fixtures/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/n_cells.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/n_cells.tdb rename to server/test/fixtures/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/n_cells.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0.tdb b/server/test/fixtures/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0.tdb rename to server/test/fixtures/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0_var.tdb b/server/test/fixtures/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0_var.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0_var.tdb rename to server/test/fixtures/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0_var.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__array_schema.tdb b/server/test/fixtures/pbmc3k.cxg/var/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__array_schema.tdb rename to server/test/fixtures/pbmc3k.cxg/var/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__lock.tdb b/server/test/fixtures/pbmc3k.cxg/var/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__lock.tdb rename to server/test/fixtures/pbmc3k.cxg/var/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__meta/__1587182255771_1587182255771_5a3a50f0b360403eab0e427c429f574e b/server/test/fixtures/pbmc3k.cxg/var/__meta/__1587182255771_1587182255771_5a3a50f0b360403eab0e427c429f574e similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__meta/__1587182255771_1587182255771_5a3a50f0b360403eab0e427c429f574e rename to server/test/fixtures/pbmc3k.cxg/var/__meta/__1587182255771_1587182255771_5a3a50f0b360403eab0e427c429f574e diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__attr.tdb b/server/test/fixtures/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__attr.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/X/__array_schema.tdb b/server/test/fixtures/pbmc3k_v0.cxg/X/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/X/__array_schema.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/X/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/X/__lock.tdb b/server/test/fixtures/pbmc3k_v0.cxg/X/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/X/__lock.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/X/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/__tiledb_group.tdb b/server/test/fixtures/pbmc3k_v0.cxg/__tiledb_group.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/__tiledb_group.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/__tiledb_group.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/__tiledb_group.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/__tiledb_group.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/__tiledb_group.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/__tiledb_group.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__attr.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__attr.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/draw_graph_fr/__array_schema.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/draw_graph_fr/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/draw_graph_fr/__array_schema.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/draw_graph_fr/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/draw_graph_fr/__lock.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/draw_graph_fr/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/draw_graph_fr/__lock.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/draw_graph_fr/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__attr.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__attr.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/pca/__array_schema.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/pca/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/pca/__array_schema.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/pca/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/pca/__lock.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/pca/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/pca/__lock.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/pca/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__attr.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__attr.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/tsne/__array_schema.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/tsne/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/tsne/__array_schema.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/tsne/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/tsne/__lock.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/tsne/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/tsne/__lock.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/tsne/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__attr.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__attr.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__attr.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/umap/__array_schema.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/umap/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/umap/__array_schema.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/umap/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/emb/umap/__lock.tdb b/server/test/fixtures/pbmc3k_v0.cxg/emb/umap/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/emb/umap/__lock.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/emb/umap/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain_var.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain_var.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain_var.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain_var.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_counts.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_counts.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_counts.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_counts.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_genes.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_genes.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_genes.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_genes.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0_var.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0_var.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0_var.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0_var.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/percent_mito.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/percent_mito.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/percent_mito.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/percent_mito.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__array_schema.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__array_schema.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__lock.tdb b/server/test/fixtures/pbmc3k_v0.cxg/obs/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__lock.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/obs/__meta/__1576858534024_1576858534024_e856aef5b0e244b1a4b7dde70cd864d0 b/server/test/fixtures/pbmc3k_v0.cxg/obs/__meta/__1576858534024_1576858534024_e856aef5b0e244b1a4b7dde70cd864d0 similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/obs/__meta/__1576858534024_1576858534024_e856aef5b0e244b1a4b7dde70cd864d0 rename to server/test/fixtures/pbmc3k_v0.cxg/obs/__meta/__1576858534024_1576858534024_e856aef5b0e244b1a4b7dde70cd864d0 diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/__fragment_metadata.tdb b/server/test/fixtures/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/__fragment_metadata.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/__fragment_metadata.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/n_cells.tdb b/server/test/fixtures/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/n_cells.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/n_cells.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/n_cells.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0.tdb b/server/test/fixtures/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0_var.tdb b/server/test/fixtures/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0_var.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0_var.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0_var.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/var/__array_schema.tdb b/server/test/fixtures/pbmc3k_v0.cxg/var/__array_schema.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/var/__array_schema.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/var/__array_schema.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/var/__lock.tdb b/server/test/fixtures/pbmc3k_v0.cxg/var/__lock.tdb similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/var/__lock.tdb rename to server/test/fixtures/pbmc3k_v0.cxg/var/__lock.tdb diff --git a/server/test/test_datasets/pbmc3k_v0.cxg/var/__meta/__1576858533966_1576858533966_1362646d804b4982b35052a367633436 b/server/test/fixtures/pbmc3k_v0.cxg/var/__meta/__1576858533966_1576858533966_1362646d804b4982b35052a367633436 similarity index 100% rename from server/test/test_datasets/pbmc3k_v0.cxg/var/__meta/__1576858533966_1576858533966_1362646d804b4982b35052a367633436 rename to server/test/fixtures/pbmc3k_v0.cxg/var/__meta/__1576858533966_1576858533966_1362646d804b4982b35052a367633436 diff --git a/server/test/schema.json b/server/test/fixtures/schema.json similarity index 100% rename from server/test/schema.json rename to server/test/fixtures/schema.json diff --git a/server/locust/README.md b/server/test/locust/README.md similarity index 100% rename from server/locust/README.md rename to server/test/locust/README.md diff --git a/server/test/locust/__init__.py b/server/test/locust/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/locust/config.py b/server/test/locust/config.py similarity index 100% rename from server/locust/config.py rename to server/test/locust/config.py diff --git a/server/locust/locustfile.py b/server/test/locust/locustfile.py similarity index 99% rename from server/locust/locustfile.py rename to server/test/locust/locustfile.py index 092f35f5..2e6a796f 100644 --- a/server/locust/locustfile.py +++ b/server/test/locust/locustfile.py @@ -4,7 +4,7 @@ import random import json from gevent.pool import Group -import server.test.decode_fbs as decode_fbs +import server.test.unit.decode_fbs as decode_fbs from config import DataSets diff --git a/server/locust/requirements-locust.txt b/server/test/locust/requirements-locust.txt similarity index 100% rename from server/locust/requirements-locust.txt rename to server/test/locust/requirements-locust.txt diff --git a/server/test/performance/__init__.py b/server/test/performance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/create_test_matrix.py b/server/test/performance/create_test_matrix.py similarity index 100% rename from server/test/create_test_matrix.py rename to server/test/performance/create_test_matrix.py diff --git a/server/test/run_diffexp.py b/server/test/performance/run_diffexp.py similarity index 100% rename from server/test/run_diffexp.py rename to server/test/performance/run_diffexp.py diff --git a/server/test/unit/__init__.py b/server/test/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/unit/auth/__init__.py b/server/test/unit/auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_auth.py b/server/test/unit/auth/test_auth.py similarity index 92% rename from server/test/test_auth.py rename to server/test/unit/auth/test_auth.py index bd4c776e..5d6eb748 100644 --- a/server/test/test_auth.py +++ b/server/test/unit/auth/test_auth.py @@ -1,14 +1,19 @@ import unittest -from server.common.app_config import AppConfig -from server.test import PROJECT_ROOT, test_server + import requests +from server.common.app_config import AppConfig +from server.test import FIXTURES_ROOT, test_server + class AuthTest(unittest.TestCase): + def setUp(self): + self.dataset_dataroot = FIXTURES_ROOT + def test_auth_none(self): c = AppConfig() c.update_server_config( - authentication__type=None, multi_dataset__dataroot=f"{PROJECT_ROOT}/server/test/test_datasets" + authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot ) c.update_default_dataset_config(user_annotations__enable=False) @@ -23,7 +28,7 @@ class AuthTest(unittest.TestCase): def test_auth_session(self): c = AppConfig() c.update_server_config( - authentication__type="session", multi_dataset__dataroot=f"{PROJECT_ROOT}/server/test/test_datasets" + authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot ) c.update_default_dataset_config(user_annotations__enable=True) c.complete_config() @@ -41,8 +46,8 @@ class AuthTest(unittest.TestCase): c.update_server_config(authentication__type="test") c.update_server_config( multi_dataset__dataroot=dict( - a1=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="auth"), - a2=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="no-auth"), + a1=dict(dataroot=self.dataset_dataroot, base_url="auth"), + a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"), ) ) @@ -100,7 +105,7 @@ class AuthTest(unittest.TestCase): c = AppConfig() c.update_server_config( authentication__type="test", - single_dataset__datapath=f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg") + single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg") c.complete_config() diff --git a/server/test/unit/cli/__init__.py b/server/test/unit/cli/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_cli_prepare.py b/server/test/unit/cli/test_prepare.py similarity index 100% rename from server/test/test_cli_prepare.py rename to server/test/unit/cli/test_prepare.py diff --git a/server/test/test_cli_upgrade.py b/server/test/unit/cli/test_upgrade.py similarity index 100% rename from server/test/test_cli_upgrade.py rename to server/test/unit/cli/test_upgrade.py diff --git a/server/test/unit/common/__init__.py b/server/test/unit/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_api.py b/server/test/unit/common/test_api.py similarity index 98% rename from server/test/test_api.py rename to server/test/unit/common/test_api.py index 7b50c426..3e2c79b9 100644 --- a/server/test/test_api.py +++ b/server/test/unit/common/test_api.py @@ -6,15 +6,15 @@ from http import HTTPStatus import pandas as pd import requests -import server.test.decode_fbs as decode_fbs +import server.test.unit.decode_fbs as decode_fbs from server.data_common.matrix_loader import MatrixDataType -from server.test import data_with_tmp_annotations, make_fbs, PROJECT_ROOT -from server.test.test_datasets.fixtures import pbmc3k_colors -from server.test import start_test_server, stop_test_server - +from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server, + stop_test_server) +from server.test.fixtures.fixtures import pbmc3k_colors BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} + # TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental @@ -404,7 +404,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints): @classmethod def setUpClass(cls): cls._setupClass(cls, [ - f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg", + f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations", ]) diff --git a/server/test/test_app_config.py b/server/test/unit/common/test_app_config.py similarity index 94% rename from server/test/test_app_config.py rename to server/test/unit/common/test_app_config.py index c55f927a..d1cc9496 100644 --- a/server/test/test_app_config.py +++ b/server/test/unit/common/test_app_config.py @@ -1,7 +1,7 @@ import unittest from server.common.app_config import AppConfig from server.common.errors import ConfigurationError -from server.test import PROJECT_ROOT, test_server +from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT import requests # NOTE, there are more tests that should be written for AppConfig. @@ -52,8 +52,8 @@ class AppConfigTest(unittest.TestCase): c.update_server_config( multi_dataset__dataroot=dict( s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), - s2=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="set2"), - s3=dict(dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", base_url="set3"), + s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), + s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), ) ) diff --git a/server/test/test_colors.py b/server/test/unit/common/test_colors.py similarity index 96% rename from server/test/test_colors.py rename to server/test/unit/common/test_colors.py index 2970587c..efef25ec 100644 --- a/server/test/test_colors.py +++ b/server/test/unit/common/test_colors.py @@ -4,7 +4,7 @@ import anndata from server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors from server.common.errors import ColorFormatException from server.test import PROJECT_ROOT -from server.test.test_datasets.fixtures import pbmc3k_colors +from server.test.fixtures.fixtures import pbmc3k_colors class ColorsTest(unittest.TestCase): diff --git a/server/test/test_corpora.py b/server/test/unit/common/test_corpora.py similarity index 100% rename from server/test/test_corpora.py rename to server/test/unit/common/test_corpora.py diff --git a/server/test/test_nan_rest.py b/server/test/unit/common/test_nan_rest.py similarity index 94% rename from server/test/test_nan_rest.py rename to server/test/unit/common/test_nan_rest.py index 00c072e5..1dd28a54 100644 --- a/server/test/test_nan_rest.py +++ b/server/test/unit/common/test_nan_rest.py @@ -3,7 +3,7 @@ import unittest import math from server.test import start_test_server, stop_test_server -import server.test.decode_fbs as decode_fbs +import server.test.unit.decode_fbs as decode_fbs import requests @@ -16,7 +16,7 @@ class WithNaNs(unittest.TestCase): @classmethod def setUpClass(cls): - cls.ps, cls.server = start_test_server(["test/test_datasets/nan.h5ad"]) + cls.ps, cls.server = start_test_server(["test/fixtures/nan.h5ad"]) @classmethod def tearDownClass(cls): diff --git a/server/test/test_filter.py b/server/test/unit/common/test_rest.py similarity index 100% rename from server/test/test_filter.py rename to server/test/unit/common/test_rest.py diff --git a/server/test/test_plugins.py b/server/test/unit/common/test_utils.py similarity index 100% rename from server/test/test_plugins.py rename to server/test/unit/common/test_utils.py diff --git a/server/test/test_writable_annotation.py b/server/test/unit/common/test_writable_annotation.py similarity index 99% rename from server/test/test_writable_annotation.py rename to server/test/unit/common/test_writable_annotation.py index 6ca0419d..3178f64c 100644 --- a/server/test/test_writable_annotation.py +++ b/server/test/unit/common/test_writable_annotation.py @@ -1,7 +1,7 @@ import json from os import path, listdir import unittest -import server.test.decode_fbs as decode_fbs +import server.test.unit.decode_fbs as decode_fbs import shutil import numpy as np diff --git a/server/test/unit/compute/__init__.py b/server/test/unit/compute/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_diffexp.py b/server/test/unit/compute/test_diffexp_cxg.py similarity index 96% rename from server/test/test_diffexp.py rename to server/test/unit/compute/test_diffexp_cxg.py index ee49681f..7a6aabdd 100644 --- a/server/test/test_diffexp.py +++ b/server/test/unit/compute/test_diffexp_cxg.py @@ -1,10 +1,10 @@ import unittest from server.data_common.matrix_loader import MatrixDataLoader -from server.test import PROJECT_ROOT, app_config +from server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT import server.compute.diffexp_cxg as diffexp_cxg import server.compute.diffexp_generic as diffexp_generic from server.converters.cxgtool import write_cxg, create_cxg_group_metadata -from server.test.create_test_matrix import create_test_h5ad +from server.test.performance.create_test_matrix import create_test_h5ad from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs import numpy as np import tempfile @@ -68,7 +68,7 @@ class DiffExpTest(unittest.TestCase): def test_cxg_default(self): """Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)""" - adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg") + adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg") maskA = self.get_mask(adaptor, 1, 10) maskB = self.get_mask(adaptor, 2, 10) @@ -82,7 +82,7 @@ class DiffExpTest(unittest.TestCase): def test_cxg_generic(self): """Test a cxg adaptor with the generic adaptor""" - adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg") + adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg") maskA = self.get_mask(adaptor, 1, 10) maskB = self.get_mask(adaptor, 2, 10) # run it directly diff --git a/server/test/unit/converters/__init__.py b/server/test/unit/converters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_cxgtool.py b/server/test/unit/converters/test_cxgtool.py similarity index 96% rename from server/test/test_cxgtool.py rename to server/test/unit/converters/test_cxgtool.py index 91ab8605..46e26b48 100644 --- a/server/test/test_cxgtool.py +++ b/server/test/unit/converters/test_cxgtool.py @@ -7,7 +7,7 @@ from server.common.data_locator import DataLocator from server.converters.cxgtool import write_cxg, create_cxg_group_metadata from server.data_cxg.cxg_adaptor import CxgAdaptor from server.test import PROJECT_ROOT, app_config, random_string -from server.test.test_datasets.fixtures import pbmc3k_colors +from server.test.fixtures.fixtures import pbmc3k_colors class TestCxgAdaptor(unittest.TestCase): diff --git a/server/test/unit/data_anndata/__init__.py b/server/test/unit/data_anndata/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_anndata_adaptor.py b/server/test/unit/data_anndata/test_anndata_adaptor.py similarity index 94% rename from server/test/test_anndata_adaptor.py rename to server/test/unit/data_anndata/test_anndata_adaptor.py index 440e9b82..d4a3a778 100644 --- a/server/test/test_anndata_adaptor.py +++ b/server/test/unit/data_anndata/test_anndata_adaptor.py @@ -1,20 +1,19 @@ import json -from os import path -import pytest +import sys import time import unittest -import sys -import server.test.decode_fbs as decode_fbs -from parameterized import parameterized_class import numpy as np import pandas as pd +import pytest +from parameterized import parameterized_class +import server.test.unit.decode_fbs as decode_fbs from server.common.data_locator import DataLocator from server.common.errors import FilterError from server.data_anndata.anndata_adaptor import AnndataAdaptor -from server.test import PROJECT_ROOT, app_config -from server.test.test_datasets.fixtures import pbmc3k_colors +from server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT +from server.test.fixtures.fixtures import pbmc3k_colors """ Test the anndata adaptor using the pbmc3k data set. @@ -25,11 +24,11 @@ Test the anndata adaptor using the pbmc3k data set. ("data_locator", "backed"), [ (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False), - (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False), - (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False), (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True), - (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True), - (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True), + (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True), + (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True), ], ) class AdaptorTest(unittest.TestCase): @@ -84,7 +83,7 @@ class AdaptorTest(unittest.TestCase): self.assertEqual(self.data.get_colors(), pbmc3k_colors) def test_get_schema(self): - with open(path.join(path.dirname(__file__), "schema.json")) as fh: + with open(f"{FIXTURES_ROOT}/schema.json") as fh: schema = json.load(fh) self.assertDictEqual(self.data.get_schema(), schema) diff --git a/server/test/test_anndata_adaptor_data_load.py b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py similarity index 100% rename from server/test/test_anndata_adaptor_data_load.py rename to server/test/unit/data_anndata/test_anndata_adaptor_data_load.py diff --git a/server/test/test_nan_anndata_adaptor.py b/server/test/unit/data_anndata/test_nan_anndata_adaptor.py similarity index 93% rename from server/test/test_nan_anndata_adaptor.py rename to server/test/unit/data_anndata/test_nan_anndata_adaptor.py index 6471dd8b..29ca178f 100644 --- a/server/test/test_nan_anndata_adaptor.py +++ b/server/test/unit/data_anndata/test_nan_anndata_adaptor.py @@ -1,19 +1,19 @@ -import pytest +import math import unittest import warnings -import math -import server.test.decode_fbs as decode_fbs +import pytest -from server.data_anndata.anndata_adaptor import AnndataAdaptor -from server.common.errors import FilterError +import server.test.unit.decode_fbs as decode_fbs from server.common.data_locator import DataLocator -from server.test import PROJECT_ROOT, app_config +from server.common.errors import FilterError +from server.data_anndata.anndata_adaptor import AnndataAdaptor +from server.test import app_config, FIXTURES_ROOT class NaNTest(unittest.TestCase): def setUp(self): - self.data_locator = DataLocator(f"{PROJECT_ROOT}/server/test/test_datasets/nan.h5ad") + self.data_locator = DataLocator(f"{FIXTURES_ROOT}/nan.h5ad") self.config = app_config(self.data_locator.path) with warnings.catch_warnings(): diff --git a/server/test/unit/data_common/__init__.py b/server/test/unit/data_common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/unit/data_common/fbs/__init__.py b/server/test/unit/data_common/fbs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_fbs.py b/server/test/unit/data_common/fbs/test_matrix.py similarity index 98% rename from server/test/test_fbs.py rename to server/test/unit/data_common/fbs/test_matrix.py index 59ad989a..1e23cb6b 100644 --- a/server/test/test_fbs.py +++ b/server/test/unit/data_common/fbs/test_matrix.py @@ -3,7 +3,7 @@ import pandas as pd import numpy as np from scipy import sparse -import server.test.decode_fbs as decode_fbs +import server.test.unit.decode_fbs as decode_fbs from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs diff --git a/server/test/test_matrixcache.py b/server/test/unit/data_common/test_matrix_loader.py similarity index 96% rename from server/test/test_matrixcache.py rename to server/test/unit/data_common/test_matrix_loader.py index 6c68cfc3..d365b001 100644 --- a/server/test/test_matrixcache.py +++ b/server/test/unit/data_common/test_matrix_loader.py @@ -1,13 +1,13 @@ +import os +import shutil +import tempfile +import time import unittest -from server.data_common.matrix_loader import MatrixDataCacheManager + from server.common.app_config import AppConfig from server.common.errors import DatasetAccessError -import tempfile -import shutil -import os -import time - -from server.test import PROJECT_ROOT +from server.data_common.matrix_loader import MatrixDataCacheManager +from server.test import FIXTURES_ROOT class MatrixCacheTest(unittest.TestCase): @@ -15,7 +15,7 @@ class MatrixCacheTest(unittest.TestCase): pass def make_temporay_datasets(self, dirname, num): - source = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg" + source = f"{FIXTURES_ROOT}/pbmc3k.cxg" for i in range(num): target = os.path.join(dirname, str(i) + ".cxg") shutil.copytree(source, target) @@ -38,7 +38,7 @@ class MatrixCacheTest(unittest.TestCase): result = {} for k, v in datasets.items(): # filter out the dirname and the .cxg from the name - newk = int(k[1][len(dirname) + 1 : -4]) + newk = int(k[1][len(dirname) + 1: -4]) result[newk] = v return result diff --git a/server/test/unit/data_cxg/__init__.py b/server/test/unit/data_cxg/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_cxg_adaptor.py b/server/test/unit/data_cxg/test_cxg_adaptor.py similarity index 74% rename from server/test/test_cxg_adaptor.py rename to server/test/unit/data_cxg/test_cxg_adaptor.py index 205ce24b..3544138c 100644 --- a/server/test/test_cxg_adaptor.py +++ b/server/test/unit/data_cxg/test_cxg_adaptor.py @@ -2,8 +2,8 @@ import unittest from server.common.data_locator import DataLocator from server.data_cxg.cxg_adaptor import CxgAdaptor -from server.test import PROJECT_ROOT, app_config -from server.test.test_datasets.fixtures import pbmc3k_colors +from server.test import FIXTURES_ROOT, app_config +from server.test.fixtures.fixtures import pbmc3k_colors class TestCxgAdaptor(unittest.TestCase): @@ -14,6 +14,6 @@ class TestCxgAdaptor(unittest.TestCase): self.assertDictEqual(data.get_colors(), dict()) def get_data(self, fixture): - data_locator = f"{PROJECT_ROOT}/server/test/test_datasets/{fixture}" + data_locator = f"{FIXTURES_ROOT}/{fixture}" config = app_config(data_locator) return CxgAdaptor(DataLocator(data_locator), config) diff --git a/server/test/decode_fbs.py b/server/test/unit/decode_fbs.py similarity index 99% rename from server/test/decode_fbs.py rename to server/test/unit/decode_fbs.py index 5debe7ae..477ce189 100644 --- a/server/test/decode_fbs.py +++ b/server/test/unit/decode_fbs.py @@ -1,18 +1,17 @@ """ Code to decode, for testing purposes, the flatbuffer encoded blobs. This code will need to be updated if fbs/matrix.fbs changes. - For more information, see fbs/matrix.fbs and server/data_common/fbs/ """ import json -import server.data_common.fbs.NetEncoding.TypedArray as TypedArray -import server.data_common.fbs.NetEncoding.Matrix as Matrix -import server.data_common.fbs.NetEncoding.Int32Array as Int32Array -import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array import server.data_common.fbs.NetEncoding.Float32Array as Float32Array import server.data_common.fbs.NetEncoding.Float64Array as Float64Array +import server.data_common.fbs.NetEncoding.Int32Array as Int32Array import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray +import server.data_common.fbs.NetEncoding.Matrix as Matrix +import server.data_common.fbs.NetEncoding.TypedArray as TypedArray +import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array def decode_typed_array(tarr): @@ -42,7 +41,6 @@ def decode_matrix_FBS(buf): """ Given a FBS Matrix, return an decoded Python dict containing same info in native format. - NOTE / TODO: row_idx not currently implemented """ df = Matrix.Matrix.GetRootAsMatrix(buf, 0) diff --git a/server/test/unit/eb/__init__.py b/server/test/unit/eb/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/test_eb.py b/server/test/unit/eb/test_eb.py similarity index 90% rename from server/test/test_eb.py rename to server/test/unit/eb/test_eb.py index f1e1ef9d..b43fda24 100644 --- a/server/test/test_eb.py +++ b/server/test/unit/eb/test_eb.py @@ -2,7 +2,7 @@ import unittest import tempfile import requests import subprocess -from server.test import PROJECT_ROOT +from server.test import PROJECT_ROOT, FIXTURES_ROOT from server.common.app_config import AppConfig from contextlib import contextmanager import time @@ -37,7 +37,7 @@ class Elastic_Beanstalk_Test(unittest.TestCase): c = AppConfig() # test that eb works c.update_server_config( - multi_dataset__dataroot=f"{PROJECT_ROOT}/server/test/test_datasets", app__flask_secret_key="open sesame" + multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame" ) c.complete_config() From 0d94c9e092f24140f57b62f969c7c6f6f38a6be7 Mon Sep 17 00:00:00 2001 From: maniarathi Date: Wed, 5 Aug 2020 11:56:34 -0700 Subject: [PATCH 31/55] DRY-ing flatbuffer code (#1716) --- server/data_common/fbs/matrix.py | 124 +++++++++++-------------------- server/requirements.txt | 2 +- server/test/unit/decode_fbs.py | 43 ++--------- 3 files changed, 53 insertions(+), 116 deletions(-) diff --git a/server/data_common/fbs/matrix.py b/server/data_common/fbs/matrix.py index e10bc56a..213e3ce4 100644 --- a/server/data_common/fbs/matrix.py +++ b/server/data_common/fbs/matrix.py @@ -1,58 +1,24 @@ -import flatbuffers -import numpy as np -from scipy import sparse -import pandas as pd import json +import numpy as np +import pandas as pd +from flatbuffers import Builder +from scipy import sparse + import server.data_common.fbs.NetEncoding.Column as Column -import server.data_common.fbs.NetEncoding.TypedArray as TypedArray -import server.data_common.fbs.NetEncoding.Matrix as Matrix -import server.data_common.fbs.NetEncoding.Int32Array as Int32Array -import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array import server.data_common.fbs.NetEncoding.Float32Array as Float32Array import server.data_common.fbs.NetEncoding.Float64Array as Float64Array +import server.data_common.fbs.NetEncoding.Int32Array as Int32Array import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray - - -# Placeholder until recent enhancements to flatbuffers Python -# runtime are released, at which point we can use the default -# version. This code is a port of the head. See: -# -# https://github.com/google/flatbuffers/pull/4829 -# -def CreateNumpyVector(builder, x): - """CreateNumpyVector writes a numpy array into the buffer.""" - - if not isinstance(x, np.ndarray): - raise TypeError(f"non-numpy-ndarray passed to CreateNumpyVector ({type(x)}") - - if x.dtype.kind not in ["b", "i", "u", "f"]: - raise TypeError("numpy-ndarray holds elements of unsupported datatype") - - if x.ndim > 1: - raise TypeError("multidimensional-ndarray passed to CreateNumpyVector") - - builder.StartVector(x.itemsize, x.size, x.dtype.alignment) - - # Ensure little endian byte ordering - if x.dtype.str[0] == "<": - x_little_endian = x - else: - x_little_endian = x.byteswap(inplace=False) - - # Calculate total length - length = int(x_little_endian.itemsize * x_little_endian.size) - builder.head = int(builder.Head() - length) - - # tobytes ensures c_contiguous ordering - builder.Bytes[builder.Head() : builder.Head() + length] = x_little_endian.tobytes(order="C") - - return builder.EndVector(x.size) +import server.data_common.fbs.NetEncoding.Matrix as Matrix +import server.data_common.fbs.NetEncoding.TypedArray as TypedArray +import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array # Serialization helper def serialize_column(builder, typed_arr): """ Serialize NetEncoding.Column """ + (u_type, u_value) = typed_arr Column.ColumnStart(builder) Column.ColumnAddUType(builder, u_type) @@ -63,6 +29,7 @@ def serialize_column(builder, typed_arr): # Serialization helper def serialize_matrix(builder, n_rows, n_cols, columns, col_idx): """ Serialize NetEncoding.Matrix """ + Matrix.MatrixStart(builder) Matrix.MatrixAddNRows(builder, n_rows) Matrix.MatrixAddNCols(builder, n_cols) @@ -77,9 +44,10 @@ def serialize_matrix(builder, n_rows, n_cols, columns, col_idx): # Serialization helper def serialize_typed_array(builder, source_array, encoding_info): """ - Serialize any of the various typed arrays, eg, Float32Array. Specific - means of serialization and type conversion are provided by type_info. + Serialize any of the various typed arrays, eg, Float32Array. Specific means of serialization and type conversion + are provided by type_info. """ + arr = source_array (array_type, as_type) = encoding_info(source_array) @@ -104,7 +72,8 @@ def serialize_typed_array(builder, source_array, encoding_info): arr = arr[0] elif arr.shape[1] == 1: arr = arr.T[0] - vec = CreateNumpyVector(builder, arr) + + vec = builder.CreateNumpyVector(arr) # serialize the typed array table builder.StartObject(1) @@ -113,38 +82,36 @@ def serialize_typed_array(builder, source_array, encoding_info): return (array_type, array_value) -column_encoding_type_map = { - # array protocol string: ( array_type, as_type ) - np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32), - np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), - np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), - np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32), - np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32), - np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), - np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), -} -column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") - - def column_encoding(arr): + column_encoding_type_map = { + # array protocol string: ( array_type, as_type ) + np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32), + np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), + np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), + np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int16).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.uint8).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint16).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + } + column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") + return column_encoding_type_map.get(arr.dtype.str, column_encoding_default) -index_encoding_type_map = { - # array protocol string: ( array_type, as_type ) - np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), - np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), - np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), -} -index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") - - def index_encoding(arr): + index_encoding_type_map = { + # array protocol string: ( array_type, as_type ) + np.dtype(np.int32).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.int64).str: (TypedArray.TypedArray.Int32Array, np.int32), + np.dtype(np.uint32).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + np.dtype(np.uint64).str: (TypedArray.TypedArray.Uint32Array, np.uint32), + } + index_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") + return index_encoding_type_map.get(arr.dtype.str, index_encoding_default) @@ -165,8 +132,7 @@ def guess_at_mem_needed(matrix): def encode_matrix_fbs(matrix, row_idx=None, col_idx=None): """ - Given a 2D DataFrame, ndarray or sparse equivalent, create and return a - Matrix flatbuffer. + Given a 2D DataFrame, ndarray or sparse equivalent, create and return a Matrix flatbuffer. :param matrix: 2D DataFrame, ndarray or sparse equivalent :param row_idx: index for row dimension, Index or ndarray @@ -183,7 +149,7 @@ def encode_matrix_fbs(matrix, row_idx=None, col_idx=None): (n_rows, n_cols) = matrix.shape # estimate size needed, so we don't unnecessarily realloc. - builder = flatbuffers.Builder(guess_at_mem_needed(matrix)) + builder = Builder(guess_at_mem_needed(matrix)) columns = [] for cidx in range(n_cols - 1, -1, -1): @@ -239,9 +205,9 @@ def deserialize_typed_array(tarr): def decode_matrix_fbs(fbs): """ - Given an FBS-encoded Matrix, return a Pandas DataFrame the contains the data - and indices. + Given an FBS-encoded Matrix, return a Pandas DataFrame the contains the data and indices. """ + matrix = Matrix.Matrix.GetRootAsMatrix(fbs, 0) n_rows = matrix.NRows() n_cols = matrix.NCols() diff --git a/server/requirements.txt b/server/requirements.txt index a4948f7d..cbac485c 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -8,7 +8,7 @@ Flask-Cors>=3.0.6 Flask-RESTful>=0.3.6 flask-server-timing>=0.1.2 flask-talisman>=0.7.0 -flatbuffers>=1.10.0 +flatbuffers>=1.11.0 flatten-dict>=0.2.0 fsspec>=0.4.4,<0.8.0 numba>=0.49.1 diff --git a/server/test/unit/decode_fbs.py b/server/test/unit/decode_fbs.py index 477ce189..6fb1fcec 100644 --- a/server/test/unit/decode_fbs.py +++ b/server/test/unit/decode_fbs.py @@ -1,46 +1,17 @@ """ Code to decode, for testing purposes, the flatbuffer encoded blobs. -This code will need to be updated if fbs/matrix.fbs changes. -For more information, see fbs/matrix.fbs and server/data_common/fbs/ + +This code will need to be updated if fbs/matrix.fbs changes. For more information, see fbs/matrix.fbs and +server/data_common/fbs/ """ -import json -import server.data_common.fbs.NetEncoding.Float32Array as Float32Array -import server.data_common.fbs.NetEncoding.Float64Array as Float64Array -import server.data_common.fbs.NetEncoding.Int32Array as Int32Array -import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray import server.data_common.fbs.NetEncoding.Matrix as Matrix -import server.data_common.fbs.NetEncoding.TypedArray as TypedArray -import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array - - -def decode_typed_array(tarr): - type_map = { - TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array, - TypedArray.TypedArray.Int32Array: Int32Array.Int32Array, - TypedArray.TypedArray.Float32Array: Float32Array.Float32Array, - TypedArray.TypedArray.Float64Array: Float64Array.Float64Array, - TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray, - } - (u_type, u) = tarr - if u_type == TypedArray.TypedArray.NONE: - return None - - TarType = type_map.get(u_type, None) - assert TarType is not None - - arr = TarType() - arr.Init(u.Bytes, u.Pos) - narr = arr.DataAsNumpy() - if u_type == TypedArray.TypedArray.JSONEncodedArray: - narr = json.loads(narr.tostring().decode("utf-8")) - return narr +from server.data_common.fbs.matrix import deserialize_typed_array def decode_matrix_FBS(buf): """ - Given a FBS Matrix, return an decoded Python dict containing - same info in native format. + Given a FBS Matrix, return an decoded Python dict containing same info in native format. NOTE / TODO: row_idx not currently implemented """ df = Matrix.Matrix.GetRootAsMatrix(buf, 0) @@ -53,8 +24,8 @@ def decode_matrix_FBS(buf): for col_idx in range(0, columns_length): col = df.Columns(col_idx) tarr = (col.UType(), col.U()) - decoded_columns.append(decode_typed_array(tarr)) + decoded_columns.append(deserialize_typed_array(tarr)) - cidx = decode_typed_array((df.ColIndexType(), df.ColIndex())) + cidx = deserialize_typed_array((df.ColIndexType(), df.ColIndex())) return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None} From b5e5ee01683191e96a55926825c54a86b19a2cb4 Mon Sep 17 00:00:00 2001 From: bmccandless Date: Wed, 5 Aug 2020 12:00:35 -0700 Subject: [PATCH 32/55] Update hosted app to get the oauth client secret from the secret manager (#1713) * Update the hosted app to get the oauth client secret from the secret manager * fix to eb app, and set no cache on oauth endpoints --- server/auth/auth_oauth.py | 14 +++++++-- server/eb/app.py | 64 +++++++++++++++++++++++++++------------ 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py index ca2adf95..bb224e21 100644 --- a/server/auth/auth_oauth.py +++ b/server/auth/auth_oauth.py @@ -122,13 +122,19 @@ class AuthTypeOAuth(AuthTypeClientBase): return payload.get("email") return None + def update_response(self, response): + response.cache_control.update( + dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True)) + def login(self): callbackurl = f'{self.callback_base_url}/oauth2/callback' return_path = request.args.get("dataset", "") return_to = f"{self.callback_base_url}/{return_path}" # save the return path in the session cookie, accessed in the callback function session["oauth_callback_redirect"] = return_to - return self.client.authorize_redirect(redirect_uri=callbackurl) + response = self.client.authorize_redirect(redirect_uri=callbackurl) + self.update_response(response) + return response def logout(self): if self.session_cookie: @@ -138,12 +144,15 @@ class AuthTypeOAuth(AuthTypeClientBase): @after_this_request def remove_cookie(response): response.set_cookie(self.cookie_params["key"], "", expires=0) + self.update_response() return response return_path = request.args.get("dataset", "") return_to = f"{self.callback_base_url}/{return_path}" params = {'returnTo' : return_to, 'client_id' : self.client_id} - return redirect(self.client.api_base_url + '/v2/logout?' + urlencode(params)) + response = redirect(self.client.api_base_url + '/v2/logout?' + urlencode(params)) + self.update_response() + return response def callback(self): token = self.client.authorize_access_token() @@ -165,6 +174,7 @@ class AuthTypeOAuth(AuthTypeClientBase): except Exception as e: raise AuthenticationError(f"unable to set_cookie {self.cookie_params}") from e + self.update_response(resp) return resp def get_login_url(self, data_adaptor): diff --git a/server/eb/app.py b/server/eb/app.py index 64645fc3..2e451e77 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -31,7 +31,7 @@ except Exception: sys.exit(1) -def get_secret_key(region_name, secret_name, secret_key): +def get_secret_key(region_name, secret_name): session = boto3.session.Session() client = session.client(service_name="secretsmanager", region_name=region_name) @@ -40,7 +40,7 @@ def get_secret_key(region_name, secret_name, secret_key): if "SecretString" in get_secret_value_response: var = get_secret_value_response["SecretString"] secret = json.loads(var) - return secret.get(secret_key) + return secret except Exception: logging.critical("Caught exception during get_secret_key", exc_info=True) sys.exit(1) @@ -48,6 +48,46 @@ def get_secret_key(region_name, secret_name, secret_key): return None +def handle_config_from_secret(app_config): + """Update configuration from the secret manager""" + secret_name = os.getenv("CXG_AWS_SECRET_NAME") + if not secret_name: + return + + # need to find the secret manager region. + # 1. from CXG_AWS_SECRET_REGION_NAME + # 2. discover from dataroot location (if on s3) + # 3. discover from config file location (if on s3) + secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME") + if secret_region_name is None: + secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) + if not secret_region_name: + secret_region_name = discover_s3_region_name(config_file) + if not secret_region_name: + logging.error("Could not determine the AWS Secret Manager region") + sys.exit(1) + + secrets = get_secret_key(secret_region_name, secret_name) + if not secrets: + return + + keyattrs = ( + ("flask_secret_key", "app__flask_secret_key"), + ("oauth_client_secret", "authentication__params_oauth__client_secret") + ) + + for key, attr in keyattrs: + curval = getattr(app_config.server_config, attr) + if curval: + continue + + # replace the attr with the secret if it is not set + val = secrets.get(key) + if val: + logging.error(f"set {attr} from secret") + app_config.update_server_config(**{attr : val}) + + class WSGIServer(Server): def __init__(self, app_config): super().__init__(app_config) @@ -158,30 +198,14 @@ try: logging.info("Configuration from CXG_DATAROOT") app_config.update_server_config(multi_dataset__dataroot=dataroot) - secret_name = os.getenv("CXG_AWS_SECRET_NAME") - if secret_name: - # need to find the secret manager region. - # 1. from CXG_AWS_SECRET_REGION_NAME - # 2. discover from dataroot location (if on s3) - # 3. discover from config file location (if on s3) - secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME") - if secret_region_name is None: - secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) - if not secret_region_name: - secret_region_name = discover_s3_region_name(config_file) - if not secret_region_name: - logging.error("Could not determine the AWS Secret Manager region") - sys.exit(1) - - flask_secret_key = get_secret_key(secret_region_name, secret_name, 'flask_secret_key') - app_config.update_server_config(app__flask_secret_key=flask_secret_key) + # update from secret manager + handle_config_from_secret(app_config) # features are unsupported in the current hosted server app_config.update_default_dataset_config( user_annotations__enable=False, embeddings__enable_reembedding=False, ) app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],) - app_config.complete_config(logging.info) if not app_config.server_config.app__flask_secret_key: From b18f96da77be81239f463fd5e86ee4b47fe463f6 Mon Sep 17 00:00:00 2001 From: Severiano Badajoz Date: Wed, 5 Aug 2020 15:20:03 -0700 Subject: [PATCH 33/55] check for bins change for canvas draw (#1693) * check for bins change for canvas draw * PR feedback --- client/src/components/categorical/value/index.js | 4 +++- client/src/components/miniHistogram/index.js | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/client/src/components/categorical/value/index.js b/client/src/components/categorical/value/index.js index f40a9a99..c04af970 100644 --- a/client/src/components/categorical/value/index.js +++ b/client/src/components/categorical/value/index.js @@ -453,7 +453,9 @@ class CategoryValue extends React.Component { label, CHART_WIDTH, VALUE_HEIGHT - ) ?? {}; + ) ?? {}; // if createHistogramBins returns empty object assign null to deconstructed + + if (!xScale || !yScale || !bins) return null; return ( { - const { obsOrVarContinuousFieldDisplayName } = this.props; + const { obsOrVarContinuousFieldDisplayName, bins } = this.props; if ( prevProps.obsOrVarContinuousFieldDisplayName !== - obsOrVarContinuousFieldDisplayName + obsOrVarContinuousFieldDisplayName || + prevProps.bins !== bins ) this.drawHistogram(); }; From 8d96477fae90ad798ff651cc3cf16e123c80b935 Mon Sep 17 00:00:00 2001 From: Severiano Badajoz Date: Wed, 5 Aug 2020 16:43:35 -0700 Subject: [PATCH 34/55] Remove hash source from CSP style-src directive (#1717) * remove style csp hash generation + lint * remove references to style_hashes --- client/configuration/webpack/cspHashPlugin.js | 32 +++++++------------ server/eb/app.py | 18 +++++------ 2 files changed, 20 insertions(+), 30 deletions(-) diff --git a/client/configuration/webpack/cspHashPlugin.js b/client/configuration/webpack/cspHashPlugin.js index 99afd46f..3b32efc0 100644 --- a/client/configuration/webpack/cspHashPlugin.js +++ b/client/configuration/webpack/cspHashPlugin.js @@ -1,7 +1,12 @@ +/* eslint-disable import/no-extraneous-dependencies -- this file is a devDependency*/ const cheerio = require("cheerio"); const crypto = require("crypto"); -HtmlWebpackPlugin = require("html-webpack-plugin"); +const HtmlWebpackPlugin = require("html-webpack-plugin"); +const digest = (str) => { + const hash = crypto.createHash("sha256").update(str, "utf8").digest("base64"); + return `sha256-${hash}`; +}; class CspHashPlugin { constructor(opts) { this.opts = { ...opts }; @@ -19,10 +24,7 @@ class CspHashPlugin { if (filename) { const results = {}; results["script-hashes"] = $("script:not([src]):not([no-csp-hash])") - .map((i, elmt) => this.digest($(elmt).html())) - .get(); - results["style-hashes"] = $("style:not([href]):not([no-csp-hash])") - .map((i, elmt) => this.digest($(elmt).html())) + .map((i, elmt) => digest($(elmt).html())) .get(); const json = JSON.stringify(results); @@ -34,13 +36,10 @@ class CspHashPlugin { // Remove no-csp-hash attributes. Cheerio does not parse Jinja templates // correctly, so we brute force this with a regular expression. - data.html = data.html - .replace(/( From 80f6137528354d3c05180b43e92b5f9613a556a1 Mon Sep 17 00:00:00 2001 From: Madison Dunitz Date: Tue, 11 Aug 2020 15:48:12 -0500 Subject: [PATCH 39/55] retrieve latest annotation from db (#1723) * add function to retrieve latest annotation from db, db updates * dont create directory in s3 --- server/common/annotations.py | 58 ++++++++++++++++++++++ server/db/cellxgene_orm.py | 25 ++++++---- server/db/db_utils.py | 3 ++ server/test/fixtures/database/__init__.py | 14 +++--- server/test/test_database/test_database.py | 36 ++++++++++++-- 5 files changed, 114 insertions(+), 22 deletions(-) diff --git a/server/common/annotations.py b/server/common/annotations.py index 592e4805..c1a0f735 100644 --- a/server/common/annotations.py +++ b/server/common/annotations.py @@ -1,3 +1,6 @@ +import json +import uuid +import time from datetime import datetime import re import os @@ -13,6 +16,9 @@ import fastobo from flask import session, current_app, has_request_context from abc import ABCMeta, abstractmethod +from server.db.cellxgene_orm import CellxGeneDataset, Annotation +from server.db.db_utils import DbUtils + class Annotations(metaclass=ABCMeta): """ baseclass for annotations, including ontologies""" @@ -259,3 +265,55 @@ class AnnotationsLocalFile(Annotations): params["annotations-data-collection-name"] = collection parameters.update(params) + + +class AnnotationsHostedTileDB(Annotations): + def __init__(self, directory_path: str, db: DbUtils): + super().__init__() + self.db = db + self.directory_path = directory_path + + def set_collection(self, name): + pass + + def read_labels(self, data_adaptor): + uid = current_app.auth.get_user_id() + dataset_name = data_adaptor.get_location() + dataset = self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]) + # Todo @madison retrieve latest based on timestamp + annotation_object = self.db.query_for_most_recent( # noqa F841 + Annotation, [Annotation.user_id == uid, Annotation.dataset == dataset] + ) + # Todo in future pr, retrieve dataframe from tiledb uri + + def write_labels(self, df, data_adaptor): + uid = current_app.auth.get_user_id() + timestamp = time.time() + dataset_name = data_adaptor.get_location() + try: + dataset_id = self.db.query( + table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name] + )[0].id + except IndexError: + dataset_id = uuid.uuid4() + dataset = CellxGeneDataset(id=dataset_id, name=dataset_name) + self.db.session.add(dataset) + + uri = f"{self.directory_path}/{dataset_name}/{uid}/{timestamp}" + if "s3" in uri: + pass + else: + os.makedirs(uri, exist_ok=True) + schema_hints = {} + annotation = Annotation( + tiledb_uri=uri, + user_id=uid, + dataset_id=str(dataset_id), + schema_hints=json.dumps(schema_hints) + ) + # todo in future pr -- write df to tiledb, store at uri + self.db.session.add(annotation) + self.db.session.commit() + + def update_parameters(self, parameters, data_adaptor): + pass diff --git a/server/db/cellxgene_orm.py b/server/db/cellxgene_orm.py index 27823bc2..f2209b63 100644 --- a/server/db/cellxgene_orm.py +++ b/server/db/cellxgene_orm.py @@ -1,11 +1,12 @@ -from datetime import datetime +import uuid from sqlalchemy import ( Column, DateTime, ForeignKey, String, -) + func, JSON) +from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship @@ -21,8 +22,8 @@ class CellxGeneUser(Base): __tablename__ = "cxguser" id = Column(String, primary_key=True) - created_at = Column(DateTime, nullable=False, default=datetime.utcnow) - updated_at = Column(DateTime, nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + updated_at = Column(DateTime, nullable=False, server_default=func.now(), onupdate=func.now()) # Relationships annotations = relationship("Annotation", back_populates="cxguser") @@ -36,12 +37,13 @@ class Annotation(Base): __tablename__ = "annotation" - id = Column(String, primary_key=True) + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False) tiledb_uri = Column(String) user_id = Column(String, ForeignKey("cxguser.id"), nullable=False) - dataset_id = Column(String, ForeignKey("cxgdataset.id"), nullable=False) - created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + dataset_id = Column(UUID, ForeignKey("cxgdataset.id"), nullable=False) + created_at = Column(DateTime, nullable=False, server_default=func.now()) + schema_hints = Column(JSON) # Relationships cxguser = relationship("CellxGeneUser", back_populates="annotations") dataset = relationship("CellxGeneDataset", back_populates="annotations") @@ -49,12 +51,13 @@ class Annotation(Base): class CellxGeneDataset(Base): """ - Datasets refer to cellxgene datasets stored in tiledb + Datasets refer to datasets stored by cellxgene """ __tablename__ = "cxgdataset" - id = Column(String, primary_key=True) - name = Column(String) - created_at = Column(DateTime, nullable=False, default=datetime.utcnow) + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4, unique=True, nullable=False) + name = Column(String, unique=True, index=True) + + created_at = Column(DateTime, nullable=False, server_default=func.now()) annotations = relationship("Annotation", back_populates="dataset") diff --git a/server/db/db_utils.py b/server/db/db_utils.py index 2cf3f59e..9cae9ea2 100644 --- a/server/db/db_utils.py +++ b/server/db/db_utils.py @@ -33,6 +33,9 @@ class DbUtils: else self.session.query(*table_args).all() ) + def query_for_most_recent(self, table: Base, filter_args: typing.List[bool] = None) -> Base: + return self.session.query(table).filter(*filter_args).order_by(table.created_at.desc()).limit(1).all()[0] + class DBSessionMaker: def __init__(self, database_uri): diff --git a/server/test/fixtures/database/__init__.py b/server/test/fixtures/database/__init__.py index 82b17b58..5a7fa984 100644 --- a/server/test/fixtures/database/__init__.py +++ b/server/test/fixtures/database/__init__.py @@ -29,23 +29,26 @@ class TestDatabase: def _create_test_user(self): user = CellxGeneUser(id="test_user_id") + user2 = CellxGeneUser(id='1234') self.db.session.add(user) + self.db.session.add(user2) self.db.session.commit() def _create_test_dataset(self): dataset = CellxGeneDataset( - id="test_dataset_id", name="test_dataset", ) self.db.session.add(dataset) self.db.session.commit() def _create_test_annotation(self): + dataset = self.db.query([CellxGeneDataset], + [CellxGeneDataset.name == "test_dataset"], + )[0] annotation = Annotation( - id="test_annotation_id", tiledb_uri="tiledb_uri", user_id="test_user_id", - dataset_id="test_dataset_id" + dataset_id=str(dataset.id) ) self.db.session.add(annotation) self.db.session.commit() @@ -65,7 +68,7 @@ class TestDatabase: def _create_test_datasets(self, dataset_count: int = 10): datasets = [] for i in range(dataset_count): - datasets.append(CellxGeneDataset(id=self.get_random_string(), name=self.get_random_string())) + datasets.append(CellxGeneDataset(name=self.get_random_string())) self.db.session.add_all(datasets) self.db.session.commit() @@ -78,10 +81,9 @@ class TestDatabase: dataset = self.order_by_random(CellxGeneDataset) user = self.order_by_random(CellxGeneUser) annotations.append(Annotation( - id=self.get_random_string(), tiledb_uri=self.get_random_string(), user_id=user.id, - dataset_id=dataset.id + dataset_id=str(dataset.id) )) self.db.session.add_all(annotations) self.db.session.commit() diff --git a/server/test/test_database/test_database.py b/server/test/test_database/test_database.py index 706d2fdc..4ecb4f27 100644 --- a/server/test/test_database/test_database.py +++ b/server/test/test_database/test_database.py @@ -4,7 +4,7 @@ from server.db.db_utils import DbUtils from server.test.fixtures.database import TestDatabase -class AppConfigTest(unittest.TestCase): +class DatabaseTest(unittest.TestCase): db = DbUtils("postgresql://postgres:test_pw@localhost:5432") @classmethod @@ -22,13 +22,39 @@ class AppConfigTest(unittest.TestCase): self.assertGreater(user_count, 10) def test_dataset_creation(self): - one_dataset = self.db.get(table=CellxGeneDataset, entity_id='test_dataset_id') - self.assertEqual(one_dataset.id, 'test_dataset_id') + one_dataset = self.db.query(table_args=[CellxGeneDataset], + filter_args=[CellxGeneDataset.name == 'test_dataset']) + self.assertEqual(one_dataset[0].name, 'test_dataset') dataset_count = self.db.session.query(CellxGeneDataset).count() self.assertGreater(dataset_count, 10) def test_annotation_creation(self): - one_annotation = self.db.get(table=Annotation, entity_id='test_annotation_id') - self.assertEqual(one_annotation.id, 'test_annotation_id') + one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0] + self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri') annotation_count = self.db.session.query(Annotation).count() self.assertGreater(annotation_count, 10) + + def test_get_most_recent_annotation_for_user_dataset(self): + dataset_id = str(self.db.query(table_args=[CellxGeneDataset], + filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id) + + # have to commit separately because created_at time written on the db server + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0')) + self.db.session.commit() + + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1')) + self.db.session.commit() + + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2')) + self.db.session.commit() + + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3')) + self.db.session.commit() + + self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4')) + self.db.session.commit() + + most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id, + Annotation.user_id == 'test_user_id']) + + self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4') From f221856ae1d5535a6604a9394cb2c2da999c867d Mon Sep 17 00:00:00 2001 From: Severiano Badajoz Date: Tue, 11 Aug 2020 16:50:46 -0700 Subject: [PATCH 40/55] add CSP sources for obsolete browser prompt (#1731) Adds script hash and explicit domain to `img-src` directive --- client/configuration/webpack/obsoleteHTMLTemplate.html | 1 + server/eb/app.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/client/configuration/webpack/obsoleteHTMLTemplate.html b/client/configuration/webpack/obsoleteHTMLTemplate.html index e5c96849..35d7be95 100644 --- a/client/configuration/webpack/obsoleteHTMLTemplate.html +++ b/client/configuration/webpack/obsoleteHTMLTemplate.html @@ -1,4 +1,5 @@