fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-24 11:48:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into release-version-0.16.5

Browse Source
This commit is contained in:
maniarathi
2020-10-05 10:34:10 -07:00
parent b37dafa99b 3718e894ed
commit e8b35a36fd
106 changed files with 3615 additions and 1960 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/push_tests.yml
+2 -1
View File
@@ -31,9 +31,10 @@ jobs:
- name: Install dependencies
run: |
pip install flake8
pip install black
cd client
npm install
- name: Lint with flake8
- name: Format with black and lint with flake8
run: |
make lint-server
- name: Lint src with eslint
LICENSE.txt
+2 -2
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2013
Copyright (c) 2017-2020 Chan Zuckerberg Initiative
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Makefile
+3 -2
View File
@@ -82,8 +82,9 @@ fmt-py:
lint: lint-server lint-client
.PHONY: lint-server
lint-server:
flake8 server
lint-server: fmt-py
flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821'
.PHONY: lint-client
lint-client:
README.md
+1 -1
View File
@@ -81,7 +81,7 @@ If you believe you have found a security issue, we would appreciate notification
# Inspiration
We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap
+2 -2
View File
@@ -3,14 +3,14 @@
exports[`annotations stacked bar graph renders 1`] = `
Array [
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
]
`;
exports[`annotations stacked bar graph renders 2`] = `
Array [
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
]
`;
client/__tests__/e2e/cellxgeneActions.js
+74
View File
@@ -13,8 +13,11 @@ import {
getTestClass,
getTestId,
isElementPresent,
goToPage,
} from "./puppeteerUtils";
import { appUrlBase } from "./config";
export async function drag(testId, start, end, lasso = false) {
const layout = await waitByID(testId);
const elBox = await layout.boxModel();
@@ -312,4 +315,75 @@ export async function assertCategoryDoesNotExist(categoryName) {
await expect(result).toBe(false);
}
export async function login() {
const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
const password = "Test1111";
await goToPage(appUrlBase);
await clickOn("auth-button");
// (thuang): Auth0 form is unstable and unsafe for input until verified
await waitUntilFormFieldStable('[name="email"]');
await expect(page).toFillForm("form", {
email,
password,
});
await Promise.all([
page.waitForNavigation({ waitUntil: "networkidle0" }),
expect(page).toClick('[name="submit"]'),
]);
expect(page.url()).toContain(appUrlBase);
}
export async function logout() {
await clickOnUntil("menu", async () => {
await expect(page).toMatch("Log Out");
await Promise.all([
page.waitForNavigation({ waitUntil: "networkidle0" }),
expect(page).toClick("a", { text: "Log Out" }),
]);
});
await expect(page).toMatch("Log In");
}
async function waitUntilFormFieldStable(selector) {
const MAX_RETRY = 10;
const WAIT_FOR_MS = 200;
const EXPECTED_VALUE = "aaa";
let retry = 0;
while (retry < MAX_RETRY) {
try {
await expect(page).toFill(selector, EXPECTED_VALUE);
const fieldHandle = await expect(page).toMatchElement(selector);
const fieldValue = await page.evaluate(
(input) => input.value,
fieldHandle
);
expect(fieldValue).toBe(EXPECTED_VALUE);
break;
} catch (error) {
retry += 1;
await page.waitFor(WAIT_FOR_MS);
}
}
if (retry === MAX_RETRY) {
throw Error("clickOnUntil() assertion failed!");
}
}
/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
client/__tests__/e2e/e2e.test.js
+16
View File
@@ -31,6 +31,8 @@ import {
runDiffExp,
selectCategory,
subset,
login,
logout,
} from "./cellxgeneActions";
const data = datasets[DATASET];
@@ -518,4 +520,18 @@ test("lasso moves after pan", async () => {
expect(panCount).toBe(initialCount);
});
const conditionalDescribe =
process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
conditionalDescribe("AuthN Integration", () => {
it("logs in", async () => {
await login();
});
it("logs out", async () => {
await login();
await logout();
});
});
/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
client/__tests__/e2e/puppeteer.setup.js
+3 -3
View File
@@ -17,7 +17,9 @@ setDefaultOptions({ timeout: 20 * 1000 });
jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
(async () => {
beforeEach(async () => {
await jestPuppeteer.resetBrowser();
const userAgent = await browser.userAgent();
await page.setUserAgent(`${userAgent}bot`);
@@ -53,6 +55,4 @@ jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
}
}
});
})().catch((error) => {
console.error("puppeteer.setup.js error", error);
});
client/configuration/eslint/eslint.js
+1
View File
@@ -4,6 +4,7 @@ module.exports = {
extends: [
"airbnb",
"plugin:eslint-comments/recommended",
"plugin:@blueprintjs/recommended",
"plugin:compat/recommended",
"plugin:prettier/recommended",
"prettier/react",
client/package-lock.json
Generated
+438 -13
View File
@@ -4156,6 +4156,216 @@
"tslib": "~1.10.0"
}
},
"@blueprintjs/eslint-plugin": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@blueprintjs/eslint-plugin/-/eslint-plugin-0.3.0.tgz",
"integrity": "sha512-bQEdE4ApEHxCDV8hT9uIxeRbDFKOtRLBT3/Zy3Ku+nowDAYl/8jwZKp6lJuR/nqvsfuIXTnVef6ivwdBEieQfA==",
"dev": true,
"requires": {
"@typescript-eslint/experimental-utils": "^4.2.0",
"eslint": "^7.9.0"
},
"dependencies": {
"@typescript-eslint/experimental-utils": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.3.0.tgz",
"integrity": "sha512-cmmIK8shn3mxmhpKfzMMywqiEheyfXLV/+yPDnOTvQX/ztngx7Lg/OD26J8gTZfkLKUmaEBxO2jYP3keV7h2OQ==",
"dev": true,
"requires": {
"@types/json-schema": "^7.0.3",
"@typescript-eslint/scope-manager": "4.3.0",
"@typescript-eslint/types": "4.3.0",
"@typescript-eslint/typescript-estree": "4.3.0",
"eslint-scope": "^5.0.0",
"eslint-utils": "^2.0.0"
}
},
"@typescript-eslint/typescript-estree": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.3.0.tgz",
"integrity": "sha512-ZAI7xjkl+oFdLV/COEz2tAbQbR3XfgqHEGy0rlUXzfGQic6EBCR4s2+WS3cmTPG69aaZckEucBoTxW9PhzHxxw==",
"dev": true,
"requires": {
"@typescript-eslint/types": "4.3.0",
"@typescript-eslint/visitor-keys": "4.3.0",
"debug": "^4.1.1",
"globby": "^11.0.1",
"is-glob": "^4.0.1",
"lodash": "^4.17.15",
"semver": "^7.3.2",
"tsutils": "^3.17.1"
}
},
"acorn": {
"version": "7.4.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
"integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
"dev": true
},
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
"which": "^2.0.1"
}
},
"eslint": {
"version": "7.10.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz",
"integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==",
"dev": true,
"requires": {
"@babel/code-frame": "^7.0.0",
"@eslint/eslintrc": "^0.1.3",
"ajv": "^6.10.0",
"chalk": "^4.0.0",
"cross-spawn": "^7.0.2",
"debug": "^4.0.1",
"doctrine": "^3.0.0",
"enquirer": "^2.3.5",
"eslint-scope": "^5.1.1",
"eslint-utils": "^2.1.0",
"eslint-visitor-keys": "^1.3.0",
"espree": "^7.3.0",
"esquery": "^1.2.0",
"esutils": "^2.0.2",
"file-entry-cache": "^5.0.1",
"functional-red-black-tree": "^1.0.1",
"glob-parent": "^5.0.0",
"globals": "^12.1.0",
"ignore": "^4.0.6",
"import-fresh": "^3.0.0",
"imurmurhash": "^0.1.4",
"is-glob": "^4.0.0",
"js-yaml": "^3.13.1",
"json-stable-stringify-without-jsonify": "^1.0.1",
"levn": "^0.4.1",
"lodash": "^4.17.19",
"minimatch": "^3.0.4",
"natural-compare": "^1.4.0",
"optionator": "^0.9.1",
"progress": "^2.0.0",
"regexpp": "^3.1.0",
"semver": "^7.2.1",
"strip-ansi": "^6.0.0",
"strip-json-comments": "^3.1.0",
"table": "^5.2.3",
"text-table": "^0.2.0",
"v8-compile-cache": "^2.0.3"
},
"dependencies": {
"eslint-scope": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
"integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
"dev": true,
"requires": {
"esrecurse": "^4.3.0",
"estraverse": "^4.1.1"
}
}
}
},
"eslint-visitor-keys": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
"dev": true
},
"espree": {
"version": "7.3.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
"integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
"dev": true,
"requires": {
"acorn": "^7.4.0",
"acorn-jsx": "^5.2.0",
"eslint-visitor-keys": "^1.3.0"
}
},
"esrecurse": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
"integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
"dev": true,
"requires": {
"estraverse": "^5.2.0"
},
"dependencies": {
"estraverse": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
"integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
"dev": true
}
}
},
"glob-parent": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
"integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
"dev": true,
"requires": {
"is-glob": "^4.0.1"
}
},
"globals": {
"version": "12.4.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
"integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
"dev": true,
"requires": {
"type-fest": "^0.8.1"
}
},
"ignore": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
"integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
"dev": true
},
"path-key": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
"integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
"dev": true
},
"semver": {
"version": "7.3.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
"integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
"dev": true
},
"shebang-command": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
"integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
"dev": true,
"requires": {
"shebang-regex": "^3.0.0"
}
},
"shebang-regex": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
"dev": true
},
"which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
"dev": true,
"requires": {
"isexe": "^2.0.0"
}
}
}
},
"@blueprintjs/icons": {
"version": "3.19.0",
"resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.19.0.tgz",
@@ -4184,6 +4394,76 @@
"minimist": "^1.2.0"
}
},
"@eslint/eslintrc": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz",
"integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==",
"dev": true,
"requires": {
"ajv": "^6.12.4",
"debug": "^4.1.1",
"espree": "^7.3.0",
"globals": "^12.1.0",
"ignore": "^4.0.6",
"import-fresh": "^3.2.1",
"js-yaml": "^3.13.1",
"lodash": "^4.17.19",
"minimatch": "^3.0.4",
"strip-json-comments": "^3.1.1"
},
"dependencies": {
"acorn": {
"version": "7.4.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
"integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
"dev": true
},
"ajv": {
"version": "6.12.5",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz",
"integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==",
"dev": true,
"requires": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"eslint-visitor-keys": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
"dev": true
},
"espree": {
"version": "7.3.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
"integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
"dev": true,
"requires": {
"acorn": "^7.4.0",
"acorn-jsx": "^5.2.0",
"eslint-visitor-keys": "^1.3.0"
}
},
"globals": {
"version": "12.4.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
"integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
"dev": true,
"requires": {
"type-fest": "^0.8.1"
}
},
"ignore": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
"integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
"dev": true
}
}
},
"@hapi/address": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz",
@@ -5137,6 +5417,32 @@
}
}
},
"@nodelib/fs.scandir": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz",
"integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==",
"dev": true,
"requires": {
"@nodelib/fs.stat": "2.0.3",
"run-parallel": "^1.1.9"
}
},
"@nodelib/fs.stat": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz",
"integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==",
"dev": true
},
"@nodelib/fs.walk": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz",
"integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==",
"dev": true,
"requires": {
"@nodelib/fs.scandir": "2.1.3",
"fastq": "^1.6.0"
}
},
"@npmcli/move-file": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.0.1.tgz",
@@ -5489,6 +5795,22 @@
"eslint-utils": "^2.0.0"
}
},
"@typescript-eslint/scope-manager": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.3.0.tgz",
"integrity": "sha512-cTeyP5SCNE8QBRfc+Lgh4Xpzje46kNUhXYfc3pQWmJif92sjrFuHT9hH4rtOkDTo/si9Klw53yIr+djqGZS1ig==",
"dev": true,
"requires": {
"@typescript-eslint/types": "4.3.0",
"@typescript-eslint/visitor-keys": "4.3.0"
}
},
"@typescript-eslint/types": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.3.0.tgz",
"integrity": "sha512-Cx9TpRvlRjOppGsU6Y6KcJnUDOelja2NNCX6AZwtVHRzaJkdytJWMuYiqi8mS35MRNA3cJSwDzXePfmhU6TANw==",
"dev": true
},
"@typescript-eslint/typescript-estree": {
"version": "2.34.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz",
@@ -5512,6 +5834,24 @@
}
}
},
"@typescript-eslint/visitor-keys": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.3.0.tgz",
"integrity": "sha512-xZxkuR7XLM6RhvLkgv9yYlTcBHnTULzfnw4i6+z2TGBLy9yljAypQaZl9c3zFvy7PNI7fYWyvKYtohyF8au3cw==",
"dev": true,
"requires": {
"@typescript-eslint/types": "4.3.0",
"eslint-visitor-keys": "^2.0.0"
},
"dependencies": {
"eslint-visitor-keys": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
"integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==",
"dev": true
}
}
},
"@webassemblyjs/ast": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz",
@@ -6387,9 +6727,9 @@
}
},
"bl": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz",
"integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==",
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
"integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
"dev": true,
"requires": {
"buffer": "^5.5.0",
@@ -6397,16 +6737,6 @@
"readable-stream": "^3.4.0"
},
"dependencies": {
"buffer": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz",
"integrity": "sha512-9FTEDjLjwoAkEwyMGDjYJQN2gfRgOKBKRfiglhvibGbpeeU/pQn1bJxQqm32OD/AIeEuHxU9roxXxg34Byp/Ww==",
"dev": true,
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4"
}
},
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -8796,6 +9126,15 @@
}
}
},
"dir-glob": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
"integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",
"dev": true,
"requires": {
"path-type": "^4.0.0"
}
},
"doctrine": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
@@ -10187,6 +10526,31 @@
"integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==",
"dev": true
},
"fast-glob": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz",
"integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==",
"dev": true,
"requires": {
"@nodelib/fs.stat": "^2.0.2",
"@nodelib/fs.walk": "^1.2.3",
"glob-parent": "^5.1.0",
"merge2": "^1.3.0",
"micromatch": "^4.0.2",
"picomatch": "^2.2.1"
},
"dependencies": {
"glob-parent": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
"integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
"dev": true,
"requires": {
"is-glob": "^4.0.1"
}
}
}
},
"fast-json-stable-stringify": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@@ -10202,6 +10566,15 @@
"resolved": "https://registry.npmjs.org/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz",
"integrity": "sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw=="
},
"fastq": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz",
"integrity": "sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==",
"dev": true,
"requires": {
"reusify": "^1.0.4"
}
},
"favicons": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/favicons/-/favicons-5.5.0.tgz",
@@ -11159,6 +11532,28 @@
"resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
"integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA=="
},
"globby": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz",
"integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==",
"dev": true,
"requires": {
"array-union": "^2.1.0",
"dir-glob": "^3.0.1",
"fast-glob": "^3.1.1",
"ignore": "^5.1.4",
"merge2": "^1.3.0",
"slash": "^3.0.0"
},
"dependencies": {
"array-union": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
"integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",
"dev": true
}
}
},
"got": {
"version": "6.7.1",
"resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
@@ -11691,6 +12086,12 @@
"integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=",
"dev": true
},
"ignore": {
"version": "5.1.8",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz",
"integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==",
"dev": true
},
"ignore-walk": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
@@ -14189,6 +14590,12 @@
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="
},
"merge2": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
"dev": true
},
"methods": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
@@ -15251,6 +15658,12 @@
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=",
"dev": true
},
"path-type": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
"integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",
"dev": true
},
"pbkdf2": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz",
@@ -17065,6 +17478,12 @@
"resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz",
"integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg=="
},
"reusify": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
"integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
"dev": true
},
"rgb-regex": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/rgb-regex/-/rgb-regex-1.0.1.tgz",
@@ -17100,6 +17519,12 @@
"resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz",
"integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA=="
},
"run-parallel": {
"version": "1.1.9",
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz",
"integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==",
"dev": true
},
"run-queue": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz",
client/package.json
+1
View File
@@ -84,6 +84,7 @@
"@babel/preset-react": "^7.10.4",
"@babel/register": "^7.10.5",
"@babel/runtime": "^7.10.5",
"@blueprintjs/eslint-plugin": "^0.3.0",
"@sentry/webpack-plugin": "^1.12.0",
"babel-eslint": "^10.1.0",
"babel-jest": "^26.1.0",
client/src/actions/embedding.js
+15 -9
View File
@@ -5,20 +5,23 @@ action creators related to embeddings choice
import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
export async function _switchEmbedding(prevAnnoMatrix, prevCrossfilter, newEmbeddingName) {
export async function _switchEmbedding(
prevAnnoMatrix,
prevCrossfilter,
newEmbeddingName
) {
/*
DRY helper used by this and reembedding action creators
*/
const base = prevAnnoMatrix.base();
const embeddingDf = await base.fetch("emb", newEmbeddingName);
const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix, prevCrossfilter.obsCrossfilter).select(
"emb",
newEmbeddingName,
{
mode: "all",
}
);
const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
annoMatrix,
prevCrossfilter.obsCrossfilter
).select("emb", newEmbeddingName, {
mode: "all",
});
return [annoMatrix, obsCrossfilter];
}
@@ -30,7 +33,10 @@ export const layoutChoiceAction = (newLayoutChoice) => async (
On layout choice, make sure we have selected all on the previous layout, AND the new
layout.
*/
const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
const {
annoMatrix: prevAnnoMatrix,
obsCrossfilter: prevCrossfilter,
} = getState();
const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
prevAnnoMatrix,
prevCrossfilter,
client/src/components/autosave/filenameDialog.js
+6 -5
View File
@@ -3,11 +3,12 @@ import { connect } from "react-redux";
import {
Button,
Tooltip,
InputGroup,
Dialog,
Classes,
Code,
Colors,
Dialog,
InputGroup,
Tooltip,
} from "@blueprintjs/core";
@connect((state) => ({
@@ -145,9 +146,9 @@ class FilenameDialog extends React.Component {
<div>
<p>
Your annotations are stored in this file:
<code className="bp3-code">
<Code>
{filenameText}-{idhash}.csv
</code>
</Code>
</p>
<p style={{ fontStyle: "italic" }}>
(We added a unique ID to your filename)
client/src/components/brushableHistogram/index.js
+5 -7
View File
@@ -5,12 +5,13 @@ https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
https://bl.ocks.org/mbostock/3019563
*/
import React, { useEffect, useRef, useState, useCallback } from "react";
import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
import { Button, ButtonGroup, Icon, Tooltip } from "@blueprintjs/core";
import { connect } from "react-redux";
import * as d3 from "d3";
import { interpolateCool } from "d3-scale-chromatic";
import Async from "react-async";
import memoize from "memoize-one";
import { IconNames } from "@blueprintjs/icons";
import * as globals from "../../globals";
import actions from "../../actions";
import { histogramContinuous } from "../../util/dataframe/histogram";
@@ -26,7 +27,7 @@ function maybeScientific(x) {
const _ticks = x.ticks(4);
if (x.domain().some((n) => Math.abs(n) >= 10000)) {
/*
/*
heuristic: if the last tick d3 wants to render has one significant
digit ie., 2000, render 2e+3, but if it's anything else ie., 42000000 render
4.20e+n
@@ -99,7 +100,7 @@ const HistogramFooter = React.memo(
pvalAdj,
}) => {
/*
Footer of each histogram. Will render range, title, and optionally
Footer of each histogram. Will render range, title, and optionally
differential expression info.
Required props:
@@ -214,10 +215,7 @@ const HistogramHeader = React.memo(
>
{onScatterPlotXClick && onScatterPlotYClick ? (
<span>
<span
style={{ marginRight: 7 }}
className="bp3-icon-standard bp3-icon-scatter-plot"
/>
<Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
<ButtonGroup style={{ marginRight: 7 }}>
<Button
data-testid={`plot-x-${fieldId}`}
client/src/components/categorical/category/index.js
+17 -5
View File
@@ -1,7 +1,13 @@
import React, { useRef, useEffect } from "react";
import { connect, shallowEqual } from "react-redux";
import { FaChevronRight, FaChevronDown } from "react-icons/fa";
import { AnchorButton, Button, Tooltip, Position } from "@blueprintjs/core";
import {
AnchorButton,
Button,
Classes,
Position,
Tooltip,
} from "@blueprintjs/core";
import { Flipper, Flipped } from "react-flip-toolkit";
import Async from "react-async";
import memoize from "memoize-one";
@@ -301,9 +307,12 @@ const StillLoading = ({ metadataField, checkboxID }) => {
alignItems: "flex-start",
}}
>
<label htmlFor={checkboxID} className="bp3-control bp3-checkbox">
<label
htmlFor={checkboxID}
className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
>
<input disabled id={checkboxID} checked type="checkbox" />
<span className="bp3-control-indicator" />
<span className={Classes.CONTROL_INDICATOR} />
</label>
<Truncate>
<span
@@ -375,7 +384,10 @@ const CategoryHeader = React.memo(
alignItems: "flex-start",
}}
>
<label className="bp3-control bp3-checkbox" htmlFor={checkboxID}>
<label
className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
htmlFor={checkboxID}
>
<input
id={checkboxID}
data-testclass="category-select"
@@ -385,7 +397,7 @@ const CategoryHeader = React.memo(
checked={selectionState === "all"}
type="checkbox"
/>
<span className="bp3-control-indicator" />
<span className={Classes.CONTROL_INDICATOR} />
</label>
<span
role="menuitem"
client/src/components/categorical/index.js
+3 -1
View File
@@ -185,7 +185,9 @@ class Categories extends React.Component {
{/* READ ONLY CATEGORICAL FIELDS */}
{/* this is duplicative but flat, could be abstracted */}
{allCategoryNames.map((catName) =>
!schema.annotations.obsByName[catName].writable ? (
!schema.annotations.obsByName[catName].writable &&
(schema.annotations.obsByName[catName].categories?.length > 1 ||
!schema.annotations.obsByName[catName].categories) ? (
<Category
key={catName}
metadataField={catName}
client/src/components/categorical/value/index.js
+5 -8
View File
@@ -4,12 +4,13 @@ import * as d3 from "d3";
import {
Button,
Classes,
Icon,
Menu,
MenuItem,
Popover,
Position,
Icon,
PopoverInteractionKind,
Position,
} from "@blueprintjs/core";
import * as globals from "../../../globals";
import styles from "../categorical.css";
@@ -410,7 +411,6 @@ class CategoryValue extends React.Component {
return (
<MiniStackedBar
/* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
{...{
colorTable,
domainValues,
@@ -418,7 +418,6 @@ class CategoryValue extends React.Component {
domain,
occupancy,
}}
/* eslint-enable react/jsx-props-no-spreading -- enable */
height={VALUE_HEIGHT}
width={CHART_WIDTH}
/>
@@ -461,14 +460,12 @@ class CategoryValue extends React.Component {
return (
<MiniHistogram
/* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
{...{
colorScale,
xScale,
yScale,
bins,
}}
/* eslint-enable react/jsx-props-no-spreading -- enable */
obsOrVarContinuousFieldDisplayName={colorAccessor}
domainLabel={label}
height={VALUE_HEIGHT}
@@ -564,7 +561,7 @@ class CategoryValue extends React.Component {
<div style={{ display: "flex", alignItems: "baseline" }}>
<label
htmlFor={valueToggleLabel}
className="bp3-control bp3-checkbox"
className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
style={{ margin: 0 }}
>
<input
@@ -576,7 +573,7 @@ class CategoryValue extends React.Component {
type="checkbox"
/>
<span
className="bp3-control-indicator"
className={Classes.CONTROL_INDICATOR}
onMouseEnter={this.handleMouseExit}
onMouseLeave={this.handleMouseEnter}
/>
client/src/components/categorical/value/occupancy.js
+6 -6
View File
@@ -3,10 +3,10 @@ import React from "react";
import { connect } from "react-redux";
import * as d3 from "d3";
import {
Classes,
Popover,
PopoverInteractionKind,
Position,
Classes,
} from "@blueprintjs/core";
@connect((state) => ({
@@ -18,8 +18,8 @@ class Occupancy extends React.PureComponent {
_HEIGHT = 11;
createHistogram = () => {
/*
Knowing that colorScale is based off continous data,
/*
Knowing that colorScale is based off continous data,
createHistogram fetches the continous data in relation to the cells releveant to the catagory value.
It then seperates that data into 50 bins for drawing the mini-histogram
*/
@@ -75,8 +75,8 @@ class Occupancy extends React.PureComponent {
};
createOccupancyStack = () => {
/*
Knowing that the color scale is based off of catagorical data,
/*
Knowing that the color scale is based off of catagorical data,
createOccupancyStack obtains a map showing the number if cells per colored value
Using the colorScale a stack of colored bars is drawn representing the map
*/
@@ -155,7 +155,7 @@ class Occupancy extends React.PureComponent {
popoverClassName={Classes.POPOVER_CONTENT_SIZING}
>
<canvas
className="bp3-popover-targer"
className={Classes.POPOVER_TARGET}
style={{
marginRight: 5,
width: this._WIDTH,
client/src/components/embedding/index.js
+5 -4
View File
@@ -2,13 +2,14 @@ import React from "react";
import { connect } from "react-redux";
import { useAsync } from "react-async";
import {
ButtonGroup,
Popover,
Button,
ButtonGroup,
H4,
Popover,
Position,
Radio,
RadioGroup,
Tooltip,
Position,
} from "@blueprintjs/core";
import * as globals from "../../globals";
import actions from "../../actions";
@@ -80,7 +81,7 @@ class Embedding extends React.PureComponent {
width: 400,
}}
>
<h1>Embedding Choice</h1>
<H4>Embedding Choice</H4>
<p style={{ fontStyle: "italic" }}>
There are {schema?.dataframe?.nObs} cells in the entire dataset.
</p>
client/src/components/geneExpression/addGenes.js
+4 -3
View File
@@ -6,11 +6,12 @@ import fuzzysort from "fuzzysort";
import { connect } from "react-redux";
import { Suggest } from "@blueprintjs/select";
import {
MenuItem,
Button,
ControlGroup,
FormGroup,
InputGroup,
ControlGroup,
Intent,
MenuItem,
} from "@blueprintjs/core";
import * as globals from "../../globals";
import actions from "../../actions";
@@ -278,7 +279,7 @@ class AddGenes extends React.Component {
popoverProps={{ minimal: true }}
/>
<Button
className="bp3-button bp3-intent-primary"
intent={Intent.PRIMARY}
data-testid="add-gene"
loading={userDefinedGenesLoading}
onClick={() => this.handleClick(activeItem)}
client/src/components/graph/setupLasso.js
+24 -5
View File
@@ -10,6 +10,7 @@ const Lasso = () => {
let lassoPolygon;
let lassoPath;
let closePath;
let lassoInProgress;
const polygonToPath = (polygon) =>
`M${polygon.map((d) => d.join(",")).join("L")}`;
@@ -25,8 +26,18 @@ const Lasso = () => {
lassoPolygon = [d3.mouse(svg.node())]; // current x y of mouse within element
if (lassoPath) {
// If the existing path is in progress
if (lassoInProgress) {
// cancel the existing lasso
handleCancel();
// Don't continue with current drag start
return;
}
lassoPath.remove();
}
// We're starting a new drag
lassoInProgress = true;
lassoPath = g
.append("path")
@@ -67,25 +78,33 @@ const Lasso = () => {
}
};
const handleCancel = () => {
lassoPath.remove();
closePath = closePath?.remove();
lassoPath = null;
lassoPolygon = null;
closePath = null;
dispatch.call("cancel");
};
const handleDragEnd = () => {
// remove the close path
closePath.remove();
closePath = null;
// succesfully closed
// successfully closed
if (
distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
closeDistance
) {
lassoInProgress = false;
lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
dispatch.call("end", lasso, lassoPolygon);
// otherwise cancel
} else {
lassoPath.remove();
lassoPath = null;
lassoPolygon = null;
dispatch.call("cancel");
handleCancel();
}
};
client/src/components/infoDrawer/infoDrawer.js
+2
View File
@@ -38,6 +38,8 @@ class InfoDrawer extends PureComponent {
const singleValueCategories = (
await Promise.all(nonUserAnnoCategories)
).reduce((acc, categoryData, i) => {
// Actually check to see if it is null(user anno)
if (!categoryData) return acc;
const catName = allCategoryNames[i];
const column = categoryData.icol(0);
client/src/components/leftSidebar/topLeftLogoAndTitle.js
+74 -36
View File
@@ -7,11 +7,19 @@ import * as globals from "../../globals";
import Logo from "../framework/logo";
import Truncate from "../util/truncate";
import InfoDrawer from "../infoDrawer/infoDrawer";
import AuthButtons from "../menubar/authButtons";
import InformationMenu from "../menubar/infoMenu";
const DATASET_TITLE_FONT_SIZE = 14;
@connect((state) => ({
datasetTitle: state.config?.displayNames?.dataset ?? "",
auth: state.config?.authentication,
userinfo: state.userinfo,
libraryVersions: state.config?.["library_versions"],
aboutLink: state.config?.links?.["about-dataset"],
tosURL: state.config?.parameters?.["about_legal_tos"],
privacyURL: state.config?.parameters?.["about_legal_privacy"],
}))
class LeftSideBar extends React.Component {
handleClick = () => {
@@ -20,7 +28,16 @@ class LeftSideBar extends React.Component {
};
render() {
const { datasetTitle } = this.props;
const {
datasetTitle,
auth,
userinfo,
libraryVersions,
aboutLink,
privacyURL,
tosURL,
dispatch,
} = this.props;
return (
<div
@@ -30,49 +47,70 @@ class LeftSideBar extends React.Component {
width: globals.leftSidebarWidth,
zIndex: 1,
borderBottom: `1px solid ${globals.lighterGrey}`,
display: "flex",
justifyContent: "space-between",
alignItems: "center",
}}
>
<Logo size={30} />
<span
style={{
fontSize: 28,
position: "relative",
top: -6,
fontWeight: "bold",
marginLeft: 5,
color: globals.logoColor,
userSelect: "none",
}}
>
cell
<div>
<Logo size={28} />
<span
style={{
position: "relative",
top: 1,
fontWeight: 300,
fontSize: 24,
position: "relative",
top: -6,
fontWeight: "bold",
marginLeft: 5,
color: globals.logoColor,
userSelect: "none",
}}
>
×
</span>
gene
</span>
<Button
minimal
style={{
fontSize: DATASET_TITLE_FONT_SIZE,
position: "absolute",
right: 10,
}}
onClick={this.handleClick}
>
<Truncate>
<span style={{ maxWidth: 155 }} data-testid="header">
{datasetTitle}
cell
<span
style={{
position: "relative",
top: 1,
fontWeight: 300,
fontSize: 24,
}}
>
×
</span>
</Truncate>
</Button>
<InfoDrawer />
gene
</span>
</div>
<div style={{ marginRight: 5, position: "relative", top: -7 }}>
<Button
minimal
style={{
fontSize: DATASET_TITLE_FONT_SIZE,
position: "relative",
top: -1,
}}
onClick={this.handleClick}
>
<Truncate>
<span style={{ maxWidth: 155 }} data-testid="header">
{datasetTitle}
</span>
</Truncate>
</Button>
<InfoDrawer />
<InformationMenu
{...{
libraryVersions,
aboutLink,
tosURL,
privacyURL,
auth,
dispatch,
userinfo,
}}
/>
{!userinfo.is_authenticated ? (
<AuthButtons auth={auth} userinfo={userinfo} />
) : null}
</div>
</div>
);
}
client/src/components/menubar/authButtons.js
+4 -5
View File
@@ -1,5 +1,5 @@
import React from "react";
import { AnchorButton, Tooltip } from "@blueprintjs/core";
import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
import * as globals from "../../globals";
import styles from "./menubar.css";
@@ -9,9 +9,9 @@ const Auth = React.memo((props) => {
if (!auth || (auth && !auth.requires_client_login)) return null;
return (
<div className={`bp3-button-group ${styles.menubarButton}`}>
<ButtonGroup className={styles.menubarButton}>
<Tooltip
content="Log in or log out of cellxgene"
content="Log in to cellxgene"
position="bottom"
hoverOpenDelay={globals.tooltipHoverOpenDelay}
>
@@ -19,13 +19,12 @@ const Auth = React.memo((props) => {
type="button"
data-testid="auth-button"
disabled={false}
icon={!userinfo.is_authenticated ? "log-in" : "log-out"}
href={!userinfo.is_authenticated ? auth.login : auth.logout}
>
{!userinfo.is_authenticated ? "Log In" : "Log Out"}
</AnchorButton>
</Tooltip>
</div>
</ButtonGroup>
);
});
client/src/components/menubar/clip.js
+14 -9
View File
@@ -1,12 +1,16 @@
import React from "react";
import {
Position,
Button,
Popover,
NumericInput,
ButtonGroup,
Icon,
Intent,
NumericInput,
Popover,
Position,
Tooltip,
} from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import { tooltipHoverOpenDelay } from "../../globals";
import styles from "./menubar.css";
@@ -28,13 +32,13 @@ const Clip = React.memo((props) => {
pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin;
const clipMax =
pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax;
const activeClipClass =
const intent =
clipPercentileMin > 0 || clipPercentileMax < 100
? " bp3-intent-warning"
: "";
? Intent.INTENT_WARNING
: Intent.NONE;
return (
<div className={`bp3-button-group ${styles.menubarButton}`}>
<ButtonGroup className={`${styles.menubarButton}`}>
<Popover
target={
<Tooltip
@@ -45,7 +49,8 @@ const Clip = React.memo((props) => {
<Button
type="button"
data-testid="visualization-settings"
className={`bp3-button bp3-icon-timeline-bar-chart ${activeClipClass}`}
intent={intent}
icon={IconNames.TIMELINE_BAR_CHART}
style={{
cursor: "pointer",
}}
@@ -126,7 +131,7 @@ const Clip = React.memo((props) => {
</div>
}
/>
</div>
</ButtonGroup>
);
});
client/src/components/menubar/index.js
+1 -12
View File
@@ -6,8 +6,7 @@ import * as globals from "../../globals";
import styles from "./menubar.css";
import actions from "../../actions";
import Clip from "./clip";
import AuthButtons from "./authButtons";
import InformationMenu from "./infoMenu";
import Subset from "./subset";
import UndoRedoReset from "./undoRedo";
import DiffexpButtons from "./diffexpButtons";
@@ -204,7 +203,6 @@ class MenuBar extends React.PureComponent {
render() {
const {
dispatch,
libraryVersions,
disableDiffexp,
undoDisabled,
redoDisabled,
@@ -212,17 +210,12 @@ class MenuBar extends React.PureComponent {
clipPercentileMin,
clipPercentileMax,
graphInteractionMode,
aboutLink,
showCentroidLabels,
privacyURL,
tosURL,
categoricalSelection,
colorAccessor,
subsetPossible,
subsetResetPossible,
enableReembedding,
auth,
userinfo,
} = this.props;
const { pendingClipPercentiles } = this.state;
@@ -248,10 +241,6 @@ class MenuBar extends React.PureComponent {
zIndex: 3,
}}
>
<AuthButtons auth={auth} userinfo={userinfo} />
<InformationMenu
{...{ libraryVersions, aboutLink, tosURL, privacyURL, dispatch }}
/>
<UndoRedoReset
dispatch={dispatch}
undoDisabled={undoDisabled}
client/src/components/menubar/infoMenu.js
+40 -9
View File
@@ -1,7 +1,14 @@
// jshint esversion: 6
import React from "react";
import { Button, Popover, Menu, MenuItem, Position } from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import {
Button,
ButtonGroup,
Classes,
Menu,
MenuItem,
Popover,
Position,
} from "@blueprintjs/core";
import styles from "./menubar.css";
const handleClick = (dispatch) => {
@@ -9,35 +16,45 @@ const handleClick = (dispatch) => {
};
const InformationMenu = React.memo((props) => {
const { libraryVersions, tosURL, privacyURL, dispatch } = props;
const {
libraryVersions,
tosURL,
privacyURL,
auth,
userinfo,
dispatch,
} = props;
return (
<div className={`bp3-button-group ${styles.menubarButton}`}>
<ButtonGroup className={`${styles.menubarButton}`}>
<Popover
content={
<Menu>
<MenuItem
onClick={() => handleClick(dispatch)}
icon={IconNames.INFO_SIGN}
icon="info-sign"
text="Dataset Overview"
/>
<MenuItem
href="https://chanzuckerberg.github.io/cellxgene/"
target="_blank"
icon="help"
text="Help"
icon="book"
text="Documentation"
rel="noopener"
/>
<MenuItem
href="https://join-cellxgene-users.herokuapp.com/"
target="_blank"
icon="chat"
text="Chat"
rel="noopener"
/>
<MenuItem
href="https://github.com/chanzuckerberg/cellxgene"
target="_blank"
icon="git-branch"
text="Github"
rel="noopener"
/>
<MenuItem
target="_blank"
@@ -56,21 +73,35 @@ const InformationMenu = React.memo((props) => {
href={privacyURL}
target="_blank"
text="Privacy Policy"
rel="noopener"
/>
) : null}
{auth?.["requires_client_login"] &&
userinfo?.["is_authenticated"] ? (
<>
<MenuItem text={`Logged in as: ${userinfo.email}`} />
<MenuItem text="Log Out" href={auth.logout} />
</>
) : null}
</Menu>
}
position={Position.BOTTOM_RIGHT}
modifiers={{
preventOverflow: { enabled: false },
hide: { enabled: false },
}}
>
<Button
data-testid="menu"
type="button"
className="bp3-button bp3-icon-info-sign"
className={`${Classes.BUTTON} bp3-icon-info-sign`}
style={{
cursor: "pointer",
}}
/>
</Popover>
</div>
</ButtonGroup>
);
});
client/src/components/menubar/undoRedo.js
+6 -5
View File
@@ -1,12 +1,13 @@
import React from "react";
import { AnchorButton, Tooltip } from "@blueprintjs/core";
import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import { tooltipHoverOpenDelay } from "../../globals";
import styles from "./menubar.css";
const UndoRedo = React.memo((props) => {
const { undoDisabled, redoDisabled, dispatch } = props;
return (
<div className={`bp3-button-group ${styles.menubarButton}`}>
<ButtonGroup className={`${styles.menubarButton}`}>
<Tooltip
content="Undo"
position="bottom"
@@ -14,7 +15,7 @@ const UndoRedo = React.memo((props) => {
>
<AnchorButton
type="button"
className="bp3-button bp3-icon-undo"
icon={IconNames.UNDO}
disabled={undoDisabled}
onClick={() => {
dispatch({ type: "@@undoable/undo" });
@@ -32,7 +33,7 @@ const UndoRedo = React.memo((props) => {
>
<AnchorButton
type="button"
className="bp3-button bp3-icon-redo"
icon={IconNames.REDO}
disabled={redoDisabled}
onClick={() => {
dispatch({ type: "@@undoable/redo" });
@@ -43,7 +44,7 @@ const UndoRedo = React.memo((props) => {
data-testid="redo"
/>
</Tooltip>
</div>
</ButtonGroup>
);
});
client/src/components/miniHistogram/index.js
-1
View File
@@ -72,7 +72,6 @@ export default class MiniHistogram extends React.PureComponent {
popoverClassName={Classes.POPOVER_CONTENT_SIZING}
>
<canvas
className="bp3-popover-targer"
style={{
marginRight: 5,
width,
client/src/components/miniStackedBar/index.js
-2
View File
@@ -1,4 +1,3 @@
// jshint esversion: 6
import React from "react";
export default class MiniStackedBar extends React.PureComponent {
@@ -59,7 +58,6 @@ export default class MiniStackedBar extends React.PureComponent {
return (
<canvas
className="bp3-popover-targer"
style={{
marginRight: 5,
width,
dev_docs/cxg.md
+2
View File
@@ -1,3 +1,5 @@
## UPDATE (9/30/2020): Starting today, the name Corpora will only be used as the internal project name, with cellxgene Data Portal being the official product name
# CXG Data Format Specification
Document Status: _draft_
docs/_config.yml
+2 -2
View File
@@ -15,6 +15,8 @@ nav:
url: posts/gallery
- title: Demo datasets
url: posts/demo-data
- title: All other datasets
url: https://cellxgene.cziscience.com/
- title: Preparing your data
url: posts/prepare
- title: Launching cellxgene
@@ -33,5 +35,3 @@ nav:
url: posts/contribute
- title: Contact & finding help
url: posts/contact
- title: cellxgene.cziscience.com
url: posts/cellxgene_cziscience_com
docs/_site/index.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Index | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Index" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/" />
<link rel="canonical" href="http://localhost:4000/cellxgene/" />
<meta property="og:url" content="http://localhost:4000/cellxgene/" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","headline":"Index","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/annotations.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>annotations | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="annotations" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Creating annotations" />
<meta property="og:description" content="Creating annotations" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/annotations.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/annotations.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","headline":"annotations","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/annotations.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"annotations","description":"Creating annotations","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/contact.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Contact | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Contact" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Contact" />
<meta property="og:description" content="Contact" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/contact.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/contact.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","headline":"Contact","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/contact.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Contact","description":"Contact","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn"><b>Contact & finding help</b></a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/contribute.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Code of conduct | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Code of conduct" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/contribute.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/contribute.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Code of conduct","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/contribute.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Code of conduct","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/demo-data.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>demo-data | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="demo-data" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Demo datasets" />
<meta property="og:description" content="Demo datasets" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/demo-data.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/demo-data.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/demo-data.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/gallery.html
+10 -10
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Gallery | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Gallery" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/gallery.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/gallery.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Gallery","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/gallery.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Gallery","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
@@ -130,7 +130,7 @@ Check out the cool data that our users are using cellxgene to explore!</p>
<h3 id="melanoma"><a href="https://melanoma.cellgeni.sanger.ac.uk/">Melanoma</a></h3>
<h3 id="czis-own-cellxgene-site"><a href="cellxgene_cziscience_com">CZI’s own cellxgene site</a></h3>
<h3 id="czis-own-cellxgene-site"><a href="https://cellxgene.cziscience.com/">CZI’s own cellxgene site</a></h3>
<p><em>Want us to link to your dataset here? <a href="contact">Just send us a note!</a></em></p>
docs/_site/posts/hosted.html
+30 -29
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Hosting cellxgene on the web | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Hosting cellxgene on the web" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/hosted.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/hosted.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Hosting cellxgene on the web","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/hosted.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Hosting cellxgene on the web","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
@@ -139,35 +139,36 @@
<h1 id="deploying-cellxgene-with-heroku">Deploying cellxgene with Heroku</h1>
<h2 id="quickstart">Quickstart</h2>
<h2 id="heroku-support">Heroku Support</h2>
<p>Clicking on the following button will forward you to Heroku to begin the deployment process:</p>
<p>The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.</p>
<p><a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
<img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy" />
</a></p>
<p>While we no longer directly support Heroku, it is still possible to create a Heroku app via <a href="https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile">our provided Dockerfile here</a> and <a href="https://devcenter.heroku.com/articles/build-docker-images-heroku-yml">Heroku’s documentation</a>.</p>
<p>If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.</p>
<p>You may have to tweak the <code class="language-plaintext highlighter-rouge">Dockerfile</code> like so:</p>
<p>Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:</p>
<pre><code class="language-Dockerfile">FROM ubuntu:bionic
<h3 id="default-settings">Default settings</h3>
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
<ul>
<li><code class="language-plaintext highlighter-rouge">App name</code>: the unique name for your deployment</li>
<li>This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)</li>
<li><code class="language-plaintext highlighter-rouge">App owner</code>: Who will own this app. Either you personally or an organization/team</li>
<li><code class="language-plaintext highlighter-rouge">Region</code>: Location of the server where the app will be deployed (EU or US)</li>
</ul>
RUN apt-get update &amp;&amp; \
apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests &amp;&amp; \
pip3 install cellxgene
<h3 id="configuration">Configuration</h3>
# ENTRYPOINT ["cellxgene"] # Heroku doesn't work well with ENTRYPOINT
</code></pre>
<ul>
<li><code class="language-plaintext highlighter-rouge">DATASET</code>: A <em>publicly</em> accessible URL pointing to a .h5ad file to view</li>
<li>This defaults to pbm3k.h5ad</li>
</ul>
<p>and provide a <code class="language-plaintext highlighter-rouge">heroku.yml</code> file similar to this:</p>
<p>After filling out the settings and pressing the <code class="language-plaintext highlighter-rouge">Deploy app</code> button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!</p>
<div class="language-yml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="na">build</span><span class="pi">:</span>
<span class="na">docker</span><span class="pi">:</span>
<span class="na">web</span><span class="pi">:</span> <span class="s">Dockerfile</span>
<span class="na">run</span><span class="pi">:</span>
<span class="na">web</span><span class="pi">:</span>
<span class="na">command</span><span class="pi">:</span>
<span class="pi">-</span> <span class="s">cellxgene launch --host 0.0.0.0 --port $PORT $DATASET</span> <span class="c1"># the DATATSET config var must be defined in your dashboard settings.</span>
</code></pre></div></div>
<h2 id="what-is-heroku">What is Heroku?</h2>
docs/_site/posts/hosted.md
+23 -16
View File
@@ -38,31 +38,38 @@ If you know of other solutions, drop us a note and we'll add to this list.
# Deploying cellxgene with Heroku
## Quickstart
## Heroku Support
Clicking on the following button will forward you to Heroku to begin the deployment process:
The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.
<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
<img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
</a>
While we no longer directly support Heroku, it is still possible to create a Heroku app via [our provided Dockerfile here](https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile) and [Heroku's documentation](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml).
If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.
You may have to tweak the `Dockerfile` like so:
Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:
```Dockerfile
FROM ubuntu:bionic
### Default settings
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
- `App name`: the unique name for your deployment
- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
- `App owner`: Who will own this app. Either you personally or an organization/team
- `Region`: Location of the server where the app will be deployed (EU or US)
RUN apt-get update && \
apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
pip3 install cellxgene
### Configuration
# ENTRYPOINT ["cellxgene"] # Heroku doesn't work well with ENTRYPOINT
```
- `DATASET`: A _publicly_ accessible URL pointing to a .h5ad file to view
- This defaults to pbm3k.h5ad
and provide a `heroku.yml` file similar to this:
After filling out the settings and pressing the `Deploy app` button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!
```yml
build:
docker:
web: Dockerfile
run:
web:
command:
- cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings.
```
## What is Heroku?
docs/_site/posts/install.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Install | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Install" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/install.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/install.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Install","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/install.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Install","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/launch.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>demo-data | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="demo-data" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Demo datasets" />
<meta property="og:description" content="Demo datasets" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/launch.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/launch.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/launch.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/methods.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Methods | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Methods" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/methods.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/methods.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Methods","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/methods.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Methods","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/prepare.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>prepare | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="prepare" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Preparing your data" />
<meta property="og:description" content="Preparing your data" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/prepare.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/prepare.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","headline":"prepare","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/prepare.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"prepare","description":"Preparing your data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn"><b>Preparing your data</b></a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/roadmap.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>roadmap | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="roadmap" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Roadmap" />
<meta property="og:description" content="Roadmap" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/roadmap.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/roadmap.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","headline":"roadmap","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/roadmap.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"roadmap","description":"Roadmap","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/troubleshooting.html
+9 -9
View File
@@ -7,19 +7,19 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Troubleshooting | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Troubleshooting" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Troubleshooting" />
<meta property="og:description" content="Troubleshooting" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/troubleshooting.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/troubleshooting.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","headline":"Troubleshooting","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","@context":"https://schema.org"}</script>
{"url":"http://localhost:4000/cellxgene/posts/troubleshooting.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Troubleshooting","description":"Troubleshooting","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -50,6 +50,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
<a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/posts/cellxgene_cziscience_com.md → docs/deprecated/cellxgene_cziscience_com.md
View File
docs/posts/gallery.md
+1 -1
View File
@@ -39,6 +39,6 @@ Check out the cool data that our users are using cellxgene to explore!
### [Melanoma](https://melanoma.cellgeni.sanger.ac.uk/)
### [CZI's own cellxgene site](cellxgene_cziscience_com)
### [CZI's own cellxgene site](https://cellxgene.cziscience.com/)
_Want us to link to your dataset here? [Just send us a note!](contact)_
server/__init__.py
-1
View File
@@ -1,6 +1,5 @@
import logging
import sys
from server.common.utils.utils import import_plugins
__version__ = "0.16.5-rc.0"
server/app/app.py
+17 -10
View File
@@ -6,8 +6,17 @@ from urllib.parse import urlparse
import hashlib
import os
from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \
send_from_directory
from flask import (
Flask,
redirect,
current_app,
make_response,
render_template,
abort,
Blueprint,
request,
send_from_directory,
)
from flask_restful import Api, Resource
from server_timing import Timing as ServerTiming
@@ -87,10 +96,7 @@ def dataset_index(url_dataroot=None, dataset=None):
cache_manager = current_app.matrix_data_cache_manager
with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
args = {
"SCRIPTS" : scripts,
"INLINE_SCRIPTS" : inline_scripts
}
args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
return render_template("index.html", **args)
except DatasetAccessError as e:
@@ -417,8 +423,9 @@ class Server:
for dataroot_dict in server_config.multi_dataset__dataroot.values():
url_dataroot = dataroot_dict["base_url"]
bp_dataroot = Blueprint(
f"api_dataset_{url_dataroot}", __name__,
url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version
f"api_dataset_{url_dataroot}",
__name__,
url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version,
)
dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
self.app.register_blueprint(dataroot_resources.blueprint)
@@ -433,7 +440,7 @@ class Server:
f"/{url_dataroot}/<dataset>/static/<path:filename>",
f"static_assets_{url_dataroot}",
view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename),
methods=["GET"]
methods=["GET"],
)
else:
@@ -444,7 +451,7 @@ class Server:
"/static/<path:filename>",
"static_assets",
view_func=lambda filename: send_from_directory("../common/web/static", filename),
methods=["GET"]
methods=["GET"],
)
self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager
server/auth/__init__.py
-1
View File
@@ -1,4 +1,3 @@
# import the built in auth types so they can be registered
import server.auth.auth_none # noqa: F401
server/auth/auth.py
+1 -1
View File
@@ -76,7 +76,7 @@ class AuthTypeFactory:
@staticmethod
def register(name, auth_type):
assert(issubclass(auth_type, AuthTypeBase))
assert issubclass(auth_type, AuthTypeBase)
AuthTypeFactory.auth_types[name] = auth_type
@staticmethod
server/auth/auth_none.py
-1
View File
@@ -2,7 +2,6 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory
class AuthTypeNone(AuthTypeBase):
def __init__(self, app_config):
super().__init__()
server/auth/auth_oauth.py
+34 -4
View File
@@ -97,8 +97,17 @@ class AuthTypeOAuth(AuthTypeClientBase):
return
valid_keys = {
"verify_signature", "verify_aud", "verify_iat", "verify_exp", "verify_nbf", "verify_iss",
"verify_sub", "verify_jti", "verify_at_hash", "leeway"}
"verify_signature",
"verify_aud",
"verify_iat",
"verify_exp",
"verify_nbf",
"verify_iss",
"verify_sub",
"verify_jti",
"verify_at_hash",
"leeway",
}
keys = set(self.jwt_decode_options.keys())
unknown = keys - valid_keys
if unknown:
@@ -114,6 +123,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
parse = urlparse(self.api_base_url)
app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"])
app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"])
app.add_url_rule(f"{parse.path}/logout_redirect", "logout_redirect", self.logout_redirect, methods=["GET"])
app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"])
def complete_setup(self, flask_app):
@@ -166,12 +176,29 @@ class AuthTypeOAuth(AuthTypeClientBase):
return response
def logout(self):
"""
We would like for the user to remain on the same dataset after logout. oauth requires that
the redirect `returnTo` path be whitelisted by the oauth server, therefore a level of
indirection is used. We first redirect to a single path "logout_redirect", and logout_redirect
will redirect the user's browser back to the current page.
"""
self.remove_tokens()
params = {"returnTo": self.web_base_url, "client_id": self.client_id}
redirect_path = request.args.get("dataset", "")
redirect_to = f"{self.web_base_url}/{redirect_path}"
session["oauth_logout_redirect"] = redirect_to
return_to = f"{self.api_base_url}/logout_redirect"
params = {"returnTo": return_to, "client_id": self.client_id}
response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params))
self.update_response(response)
return response
def logout_redirect(self):
oauth_logout_redirect = session.pop("oauth_logout_redirect", "/")
response = redirect(oauth_logout_redirect)
self.update_response(response)
return response
def callback(self):
data = self.client.authorize_access_token()
tokens = Tokens(
@@ -253,7 +280,10 @@ class AuthTypeOAuth(AuthTypeClientBase):
def get_logout_url(self, data_adaptor):
"""Return the url for the logout route"""
return f"{self.api_base_url}/logout"
if data_adaptor and current_app.app_config.is_multi_dataset():
return f"{self.api_base_url}/logout?dataset={data_adaptor.uri_path}/"
else:
return f"{self.api_base_url}/logout"
def check_jwt_payload(self, id_token):
try:
server/auth/auth_test.py
+1
View File
@@ -47,6 +47,7 @@ class AuthTypeTest(AuthTypeClientBase):
return_to = args.get("dataset", "/")
session[self.CXGUID] = args.get("userid", self.user_id)
session[self.CXGUNAME] = args.get("username", self.user_name)
session[self.CXGUEMAIL] = args.get("email", self.user_email)
return redirect(return_to)
def logout(self):
server/cli/convert_to_cxg.py
+39 -35
View File
@@ -9,26 +9,24 @@ from server.converters.h5ad_data_file import H5ADDataFile
name="convert",
short_help="Converts an H5AD dataset to the CXG format.",
help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
"that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
"environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
"usually with the generated CXG file.",
"that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
"environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
"usually with the generated CXG file.",
)
@click.argument(
"input-file",
nargs=1,
type=click.Path(exists=True, dir_okay=False),
"input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"-o",
"--output-directory",
help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
"CXG extension.",
"CXG extension.",
)
@click.option(
"-b",
"--backed",
help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
"but will use less memory.",
"but will use less memory.",
default=False,
show_default=True,
is_flag=True,
@@ -37,29 +35,33 @@ from server.converters.h5ad_data_file import H5ADDataFile
"-t",
"--title",
help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
"the dataset title will be the filename.",
"the dataset title will be the filename.",
)
@click.option(
"-a",
"--about",
help="A fully qualified URL that provides more information about the dataset and will be included as "
"metadata about the CXG file.",
"metadata about the CXG file.",
)
@click.option(
"-s",
"--sparse-threshold",
help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
"array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
"convert to dense array.",
"array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
"convert to dense array.",
default=0.0,
show_default=True,
)
@click.option("--obs-names",
help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.")
@click.option("--var-names",
help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.")
@click.option(
"--obs-names",
help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.",
)
@click.option(
"--var-names",
help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.",
)
@click.option(
"--disable-custom-colors",
help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
@@ -70,8 +72,8 @@ from server.converters.h5ad_data_file import H5ADDataFile
@click.option(
"--disable-corpora-schema",
help="When set, conversion process will neither extract nor store Corpora schema information. See "
"https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
"information.",
"https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
"information.",
default=False,
show_default=True,
is_flag=True,
@@ -85,30 +87,32 @@ from server.converters.h5ad_data_file import H5ADDataFile
)
@click.help_option("--help", "-h", help="Show this message and exit.")
def convert_to_cxg(
input_file,
output_directory,
backed,
title,
about,
sparse_threshold,
obs_names,
var_names,
disable_custom_colors,
disable_corpora_schema,
overwrite,
input_file,
output_directory,
backed,
title,
about,
sparse_threshold,
obs_names,
var_names,
disable_custom_colors,
disable_corpora_schema,
overwrite,
):
"""
Convert a dataset file into CXG.
"""
h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names,
use_corpora_schema=not disable_corpora_schema)
h5ad_data_file = H5ADDataFile(
input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
)
# Get the directory that will hold all the CXG files
cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold,
convert_anndata_colors_to_cxg_colors=not disable_custom_colors)
h5ad_data_file.to_cxg(
cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
)
def get_output_directory(input_filename, output_directory, should_overwrite):
server/cli/launch.py
+35 -36
View File
@@ -3,15 +3,14 @@ import functools
import logging
import sys
import webbrowser
from os import devnull
import os
import click
from flask_compress import Compress
from flask_cors import CORS
from server.default_config import default_config
from server.app.app import Server
from server.common.app_config import AppConfig
from server.common.default_config import default_config
from server.common.config.app_config import AppConfig
from server.common.errors import DatasetAccessError, ConfigurationError
from server.common.utils.utils import sort_options
@@ -33,7 +32,7 @@ def annotation_args(func):
multiple=False,
metavar="<path>",
help="CSV file to initialize editing of existing annotations; will be altered in-place. "
"Incompatible with --annotations-dir.",
"Incompatible with --annotations-dir.",
)
@click.option(
"--annotations-dir",
@@ -42,7 +41,7 @@ def annotation_args(func):
multiple=False,
metavar="<directory path>",
help="Directory of where to save output annotations; filename will be specified in the application. "
"Incompatible with --annotations-file.",
"Incompatible with --annotations-file.",
)
@click.option(
"--experimental-annotations-ontology",
@@ -170,7 +169,7 @@ def server_args(func):
default=DEFAULT_CONFIG.server_config.app__debug,
show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",
"or when you want more information about an error condition.",
)
@click.option(
"--verbose",
@@ -203,7 +202,7 @@ def server_args(func):
multiple=True,
metavar="<text>",
help="Additional script files to include in HTML page. If not specified, "
"no additional script files will be included.",
"no additional script files will be included.",
show_default=False,
)
@functools.wraps(func)
@@ -223,7 +222,7 @@ def launch_args(func):
default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
metavar="<data directory>",
help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
" to folder containing H5AD and/or CXG datasets.",
" to folder containing H5AD and/or CXG datasets.",
hidden=True,
) # TODO, unhide when dataroot is supported)
@click.argument("datapath", required=False, metavar="<path to data file>")
@@ -307,32 +306,32 @@ class CliLaunchServer(Server):
)
@launch_args
def launch(
datapath,
dataroot,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
disable_custom_colors,
diffexp_lfc_cutoff,
title,
scripts,
about,
disable_annotations,
annotations_file,
annotations_dir,
backed,
disable_diffexp,
experimental_annotations_ontology,
experimental_annotations_ontology_obo,
experimental_enable_reembedding,
config_file,
dump_default_config,
datapath,
dataroot,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
disable_custom_colors,
diffexp_lfc_cutoff,
title,
scripts,
about,
disable_annotations,
annotations_file,
annotations_dir,
backed,
disable_diffexp,
experimental_annotations_ontology,
experimental_annotations_ontology_obo,
experimental_enable_reembedding,
config_file,
dump_default_config,
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
@@ -443,7 +442,7 @@ def launch(
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not server_config.app__verbose:
f = open(devnull, "w")
f = open(os.devnull, "w")
sys.stdout = f
try:
server/cli/prepare.py
+13 -13
View File
@@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options
default=False,
is_flag=True,
help="Do not run quality control metrics. By default cellxgene runs them "
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
)
@click.option(
"--make-obs-names-unique/--no-make-obs-names-unique",
@@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options
)
@click.help_option("--help", "-h", help="Show this message and exit.")
def prepare(
data,
embedding,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
skip_qc,
make_obs_names_unique,
make_var_names_unique,
data,
embedding,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
skip_qc,
make_obs_names_unique,
make_var_names_unique,
):
"""
Preprocess data for use with cellxgene.
server/cli/upgrade.py
+2 -1
View File
@@ -10,7 +10,8 @@ from .. import __version__
SEMVER_FORMAT = re.compile(
r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
r"?:\.[0-9a-zA-Z-]+)*))?$")
r"?:\.[0-9a-zA-Z-]+)*))?$"
)
def log_upgrade_check():
server/common/annotations/hosted_tiledb.py
+9 -8
View File
@@ -31,7 +31,8 @@ class AnnotationsHostedTileDB(Annotations):
unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
if unsanitary_original_category_names:
raise AnnotationCategoryNameError(
f"{unsanitary_original_category_names} are not valid category names, please resubmit")
f"{unsanitary_original_category_names} are not valid category names, please resubmit"
)
def is_safe_collection_name(self, name):
"""
@@ -68,11 +69,11 @@ class AnnotationsHostedTileDB(Annotations):
index_dims = None
schema_hints = json.loads(schema_hints)
if '__pandas_attribute_repr' in tileDBArray.meta:
if "__pandas_attribute_repr" in tileDBArray.meta:
# backwards compatibility... unsure if necessary at this point
repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr'])
if '__pandas_index_dims' in tileDBArray.meta:
index_dims = json.loads(tileDBArray.meta['__pandas_index_dims'])
repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
if "__pandas_index_dims" in tileDBArray.meta:
index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
data = tileDBArray[:]
indexes = list()
@@ -80,12 +81,12 @@ class AnnotationsHostedTileDB(Annotations):
for col_name, col_val in data.items():
# If the column values are byte literals, decode them
if isinstance(col_val[0], bytes):
col_val = [value.decode('utf-8') for value in col_val]
col_val = [value.decode("utf-8") for value in col_val]
if schema_hints and col_name in schema_hints:
type = schema_hints.get(col_name).get("type")
if type and type == "categorical":
new_col = pd.Series(col_val, dtype='category')
new_col = pd.Series(col_val, dtype="category")
data[col_name] = new_col
elif repr_meta and col_name in repr_meta:
new_col = pd.Series(col_val, dtype=repr_meta[col_name])
@@ -127,7 +128,7 @@ class AnnotationsHostedTileDB(Annotations):
tiledb_uri=uri,
user_id=user_id,
dataset_id=str(dataset_id),
schema_hints=json.dumps(dataframe_schema_type_hints)
schema_hints=json.dumps(dataframe_schema_type_hints),
)
if not df.empty:
self.check_category_names(df)
server/common/app_config.py
-962
View File
@@ -1,962 +0,0 @@
import copy
import os
import sys
import warnings
from os.path import splitext, basename, isdir
from urllib.parse import urlparse, quote_plus
import yaml
from flatten_dict import flatten, unflatten
import server.compute.diffexp_cxg as diffexp_tiledb
import server.compute.scanpy
from server import display_version as cellxgene_display_version
from server.auth.auth import AuthTypeFactory
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from server.common.annotations.local_file_csv import AnnotationsLocalFile
from server.common.data_locator import discover_s3_region_name
from server.common.default_config import get_default_config
from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
from server.db.db_utils import DbUtils
DEFAULT_SERVER_PORT = 5005
# anything bigger than this will generate a special message
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
class AppFeature(object):
def __init__(self, path, available=False, method="POST", extra={}):
self.path = path
self.available = available
self.method = method
self.extra = extra
for k, v in extra.items():
setattr(self, k, v)
def todict(self):
d = dict(available=self.available, method=self.method, path=self.path)
d.update(self.extra)
return d
class AppConfig(object):
"""AppConfig stores all the configuration for cellxgene. The configuration is divided into two main parts:
server attributes, and dataset attributes. The server_config contains attributes that refer to the server process
as a whole. The default_dataset_config referes to attributes that are associated with the features and
presentations of a dataset. The dataset config attributes can be overridden depending on the url by which the
dataset was accessed. These are stored in dataroot_config.
AppConfig has methods to initialize, modify, and access the configuration.
"""
def __init__(self):
# the default configuration (see default_config.py)
self.default_config = get_default_config()
# the server configuration
self.server_config = ServerConfig(self, self.default_config["server"])
# the dataset config, unless overridden by an entry in dataroot_config
self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
# a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
# attribute of the server_config.
self.dataroot_config = {}
# Set to true when config_completed is called
self.is_completed = False
def get_dataset_config(self, dataroot_key):
if self.server_config.single_dataset__datapath:
return self.default_dataset_config
else:
return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
def check_config(self):
"""Verify all the attributes have been checked"""
if not self.is_completed:
raise ConfigurationError("The configuration has not been completed")
self.server_config.check_config()
self.default_dataset_config.check_config()
for dataset_config in self.dataroot_config.values():
dataset_config.check_config()
def update_server_config(self, **kw):
self.server_config.update(**kw)
self.is_complete = False
def update_default_dataset_config(self, **kw):
self.default_dataset_config.update(**kw)
# update all the other dataset configs, if any
for value in self.dataroot_config.values():
value.update(**kw)
self.is_complete = False
def update_from_config_file(self, config_file):
with open(config_file) as fyaml:
config = yaml.load(fyaml, Loader=yaml.FullLoader)
if config.get("server"):
self.server_config.update_from_config(config["server"], "server")
if config.get("dataset"):
self.default_dataset_config.update_from_config(config["dataset"], "dataset")
per_dataset_config = config.get("per_dataset_config", {})
for key, dataroot_config in per_dataset_config.items():
# first create and initialize the dataroot with the default config
self.add_dataroot_config(key, **config["dataset"])
# then apply the per dataset configuration
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
self.is_complete = False
def write_config(self, config_file):
"""output the config to a yaml file"""
server = self.server_config.create_mapping(self.server_config.default_config)
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = dict(server={}, dataset={})
for attrname in server.keys():
config["server__" + attrname] = getattr(self.server_config, attrname)
for attrname in dataset.keys():
config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
if self.dataroot_config:
config["per_dataset_config"] = {}
for dataroot_tag, dataroot_config in self.dataroot_config.items():
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
for attrname in dataset.keys():
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
config = unflatten(config, splitter=lambda key: key.split("__"))
yaml.dump(config, open(config_file, "w"))
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
diff_server = self.server_config.changes_from_default()
diff_dataset = self.default_dataset_config.changes_from_default()
diff = dict(server=diff_server, dataset=diff_dataset)
return diff
def add_dataroot_config(self, dataroot_tag, **kw):
"""Create a new dataset config object based on the default dataset config, and kw parameters"""
if dataroot_tag in self.dataroot_config:
raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
if type(self.server_config.multi_dataset__dataroot) != dict:
raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
if dataroot_tag not in self.server_config.multi_dataset__dataroot:
raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
self.is_completed = False
self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = {key: value[1] for key, value in flat_config.items()}
self.dataroot_config[dataroot_tag].update(**config)
self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
def complete_config(self, messagefn=None):
"""The configure options are checked, and any additional setup based on the config
parameters is done"""
if messagefn is None:
def noop(message):
pass
messagefn = noop
# TODO: to give better error messages we can add a mapping between where each config
# attribute originated (e.g. command line argument or config file), then in the error
# messages we can give correct context for attributes with bad value.
context = dict(messagefn=messagefn)
self.server_config.complete_config(context)
self.default_dataset_config.complete_config(context)
for dataroot_config in self.dataroot_config.values():
dataroot_config.complete_config(context)
self.is_completed = True
self.check_config()
def get_matrix_data_cache_manager(self):
return self.server_config.matrix_data_cache_manager
def is_multi_dataset(self):
return self.server_config.multi_dataset__dataroot is not None
def get_title(self, data_adaptor):
return (
self.server_config.single_dataset__title
if self.server_config.single_dataset__title
else data_adaptor.get_title()
)
def get_about(self, data_adaptor):
return (
self.server_config.single_dataset__about
if self.server_config.single_dataset__about
else data_adaptor.get_about()
)
def get_client_config(self, data_adaptor):
"""
Return the configuration as required by the /config REST route
"""
server_config = self.server_config
dataset_config = data_adaptor.dataset_config
annotation = dataset_config.user_annotations
auth = server_config.auth
# FIXME The current set of config is not consistently presented:
# we have camalCase, hyphen-text, and underscore_text
# make sure the configuration has been checked.
self.check_config()
# features
features = [f.todict() for f in data_adaptor.get_features(annotation)]
# display_names
title = self.get_title(data_adaptor)
about = self.get_about(data_adaptor)
display_names = dict(engine=data_adaptor.get_name(), dataset=title)
# library_versions
library_versions = {}
library_versions.update(data_adaptor.get_library_versions())
library_versions["cellxgene"] = cellxgene_display_version
# links
links = {"about-dataset": about}
# parameters
parameters = {
"layout": dataset_config.embeddings__names,
"max-category-items": dataset_config.presentation__max_categories,
"obs_names": server_config.single_dataset__obs_names,
"var_names": server_config.single_dataset__var_names,
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
"backed": server_config.adaptor__anndata_adaptor__backed,
"disable-diffexp": not dataset_config.diffexp__enable,
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
"annotations_cell_ontology_enabled": False,
"annotations_cell_ontology_obopath": None,
"annotations_cell_ontology_terms": None,
"custom_colors": dataset_config.presentation__custom_colors,
"diffexp-may-be-slow": False,
"about_legal_tos": dataset_config.app__about_legal_tos,
"about_legal_privacy": dataset_config.app__about_legal_privacy,
}
# corpora dataset_props
# TODO/Note: putting info from the dataset into the /config is not ideal.
# However, it is definitely not part of /schema, and we do not have a top-level
# route for data properties. Consider creating one at some point.
corpora_props = data_adaptor.get_corpora_props()
if corpora_props and "default_embedding" in corpora_props:
default_embedding = corpora_props["default_embedding"]
if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
default_embedding = default_embedding[2:] # drop X_ prefix
if default_embedding in data_adaptor.get_embedding_names():
parameters["default_embedding"] = default_embedding
data_adaptor.update_parameters(parameters)
if annotation:
annotation.update_parameters(parameters, data_adaptor)
# gather it all together
c = {}
config = c["config"] = {}
config["features"] = features
config["displayNames"] = display_names
config["library_versions"] = library_versions
config["links"] = links
config["parameters"] = parameters
config["corpora_props"] = corpora_props
config["limits"] = {
"column_request_max": server_config.limits__column_request_max,
"diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
}
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
config["authentication"] = {
"requires_client_login": auth.requires_client_login(),
}
if auth.requires_client_login():
config["authentication"].update({
"login": auth.get_login_url(data_adaptor),
"logout": auth.get_logout_url(data_adaptor),
})
return c
def get_client_userinfo(self, data_adaptor):
"""
Return the userinfo as required by the /userinfo REST route
"""
server_config = self.server_config
dataset_config = data_adaptor.dataset_config
auth = server_config.auth
# make sure the configuration has been checked.
self.check_config()
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
userinfo = {}
userinfo["userinfo"] = {
"is_authenticated": auth.is_user_authenticated(),
"username": auth.get_user_name(),
"user_id": auth.get_user_id()
}
return userinfo
else:
return None
class BaseConfig(object):
"""This class handles the mechanics of updating and checking attributes.
Derived classes are expected to store the actual attributes"""
def __init__(self, app_config, default_config, dictval_cases={}):
# reference back to the app_config
self.app_config = app_config
# the complete set of attribute and their default values (unflattened)
self.default_config = default_config
# attributes where the value may be a dict (and therefore are not flattened)
self.dictval_cases = dictval_cases
# used to make sure every attribute value is checked
self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()}
def create_mapping(self, config):
"""Create a mapping from attribute names to (location in the config tree, value)"""
dc = copy.deepcopy(config)
mapping = {}
# special cases where the value could be a dict.
# If its value is not None, the entry is added to the mapping, and not included
# in the flattening below.
for dictval_case in self.dictval_cases:
cur = dc
for part in dictval_case[:-1]:
cur = cur.get(part, {})
val = cur.get(dictval_case[-1])
if val is not None:
key = "__".join(dictval_case)
mapping[key] = (dictval_case, val)
del cur[dictval_case[-1]]
flat_config = flatten(dc)
for key, value in flat_config.items():
# name of the attribute
attr = "__".join(key)
mapping[attr] = (key, value)
return mapping
def check_attr(self, attrname, vtype):
val = getattr(self, attrname)
if type(vtype) in (list, tuple):
if type(val) not in vtype:
tnames = ",".join([x.__name__ for x in vtype])
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
)
else:
if type(val) != vtype:
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, "
f"expected type {vtype.__name__}, got {type(val).__name__}"
)
self.attr_checked[attrname] = True
def check_config(self):
mapping = self.create_mapping(self.default_config)
for key in mapping.keys():
if not self.attr_checked[key]:
raise ConfigurationError(f"The attr '{key}' has not been checked")
def update(self, **kw):
for key, value in kw.items():
if not hasattr(self, key):
raise ConfigurationError(f"unknown config parameter {key}.")
try:
if type(value) == tuple:
# convert tuple values to list values
value = list(value)
setattr(self, key, value)
except KeyError:
raise ConfigurationError(f"Unable to set config parameter {key}.")
self.attr_checked[key] = False
def update_from_config(self, config, prefix):
mapping = self.create_mapping(config)
for attr, (key, value) in mapping.items():
if not hasattr(self, attr):
raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
try:
setattr(self, attr, value)
except KeyError:
raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}")
self.attr_checked[attr] = False
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
mapping = self.create_mapping(self.default_config)
diff = []
for attrname, (key, defval) in mapping.items():
curval = getattr(self, attrname)
if curval != defval:
diff.append((attrname, curval, defval))
return diff
class ServerConfig(BaseConfig):
"""Manages the config attribute associated with the server."""
def __init__(self, app_config, default_config):
dictval_cases = [
("app", "csp_directives"),
("authentication", "params_oauth", "cookie"),
("authentication", "params_oauth", "jwt_decode_options"),
("adaptor", "cxg_adaptor", "tiledb_ctx"),
("multi_dataset", "dataroot"),
]
super().__init__(app_config, default_config, dictval_cases)
dc = default_config
try:
self.app__verbose = dc["app"]["verbose"]
self.app__debug = dc["app"]["debug"]
self.app__host = dc["app"]["host"]
self.app__port = dc["app"]["port"]
self.app__open_browser = dc["app"]["open_browser"]
self.app__force_https = dc["app"]["force_https"]
self.app__flask_secret_key = dc["app"]["flask_secret_key"]
self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"]
self.app__server_timing_headers = dc["app"]["server_timing_headers"]
self.app__csp_directives = dc["app"]["csp_directives"]
self.app__api_base_url = dc["app"]["api_base_url"]
self.app__web_base_url = dc["app"]["web_base_url"]
self.authentication__type = dc["authentication"]["type"]
self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][
"oauth_api_base_url"
]
self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"]
self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"]
self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][
"jwt_decode_options"]
self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"]
self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"]
self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
self.multi_dataset__index = dc["multi_dataset"]["index"]
self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"]
self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"]
self.single_dataset__datapath = dc["single_dataset"]["datapath"]
self.single_dataset__obs_names = dc["single_dataset"]["obs_names"]
self.single_dataset__var_names = dc["single_dataset"]["var_names"]
self.single_dataset__about = dc["single_dataset"]["about"]
self.single_dataset__title = dc["single_dataset"]["title"]
self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"]
self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"]
self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"]
self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"]
self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"]
self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"]
self.limits__column_request_max = dc["limits"]["column_request_max"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The matrix data cache manager is created during the complete_config and stored here.
self.matrix_data_cache_manager = None
# The authentication object
self.auth = None
def complete_config(self, context):
self.handle_app(context)
self.handle_data_source(context)
self.handle_authentication(context)
self.handle_data_locator(context)
self.handle_adaptor(context) # may depend on data_locator
self.handle_single_dataset(context) # may depend on adaptor
self.handle_multi_dataset(context) # may depend on adaptor
self.handle_diffexp(context)
self.handle_limits(context)
self.check_config()
def handle_app(self, context):
self.check_attr("app__verbose", bool)
self.check_attr("app__debug", bool)
self.check_attr("app__host", str)
self.check_attr("app__port", (type(None), int))
self.check_attr("app__open_browser", bool)
self.check_attr("app__force_https", bool)
self.check_attr("app__flask_secret_key", (type(None), str))
self.check_attr("app__generate_cache_control_headers", bool)
self.check_attr("app__server_timing_headers", bool)
self.check_attr("app__csp_directives", (type(None), dict))
self.check_attr("app__api_base_url", (type(None), str))
self.check_attr("app__web_base_url", (type(None), str))
if self.app__port:
try:
if not is_port_available(self.app__host, self.app__port):
raise ConfigurationError(
f"The port selected {self.app__port} is in use, please configure an open port."
)
except OverflowError:
raise ConfigurationError(f"Invalid port: {self.app__port}")
else:
try:
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
except ValueError:
raise ConfigurationError(
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
)
try:
self.app__port = find_available_port(self.app__host, default_server_port)
except OverflowError:
raise ConfigurationError(f"Invalid port: {default_server_port}")
if self.app__debug:
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
self.app__verbose = True
self.app__open_browser = False
else:
warnings.formatwarning = custom_format_warning
if not self.app__verbose:
sys.tracebacklimit = 0
# secret key:
# first, from CXG_SECRET_KEY environment variable
# second, from config file
self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
# CSP Directives are a dict of string: list(string) or string: string
if self.app__csp_directives is not None:
for k, v in self.app__csp_directives.items():
if not isinstance(k, str):
raise ConfigurationError("CSP directive names must be a string.")
if isinstance(v, list):
for policy in v:
if not isinstance(policy, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
elif not isinstance(v, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
if self.app__web_base_url is None:
self.app__web_base_url = self.app__api_base_url
def handle_authentication(self, context):
self.check_attr("authentication__type", (type(None), str))
# oauth
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes)
self.check_attr("authentication__params_oauth__client_id", ptypes)
self.check_attr("authentication__params_oauth__client_secret", ptypes)
self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict))
self.check_attr("authentication__params_oauth__session_cookie", bool)
if self.authentication__params_oauth__session_cookie:
self.check_attr("authentication__params_oauth__cookie", (type(None), dict))
else:
self.check_attr("authentication__params_oauth__cookie", dict)
# secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
# second, from config file
self.authentication__params__oauth__client_secret = os.environ.get(
"CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret)
self.auth = AuthTypeFactory.create(self.authentication__type, self)
if self.auth is None:
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
def handle_data_locator(self, context):
self.check_attr("data_locator__s3__region_name", (type(None), bool, str))
if self.data_locator__s3__region_name is True:
path = self.single_dataset__datapath or self.multi_dataset__dataroot
if type(path) == dict:
# if multi_dataset__dataroot is a dict, then use the first key
# that is in s3. NOTE: it is not supported to have dataroots
# in different regions.
paths = [val.get("dataroot") for val in path.values()]
for path in paths:
if path.startswith("s3://"):
break
if path.startswith("s3://"):
region_name = discover_s3_region_name(path)
if region_name is None:
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
else:
region_name = None
self.data_locator__s3__region_name = region_name
def handle_data_source(self, context):
self.check_attr("single_dataset__datapath", (str, type(None)))
self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
if self.single_dataset__datapath is None:
if self.multi_dataset__dataroot is None:
# TODO: change the error message once dataroot is fully supported
raise ConfigurationError("missing datapath")
return
else:
if self.multi_dataset__dataroot is not None:
raise ConfigurationError("must supply only one of datapath or dataroot")
def handle_single_dataset(self, context):
self.check_attr("single_dataset__datapath", (str, type(None)))
self.check_attr("single_dataset__title", (str, type(None)))
self.check_attr("single_dataset__about", (str, type(None)))
self.check_attr("single_dataset__obs_names", (str, type(None)))
self.check_attr("single_dataset__var_names", (str, type(None)))
if self.single_dataset__datapath is None:
return
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
def handle_multi_dataset(self, context):
self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
self.check_attr("multi_dataset__index", (type(None), bool, str))
self.check_attr("multi_dataset__allowed_matrix_types", list)
self.check_attr("multi_dataset__matrix_cache__max_datasets", int)
self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float))
if self.multi_dataset__dataroot is None:
return
if type(self.multi_dataset__dataroot) == str:
default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
self.multi_dataset__dataroot = dict(d=default_dict)
for tag, dataroot_dict in self.multi_dataset__dataroot.items():
if "base_url" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
if "dataroot" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
base_url = dataroot_dict["base_url"]
# sanity check for well formed base urls
bad = False
if type(base_url) != str:
bad = True
elif os.path.normpath(base_url) != base_url:
bad = True
else:
base_url_parts = base_url.split("/")
if [quote_plus(part) for part in base_url_parts] != base_url_parts:
bad = True
if ".." in base_url_parts:
bad = True
if bad:
raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
# verify all the base_urls are unique
base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
if len(base_urls) > len(set(base_urls)):
raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
# error checking
for mtype in self.multi_dataset__allowed_matrix_types:
try:
MatrixDataType(mtype)
except ValueError:
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(
max_cached=self.multi_dataset__matrix_cache__max_datasets,
timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
)
def handle_diffexp(self, context):
self.check_attr("diffexp__alg_cxg__max_workers", (str, int))
self.check_attr("diffexp__alg_cxg__cpu_multiplier", int)
self.check_attr("diffexp__alg_cxg__target_workunit", int)
max_workers = self.diffexp__alg_cxg__max_workers
cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
cpu_count = os.cpu_count()
max_workers = min(max_workers, cpu_multiplier * cpu_count)
diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
def handle_adaptor(self, context):
# cxg
self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict)
regionkey = "vfs.s3.region"
if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
if type(self.data_locator__s3__region_name) == str:
self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
from server.data_cxg.cxg_adaptor import CxgAdaptor
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
# anndata
self.check_attr("adaptor__anndata_adaptor__backed", bool)
def handle_limits(self, context):
self.check_attr("limits__diffexp_cellcount_max", (type(None), int))
self.check_attr("limits__column_request_max", (type(None), int))
def exceeds_limit(self, limit_name, value):
limit_value = getattr(self, "limits__" + limit_name, None)
if limit_value is None: # disabled
return False
return value > limit_value
def get_api_base_url(self):
if self.app__api_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__api_base_url and self.app__api_base_url.endswith("/"):
return self.app__api_base_url[:-1]
return self.app__api_base_url
def get_web_base_url(self):
if self.app__web_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__web_base_url is None:
return self.get_api_base_url()
if self.app__web_base_url.endswith("/"):
return self.app__web_base_url[:-1]
return self.api__web_base_url
class DatasetConfig(BaseConfig):
"""Manages the config attribute associated with a dataset."""
def __init__(self, tag, app_config, default_config):
super().__init__(app_config, default_config)
self.tag = tag
dc = default_config
try:
self.app__scripts = dc["app"]["scripts"]
self.app__inline_scripts = dc["app"]["inline_scripts"]
self.app__about_legal_tos = dc["app"]["about_legal_tos"]
self.app__about_legal_privacy = dc["app"]["about_legal_privacy"]
self.app__authentication_enable = dc["app"]["authentication_enable"]
self.presentation__max_categories = dc["presentation"]["max_categories"]
self.presentation__custom_colors = dc["presentation"]["custom_colors"]
self.user_annotations__enable = dc["user_annotations"]["enable"]
self.user_annotations__type = dc["user_annotations"]["type"]
self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"]
self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ] # noqa E501
self.embeddings__names = dc["embeddings"]["names"]
self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
self.diffexp__enable = dc["diffexp"]["enable"]
self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = dc["diffexp"]["top_n"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The annotation object is created during complete_config and stored here.
self.user_annotations = None
def complete_config(self, context):
self.handle_app(context)
self.handle_presentation(context)
self.handle_user_annotations(context)
self.handle_embeddings(context)
self.handle_diffexp(context)
def handle_app(self, context):
self.check_attr("app__scripts", list)
self.check_attr("app__inline_scripts", list)
self.check_attr("app__about_legal_tos", (type(None), str))
self.check_attr("app__about_legal_privacy", (type(None), str))
self.check_attr("app__authentication_enable", bool)
# scripts can be string (filename) or dict (attributes). Convert string to dict.
scripts = []
for s in self.app__scripts:
if isinstance(s, str):
scripts.append({"src": s})
elif isinstance(s, dict) and isinstance(s["src"], str):
scripts.append(s)
else:
raise ConfigurationError("Scripts must be string or dict")
self.app__scripts = scripts
def handle_presentation(self, context):
self.check_attr("presentation__max_categories", int)
self.check_attr("presentation__custom_colors", bool)
def handle_user_annotations(self, context):
self.check_attr("user_annotations__enable", bool)
self.check_attr("user_annotations__type", str)
self.check_attr("user_annotations__local_file_csv__directory", (type(None), str))
self.check_attr("user_annotations__local_file_csv__file", (type(None), str))
self.check_attr("user_annotations__ontology__enable", bool)
self.check_attr("user_annotations__ontology__obo_location", (type(None), str))
self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str))
self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str))
if self.user_annotations__enable:
server_config = self.app_config.server_config
if not self.app__authentication_enable:
raise ConfigurationError("user annotations requires authentication to be enabled")
if not server_config.auth.is_valid_authentication_type():
auth_type = server_config.authentication__type
raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
# TODO, replace this with a factory pattern once we have more than one way
# to do annotations. currently only local_file_csv
if self.user_annotations__type == "local_file_csv":
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None and dirname is not None:
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
if filename is not None:
lf_name, lf_ext = splitext(filename)
if lf_ext and lf_ext != ".csv":
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
if dirname is not None and not isdir(dirname):
try:
os.mkdir(dirname)
except OSError:
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
self.user_annotations = AnnotationsLocalFile(dirname, filename)
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
server_config = self.app_config.server_config
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
try:
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
except OntologyLoadFailure as e:
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
elif self.user_annotations__type == "hosted_tiledb_array":
self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str)
self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str)
self.user_annotations = AnnotationsHostedTileDB(
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
)
else:
raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
else:
if self.user_annotations__type == "local_file_csv":
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None:
context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
if dirname is not None:
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
if self.user_annotations__ontology__enable:
context["messagefn"](
"Warning: --experimental-annotations-ontology" " ignored as annotations are disabled."
)
if self.user_annotations__ontology__obo_location is not None:
context["messagefn"](
"Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled."
)
def handle_embeddings(self, context):
self.check_attr("embeddings__names", list)
self.check_attr("embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
matrix_data_loader = MatrixDataLoader(
server_config.single_dataset__datapath, app_config=self.app_config
)
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
try:
server.compute.scanpy.get_scanpy_module()
except NotImplementedError:
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
def handle_diffexp(self, context):
self.check_attr("diffexp__enable", bool)
self.check_attr("diffexp__lfc_cutoff", float)
self.check_attr("diffexp__top_n", int)
server_config = self.app_config.server_config
if server_config.single_dataset__datapath:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
context["messagefn"](
"CAUTION: due to the size of your dataset, "
"running differential expression may take longer or fail."
)
server/common/aws_secret_utils.py
-61
View File
@@ -1,72 +1,11 @@
import logging
import os
import sys
import boto3
from flask import json
from server.common.data_locator import discover_s3_region_name
from server.common.errors import SecretKeyRetrievalError
def handle_config_from_secret(app_config):
"""Update configuration from the secret manager"""
secret_name = os.getenv("CXG_AWS_SECRET_NAME")
if not secret_name:
return
# need to find the secret manager region.
# 1. from CXG_AWS_SECRET_REGION_NAME
# 2. discover from dataroot location (if on s3)
# 3. discover from config file location (if on s3)
secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
if secret_region_name is None:
secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
if not secret_region_name:
from server.eb.app import config_file
secret_region_name = discover_s3_region_name(config_file)
if not secret_region_name:
logging.error("Could not determine the AWS Secret Manager region")
sys.exit(1)
secrets = get_secret_key(secret_region_name, secret_name)
if not secrets:
return
server_attrs = (
("flask_secret_key", "app__flask_secret_key"),
("oauth_client_secret", "authentication__params_oauth__client_secret"),
)
default_dataset_attrs = (
("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),
)
# update server configuration attributes
for key, attr in server_attrs:
cur_val = getattr(app_config.server_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_server_config(**{attr : val})
# update default dataset configuration attributes
for key, attr in default_dataset_attrs:
cur_val = getattr(app_config.default_dataset_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_default_dataset_config(**{attr : val})
def get_secret_key(region_name, secret_name):
session = boto3.session.Session()
client = session.client(service_name="secretsmanager", region_name=region_name)
server/common/config/__init__.py
+66
View File
@@ -0,0 +1,66 @@
import logging
import os
import sys
from server.common.aws_secret_utils import get_secret_key
from server.common.data_locator import discover_s3_region_name
DEFAULT_SERVER_PORT = 5005
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
def handle_config_from_secret(app_config):
"""Update configuration from the secret manager"""
secret_name = os.getenv("CXG_AWS_SECRET_NAME")
if not secret_name:
return
# need to find the secret manager region.
# 1. from CXG_AWS_SECRET_REGION_NAME
# 2. discover from dataroot location (if on s3)
# 3. discover from config file location (if on s3)
secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
if secret_region_name is None:
secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
if not secret_region_name:
from server.eb.app import config_file
secret_region_name = discover_s3_region_name(config_file)
if not secret_region_name:
logging.error("Could not determine the AWS Secret Manager region")
sys.exit(1)
secrets = get_secret_key(secret_region_name, secret_name)
if not secrets:
return
server_attrs = (
("flask_secret_key", "app__flask_secret_key"),
("oauth_client_secret", "authentication__params_oauth__client_secret"),
)
default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),)
# update server configuration attributes
for key, attr in server_attrs:
cur_val = getattr(app_config.server_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_server_config(**{attr: val})
# update default dataset configuration attributes
for key, attr in default_dataset_attrs:
cur_val = getattr(app_config.default_dataset_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_default_dataset_config(**{attr: val})
server/common/config/app_config.py
+183
View File
@@ -0,0 +1,183 @@
import yaml
from flatten_dict import unflatten
from server.default_config import get_default_config
from server.common.config.dataset_config import DatasetConfig
from server.common.config.server_config import ServerConfig
from server.common.errors import ConfigurationError
class AppConfig(object):
"""
AppConfig stores all the configuration for cellxgene.
AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
The server_config contains attributes that refer to the server process as a whole.
The default_dataset_config refers to attributes that are associated with the features and
presentations of a dataset.
The dataset config attributes can be overridden depending on the url by which the
dataset was accessed. These are stored in dataroot_config.
AppConfig has methods to initialize, modify, and access the configuration.
"""
def __init__(self):
# the default configuration (see default_config.py)
# TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
# defaults within the config class?
self.default_config = get_default_config()
# the server configuration
self.server_config = ServerConfig(self, self.default_config["server"])
# the dataset config, unless overridden by an entry in dataroot_config
self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
# a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
# attribute of the server_config. The default dataset config will apply to all datasets unless a different set
# of config vars was passed for a specific dataset under the multidataset config. For example:
"""
per_dataset_config:
d1:
user_annotations:
enable: false
d2:
user_annotations:
enable: true
"""
# dataroot config
self.dataroot_config = {}
# Set to true when config_completed is called
self.is_completed = False
def get_dataset_config(self, dataroot_key):
if self.server_config.single_dataset__datapath:
return self.default_dataset_config
else:
return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
def check_config(self):
"""Verify all the attributes in the config have been type checked"""
if not self.is_completed:
raise ConfigurationError("The configuration has not been completed")
self.server_config.check_config()
self.default_dataset_config.check_config()
for dataset_config in self.dataroot_config.values():
dataset_config.check_config()
def update_server_config(self, **kw):
self.server_config.update(**kw)
self.is_complete = False
def update_default_dataset_config(self, **kw):
self.default_dataset_config.update(**kw)
# update all the other dataset configs, if any
for value in self.dataroot_config.values():
value.update(**kw)
self.is_complete = False
def update_from_config_file(self, config_file):
try:
with open(config_file) as yml_file:
config = yaml.safe_load(yml_file)
except yaml.YAMLError as e:
raise ConfigurationError(f"The specified config file contained an error: {e}")
except OSError as e:
raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
if config.get("server"):
self.server_config.update_from_config(config["server"], "server")
if config.get("dataset"):
self.default_dataset_config.update_from_config(config["dataset"], "dataset")
per_dataset_config = config.get("per_dataset_config", {})
for key, dataroot_config in per_dataset_config.items():
# first create and initialize the dataroot with the default config
self.add_dataroot_config(key, **config["dataset"])
# then apply the per dataset configuration
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
self.is_complete = False
def write_config(self, config_file):
"""output the config to a yaml file"""
server = self.server_config.create_mapping(self.server_config.default_config)
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = dict(server={}, dataset={})
for attrname in server.keys():
config["server__" + attrname] = getattr(self.server_config, attrname)
for attrname in dataset.keys():
config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
if self.dataroot_config:
config["per_dataset_config"] = {}
for dataroot_tag, dataroot_config in self.dataroot_config.items():
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
for attrname in dataset.keys():
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
config = unflatten(config, splitter=lambda key: key.split("__"))
yaml.dump(config, open(config_file, "w"))
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
diff_server = self.server_config.changes_from_default()
diff_dataset = self.default_dataset_config.changes_from_default()
diff = dict(server=diff_server, dataset=diff_dataset)
return diff
def add_dataroot_config(self, dataroot_tag, **kw):
"""Create a new dataset config object based on the default dataset config, and kw parameters"""
if dataroot_tag in self.dataroot_config:
raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
if type(self.server_config.multi_dataset__dataroot) != dict:
raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
if dataroot_tag not in self.server_config.multi_dataset__dataroot:
raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
self.is_completed = False
self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = {key: value[1] for key, value in flat_config.items()}
self.dataroot_config[dataroot_tag].update(**config)
self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
def complete_config(self, messagefn=None):
"""The configure options are checked, and any additional setup based on the config
parameters is done"""
if messagefn is None:
def noop(message):
pass
messagefn = noop
# TODO: to give better error messages we can add a mapping between where each config
# attribute originated (e.g. command line argument or config file), then in the error
# messages we can give correct context for attributes with bad value.
context = dict(messagefn=messagefn)
self.server_config.complete_config(context)
self.default_dataset_config.complete_config(context)
for dataroot_config in self.dataroot_config.values():
dataroot_config.complete_config(context)
self.is_completed = True
self.check_config()
def get_matrix_data_cache_manager(self):
return self.server_config.matrix_data_cache_manager
def is_multi_dataset(self):
return self.server_config.multi_dataset__dataroot is not None
def get_title(self, data_adaptor):
return (
self.server_config.single_dataset__title
if self.server_config.single_dataset__title
else data_adaptor.get_title()
)
def get_about(self, data_adaptor):
return (
self.server_config.single_dataset__about
if self.server_config.single_dataset__about
else data_adaptor.get_about()
)
server/common/config/base_config.py
+113
View File
@@ -0,0 +1,113 @@
import copy
from flatten_dict import flatten
from server.common.errors import ConfigurationError
class BaseConfig(object):
"""
This class handles the mechanics of updating and checking attributes.
Derived classes are expected to store the actual attributes
Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
"""
def __init__(self, app_config, default_config, dictval_cases={}):
# reference back to the app_config
self.app_config = app_config
# the complete set of attributes and their default values (unflattened)
self.default_config = default_config
# attributes where the value may be a dict (and therefore are not flattened)
self.dictval_cases = dictval_cases
# used to make sure every attribute value is checked
self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
def create_mapping(self, config):
"""
Create a dictionary where the keys are the name of attributes (using double underscore convention)
For example: authentication__type
The values are a tuple,
- the first item of the tuple is a tuple of path elements (location in config 'tree')
- the second item is the value of the config parameter
For example: (('authentication', 'type'), 'session'))
"""
config_copy = copy.deepcopy(config)
mapping = {}
# special cases where the value could be a dict.
# If its value is not None, the entry is added to the mapping, and not included
# in the flattening below.
for dictval_case in self.dictval_cases:
cur = config_copy
for part in dictval_case[:-1]:
cur = cur.get(part, {})
val = cur.get(dictval_case[-1])
if val is not None:
key = "__".join(dictval_case)
mapping[key] = (dictval_case, val)
del cur[dictval_case[-1]]
flat_config = flatten(config_copy)
for key, value in flat_config.items():
# name of the attribute
attr = "__".join(key)
mapping[attr] = (key, value)
return mapping
def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
val = getattr(self, attrname)
if type(vtype) in (list, tuple):
if type(val) not in vtype:
tnames = ",".join([x.__name__ for x in vtype])
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
)
else:
if type(val) != vtype:
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, "
f"expected type {vtype.__name__}, got {type(val).__name__}"
)
self.attr_checked[attrname] = True
def check_config(self):
mapping = self.create_mapping(self.default_config)
for key in mapping.keys():
if not self.attr_checked[key]:
raise ConfigurationError(f"The attr '{key}' has not been checked")
def update(self, **kw):
for key, value in kw.items():
if not hasattr(self, key):
raise ConfigurationError(f"unknown config parameter {key}.")
try:
if type(value) == tuple:
# convert tuple values to list values
value = list(value)
setattr(self, key, value)
except KeyError:
raise ConfigurationError(f"Unable to set config parameter {key}.")
self.attr_checked[key] = False
def update_from_config(self, config, prefix):
mapping = self.create_mapping(config)
for attr, (key, value) in mapping.items():
if not hasattr(self, attr):
raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
setattr(self, attr, value)
self.attr_checked[attr] = False
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
mapping = self.create_mapping(self.default_config)
diff = []
for attrname, (key, defval) in mapping.items():
curval = getattr(self, attrname)
if curval != defval:
diff.append((attrname, curval, defval))
return diff
server/common/config/client_config.py
+121
View File
@@ -0,0 +1,121 @@
from server import display_version as cellxgene_display_version
def get_client_config(app_config, data_adaptor):
"""
Return the configuration as required by the /config REST route
"""
server_config = app_config.server_config
dataset_config = data_adaptor.dataset_config
annotation = dataset_config.user_annotations
auth = server_config.auth
# FIXME The current set of config is not consistently presented:
# we have camalCase, hyphen-text, and underscore_text
# make sure the configuration has been checked.
app_config.check_config()
# display_names
title = app_config.get_title(data_adaptor)
about = app_config.get_about(data_adaptor)
display_names = dict(engine=data_adaptor.get_name(), dataset=title)
# library_versions
library_versions = {}
library_versions.update(data_adaptor.get_library_versions())
library_versions["cellxgene"] = cellxgene_display_version
# links
links = {"about-dataset": about}
# parameters
parameters = {
"layout": dataset_config.embeddings__names,
"max-category-items": dataset_config.presentation__max_categories,
"obs_names": server_config.single_dataset__obs_names,
"var_names": server_config.single_dataset__var_names,
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
"backed": server_config.adaptor__anndata_adaptor__backed,
"disable-diffexp": not dataset_config.diffexp__enable,
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
"annotations_cell_ontology_enabled": False,
"annotations_cell_ontology_obopath": None,
"annotations_cell_ontology_terms": None,
"custom_colors": dataset_config.presentation__custom_colors,
"diffexp-may-be-slow": False,
"about_legal_tos": dataset_config.app__about_legal_tos,
"about_legal_privacy": dataset_config.app__about_legal_privacy,
}
# corpora dataset_props
# TODO/Note: putting info from the dataset into the /config is not ideal.
# However, it is definitely not part of /schema, and we do not have a top-level
# route for data properties. Consider creating one at some point.
corpora_props = data_adaptor.get_corpora_props()
if corpora_props and "default_embedding" in corpora_props:
default_embedding = corpora_props["default_embedding"]
if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
default_embedding = default_embedding[2:] # drop X_ prefix
if default_embedding in data_adaptor.get_embedding_names():
parameters["default_embedding"] = default_embedding
data_adaptor.update_parameters(parameters)
if annotation:
annotation.update_parameters(parameters, data_adaptor)
# gather it all together
client_config = {}
config = client_config["config"] = {}
config["displayNames"] = display_names
config["library_versions"] = library_versions
config["links"] = links
config["parameters"] = parameters
config["corpora_props"] = corpora_props
config["limits"] = {
"column_request_max": server_config.limits__column_request_max,
"diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
}
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
config["authentication"] = {
"requires_client_login": auth.requires_client_login(),
}
if auth.requires_client_login():
config["authentication"].update(
{
# Todo why are these stored on the data_adaptor?
"login": auth.get_login_url(data_adaptor),
"logout": auth.get_logout_url(data_adaptor),
}
)
return client_config
def get_client_userinfo(app_config, data_adaptor):
"""
Return the userinfo as required by the /userinfo REST route
"""
server_config = app_config.server_config
dataset_config = data_adaptor.dataset_config
auth = server_config.auth
# make sure the configuration has been checked.
app_config.check_config()
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
userinfo = {}
userinfo["userinfo"] = {
"is_authenticated": auth.is_user_authenticated(),
"username": auth.get_user_name(),
"user_id": auth.get_user_id(),
"email": auth.get_user_email(),
}
return userinfo
server/common/config/dataset_config.py
+234
View File
@@ -0,0 +1,234 @@
import os
from os.path import splitext, isdir
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from server.common.annotations.local_file_csv import AnnotationsLocalFile
from server.common.config.base_config import BaseConfig
from server.common.errors import ConfigurationError, OntologyLoadFailure
from server.compute.scanpy import get_scanpy_module
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
from server.db.db_utils import DbUtils
class DatasetConfig(BaseConfig):
"""Manages the config attribute associated with a dataset."""
def __init__(self, tag, app_config, default_config):
super().__init__(app_config, default_config)
self.tag = tag
try:
self.app__scripts = default_config["app"]["scripts"]
self.app__inline_scripts = default_config["app"]["inline_scripts"]
self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
self.app__authentication_enable = default_config["app"]["authentication_enable"]
self.presentation__max_categories = default_config["presentation"]["max_categories"]
self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
self.user_annotations__enable = default_config["user_annotations"]["enable"]
self.user_annotations__type = default_config["user_annotations"]["type"]
self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
"directory"
] # noqa E501
self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
"obo_location"
] # noqa E501
self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
"hosted_tiledb_array"
][
"db_uri"
] # noqa E501
self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
"hosted_tiledb_array"
][
"hosted_file_directory"
] # noqa E501
self.embeddings__names = default_config["embeddings"]["names"]
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
self.diffexp__enable = default_config["diffexp"]["enable"]
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = default_config["diffexp"]["top_n"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The annotation object is created during complete_config and stored here.
self.user_annotations = None
def complete_config(self, context):
self.handle_app()
self.handle_presentation()
self.handle_user_annotations(context)
self.handle_embeddings()
self.handle_diffexp(context)
def handle_app(self):
self.validate_correct_type_of_configuration_attribute("app__scripts", list)
self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
# scripts can be string (filename) or dict (attributes). Convert string to dict.
scripts = []
for script in self.app__scripts:
try:
if isinstance(script, str):
scripts.append({"src": script})
elif isinstance(script, dict) and isinstance(script["src"], str):
scripts.append(script)
else:
raise Exception
except Exception as e:
raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
self.app__scripts = scripts
def handle_presentation(self):
self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
def handle_user_annotations(self, context):
self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__directory", (type(None), str)
) # noqa E501
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__file", (type(None), str)
) # noqa E501
self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__ontology__obo_location", (type(None), str)
) # noqa E501
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
) # noqa E501
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
) # noqa E501
if self.user_annotations__enable:
server_config = self.app_config.server_config
if not self.app__authentication_enable:
raise ConfigurationError("user annotations requires authentication to be enabled")
if not server_config.auth.is_valid_authentication_type():
auth_type = server_config.authentication__type
raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
if self.user_annotations__type == "local_file_csv":
self.handle_local_file_csv_annotations()
elif self.user_annotations__type == "hosted_tiledb_array":
self.handle_hosted_tiledb_annotations()
else:
raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
try:
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
except OntologyLoadFailure as e:
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
else:
self.check_annotation_config_vars_not_set(context)
def handle_local_file_csv_annotations(self):
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None and dirname is not None:
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
if filename is not None:
lf_name, lf_ext = splitext(filename)
if lf_ext and lf_ext != ".csv":
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
if dirname is not None and not isdir(dirname):
try:
os.mkdir(dirname)
except OSError:
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
self.user_annotations = AnnotationsLocalFile(dirname, filename)
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
server_config = self.app_config.server_config
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
def handle_hosted_tiledb_annotations(self):
self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", str
) # noqa E501
self.user_annotations = AnnotationsHostedTileDB(
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
)
def check_annotation_config_vars_not_set(self, context):
if self.user_annotations__type is not None:
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
db_uri = self.user_annotations__hosted_tiledb_array__db_uri
hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
if filename is not None:
context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
if dirname is not None:
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
if db_uri is not None:
context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
if hosted_file_dirname is not None:
context["messagefn"](
"Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
)
if self.user_annotations__ontology__enable:
context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
if self.user_annotations__ontology__obo_location is not None:
context["messagefn"](
"Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
)
def handle_embeddings(self):
self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
matrix_data_loader = MatrixDataLoader(
server_config.single_dataset__datapath, app_config=self.app_config
)
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
try:
get_scanpy_module()
except NotImplementedError:
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
def handle_diffexp(self, context):
self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
server_config = self.app_config.server_config
if server_config.single_dataset__datapath:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
context["messagefn"](
"CAUTION: due to the size of your dataset, "
"running differential expression may take longer or fail."
)
server/common/config/server_config.py
+390
View File
@@ -0,0 +1,390 @@
import os
import sys
import warnings
from os.path import basename
from urllib.parse import urlparse, quote_plus
from server.auth.auth import AuthTypeFactory
from server.common.config.base_config import BaseConfig
from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
from server.common.errors import ConfigurationError, DatasetAccessError
from server.common.data_locator import discover_s3_region_name
from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
from server.compute import diffexp_cxg as diffexp_tiledb
from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
class ServerConfig(BaseConfig):
"""Manages the config attribute associated with the server."""
def __init__(self, app_config, default_config):
dictval_cases = [
("app", "csp_directives"),
("authentication", "params_oauth", "cookie"),
("authentication", "params_oauth", "jwt_decode_options"),
("adaptor", "cxg_adaptor", "tiledb_ctx"),
("multi_dataset", "dataroot"),
]
super().__init__(app_config, default_config, dictval_cases)
try:
self.app__verbose = default_config["app"]["verbose"]
self.app__debug = default_config["app"]["debug"]
self.app__host = default_config["app"]["host"]
self.app__port = default_config["app"]["port"]
self.app__open_browser = default_config["app"]["open_browser"]
self.app__force_https = default_config["app"]["force_https"]
self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
self.app__csp_directives = default_config["app"]["csp_directives"]
self.app__api_base_url = default_config["app"]["api_base_url"]
self.app__web_base_url = default_config["app"]["web_base_url"]
self.authentication__type = default_config["authentication"]["type"]
self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
"oauth_api_base_url"
] # noqa E501
self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
"client_secret"
] # noqa E501
self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
"jwt_decode_options"
] # noqa E501
self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
"session_cookie"
] # noqa E501
self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
self.multi_dataset__index = default_config["multi_dataset"]["index"]
self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
"max_datasets"
] # noqa E501
self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
"timelimit_s"
] # noqa E501
self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
self.single_dataset__about = default_config["single_dataset"]["about"]
self.single_dataset__title = default_config["single_dataset"]["title"]
self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
self.limits__column_request_max = default_config["limits"]["column_request_max"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The matrix data cache manager is created during the complete_config and stored here.
self.matrix_data_cache_manager = None
# The authentication object
self.auth = None
def complete_config(self, context):
self.handle_app(context)
self.handle_data_source()
self.handle_authentication()
self.handle_data_locator()
self.handle_adaptor() # may depend on data_locator
self.handle_single_dataset(context) # may depend on adaptor
self.handle_multi_dataset() # may depend on adaptor
self.handle_diffexp()
self.handle_limits()
self.check_config()
def handle_app(self, context):
self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
self.validate_correct_type_of_configuration_attribute("app__debug", bool)
self.validate_correct_type_of_configuration_attribute("app__host", str)
self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
if self.app__port:
try:
if not is_port_available(self.app__host, self.app__port):
raise ConfigurationError(
f"The port selected {self.app__port} is in use, please configure an open port."
)
except OverflowError:
raise ConfigurationError(f"Invalid port: {self.app__port}")
else:
try:
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
except ValueError:
raise ConfigurationError(
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
)
try:
self.app__port = find_available_port(self.app__host, default_server_port)
except OverflowError:
raise ConfigurationError(f"Invalid port: {default_server_port}")
if self.app__debug:
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
self.app__verbose = True
self.app__open_browser = False
else:
warnings.formatwarning = custom_format_warning
if not self.app__verbose:
sys.tracebacklimit = 0
# secret key:
# first, from CXG_SECRET_KEY environment variable
# second, from config file
self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
# CSP Directives are a dict of string: list(string) or string: string
if self.app__csp_directives is not None:
for k, v in self.app__csp_directives.items():
if not isinstance(k, str):
raise ConfigurationError("CSP directive names must be a string.")
if isinstance(v, list):
for policy in v:
if not isinstance(policy, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
elif not isinstance(v, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
if self.app__web_base_url is None:
self.app__web_base_url = self.app__api_base_url
def handle_authentication(self):
self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
# oauth
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__oauth_api_base_url", ptypes
) # noqa E501
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__jwt_decode_options", (type(None), dict)
) # noqa E501
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
if self.authentication__params_oauth__session_cookie:
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__cookie", (type(None), dict)
) # noqa E501
else:
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
# secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
# second, from config file
self.authentication__params_oauth__client_secret = os.environ.get(
"CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret
)
self.auth = AuthTypeFactory.create(self.authentication__type, self)
if self.auth is None:
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
def handle_data_locator(self):
self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
if self.data_locator__s3__region_name is True:
path = self.single_dataset__datapath or self.multi_dataset__dataroot
if type(path) == dict:
# if multi_dataset__dataroot is a dict, then use the first key
# that is in s3. NOTE: it is not supported to have dataroots
# in different regions.
paths = [val.get("dataroot") for val in path.values()]
for path in paths:
if path.startswith("s3://"):
break
if path.startswith("s3://"):
region_name = discover_s3_region_name(path)
if region_name is None:
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
else:
region_name = None
self.data_locator__s3__region_name = region_name
def handle_data_source(self):
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
if self.single_dataset__datapath and self.multi_dataset__dataroot:
raise ConfigurationError(
"You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
)
if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
def handle_single_dataset(self, context):
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
if self.single_dataset__datapath is None:
return
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
def handle_multi_dataset(self):
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
self.validate_correct_type_of_configuration_attribute(
"multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
) # noqa E501
if self.multi_dataset__dataroot is None:
return
if type(self.multi_dataset__dataroot) == str:
default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
self.multi_dataset__dataroot = dict(d=default_dict)
for tag, dataroot_dict in self.multi_dataset__dataroot.items():
if "base_url" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
if "dataroot" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
base_url = dataroot_dict["base_url"]
# sanity check for well formed base urls
bad = False
if type(base_url) != str:
bad = True
elif os.path.normpath(base_url) != base_url:
bad = True
else:
base_url_parts = base_url.split("/")
if [quote_plus(part) for part in base_url_parts] != base_url_parts:
bad = True
if ".." in base_url_parts:
bad = True
if bad:
raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
# verify all the base_urls are unique
base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
if len(base_urls) > len(set(base_urls)):
raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
# error checking
for mtype in self.multi_dataset__allowed_matrix_types:
try:
MatrixDataType(mtype)
except ValueError:
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(
max_cached=self.multi_dataset__matrix_cache__max_datasets,
timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
)
def handle_diffexp(self):
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
max_workers = self.diffexp__alg_cxg__max_workers
cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
cpu_count = os.cpu_count()
max_workers = min(max_workers, cpu_multiplier * cpu_count)
diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
def handle_adaptor(self):
# cxg
self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
regionkey = "vfs.s3.region"
if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
if type(self.data_locator__s3__region_name) == str:
self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
from server.data_cxg.cxg_adaptor import CxgAdaptor
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
# anndata
self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
def handle_limits(self):
self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
def exceeds_limit(self, limit_name, value):
limit_value = getattr(self, "limits__" + limit_name, None)
if limit_value is None: # disabled
return False
return value > limit_value
def get_api_base_url(self):
if self.app__api_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__api_base_url and self.app__api_base_url.endswith("/"):
return self.app__api_base_url[:-1]
return self.app__api_base_url
def get_web_base_url(self):
if self.app__web_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__web_base_url is None:
return self.get_api_base_url()
if self.app__web_base_url.endswith("/"):
return self.app__web_base_url[:-1]
return self.app__web_base_url
server/common/errors.py
+4 -4
View File
@@ -42,14 +42,14 @@ define_request_exception(
define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
define_request_exception(
"AuthenticationError",
"Raised when there is an authentication error",
default_status_code=HTTPStatus.UNAUTHORIZED)
"AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
)
define_request_exception(
"AnnotationCategoryNameError",
"Raised when an annotation category name cant be saved",
default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY)
default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
)
define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
define_exception("ConfigurationError", "Raised when checking configuration errors")
server/common/rest.py
+3 -2
View File
@@ -6,6 +6,7 @@ from http import HTTPStatus
from flask import make_response, jsonify, current_app, abort
from werkzeug.urls import url_unquote
from server.common.config.client_config import get_client_config, get_client_userinfo
from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
from server.common.errors import (
FilterError,
@@ -117,12 +118,12 @@ def schema_get(data_adaptor):
def config_get(app_config, data_adaptor):
config = app_config.get_client_config(data_adaptor)
config = get_client_config(app_config, data_adaptor)
return make_response(jsonify(config), HTTPStatus.OK)
def userinfo_get(app_config, data_adaptor):
config = app_config.get_client_userinfo(data_adaptor)
config = get_client_userinfo(app_config, data_adaptor)
return make_response(jsonify(config), HTTPStatus.OK)
server/common/utils/cxg_generation_utils.py
+1 -1
View File
@@ -111,7 +111,7 @@ def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
def convert_matrix_to_cxg_array(
matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
):
"""
Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
server/common/utils/matrix_utils.py
+7 -4
View File
@@ -41,16 +41,19 @@ def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
number_of_non_zero_elements += np.count_nonzero(matrix_subset)
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
if end_row_index != total_number_of_rows:
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / (
end_row_index * total_number_of_columns)
percentage_of_non_zero_elements = (
100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
)
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
f"{percentage_of_non_zero_elements:6.2f}")
f"{percentage_of_non_zero_elements:6.2f}"
)
else:
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (exact): "
f"{percentage_of_non_zero_elements:6.2f}")
f"{percentage_of_non_zero_elements:6.2f}"
)
return False
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
server/common/utils/type_conversion_utils.py
+13 -7
View File
@@ -9,8 +9,10 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
schema_type_hints_by_column_name = {}
for column_name, column_values in dataframe.items():
dtypes_by_column_name[column_name], schema_type_hints_by_column_name[column_name] = \
get_dtype_and_schema_of_array(column_values)
(
dtypes_by_column_name[column_name],
schema_type_hints_by_column_name[column_name],
) = get_dtype_and_schema_of_array(column_values)
return dtypes_by_column_name, schema_type_hints_by_column_name
@@ -24,8 +26,10 @@ def get_schema_type_hint_of_array(array: pd.Series):
def get_dtype_and_schema_of_array(array: pd.Series):
return (get_dtype_from_dtype(array.dtype, array_values=array),
get_schema_type_hint_from_dtype(array.dtype, array_values=array))
return (
get_dtype_from_dtype(array.dtype, array_values=array),
get_schema_type_hint_from_dtype(array.dtype, array_values=array),
)
def get_dtype_from_dtype(dtype, array_values=None):
@@ -133,9 +137,11 @@ def can_cast_to_int32(dtype, array_values=None):
if np.can_cast(dtype, np.int32):
return True
ii32 = np.iinfo(np.int32)
if not ordered_array_values.empty and (
ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \
ordered_array_values.empty:
if (
not ordered_array_values.empty
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
or ordered_array_values.empty
):
return True
return False
server/converters/h5ad_data_file.py
+8 -8
View File
@@ -24,14 +24,14 @@ class H5ADDataFile:
another format (currently just CXG is supported). """
def __init__(
self,
input_filename,
backed=False,
dataset_title=None,
dataset_about=None,
obs_index_column_name=None,
vars_index_column_name=None,
use_corpora_schema=True,
self,
input_filename,
backed=False,
dataset_title=None,
dataset_about=None,
obs_index_column_name=None,
vars_index_column_name=None,
use_corpora_schema=True,
):
self.input_filename = input_filename
self.backed = backed
server/data_common/data_adaptor.py
+3 -14
View File
@@ -5,7 +5,7 @@ import numpy as np
import pandas as pd
from server_timing import Timing as ServerTiming
from server.common.app_config import AppFeature, AppConfig
from server.common.config.app_config import AppConfig
from server.common.constants import Axis
from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
from server.common.utils.utils import jsonify_numpy
@@ -155,17 +155,6 @@ class DataAdaptor(metaclass=ABCMeta):
"""
pass
def get_features(self, annotations=None):
"""Return list of features, to return as part of the config route"""
features = [
AppFeature("/cluster/", method="POST", available=False),
AppFeature("/layout/obs", method="GET", available=self.get_embedding_names() is not None),
AppFeature("/layout/obs", method="PUT", available=self.dataset_config.embeddings__enable_reembedding),
AppFeature("/diffexp/", method="POST", available=self.dataset_config.diffexp__enable),
AppFeature("/annotations/obs", method="PUT", available=annotations is not None),
]
return features
def update_parameters(self, parameters):
parameters.update(self.parameters)
@@ -173,7 +162,7 @@ class DataAdaptor(metaclass=ABCMeta):
mask = np.zeros((count,), dtype=np.bool)
for i in filter:
if type(i) == list:
mask[i[0]: i[1]] = True
mask[i[0] : i[1]] = True
else:
mask[i] = True
return mask
@@ -314,7 +303,7 @@ class DataAdaptor(metaclass=ABCMeta):
top_n = self.dataset_config.diffexp__top_n
if self.server_config.exceeds_limit(
"diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
"diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
):
raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
server/db/cellxgene_orm.py
+1 -6
View File
@@ -1,11 +1,6 @@
import uuid
from sqlalchemy import (
Column,
DateTime,
ForeignKey,
String,
func, JSON)
from sqlalchemy import Column, DateTime, ForeignKey, String, func, JSON
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
server/db/db_utils.py
+4 -6
View File
@@ -42,9 +42,9 @@ class DbUtils:
def get_or_create_dataset(self, dataset_name):
try:
dataset_id = self.query(
table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]
)[0].id
dataset_id = self.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])[
0
].id
except IndexError:
dataset_id = uuid.uuid4()
dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
@@ -54,9 +54,7 @@ class DbUtils:
def get_or_create_user(self, user_id):
try:
user_id = self.query(
table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id]
)[0].id
user_id = self.query(table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id])[0].id
except IndexError:
user = CellxGeneUser(id=user_id)
self.session.add(user)
server/common/default_config.py → server/default_config.py
-1
View File
@@ -204,7 +204,6 @@ dataset:
enable: true
lfc_cutoff: 0.01
top_n: 10
"""
server/eb/app.py
+5 -8
View File
@@ -9,7 +9,7 @@ from flask import json
import logging
from flask_talisman import Talisman
from flask_cors import CORS
from server.common.aws_secret_utils import handle_config_from_secret
from server.common.config import handle_config_from_secret
from server.common.errors import SecretKeyRetrievalError
@@ -26,7 +26,7 @@ SERVERDIR = os.path.dirname(os.path.realpath(__file__))
sys.path.append(SERVERDIR)
try:
from server.common.app_config import AppConfig
from server.common.config.app_config import AppConfig
from server.app.app import Server
from server.common.data_locator import DataLocator, discover_s3_region_name
except Exception:
@@ -61,8 +61,7 @@ class WSGIServer(Server):
csp = {
"default-src": ["'self'"],
"connect-src": ["'self'"] + extra_connect_src,
"script-src": ["'self'", "'unsafe-eval'"]
+ obsolete_browser_script_hash + script_hashes,
"script-src": ["'self'", "'unsafe-eval'"] + obsolete_browser_script_hash + script_hashes,
"style-src": ["'self'", "'unsafe-inline'"],
"img-src": ["'self'", "https://cellxgene.cziscience.com", "data:"],
"object-src": ["'none'"],
@@ -104,7 +103,7 @@ class WSGIServer(Server):
if len(script_hashes) == 0:
logging.error("Content security policy hashes are missing, falling back to unsafe-inline policy")
return (script_hashes)
return script_hashes
@staticmethod
def compute_inline_csp_hashes(app, app_config):
@@ -173,9 +172,7 @@ try:
sys.exit(1)
# features are unsupported in the current hosted server
app_config.update_default_dataset_config(
embeddings__enable_reembedding=False,
)
app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
app_config.complete_config(logging.info)
server/test/__init__.py
+5 -8
View File
@@ -13,7 +13,8 @@ import requests
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from server.common.annotations.local_file_csv import AnnotationsLocalFile
from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT
from server.common.config.app_config import AppConfig
from server.common.config import DEFAULT_SERVER_PORT
from server.common.data_locator import DataLocator
from server.common.utils.utils import find_available_port
from server.data_common.fbs.matrix import encode_matrix_fbs
@@ -33,8 +34,7 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
multi_dataset__dataroot=data_locator.path,
authentication__type="test",
multi_dataset__dataroot=data_locator.path, authentication__type="test",
)
config.update_default_dataset_config(
embeddings__names=["umap"],
@@ -42,16 +42,13 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
diffexp__lfc_cutoff=0.01,
user_annotations__type="hosted_tiledb_array",
user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432",
user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir
user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsHostedTileDB(
tmp_dir,
DbUtils("postgresql://postgres:test_pw@localhost:5432"),
)
annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),)
return data, tmp_dir, annotations
server/test/fixtures/database/__init__.py
Vendored
+8 -18
View File
@@ -29,34 +29,26 @@ class TestDatabase:
def _create_test_user(self):
user = CellxGeneUser(id="test_user_id")
user2 = CellxGeneUser(id='1234')
user2 = CellxGeneUser(id="1234")
self.db.session.add(user)
self.db.session.add(user2)
self.db.session.commit()
def _create_test_dataset(self):
dataset = CellxGeneDataset(
name="test_dataset",
)
dataset = CellxGeneDataset(name="test_dataset",)
self.db.session.add(dataset)
self.db.session.commit()
def _create_test_annotation(self):
dataset = self.db.query([CellxGeneDataset],
[CellxGeneDataset.name == "test_dataset"],
)[0]
annotation = Annotation(
tiledb_uri="tiledb_uri",
user_id="test_user_id",
dataset_id=str(dataset.id)
)
dataset = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == "test_dataset"],)[0]
annotation = Annotation(tiledb_uri="tiledb_uri", user_id="test_user_id", dataset_id=str(dataset.id))
self.db.session.add(annotation)
self.db.session.commit()
@staticmethod
def get_random_string():
letters = string.ascii_lowercase
return ''.join(random.choice(letters) for i in range(12))
return "".join(random.choice(letters) for i in range(12))
def _create_test_users(self, user_count: int = 10):
users = []
@@ -80,10 +72,8 @@ class TestDatabase:
for i in range(annotation_count):
dataset = self.order_by_random(CellxGeneDataset)
user = self.order_by_random(CellxGeneUser)
annotations.append(Annotation(
tiledb_uri=self.get_random_string(),
user_id=user.id,
dataset_id=str(dataset.id)
))
annotations.append(
Annotation(tiledb_uri=self.get_random_string(), user_id=user.id, dataset_id=str(dataset.id))
)
self.db.session.add_all(annotations)
self.db.session.commit()
server/test/fixtures/dataset_config_outline.py
Vendored
+37
View File
@@ -0,0 +1,37 @@
f"""
dataset:
app:
scripts: {scripts} #list of strs (filenames) or dicts containing keys
inline_scripts: {inline_scripts} #list of strs (filenames)
about_legal_tos: {about_legal_tos}
about_legal_privacy: {about_legal_privacy}
authentication_enable: {authentication_enable}
presentation:
max_categories: {max_categories}
custom_colors: {custom_colors}
user_annotations:
enable: {enable_users_annotations}
type: {annotation_type}
hosted_tiledb_array:
db_uri: {db_uri}
hosted_file_directory: {hosted_file_directory}
local_file_csv:
directory: {local_file_csv_directory}
file: {local_file_csv_file}
ontology:
enable: {ontology_enabled}
obo_location: {obo_location}
embeddings:
names: {embedding_names}
enable_reembedding: {enable_reembedding}
diffexp:
enable: {enable_difexp}
lfc_cutoff: {lfc_cutoff}
top_n: {top_n}
"""
server/test/fixtures/server_config_outline.py
Vendored
+62
View File
@@ -0,0 +1,62 @@
f"""server:
app:
verbose: {verbose}
debug: {debug}
host: {host}
port: {port}
open_browser: {open_browser}
force_https: {force_https}
flask_secret_key: {flask_secret_key}
generate_cache_control_headers: {generate_cache_control_headers}
server_timing_headers: {server_timing_headers}
csp_directives: {csp_directives}
api_base_url: {api_base_url}
web_base_url: {web_base_url}
authentication:
type: {auth_type}
params_oauth:
oauth_api_base_url: {oauth_api_base_url}
client_id: {client_id}
client_secret: {client_secret}
jwt_decode_options: {jwt_decode_options}
session_cookie: {session_cookie}
cookie: {cookie}
multi_dataset:
dataroot: {dataroot}
index: {index}
allowed_matrix_types: {allowed_matrix_types}
matrix_cache:
max_datasets: {max_cached_datasets}
timelimit_s: {timelimit_s}
single_dataset:
datapath: {dataset_datapath}
obs_names: {obs_names}
var_names: {var_names}
about: {about}
title: {title}
diffexp:
alg_cxg: # number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count)
max_workers: {diffexp_max_workers}
cpu_multiplier: {cpu_multiplier}
target_workunit: {target_workunit} # The target number of matrix elements that are evaluated in one thread.
data_locator:
s3:
region_name: {data_locater_region_name}
adaptor:
cxg_adaptor:
tiledb_ctx:
sm.tile_cache_size: {cxg_tile_cache_size}
sm.num_reader_threads: {cxg_num_reader_threads}
anndata_adaptor:
backed: {anndata_backed}
limits:
column_request_max: {column_request_max}
diffexp_cellcount_max: {diffexp_cellcount_max}
"""
server/test/performance/run_diffexp.py
+1 -1
View File
@@ -7,7 +7,7 @@ import numpy as np
import server.compute.diffexp_cxg as diffexp_cxg
import server.compute.diffexp_generic as diffexp_generic
from server.common.app_config import AppConfig
from server.common.config.app_config import AppConfig
from server.data_common.matrix_loader import MatrixDataLoader
from server.data_cxg.cxg_adaptor import CxgAdaptor
server/test/test_database/test_database.py
+20 -17
View File
@@ -16,45 +16,48 @@ class DatabaseTest(unittest.TestCase):
del cls.db
def test_user_creation(self):
one_user = self.db.get(table=CellxGeneUser, entity_id='test_user_id')
self.assertEqual(one_user.id, 'test_user_id')
one_user = self.db.get(table=CellxGeneUser, entity_id="test_user_id")
self.assertEqual(one_user.id, "test_user_id")
user_count = self.db.session.query(CellxGeneUser).count()
self.assertGreater(user_count, 10)
def test_dataset_creation(self):
one_dataset = self.db.query(table_args=[CellxGeneDataset],
filter_args=[CellxGeneDataset.name == 'test_dataset'])
self.assertEqual(one_dataset[0].name, 'test_dataset')
one_dataset = self.db.query(
table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"]
)
self.assertEqual(one_dataset[0].name, "test_dataset")
dataset_count = self.db.session.query(CellxGeneDataset).count()
self.assertGreater(dataset_count, 10)
def test_annotation_creation(self):
one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0]
self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri')
one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == "tiledb_uri"])[0]
self.assertEqual(one_annotation.tiledb_uri, "tiledb_uri")
annotation_count = self.db.session.query(Annotation).count()
self.assertGreater(annotation_count, 10)
def test_get_most_recent_annotation_for_user_dataset(self):
dataset_id = str(self.db.query(table_args=[CellxGeneDataset],
filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id)
dataset_id = str(
self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"])[0].id
)
# have to commit separately because created_at time written on the db server
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0'))
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_0"))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1'))
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_1"))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2'))
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_2"))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3'))
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_3"))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4'))
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_4"))
self.db.session.commit()
most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id,
Annotation.user_id == 'test_user_id'])
most_recent_annotation = self.db.query_for_most_recent(
Annotation, [Annotation.dataset_id == dataset_id, Annotation.user_id == "test_user_id"]
)
self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4')
self.assertEqual(most_recent_annotation.tiledb_uri, "tiledb_uri_4")
server/test/unit/auth/test_auth.py
+5 -9
View File
@@ -2,7 +2,7 @@ import unittest
import requests
from server.common.app_config import AppConfig
from server.common.config.app_config import AppConfig
from server.test import FIXTURES_ROOT, test_server
@@ -12,9 +12,7 @@ class AuthTest(unittest.TestCase):
def test_auth_none(self):
c = AppConfig()
c.update_server_config(
authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot
)
c.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
c.update_default_dataset_config(user_annotations__enable=False)
c.complete_config()
@@ -28,9 +26,7 @@ class AuthTest(unittest.TestCase):
def test_auth_session(self):
c = AppConfig()
c.update_server_config(
authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot
)
c.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
c.update_default_dataset_config(user_annotations__enable=True)
c.complete_config()
@@ -107,8 +103,8 @@ class AuthTest(unittest.TestCase):
def test_auth_test_single(self):
c = AppConfig()
c.update_server_config(
authentication__type="test",
single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg")
authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
)
c.complete_config()
server/test/unit/auth/test_oauth.py
+8 -10
View File
@@ -9,7 +9,7 @@ from flask import Flask, jsonify, make_response, request, redirect
from multiprocessing import Process
import jose
from server.common.app_config import AppConfig
from server.common.config.app_config import AppConfig
from server.test import FIXTURES_ROOT, test_server
# This tests the oauth authentication type.
@@ -46,7 +46,7 @@ def token():
"scope": "openid profile email",
"expires_in": TOKEN_EXPIRES,
"token_type": "Bearer",
"expires_at": expires_at
"expires_at": expires_at,
}
return make_response(jsonify(r))
@@ -89,9 +89,8 @@ class AuthTest(unittest.TestCase):
authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}",
authentication__params_oauth__client_id="mock_client_id",
authentication__params_oauth__client_secret="mock_client_secret",
authentication__params_oauth__jwt_decode_options={
"verify_signature": False, "verify_iss": False
})
authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False},
)
app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
app_config.complete_config()
@@ -112,7 +111,7 @@ class AuthTest(unittest.TestCase):
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, f"{server}/login?dataset=d/pbmc3k.cxg/")
self.assertEqual(logout_uri, f"{server}/logout")
self.assertEqual(logout_uri, f"{server}/logout?dataset=d/pbmc3k.cxg/")
r = session.get(login_uri)
# check that the login redirect worked
@@ -122,6 +121,7 @@ class AuthTest(unittest.TestCase):
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com")
self.assertTrue(config["config"]["parameters"]["annotations"])
if cookie_key:
@@ -150,7 +150,7 @@ class AuthTest(unittest.TestCase):
r = session.get(logout_uri)
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}")
self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
@@ -160,9 +160,7 @@ class AuthTest(unittest.TestCase):
def test_auth_oauth_session(self):
# test with session cookies
app_config = AppConfig()
app_config.update_server_config(
authentication__params_oauth__session_cookie=True,
)
app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
self.auth_flow(app_config)
def test_auth_oauth_cookie(self):
server/test/unit/cli/test_launch.py
+28
View File
@@ -0,0 +1,28 @@
import filecmp
import os
import shutil
import unittest
import yaml
from server.default_config import default_config
from server.test import FIXTURES_ROOT
class CLIPLaunchTests(unittest.TestCase):
tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
@classmethod
def setUpClass(cls) -> None:
os.mkdir(cls.tmp_dir)
@classmethod
def tearDownClass(cls) -> None:
shutil.rmtree(cls.tmp_dir)
def test_dump_default_config(self):
os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
expected_config.write(yaml.dump(default_config))
filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")
server/test/unit/common/config/__init__.py
+246
View File
@@ -0,0 +1,246 @@
import os
import shutil
import unittest
import random
from unittest import mock
from server.test import FIXTURES_ROOT
def mockenv(**envvars):
return mock.patch.dict(os.environ, envvars)
class ConfigTests(unittest.TestCase):
tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
@classmethod
def tearDownClass(cls) -> None:
shutil.rmtree(cls.tmp_fixtures_directory)
@classmethod
def setUpClass(cls) -> None:
os.makedirs(cls.tmp_fixtures_directory)
def custom_server_config(
self,
verbose="false",
debug="false",
host="localhost",
port="null",
open_browser="false",
force_https="false",
flask_secret_key="null",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
api_base_url="null",
web_base_url="null",
auth_type="session",
oauth_api_base_url="null",
client_id="null",
client_secret="null",
jwt_decode_options="null",
session_cookie="true",
cookie="null",
dataroot="null",
index="false",
allowed_matrix_types=[],
max_cached_datasets=5,
timelimit_s=5,
dataset_datapath="null",
obs_names="null",
var_names="null",
about="null",
title="null",
diffexp_max_workers=64,
cpu_multiplier=4,
target_workunit="16_000_000",
data_locater_region_name="us-east-1",
cxg_tile_cache_size=8589934592,
cxg_num_reader_threads=32,
anndata_backed="false",
column_request_max=32,
diffexp_cellcount_max="null",
config_file_name="server_config.yaml",
):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py")
with open(server_config_outline_path, "r") as config_skeleton:
config = config_skeleton.read()
server_config = eval(config)
with open(configfile, "w") as server_config_file:
server_config_file.write(server_config)
return configfile
def custom_app_config(
self,
verbose="false",
debug="false",
host="localhost",
port="null",
open_browser="false",
force_https="false",
flask_secret_key="null",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
api_base_url="null",
web_base_url="null",
auth_type="session",
oauth_api_base_url="null",
client_id="null",
client_secret="null",
jwt_decode_options="null",
session_cookie="true",
cookie="null",
dataroot="null",
index="false",
allowed_matrix_types=[],
max_cached_datasets=5,
timelimit_s=5,
dataset_datapath="null",
obs_names="null",
var_names="null",
about="null",
title="null",
diffexp_max_workers=64,
cpu_multiplier=4,
target_workunit="16_000_000",
data_locater_region_name="us-east-1",
cxg_tile_cache_size=8589934592,
cxg_num_reader_threads=32,
anndata_backed="false",
column_request_max=32,
diffexp_cellcount_max="null",
scripts=[],
inline_scripts=[],
about_legal_tos="null",
about_legal_privacy="null",
authentication_enable="true",
max_categories=1000,
custom_colors="true",
enable_users_annotations="true",
annotation_type="local_file_csv",
db_uri="null",
hosted_file_directory="null",
local_file_csv_directory="null",
local_file_csv_file="null",
ontology_enabled="false",
obo_location="null",
embedding_names=[],
enable_reembedding="false",
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
config_file_name="app_config.yml",
):
random_num = random.randrange(999999)
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
server_config = self.custom_server_config(
verbose=verbose,
debug=debug,
host=host,
port=port,
open_browser=open_browser,
force_https=force_https,
flask_secret_key=flask_secret_key,
generate_cache_control_headers=generate_cache_control_headers,
server_timing_headers=server_timing_headers,
csp_directives=csp_directives,
api_base_url=api_base_url,
web_base_url=web_base_url,
auth_type=auth_type,
oauth_api_base_url=oauth_api_base_url,
client_id=client_id,
client_secret=client_secret,
jwt_decode_options=jwt_decode_options,
session_cookie=session_cookie,
cookie=cookie,
dataroot=dataroot,
index=index,
allowed_matrix_types=allowed_matrix_types,
max_cached_datasets=max_cached_datasets,
timelimit_s=timelimit_s,
dataset_datapath=dataset_datapath,
obs_names=obs_names,
var_names=var_names,
about=about,
title=title,
diffexp_max_workers=diffexp_max_workers,
cpu_multiplier=cpu_multiplier,
target_workunit=target_workunit,
data_locater_region_name=data_locater_region_name,
cxg_tile_cache_size=cxg_tile_cache_size,
cxg_num_reader_threads=cxg_num_reader_threads,
anndata_backed=anndata_backed,
column_request_max=column_request_max,
diffexp_cellcount_max=diffexp_cellcount_max,
config_file_name=f"temp_server_config_{random_num}.yml",
)
dataset_config = self.custom_dataset_config(
scripts=scripts,
inline_scripts=inline_scripts,
about_legal_tos=about_legal_tos,
about_legal_privacy=about_legal_privacy,
authentication_enable=authentication_enable,
max_categories=max_categories,
custom_colors=custom_colors,
enable_users_annotations=enable_users_annotations,
annotation_type=annotation_type,
db_uri=db_uri,
hosted_file_directory=hosted_file_directory,
local_file_csv_directory=local_file_csv_directory,
local_file_csv_file=local_file_csv_file,
ontology_enabled=ontology_enabled,
obo_location=obo_location,
embedding_names=embedding_names,
enable_reembedding=enable_reembedding,
enable_difexp=enable_difexp,
lfc_cutoff=lfc_cutoff,
top_n=top_n,
config_file_name=f"temp_dataset_config_{random_num}.yml",
)
with open(server_config) as server_config:
with open(dataset_config) as dataset_config:
with open(configfile, "w") as app_config_file:
for line in server_config:
app_config_file.write(line)
for line in dataset_config:
app_config_file.write(line)
return configfile
def custom_dataset_config(
self,
scripts=[],
inline_scripts=[],
about_legal_tos="null",
about_legal_privacy="null",
authentication_enable="true",
max_categories=1000,
custom_colors="true",
enable_users_annotations="true",
annotation_type="local_file_csv",
db_uri="null",
hosted_file_directory="null",
local_file_csv_directory="null",
local_file_csv_file="null",
ontology_enabled="false",
obo_location="null",
embedding_names=[],
enable_reembedding="false",
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
config_file_name="dataset_config.yml",
):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py")
with open(dataset_config_outline_path, "r") as config_skeleton:
config = config_skeleton.read()
dataset_config = eval(config)
with open(configfile, "w") as dataset_config_file:
dataset_config_file.write(dataset_config)
return configfile
server/test/unit/common/config/test_app_config.py
+140
View File
@@ -0,0 +1,140 @@
import os
import tempfile
import unittest
import yaml
from server.default_config import default_config
from server.common.config.app_config import AppConfig
from server.test.unit.common.config import ConfigTests
from server.common.errors import ConfigurationError
from server.test import FIXTURES_ROOT
class AppConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_get_default_config_correctly_reads_default_config_file(self):
app_default_config = AppConfig().default_config
expected_config = yaml.load(default_config, Loader=yaml.Loader)
server_config = app_default_config["server"]
dataset_config = app_default_config["dataset"]
expected_server_config = expected_config["server"]
expected_dataset_config = expected_config["dataset"]
self.assertDictEqual(app_default_config, expected_config)
self.assertDictEqual(server_config, expected_server_config)
self.assertDictEqual(dataset_config, expected_dataset_config)
def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self):
datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
config = AppConfig()
config.update_from_config_file(file_name)
self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
def test_update_server_config_updates_server_config_and_config_status(self):
config = self.get_config()
config.complete_config()
config.check_config()
config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
with self.assertRaises(ConfigurationError):
config.server_config.check_config()
def test_write_config_outputs_yaml_with_all_config_vars(self):
config = self.get_config()
config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
default_config_yml = yaml.safe_load(default_config)
with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
output_config_yml = yaml.safe_load(output_config)
self.maxDiff = None
self.assertEqual(default_config_yml, output_config_yml)
def test_update_app_config(self):
config = AppConfig()
config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
config = AppConfig()
config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
def test_configfile_no_dataset_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
multi_dataset:
dataroot: test_dataroot
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
self.assertEqual(dataset_changes, [])
def test_configfile_no_server_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
dataset:
user_annotations:
enable: false
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
server/test/unit/common/config/test_base_config.py
+63
View File
@@ -0,0 +1,63 @@
import unittest
from server.common.config.app_config import AppConfig
from server.test import FIXTURES_ROOT
from server.test.unit.common.config import ConfigTests
from server.common.errors import ConfigurationError
class BaseConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
config = AppConfig()
mapping = config.default_dataset_config.create_mapping(config.default_config)
self.assertIsNotNone(mapping["server__app__verbose"])
self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
def test_changes_from_default_returns_list_of_nondefault_config_values(self):
config = self.get_config(verbose="true", lfc_cutoff=0.05)
server_changes = config.server_config.changes_from_default()
dataset_changes = config.default_dataset_config.changes_from_default()
self.assertEqual(
server_changes,
[
("app__verbose", True, False),
("multi_dataset__dataroot", FIXTURES_ROOT, None),
("multi_dataset__matrix_cache__timelimit_s", 5, 30),
("data_locator__s3__region_name", "us-east-1", True),
],
)
self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
def test_check_config_throws_error_if_attr_has_not_been_checked(self):
config = self.get_config(verbose="true")
config.complete_config()
config.check_config()
config.update_server_config(app__verbose=False)
with self.assertRaises(ConfigurationError):
config.check_config()
server/test/unit/common/config/test_dataset_config.py
+260
View File
@@ -0,0 +1,260 @@
import os
import tempfile
import requests
import unittest
from unittest.mock import patch
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from server.common.annotations.local_file_csv import AnnotationsLocalFile
from server.common.config.app_config import AppConfig
from server.common.config.base_config import BaseConfig
from server.test import test_server, PROJECT_ROOT, FIXTURES_ROOT
from server.common.errors import ConfigurationError
from server.test.unit.common.config import ConfigTests
class TestDatasetConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.dataset_config = self.config.default_dataset_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_datatset_config_sets_vars_from_default_config(self):
config = AppConfig()
self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location)
@patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
self.dataset_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 21)
def test_app_sets_script_vars(self):
config = self.get_config(scripts=["path/to/script"])
config.default_dataset_config.handle_app()
self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}])
config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
config.default_dataset_config.handle_app()
self.assertEqual(
config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
)
config = self.get_config(scripts=["path/to/script", "different/script/path"])
config.default_dataset_config.handle_app()
# TODO @madison -- is this the desired functionality?
self.assertEqual(
config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
)
config = self.get_config(scripts=[{"more": "different/script/path"}])
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_app()
def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
config = self.get_config(enable_users_annotations="true", authentication_enable="false")
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_user_annotations(self.context)
config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
with self.assertRaises(ConfigurationError):
config.server_config.complete_config(self.context)
def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self):
config = self.get_config(
enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set"
)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."])
@patch("server.common.config.dataset_config.DbUtils")
def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils):
mock_db_utils.return_value = "123"
config = self.get_config(
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
config = self.get_config(
enable_users_annotations="true",
authentication_enable="true",
annotation_type="hosted_tiledb_array",
db_uri="gotta/set/this",
hosted_file_directory="and/this",
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB)
config = self.get_config(
enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_user_annotations(self.context)
def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
config = self.get_config(
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_local_file_csv_annotations()
self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
cwd = os.getcwd()
self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
def test_handle_embeddings__checks_data_file_types(self):
file_name = self.custom_app_config(
embedding_names=["name1", "name2"],
enable_reembedding="true",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
anndata_backed="true",
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_embeddings()
def test_handle_diffexp__raises_warning_for_large_datasets(self):
config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_diffexp(self.context)
self.assertEqual(len(self.context["messages"]), 0)
def test_multi_dataset(self):
config = AppConfig()
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
)
with self.assertRaises(ConfigurationError):
config.complete_config()
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
)
config.complete_config()
# test that multi dataroots work end to end
config.update_server_config(
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
)
)
# Change this default to test if the dataroot overrides below work.
config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
# specialize the configs for set1
config.add_dataroot_config(
"s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
)
# specialize the configs for set2
config.add_dataroot_config(
"s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
)
# no specializations for set3 (they get the default dataset config)
config.complete_config()
with test_server(app_config=config) as server:
session = requests.Session()
response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
multi_dataset:
dataroot:
test:
base_url: test
dataroot: fake_dataroot
dataset:
user_annotations:
enable: false
type: hosted_tiledb_array
hosted_tiledb_array:
db_uri: fake_db_uri
hosted_file_directory: fake_dir
per_dataset_config:
test:
user_annotations:
enable: true
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataroot_config["test"]
# test config from default
self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
# test config from specialization
self.assertTrue(test_config.user_annotations__enable)
server/test/unit/common/config/test_server_config.py
+335
View File
@@ -0,0 +1,335 @@
import os
import unittest
from unittest import mock
from unittest.mock import patch
from server.common.config.base_config import BaseConfig
from server.common.utils.utils import find_available_port
from server.test import PROJECT_ROOT, FIXTURES_ROOT
import requests
from server.common.config.app_config import AppConfig
from server.common.errors import ConfigurationError
from server.test import test_server
from server.test.unit.common.config import ConfigTests
def mockenv(**envvars):
return mock.patch.dict(os.environ, envvars)
class TestServerConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_raises_error_if_default_config_is_invalid(self):
invalid_config = self.get_config(port="not_valid")
with self.assertRaises(ConfigurationError):
invalid_config.complete_config()
@patch("server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
self.server_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 40)
def test_handle_app__throws_error_if_port_doesnt_exist(self):
config = self.get_config(port=99999999)
with self.assertRaises(ConfigurationError):
config.server_config.handle_app(self.context)
@patch("server.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
# incorrectly formatted
dataroot = {
"d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
"d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"},
}
file_name = self.custom_app_config(
dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
@patch("server.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
mock_discover_region_name.return_value = "us-west-2"
dataroot = {
"d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
"d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
}
file_name = self.custom_app_config(
dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_data_locator()
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
def test_handle_app___can_use_envar_port(self):
config = self.get_config(port=24)
self.assertEqual(config.server_config.app__port, 24)
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
os.environ["CXG_SERVER_PORT"] = "4008"
self.config = AppConfig()
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
config.server_config.handle_app(self.context)
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
def test_handle_app__sets_web_base_url(self):
config = self.get_config(web_base_url="anything.com")
self.assertEqual(config.server_config.app__web_base_url, "anything.com")
def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self):
config = self.get_config(client_secret="KEY_FROM_FILE")
config.server_config.handle_authentication()
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
config.server_config.handle_authentication()
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name="two_data_roots.yml",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
file_name = self.custom_app_config(config_file_name="zero_roots.yml")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
def test_get_api_base_url_works(self):
# test the api_base_url feature, and that it can contain a path
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
app__api_base_url=f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
)
config.complete_config()
with test_server(["-p", str(backend_port)], app_config=config) as server:
session = requests.Session()
self.assertEqual(server, f"http://localhost:{backend_port}")
response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
self.assertEqual(response.status_code, 200)
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
# test the health check at the correct url
response = session.get(f"{server}/additional/path/health")
assert response.json()["status"] == "pass"
# also check that the old URL still works.
# NOTE: this old URL location will soon be deprecated, and when that happens
# this check can be removed.
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
def test_get_web_base_url_works(self):
config = self.get_config(web_base_url="www.thisisawebsite.com")
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "www.thisisawebsite.com")
config = self.get_config(web_base_url="local", port=12)
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "http://localhost:12")
config = self.get_config(web_base_url="www.thisisawebsite.com/")
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "www.thisisawebsite.com")
config = self.get_config(api_base_url="www.api_base.com/")
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "www.api_base.com")
def test_config_for_single_dataset(self):
file_name = self.custom_app_config(
config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg"
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_single_dataset(self.context)
self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
file_name = self.custom_app_config(
config_file_name="single_dataset_with_about.yml",
about="www.cziscience.com",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_single_dataset(self.context)
def test_multi_dataset_raises_error_for_illegal_routes(self):
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
self.config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
)
with self.assertRaises(ConfigurationError):
self.config.complete_config()
def test_multidataset_works_for_legal_routes(self):
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
self.config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
)
self.config.complete_config()
def test_mulitdatasets_work_e2e(self):
# test that multi dataroots work end to end
self.config.update_server_config(
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
)
)
# Change this default to test if the dataroot overrides below work.
self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
# specialize the configs for set1
self.config.add_dataroot_config(
"s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
)
# specialize the configs for set2
self.config.add_dataroot_config(
"s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
)
# no specializations for set3 (they get the default dataset config)
self.config.complete_config()
with test_server(app_config=self.config) as server:
session = requests.Session()
response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
@patch("server.common.config.server_config.diffexp_tiledb.set_config")
def test_handle_diffexp(self, mock_tiledb_config):
custom_config_file = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
cpu_multiplier=3,
diffexp_max_workers=1,
target_workunit=4,
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(custom_config_file)
config.server_config.handle_diffexp()
# called with the min of diffexp_max_workers and cpus*cpu_multiplier
mock_tiledb_config.assert_called_once_with(1, 4)
@patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
def test_handle_adaptor(self, mock_tiledb_context):
custom_config = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
)
config = AppConfig()
config.update_from_config_file(custom_config)
config.server_config.handle_adaptor()
mock_tiledb_context.assert_called_once_with(
{"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
)
@mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
@patch("server.common.config.get_secret_key")
def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"flask_secret_key": "mock_flask_secret",
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
config = AppConfig()
with self.assertLogs(level="INFO") as logger:
from server.common.config import handle_config_from_secret
# should not throw error
# "AttributeError: 'XConfig' object has no attribute 'x'"
handle_config_from_secret(config)
# should log 3 lines (one for each var set from a secret)
self.assertEqual(len(logger.output), 3)
self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0])
self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1])
self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2])
self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
server/test/unit/common/test_api.py
+19 -18
View File
@@ -8,8 +8,14 @@ import requests
import server.test.unit.decode_fbs as decode_fbs
from server.data_common.matrix_loader import MatrixDataType
from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server,
stop_test_server)
from server.test import (
data_with_tmp_annotations,
make_fbs,
PROJECT_ROOT,
FIXTURES_ROOT,
start_test_server,
stop_test_server,
)
from server.test.fixtures.fixtures import pbmc3k_colors
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
@@ -43,7 +49,6 @@ class EndPoints(object):
result_data = result.json()
self.assertIn("library_versions", result_data["config"])
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertEqual(len(result_data["config"]["features"]), 5)
def test_get_layout_fbs(self):
endpoint = "layout/obs"
@@ -381,11 +386,14 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
@classmethod
def setUpClass(cls):
cls._setupClass(cls, [
f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
"--disable-annotations",
"--experimental-enable-reembedding",
])
cls._setupClass(
cls,
[
f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
"--disable-annotations",
"--experimental-enable-reembedding",
],
)
@classmethod
def tearDownClass(cls):
@@ -403,10 +411,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
@classmethod
def setUpClass(cls):
cls._setupClass(cls, [
f"{FIXTURES_ROOT}/pbmc3k.cxg",
"--disable-annotations",
])
cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"])
@classmethod
def tearDownClass(cls):
@@ -423,7 +428,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
MatrixDataType.H5AD, annotations_fixture=True
)
cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ])
cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
@classmethod
def tearDownClass(cls):
@@ -439,11 +444,7 @@ class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
@classmethod
def setUpClass(cls):
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
cls._setupClass(cls, [
"--annotations-file",
cls.annotations.output_file,
cls.data.get_location(),
])
cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
@classmethod
def tearDownClass(cls):
server/test/unit/common/test_app_config.py
-249
View File
@@ -1,249 +0,0 @@
import os
import unittest
from unittest import mock
from unittest.mock import patch
import tempfile
import requests
from server.common.app_config import AppConfig
from server.common.errors import ConfigurationError
from server.common.utils.utils import find_available_port
from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
# NOTE, there are more tests that should be written for AppConfig.
# this is just a start.
def mockenv(**envvars):
return mock.patch.dict(os.environ, envvars)
class AppConfigTest(unittest.TestCase):
def test_update(self):
config = AppConfig()
config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
config = AppConfig()
config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
def test_multi_dataset(self):
config = AppConfig()
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
)
with self.assertRaises(ConfigurationError):
config.complete_config()
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
)
config.complete_config()
# test that multi dataroots work end to end
config.update_server_config(
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
)
)
# Change this default to test if the dataroot overrides below work.
config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
# specialize the configs for set1
config.add_dataroot_config(
"s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
)
# specialize the configs for set2
config.add_dataroot_config(
"s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
)
# no specializations for set3 (they get the default dataset config)
config.complete_config()
with test_server(app_config=config) as server:
session = requests.Session()
response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
@mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
@patch('server.common.aws_secret_utils.get_secret_key')
def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"flask_secret_key": "mock_flask_secret",
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri"
}
config = AppConfig()
with self.assertLogs(level="INFO") as logger:
from server.common.aws_secret_utils import handle_config_from_secret
# should not throw error
# "AttributeError: 'XConfig' object has no attribute 'x'"
handle_config_from_secret(config)
# should log 3 lines (one for each var set from a secret)
self.assertEqual(len(logger.output), 3)
self.assertIn('INFO:root:set app__flask_secret_key from secret', logger.output[0])
self.assertIn('INFO:root:set authentication__params_oauth__client_secret from secret', logger.output[1])
self.assertIn('INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret', logger.output[2])
self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
def test_api_base_url(self):
# test the api_base_url feature, and that it can contain a path
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
app__api_base_url=f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset"
)
config.complete_config()
with test_server(["-p", str(backend_port)], app_config=config) as server:
session = requests.Session()
self.assertEqual(server, f"http://localhost:{backend_port}")
response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
self.assertEqual(response.status_code, 200)
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
# test the health check at the correct url
response = session.get(f"{server}/additional/path/health")
assert response.json()["status"] == "pass"
# also check that the old URL still works.
# NOTE: this old URL location will soon be deprecated, and when that happens
# this check can be removed.
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
multi_dataset:
dataroot:
test:
base_url: test
dataroot: fake_dataroot
dataset:
user_annotations:
enable: false
type: hosted_tiledb_array
hosted_tiledb_array:
db_uri: fake_db_uri
hosted_file_directory: fake_dir
per_dataset_config:
test:
user_annotations:
enable: true
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataroot_config["test"]
# test config from default
self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
# test config from specialization
self.assertTrue(test_config.user_annotations__enable)
def test_configfile_no_dataset_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
multi_dataset:
dataroot: test_dataroot
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
self.assertEqual(dataset_changes, [])
def test_configfile_no_server_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
dataset:
user_annotations:
enable: false
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
server/test/unit/common/test_corpora.py
+6 -13
View File
@@ -87,24 +87,17 @@ class CorporaRESTAPITest(unittest.TestCase):
def setCorporaFields(cls, path):
adata = anndata.read_h5ad(path)
corpora_props = {
"version": {
"corpora_schema_version": "1.0.0",
"corpora_encoding_version": "0.1.0"
},
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
"title": "PBMC3K",
"contributors": json.dumps([
{"name": "name"}
]),
"layer_descriptions": {
"X": "raw counts"
},
"contributors": json.dumps([{"name": "name"}]),
"layer_descriptions": {"X": "raw counts"},
"organism": "human",
"organism_ontology_term_id": "unknown",
"project_name": "test project",
"project_description": "test description",
"project_links": json.dumps([
{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}
]),
"project_links": json.dumps(
[{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
),
"default_embedding": "X_tsne",
}
adata.uns.update(corpora_props)
server/test/unit/common/test_writable_annotation.py
+7 -26
View File
@@ -27,7 +27,7 @@ class auth(object):
class WritableTileDBStoredAnnotationTest(unittest.TestCase):
def setUp(self):
self.user_id = '1234'
self.user_id = "1234"
self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
self.data.dataset_config.user_annotations = self.annotations
self.db = self.annotations.db
@@ -38,7 +38,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
}
self.fbs = make_fbs(self.test_dict)
self.df = pd.DataFrame(self.test_dict)
self.app = Flask('fake_app')
self.app = Flask("fake_app")
self.app.__setattr__("auth", auth)
def tearDown(self):
@@ -65,8 +65,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
self.annotations.write_labels(self.df, self.data)
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
Annotation,
[Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
)
# retrieve tiledb array
df = tiledb.open(annotation.tiledb_uri)
@@ -78,7 +77,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
with self.app.test_request_context():
new_name = 'new_dataset/location'
new_name = "new_dataset/location"
self.data.get_location = MagicMock(return_value=new_name)
num_datasets = len(self.db.query([CellxGeneDataset]))
self.annotation_put_fbs(self.fbs)
@@ -130,15 +129,14 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
with self.assertRaises(KeyError):
self.annotation_put_fbs(fbs_bad)
@patch('server.common.annotations.hosted_tiledb.current_app')
@patch("server.common.annotations.hosted_tiledb.current_app")
def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id):
mock_user_id.auth.get_user_id.return_value = '1234'
mock_user_id.auth.get_user_id.return_value = "1234"
self.annotations.write_labels(self.df, self.data)
# get uri
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
Annotation,
[Annotation.user_id == '1234', Annotation.dataset_id == str(dataset_id)]
Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
)
df = tiledb.open(annotation.tiledb_uri)
@@ -270,20 +268,3 @@ class WritableAnnotationTest(unittest.TestCase):
all_col_schema["cat_B"],
{"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
)
def test_config(self):
features = self.data.get_features(self.annotations)
# test each for singular presence and accuracy of available flag
def check_feature(method, path, available):
feature = list(
filter(lambda f: f.method == method and f.path == path and f.available == available, features)
)
self.assertIsNotNone(feature)
self.assertEqual(len(feature), 1)
check_feature("POST", "/cluster/", False)
check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable)
check_feature("GET", "/layout/obs", True)
check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding)
check_feature("PUT", "/annotations/obs", True)
server/test/unit/common/utils/test_cxg_generation_utils.py
+31 -21
View File
@@ -8,8 +8,12 @@ import numpy as np
import tiledb
from pandas import Series, DataFrame
from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array,
convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array)
from server.common.utils.cxg_generation_utils import (
convert_dictionary_to_cxg_group,
convert_dataframe_to_cxg_array,
convert_ndarray_to_cxg_dense_array,
convert_matrix_to_cxg_array,
)
PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
@@ -28,8 +32,9 @@ class TestCxgGenerationUtils(unittest.TestCase):
dictionary_name = "favorite_desserts"
expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary,
group_metadata_name=dictionary_name)
convert_dictionary_to_cxg_group(
self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
)
array = tiledb.open(expected_array_directory)
actual_stored_metadata = dict(array.meta.items())
@@ -44,13 +49,16 @@ class TestCxgGenerationUtils(unittest.TestCase):
random_dataframe_name = f"random_dataframe_{uuid4()}"
random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe,
"int_category", tiledb.Ctx())
convert_dataframe_to_cxg_array(
self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
)
expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
expected_array_metadata = {
"cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"},
"index": "int_category"})}
"cxg_schema": json.dumps(
{"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
)
}
actual_stored_dataframe_array = tiledb.open(expected_array_directory)
actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
@@ -95,7 +103,7 @@ class TestCxgGenerationUtils(unittest.TestCase):
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[:, :][''].size == 0)
self.assertTrue(actual_stored_array[:, :][""].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
matrix = np.zeros([3, 3])
@@ -110,10 +118,10 @@ class TestCxgGenerationUtils(unittest.TestCase):
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[0, 0][''] == 1)
self.assertTrue(actual_stored_array[1, 1][''] == 1)
self.assertTrue(actual_stored_array[2, 2][''] == 2)
self.assertTrue(actual_stored_array[:, :][''].size == 3)
self.assertTrue(actual_stored_array[0, 0][""] == 1)
self.assertTrue(actual_stored_array[1, 1][""] == 1)
self.assertTrue(actual_stored_array[2, 2][""] == 2)
self.assertTrue(actual_stored_array[:, :][""].size == 3)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -122,14 +130,15 @@ class TestCxgGenerationUtils(unittest.TestCase):
# a matrix of zeros which is sparse.
column_shift = np.ones((3, 2))
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
column_shift_for_sparse_encoding=column_shift)
convert_matrix_to_cxg_array(
matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
)
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[:, :][''].size == 0)
self.assertTrue(actual_stored_array[:, :][""].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -137,13 +146,14 @@ class TestCxgGenerationUtils(unittest.TestCase):
# Only column shift the first column of ones.
column_shift = np.array([[1, 0], [1, 0]])
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
column_shift_for_sparse_encoding=column_shift)
convert_matrix_to_cxg_array(
matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
)
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[0, 1][''] == 1)
self.assertTrue(actual_stored_array[1, 1][''] == 1)
self.assertTrue(actual_stored_array[:, :][''].size == 2)
self.assertTrue(actual_stored_array[0, 1][""] == 1)
self.assertTrue(actual_stored_array[1, 1][""] == 1)
self.assertTrue(actual_stored_array[:, :][""].size == 2)
server/test/unit/common/utils/test_matrix_utils.py
-1
View File
@@ -6,7 +6,6 @@ from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_
class TestMatrixUtils(unittest.TestCase):
def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
matrix = np.array([1, 2, 3])
server/test/unit/common/utils/test_sanitization_utils.py
-1
View File
@@ -4,7 +4,6 @@ from server.common.utils.sanitization_utils import sanitize_values_in_list, sani
class TestSanitizationUtils(unittest.TestCase):
def test__sanitize_values_in_list__not_strings_raises_exception(self):
keys_to_sanitize = [1, 2, 3]

Some files were not shown because too many files have changed in this diff Show More