fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-22 04:18:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge main

Browse Source
This commit is contained in:
Colin Megill
2020-12-03 15:25:49 -08:00
parent 36939d99fa c428242878
commit 0a9a78bd85
164 changed files with 8999 additions and 2586 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/PULL_REQUEST_TEMPLATE/pull request.md
+11
View File
@@ -0,0 +1,11 @@
#### Reviewers
**Functional:**
**Readability:**
---
## Changes
- add
- remove
- modify
.github/workflows/codeql-analysis.yml
+67
View File
@@ -0,0 +1,67 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL Scan"
on:
push:
branches: [ main ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ main ]
schedule:
- cron: '0 8 * * *'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
language: [ 'javascript', 'python' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
# Learn more:
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v1
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1
.github/workflows/compatibility_tests.yml
+3 -2
View File
@@ -25,10 +25,11 @@ jobs:
cellxgene-main-with-python-and-anndata-versions:
name: python versions x anndata versions
runs-on: ubuntu-latest
continue-on-error: true
strategy:
matrix:
python-version: [3.6, 3.7, 3.8]
anndata-version: [0.6.22.post1, 0.7.1]
python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4]
test-suite: [smoke-test, smoke-test-annotations]
steps:
- uses: actions/checkout@v2
.github/workflows/push_tests.yml
+2 -1
View File
@@ -31,9 +31,10 @@ jobs:
- name: Install dependencies
run: |
pip install flake8
pip install black
cd client
npm install
- name: Lint with flake8
- name: Format with black and lint with flake8
run: |
make lint-server
- name: Lint src with eslint
.github/workflows/scale-test.yml
+30
View File
@@ -0,0 +1,30 @@
name: "Scale test cellxgene APIs for initial loading"
on:
schedule:
- cron: "0 0 * * Sun"
jobs:
locust-build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.7
uses: actions/setup-python@v1
with:
python-version: 3.7
- name: Install dependencies
run: |
pip install -r server/test/locust/requirements-locust.txt
- name: Dev Scale Test
run: |
locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
- name: Slack success webhook
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
run: |
DEV_STATS=$(tail -n 61 locust_dev_stats.txt)
DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
LICENSE.txt
+2 -2
View File
@@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2013
Copyright (c) 2017-2020 Chan Zuckerberg Initiative
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
MANIFEST.in
+2
View File
@@ -3,3 +3,5 @@ recursive-include server/common/web/static *
include server/requirements.txt
include server/requirements-prepare.txt
include server/converters/schema/hgnc_complete_set.txt.gz
include server/converters/schema/schema_definitions/*
Makefile
+3 -2
View File
@@ -82,8 +82,9 @@ fmt-py:
lint: lint-server lint-client
.PHONY: lint-server
lint-server:
flake8 server
lint-server: fmt-py
flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
.PHONY: lint-client
lint-client:
README.md
+2 -2
View File
@@ -67,7 +67,7 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg
### Contributing
We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics.
This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
@@ -81,7 +81,7 @@ If you believe you have found a security issue, we would appreciate notification
# Inspiration
We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
client/Makefile
+4 -2
View File
@@ -3,6 +3,8 @@ include ../common.mk
ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)
# Packaging
.PHONY: clean
clean:
@@ -31,9 +33,9 @@ start-frontend:
.PHONY: smoke-test
smoke-test:
start_server_and_test \
'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \
'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
$(CXG_SERVER_PORT) \
'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false'
# start an instance of cellxgene and run the end-to-end annotations tests
.PHONY: smoke-test-annotations
client/__tests__/e2e/__snapshots__/e2e.test.js.snap
+2 -2
View File
@@ -1,5 +1,5 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;
exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"max-width: 265px;\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;
client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap
+8 -8
View File
@@ -2,22 +2,22 @@
exports[`annotations stacked bar graph renders 1`] = `
Array [
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
]
`;
exports[`annotations stacked bar graph renders 2`] = `
Array [
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
"<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
]
`;
exports[`annotations truncate midpoint whitespace 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
exports[`annotations truncate midpoint whitespace 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
exports[`annotations truncate midpoint whitespace 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
exports[`annotations truncate midpoint whitespace 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
exports[`annotations truncate single character 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
exports[`annotations truncate single character 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
exports[`annotations truncate single character 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
exports[`annotations truncate single character 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
client/__tests__/e2e/cellxgeneActions.js
+73
View File
@@ -13,8 +13,11 @@ import {
getTestClass,
getTestId,
isElementPresent,
goToPage,
} from "./puppeteerUtils";
import { appUrlBase } from "./config";
export async function drag(testId, start, end, lasso = false) {
const layout = await waitByID(testId);
const elBox = await layout.boxModel();
@@ -312,4 +315,74 @@ export async function assertCategoryDoesNotExist(categoryName) {
await expect(result).toBe(false);
}
export async function login() {
const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
const password = "Test1111";
await goToPage(appUrlBase);
await clickOn("log-in");
// (thuang): Auth0 form is unstable and unsafe for input until verified
await waitUntilFormFieldStable('[name="email"]');
await expect(page).toFillForm("form", {
email,
password,
});
await Promise.all([
page.waitForNavigation({ waitUntil: "networkidle0" }),
expect(page).toClick('[name="submit"]'),
]);
expect(page.url()).toContain(appUrlBase);
}
export async function logout() {
await clickOnUntil("user-info", async () => {
await waitByID("log-out");
await Promise.all([
page.waitForNavigation({ waitUntil: "networkidle0" }),
clickOn("log-out"),
]);
});
await waitByID("log-in");
}
async function waitUntilFormFieldStable(selector) {
const MAX_RETRY = 10;
const WAIT_FOR_MS = 200;
const EXPECTED_VALUE = "aaa";
let retry = 0;
while (retry < MAX_RETRY) {
try {
await expect(page).toFill(selector, EXPECTED_VALUE);
const fieldHandle = await expect(page).toMatchElement(selector);
const fieldValue = await page.evaluate(
(input) => input.value,
fieldHandle
);
expect(fieldValue).toBe(EXPECTED_VALUE);
break;
} catch (error) {
retry += 1;
await page.waitFor(WAIT_FOR_MS);
}
}
if (retry === MAX_RETRY) {
throw Error("clickOnUntil() assertion failed!");
}
}
/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
client/__tests__/e2e/e2e.test.js
+33
View File
@@ -17,6 +17,7 @@ import {
goToPage,
typeInto,
waitByID,
clickOnUntil,
} from "./puppeteerUtils";
import {
@@ -31,6 +32,8 @@ import {
runDiffExp,
selectCategory,
subset,
login,
logout,
} from "./cellxgeneActions";
const data = datasets[DATASET];
@@ -518,4 +521,34 @@ test("lasso moves after pan", async () => {
expect(panCount).toBe(initialCount);
});
const describeIfCalledByMakeFileTarget =
process.env.CXG_AUTH_TYPE?.toLowerCase() === "test"
? describe
: describe.skip;
describeIfCalledByMakeFileTarget("auth buttons", () => {
test("login then logout", async () => {
await goToPage(appUrlBase);
await clickOnUntil("log-in", async () => {
await page.waitForNavigation({ waitUntil: "networkidle0" });
await waitByID("user-info");
});
await logout();
});
});
const conditionalDescribe =
process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
conditionalDescribe("AuthN Integration", () => {
it("logs in", async () => {
await login();
});
it("logs out", async () => {
await login();
await logout();
});
});
/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
client/__tests__/e2e/puppeteer.setup.js
+3 -3
View File
@@ -17,7 +17,9 @@ setDefaultOptions({ timeout: 20 * 1000 });
jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
(async () => {
beforeEach(async () => {
await jestPuppeteer.resetBrowser();
const userAgent = await browser.userAgent();
await page.setUserAgent(`${userAgent}bot`);
@@ -53,6 +55,4 @@ jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
}
}
});
})().catch((error) => {
console.error("puppeteer.setup.js error", error);
});
client/__tests__/e2e/test_config.yaml
+47
View File
@@ -0,0 +1,47 @@
server:
app:
force_https: true
# By default, cellxgene will serve api requests from the same base url as the webpage.
# In general api_base_url and web_base_url will not need to be set.
# There are two reasons to set these parameters:
# 1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
# which then redirects back to the web_base_url. If the web_base_url is not set, it will default to
# the api_base_url. If oauth authentication is used, the api_base_url must be set.
# For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
# set to the string "local".
# 2. The cellxgene deploymnent is in an environment where the webpage and api have
# different base urls. In this case both api_base_url and web_base_url must be set.
# It is up to the server admin to ensure that the networking is setup correctly for this environment.
api_base_url: http://localhost:5005
web_base_url: http://localhost:3000
authentication:
# The authentication types may be "none", "session", "oauth"
# none: No authentication support, features like user_annotations must not be enabled.
# session: A session based userid is automatically generated. (no params needed)
# oauth: oauth2 is used for authentication; parameters are defined in params_oauth.
type: test
dataset:
app:
about_legal_tos: null
about_legal_privacy: null
presentation:
max_categories: 1000
custom_colors: true
user_annotations:
enable: false
type: local_file_csv
local_file_csv:
directory: null
file: null
ontology:
enable: false
obo_location: null
embeddings:
names: []
enable_reembedding: false
client/configuration/eslint/eslint.js
+1
View File
@@ -4,6 +4,7 @@ module.exports = {
extends: [
"airbnb",
"plugin:eslint-comments/recommended",
"plugin:@blueprintjs/recommended",
"plugin:compat/recommended",
"plugin:prettier/recommended",
"prettier/react",
client/package-lock.json
Generated
+463 -52
View File
@@ -4156,6 +4156,216 @@
"tslib": "~1.10.0"
}
},
"@blueprintjs/eslint-plugin": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@blueprintjs/eslint-plugin/-/eslint-plugin-0.3.0.tgz",
"integrity": "sha512-bQEdE4ApEHxCDV8hT9uIxeRbDFKOtRLBT3/Zy3Ku+nowDAYl/8jwZKp6lJuR/nqvsfuIXTnVef6ivwdBEieQfA==",
"dev": true,
"requires": {
"@typescript-eslint/experimental-utils": "^4.2.0",
"eslint": "^7.9.0"
},
"dependencies": {
"@typescript-eslint/experimental-utils": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.3.0.tgz",
"integrity": "sha512-cmmIK8shn3mxmhpKfzMMywqiEheyfXLV/+yPDnOTvQX/ztngx7Lg/OD26J8gTZfkLKUmaEBxO2jYP3keV7h2OQ==",
"dev": true,
"requires": {
"@types/json-schema": "^7.0.3",
"@typescript-eslint/scope-manager": "4.3.0",
"@typescript-eslint/types": "4.3.0",
"@typescript-eslint/typescript-estree": "4.3.0",
"eslint-scope": "^5.0.0",
"eslint-utils": "^2.0.0"
}
},
"@typescript-eslint/typescript-estree": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.3.0.tgz",
"integrity": "sha512-ZAI7xjkl+oFdLV/COEz2tAbQbR3XfgqHEGy0rlUXzfGQic6EBCR4s2+WS3cmTPG69aaZckEucBoTxW9PhzHxxw==",
"dev": true,
"requires": {
"@typescript-eslint/types": "4.3.0",
"@typescript-eslint/visitor-keys": "4.3.0",
"debug": "^4.1.1",
"globby": "^11.0.1",
"is-glob": "^4.0.1",
"lodash": "^4.17.15",
"semver": "^7.3.2",
"tsutils": "^3.17.1"
}
},
"acorn": {
"version": "7.4.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
"integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
"dev": true
},
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"dev": true,
"requires": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
"which": "^2.0.1"
}
},
"eslint": {
"version": "7.10.0",
"resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz",
"integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==",
"dev": true,
"requires": {
"@babel/code-frame": "^7.0.0",
"@eslint/eslintrc": "^0.1.3",
"ajv": "^6.10.0",
"chalk": "^4.0.0",
"cross-spawn": "^7.0.2",
"debug": "^4.0.1",
"doctrine": "^3.0.0",
"enquirer": "^2.3.5",
"eslint-scope": "^5.1.1",
"eslint-utils": "^2.1.0",
"eslint-visitor-keys": "^1.3.0",
"espree": "^7.3.0",
"esquery": "^1.2.0",
"esutils": "^2.0.2",
"file-entry-cache": "^5.0.1",
"functional-red-black-tree": "^1.0.1",
"glob-parent": "^5.0.0",
"globals": "^12.1.0",
"ignore": "^4.0.6",
"import-fresh": "^3.0.0",
"imurmurhash": "^0.1.4",
"is-glob": "^4.0.0",
"js-yaml": "^3.13.1",
"json-stable-stringify-without-jsonify": "^1.0.1",
"levn": "^0.4.1",
"lodash": "^4.17.19",
"minimatch": "^3.0.4",
"natural-compare": "^1.4.0",
"optionator": "^0.9.1",
"progress": "^2.0.0",
"regexpp": "^3.1.0",
"semver": "^7.2.1",
"strip-ansi": "^6.0.0",
"strip-json-comments": "^3.1.0",
"table": "^5.2.3",
"text-table": "^0.2.0",
"v8-compile-cache": "^2.0.3"
},
"dependencies": {
"eslint-scope": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
"integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
"dev": true,
"requires": {
"esrecurse": "^4.3.0",
"estraverse": "^4.1.1"
}
}
}
},
"eslint-visitor-keys": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
"dev": true
},
"espree": {
"version": "7.3.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
"integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
"dev": true,
"requires": {
"acorn": "^7.4.0",
"acorn-jsx": "^5.2.0",
"eslint-visitor-keys": "^1.3.0"
}
},
"esrecurse": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
"integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
"dev": true,
"requires": {
"estraverse": "^5.2.0"
},
"dependencies": {
"estraverse": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
"integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
"dev": true
}
}
},
"glob-parent": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
"integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
"dev": true,
"requires": {
"is-glob": "^4.0.1"
}
},
"globals": {
"version": "12.4.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
"integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
"dev": true,
"requires": {
"type-fest": "^0.8.1"
}
},
"ignore": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
"integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
"dev": true
},
"path-key": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
"integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
"dev": true
},
"semver": {
"version": "7.3.2",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
"integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
"dev": true
},
"shebang-command": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
"integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
"dev": true,
"requires": {
"shebang-regex": "^3.0.0"
}
},
"shebang-regex": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
"dev": true
},
"which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
"dev": true,
"requires": {
"isexe": "^2.0.0"
}
}
}
},
"@blueprintjs/icons": {
"version": "3.19.0",
"resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.19.0.tgz",
@@ -4184,6 +4394,76 @@
"minimist": "^1.2.0"
}
},
"@eslint/eslintrc": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz",
"integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==",
"dev": true,
"requires": {
"ajv": "^6.12.4",
"debug": "^4.1.1",
"espree": "^7.3.0",
"globals": "^12.1.0",
"ignore": "^4.0.6",
"import-fresh": "^3.2.1",
"js-yaml": "^3.13.1",
"lodash": "^4.17.19",
"minimatch": "^3.0.4",
"strip-json-comments": "^3.1.1"
},
"dependencies": {
"acorn": {
"version": "7.4.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
"integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
"dev": true
},
"ajv": {
"version": "6.12.5",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz",
"integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==",
"dev": true,
"requires": {
"fast-deep-equal": "^3.1.1",
"fast-json-stable-stringify": "^2.0.0",
"json-schema-traverse": "^0.4.1",
"uri-js": "^4.2.2"
}
},
"eslint-visitor-keys": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
"dev": true
},
"espree": {
"version": "7.3.0",
"resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
"integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
"dev": true,
"requires": {
"acorn": "^7.4.0",
"acorn-jsx": "^5.2.0",
"eslint-visitor-keys": "^1.3.0"
}
},
"globals": {
"version": "12.4.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
"integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
"dev": true,
"requires": {
"type-fest": "^0.8.1"
}
},
"ignore": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
"integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
"dev": true
}
}
},
"@hapi/address": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz",
@@ -5137,6 +5417,32 @@
}
}
},
"@nodelib/fs.scandir": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz",
"integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==",
"dev": true,
"requires": {
"@nodelib/fs.stat": "2.0.3",
"run-parallel": "^1.1.9"
}
},
"@nodelib/fs.stat": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz",
"integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==",
"dev": true
},
"@nodelib/fs.walk": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz",
"integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==",
"dev": true,
"requires": {
"@nodelib/fs.scandir": "2.1.3",
"fastq": "^1.6.0"
}
},
"@npmcli/move-file": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.0.1.tgz",
@@ -5489,6 +5795,22 @@
"eslint-utils": "^2.0.0"
}
},
"@typescript-eslint/scope-manager": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.3.0.tgz",
"integrity": "sha512-cTeyP5SCNE8QBRfc+Lgh4Xpzje46kNUhXYfc3pQWmJif92sjrFuHT9hH4rtOkDTo/si9Klw53yIr+djqGZS1ig==",
"dev": true,
"requires": {
"@typescript-eslint/types": "4.3.0",
"@typescript-eslint/visitor-keys": "4.3.0"
}
},
"@typescript-eslint/types": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.3.0.tgz",
"integrity": "sha512-Cx9TpRvlRjOppGsU6Y6KcJnUDOelja2NNCX6AZwtVHRzaJkdytJWMuYiqi8mS35MRNA3cJSwDzXePfmhU6TANw==",
"dev": true
},
"@typescript-eslint/typescript-estree": {
"version": "2.34.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz",
@@ -5512,6 +5834,24 @@
}
}
},
"@typescript-eslint/visitor-keys": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.3.0.tgz",
"integrity": "sha512-xZxkuR7XLM6RhvLkgv9yYlTcBHnTULzfnw4i6+z2TGBLy9yljAypQaZl9c3zFvy7PNI7fYWyvKYtohyF8au3cw==",
"dev": true,
"requires": {
"@typescript-eslint/types": "4.3.0",
"eslint-visitor-keys": "^2.0.0"
},
"dependencies": {
"eslint-visitor-keys": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
"integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==",
"dev": true
}
}
},
"@webassemblyjs/ast": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz",
@@ -6387,9 +6727,9 @@
}
},
"bl": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz",
"integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==",
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
"integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
"dev": true,
"requires": {
"buffer": "^5.5.0",
@@ -6397,16 +6737,6 @@
"readable-stream": "^3.4.0"
},
"dependencies": {
"buffer": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz",
"integrity": "sha512-9FTEDjLjwoAkEwyMGDjYJQN2gfRgOKBKRfiglhvibGbpeeU/pQn1bJxQqm32OD/AIeEuHxU9roxXxg34Byp/Ww==",
"dev": true,
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4"
}
},
"readable-stream": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -7633,6 +7963,15 @@
"xdg-basedir": "^3.0.0"
},
"dependencies": {
"dot-prop": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.1.tgz",
"integrity": "sha512-l0p4+mIuJIua0mhxGoh4a+iNL9bmeK5DvnSVQa6T0OhrVmaEa1XScX5Etc673FePCJOArq/4Pa2cLGODUWTPOQ==",
"dev": true,
"requires": {
"is-obj": "^1.0.0"
}
},
"make-dir": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz",
@@ -8796,6 +9135,15 @@
}
}
},
"dir-glob": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
"integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",
"dev": true,
"requires": {
"path-type": "^4.0.0"
}
},
"doctrine": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
@@ -8935,12 +9283,20 @@
}
},
"dot-prop": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz",
"integrity": "sha512-tUMXrxlExSW6U2EXiiKGSBVdYgtV8qlHL+C10TsW4PURY/ic+eaysnSkwB4kA/mBlCyy/IKDJ+Lc3wbWeaXtuQ==",
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.3.0.tgz",
"integrity": "sha512-QM8q3zDe58hqUqjraQOmzZ1LIH9SWQJTlEKCH4kJ2oQvLZk7RbQXvtDM2XEq3fwkV9CCvvH4LA0AV+ogFsBM2Q==",
"dev": true,
"requires": {
"is-obj": "^1.0.0"
"is-obj": "^2.0.0"
},
"dependencies": {
"is-obj": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
"integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
"dev": true
}
}
},
"duplexer3": {
@@ -10187,6 +10543,31 @@
"integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==",
"dev": true
},
"fast-glob": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz",
"integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==",
"dev": true,
"requires": {
"@nodelib/fs.stat": "^2.0.2",
"@nodelib/fs.walk": "^1.2.3",
"glob-parent": "^5.1.0",
"merge2": "^1.3.0",
"micromatch": "^4.0.2",
"picomatch": "^2.2.1"
},
"dependencies": {
"glob-parent": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
"integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
"dev": true,
"requires": {
"is-glob": "^4.0.1"
}
}
}
},
"fast-json-stable-stringify": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@@ -10202,6 +10583,15 @@
"resolved": "https://registry.npmjs.org/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz",
"integrity": "sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw=="
},
"fastq": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz",
"integrity": "sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==",
"dev": true,
"requires": {
"reusify": "^1.0.4"
}
},
"favicons": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/favicons/-/favicons-5.5.0.tgz",
@@ -11159,6 +11549,28 @@
"resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
"integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA=="
},
"globby": {
"version": "11.0.1",
"resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz",
"integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==",
"dev": true,
"requires": {
"array-union": "^2.1.0",
"dir-glob": "^3.0.1",
"fast-glob": "^3.1.1",
"ignore": "^5.1.4",
"merge2": "^1.3.0",
"slash": "^3.0.0"
},
"dependencies": {
"array-union": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
"integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",
"dev": true
}
}
},
"got": {
"version": "6.7.1",
"resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
@@ -11691,6 +12103,12 @@
"integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=",
"dev": true
},
"ignore": {
"version": "5.1.8",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz",
"integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==",
"dev": true
},
"ignore-walk": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
@@ -14203,6 +14621,12 @@
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="
},
"merge2": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
"integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
"dev": true
},
"methods": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
@@ -15073,8 +15497,7 @@
"pako": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
"integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==",
"dev": true
"integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="
},
"parallel-transform": {
"version": "1.2.0",
@@ -15265,6 +15688,12 @@
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=",
"dev": true
},
"path-type": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
"integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",
"dev": true
},
"pbkdf2": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz",
@@ -15596,20 +16025,10 @@
"vendors": "^1.0.0"
},
"dependencies": {
"dot-prop": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz",
"integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==",
"dev": true,
"requires": {
"is-obj": "^2.0.0"
}
},
"is-obj": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
"integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
"dev": true
"integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="
},
"postcss-selector-parser": {
"version": "3.1.2",
@@ -15696,20 +16115,10 @@
"postcss-selector-parser": "^3.0.0"
},
"dependencies": {
"dot-prop": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz",
"integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==",
"dev": true,
"requires": {
"is-obj": "^2.0.0"
}
},
"is-obj": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
"integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
"dev": true
"integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="
},
"postcss-selector-parser": {
"version": "3.1.2",
@@ -17088,6 +17497,12 @@
"resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz",
"integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg=="
},
"reusify": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
"integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
"dev": true
},
"rgb-regex": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/rgb-regex/-/rgb-regex-1.0.1.tgz",
@@ -17123,6 +17538,12 @@
"resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz",
"integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA=="
},
"run-parallel": {
"version": "1.1.9",
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz",
"integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==",
"dev": true
},
"run-queue": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz",
@@ -18324,20 +18745,10 @@
"postcss-selector-parser": "^3.0.0"
},
"dependencies": {
"dot-prop": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz",
"integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==",
"dev": true,
"requires": {
"is-obj": "^2.0.0"
}
},
"is-obj": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
"integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==",
"dev": true
"integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="
},
"postcss-selector-parser": {
"version": "3.1.2",
client/package.json
+2
View File
@@ -54,6 +54,7 @@
"is-number": "^7.0.0",
"lodash": "^4.17.20",
"memoize-one": "^5.1.1",
"pako": "^1.0.11",
"react": "^16.13.1",
"react-async": "^10.0.1",
"react-dom": "^16.13.1",
@@ -85,6 +86,7 @@
"@babel/preset-react": "^7.10.4",
"@babel/register": "^7.10.5",
"@babel/runtime": "^7.10.5",
"@blueprintjs/eslint-plugin": "^0.3.0",
"@sentry/webpack-plugin": "^1.12.0",
"babel-eslint": "^10.1.0",
"babel-jest": "^26.1.0",
client/server/development.js
+22 -8
View File
@@ -1,5 +1,3 @@
const path = require("path");
const historyApiFallback = require("connect-history-api-fallback");
const chalk = require("chalk");
const express = require("express");
const favicon = require("serve-favicon");
@@ -11,35 +9,51 @@ const utils = require("./utils");
process.env.NODE_ENV = "development";
const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
const { CXG_SERVER_PORT } = process.env;
const API = {
prefix: `http://localhost:${CXG_SERVER_PORT}/`,
};
// Set up compiler
const compiler = webpack(config);
compiler.plugin("invalid", () => {
compiler.hooks.invalid.tap("invalid", () => {
utils.clearConsole();
console.log("Compiling...");
});
compiler.plugin("done", (stats) => {
compiler.hooks.done.tap("done", (stats) => {
utils.formatStats(stats, CLIENT_PORT);
});
// Launch server
const app = express();
app.use(historyApiFallback({ verbose: false }));
app.use(
devMiddleware(compiler, {
logLevel: "warn",
publicPath: config.output.publicPath,
index: true,
})
);
app.use(favicon("./favicon.png"));
app.get("*", (req, res) => {
res.sendFile(path.resolve("index.html"));
app.get("/login", async (req, res) => {
try {
res.redirect(`${API.prefix}login?dataset=http://localhost:${CLIENT_PORT}`);
} catch (err) {
console.error(err);
}
});
app.get("/logout", async (req, res) => {
try {
res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);
} catch (err) {
console.error(err);
}
});
app.listen(CLIENT_PORT, (err) => {
client/src/actions/annotation.js
+4 -2
View File
@@ -2,6 +2,7 @@
Action creators for user annotation
*/
import _ from "lodash";
import pako from "pako";
import * as globals from "../globals";
import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
@@ -153,7 +154,7 @@ export const annotationCreateLabelInCategory = (
assignSelected
) => async (dispatch, getState) => {
/*
Add a new label to a user-defined category. If assignSelected is true, assign
Add a new label to a user-defined category. If assignSelected is true, assign
the label to all currently selected cells.
*/
const {
@@ -347,6 +348,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
const matrix = MatrixFBS.encodeMatrixFBS(df);
const compressedMatrix = pako.deflate(matrix);
try {
const queryString =
!dataCollectionNameIsReadOnly && !!dataCollectionName
@@ -358,7 +360,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
`${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
{
method: "PUT",
body: matrix,
body: compressedMatrix,
headers: new Headers({
"Content-Type": "application/octet-stream",
}),
client/src/actions/embedding.js
+15 -9
View File
@@ -5,20 +5,23 @@ action creators related to embeddings choice
import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
export async function _switchEmbedding(prevAnnoMatrix, prevCrossfilter, newEmbeddingName) {
export async function _switchEmbedding(
prevAnnoMatrix,
prevCrossfilter,
newEmbeddingName
) {
/*
DRY helper used by this and reembedding action creators
*/
const base = prevAnnoMatrix.base();
const embeddingDf = await base.fetch("emb", newEmbeddingName);
const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix, prevCrossfilter.obsCrossfilter).select(
"emb",
newEmbeddingName,
{
mode: "all",
}
);
const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
annoMatrix,
prevCrossfilter.obsCrossfilter
).select("emb", newEmbeddingName, {
mode: "all",
});
return [annoMatrix, obsCrossfilter];
}
@@ -30,7 +33,10 @@ export const layoutChoiceAction = (newLayoutChoice) => async (
On layout choice, make sure we have selected all on the previous layout, AND the new
layout.
*/
const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
const {
annoMatrix: prevAnnoMatrix,
obsCrossfilter: prevCrossfilter,
} = getState();
const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
prevAnnoMatrix,
prevCrossfilter,
client/src/actions/index.js
+4 -4
View File
@@ -43,12 +43,12 @@ async function configFetch(dispatch) {
async function userInfoFetch(dispatch) {
return fetchJson("userinfo").then((response) => {
const { userinfo } = response || {};
const { userinfo: userInfo } = response || {};
dispatch({
type: "userinfo load complete",
userinfo,
type: "userInfo load complete",
userInfo,
});
return userinfo;
return userInfo;
});
}
client/src/actions/reembed.js
+5 -1
View File
@@ -79,10 +79,14 @@ export function requestReembed() {
type: "reembed: request completed",
});
const { annoMatrix: prevAnnoMatrix } = getState();
const {
annoMatrix: prevAnnoMatrix,
obsCrossfilter: prevCrossfilter,
} = getState();
const base = prevAnnoMatrix.base().addEmbedding(schema);
const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
base,
prevCrossfilter,
schema.name
);
dispatch({
client/src/components/autosave/filenameDialog.js
+9 -8
View File
@@ -3,18 +3,19 @@ import { connect } from "react-redux";
import {
Button,
Tooltip,
InputGroup,
Dialog,
Classes,
Code,
Colors,
Dialog,
InputGroup,
Tooltip,
} from "@blueprintjs/core";
@connect((state) => ({
idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
annotations: state.annotations,
auth: state.config?.authentication,
userinfo: state.userinfo,
userInfo: state.userInfo,
writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
}))
class FilenameDialog extends React.Component {
@@ -96,7 +97,7 @@ class FilenameDialog extends React.Component {
writableCategoriesEnabled,
annotations,
idhash,
userinfo,
userInfo,
} = this.props;
const { filenameText } = this.state;
@@ -104,7 +105,7 @@ class FilenameDialog extends React.Component {
annotations.promptForFilename &&
!annotations.dataCollectionNameIsReadOnly &&
!annotations.dataCollectionName &&
userinfo.is_authenticated ? (
userInfo.is_authenticated ? (
<Dialog
icon="tag"
title="Annotations Collection"
@@ -145,9 +146,9 @@ class FilenameDialog extends React.Component {
<div>
<p>
Your annotations are stored in this file:
<code className="bp3-code">
<Code>
{filenameText}-{idhash}.csv
</code>
</Code>
</p>
<p style={{ fontStyle: "italic" }}>
(We added a unique ID to your filename)
client/src/components/categorical/category/index.js
+17 -5
View File
@@ -1,7 +1,13 @@
import React, { useRef, useEffect } from "react";
import { connect, shallowEqual } from "react-redux";
import { FaChevronRight, FaChevronDown } from "react-icons/fa";
import { AnchorButton, Button, Tooltip, Position } from "@blueprintjs/core";
import {
AnchorButton,
Button,
Classes,
Position,
Tooltip,
} from "@blueprintjs/core";
import { Flipper, Flipped } from "react-flip-toolkit";
import Async from "react-async";
import memoize from "memoize-one";
@@ -301,9 +307,12 @@ const StillLoading = ({ metadataField, checkboxID }) => {
alignItems: "flex-start",
}}
>
<label htmlFor={checkboxID} className="bp3-control bp3-checkbox">
<label
htmlFor={checkboxID}
className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
>
<input disabled id={checkboxID} checked type="checkbox" />
<span className="bp3-control-indicator" />
<span className={Classes.CONTROL_INDICATOR} />
</label>
<Truncate>
<span
@@ -375,7 +384,10 @@ const CategoryHeader = React.memo(
alignItems: "flex-start",
}}
>
<label className="bp3-control bp3-checkbox" htmlFor={checkboxID}>
<label
className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
htmlFor={checkboxID}
>
<input
id={checkboxID}
data-testclass="category-select"
@@ -385,7 +397,7 @@ const CategoryHeader = React.memo(
checked={selectionState === "all"}
type="checkbox"
/>
<span className="bp3-control-indicator" />
<span className={Classes.CONTROL_INDICATOR} />
</label>
<span
role="menuitem"
client/src/components/categorical/index.js
+7 -5
View File
@@ -15,7 +15,7 @@ import actions from "../../actions";
writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
schema: state.annoMatrix?.schema,
ontology: state.ontology,
userinfo: state.userinfo,
userInfo: state.userInfo,
}))
class Categories extends React.Component {
constructor(props) {
@@ -132,7 +132,7 @@ class Categories extends React.Component {
writableCategoriesEnabled,
schema,
ontology,
userinfo,
userInfo,
} = this.props;
const ontologyEnabled = ontology?.enabled ?? false;
/* all names, sorted in display order. Will be rendered in this order */
@@ -186,7 +186,7 @@ class Categories extends React.Component {
<div style={{ marginBottom: 10 }}>
<Tooltip
content={
userinfo.is_authenticated
userInfo.is_authenticated
? "Create a new category"
: "You must be logged in to create new categorical fields"
}
@@ -203,7 +203,7 @@ class Categories extends React.Component {
data-testid="open-annotation-dialog"
onClick={this.handleEnableAnnoMode}
intent="primary"
disabled={!userinfo.is_authenticated}
disabled={!userInfo.is_authenticated}
>
Create new <strong>category</strong>
</AnchorButton>
@@ -214,7 +214,9 @@ class Categories extends React.Component {
{/* READ ONLY CATEGORICAL FIELDS */}
{/* this is duplicative but flat, could be abstracted */}
{allCategoryNames.map((catName) =>
!schema.annotations.obsByName[catName].writable ? (
!schema.annotations.obsByName[catName].writable &&
(schema.annotations.obsByName[catName].categories?.length > 1 ||
!schema.annotations.obsByName[catName].categories) ? (
<Category
key={catName}
metadataField={catName}
client/src/components/categorical/value/index.js
+5 -8
View File
@@ -4,12 +4,13 @@ import * as d3 from "d3";
import {
Button,
Classes,
Icon,
Menu,
MenuItem,
Popover,
Position,
Icon,
PopoverInteractionKind,
Position,
} from "@blueprintjs/core";
import * as globals from "../../../globals";
import styles from "../categorical.css";
@@ -410,7 +411,6 @@ class CategoryValue extends React.Component {
return (
<MiniStackedBar
/* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
{...{
colorTable,
domainValues,
@@ -418,7 +418,6 @@ class CategoryValue extends React.Component {
domain,
occupancy,
}}
/* eslint-enable react/jsx-props-no-spreading -- enable */
height={VALUE_HEIGHT}
width={CHART_WIDTH}
/>
@@ -461,14 +460,12 @@ class CategoryValue extends React.Component {
return (
<MiniHistogram
/* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
{...{
colorScale,
xScale,
yScale,
bins,
}}
/* eslint-enable react/jsx-props-no-spreading -- enable */
obsOrVarContinuousFieldDisplayName={colorAccessor}
domainLabel={label}
height={VALUE_HEIGHT}
@@ -564,7 +561,7 @@ class CategoryValue extends React.Component {
<div style={{ display: "flex", alignItems: "baseline" }}>
<label
htmlFor={valueToggleLabel}
className="bp3-control bp3-checkbox"
className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
style={{ margin: 0 }}
>
<input
@@ -576,7 +573,7 @@ class CategoryValue extends React.Component {
type="checkbox"
/>
<span
className="bp3-control-indicator"
className={Classes.CONTROL_INDICATOR}
onMouseEnter={this.handleMouseExit}
onMouseLeave={this.handleMouseEnter}
/>
client/src/components/categorical/value/occupancy.js
+6 -6
View File
@@ -3,10 +3,10 @@ import React from "react";
import { connect } from "react-redux";
import * as d3 from "d3";
import {
Classes,
Popover,
PopoverInteractionKind,
Position,
Classes,
} from "@blueprintjs/core";
@connect((state) => ({
@@ -18,8 +18,8 @@ class Occupancy extends React.PureComponent {
_HEIGHT = 11;
createHistogram = () => {
/*
Knowing that colorScale is based off continous data,
/*
Knowing that colorScale is based off continous data,
createHistogram fetches the continous data in relation to the cells releveant to the catagory value.
It then seperates that data into 50 bins for drawing the mini-histogram
*/
@@ -75,8 +75,8 @@ class Occupancy extends React.PureComponent {
};
createOccupancyStack = () => {
/*
Knowing that the color scale is based off of catagorical data,
/*
Knowing that the color scale is based off of catagorical data,
createOccupancyStack obtains a map showing the number if cells per colored value
Using the colorScale a stack of colored bars is drawn representing the map
*/
@@ -155,7 +155,7 @@ class Occupancy extends React.PureComponent {
popoverClassName={Classes.POPOVER_CONTENT_SIZING}
>
<canvas
className="bp3-popover-targer"
className={Classes.POPOVER_TARGET}
style={{
marginRight: 5,
width: this._WIDTH,
client/src/components/continuousLegend/index.js
+25 -58
View File
@@ -11,20 +11,20 @@ import {
// create continuous color legend
// http://bl.ocks.org/syntagmatic/e8ccca52559796be775553b467593a9f
const continuous = (selectorId, colorscale, colorAccessor) => {
const legendheight = 200;
const legendwidth = 80;
const continuous = (selectorId, colorScale, colorAccessor) => {
const legendHeight = 200;
const legendWidth = 80;
const margin = { top: 10, right: 60, bottom: 10, left: 2 };
const canvas = d3
.select(selectorId)
.style("height", `${legendheight}px`)
.style("width", `${legendwidth}px`)
.style("height", `${legendHeight}px`)
.style("width", `${legendWidth}px`)
.append("canvas")
.attr("height", legendheight - margin.top - margin.bottom)
.attr("height", legendHeight - margin.top - margin.bottom)
.attr("width", 1)
.style("height", `${legendheight - margin.top - margin.bottom}px`)
.style("width", `${legendwidth - margin.left - margin.right}px`)
.style("height", `${legendHeight - margin.top - margin.bottom}px`)
.style("width", `${legendWidth - margin.left - margin.right}px`)
.style("position", "absolute")
.style("top", `${margin.top + 1}px`)
.style("left", `${margin.left + 1}px`)
@@ -37,18 +37,18 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
const ctx = canvas.getContext("2d");
const legendscale = d3
const legendScale = d3
.scaleLinear()
.range([1, legendheight - margin.top - margin.bottom])
.range([1, legendHeight - margin.top - margin.bottom])
.domain([
colorscale.domain()[1],
colorscale.domain()[0],
colorScale.domain()[1],
colorScale.domain()[0],
]); /* we flip this to make viridis colors dark if high in the color scale */
// image data hackery based on http://bl.ocks.org/mbostock/048d21cf747371b11884f75ad896e5a5
const image = ctx.createImageData(1, legendheight);
d3.range(legendheight).forEach((i) => {
const c = d3.rgb(colorscale(legendscale.invert(i)));
const image = ctx.createImageData(1, legendHeight);
d3.range(legendHeight).forEach((i) => {
const c = d3.rgb(colorScale(legendScale.invert(i)));
image.data[4 * i] = c.r;
image.data[4 * i + 1] = c.g;
image.data[4 * i + 2] = c.b;
@@ -66,20 +66,20 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
});
*/
const legendaxis = d3
.axisRight(legendscale)
const legendAxis = d3
.axisRight(legendScale)
.ticks(6)
.tickFormat(
d3.format(
legendscale.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
legendScale.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
)
);
const svg = d3
.select(selectorId)
.append("svg")
.attr("height", `${legendheight}px`)
.attr("width", `${legendwidth}px`)
.attr("height", `${legendHeight}px`)
.attr("width", `${legendWidth}px`)
.style("position", "absolute")
.style("left", "0px")
.style("top", "0px");
@@ -89,16 +89,16 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
.attr("class", "axis")
.attr(
"transform",
`translate(${legendwidth - margin.left - margin.right + 3},${margin.top})`
`translate(${legendWidth - margin.left - margin.right + 3},${margin.top})`
)
.call(legendaxis);
.call(legendAxis);
// text label for the y axis
svg
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 2)
.attr("x", 0 - legendheight / 2)
.attr("x", 0 - legendHeight / 2)
.attr("dy", "1em")
.style("text-anchor", "middle")
.style("fill", "white")
@@ -110,24 +110,7 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
colors: state.colors,
}))
class ContinuousLegend extends React.Component {
constructor(props) {
super(props);
this.ref = null;
this.state = {
colorAccessor: null,
colorScale: null,
};
}
componentDidMount() {
this.updateState(null);
}
componentDidUpdate(prevProps) {
this.updateState(prevProps);
}
async updateState(prevProps) {
async componentDidUpdate(prevProps) {
const { annoMatrix, colors } = this.props;
if (!colors || !annoMatrix) return;
@@ -161,35 +144,19 @@ class ContinuousLegend extends React.Component {
);
}
}
this.setState({
colorAccessor,
colorScale: colorTable.scale,
});
}
}
render() {
const { colorAccessor, colorScale } = this.state;
if (
colorScale?.domain &&
colorScale.domain()[1] === colorScale.domain()[0]
) {
/* it's a single value, not a distribution, min max are the same */
return null;
}
return (
<div
id="continuous_legend"
ref={(ref) => {
this.ref = ref;
}}
style={{
display: colorAccessor ? "inherit" : "none",
position: "absolute",
left: 8,
top: 35,
zIndex: 1,
pointerEvents: "none",
}}
/>
);
client/src/components/embedding/index.js
+5 -4
View File
@@ -2,13 +2,14 @@ import React from "react";
import { connect } from "react-redux";
import { useAsync } from "react-async";
import {
ButtonGroup,
Popover,
Button,
ButtonGroup,
H4,
Popover,
Position,
Radio,
RadioGroup,
Tooltip,
Position,
} from "@blueprintjs/core";
import * as globals from "../../globals";
import actions from "../../actions";
@@ -80,7 +81,7 @@ class Embedding extends React.PureComponent {
width: 400,
}}
>
<h1>Embedding Choice</h1>
<H4>Embedding Choice</H4>
<p style={{ fontStyle: "italic" }}>
There are {schema?.dataframe?.nObs} cells in the entire dataset.
</p>
client/src/components/geneExpression/menus/addGenes.js
+4 -3
View File
@@ -6,11 +6,12 @@ import fuzzysort from "fuzzysort";
import { connect } from "react-redux";
import { Suggest } from "@blueprintjs/select";
import {
MenuItem,
Button,
ControlGroup,
FormGroup,
InputGroup,
ControlGroup,
Intent,
MenuItem,
} from "@blueprintjs/core";
import * as globals from "../../../globals";
import actions from "../../../actions";
@@ -278,7 +279,7 @@ class AddGenes extends React.Component {
popoverProps={{ minimal: true }}
/>
<Button
className="bp3-button bp3-intent-primary"
intent={Intent.PRIMARY}
data-testid="add-gene"
loading={userDefinedGenesLoading}
onClick={() => this.handleClick(activeItem)}
client/src/components/graph/graph.js
+17 -6
View File
@@ -730,14 +730,25 @@ class Graph extends React.Component {
);
});
updateReglAndRender(asyncProps) {
updateReglAndRender(asyncProps, prevAsyncProps) {
const { positions, colors, flags } = asyncProps;
this.cachedAsyncProps = asyncProps;
const { pointBuffer, colorBuffer, flagBuffer } = this.state;
pointBuffer({ data: positions, dimension: 2 });
colorBuffer({ data: colors, dimension: 3 });
flagBuffer({ data: flags, dimension: 1 });
this.renderCanvas();
let needToRenderCanvas = false;
if (positions !== prevAsyncProps?.positions) {
pointBuffer({ data: positions, dimension: 2 });
needToRenderCanvas = true;
}
if (colors !== prevAsyncProps?.colors) {
colorBuffer({ data: colors, dimension: 3 });
needToRenderCanvas = true;
}
if (flags !== prevAsyncProps?.flags) {
flagBuffer({ data: flags, dimension: 1 });
needToRenderCanvas = true;
}
if (needToRenderCanvas) this.renderCanvas();
}
updateColorTable(colors, colorDf) {
@@ -906,7 +917,7 @@ class Graph extends React.Component {
<Async.Fulfilled>
{(asyncProps) => {
if (regl && !shallowEqual(asyncProps, this.cachedAsyncProps)) {
this.updateReglAndRender(asyncProps);
this.updateReglAndRender(asyncProps, this.cachedAsyncProps);
}
return null;
}}
client/src/components/graph/setupLasso.js
+24 -5
View File
@@ -10,6 +10,7 @@ const Lasso = () => {
let lassoPolygon;
let lassoPath;
let closePath;
let lassoInProgress;
const polygonToPath = (polygon) =>
`M${polygon.map((d) => d.join(",")).join("L")}`;
@@ -25,8 +26,18 @@ const Lasso = () => {
lassoPolygon = [d3.mouse(svg.node())]; // current x y of mouse within element
if (lassoPath) {
// If the existing path is in progress
if (lassoInProgress) {
// cancel the existing lasso
handleCancel();
// Don't continue with current drag start
return;
}
lassoPath.remove();
}
// We're starting a new drag
lassoInProgress = true;
lassoPath = g
.append("path")
@@ -67,25 +78,33 @@ const Lasso = () => {
}
};
const handleCancel = () => {
lassoPath.remove();
closePath = closePath?.remove();
lassoPath = null;
lassoPolygon = null;
closePath = null;
dispatch.call("cancel");
};
const handleDragEnd = () => {
// remove the close path
closePath.remove();
closePath = null;
// succesfully closed
// successfully closed
if (
distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
closeDistance
) {
lassoInProgress = false;
lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
dispatch.call("end", lasso, lassoPolygon);
// otherwise cancel
} else {
lassoPath.remove();
lassoPath = null;
lassoPolygon = null;
dispatch.call("cancel");
handleCancel();
}
};
client/src/components/infoDrawer/infoDrawer.js
+22 -77
View File
@@ -1,61 +1,20 @@
import React, { PureComponent } from "react";
import { connect, shallowEqual } from "react-redux";
import { connect } from "react-redux";
import { Drawer } from "@blueprintjs/core";
import Async from "react-async";
import InfoFormat from "./infoFormat";
import {
selectableCategoryNames,
createCategorySummaryFromDfCol,
} from "../../util/stateManager/controlsHelpers";
import { selectableCategoryNames } from "../../util/stateManager/controlsHelpers";
@connect((state) => {
return {
annoMatrix: state.annoMatrix,
schema: state.annoMatrix.schema,
datasetTitle: state.config?.displayNames?.dataset ?? "",
aboutURL: state.config?.links?.["about-dataset"],
isOpen: state.controls.datasetDrawer,
dataPortalProps: state.config?.["corpora_props"] ?? {},
dataPortalProps: state.config?.["corpora_props"],
};
})
class InfoDrawer extends PureComponent {
static watchAsync(props, prevProps) {
return !shallowEqual(props.watchProps, prevProps.watchProps);
}
fetchAsyncProps = async (props) => {
const { schema } = props.watchProps;
const { annoMatrix } = this.props;
const allCategoryNames = selectableCategoryNames(schema).sort();
const nonUserAnnoCategories = allCategoryNames.map((catName) => {
const isUserAnno = schema?.annotations?.obsByName[catName]?.writable;
if (!isUserAnno) return annoMatrix.fetch("obs", catName);
return null;
});
const singleValueCategories = (
await Promise.all(nonUserAnnoCategories)
).reduce((acc, categoryData, i) => {
const catName = allCategoryNames[i];
const column = categoryData.icol(0);
const colSchema = schema.annotations.obsByName[catName];
const categorySummary = createCategorySummaryFromDfCol(column, colSchema);
const { numCategoryValues } = categorySummary;
// Add to the array if the category has only one value
if (numCategoryValues === 1) {
acc.set(catName, categorySummary.allCategoryValues[0]);
}
return acc;
}, new Map());
return { singleValueCategories };
};
handleClose = () => {
const { dispatch } = this.props;
@@ -72,45 +31,31 @@ class InfoDrawer extends PureComponent {
dataPortalProps,
} = this.props;
const allCategoryNames = selectableCategoryNames(schema).sort();
const singleValueCategories = new Map();
allCategoryNames.forEach((catName) => {
const isUserAnno = schema?.annotations?.obsByName[catName]?.writable;
const colSchema = schema.annotations.obsByName[catName];
if (!isUserAnno && colSchema.categories?.length === 1) {
singleValueCategories.set(catName, colSchema.categories[0]);
}
});
return (
<Drawer
title="Dataset Overview"
onClose={this.handleClose}
{...{ isOpen, position }}
>
<Async
watchFn={InfoDrawer.watchAsync}
promiseFn={this.fetchAsyncProps}
watchProps={{ schema }}
>
<Async.Pending>
<InfoFormat
skeleton
{...{ datasetTitle, aboutURL, dataPortalProps }}
/>
</Async.Pending>
<Async.Rejected>
{(error) => {
console.error(error);
return <span>Failed to load info</span>;
}}
</Async.Rejected>
<Async.Fulfilled>
{(asyncProps) => {
const { singleValueCategories } = asyncProps;
return (
<InfoFormat
{...{
datasetTitle,
aboutURL,
singleValueCategories,
dataPortalProps,
}}
/>
);
}}
</Async.Fulfilled>
</Async>
<InfoFormat
{...{
datasetTitle,
aboutURL,
singleValueCategories,
dataPortalProps: dataPortalProps ?? {},
}}
/>
</Drawer>
);
}
client/src/components/infoDrawer/infoFormat.js
+80 -75
View File
@@ -1,13 +1,13 @@
import { H3, H1, UL, Classes } from "@blueprintjs/core";
import { H3, H1, UL, HTMLTable, Classes } from "@blueprintjs/core";
import React from "react";
const renderContributors = (contributors, affiliations, skeleton) => {
const renderContributors = (contributors, affiliations) => {
// eslint-disable-next-line no-constant-condition -- Temp removed contributor section to avoid publishing PII
if (!contributors || contributors.length === 0 || true) return null;
return (
<>
<H3 className={skeleton ? Classes.SKELETON : null}>Contributors</H3>
<p className={skeleton ? Classes.SKELETON : null}>
<H3>Contributors</H3>
<p>
{contributors.map((contributor) => {
const { email, name, institution } = contributor;
@@ -20,7 +20,7 @@ const renderContributors = (contributors, affiliations, skeleton) => {
);
})}
</p>
{renderAffiliations(affiliations, skeleton)}
{renderAffiliations(affiliations)}
</>
);
};
@@ -37,14 +37,14 @@ const buildAffiliations = (contributors = []) => {
return affiliations;
};
const renderAffiliations = (affiliations, skeleton) => {
const renderAffiliations = (affiliations) => {
if (affiliations.length === 0) return null;
return (
<>
<H3 className={skeleton ? Classes.SKELETON : null}>Affiliations</H3>
<H3>Affiliations</H3>
<UL>
{affiliations.map((item, index) => (
<div key={item} className={skeleton ? Classes.SKELETON : null}>
<div key={item}>
<sup>{index + 1}</sup>
{" "}
{item}
@@ -55,12 +55,12 @@ const renderAffiliations = (affiliations, skeleton) => {
);
};
const renderDOILink = (type, doi, skeleton) => {
const renderDOILink = (type, doi) => {
if (!doi) return null;
return (
<>
<H3 className={skeleton ? Classes.SKELETON : null}>{type}</H3>
<p className={skeleton ? Classes.SKELETON : null}>
<H3>{type}</H3>
<p>
<a href={doi} target="_blank" rel="noopener">
{doi}
</a>
@@ -69,54 +69,80 @@ const renderDOILink = (type, doi, skeleton) => {
);
};
const renderOrganism = (organism, skeleton) => {
if (!organism) return null;
return (
<>
<H3 className={skeleton ? Classes.SKELETON : null}>Organism</H3>
<p className={skeleton ? Classes.SKELETON : null}>{organism}</p>
</>
);
};
const ONTOLOGY_KEY = "ontology_term_id";
// Render list of metadata attributes found in categorical field
// Ignores categories with empty or null values
const renderSingleValueCategories = (singleValueCategories, skeleton) => {
const renderDatasetMetadata = (singleValueCategories, corporaMetadata) => {
if (singleValueCategories.size === 0) return null;
return (
<>
<H3 className={skeleton ? Classes.SKELETON : null}>Dataset Metadata</H3>
<UL>
{Array.from(singleValueCategories).map((pair) => {
if (!pair[1] || pair[1] === "") return null;
return (
<li
className={skeleton ? Classes.SKELETON : null}
key={pair[0]}
>{`${pair[0]}: ${pair[1]}`}</li>
);
})}
</UL>
<H3>Dataset Metadata</H3>
<HTMLTable
striped
condensed
style={{ display: "block", width: "100%", overflowX: "auto" }}
>
<thead>
<tr>
<th>Field</th>
<th>Label</th>
<th>Ontology ID</th>
</tr>
</thead>
<tbody>
{Object.entries(corporaMetadata).map(([key, value]) => {
return (
<tr {...{ key }}>
<td>{`${key}:`}</td>
<td>{value}</td>
<td />
</tr>
);
})}
{Array.from(singleValueCategories).reduce((elems, pair) => {
const [category, value] = pair;
// If the value is empty skip it
if (!value) return elems;
// If this category is a ontology term, let's add its value to the previous node
if (String(category).includes(ONTOLOGY_KEY)) {
const prevElem = elems.pop();
const newChildren = [...prevElem.props.children];
newChildren.splice(2, 1, [<td key="ontology">{value}</td>]);
// Props aren't extensible so we must clone and alter the component to append the new child
elems.push(
React.cloneElement(prevElem, prevElem.props, newChildren)
);
} else {
// Create the list item
elems.push(
<tr key={category}>
<td>{`${category}:`}</td>
<td>{value}</td>
<td />
</tr>
);
}
return elems;
}, [])}
</tbody>
</HTMLTable>
</>
);
};
// Renders any links found in the config where link_type is not "SUMMARY"
// If there are no links in the config, render the aboutURL
const renderLinks = (projectLinks, aboutURL, skeleton) => {
const renderLinks = (projectLinks, aboutURL) => {
if (!projectLinks && !aboutURL) return null;
if (projectLinks)
return (
<>
<H3 className={skeleton ? Classes.SKELETON : null}>Project Links</H3>
<H3>Project Links</H3>
<UL>
{projectLinks.map((link) => {
if (link.link_type === "SUMMARY") return null;
return (
<li
key={link.link_name}
className={skeleton ? Classes.SKELETON : null}
>
<li key={link.link_name}>
<a href={link.link_url} target="_blank" rel="noopener">
{link.link_name}
</a>
@@ -129,14 +155,9 @@ const renderLinks = (projectLinks, aboutURL, skeleton) => {
return (
<>
<H3 className={skeleton ? Classes.SKELETON : null}>More Info</H3>
<H3>More Info</H3>
<p>
<a
className={skeleton ? Classes.SKELETON : null}
href={aboutURL}
target="_blank"
rel="noopener"
>
<a href={aboutURL} target="_blank" rel="noopener">
{aboutURL}
</a>
</p>
@@ -144,24 +165,9 @@ const renderLinks = (projectLinks, aboutURL, skeleton) => {
);
};
const NUM_CATEGORIES = 8;
// Generates arbitrary placeholder array for singleValueCategories skeleton shape
const singleValueCategoriesPlaceholder = Array.from(Array(NUM_CATEGORIES)).map(
(_, index) => {
return [index, index];
}
);
const InfoFormat = React.memo(
({
datasetTitle,
singleValueCategories = new Map(singleValueCategoriesPlaceholder),
aboutURL = "thisisabouthtelengthofaurl",
dataPortalProps = {},
skeleton = false,
}) => {
if (dataPortalProps.corpora_schema_version === "1.0.0") {
({ datasetTitle, singleValueCategories, aboutURL, dataPortalProps = {} }) => {
if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") {
dataPortalProps = {};
}
const {
@@ -176,16 +182,15 @@ const InfoFormat = React.memo(
const affiliations = buildAffiliations(contributors);
return (
<div style={{ margin: 24, overflow: "auto" }}>
<H1 className={skeleton ? Classes.SKELETON : null}>
{title ?? datasetTitle}
</H1>
{renderContributors(contributors, affiliations, skeleton)}
{renderDOILink("DOI", doi, skeleton)}
{renderDOILink("Preprint DOI", preprintDOI, skeleton)}
{renderOrganism(organism, skeleton)}
{renderSingleValueCategories(singleValueCategories, skeleton)}
{renderLinks(projectLinks, aboutURL, skeleton)}
<div className={Classes.DIALOG_BODY}>
<div className={Classes.DIALOG_BODY}>
<H1>{title ?? datasetTitle}</H1>
{renderContributors(contributors, affiliations)}
{renderDatasetMetadata(singleValueCategories, { organism })}
{renderLinks(projectLinks, aboutURL)}
{renderDOILink("DOI", doi)}
{renderDOILink("Preprint DOI", preprintDOI)}
</div>
</div>
);
}
client/src/components/leftSidebar/infoMenu.js
+72
View File
@@ -0,0 +1,72 @@
// jshint esversion: 6
import React from "react";
import { Button, Menu, MenuItem, Popover, Position } from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
const InformationMenu = React.memo((props) => {
const { libraryVersions, tosURL, privacyURL } = props;
return (
<Popover
content={
<Menu>
<MenuItem
href="https://chanzuckerberg.github.io/cellxgene/"
target="_blank"
icon="book"
text="Documentation"
rel="noopener"
/>
<MenuItem
href="https://join-cellxgene-users.herokuapp.com/"
target="_blank"
icon="chat"
text="Chat"
rel="noopener"
/>
<MenuItem
href="https://github.com/chanzuckerberg/cellxgene"
target="_blank"
icon="git-branch"
text="Github"
rel="noopener"
/>
<MenuItem target="_blank" text={libraryVersions?.cellxgene || null} />
<MenuItem text="MIT License" />
{tosURL && (
<MenuItem
href={tosURL}
target="_blank"
text="Terms of Service"
rel="noopener"
/>
)}
{privacyURL && (
<MenuItem
href={privacyURL}
target="_blank"
text="Privacy Policy"
rel="noopener"
/>
)}
</Menu>
}
position={Position.BOTTOM_RIGHT}
modifiers={{
preventOverflow: { enabled: false },
hide: { enabled: false },
}}
>
<Button
data-testid="menu"
type="button"
icon={IconNames.INFO_SIGN}
style={{
cursor: "pointer",
verticalAlign: "middle",
}}
/>
</Popover>
);
});
export default InformationMenu;
client/src/components/leftSidebar/topLeftLogoAndTitle.js
+74 -42
View File
@@ -1,19 +1,28 @@
// jshint esversion: 6
import React from "react";
import { connect } from "react-redux";
import { Button } from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import * as globals from "../../globals";
import Logo from "../framework/logo";
import Truncate from "../util/truncate";
import InfoDrawer from "../infoDrawer/infoDrawer";
import InformationMenu from "./infoMenu";
const DATASET_TITLE_FONT_SIZE = 14;
@connect((state) => ({
datasetTitle: state.config?.displayNames?.dataset ?? "",
}))
@connect((state) => {
const { corpora_props: corporaProps } = state.config;
const correctVersion =
corporaProps?.version?.["corpora_schema_version"] === "1.0.0";
return {
datasetTitle: state.config?.displayNames?.dataset ?? "",
libraryVersions: state.config?.["library_versions"],
aboutLink: state.config?.links?.["about-dataset"],
tosURL: state.config?.parameters?.["about_legal_tos"],
privacyURL: state.config?.parameters?.["about_legal_privacy"],
title: correctVersion ? corporaProps?.title : undefined,
};
})
class LeftSideBar extends React.Component {
handleClick = () => {
const { dispatch } = this.props;
@@ -21,7 +30,15 @@ class LeftSideBar extends React.Component {
};
render() {
const { datasetTitle } = this.props;
const {
datasetTitle,
libraryVersions,
aboutLink,
privacyURL,
tosURL,
dispatch,
title,
} = this.props;
return (
<div
@@ -31,50 +48,65 @@ class LeftSideBar extends React.Component {
width: globals.leftSidebarWidth,
zIndex: 1,
borderBottom: `1px solid ${globals.lighterGrey}`,
display: "flex",
justifyContent: "space-between",
alignItems: "center",
}}
>
<Logo size={30} />
<span
style={{
fontSize: 28,
position: "relative",
top: -6,
fontWeight: "bold",
marginLeft: 5,
color: globals.logoColor,
userSelect: "none",
}}
>
cell
<div>
<Logo size={28} />
<span
style={{
position: "relative",
top: 1,
fontWeight: 300,
fontSize: 24,
position: "relative",
top: -6,
fontWeight: "bold",
marginLeft: 5,
color: globals.logoColor,
userSelect: "none",
}}
>
×
</span>
gene
</span>
<Button
minimal
icon={IconNames.BOOK}
style={{
fontSize: DATASET_TITLE_FONT_SIZE,
position: "absolute",
right: 10,
}}
onClick={this.handleClick}
>
<Truncate>
<span style={{ maxWidth: 155 }} data-testid="header">
{datasetTitle}
cell
<span
style={{
position: "relative",
top: 1,
fontWeight: 300,
fontSize: 24,
}}
>
×
</span>
</Truncate>
</Button>
<InfoDrawer />
gene
</span>
</div>
<div style={{ marginRight: 5, height: "100%" }}>
<Button
minimal
style={{
fontSize: DATASET_TITLE_FONT_SIZE,
position: "relative",
top: -1,
}}
onClick={this.handleClick}
>
<Truncate>
<span style={{ maxWidth: 155 }} data-testid="header">
{title ?? datasetTitle}
</span>
</Truncate>
</Button>
<InfoDrawer />
<InformationMenu
{...{
libraryVersions,
aboutLink,
tosURL,
privacyURL,
dispatch,
}}
/>
</div>
</div>
);
}
client/src/components/menubar/authButtons.js
+165 -22
View File
@@ -1,32 +1,175 @@
import React from "react";
import { AnchorButton, Tooltip } from "@blueprintjs/core";
import React, { useState } from "react";
import {
AnchorButton,
Button,
MenuItem,
Tooltip,
Popover,
Menu,
Elevation,
PopoverPosition,
Checkbox,
Card,
} from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import * as globals from "../../globals";
import styles from "./menubar.css";
import { storageGet, storageSet, KEYS } from "../util/localStorage";
const BASE_EMOJI = [0x1f9d1, 0x1f468, 0x1f469];
const SKIN_TONES = [0x1f3fb, 0x1f3fc, 0x1f3fd, 0x1f3fe, 0x1f3ff];
const MICROSCOPE = 0x1f52c;
const ZERO_WIDTH_JOINER = 0x0200d;
const LOGIN_PROMPT_OFF = "off";
const Auth = React.memo((props) => {
const { auth, userinfo } = props;
const [isPromptOpen, setIsPromptOpen] = useState(shouldShowPrompt());
if (!auth || (auth && !auth.requires_client_login)) return null;
const { auth, userInfo } = props;
return (
<div className={`bp3-button-group ${styles.menubarButton}`}>
<Tooltip
content="Log in or log out of cellxgene"
position="bottom"
hoverOpenDelay={globals.tooltipHoverOpenDelay}
>
<AnchorButton
type="button"
data-testid="auth-button"
disabled={false}
icon={!userinfo.is_authenticated ? "log-in" : "log-out"}
href={!userinfo.is_authenticated ? auth.login : auth.logout}
>
{!userinfo.is_authenticated ? "Log In" : "Log Out"}
</AnchorButton>
</Tooltip>
</div>
const isAuthenticated = userInfo && userInfo.is_authenticated;
window.userInfo = userInfo;
const randomInt = Math.random() * 15;
const sexIndex = Math.floor(randomInt / 5);
const skinToneIndex = Math.floor(randomInt % 5);
const scientist = String.fromCodePoint(
BASE_EMOJI[sexIndex],
SKIN_TONES[skinToneIndex],
ZERO_WIDTH_JOINER,
MICROSCOPE
);
if (!shouldShowAuth()) return null;
if (isAuthenticated) {
const PopoverContent = (
<Menu>
<MenuItem
data-testid="user-email"
text={`Logged in as: ${userInfo.email}`}
/>
<MenuItem
data-testid="log-out"
text="Log Out"
href={auth.logout}
icon={IconNames.LOG_OUT}
/>
</Menu>
);
return (
<Popover content={PopoverContent}>
<Button
data-testid="user-info"
className={styles.menubarButton}
style={{ padding: 0 }}
>
{/* eslint-disable-next-line no-constant-condition -- disable profile picture until CSP is tweaked */}
{userInfo?.picture && false ? (
<img alt="profile" size="21px" src={userInfo?.picture} />
) : (
<span style={{ fontSize: "18px" }}>{scientist}</span>
)}
</Button>
</Popover>
);
}
const LoginButton = (
<Tooltip
content="Log in to cellxgene"
position="bottom"
hoverOpenDelay={globals.tooltipHoverOpenDelay}
>
<AnchorButton
type="button"
data-testid="log-in"
href={auth.login}
className={styles.menubarButton}
>
Log In
</AnchorButton>
</Tooltip>
);
if (isPromptOpen) {
return (
<Popover
position={PopoverPosition.AUTO_END}
isOpen
content={<PromptContent setIsPromptOpen={setIsPromptOpen} />}
onInteraction={setIsPromptOpen}
>
{LoginButton}
</Popover>
);
}
return LoginButton;
function shouldShowAuth() {
return auth && auth.requires_client_login;
}
function shouldShowPrompt() {
if (storageGet(KEYS.LOGIN_PROMPT) === LOGIN_PROMPT_OFF) return false;
return shouldShowAuth && !isAuthenticated;
}
});
function PromptContent({ setIsPromptOpen }) {
const [isChecked, setIsChecked] = useState(false);
function handleOKClick() {
if (isChecked) {
storageSet(KEYS.LOGIN_PROMPT, LOGIN_PROMPT_OFF);
}
setIsPromptOpen(false);
}
function handleCheckboxChange() {
setIsChecked(!isChecked);
}
return (
<Card style={{ width: "500px" }} elevation={Elevation.TWO}>
<p>
Logging in will enable you to create your own categories and labels.
Logging in later will reset cellxgene to the default view and cause you
to lose progress.
</p>
<Checkbox
style={{ width: "230px" }}
checked={isChecked}
onChange={handleCheckboxChange}
data-testid="login-hint-do-not-show-again"
>
Do not show me this message again
</Checkbox>
<div
style={{ display: "flex", justifyContent: "flex-end", marginTop: 15 }}
>
<Button
onClick={handleOKClick}
intent="primary"
data-testid="login-hint-yes"
>
Acknowledge
</Button>
</div>
</Card>
);
}
export default Auth;
client/src/components/menubar/clip.js
+14 -9
View File
@@ -1,12 +1,16 @@
import React from "react";
import {
Position,
Button,
Popover,
NumericInput,
ButtonGroup,
Icon,
Intent,
NumericInput,
Popover,
Position,
Tooltip,
} from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import { tooltipHoverOpenDelay } from "../../globals";
import styles from "./menubar.css";
@@ -28,13 +32,13 @@ const Clip = React.memo((props) => {
pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin;
const clipMax =
pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax;
const activeClipClass =
const intent =
clipPercentileMin > 0 || clipPercentileMax < 100
? " bp3-intent-warning"
: "";
? Intent.INTENT_WARNING
: Intent.NONE;
return (
<div className={`bp3-button-group ${styles.menubarButton}`}>
<ButtonGroup className={`${styles.menubarButton}`}>
<Popover
target={
<Tooltip
@@ -45,7 +49,8 @@ const Clip = React.memo((props) => {
<Button
type="button"
data-testid="visualization-settings"
className={`bp3-button bp3-icon-timeline-bar-chart ${activeClipClass}`}
intent={intent}
icon={IconNames.TIMELINE_BAR_CHART}
style={{
cursor: "pointer",
}}
@@ -126,7 +131,7 @@ const Clip = React.memo((props) => {
</div>
}
/>
</div>
</ButtonGroup>
);
});
client/src/components/menubar/index.js
+4 -11
View File
@@ -6,8 +6,8 @@ import * as globals from "../../globals";
import styles from "./menubar.css";
import actions from "../../actions";
import Clip from "./clip";
import AuthButtons from "./authButtons";
import InformationMenu from "./infoMenu";
import Subset from "./subset";
import UndoRedoReset from "./undoRedo";
import DiffexpButtons from "./diffexpButtons";
@@ -42,7 +42,7 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
celllist2: state.differential.celllist2,
libraryVersions: state.config?.["library_versions"],
auth: state.config?.authentication,
userinfo: state.userinfo,
userInfo: state.userInfo,
undoDisabled: state["@@undoable/past"].length === 0,
redoDisabled: state["@@undoable/future"].length === 0,
aboutLink: state.config?.links?.["about-dataset"],
@@ -204,7 +204,6 @@ class MenuBar extends React.PureComponent {
render() {
const {
dispatch,
libraryVersions,
disableDiffexp,
undoDisabled,
redoDisabled,
@@ -212,17 +211,14 @@ class MenuBar extends React.PureComponent {
clipPercentileMin,
clipPercentileMax,
graphInteractionMode,
aboutLink,
showCentroidLabels,
privacyURL,
tosURL,
categoricalSelection,
colorAccessor,
subsetPossible,
subsetResetPossible,
enableReembedding,
userInfo,
auth,
userinfo,
} = this.props;
const { pendingClipPercentiles } = this.state;
@@ -248,10 +244,7 @@ class MenuBar extends React.PureComponent {
zIndex: 3,
}}
>
<AuthButtons auth={auth} userinfo={userinfo} />
<InformationMenu
{...{ libraryVersions, aboutLink, tosURL, privacyURL, dispatch }}
/>
<AuthButtons {...{ auth, userInfo }} />
<UndoRedoReset
dispatch={dispatch}
undoDisabled={undoDisabled}
client/src/components/menubar/infoMenu.js
-77
View File
@@ -1,77 +0,0 @@
// jshint esversion: 6
import React from "react";
import { Button, Popover, Menu, MenuItem, Position } from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import styles from "./menubar.css";
const handleClick = (dispatch) => {
dispatch({ type: "toggle dataset drawer" });
};
const InformationMenu = React.memo((props) => {
const { libraryVersions, tosURL, privacyURL, dispatch } = props;
return (
<div className={`bp3-button-group ${styles.menubarButton}`}>
<Popover
content={
<Menu>
<MenuItem
onClick={() => handleClick(dispatch)}
icon={IconNames.BOOK}
text="Dataset Overview"
/>
<MenuItem
href="https://chanzuckerberg.github.io/cellxgene/"
target="_blank"
icon="help"
text="Help"
/>
<MenuItem
href="https://join-cellxgene-users.herokuapp.com/"
target="_blank"
icon="chat"
text="Chat"
/>
<MenuItem
href="https://github.com/chanzuckerberg/cellxgene"
target="_blank"
icon="git-branch"
text="Github"
/>
<MenuItem
target="_blank"
text={
libraryVersions && libraryVersions.cellxgene
? libraryVersions.cellxgene
: null
}
/>
<MenuItem text="MIT License" />
{tosURL ? (
<MenuItem href={tosURL} target="_blank" text="Terms of Service" />
) : null}
{privacyURL ? (
<MenuItem
href={privacyURL}
target="_blank"
text="Privacy Policy"
/>
) : null}
</Menu>
}
position={Position.BOTTOM_RIGHT}
>
<Button
type="button"
className="bp3-button bp3-icon-info-sign"
style={{
cursor: "pointer",
}}
/>
</Popover>
</div>
);
});
export default InformationMenu;
client/src/components/menubar/undoRedo.js
+6 -5
View File
@@ -1,12 +1,13 @@
import React from "react";
import { AnchorButton, Tooltip } from "@blueprintjs/core";
import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
import { IconNames } from "@blueprintjs/icons";
import { tooltipHoverOpenDelay } from "../../globals";
import styles from "./menubar.css";
const UndoRedo = React.memo((props) => {
const { undoDisabled, redoDisabled, dispatch } = props;
return (
<div className={`bp3-button-group ${styles.menubarButton}`}>
<ButtonGroup className={`${styles.menubarButton}`}>
<Tooltip
content="Undo"
position="bottom"
@@ -14,7 +15,7 @@ const UndoRedo = React.memo((props) => {
>
<AnchorButton
type="button"
className="bp3-button bp3-icon-undo"
icon={IconNames.UNDO}
disabled={undoDisabled}
onClick={() => {
dispatch({ type: "@@undoable/undo" });
@@ -32,7 +33,7 @@ const UndoRedo = React.memo((props) => {
>
<AnchorButton
type="button"
className="bp3-button bp3-icon-redo"
icon={IconNames.REDO}
disabled={redoDisabled}
onClick={() => {
dispatch({ type: "@@undoable/redo" });
@@ -43,7 +44,7 @@ const UndoRedo = React.memo((props) => {
data-testid="redo"
/>
</Tooltip>
</div>
</ButtonGroup>
);
});
client/src/components/miniHistogram/index.js
-1
View File
@@ -72,7 +72,6 @@ export default class MiniHistogram extends React.PureComponent {
popoverClassName={Classes.POPOVER_CONTENT_SIZING}
>
<canvas
className="bp3-popover-targer"
style={{
marginRight: 5,
width,
client/src/components/miniStackedBar/index.js
-2
View File
@@ -1,4 +1,3 @@
// jshint esversion: 6
import React from "react";
export default class MiniStackedBar extends React.PureComponent {
@@ -59,7 +58,6 @@ export default class MiniStackedBar extends React.PureComponent {
return (
<canvas
className="bp3-popover-targer"
style={{
marginRight: 5,
width,
client/src/components/termsPrompt/index.js
+4 -23
View File
@@ -8,26 +8,7 @@ import {
Colors,
Icon,
} from "@blueprintjs/core";
const CookieDecision = "cxg.cookieDecision";
function storageGet(key, defaultValue = null) {
try {
const val = window.localStorage.getItem(key);
if (val === null) return defaultValue;
return val;
} catch (e) {
return defaultValue;
}
}
function storageSet(key, value) {
try {
window.localStorage.setItem(key, value);
} catch {
// continue
}
}
import { storageGet, storageSet, KEYS } from "../util/localStorage";
@connect((state) => ({
tosURL: state.config?.parameters?.["about_legal_tos"],
@@ -37,7 +18,7 @@ class TermsPrompt extends React.PureComponent {
constructor(props) {
super(props);
const { tosURL, privacyURL } = this.props;
const cookieDecision = storageGet(CookieDecision, null);
const cookieDecision = storageGet(KEYS.COOKIE_DECISION, null);
const hasDecided = cookieDecision !== null;
this.state = {
hasDecided,
@@ -55,7 +36,7 @@ class TermsPrompt extends React.PureComponent {
handleOK = () => {
this.setState({ isOpen: false });
storageSet(CookieDecision, "yes");
storageSet(KEYS.COOKIE_DECISION, "yes");
if (window.cookieDecisionCallback instanceof Function) {
try {
window.cookieDecisionCallback();
@@ -67,7 +48,7 @@ class TermsPrompt extends React.PureComponent {
handleNo = () => {
this.setState({ isOpen: false });
storageSet(CookieDecision, "no");
storageSet(KEYS.COOKIE_DECISION, "no");
};
renderTos() {
client/src/components/util/localStorage.js
+22
View File
@@ -0,0 +1,22 @@
export const KEYS = {
COOKIE_DECISION: "cxg.cookieDecision",
LOGIN_PROMPT: "cxg.LOGIN_PROMPT",
};
export function storageGet(key, defaultValue = null) {
try {
const val = window.localStorage.getItem(key);
if (val === null) return defaultValue;
return val;
} catch (e) {
return defaultValue;
}
}
export function storageSet(key, value) {
try {
window.localStorage.setItem(key, value);
} catch {
// continue
}
}
client/src/components/util/truncate.js
+5 -2
View File
@@ -7,6 +7,8 @@ const SPLIT_STYLE = {
display: "flex",
overflow: "hidden",
justifyContent: "flex-start",
width: "100%", // There are probably additional styles that we don't want to stack
padding: 0,
};
const FIRST_HALF_STYLE = {
@@ -40,7 +42,7 @@ export default (props) => {
) {
throw Error("Only pass a single child with text to Truncate");
}
const originalString = children.props.children;
const originalString = String(children.props.children);
let firstString;
let secondString;
@@ -58,7 +60,7 @@ export default (props) => {
}
}
const inheritedColor = children.props.style.color;
const inheritedColor = children.props.style?.color;
const splitStyle = { ...children.props.style, ...SPLIT_STYLE };
const secondHalfContentStyle = {
@@ -93,6 +95,7 @@ export default (props) => {
preventOverflow: { enabled: false },
hide: { enabled: false },
}}
targetProps={{ style: children.props.style }}
>
{newChildren}
</Tooltip>
client/src/reducers/index.js
+2 -2
View File
@@ -4,7 +4,7 @@ import thunk from "redux-thunk";
import cascadeReducers from "./cascade";
import undoable from "./undoable";
import config from "./config";
import userinfo from "./userinfo";
import userInfo from "./userinfo";
import annoMatrix from "./annoMatrix";
import obsCrossfilter from "./obsCrossfilter";
import categoricalSelection from "./categoricalSelection";
@@ -44,7 +44,7 @@ const Reducer = undoable(
["pointDilation", pointDialation],
["reembedController", reembedController],
["autosave", autosave],
["userinfo", userinfo],
["userInfo", userInfo],
]),
[
"annoMatrix",
client/src/reducers/userinfo.js → client/src/reducers/userInfo.js
+2 -3
View File
@@ -1,4 +1,3 @@
// jshint esversion: 6
const UserInfo = (state = {}, action) => {
switch (action.type) {
case "initial data load start":
@@ -7,12 +6,12 @@ const UserInfo = (state = {}, action) => {
loading: true,
error: null,
};
case "userinfo load complete":
case "userInfo load complete":
return {
...state,
loading: false,
error: null,
...action.userinfo,
...action.userInfo,
};
case "initial data load error":
return {
client/src/util/stateManager/colorHelpers.js
+7 -3
View File
@@ -106,15 +106,19 @@ export function loadUserColorConfig(userColors) {
return -1;
})
.reduce(
(acc, label, i) => {
(acc, label) => {
const color = parseRGB(userColors[category][label]);
acc[0][label] = color;
acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
acc[1][label] = d3.rgb(
255 * color[0],
255 * color[1],
255 * color[2]
);
return acc;
},
[{}, {}]
);
const scale = (i) => scaleMap[i];
const scale = (label) => scaleMap[label];
convertedUserColors[category] = { colors, scale };
});
return convertedUserColors;
client/src/util/stateManager/matrix.js
+5 -5
View File
@@ -178,26 +178,26 @@ function promoteTypedArray(o) {
*/
if (isFpTypedArray(o) || Array.isArray(o)) return o;
let TyepdArrayCtor;
let TypedArrayCtor;
switch (o.constructor) {
case Int8Array:
case Uint8Array:
case Uint8ClampedArray:
case Int16Array:
case Uint16Array:
TyepdArrayCtor = Float32Array;
TypedArrayCtor = Float32Array;
break;
case Int32Array:
case Uint32Array:
TyepdArrayCtor = Float64Array;
TypedArrayCtor = Float64Array;
break;
default:
throw new Error("Unexpected data type returned from server.");
}
if (o.constructor === TyepdArrayCtor) return o;
return new TyepdArrayCtor(o);
if (o.constructor === TypedArrayCtor) return o;
return new TypedArrayCtor(o);
}
export function matrixFBSToDataframe(arrayBuffers) {
dev_docs/cxg.md
+2
View File
@@ -1,3 +1,5 @@
## UPDATE (9/30/2020): Starting today, the name Corpora will only be used as the internal project name, with cellxgene Data Portal being the official product name
# CXG Data Format Specification
Document Status: _draft_
dev_docs/schema_guide.md
+175
View File
@@ -0,0 +1,175 @@
# Cellxgene Schema Guide
Datasets included in the [data portal](https://cellxgene.cziscience.com/) and hosted cellxgene need to follow the schema
described [here](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md). That
schema defines some required fields, requirements about feature labels, and some optional fields that mostly help with
presentation.
The number of fields is rather low, and we expect that information needed to populate those fields should either already
be present in datasets prepared by a submitter or be easy to obtain. However, this still leaves the task of actually
manipulating the dataset so that it follows the schema: adjusting field names, ensuring proper ontologies are used,
converting gene symbols to a common set, etc. This can be tedious and error-prone, and at the beginning of the hosted
cellxgene project, this was always done with engineering support. As we increase the rate at which we add data, we want
to eliminate the need for engineering support so that ultimately submitters themselves can create files that follow the
schema.
## `cellxgene schema apply`
To enable this, we have a new cellxgene subcommand, `cellxgene schema`, that handles applying and verifying the schema.
Its first subcommand, `cellxgene schema apply`, takes three inputs:
1. A source h5ad file. The input needs to be an AnnData file, so if a submitter has, say, a serialized Seurat or
SingleCellExperiment object, it needs to be converted to AnnData first. This can be done with
[sceasy](https://github.com/cellgeni/sceasy) or via
[Seurat](https://satijalab.org/seurat/v3.1/conversion_vignette.html).
2. A configuration yaml file that describes the fields to add and conversions to apply (see below).
3. A name for the new h5ad file that should follow the schema.
### Configuration yaml
The configuration yaml file describes how to apply the schema. This is an example of a "skeleton" yaml that has all the
fields required for the 1.0.0 schema but is not yet filled in with any logic:
```
uns:
version:
corpora_schema_version: 1.0.0
corpora_encoding_version: 0.1.0
contributors:
title:
layer_descriptions:
preprint_doi:
publication_doi:
organism_ontology_term_id:
obs:
tissue_ontology_term_id:
assay_ontology_term_id:
disease_ontology_term_id:
cell_type_ontology_term_id:
sex:
ethnicity_ontology_term_id:
development_stage_ontology_term_id:
fixup_gene_symbols:
```
#### Unstructured metadata
The first section is `uns`, which includes metadata fields that describe the whole dataset (see
[here](https://anndata.readthedocs.io/en/latest/) for further description of `uns` and `obs`.).
The first line is `version`, which is required for most of our tooling to work. The schema version is set at
1.0.0 in the example above, but of course for future versions that should be changed.
Next is `contributors` which describes who is adding the dataset to the portal. If you consult the schema, you see that
contributors is a list where each element can have `name`, `email`, and `institution`. So when filled out, the
`contributors` field should look like this:
```
contributors:
- name: Mary B. Scientist
email: mbs@singlecell.edu
institution: Single-Cell University
- name: Robert J. Scientist
email: rjs@usingle.edu
institution: University of Single Cell
```
`title` is the name of the dataset, and is just a string that gets displayed in the portal and cellxgene to identify the
dataset.
`layer_descriptions` is free text descriptions of the different
[layers](https://anndata.readthedocs.io/en/latest/anndata.AnnData.layers.html) of the AnnData file. It should look like
this when complete, depending on what layers are present:
```
layer_descriptions:
X: CPM and logged
raw.X: raw
```
Note that one of the layers needs to be "raw", that is, the AnnData file must contain raw counts.
The two DOI fields are optional but can be included if the dataset is associated with a publication or preprint. Note
that the DOI should be a full url:
```
publication_doi: https://doi.org/10.1073%2Fpnas.83.15.5372
```
Finally, the `organism_ontology_term_id` field is the species of the donor organism from the NCBITaxon ontology. The
value for _Homo sapiens_ is `NCBITaxon:9606`:
```
organism_ontology_term_id: NCBITaxon:9606
```
Note that the schema also requires a human-readable `organism` field, but this doesn't need to be included in the yaml.
When the `cellxgene schema apply` script encounters an ontology field, it looks up the label for the term(s) and inserts it
into the appropriate field.
#### Observation metadata
The next section is `obs`, which is metadata than can vary for each observation (and "observation" usually means cell).
These fields are all ontology fields except for `sex`, which has its own enumerated set of permitted values.
There are two ways to fill in the `obs` fields. The first is useful when there is only one value for all the
observations in the dataset. This is not uncommon, for example all cells often come from the same assay. In that case
just insert the ontology term:
```
assay_ontology_term_id: EFO:0009922
```
The second is for when there is an existing field in the dataset that needs to be mapped to the schema field. For
example, the submitter may have included cell type annotations in a field called `CellType`, and those annotations may
just be free text. This doesn't follow the schema because it needs to be in `cell_type_ontology_term_id` and
`cell_type`, and it needs ontology terms and labels, not just any text. In that case the field can be a dictionary:
```
cell_type_ontology_term_id:
CellType:
t-cell: CL:0000084
b-cell: CL:0000236
```
This will look at the `obs.CellType` field in the dataset, and where it has the value "t-cell", it will insert
`CL:0000084` into `cell_type_ontology_term_id` and its label `T cell` into `cell_type`.
Now there are often situations where there is no valid ontology term for some field. For example, the dataset may have
been produced via an assay not present in `EFO`. Or, a particular cell type may have no entry in `CL`. In that case, a
free text description can be used in the `ontology_term_id` field:
```
assay_ontology_term_id: Sci-Plex
cell_type_ontology_term_id:
CellType:
t-cell: CL:0000084
b-cell: CL:0000236
new cell type: new cell type
```
In these cases, the `cellxgene schema apply` script will leave the ontology field blank and move the free text
description into the label field. So the `assay_ontology_term_id` in the new dataset would be `""` but `assay` would be
`Sci-Plex`.
#### Gene symbol harmonization
The last section describes how gene symbol conversion should be applied to each of the layers. This is similar to the
`layer_descriptions` field above, but there are only three permitted values: `raw`, `log1p`, and `sqrt`:
```
fixup_gene_symbols:
X: log1p
raw.X: raw
```
This tells the script how each each layer was transformed from raw values that can be directly summed. `raw` means that
the layer contains raw counts or some linear tranformation of raw counts. `log1p` means that the layer has `log(X + 1)`
for each the raw `X` values. `sqrt` means `sqrt(X)` (this is not common). For layers produced by Seurat's normalization
or SCTransform functions, the correct choice is usually `log1p`.
### `cellxgene schema validate`
The next `cellxgene schema` subcommand is `cellxgene schema validate`, and it validates that a given h5ad follows a
version of the schema. It accepts two parameters:
1. The h5ad file to check
2. The version of the schema to check against.
If the validation succeeds, the command will have a zero exit code. If it does not, it will have a non-zero exit code
and will print validation failure messages.
docs/_config.yml
+2 -2
View File
@@ -13,6 +13,8 @@ nav:
url: posts/install
- title: Gallery
url: posts/gallery
- title: Cellxgene data portal
url: https://cellxgene.cziscience.com/
- title: Demo datasets
url: posts/demo-data
- title: Preparing your data
@@ -33,5 +35,3 @@ nav:
url: posts/contribute
- title: Contact & finding help
url: posts/contact
- title: cellxgene.cziscience.com
url: posts/cellxgene_cziscience_com
docs/_site/index.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Index | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Index" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","headline":"Index","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/annotations.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>annotations | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="annotations" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Creating annotations" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","headline":"annotations","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"annotations","description":"Creating annotations","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/contact.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Contact | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Contact" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Contact" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","headline":"Contact","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Contact","description":"Contact","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn"><b>Contact & finding help</b></a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/contribute.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Code of conduct | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Code of conduct" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Code of conduct","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Code of conduct","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/demo-data.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>demo-data | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="demo-data" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Demo datasets" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn"><b>Demo datasets</b></a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/gallery.html
+8 -8
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Gallery | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Gallery" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Gallery","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Gallery","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
@@ -130,7 +130,7 @@ Check out the cool data that our users are using cellxgene to explore!</p>
<h3 id="melanoma"><a href="https://melanoma.cellgeni.sanger.ac.uk/">Melanoma</a></h3>
<h3 id="czis-own-cellxgene-site"><a href="cellxgene_cziscience_com">CZI’s own cellxgene site</a></h3>
<h3 id="czis-own-cellxgene-site"><a href="https://cellxgene.cziscience.com/">CZI’s own cellxgene site</a></h3>
<p><em>Want us to link to your dataset here? <a href="contact">Just send us a note!</a></em></p>
docs/_site/posts/hosted.html
+28 -27
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Hosting cellxgene on the web | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Hosting cellxgene on the web" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Hosting cellxgene on the web","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Hosting cellxgene on the web","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
@@ -139,35 +139,36 @@
<h1 id="deploying-cellxgene-with-heroku">Deploying cellxgene with Heroku</h1>
<h2 id="quickstart">Quickstart</h2>
<h2 id="heroku-support">Heroku Support</h2>
<p>Clicking on the following button will forward you to Heroku to begin the deployment process:</p>
<p>The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.</p>
<p><a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
<img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy" />
</a></p>
<p>While we no longer directly support Heroku, it is still possible to create a Heroku app via <a href="https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile">our provided Dockerfile here</a> and <a href="https://devcenter.heroku.com/articles/build-docker-images-heroku-yml">Heroku’s documentation</a>.</p>
<p>If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.</p>
<p>You may have to tweak the <code class="language-plaintext highlighter-rouge">Dockerfile</code> like so:</p>
<p>Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:</p>
<pre><code class="language-Dockerfile">FROM ubuntu:bionic
<h3 id="default-settings">Default settings</h3>
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
<ul>
<li><code class="language-plaintext highlighter-rouge">App name</code>: the unique name for your deployment</li>
<li>This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)</li>
<li><code class="language-plaintext highlighter-rouge">App owner</code>: Who will own this app. Either you personally or an organization/team</li>
<li><code class="language-plaintext highlighter-rouge">Region</code>: Location of the server where the app will be deployed (EU or US)</li>
</ul>
RUN apt-get update &amp;&amp; \
apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests &amp;&amp; \
pip3 install cellxgene
<h3 id="configuration">Configuration</h3>
# ENTRYPOINT ["cellxgene"] # Heroku doesn't work well with ENTRYPOINT
</code></pre>
<ul>
<li><code class="language-plaintext highlighter-rouge">DATASET</code>: A <em>publicly</em> accessible URL pointing to a .h5ad file to view</li>
<li>This defaults to pbm3k.h5ad</li>
</ul>
<p>and provide a <code class="language-plaintext highlighter-rouge">heroku.yml</code> file similar to this:</p>
<p>After filling out the settings and pressing the <code class="language-plaintext highlighter-rouge">Deploy app</code> button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!</p>
<div class="language-yml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="na">build</span><span class="pi">:</span>
<span class="na">docker</span><span class="pi">:</span>
<span class="na">web</span><span class="pi">:</span> <span class="s">Dockerfile</span>
<span class="na">run</span><span class="pi">:</span>
<span class="na">web</span><span class="pi">:</span>
<span class="na">command</span><span class="pi">:</span>
<span class="pi">-</span> <span class="s">cellxgene launch --host 0.0.0.0 --port $PORT $DATASET</span> <span class="c1"># the DATATSET config var must be defined in your dashboard settings.</span>
</code></pre></div></div>
<h2 id="what-is-heroku">What is Heroku?</h2>
docs/_site/posts/hosted.md
+23 -16
View File
@@ -38,31 +38,38 @@ If you know of other solutions, drop us a note and we'll add to this list.
# Deploying cellxgene with Heroku
## Quickstart
## Heroku Support
Clicking on the following button will forward you to Heroku to begin the deployment process:
The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.
<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
<img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
</a>
While we no longer directly support Heroku, it is still possible to create a Heroku app via [our provided Dockerfile here](https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile) and [Heroku's documentation](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml).
If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.
You may have to tweak the `Dockerfile` like so:
Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:
```Dockerfile
FROM ubuntu:bionic
### Default settings
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
- `App name`: the unique name for your deployment
- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
- `App owner`: Who will own this app. Either you personally or an organization/team
- `Region`: Location of the server where the app will be deployed (EU or US)
RUN apt-get update && \
apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
pip3 install cellxgene
### Configuration
# ENTRYPOINT ["cellxgene"] # Heroku doesn't work well with ENTRYPOINT
```
- `DATASET`: A _publicly_ accessible URL pointing to a .h5ad file to view
- This defaults to pbm3k.h5ad
and provide a `heroku.yml` file similar to this:
After filling out the settings and pressing the `Deploy app` button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!
```yml
build:
docker:
web: Dockerfile
run:
web:
command:
- cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings.
```
## What is Heroku?
docs/_site/posts/install.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Install | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Install" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Install","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Install","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/launch.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>demo-data | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="demo-data" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Demo datasets" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/methods.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Methods | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Methods" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Methods","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Methods","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/prepare.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>prepare | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="prepare" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Preparing your data" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","headline":"prepare","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"prepare","description":"Preparing your data","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/roadmap.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>roadmap | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="roadmap" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Roadmap" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","headline":"roadmap","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"roadmap","description":"Roadmap","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/_site/posts/troubleshooting.html
+7 -7
View File
@@ -7,7 +7,7 @@
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Troubleshooting | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.7" />
<meta name="generator" content="Jekyll v3.9.0" />
<meta property="og:title" content="Troubleshooting" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Troubleshooting" />
@@ -16,10 +16,10 @@
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","headline":"Troubleshooting","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","@context":"https://schema.org"}</script>
{"url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Troubleshooting","description":"Troubleshooting","@type":"WebPage","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -46,6 +46,10 @@
<a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
<a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
@@ -85,10 +89,6 @@
<a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
<a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
<a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
</p>
docs/posts/cellxgene_cziscience_com.md → docs/deprecated/cellxgene_cziscience_com.md
View File
docs/posts/gallery.md
+1 -1
View File
@@ -39,6 +39,6 @@ Check out the cool data that our users are using cellxgene to explore!
### [Melanoma](https://melanoma.cellgeni.sanger.ac.uk/)
### [CZI's own cellxgene site](cellxgene_cziscience_com)
### [CZI's own cellxgene site](https://cellxgene.cziscience.com/)
_Want us to link to your dataset here? [Just send us a note!](contact)_
package-lock.json
Generated
+1191
View File
File diff suppressed because it is too large Load Diff
package.json
+3
View File
@@ -2,5 +2,8 @@
"name": "cellxgene",
"scripts": {
"postinstall": "npm install --prefix client && npm run build --prefix client && make copy-client-assets"
},
"devDependencies": {
"@blueprintjs/eslint-plugin": "^0.3.1"
}
}
server/Makefile
+8
View File
@@ -39,3 +39,11 @@ create-test-db:
clean-test-db:
-docker stop test_db
-docker rm test_db
.PHONY: test-annotations-performance
test-annotations-performance:
python test/performance/performance_test_annotations_backend.py
.PHONY: test-annotations-scale
test-annotations-scale:
locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
server/__init__.py
-1
View File
@@ -1,6 +1,5 @@
import logging
import sys
from server.common.utils.utils import import_plugins
__version__ = "0.16.0"
server/app/app.py
+45 -23
View File
@@ -6,8 +6,17 @@ from urllib.parse import urlparse
import hashlib
import os
from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \
send_from_directory
from flask import (
Flask,
redirect,
current_app,
make_response,
render_template,
abort,
Blueprint,
request,
send_from_directory,
)
from flask_restful import Api, Resource
from server_timing import Timing as ServerTiming
@@ -87,10 +96,7 @@ def dataset_index(url_dataroot=None, dataset=None):
cache_manager = current_app.matrix_data_cache_manager
with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
args = {
"SCRIPTS" : scripts,
"INLINE_SCRIPTS" : inline_scripts
}
args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
return render_template("index.html", **args)
except DatasetAccessError as e:
@@ -99,13 +105,6 @@ def dataset_index(url_dataroot=None, dataset=None):
)
@webbp.route("/health", methods=["GET"])
@cache_control_always(no_store=True)
def health():
config = current_app.app_config
return health_check(config)
@webbp.errorhandler(RequestException)
def handle_request_exception(error):
return common_rest.abort_and_log(error.status_code, error.message, loglevel=logging.INFO, include_exc_info=True)
@@ -224,6 +223,13 @@ def dataroot_index():
return redirect(config.server_config.multi_dataset__index)
class HealthAPI(Resource):
@cache_control(no_store=True)
def get(self):
config = current_app.app_config
return health_check(config)
class DatasetResource(Resource):
"""Base class for all Resources that act on datasets."""
@@ -312,8 +318,18 @@ class LayoutObsAPI(DatasetResource):
return common_rest.layout_obs_put(request, data_adaptor)
def get_api_resources(bp_api, url_dataroot=None):
api = Api(bp_api)
def get_api_base_resources(bp_base):
"""Add resources that are accessed from the api_base_url"""
api = Api(bp_base)
# Diagnostics routes
api.add_resource(HealthAPI, "/health")
return api
def get_api_dataroot_resources(bp_dataroot, url_dataroot=None):
"""Add resources that refer to a dataset"""
api = Api(bp_dataroot)
def add_resource(resource, url):
"""convenience function to make the outer function less verbose"""
@@ -385,18 +401,24 @@ class Server:
parse = urlparse(api_base_url)
api_path = parse.path
bp_base = Blueprint("bp_base", __name__, url_prefix=api_path)
base_resources = get_api_base_resources(bp_base)
self.app.register_blueprint(base_resources.blueprint)
if app_config.is_multi_dataset():
# NOTE: These routes only allow the dataset to be in the directory
# of the dataroot, and not a subdirectory. We may want to change
# the route format at some point
for dataroot_dict in server_config.multi_dataset__dataroot.values():
url_dataroot = dataroot_dict["base_url"]
bp_api = Blueprint(
f"api_dataset_{url_dataroot}", __name__,
url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version
bp_dataroot = Blueprint(
f"api_dataset_{url_dataroot}",
__name__,
url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version,
)
resources = get_api_resources(bp_api, url_dataroot)
self.app.register_blueprint(resources.blueprint)
dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
self.app.register_blueprint(dataroot_resources.blueprint)
self.app.add_url_rule(
f"/{url_dataroot}/<dataset>/",
f"dataset_index_{url_dataroot}",
@@ -407,18 +429,18 @@ class Server:
f"/{url_dataroot}/<dataset>/static/<path:filename>",
f"static_assets_{url_dataroot}",
view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename),
methods=["GET"]
methods=["GET"],
)
else:
bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}")
resources = get_api_resources(bp_api)
resources = get_api_dataroot_resources(bp_api)
self.app.register_blueprint(resources.blueprint)
self.app.add_url_rule(
"/static/<path:filename>",
"static_assets",
view_func=lambda filename: send_from_directory("../common/web/static", filename),
methods=["GET"]
methods=["GET"],
)
self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager
server/auth/__init__.py
-1
View File
@@ -1,4 +1,3 @@
# import the built in auth types so they can be registered
import server.auth.auth_none # noqa: F401
server/auth/auth.py
+5 -1
View File
@@ -43,6 +43,10 @@ class AuthTypeBase(ABC):
"""Return the name of the user (string)"""
pass
def get_user_picture(self):
"""Return the location to the user's picture"""
return None
class AuthTypeClientBase(AuthTypeBase):
"""Base type for all authentication types that require the client to login"""
@@ -76,7 +80,7 @@ class AuthTypeFactory:
@staticmethod
def register(name, auth_type):
assert(issubclass(auth_type, AuthTypeBase))
assert issubclass(auth_type, AuthTypeBase)
AuthTypeFactory.auth_types[name] = auth_type
@staticmethod
server/auth/auth_none.py
-1
View File
@@ -2,7 +2,6 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory
class AuthTypeNone(AuthTypeBase):
def __init__(self, app_config):
super().__init__()
server/auth/auth_oauth.py
+56 -25
View File
@@ -24,7 +24,7 @@ except ModuleNotFoundError:
class Tokens:
"""Simple class to represent the tokens that are saved/restored from the cookie"""
def __init__(self, access_token, id_token, refresh_token, expires_at):
def __init__(self, access_token, id_token, refresh_token, expires_at, **kwargs):
self.access_token = access_token
self.id_token = id_token
self.refresh_token = refresh_token
@@ -97,8 +97,17 @@ class AuthTypeOAuth(AuthTypeClientBase):
return
valid_keys = {
"verify_signature", "verify_aud", "verify_iat", "verify_exp", "verify_nbf", "verify_iss",
"verify_sub", "verify_jti", "verify_at_hash", "leeway"}
"verify_signature",
"verify_aud",
"verify_iat",
"verify_exp",
"verify_nbf",
"verify_iss",
"verify_sub",
"verify_jti",
"verify_at_hash",
"leeway",
}
keys = set(self.jwt_decode_options.keys())
unknown = keys - valid_keys
if unknown:
@@ -114,6 +123,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
parse = urlparse(self.api_base_url)
app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"])
app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"])
app.add_url_rule(f"{parse.path}/logout_redirect", "logout_redirect", self.logout_redirect, methods=["GET"])
app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"])
def complete_setup(self, flask_app):
@@ -136,21 +146,19 @@ class AuthTypeOAuth(AuthTypeClientBase):
def get_user_id(self):
payload = self.get_userinfo()
if payload and payload.get("sub"):
return payload.get("sub")
return None
return payload.get("sub") if payload else None
def get_user_name(self):
payload = self.get_userinfo()
if payload and payload.get("name"):
return payload.get("name")
return None
return payload.get("name") if payload else None
def get_user_email(self):
payload = self.get_userinfo()
if payload and payload.get("email"):
return payload.get("email")
return None
return payload.get("email") if payload else None
def get_user_picture(self):
payload = self.get_userinfo()
return payload.get("picture") if payload else None
def update_response(self, response):
response.cache_control.update(dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
@@ -158,7 +166,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
def login(self):
callbackurl = f"{self.api_base_url}/oauth2/callback"
return_path = request.args.get("dataset", "")
return_to = f"{self.web_base_url}/{return_path}/"
return_to = f"{self.web_base_url}/{return_path}"
# save the return path in the session cookie, accessed in the callback function
session["oauth_callback_redirect"] = return_to
response = self.client.authorize_redirect(redirect_uri=callbackurl)
@@ -166,12 +174,29 @@ class AuthTypeOAuth(AuthTypeClientBase):
return response
def logout(self):
"""
We would like for the user to remain on the same dataset after logout. oauth requires that
the redirect `returnTo` path be whitelisted by the oauth server, therefore a level of
indirection is used. We first redirect to a single path "logout_redirect", and logout_redirect
will redirect the user's browser back to the current page.
"""
self.remove_tokens()
params = {"returnTo": self.web_base_url, "client_id": self.client_id}
redirect_path = request.args.get("dataset", "")
redirect_to = f"{self.web_base_url}/{redirect_path}"
session["oauth_logout_redirect"] = redirect_to
return_to = f"{self.api_base_url}/logout_redirect"
params = {"returnTo": return_to, "client_id": self.client_id}
response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params))
self.update_response(response)
return response
def logout_redirect(self):
oauth_logout_redirect = session.pop("oauth_logout_redirect", "/")
response = redirect(oauth_logout_redirect)
self.update_response(response)
return response
def callback(self):
data = self.client.authorize_access_token()
tokens = Tokens(
@@ -193,22 +218,24 @@ class AuthTypeOAuth(AuthTypeClientBase):
try:
if self.session_cookie:
tokensdict = session.get(self.CXG_TOKENS)
if tokensdict:
g.tokens = Tokens(**tokensdict)
value = session.get(self.CXG_TOKENS)
if value:
g.tokens = Tokens(**value)
else:
return None
else:
value = request.cookies.get(self.cookie_params["key"])
value = base64.b64decode(value)
try:
tokensdict = json.loads(value)
g.tokens = Tokens(**tokensdict)
except (TypeError, KeyError, json.decoder.JSONDecodeError):
g.pop("tokens", None)
if value is None:
return None
value = base64.b64decode(value)
value = json.loads(value)
g.tokens = Tokens(**value)
except (TypeError, KeyError):
except Exception:
# there are many types of exceptions that can be raise in the above section.
# It is impractical to list all the exceptions here, since that would be brittle.
# If an exception occurs, then return None, meaning that no token could be retrieved.
current_app.logger.warning(f"auth cookie is in the wrong format: {str(value)}")
g.pop("tokens", None)
return None
@@ -253,7 +280,10 @@ class AuthTypeOAuth(AuthTypeClientBase):
def get_logout_url(self, data_adaptor):
"""Return the url for the logout route"""
return f"{self.api_base_url}/logout"
if data_adaptor and current_app.app_config.is_multi_dataset():
return f"{self.api_base_url}/logout?dataset={data_adaptor.uri_path}/"
else:
return f"{self.api_base_url}/logout"
def check_jwt_payload(self, id_token):
try:
@@ -303,6 +333,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
# if there is no id_token, return None (user is not authenticated)
tokens = self.get_tokens()
if tokens is None or tokens.id_token is None:
return None
server/auth/auth_test.py
+7
View File
@@ -10,12 +10,14 @@ class AuthTypeTest(AuthTypeClientBase):
CXGUID = "cxguid_test"
CXGUNAME = "cxguname_test"
CXGUEMAIL = "cxguemail_test"
CXGUPICTURE = "cxgupicture_test"
def __init__(self, app_config):
super().__init__()
self.user_name = "test_account"
self.user_id = "id0001"
self.user_email = "test_account@test.com"
self.user_picture = None
def is_valid_authentication_type(self):
return True
@@ -42,11 +44,16 @@ class AuthTypeTest(AuthTypeClientBase):
def get_user_email(self):
return session.get(self.CXGUEMAIL)
def get_user_picture(self):
return session.get(self.CXGUPICTURE)
def login(self):
args = request.args
return_to = args.get("dataset", "/")
session[self.CXGUID] = args.get("userid", self.user_id)
session[self.CXGUNAME] = args.get("username", self.user_name)
session[self.CXGUEMAIL] = args.get("email", self.user_email)
session[self.CXGUPICTURE] = args.get("picture", self.user_picture)
return redirect(return_to)
def logout(self):
server/cli/cli.py
+2
View File
@@ -4,6 +4,7 @@ from .convert_to_cxg import convert_to_cxg
from .launch import launch
from .prepare import prepare
from .upgrade import log_upgrade_check
from .schema import schema_cli
from .. import __version__
@@ -31,3 +32,4 @@ def cli(upgrade_check):
cli.add_command(launch)
cli.add_command(prepare)
cli.add_command(convert_to_cxg)
cli.add_command(schema_cli)
server/cli/convert_to_cxg.py
+39 -35
View File
@@ -9,26 +9,24 @@ from server.converters.h5ad_data_file import H5ADDataFile
name="convert",
short_help="Converts an H5AD dataset to the CXG format.",
help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
"that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
"environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
"usually with the generated CXG file.",
"that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
"environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
"usually with the generated CXG file.",
)
@click.argument(
"input-file",
nargs=1,
type=click.Path(exists=True, dir_okay=False),
"input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"-o",
"--output-directory",
help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
"CXG extension.",
"CXG extension.",
)
@click.option(
"-b",
"--backed",
help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
"but will use less memory.",
"but will use less memory.",
default=False,
show_default=True,
is_flag=True,
@@ -37,29 +35,33 @@ from server.converters.h5ad_data_file import H5ADDataFile
"-t",
"--title",
help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
"the dataset title will be the filename.",
"the dataset title will be the filename.",
)
@click.option(
"-a",
"--about",
help="A fully qualified URL that provides more information about the dataset and will be included as "
"metadata about the CXG file.",
"metadata about the CXG file.",
)
@click.option(
"-s",
"--sparse-threshold",
help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
"array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
"convert to dense array.",
"array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
"convert to dense array.",
default=0.0,
show_default=True,
)
@click.option("--obs-names",
help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.")
@click.option("--var-names",
help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.")
@click.option(
"--obs-names",
help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.",
)
@click.option(
"--var-names",
help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.",
)
@click.option(
"--disable-custom-colors",
help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
@@ -70,8 +72,8 @@ from server.converters.h5ad_data_file import H5ADDataFile
@click.option(
"--disable-corpora-schema",
help="When set, conversion process will neither extract nor store Corpora schema information. See "
"https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
"information.",
"https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
"information.",
default=False,
show_default=True,
is_flag=True,
@@ -85,30 +87,32 @@ from server.converters.h5ad_data_file import H5ADDataFile
)
@click.help_option("--help", "-h", help="Show this message and exit.")
def convert_to_cxg(
input_file,
output_directory,
backed,
title,
about,
sparse_threshold,
obs_names,
var_names,
disable_custom_colors,
disable_corpora_schema,
overwrite,
input_file,
output_directory,
backed,
title,
about,
sparse_threshold,
obs_names,
var_names,
disable_custom_colors,
disable_corpora_schema,
overwrite,
):
"""
Convert a dataset file into CXG.
"""
h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names,
use_corpora_schema=not disable_corpora_schema)
h5ad_data_file = H5ADDataFile(
input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
)
# Get the directory that will hold all the CXG files
cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold,
convert_anndata_colors_to_cxg_colors=not disable_custom_colors)
h5ad_data_file.to_cxg(
cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
)
def get_output_directory(input_filename, output_directory, should_overwrite):
server/cli/launch.py
+35 -36
View File
@@ -3,15 +3,14 @@ import functools
import logging
import sys
import webbrowser
from os import devnull
import os
import click
from flask_compress import Compress
from flask_cors import CORS
from server.default_config import default_config
from server.app.app import Server
from server.common.app_config import AppConfig
from server.common.default_config import default_config
from server.common.config.app_config import AppConfig
from server.common.errors import DatasetAccessError, ConfigurationError
from server.common.utils.utils import sort_options
@@ -33,7 +32,7 @@ def annotation_args(func):
multiple=False,
metavar="<path>",
help="CSV file to initialize editing of existing annotations; will be altered in-place. "
"Incompatible with --annotations-dir.",
"Incompatible with --annotations-dir.",
)
@click.option(
"--annotations-dir",
@@ -42,7 +41,7 @@ def annotation_args(func):
multiple=False,
metavar="<directory path>",
help="Directory of where to save output annotations; filename will be specified in the application. "
"Incompatible with --annotations-file.",
"Incompatible with --annotations-file.",
)
@click.option(
"--experimental-annotations-ontology",
@@ -170,7 +169,7 @@ def server_args(func):
default=DEFAULT_CONFIG.server_config.app__debug,
show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",
"or when you want more information about an error condition.",
)
@click.option(
"--verbose",
@@ -203,7 +202,7 @@ def server_args(func):
multiple=True,
metavar="<text>",
help="Additional script files to include in HTML page. If not specified, "
"no additional script files will be included.",
"no additional script files will be included.",
show_default=False,
)
@functools.wraps(func)
@@ -223,7 +222,7 @@ def launch_args(func):
default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
metavar="<data directory>",
help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
" to folder containing H5AD and/or CXG datasets.",
" to folder containing H5AD and/or CXG datasets.",
hidden=True,
) # TODO, unhide when dataroot is supported)
@click.argument("datapath", required=False, metavar="<path to data file>")
@@ -307,32 +306,32 @@ class CliLaunchServer(Server):
)
@launch_args
def launch(
datapath,
dataroot,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
disable_custom_colors,
diffexp_lfc_cutoff,
title,
scripts,
about,
disable_annotations,
annotations_file,
annotations_dir,
backed,
disable_diffexp,
experimental_annotations_ontology,
experimental_annotations_ontology_obo,
experimental_enable_reembedding,
config_file,
dump_default_config,
datapath,
dataroot,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
disable_custom_colors,
diffexp_lfc_cutoff,
title,
scripts,
about,
disable_annotations,
annotations_file,
annotations_dir,
backed,
disable_diffexp,
experimental_annotations_ontology,
experimental_annotations_ontology_obo,
experimental_enable_reembedding,
config_file,
dump_default_config,
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
@@ -443,7 +442,7 @@ def launch(
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not server_config.app__verbose:
f = open(devnull, "w")
f = open(os.devnull, "w")
sys.stdout = f
try:
server/cli/prepare.py
+13 -13
View File
@@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options
default=False,
is_flag=True,
help="Do not run quality control metrics. By default cellxgene runs them "
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
)
@click.option(
"--make-obs-names-unique/--no-make-obs-names-unique",
@@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options
)
@click.help_option("--help", "-h", help="Show this message and exit.")
def prepare(
data,
embedding,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
skip_qc,
make_obs_names_unique,
make_var_names_unique,
data,
embedding,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
skip_qc,
make_obs_names_unique,
make_var_names_unique,
):
"""
Preprocess data for use with cellxgene.
server/cli/schema.py
+72
View File
@@ -0,0 +1,72 @@
import click
from server.converters.schema import remix, validate
@click.group(
name="schema",
subcommand_metavar="COMMAND <args>",
short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
)
def schema_cli():
try:
import scanpy # noqa: F401
except ImportError:
raise click.ClickException(
"[cellxgene] cellxgene schema requires scanpy"
)
@click.command(
name="apply",
short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
"h5ad file, apply the schema changes and create a new, conforming h5ad.",
)
@click.option(
"--source-h5ad",
help="Input h5ad file.",
nargs=1,
required=True,
type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"--remix-config",
help="Config yaml with information on how to apply the schema.",
nargs=1,
required=True,
type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"--output-filename",
help="Filename for the new, schema-conforming h5ad file.",
required=True,
nargs=1
)
def schema_apply(source_h5ad, remix_config, output_filename):
remix.apply_schema(source_h5ad, remix_config, output_filename)
@click.command(
name="validate",
short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
)
@click.argument(
"h5ad",
nargs=1,
type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"--shallow",
help="When true, just check that the correct version information is present.",
default=False,
show_default=True,
is_flag=True,
)
def schema_validate(h5ad, shallow):
validate.validate(h5ad, shallow)
schema_cli.add_command(schema_apply)
schema_cli.add_command(schema_validate)
server/cli/upgrade.py
+2 -1
View File
@@ -10,7 +10,8 @@ from .. import __version__
SEMVER_FORMAT = re.compile(
r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
r"?:\.[0-9a-zA-Z-]+)*))?$")
r"?:\.[0-9a-zA-Z-]+)*))?$"
)
def log_upgrade_check():
server/common/annotations/hosted_tiledb.py
+35 -18
View File
@@ -31,7 +31,14 @@ class AnnotationsHostedTileDB(Annotations):
unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
if unsanitary_original_category_names:
raise AnnotationCategoryNameError(
f"{unsanitary_original_category_names} are not valid category names, please resubmit")
f"{unsanitary_original_category_names} are not valid category names, please resubmit"
)
def get_user_name(self):
return current_app.auth.get_user_name()
def get_user_id(self):
return current_app.auth.get_user_id()
def is_safe_collection_name(self, name):
"""
@@ -47,7 +54,7 @@ class AnnotationsHostedTileDB(Annotations):
self.CXG_ANNO_COLLECTION = name
def read_labels(self, data_adaptor):
user_id = current_app.auth.get_user_id()
user_id = self.get_user_id()
if user_id is None:
return
dataset_name = data_adaptor.get_location()
@@ -57,7 +64,15 @@ class AnnotationsHostedTileDB(Annotations):
Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
)
if annotation_object:
df = tiledb.open(annotation_object.tiledb_uri)
if annotation_object.tiledb_uri == "":
# this mean the user has removed all the categories.
return None
try:
df = tiledb.open(annotation_object.tiledb_uri)
except tiledb.TileDBError:
# don't crash if the annotations file is missing or can't be read.
current_app.logger.warning(f"Cannot read annotation file: {annotation_object.tiledb_uri}")
return None
pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
return pandas_df
else:
@@ -68,11 +83,11 @@ class AnnotationsHostedTileDB(Annotations):
index_dims = None
schema_hints = json.loads(schema_hints)
if '__pandas_attribute_repr' in tileDBArray.meta:
if "__pandas_attribute_repr" in tileDBArray.meta:
# backwards compatibility... unsure if necessary at this point
repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr'])
if '__pandas_index_dims' in tileDBArray.meta:
index_dims = json.loads(tileDBArray.meta['__pandas_index_dims'])
repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
if "__pandas_index_dims" in tileDBArray.meta:
index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
data = tileDBArray[:]
indexes = list()
@@ -80,12 +95,12 @@ class AnnotationsHostedTileDB(Annotations):
for col_name, col_val in data.items():
# If the column values are byte literals, decode them
if isinstance(col_val[0], bytes):
col_val = [value.decode('utf-8') for value in col_val]
col_val = [value.decode("utf-8") for value in col_val]
if schema_hints and col_name in schema_hints:
type = schema_hints.get(col_name).get("type")
if type and type == "categorical":
new_col = pd.Series(col_val, dtype='category')
new_col = pd.Series(col_val, dtype="category")
data[col_name] = new_col
elif repr_meta and col_name in repr_meta:
new_col = pd.Series(col_val, dtype=repr_meta[col_name])
@@ -102,8 +117,8 @@ class AnnotationsHostedTileDB(Annotations):
return new_df
def write_labels(self, df, data_adaptor):
auth_user_id = current_app.auth.get_user_id()
user_name = current_app.auth.get_user_name()
auth_user_id = self.get_user_id()
user_name = self.get_user_name()
timestamp = time.time()
dataset_location = data_adaptor.get_location()
dataset_id = self.db.get_or_create_dataset(dataset_location)
@@ -123,20 +138,22 @@ class AnnotationsHostedTileDB(Annotations):
else:
os.makedirs(uri, exist_ok=True)
_, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
annotation = Annotation(
tiledb_uri=uri,
user_id=user_id,
dataset_id=str(dataset_id),
schema_hints=json.dumps(dataframe_schema_type_hints)
)
if not df.empty:
self.check_category_names(df)
# convert to tiledb datatypes
for col in df:
df[col] = df[col].astype(get_dtype_of_array(df[col]))
tiledb.from_pandas(uri, df)
tiledb.from_pandas(uri, df, sparse=True)
else:
uri = ""
annotation = Annotation(
tiledb_uri=uri,
user_id=user_id,
dataset_id=str(dataset_id),
schema_hints=json.dumps(dataframe_schema_type_hints),
)
self.db.session.add(annotation)
self.db.session.commit()
server/common/app_config.py
-960
View File
@@ -1,960 +0,0 @@
import copy
import os
import sys
import warnings
from os.path import splitext, basename, isdir
from urllib.parse import urlparse, quote_plus
import yaml
from flatten_dict import flatten, unflatten
import server.compute.diffexp_cxg as diffexp_tiledb
import server.compute.scanpy
from server import display_version as cellxgene_display_version
from server.auth.auth import AuthTypeFactory
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from server.common.annotations.local_file_csv import AnnotationsLocalFile
from server.common.data_locator import discover_s3_region_name
from server.common.default_config import get_default_config
from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
from server.db.db_utils import DbUtils
DEFAULT_SERVER_PORT = 5005
# anything bigger than this will generate a special message
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
class AppFeature(object):
def __init__(self, path, available=False, method="POST", extra={}):
self.path = path
self.available = available
self.method = method
self.extra = extra
for k, v in extra.items():
setattr(self, k, v)
def todict(self):
d = dict(available=self.available, method=self.method, path=self.path)
d.update(self.extra)
return d
class AppConfig(object):
"""AppConfig stores all the configuration for cellxgene. The configuration is divided into two main parts:
server attributes, and dataset attributes. The server_config contains attributes that refer to the server process
as a whole. The default_dataset_config referes to attributes that are associated with the features and
presentations of a dataset. The dataset config attributes can be overridden depending on the url by which the
dataset was accessed. These are stored in dataroot_config.
AppConfig has methods to initialize, modify, and access the configuration.
"""
def __init__(self):
# the default configuration (see default_config.py)
self.default_config = get_default_config()
# the server configuration
self.server_config = ServerConfig(self, self.default_config["server"])
# the dataset config, unless overridden by an entry in dataroot_config
self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
# a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
# attribute of the server_config.
self.dataroot_config = {}
# Set to true when config_completed is called
self.is_completed = False
def get_dataset_config(self, dataroot_key):
if self.server_config.single_dataset__datapath:
return self.default_dataset_config
else:
return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
def check_config(self):
"""Verify all the attributes have been checked"""
if not self.is_completed:
raise ConfigurationError("The configuration has not been completed")
self.server_config.check_config()
self.default_dataset_config.check_config()
for dataset_config in self.dataroot_config.values():
dataset_config.check_config()
def update_server_config(self, **kw):
self.server_config.update(**kw)
self.is_complete = False
def update_default_dataset_config(self, **kw):
self.default_dataset_config.update(**kw)
# update all the other dataset configs, if any
for value in self.dataroot_config.values():
value.update(**kw)
self.is_complete = False
def update_from_config_file(self, config_file):
with open(config_file) as fyaml:
config = yaml.load(fyaml, Loader=yaml.FullLoader)
self.server_config.update_from_config(config["server"], "server")
self.default_dataset_config.update_from_config(config["dataset"], "dataset")
per_dataset_config = config.get("per_dataset_config", {})
for key, dataroot_config in per_dataset_config.items():
# first create and initialize the dataroot with the default config
self.add_dataroot_config(key, **config["dataset"])
# then apply the per dataset configuration
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
self.is_complete = False
def write_config(self, config_file):
"""output the config to a yaml file"""
server = self.server_config.create_mapping(self.server_config.default_config)
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = dict(server={}, dataset={})
for attrname in server.keys():
config["server__" + attrname] = getattr(self.server_config, attrname)
for attrname in dataset.keys():
config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
if self.dataroot_config:
config["per_dataset_config"] = {}
for dataroot_tag, dataroot_config in self.dataroot_config.items():
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
for attrname in dataset.keys():
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
config = unflatten(config, splitter=lambda key: key.split("__"))
yaml.dump(config, open(config_file, "w"))
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
diff_server = self.server_config.changes_from_default()
diff_dataset = self.default_dataset_config.changes_from_default()
diff = dict(server=diff_server, dataset=diff_dataset)
return diff
def add_dataroot_config(self, dataroot_tag, **kw):
"""Create a new dataset config object based on the default dataset config, and kw parameters"""
if dataroot_tag in self.dataroot_config:
raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
if type(self.server_config.multi_dataset__dataroot) != dict:
raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
if dataroot_tag not in self.server_config.multi_dataset__dataroot:
raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
self.is_completed = False
self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = {key: value[1] for key, value in flat_config.items()}
self.dataroot_config[dataroot_tag].update(**config)
self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
def complete_config(self, messagefn=None):
"""The configure options are checked, and any additional setup based on the config
parameters is done"""
if messagefn is None:
def noop(message):
pass
messagefn = noop
# TODO: to give better error messages we can add a mapping between where each config
# attribute originated (e.g. command line argument or config file), then in the error
# messages we can give correct context for attributes with bad value.
context = dict(messagefn=messagefn)
self.server_config.complete_config(context)
self.default_dataset_config.complete_config(context)
for dataroot_config in self.dataroot_config.values():
dataroot_config.complete_config(context)
self.is_completed = True
self.check_config()
def get_matrix_data_cache_manager(self):
return self.server_config.matrix_data_cache_manager
def is_multi_dataset(self):
return self.server_config.multi_dataset__dataroot is not None
def get_title(self, data_adaptor):
return (
self.server_config.single_dataset__title
if self.server_config.single_dataset__title
else data_adaptor.get_title()
)
def get_about(self, data_adaptor):
return (
self.server_config.single_dataset__about
if self.server_config.single_dataset__about
else data_adaptor.get_about()
)
def get_client_config(self, data_adaptor):
"""
Return the configuration as required by the /config REST route
"""
server_config = self.server_config
dataset_config = data_adaptor.dataset_config
annotation = dataset_config.user_annotations
auth = server_config.auth
# FIXME The current set of config is not consistently presented:
# we have camalCase, hyphen-text, and underscore_text
# make sure the configuration has been checked.
self.check_config()
# features
features = [f.todict() for f in data_adaptor.get_features(annotation)]
# display_names
title = self.get_title(data_adaptor)
about = self.get_about(data_adaptor)
display_names = dict(engine=data_adaptor.get_name(), dataset=title)
# library_versions
library_versions = {}
library_versions.update(data_adaptor.get_library_versions())
library_versions["cellxgene"] = cellxgene_display_version
# links
links = {"about-dataset": about}
# parameters
parameters = {
"layout": dataset_config.embeddings__names,
"max-category-items": dataset_config.presentation__max_categories,
"obs_names": server_config.single_dataset__obs_names,
"var_names": server_config.single_dataset__var_names,
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
"backed": server_config.adaptor__anndata_adaptor__backed,
"disable-diffexp": not dataset_config.diffexp__enable,
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
"annotations_cell_ontology_enabled": False,
"annotations_cell_ontology_obopath": None,
"annotations_cell_ontology_terms": None,
"custom_colors": dataset_config.presentation__custom_colors,
"diffexp-may-be-slow": False,
"about_legal_tos": dataset_config.app__about_legal_tos,
"about_legal_privacy": dataset_config.app__about_legal_privacy,
}
# corpora dataset_props
# TODO/Note: putting info from the dataset into the /config is not ideal.
# However, it is definitely not part of /schema, and we do not have a top-level
# route for data properties. Consider creating one at some point.
corpora_props = data_adaptor.get_corpora_props()
if corpora_props and "default_embedding" in corpora_props:
default_embedding = corpora_props["default_embedding"]
if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
default_embedding = default_embedding[2:] # drop X_ prefix
if default_embedding in data_adaptor.get_embedding_names():
parameters["default_embedding"] = default_embedding
data_adaptor.update_parameters(parameters)
if annotation:
annotation.update_parameters(parameters, data_adaptor)
# gather it all together
c = {}
config = c["config"] = {}
config["features"] = features
config["displayNames"] = display_names
config["library_versions"] = library_versions
config["links"] = links
config["parameters"] = parameters
config["corpora_props"] = corpora_props
config["limits"] = {
"column_request_max": server_config.limits__column_request_max,
"diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
}
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
config["authentication"] = {
"requires_client_login": auth.requires_client_login(),
}
if auth.requires_client_login():
config["authentication"].update({
"login": auth.get_login_url(data_adaptor),
"logout": auth.get_logout_url(data_adaptor),
})
return c
def get_client_userinfo(self, data_adaptor):
"""
Return the userinfo as required by the /userinfo REST route
"""
server_config = self.server_config
dataset_config = data_adaptor.dataset_config
auth = server_config.auth
# make sure the configuration has been checked.
self.check_config()
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
userinfo = {}
userinfo["userinfo"] = {
"is_authenticated": auth.is_user_authenticated(),
"username": auth.get_user_name(),
"user_id": auth.get_user_id()
}
return userinfo
else:
return None
class BaseConfig(object):
"""This class handles the mechanics of updating and checking attributes.
Derived classes are expected to store the actual attributes"""
def __init__(self, app_config, default_config, dictval_cases={}):
# reference back to the app_config
self.app_config = app_config
# the complete set of attribute and their default values (unflattened)
self.default_config = default_config
# attributes where the value may be a dict (and therefore are not flattened)
self.dictval_cases = dictval_cases
# used to make sure every attribute value is checked
self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()}
def create_mapping(self, config):
"""Create a mapping from attribute names to (location in the config tree, value)"""
dc = copy.deepcopy(config)
mapping = {}
# special cases where the value could be a dict.
# If its value is not None, the entry is added to the mapping, and not included
# in the flattening below.
for dictval_case in self.dictval_cases:
cur = dc
for part in dictval_case[:-1]:
cur = cur.get(part, {})
val = cur.get(dictval_case[-1])
if val is not None:
key = "__".join(dictval_case)
mapping[key] = (dictval_case, val)
del cur[dictval_case[-1]]
flat_config = flatten(dc)
for key, value in flat_config.items():
# name of the attribute
attr = "__".join(key)
mapping[attr] = (key, value)
return mapping
def check_attr(self, attrname, vtype):
val = getattr(self, attrname)
if type(vtype) in (list, tuple):
if type(val) not in vtype:
tnames = ",".join([x.__name__ for x in vtype])
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
)
else:
if type(val) != vtype:
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, "
f"expected type {vtype.__name__}, got {type(val).__name__}"
)
self.attr_checked[attrname] = True
def check_config(self):
mapping = self.create_mapping(self.default_config)
for key in mapping.keys():
if not self.attr_checked[key]:
raise ConfigurationError(f"The attr '{key}' has not been checked")
def update(self, **kw):
for key, value in kw.items():
if not hasattr(self, key):
raise ConfigurationError(f"unknown config parameter {key}.")
try:
if type(value) == tuple:
# convert tuple values to list values
value = list(value)
setattr(self, key, value)
except KeyError:
raise ConfigurationError(f"Unable to set config parameter {key}.")
self.attr_checked[key] = False
def update_from_config(self, config, prefix):
mapping = self.create_mapping(config)
for attr, (key, value) in mapping.items():
if not hasattr(self, attr):
raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
try:
setattr(self, attr, value)
except KeyError:
raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}")
self.attr_checked[attr] = False
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
mapping = self.create_mapping(self.default_config)
diff = []
for attrname, (key, defval) in mapping.items():
curval = getattr(self, attrname)
if curval != defval:
diff.append((attrname, curval, defval))
return diff
class ServerConfig(BaseConfig):
"""Manages the config attribute associated with the server."""
def __init__(self, app_config, default_config):
dictval_cases = [
("app", "csp_directives"),
("authentication", "params_oauth", "cookie"),
("authentication", "params_oauth", "jwt_decode_options"),
("adaptor", "cxg_adaptor", "tiledb_ctx"),
("multi_dataset", "dataroot"),
]
super().__init__(app_config, default_config, dictval_cases)
dc = default_config
try:
self.app__verbose = dc["app"]["verbose"]
self.app__debug = dc["app"]["debug"]
self.app__host = dc["app"]["host"]
self.app__port = dc["app"]["port"]
self.app__open_browser = dc["app"]["open_browser"]
self.app__force_https = dc["app"]["force_https"]
self.app__flask_secret_key = dc["app"]["flask_secret_key"]
self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"]
self.app__server_timing_headers = dc["app"]["server_timing_headers"]
self.app__csp_directives = dc["app"]["csp_directives"]
self.app__api_base_url = dc["app"]["api_base_url"]
self.app__web_base_url = dc["app"]["web_base_url"]
self.authentication__type = dc["authentication"]["type"]
self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][
"oauth_api_base_url"
]
self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"]
self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"]
self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][
"jwt_decode_options"]
self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"]
self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"]
self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
self.multi_dataset__index = dc["multi_dataset"]["index"]
self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"]
self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"]
self.single_dataset__datapath = dc["single_dataset"]["datapath"]
self.single_dataset__obs_names = dc["single_dataset"]["obs_names"]
self.single_dataset__var_names = dc["single_dataset"]["var_names"]
self.single_dataset__about = dc["single_dataset"]["about"]
self.single_dataset__title = dc["single_dataset"]["title"]
self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"]
self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"]
self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"]
self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"]
self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"]
self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"]
self.limits__column_request_max = dc["limits"]["column_request_max"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The matrix data cache manager is created during the complete_config and stored here.
self.matrix_data_cache_manager = None
# The authentication object
self.auth = None
def complete_config(self, context):
self.handle_app(context)
self.handle_data_source(context)
self.handle_authentication(context)
self.handle_data_locator(context)
self.handle_adaptor(context) # may depend on data_locator
self.handle_single_dataset(context) # may depend on adaptor
self.handle_multi_dataset(context) # may depend on adaptor
self.handle_diffexp(context)
self.handle_limits(context)
self.check_config()
def handle_app(self, context):
self.check_attr("app__verbose", bool)
self.check_attr("app__debug", bool)
self.check_attr("app__host", str)
self.check_attr("app__port", (type(None), int))
self.check_attr("app__open_browser", bool)
self.check_attr("app__force_https", bool)
self.check_attr("app__flask_secret_key", (type(None), str))
self.check_attr("app__generate_cache_control_headers", bool)
self.check_attr("app__server_timing_headers", bool)
self.check_attr("app__csp_directives", (type(None), dict))
self.check_attr("app__api_base_url", (type(None), str))
self.check_attr("app__web_base_url", (type(None), str))
if self.app__port:
try:
if not is_port_available(self.app__host, self.app__port):
raise ConfigurationError(
f"The port selected {self.app__port} is in use, please configure an open port."
)
except OverflowError:
raise ConfigurationError(f"Invalid port: {self.app__port}")
else:
try:
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
except ValueError:
raise ConfigurationError(
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
)
try:
self.app__port = find_available_port(self.app__host, default_server_port)
except OverflowError:
raise ConfigurationError(f"Invalid port: {default_server_port}")
if self.app__debug:
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
self.app__verbose = True
self.app__open_browser = False
else:
warnings.formatwarning = custom_format_warning
if not self.app__verbose:
sys.tracebacklimit = 0
# secret key:
# first, from CXG_SECRET_KEY environment variable
# second, from config file
self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
# CSP Directives are a dict of string: list(string) or string: string
if self.app__csp_directives is not None:
for k, v in self.app__csp_directives.items():
if not isinstance(k, str):
raise ConfigurationError("CSP directive names must be a string.")
if isinstance(v, list):
for policy in v:
if not isinstance(policy, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
elif not isinstance(v, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
if self.app__web_base_url is None:
self.app__web_base_url = self.app__api_base_url
def handle_authentication(self, context):
self.check_attr("authentication__type", (type(None), str))
# oauth
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes)
self.check_attr("authentication__params_oauth__client_id", ptypes)
self.check_attr("authentication__params_oauth__client_secret", ptypes)
self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict))
self.check_attr("authentication__params_oauth__session_cookie", bool)
if self.authentication__params_oauth__session_cookie:
self.check_attr("authentication__params_oauth__cookie", (type(None), dict))
else:
self.check_attr("authentication__params_oauth__cookie", dict)
# secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
# second, from config file
self.authentication__params__oauth__client_secret = os.environ.get(
"CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret)
self.auth = AuthTypeFactory.create(self.authentication__type, self)
if self.auth is None:
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
def handle_data_locator(self, context):
self.check_attr("data_locator__s3__region_name", (type(None), bool, str))
if self.data_locator__s3__region_name is True:
path = self.single_dataset__datapath or self.multi_dataset__dataroot
if type(path) == dict:
# if multi_dataset__dataroot is a dict, then use the first key
# that is in s3. NOTE: it is not supported to have dataroots
# in different regions.
paths = [val.get("dataroot") for val in path.values()]
for path in paths:
if path.startswith("s3://"):
break
if path.startswith("s3://"):
region_name = discover_s3_region_name(path)
if region_name is None:
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
else:
region_name = None
self.data_locator__s3__region_name = region_name
def handle_data_source(self, context):
self.check_attr("single_dataset__datapath", (str, type(None)))
self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
if self.single_dataset__datapath is None:
if self.multi_dataset__dataroot is None:
# TODO: change the error message once dataroot is fully supported
raise ConfigurationError("missing datapath")
return
else:
if self.multi_dataset__dataroot is not None:
raise ConfigurationError("must supply only one of datapath or dataroot")
def handle_single_dataset(self, context):
self.check_attr("single_dataset__datapath", (str, type(None)))
self.check_attr("single_dataset__title", (str, type(None)))
self.check_attr("single_dataset__about", (str, type(None)))
self.check_attr("single_dataset__obs_names", (str, type(None)))
self.check_attr("single_dataset__var_names", (str, type(None)))
if self.single_dataset__datapath is None:
return
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
def handle_multi_dataset(self, context):
self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
self.check_attr("multi_dataset__index", (type(None), bool, str))
self.check_attr("multi_dataset__allowed_matrix_types", list)
self.check_attr("multi_dataset__matrix_cache__max_datasets", int)
self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float))
if self.multi_dataset__dataroot is None:
return
if type(self.multi_dataset__dataroot) == str:
default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
self.multi_dataset__dataroot = dict(d=default_dict)
for tag, dataroot_dict in self.multi_dataset__dataroot.items():
if "base_url" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
if "dataroot" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
base_url = dataroot_dict["base_url"]
# sanity check for well formed base urls
bad = False
if type(base_url) != str:
bad = True
elif os.path.normpath(base_url) != base_url:
bad = True
else:
base_url_parts = base_url.split("/")
if [quote_plus(part) for part in base_url_parts] != base_url_parts:
bad = True
if ".." in base_url_parts:
bad = True
if bad:
raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
# verify all the base_urls are unique
base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
if len(base_urls) > len(set(base_urls)):
raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
# error checking
for mtype in self.multi_dataset__allowed_matrix_types:
try:
MatrixDataType(mtype)
except ValueError:
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(
max_cached=self.multi_dataset__matrix_cache__max_datasets,
timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
)
def handle_diffexp(self, context):
self.check_attr("diffexp__alg_cxg__max_workers", (str, int))
self.check_attr("diffexp__alg_cxg__cpu_multiplier", int)
self.check_attr("diffexp__alg_cxg__target_workunit", int)
max_workers = self.diffexp__alg_cxg__max_workers
cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
cpu_count = os.cpu_count()
max_workers = min(max_workers, cpu_multiplier * cpu_count)
diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
def handle_adaptor(self, context):
# cxg
self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict)
regionkey = "vfs.s3.region"
if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
if type(self.data_locator__s3__region_name) == str:
self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
from server.data_cxg.cxg_adaptor import CxgAdaptor
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
# anndata
self.check_attr("adaptor__anndata_adaptor__backed", bool)
def handle_limits(self, context):
self.check_attr("limits__diffexp_cellcount_max", (type(None), int))
self.check_attr("limits__column_request_max", (type(None), int))
def exceeds_limit(self, limit_name, value):
limit_value = getattr(self, "limits__" + limit_name, None)
if limit_value is None: # disabled
return False
return value > limit_value
def get_api_base_url(self):
if self.app__api_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__api_base_url and self.app__api_base_url.endswith("/"):
return self.app__api_base_url[:-1]
return self.app__api_base_url
def get_web_base_url(self):
if self.app__web_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__web_base_url is None:
return self.get_api_base_url()
if self.app__web_base_url.endswith("/"):
return self.app__web_base_url[:-1]
return self.api__web_base_url
class DatasetConfig(BaseConfig):
"""Manages the config attribute associated with a dataset."""
def __init__(self, tag, app_config, default_config):
super().__init__(app_config, default_config)
self.tag = tag
dc = default_config
try:
self.app__scripts = dc["app"]["scripts"]
self.app__inline_scripts = dc["app"]["inline_scripts"]
self.app__about_legal_tos = dc["app"]["about_legal_tos"]
self.app__about_legal_privacy = dc["app"]["about_legal_privacy"]
self.app__authentication_enable = dc["app"]["authentication_enable"]
self.presentation__max_categories = dc["presentation"]["max_categories"]
self.presentation__custom_colors = dc["presentation"]["custom_colors"]
self.user_annotations__enable = dc["user_annotations"]["enable"]
self.user_annotations__type = dc["user_annotations"]["type"]
self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"]
self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ] # noqa E501
self.embeddings__names = dc["embeddings"]["names"]
self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
self.diffexp__enable = dc["diffexp"]["enable"]
self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = dc["diffexp"]["top_n"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The annotation object is created during complete_config and stored here.
self.user_annotations = None
def complete_config(self, context):
self.handle_app(context)
self.handle_presentation(context)
self.handle_user_annotations(context)
self.handle_embeddings(context)
self.handle_diffexp(context)
def handle_app(self, context):
self.check_attr("app__scripts", list)
self.check_attr("app__inline_scripts", list)
self.check_attr("app__about_legal_tos", (type(None), str))
self.check_attr("app__about_legal_privacy", (type(None), str))
self.check_attr("app__authentication_enable", bool)
# scripts can be string (filename) or dict (attributes). Convert string to dict.
scripts = []
for s in self.app__scripts:
if isinstance(s, str):
scripts.append({"src": s})
elif isinstance(s, dict) and isinstance(s["src"], str):
scripts.append(s)
else:
raise ConfigurationError("Scripts must be string or dict")
self.app__scripts = scripts
def handle_presentation(self, context):
self.check_attr("presentation__max_categories", int)
self.check_attr("presentation__custom_colors", bool)
def handle_user_annotations(self, context):
self.check_attr("user_annotations__enable", bool)
self.check_attr("user_annotations__type", str)
self.check_attr("user_annotations__local_file_csv__directory", (type(None), str))
self.check_attr("user_annotations__local_file_csv__file", (type(None), str))
self.check_attr("user_annotations__ontology__enable", bool)
self.check_attr("user_annotations__ontology__obo_location", (type(None), str))
self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str))
self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str))
if self.user_annotations__enable:
server_config = self.app_config.server_config
if not self.app__authentication_enable:
raise ConfigurationError("user annotations requires authentication to be enabled")
if not server_config.auth.is_valid_authentication_type():
auth_type = server_config.authentication__type
raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
# TODO, replace this with a factory pattern once we have more than one way
# to do annotations. currently only local_file_csv
if self.user_annotations__type == "local_file_csv":
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None and dirname is not None:
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
if filename is not None:
lf_name, lf_ext = splitext(filename)
if lf_ext and lf_ext != ".csv":
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
if dirname is not None and not isdir(dirname):
try:
os.mkdir(dirname)
except OSError:
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
self.user_annotations = AnnotationsLocalFile(dirname, filename)
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
server_config = self.app_config.server_config
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
try:
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
except OntologyLoadFailure as e:
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
elif self.user_annotations__type == "hosted_tiledb_array":
self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str)
self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str)
self.user_annotations = AnnotationsHostedTileDB(
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
)
else:
raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
else:
if self.user_annotations__type == "local_file_csv":
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None:
context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
if dirname is not None:
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
if self.user_annotations__ontology__enable:
context["messagefn"](
"Warning: --experimental-annotations-ontology" " ignored as annotations are disabled."
)
if self.user_annotations__ontology__obo_location is not None:
context["messagefn"](
"Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled."
)
def handle_embeddings(self, context):
self.check_attr("embeddings__names", list)
self.check_attr("embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
matrix_data_loader = MatrixDataLoader(
server_config.single_dataset__datapath, app_config=self.app_config
)
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
try:
server.compute.scanpy.get_scanpy_module()
except NotImplementedError:
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
def handle_diffexp(self, context):
self.check_attr("diffexp__enable", bool)
self.check_attr("diffexp__lfc_cutoff", float)
self.check_attr("diffexp__top_n", int)
server_config = self.app_config.server_config
if server_config.single_dataset__datapath:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
context["messagefn"](
"CAUTION: due to the size of your dataset, "
"running differential expression may take longer or fail."
)
server/common/aws_secret_utils.py
+1 -62
View File
@@ -1,72 +1,11 @@
import logging
import os
import sys
import boto3
from flask import json
from server.common.data_locator import discover_s3_region_name
from server.common.errors import SecretKeyRetrievalError
def handle_config_from_secret(app_config):
"""Update configuration from the secret manager"""
secret_name = os.getenv("CXG_AWS_SECRET_NAME")
if not secret_name:
return
# need to find the secret manager region.
# 1. from CXG_AWS_SECRET_REGION_NAME
# 2. discover from dataroot location (if on s3)
# 3. discover from config file location (if on s3)
secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
if secret_region_name is None:
secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
if not secret_region_name:
from server.eb.app import config_file
secret_region_name = discover_s3_region_name(config_file)
if not secret_region_name:
logging.error("Could not determine the AWS Secret Manager region")
sys.exit(1)
secrets = get_secret_key(secret_region_name, secret_name)
if not secrets:
return
server_attrs = (
("flask_secret_key", "app__flask_secret_key"),
("oauth_client_secret", "authentication__params_oauth__client_secret"),
)
default_dataset_attrs = (
("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),
)
# update server configuration attributes
for key, attr in server_attrs:
cur_val = getattr(app_config.server_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_server_config(**{attr : val})
# update default dataset configuration attributes
for key, attr in default_dataset_attrs:
cur_val = getattr(app_config.default_dataset_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_default_dataset_config(**{attr : val})
def get_secret_key(region_name, secret_name):
session = boto3.session.Session()
client = session.client(service_name="secretsmanager", region_name=region_name)
@@ -79,6 +18,6 @@ def get_secret_key(region_name, secret_name):
return secret
except Exception as e:
logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
raise SecretKeyRetrievalError
raise SecretKeyRetrievalError(str(e))
return None
server/common/config/__init__.py
+4
View File
@@ -0,0 +1,4 @@
from server.common.aws_secret_utils import get_secret_key # noqa F504
DEFAULT_SERVER_PORT = 5005
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
server/common/config/app_config.py
+247
View File
@@ -0,0 +1,247 @@
import yaml
from flatten_dict import unflatten
from server.default_config import get_default_config
from server.common.config.dataset_config import DatasetConfig
from server.common.config.server_config import ServerConfig
from server.common.config.external_config import ExternalConfig
from server.common.errors import ConfigurationError
class AppConfig(object):
"""
AppConfig stores all the configuration for cellxgene.
AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
The server_config contains attributes that refer to the server process as a whole.
The default_dataset_config refers to attributes that are associated with the features and
presentations of a dataset.
The dataset config attributes can be overridden depending on the url by which the
dataset was accessed. These are stored in dataroot_config.
AppConfig has methods to initialize, modify, and access the configuration.
"""
def __init__(self):
# the default configuration (see default_config.py)
# TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
# defaults within the config class?
self.default_config = get_default_config()
# the server configuration
self.server_config = ServerConfig(self, self.default_config["server"])
# the dataset config, unless overridden by an entry in dataroot_config
self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
# a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
# attribute of the server_config. The default dataset config will apply to all datasets unless a different set
# of config vars was passed for a specific dataset under the multidataset config. For example:
"""
per_dataset_config:
d1:
user_annotations:
enable: false
d2:
user_annotations:
enable: true
"""
# dataroot config
self.dataroot_config = {}
# external config
self.external_config = ExternalConfig(self, self.default_config["external"])
# Set to true when config_completed is called
self.is_completed = False
def get_dataset_config(self, dataroot_key):
if self.server_config.single_dataset__datapath:
return self.default_dataset_config
else:
return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
def check_config(self):
"""Verify all the attributes in the config have been type checked"""
if not self.is_completed:
raise ConfigurationError("The configuration has not been completed")
self.server_config.check_config()
self.default_dataset_config.check_config()
for dataset_config in self.dataroot_config.values():
dataset_config.check_config()
self.external_config.check_config()
def update_server_config(self, **kw):
self.server_config.update(**kw)
self.is_complete = False
def update_default_dataset_config(self, **kw):
self.default_dataset_config.update(**kw)
# update all the other dataset configs, if any
for value in self.dataroot_config.values():
value.update(**kw)
self.is_complete = False
def update_single_config_from_path_and_value(self, path, value):
"""Update a single config parameter with the value.
Path is a list of string, that gives a path to the config parameter to be updated.
For example, path may be ["server","app","port"].
"""
self.is_complete = False
if not isinstance(path, list):
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
for part in path:
if not isinstance(part, str):
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
if path[0] == "server":
attr = "__".join(path[1:])
try:
self.update_server_config(**{attr: value})
except ConfigurationError:
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
elif path[0] == "dataset":
attr = "__".join(path[1:])
try:
self.update_default_dataset_config(**{attr: value})
except ConfigurationError:
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
elif path[0] == "per_dataset_config":
if len(path) < 2:
raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
dataroot = path[1]
if dataroot not in self.dataroot_config:
dataroots = str(list(self.dataroot_config.keys()))
raise ConfigurationError(
f"unknown dataroot when using per_dataset_config: got '{path}',"
f" dataroots specified in config are {dataroots}"
)
attr = "__".join(path[2:])
try:
self.dataroot_config[dataroot].update(**{attr: value})
except ConfigurationError:
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
def update_from_config_file(self, config_file):
try:
with open(config_file) as yml_file:
config = yaml.safe_load(yml_file)
except yaml.YAMLError as e:
raise ConfigurationError(f"The specified config file contained an error: {e}")
except OSError as e:
raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
if config.get("server"):
self.server_config.update_from_config(config["server"], "server")
if config.get("dataset"):
self.default_dataset_config.update_from_config(config["dataset"], "dataset")
per_dataset_config = config.get("per_dataset_config", {})
for key, dataroot_config in per_dataset_config.items():
# first create and initialize the dataroot with the default config
self.add_dataroot_config(key, **config["dataset"])
# then apply the per dataset configuration
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
if config.get("external"):
self.external_config.update_from_config(config["external"], "external")
self.is_complete = False
def config_to_dict(self):
"""return the configuration as an unflattened dict"""
server = self.server_config.create_mapping(self.server_config.default_config)
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
external = self.external_config.create_mapping(self.external_config.default_config)
config = dict(server={}, dataset={})
for attrname in server.keys():
config["server__" + attrname] = getattr(self.server_config, attrname)
for attrname in dataset.keys():
config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
if self.dataroot_config:
config["per_dataset_config"] = {}
for dataroot_tag, dataroot_config in self.dataroot_config.items():
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
for attrname in dataset.keys():
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
for attrname in external.keys():
config["external__" + attrname] = getattr(self.external_config, attrname)
config = unflatten(config, splitter=lambda key: key.split("__"))
return config
def write_config(self, config_file):
"""output the config to a yaml file"""
config = self.config_to_dict()
yaml.dump(config, open(config_file, "w"))
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
diff_server = self.server_config.changes_from_default()
diff_dataset = self.default_dataset_config.changes_from_default()
diff_external = self.external.changes_from_default()
diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
return diff
def add_dataroot_config(self, dataroot_tag, **kw):
"""Create a new dataset config object based on the default dataset config, and kw parameters"""
if dataroot_tag in self.dataroot_config:
raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
if type(self.server_config.multi_dataset__dataroot) != dict:
raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
if dataroot_tag not in self.server_config.multi_dataset__dataroot:
raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
self.is_completed = False
self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = {key: value[1] for key, value in flat_config.items()}
self.dataroot_config[dataroot_tag].update(**config)
self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
def complete_config(self, messagefn=None):
"""The configure options are checked, and any additional setup based on the config
parameters is done"""
if messagefn is None:
def noop(message):
pass
messagefn = noop
# TODO: to give better error messages we can add a mapping between where each config
# attribute originated (e.g. command line argument or config file), then in the error
# messages we can give correct context for attributes with bad value.
context = dict(messagefn=messagefn)
# complete config for external_config first, since this may update values in the other sections
self.external_config.complete_config(context)
self.server_config.complete_config(context)
self.default_dataset_config.complete_config(context)
for dataroot_config in self.dataroot_config.values():
dataroot_config.complete_config(context)
self.is_completed = True
self.check_config()
def get_matrix_data_cache_manager(self):
return self.server_config.matrix_data_cache_manager
def is_multi_dataset(self):
return self.server_config.multi_dataset__dataroot is not None
def get_title(self, data_adaptor):
return (
self.server_config.single_dataset__title
if self.server_config.single_dataset__title
else data_adaptor.get_title()
)
def get_about(self, data_adaptor):
return (
self.server_config.single_dataset__about
if self.server_config.single_dataset__about
else data_adaptor.get_about()
)
server/common/config/base_config.py
+132
View File
@@ -0,0 +1,132 @@
import copy
from flatten_dict import flatten
from server.common.errors import ConfigurationError
class BaseConfig(object):
"""
This class handles the mechanics of updating and checking attributes.
Derived classes are expected to store the actual attributes
Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
"""
def __init__(self, app_config, default_config, dictval_cases={}):
# reference back to the app_config
self.app_config = app_config
# the complete set of attributes and their default values (unflattened)
self.default_config = default_config
# attributes where the value may be a dict (and therefore are not flattened)
self.dictval_cases = dictval_cases
# used to make sure every attribute value is checked
self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
def create_mapping(self, config):
"""
Create a dictionary where the keys are the name of attributes (using double underscore convention)
For example: authentication__type
The values are a tuple,
- the first item of the tuple is a tuple of path elements (location in config 'tree')
- the second item is the value of the config parameter
For example: (('authentication', 'type'), 'session'))
"""
config_copy = copy.deepcopy(config)
mapping = {}
# special cases where the value could be a dict.
# If its value is not None, the entry is added to the mapping, and not included
# in the flattening below.
for dictval_case in self.dictval_cases:
cur = config_copy
for part in dictval_case[:-1]:
cur = cur.get(part, {})
val = cur.get(dictval_case[-1])
if val is not None:
key = "__".join(dictval_case)
mapping[key] = (dictval_case, val)
del cur[dictval_case[-1]]
flat_config = flatten(config_copy)
for key, value in flat_config.items():
# name of the attribute
attr = "__".join(key)
mapping[attr] = (key, value)
return mapping
def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
val = getattr(self, attrname)
if type(vtype) in (list, tuple):
if type(val) not in vtype:
tnames = ",".join([x.__name__ for x in vtype])
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
)
else:
if type(val) != vtype:
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, "
f"expected type {vtype.__name__}, got {type(val).__name__}"
)
self.attr_checked[attrname] = True
def check_config(self):
mapping = self.create_mapping(self.default_config)
for key in mapping.keys():
if not self.attr_checked[key]:
raise ConfigurationError(f"The attr '{key}' has not been checked")
def update(self, **kw):
"""Update the attributes defined in kw with their new values."""
for key, value in kw.items():
if not hasattr(self, key):
# check if the key is setting into a dictval entry.
found_dictval = False
for dictval in self.dictval_cases:
dictvalname = "__".join(dictval)
if dictvalname + "__" in key:
dictkey = key[len(dictvalname) + 2 :]
curdictval = getattr(self, dictvalname)
if curdictval is None:
setattr(self, dictvalname, dict(dictkey=value))
else:
curdictval[dictkey] = value
found_dictval = True
break
if found_dictval:
continue
raise ConfigurationError(f"unknown config parameter {key}.")
try:
if type(value) == tuple:
# convert tuple values to list values
value = list(value)
setattr(self, key, value)
except KeyError:
raise ConfigurationError(f"Unable to set config parameter {key}.")
self.attr_checked[key] = False
def update_from_config(self, config, prefix):
mapping = self.create_mapping(config)
for attr, (key, value) in mapping.items():
if not hasattr(self, attr):
raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
setattr(self, attr, value)
self.attr_checked[attr] = False
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
mapping = self.create_mapping(self.default_config)
diff = []
for attrname, (key, defval) in mapping.items():
curval = getattr(self, attrname)
if curval != defval:
diff.append((attrname, curval, defval))
return diff
server/common/config/client_config.py
+122
View File
@@ -0,0 +1,122 @@
from server import display_version as cellxgene_display_version
def get_client_config(app_config, data_adaptor):
"""
Return the configuration as required by the /config REST route
"""
server_config = app_config.server_config
dataset_config = data_adaptor.dataset_config
annotation = dataset_config.user_annotations
auth = server_config.auth
# FIXME The current set of config is not consistently presented:
# we have camalCase, hyphen-text, and underscore_text
# make sure the configuration has been checked.
app_config.check_config()
# display_names
title = app_config.get_title(data_adaptor)
about = app_config.get_about(data_adaptor)
display_names = dict(engine=data_adaptor.get_name(), dataset=title)
# library_versions
library_versions = {}
library_versions.update(data_adaptor.get_library_versions())
library_versions["cellxgene"] = cellxgene_display_version
# links
links = {"about-dataset": about}
# parameters
parameters = {
"layout": dataset_config.embeddings__names,
"max-category-items": dataset_config.presentation__max_categories,
"obs_names": server_config.single_dataset__obs_names,
"var_names": server_config.single_dataset__var_names,
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
"backed": server_config.adaptor__anndata_adaptor__backed,
"disable-diffexp": not dataset_config.diffexp__enable,
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
"annotations_cell_ontology_enabled": False,
"annotations_cell_ontology_obopath": None,
"annotations_cell_ontology_terms": None,
"custom_colors": dataset_config.presentation__custom_colors,
"diffexp-may-be-slow": False,
"about_legal_tos": dataset_config.app__about_legal_tos,
"about_legal_privacy": dataset_config.app__about_legal_privacy,
}
# corpora dataset_props
# TODO/Note: putting info from the dataset into the /config is not ideal.
# However, it is definitely not part of /schema, and we do not have a top-level
# route for data properties. Consider creating one at some point.
corpora_props = data_adaptor.get_corpora_props()
if corpora_props and "default_embedding" in corpora_props:
default_embedding = corpora_props["default_embedding"]
if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
default_embedding = default_embedding[2:] # drop X_ prefix
if default_embedding in data_adaptor.get_embedding_names():
parameters["default_embedding"] = default_embedding
data_adaptor.update_parameters(parameters)
if annotation:
annotation.update_parameters(parameters, data_adaptor)
# gather it all together
client_config = {}
config = client_config["config"] = {}
config["displayNames"] = display_names
config["library_versions"] = library_versions
config["links"] = links
config["parameters"] = parameters
config["corpora_props"] = corpora_props
config["limits"] = {
"column_request_max": server_config.limits__column_request_max,
"diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
}
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
config["authentication"] = {
"requires_client_login": auth.requires_client_login(),
}
if auth.requires_client_login():
config["authentication"].update(
{
# Todo why are these stored on the data_adaptor?
"login": auth.get_login_url(data_adaptor),
"logout": auth.get_logout_url(data_adaptor),
}
)
return client_config
def get_client_userinfo(app_config, data_adaptor):
"""
Return the userinfo as required by the /userinfo REST route
"""
server_config = app_config.server_config
dataset_config = data_adaptor.dataset_config
auth = server_config.auth
# make sure the configuration has been checked.
app_config.check_config()
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
userinfo = {}
userinfo["userinfo"] = {
"is_authenticated": auth.is_user_authenticated(),
"username": auth.get_user_name(),
"user_id": auth.get_user_id(),
"email": auth.get_user_email(),
"picture": auth.get_user_picture(),
}
return userinfo
server/common/config/dataset_config.py
+230
View File
@@ -0,0 +1,230 @@
import os
from os.path import splitext, isdir
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from server.common.annotations.local_file_csv import AnnotationsLocalFile
from server.common.config.base_config import BaseConfig
from server.common.errors import ConfigurationError, OntologyLoadFailure
from server.compute.scanpy import get_scanpy_module
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
from server.db.db_utils import DbUtils
class DatasetConfig(BaseConfig):
"""Manages the config attribute associated with a dataset."""
def __init__(self, tag, app_config, default_config):
super().__init__(app_config, default_config)
self.tag = tag
try:
self.app__scripts = default_config["app"]["scripts"]
self.app__inline_scripts = default_config["app"]["inline_scripts"]
self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
self.app__authentication_enable = default_config["app"]["authentication_enable"]
self.presentation__max_categories = default_config["presentation"]["max_categories"]
self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
self.user_annotations__enable = default_config["user_annotations"]["enable"]
self.user_annotations__type = default_config["user_annotations"]["type"]
self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
"directory"
]
self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
"obo_location"
]
self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
"hosted_tiledb_array"
]["db_uri"]
self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
"hosted_tiledb_array"
]["hosted_file_directory"]
self.embeddings__names = default_config["embeddings"]["names"]
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
self.diffexp__enable = default_config["diffexp"]["enable"]
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = default_config["diffexp"]["top_n"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The annotation object is created during complete_config and stored here.
self.user_annotations = None
def complete_config(self, context):
self.handle_app()
self.handle_presentation()
self.handle_user_annotations(context)
self.handle_embeddings()
self.handle_diffexp(context)
def handle_app(self):
self.validate_correct_type_of_configuration_attribute("app__scripts", list)
self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
# scripts can be string (filename) or dict (attributes). Convert string to dict.
scripts = []
for script in self.app__scripts:
try:
if isinstance(script, str):
scripts.append({"src": script})
elif isinstance(script, dict) and isinstance(script["src"], str):
scripts.append(script)
else:
raise Exception
except Exception as e:
raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
self.app__scripts = scripts
def handle_presentation(self):
self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
def handle_user_annotations(self, context):
self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__directory", (type(None), str)
)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__file", (type(None), str)
)
self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__ontology__obo_location", (type(None), str)
)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
)
if self.user_annotations__enable:
server_config = self.app_config.server_config
if not self.app__authentication_enable:
raise ConfigurationError("user annotations requires authentication to be enabled")
if not server_config.auth.is_valid_authentication_type():
auth_type = server_config.authentication__type
raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
if self.user_annotations__type == "local_file_csv":
self.handle_local_file_csv_annotations()
elif self.user_annotations__type == "hosted_tiledb_array":
self.handle_hosted_tiledb_annotations()
else:
raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
try:
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
except OntologyLoadFailure as e:
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
else:
self.check_annotation_config_vars_not_set(context)
def handle_local_file_csv_annotations(self):
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None and dirname is not None:
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
if filename is not None:
lf_name, lf_ext = splitext(filename)
if lf_ext and lf_ext != ".csv":
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
if dirname is not None and not isdir(dirname):
try:
os.mkdir(dirname)
except OSError:
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
self.user_annotations = AnnotationsLocalFile(dirname, filename)
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
server_config = self.app_config.server_config
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
def handle_hosted_tiledb_annotations(self):
self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", str
)
self.user_annotations = AnnotationsHostedTileDB(
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
)
def check_annotation_config_vars_not_set(self, context):
if self.user_annotations__type is not None:
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
db_uri = self.user_annotations__hosted_tiledb_array__db_uri
hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
if filename is not None:
context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
if dirname is not None:
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
if db_uri is not None:
context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
if hosted_file_dirname is not None:
context["messagefn"](
"Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
)
if self.user_annotations__ontology__enable:
context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
if self.user_annotations__ontology__obo_location is not None:
context["messagefn"](
"Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
)
def handle_embeddings(self):
self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
matrix_data_loader = MatrixDataLoader(
server_config.single_dataset__datapath, app_config=self.app_config
)
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
try:
get_scanpy_module()
except NotImplementedError:
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
def handle_diffexp(self, context):
self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
server_config = self.app_config.server_config
if server_config.single_dataset__datapath:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
context["messagefn"](
"CAUTION: due to the size of your dataset, "
"running differential expression may take longer or fail."
)
server/common/config/external_config.py
+96
View File
@@ -0,0 +1,96 @@
import os
from server.common.config.base_config import BaseConfig
from server.common.errors import ConfigurationError
from server.common.config import get_secret_key
from server.common.errors import SecretKeyRetrievalError
from server.common.utils.type_conversion_utils import convert_string_to_value
class ExternalConfig(BaseConfig):
"""Manages the config attribute associated with external configuration sources, such as
environment variables or the AWS Secrets Manager."""
def __init__(self, app_config, default_config):
super().__init__(app_config, default_config)
try:
self.environment = default_config["environment"]
self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
def complete_config(self, context):
self.handle_environment(context)
self.handle_aws_secrets_manager(context)
def handle_environment(self, context):
"""For each environment variable defined, get the value (if it is set),
and set the specified config parameter"""
self.validate_correct_type_of_configuration_attribute("environment", list)
for envdict in self.environment:
name = envdict.get("name")
if name is None:
raise ConfigurationError("environment: 'name' is missing")
required = envdict.get("required", False)
if type(required) != bool:
raise ConfigurationError("environment: 'required' must be a bool")
path = envdict.get("path")
if path is None:
raise ConfigurationError("environment: 'path' is missing")
value = os.environ.get(name)
if value is None:
if required:
raise ConfigurationError(f"required environment variable '{name}' not set")
else:
value = convert_string_to_value(value)
self.app_config.update_single_config_from_path_and_value(path, value)
def handle_aws_secrets_manager(self, context):
"""For each aws secret defined, get the key/values, and set the specified config parameter"""
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
if not self.aws_secrets_manager__secrets:
return
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
for secret in self.aws_secrets_manager__secrets:
secret_name = secret.get("name")
if secret_name is None:
raise ConfigurationError("aws_secrets_manager: 'name' is missing")
if not isinstance(secret_name, str):
raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
try:
secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
except SecretKeyRetrievalError as e:
raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
values = secret.get("values")
if values is None:
raise ConfigurationError("aws_secrets_manager: 'values' is missing")
if not isinstance(values, list):
raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
for value in values:
key = value.get("key")
if key is None:
raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
path = value.get("path")
if path is None:
raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
required = value.get("required", False)
if type(required) != bool:
raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
secret_value = secret_dict.get(key)
if secret_value is None:
if required:
raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
else:
secret_value = convert_string_to_value(secret_value)
self.app_config.update_single_config_from_path_and_value(path, secret_value)
server/common/config/server_config.py
+380
View File
@@ -0,0 +1,380 @@
import os
import sys
import warnings
from os.path import basename
from urllib.parse import urlparse, quote_plus
from server.auth.auth import AuthTypeFactory
from server.common.config.base_config import BaseConfig
from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
from server.common.errors import ConfigurationError, DatasetAccessError
from server.common.data_locator import discover_s3_region_name
from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
from server.compute import diffexp_cxg as diffexp_tiledb
from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
class ServerConfig(BaseConfig):
"""Manages the config attribute associated with the server."""
def __init__(self, app_config, default_config):
dictval_cases = [
("app", "csp_directives"),
("authentication", "params_oauth", "cookie"),
("authentication", "params_oauth", "jwt_decode_options"),
("adaptor", "cxg_adaptor", "tiledb_ctx"),
("multi_dataset", "dataroot"),
]
super().__init__(app_config, default_config, dictval_cases)
try:
self.app__verbose = default_config["app"]["verbose"]
self.app__debug = default_config["app"]["debug"]
self.app__host = default_config["app"]["host"]
self.app__port = default_config["app"]["port"]
self.app__open_browser = default_config["app"]["open_browser"]
self.app__force_https = default_config["app"]["force_https"]
self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
self.app__csp_directives = default_config["app"]["csp_directives"]
self.app__api_base_url = default_config["app"]["api_base_url"]
self.app__web_base_url = default_config["app"]["web_base_url"]
self.authentication__type = default_config["authentication"]["type"]
self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
"oauth_api_base_url"
]
self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
"client_secret"
]
self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
"jwt_decode_options"
]
self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
"session_cookie"
]
self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
self.multi_dataset__index = default_config["multi_dataset"]["index"]
self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
"max_datasets"
]
self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
"timelimit_s"
]
self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
self.single_dataset__about = default_config["single_dataset"]["about"]
self.single_dataset__title = default_config["single_dataset"]["title"]
self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
self.limits__column_request_max = default_config["limits"]["column_request_max"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The matrix data cache manager is created during the complete_config and stored here.
self.matrix_data_cache_manager = None
# The authentication object
self.auth = None
def complete_config(self, context):
self.handle_app(context)
self.handle_data_source()
self.handle_authentication()
self.handle_data_locator()
self.handle_adaptor() # may depend on data_locator
self.handle_single_dataset(context) # may depend on adaptor
self.handle_multi_dataset() # may depend on adaptor
self.handle_diffexp()
self.handle_limits()
self.check_config()
def handle_app(self, context):
self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
self.validate_correct_type_of_configuration_attribute("app__debug", bool)
self.validate_correct_type_of_configuration_attribute("app__host", str)
self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
if self.app__port:
try:
if not is_port_available(self.app__host, self.app__port):
raise ConfigurationError(
f"The port selected {self.app__port} is in use, please configure an open port."
)
except OverflowError:
raise ConfigurationError(f"Invalid port: {self.app__port}")
else:
try:
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
except ValueError:
raise ConfigurationError(
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
)
try:
self.app__port = find_available_port(self.app__host, default_server_port)
except OverflowError:
raise ConfigurationError(f"Invalid port: {default_server_port}")
if self.app__debug:
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
self.app__verbose = True
self.app__open_browser = False
else:
warnings.formatwarning = custom_format_warning
if not self.app__verbose:
sys.tracebacklimit = 0
# CSP Directives are a dict of string: list(string) or string: string
if self.app__csp_directives is not None:
for k, v in self.app__csp_directives.items():
if not isinstance(k, str):
raise ConfigurationError("CSP directive names must be a string.")
if isinstance(v, list):
for policy in v:
if not isinstance(policy, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
elif not isinstance(v, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
if self.app__web_base_url is None:
self.app__web_base_url = self.app__api_base_url
def handle_authentication(self):
self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
# oauth
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__oauth_api_base_url", ptypes
)
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__jwt_decode_options", (type(None), dict)
)
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
if self.authentication__params_oauth__session_cookie:
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__cookie", (type(None), dict)
)
else:
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
self.auth = AuthTypeFactory.create(self.authentication__type, self)
if self.auth is None:
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
def handle_data_locator(self):
self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
if self.data_locator__s3__region_name is True:
path = self.single_dataset__datapath or self.multi_dataset__dataroot
if type(path) == dict:
# if multi_dataset__dataroot is a dict, then use the first key
# that is in s3. NOTE: it is not supported to have dataroots
# in different regions.
paths = [val.get("dataroot") for val in path.values()]
for path in paths:
if path.startswith("s3://"):
break
if path.startswith("s3://"):
region_name = discover_s3_region_name(path)
if region_name is None:
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
else:
region_name = None
self.data_locator__s3__region_name = region_name
def handle_data_source(self):
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
if self.single_dataset__datapath and self.multi_dataset__dataroot:
raise ConfigurationError(
"You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
)
if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
def handle_single_dataset(self, context):
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
if self.single_dataset__datapath is None:
return
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
def handle_multi_dataset(self):
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
self.validate_correct_type_of_configuration_attribute(
"multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
)
if self.multi_dataset__dataroot is None:
return
if type(self.multi_dataset__dataroot) == str:
default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
self.multi_dataset__dataroot = dict(d=default_dict)
for tag, dataroot_dict in self.multi_dataset__dataroot.items():
if "base_url" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
if "dataroot" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
base_url = dataroot_dict["base_url"]
# sanity check for well formed base urls
bad = False
if type(base_url) != str:
bad = True
elif os.path.normpath(base_url) != base_url:
bad = True
else:
base_url_parts = base_url.split("/")
if [quote_plus(part) for part in base_url_parts] != base_url_parts:
bad = True
if ".." in base_url_parts:
bad = True
if bad:
raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
# verify all the base_urls are unique
base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
if len(base_urls) > len(set(base_urls)):
raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
# error checking
for mtype in self.multi_dataset__allowed_matrix_types:
try:
MatrixDataType(mtype)
except ValueError:
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(
max_cached=self.multi_dataset__matrix_cache__max_datasets,
timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
)
def handle_diffexp(self):
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
max_workers = self.diffexp__alg_cxg__max_workers
cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
cpu_count = os.cpu_count()
max_workers = min(max_workers, cpu_multiplier * cpu_count)
diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
def handle_adaptor(self):
# cxg
self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
regionkey = "vfs.s3.region"
if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
if type(self.data_locator__s3__region_name) == str:
self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
from server.data_cxg.cxg_adaptor import CxgAdaptor
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
# anndata
self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
def handle_limits(self):
self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
def exceeds_limit(self, limit_name, value):
limit_value = getattr(self, "limits__" + limit_name, None)
if limit_value is None: # disabled
return False
return value > limit_value
def get_api_base_url(self):
if self.app__api_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__api_base_url and self.app__api_base_url.endswith("/"):
return self.app__api_base_url[:-1]
return self.app__api_base_url
def get_web_base_url(self):
if self.app__web_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__web_base_url is None:
return self.get_api_base_url()
if self.app__web_base_url.endswith("/"):
return self.app__web_base_url[:-1]
return self.app__web_base_url
server/common/errors.py
+4 -4
View File
@@ -42,14 +42,14 @@ define_request_exception(
define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
define_request_exception(
"AuthenticationError",
"Raised when there is an authentication error",
default_status_code=HTTPStatus.UNAUTHORIZED)
"AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
)
define_request_exception(
"AnnotationCategoryNameError",
"Raised when an annotation category name cant be saved",
default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY)
default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
)
define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
define_exception("ConfigurationError", "Raised when checking configuration errors")
server/common/rest.py
+10 -4
View File
@@ -2,10 +2,12 @@ import copy
import logging
import sys
from http import HTTPStatus
import zlib
from flask import make_response, jsonify, current_app, abort
from werkzeug.urls import url_unquote
from server.common.config.client_config import get_client_config, get_client_userinfo
from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
from server.common.errors import (
FilterError,
@@ -117,12 +119,12 @@ def schema_get(data_adaptor):
def config_get(app_config, data_adaptor):
config = app_config.get_client_config(data_adaptor)
config = get_client_config(app_config, data_adaptor)
return make_response(jsonify(config), HTTPStatus.OK)
def userinfo_get(app_config, data_adaptor):
config = app_config.get_client_userinfo(data_adaptor)
config = get_client_userinfo(app_config, data_adaptor)
return make_response(jsonify(config), HTTPStatus.OK)
@@ -154,17 +156,21 @@ def annotations_put_fbs_helper(data_adaptor, fbs):
new_label_df = decode_matrix_fbs(fbs)
if not new_label_df.empty:
data_adaptor.check_new_labels(new_label_df)
new_label_df = data_adaptor.check_new_labels(new_label_df)
annotations.write_labels(new_label_df, data_adaptor)
def inflate(data):
return zlib.decompress(data)
def annotations_obs_put(request, data_adaptor):
annotations = data_adaptor.dataset_config.user_annotations
if annotations is None:
return abort(HTTPStatus.NOT_IMPLEMENTED)
anno_collection = request.args.get("annotation-collection-name", default=None)
fbs = request.get_data()
fbs = inflate(request.get_data())
if anno_collection is not None:
if not annotations.is_safe_collection_name(anno_collection):
server/common/utils/cxg_generation_utils.py
+1 -1
View File
@@ -111,7 +111,7 @@ def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
def convert_matrix_to_cxg_array(
matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
):
"""
Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`

Some files were not shown because too many files have changed in this diff Show More