mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-26 16:38:11 +08:00
Compare commits
15
Commits
1.1.0
..
visium-beta
| Author | SHA1 | Date | |
|---|---|---|---|
 Emanuele Bezzi
|
11690eb745 | ||
|
Emanuele Bezzi
|
eac84c5d74 | ||
|
Emanuele Bezzi
|
cbf3ba240a | ||
|
Emanuele Bezzi
|
2fe9cc4aac | ||
|
Emanuele Bezzi
|
8b07e57257 | ||
|
Colin Megill
|
f4c4ac5bda | ||
|
Emanuele Bezzi
|
febf582a0b | ||
|
Emanuele Bezzi
|
efe3bf7a72 | ||
|
Colin Megill
|
b411fca5a3 | ||
|
Colin Megill
|
caa1526eb6 | ||
|
Colin Megill
|
99c8f37a60 | ||
|
Emanuele Bezzi
|
b048bbfd0c | ||
|
Emanuele Bezzi
|
b1ff638879 | ||
|
Emanuele Bezzi
|
db0f50d011 | ||
|
Emanuele Bezzi
|
54d4de431c |
+1
-1
@@ -1,5 +1,5 @@
|
||||
[bumpversion]
|
||||
current_version = 1.0.1
|
||||
current_version = 1.0.0
|
||||
commit = True
|
||||
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
|
||||
serialize =
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Build docker image
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Cache env vars
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
- run: |
|
||||
git fetch --depth=1 origin +${{github.base_ref}}
|
||||
- name: Set up Python 3.7
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.7
|
||||
- name: Node cache
|
||||
@@ -46,12 +46,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.7 (pyenv) # pyenv needed for mlflow in cli annotate tests
|
||||
uses: gabrielfalcao/pyenv-action@v9
|
||||
- name: Set up Python 3.7
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
default: 3.7
|
||||
command: pip install -U pip # upgrade pip after installing python
|
||||
- run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
|
||||
python-version: 3.7
|
||||
- name: Python cache
|
||||
uses: actions/cache@v1
|
||||
with:
|
||||
@@ -80,7 +78,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.7
|
||||
uses: actions/setup-python@v4
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.7
|
||||
- name: Python cache
|
||||
@@ -104,33 +102,32 @@ jobs:
|
||||
cd client && make smoke-test
|
||||
./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
|
||||
|
||||
# TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
|
||||
# smoke-tests-annotations:
|
||||
# runs-on: ubuntu-latest
|
||||
# timeout-minutes: 20
|
||||
# steps:
|
||||
# - uses: actions/checkout@v2
|
||||
# - name: Set up Python 3.7
|
||||
# uses: actions/setup-python@v4
|
||||
# with:
|
||||
# python-version: 3.7
|
||||
# - name: Python cache
|
||||
# uses: actions/cache@v1
|
||||
# with:
|
||||
# path: ~/.cache/pip
|
||||
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
||||
# restore-keys: |
|
||||
# ${{ runner.os }}-pip-
|
||||
# - name: Node cache
|
||||
# uses: actions/cache@v1
|
||||
# with:
|
||||
# path: ~/.npm
|
||||
# key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||
# restore-keys: |
|
||||
# ${{ runner.os }}-node-
|
||||
# - name: Install dependencies
|
||||
# run: make pydist install-dist
|
||||
# - name: Smoke tests (with annotations feature)
|
||||
# run: |
|
||||
# cd client && make smoke-test-annotations
|
||||
# ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
|
||||
smoke-tests-annotations:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.7
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: 3.7
|
||||
- name: Python cache
|
||||
uses: actions/cache@v1
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-
|
||||
- name: Node cache
|
||||
uses: actions/cache@v1
|
||||
with:
|
||||
path: ~/.npm
|
||||
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-node-
|
||||
- name: Install dependencies
|
||||
run: make pydist install-dist
|
||||
- name: Smoke tests (with annotations feature)
|
||||
run: |
|
||||
cd client && make smoke-test-annotations
|
||||
./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
|
||||
|
||||
@@ -54,6 +54,3 @@ client/.eslintcache
|
||||
|
||||
# E2E Testing
|
||||
ignoreE2E*
|
||||
|
||||
# annotate subcmd
|
||||
.models_cache
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2017-2022 Chan Zuckerberg Initiative
|
||||
Copyright (c) 2017-2021 Chan Zuckerberg Initiative
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
|
||||
@@ -3,6 +3,5 @@ recursive-include server/common/web/static *
|
||||
|
||||
include server/requirements.txt
|
||||
include server/requirements-prepare.txt
|
||||
include server/requirements-annotate.txt
|
||||
include server/converters/schema/hgnc_complete_set.txt.gz
|
||||
include server/converters/schema/schema_definitions/*
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Cellxgene Visium Beta
|
||||
|
||||
## How it works
|
||||
1. Launch `cellxgene` as normal.
|
||||
1. If the loaded dataset has spatial information available, the image data will be loaded on startup.
|
||||
1. On the toolbar, next to the Zoom icon, a `Toggle image` button will now appear. Click on it and the image will be added as an underlay.
|
||||
1. You can now use any `cellxgene` functionality and the image will still be present. If you pan and zoom, the image will also be panned and zoomed.
|
||||
1. If you want to hide the image, you can click on `Toggle image` again
|
||||
|
||||
In order for the image to be displayed with the correct size and alignment, the H5AD needs to have a few requirements. See the following section to learn more.
|
||||
|
||||
## h5ad requirements
|
||||
1. The spatial embedding layer should be contained in `obsm` and be named `X_spatial`. Other layers can exist, but only this one will have the spatial feature enabled.
|
||||
2. A `spatial` dict needs to be defined in the `uns` dictionary.
|
||||
3. Inside the `spatial` dict, an `images` dict must be defined.
|
||||
4. The `images` dict must contain a `hires` key, which should reference an image encoded as an RGB matrix (i.e., a three-dimensional matrix of size `height x width x 3` where the final dimension has the RGB values for each pixel)
|
||||
5. The `images` dict must contain a `scalefactors` dict. This should in turn contain a `tissue_hires_scalef` key, which should reference a floating point number.
|
||||
|
||||
Moreover, in order to have the image correctly aligned with the dots, the following must be true:
|
||||
1. `tissue_hires_scalef` should represent the ratio between the embedding layer `X_spatial` and the image matrix. In particular, if you multiply `X_spatial` by `tissue_hires_scalef`, you should obtain an array of points that ovelap the tissue image if you plot them in a plane.
|
||||
@@ -2,4 +2,4 @@
|
||||
|
||||
exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
|
||||
|
||||
exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" aria-hidden=\\"true\\" tabindex=\\"0\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;
|
||||
exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,6 +1,8 @@
|
||||
const path = require("path");
|
||||
const webpack = require("webpack");
|
||||
const HtmlWebpackPlugin = require("html-webpack-plugin");
|
||||
const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
|
||||
const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
|
||||
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
|
||||
|
||||
const { merge } = require("webpack-merge");
|
||||
@@ -42,6 +44,21 @@ const devConfig = {
|
||||
inject: true,
|
||||
template: path.resolve("index.html"),
|
||||
}),
|
||||
new FaviconsWebpackPlugin({
|
||||
logo: "./favicon.png",
|
||||
prefix: "static/img/",
|
||||
favicons: {
|
||||
icons: {
|
||||
android: false,
|
||||
appleIcon: false,
|
||||
appleStartup: false,
|
||||
coast: false,
|
||||
firefox: false,
|
||||
windows: false,
|
||||
yandex: false,
|
||||
},
|
||||
},
|
||||
}),
|
||||
new MiniCssExtractPlugin({
|
||||
filename: "static/[name].css",
|
||||
}),
|
||||
@@ -56,6 +73,9 @@ const devConfig = {
|
||||
CXG_SERVER_PORT: process.env.CXG_SERVER_PORT || "5005",
|
||||
}),
|
||||
}),
|
||||
new ScriptExtHtmlWebpackPlugin({
|
||||
async: "obsolete",
|
||||
}),
|
||||
],
|
||||
infrastructureLogging: {
|
||||
level: "warn",
|
||||
|
||||
@@ -3,7 +3,9 @@ const webpack = require("webpack");
|
||||
const HtmlWebpackPlugin = require("html-webpack-plugin");
|
||||
const { CleanWebpackPlugin } = require("clean-webpack-plugin");
|
||||
const TerserJSPlugin = require("terser-webpack-plugin");
|
||||
const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
|
||||
const CleanCss = require("clean-css");
|
||||
const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
|
||||
const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
|
||||
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
|
||||
|
||||
const { merge } = require("webpack-merge");
|
||||
@@ -27,8 +29,8 @@ const prodConfig = {
|
||||
minimize: true,
|
||||
minimizer: [
|
||||
new TerserJSPlugin({}),
|
||||
new CssMinimizerPlugin({
|
||||
minify: CssMinimizerPlugin.cleanCssMinify,
|
||||
new OptimizeCSSAssetsPlugin({
|
||||
cssProcessor: CleanCss,
|
||||
}),
|
||||
],
|
||||
},
|
||||
@@ -64,6 +66,21 @@ const prodConfig = {
|
||||
protectWebpackAssets: false,
|
||||
cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
|
||||
}),
|
||||
new FaviconsWebpackPlugin({
|
||||
logo: "./favicon.png",
|
||||
prefix: "static/assets/",
|
||||
favicons: {
|
||||
icons: {
|
||||
android: false,
|
||||
appleIcon: false,
|
||||
appleStartup: false,
|
||||
coast: false,
|
||||
firefox: false,
|
||||
windows: false,
|
||||
yandex: false,
|
||||
},
|
||||
},
|
||||
}),
|
||||
new MiniCssExtractPlugin({
|
||||
filename: "static/[name]-[contenthash].css",
|
||||
}),
|
||||
|
||||
@@ -2,6 +2,8 @@ const path = require("path");
|
||||
const fs = require("fs");
|
||||
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
|
||||
const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
|
||||
// eslint-disable-next-line @blueprintjs/classes-constants -- incorrect match
|
||||
const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
|
||||
|
||||
const src = path.resolve("src");
|
||||
const nodeModules = path.resolve("node_modules");
|
||||
@@ -65,5 +67,8 @@ module.exports = {
|
||||
template: obsoleteHTMLTemplate,
|
||||
promptOnNonTargetBrowser: false,
|
||||
}),
|
||||
new ScriptExtHtmlWebpackPlugin({
|
||||
async: "obsolete",
|
||||
}),
|
||||
],
|
||||
};
|
||||
|
||||
Generated
+19041
-7139
File diff suppressed because it is too large
Load Diff
+5
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "cellxgene",
|
||||
"version": "1.0.1",
|
||||
"version": "1.0.0",
|
||||
"license": "MIT",
|
||||
"description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
|
||||
"repository": "https://github.com/chanzuckerberg/cellxgene",
|
||||
@@ -101,7 +101,6 @@
|
||||
"clean-webpack-plugin": "^4.0.0-alpha.0",
|
||||
"codecov": "^3.7.1",
|
||||
"css-loader": "^5.2.4",
|
||||
"css-minimizer-webpack-plugin": "^4.0.0",
|
||||
"eslint": "^7.24.0",
|
||||
"eslint-config-airbnb": "^18.2.0",
|
||||
"eslint-config-prettier": "^8.2.0",
|
||||
@@ -115,6 +114,8 @@
|
||||
"eslint-plugin-react-hooks": "^4.0.8",
|
||||
"expect-puppeteer": "^5.0.0",
|
||||
"express": "^4.17.1",
|
||||
"favicons": "^6.2.2",
|
||||
"favicons-webpack-plugin": "^5.0.2",
|
||||
"file-loader": "^6.0.0",
|
||||
"html-webpack-plugin": "^5.3.1",
|
||||
"husky": "^7.0.2",
|
||||
@@ -133,9 +134,11 @@
|
||||
"lodash.zip": "^4.2.0",
|
||||
"mini-css-extract-plugin": "^1.5.0",
|
||||
"obsolete-webpack-plugin": "^0.5.6",
|
||||
"optimize-css-assets-webpack-plugin": "^5.0.3",
|
||||
"prettier": "^2.0.5",
|
||||
"puppeteer": "^8.0.0",
|
||||
"rimraf": "^3.0.2",
|
||||
"script-ext-html-webpack-plugin": "^2.1.4",
|
||||
"serve-favicon": "^2.5.0",
|
||||
"terser-webpack-plugin": "^5.1.1",
|
||||
"webpack": "^5.34.0",
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
|
||||
import * as selnActions from "./selection";
|
||||
import * as annoActions from "./annotation";
|
||||
import * as spatialActions from "./spatial";
|
||||
import * as viewActions from "./viewStack";
|
||||
import * as embActions from "./embedding";
|
||||
import * as genesetActions from "./geneset";
|
||||
@@ -272,4 +273,5 @@ export default {
|
||||
genesetDelete: genesetActions.genesetDelete,
|
||||
genesetAddGenes: genesetActions.genesetAddGenes,
|
||||
genesetDeleteGenes: genesetActions.genesetDeleteGenes,
|
||||
requestSpatialMetadata: spatialActions.requestSpatialMetadata,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import * as globals from "../globals";
|
||||
|
||||
export const requestSpatialMetadata = () => async (dispatch) => {
|
||||
dispatch({ type: "request spatial metadata started" });
|
||||
try {
|
||||
const res = await fetch(
|
||||
`${globals.API.prefix}${globals.API.version}spatial/meta`,
|
||||
{
|
||||
method: "GET",
|
||||
headers: new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
}),
|
||||
credentials: "include",
|
||||
}
|
||||
);
|
||||
|
||||
if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
|
||||
return null; // TODO need a dispatch //dispatchDiffExpErrors(dispatch, res);
|
||||
}
|
||||
|
||||
const response = await res.json();
|
||||
|
||||
/* then send the success case action through */
|
||||
return dispatch({
|
||||
type: "request spatial metadata success",
|
||||
data: response,
|
||||
});
|
||||
} catch (error) {
|
||||
return dispatch({
|
||||
type: "request spatial metadata error",
|
||||
error,
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -58,11 +58,10 @@ function _maskToList(mask) {
|
||||
if (!mask) {
|
||||
return null;
|
||||
}
|
||||
const [...m] = mask;
|
||||
const list = new Int32Array(m.length);
|
||||
const list = new Int32Array(mask.length);
|
||||
let elems = 0;
|
||||
for (let i = 0, l = m.length; i < l; i += 1) {
|
||||
if (m[i]) {
|
||||
for (let i = 0, l = mask.length; i < l; i += 1) {
|
||||
if (mask[i]) {
|
||||
list[elems] = i;
|
||||
elems += 1;
|
||||
}
|
||||
|
||||
@@ -91,9 +91,12 @@ export function _whereCacheCreate(field, query, columnLabels) {
|
||||
*/
|
||||
if (typeof query !== "object") return null;
|
||||
|
||||
const { where, summarize } = query;
|
||||
if (where) {
|
||||
const { field: queryField, column: queryColumn, value: queryValue } = where;
|
||||
if (query.where) {
|
||||
const {
|
||||
field: queryField,
|
||||
column: queryColumn,
|
||||
value: queryValue,
|
||||
} = query.where;
|
||||
return {
|
||||
where: {
|
||||
[field]: {
|
||||
@@ -104,13 +107,13 @@ export function _whereCacheCreate(field, query, columnLabels) {
|
||||
},
|
||||
};
|
||||
}
|
||||
if (summarize) {
|
||||
if (query.summarize) {
|
||||
const {
|
||||
method,
|
||||
field: queryField,
|
||||
column: queryColumn,
|
||||
values: queryValues,
|
||||
} = summarize;
|
||||
} = query.summarize;
|
||||
const queryValueHash = _hashStringValues(queryValues);
|
||||
return {
|
||||
summarize: {
|
||||
|
||||
@@ -23,6 +23,8 @@ class App extends React.Component {
|
||||
componentDidMount() {
|
||||
const { dispatch } = this.props;
|
||||
|
||||
dispatch(actions.requestSpatialMetadata());
|
||||
|
||||
/* listen for url changes, fire one when we start the app up */
|
||||
window.addEventListener("popstate", this._onURLChanged);
|
||||
this._onURLChanged();
|
||||
|
||||
@@ -16,10 +16,11 @@ import actions from "../../actions";
|
||||
import { getDiscreteCellEmbeddingRowIndex } from "../../util/stateManager/viewStackHelpers";
|
||||
|
||||
@connect((state) => ({
|
||||
layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
|
||||
schema: state.annoMatrix?.schema,
|
||||
crossfilter: state.obsCrossfilter,
|
||||
}))
|
||||
imageUnderlay: state.imageUnderlay,
|
||||
layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
|
||||
schema: state.annoMatrix?.schema,
|
||||
crossfilter: state.obsCrossfilter,
|
||||
}))
|
||||
class Embedding extends React.PureComponent {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
@@ -27,8 +28,18 @@ class Embedding extends React.PureComponent {
|
||||
}
|
||||
|
||||
handleLayoutChoiceChange = (e) => {
|
||||
const { dispatch } = this.props;
|
||||
const { dispatch, imageUnderlay } = this.props;
|
||||
dispatch(actions.layoutChoiceAction(e.currentTarget.value));
|
||||
|
||||
// if we just switched off spatial, if the image is on, turn it off
|
||||
if (
|
||||
imageUnderlay.isActive &&
|
||||
e.target.value !== globals.spatialEmbeddingKeyword
|
||||
) {
|
||||
dispatch({
|
||||
type: "toggle image underlay",
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
render() {
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
export default function drawSpatialImageRegl(regl) {
|
||||
return regl({
|
||||
frag: `
|
||||
precision mediump float;
|
||||
|
||||
// our texture
|
||||
uniform sampler2D u_image;
|
||||
|
||||
// the texCoords passed in from the vertex shader.
|
||||
varying vec2 v_texCoord;
|
||||
|
||||
void main() {
|
||||
gl_FragColor = texture2D(u_image, v_texCoord);
|
||||
}`,
|
||||
|
||||
vert: `
|
||||
attribute vec2 a_position;
|
||||
attribute vec2 a_texCoord;
|
||||
|
||||
uniform vec2 u_resolution;
|
||||
|
||||
uniform mat3 projView;
|
||||
|
||||
varying vec2 v_texCoord;
|
||||
|
||||
void main() {
|
||||
// convert the rectangle from pixels to 0.0 to 1.0
|
||||
vec3 pos = vec3(a_position, 1.);
|
||||
vec2 zeroToOne = pos.xy / u_resolution;
|
||||
|
||||
// convert from 0->1 to 0->2
|
||||
vec2 zeroToTwo = zeroToOne * 2.0;
|
||||
|
||||
// convert from 0->2 to -1->+1 (clipspace)
|
||||
vec2 clipSpace = zeroToTwo - 1.0;
|
||||
|
||||
vec3 pos2 = projView * vec3(clipSpace, 1.);
|
||||
|
||||
gl_Position = vec4(pos2.xy , 0, 1);
|
||||
|
||||
// pass the texCoord to the fragment shader
|
||||
// The GPU will interpolate this value between points.
|
||||
v_texCoord = a_texCoord;
|
||||
}`,
|
||||
|
||||
attributes: {
|
||||
a_texCoord: [0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0],
|
||||
a_position: regl.prop("rectCoords"),
|
||||
},
|
||||
|
||||
uniforms: {
|
||||
projView: regl.prop("projView"),
|
||||
u_image: regl.prop("spatialImageAsTexture"),
|
||||
color: [1, 0, 0, 1],
|
||||
u_resolution: [regl.prop("imageWidth"), regl.prop("imageHeight")],
|
||||
image_width: regl.prop("imageWidth"),
|
||||
// translate:
|
||||
},
|
||||
|
||||
count: 6,
|
||||
});
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
createColorTable,
|
||||
createColorQuery,
|
||||
} from "../../util/stateManager/colorHelpers";
|
||||
import _drawSpatialImage from "./drawSpatialImageRegl";
|
||||
import * as globals from "../../globals";
|
||||
|
||||
import GraphOverlayLayer from "./overlays/graphOverlayLayer";
|
||||
@@ -77,6 +78,8 @@ function createModelTF() {
|
||||
colors: state.colors,
|
||||
pointDilation: state.pointDilation,
|
||||
genesets: state.genesets.genesets,
|
||||
spatial: state.spatial.metadata,
|
||||
imageUnderlay: state.imageUnderlay,
|
||||
}))
|
||||
class Graph extends React.Component {
|
||||
static createReglState(canvas) {
|
||||
@@ -87,6 +90,7 @@ class Graph extends React.Component {
|
||||
const camera = _camera(canvas);
|
||||
const regl = _regl(canvas);
|
||||
const drawPoints = _drawPoints(regl);
|
||||
const drawSpatialImage = _drawSpatialImage(regl);
|
||||
|
||||
// preallocate webgl buffers
|
||||
const pointBuffer = regl.buffer();
|
||||
@@ -100,6 +104,7 @@ class Graph extends React.Component {
|
||||
pointBuffer,
|
||||
colorBuffer,
|
||||
flagBuffer,
|
||||
drawSpatialImage,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -232,6 +237,8 @@ class Graph extends React.Component {
|
||||
pointBuffer: null,
|
||||
colorBuffer: null,
|
||||
flagBuffer: null,
|
||||
drawSpatialImage: null,
|
||||
spatial: null,
|
||||
|
||||
// component rendering derived state - these must stay synchronized
|
||||
// with the reducer state they were generated from.
|
||||
@@ -317,7 +324,10 @@ class Graph extends React.Component {
|
||||
if (e.type !== "wheel") e.preventDefault();
|
||||
if (camera.handleEvent(e, projectionTF)) {
|
||||
this.renderCanvas();
|
||||
this.setState((state) => ({ ...state, updateOverlay: !state.updateOverlay }));
|
||||
this.setState((state) => ({
|
||||
...state,
|
||||
updateOverlay: !state.updateOverlay,
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
@@ -509,6 +519,14 @@ class Graph extends React.Component {
|
||||
return { toolSVG: newToolSVG, tool, container };
|
||||
};
|
||||
|
||||
loadTextureFromUrl = (src) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
img.onload = () => resolve(img);
|
||||
img.onerror = reject;
|
||||
img.src = src;
|
||||
});
|
||||
|
||||
fetchAsyncProps = async (props) => {
|
||||
const {
|
||||
annoMatrix,
|
||||
@@ -517,6 +535,8 @@ class Graph extends React.Component {
|
||||
crossfilter,
|
||||
pointDilation,
|
||||
viewport,
|
||||
spatial,
|
||||
imageUnderlay,
|
||||
} = props.watchProps;
|
||||
const { modelTF } = this.state;
|
||||
|
||||
@@ -524,7 +544,8 @@ class Graph extends React.Component {
|
||||
annoMatrix,
|
||||
layoutChoice,
|
||||
colorsProp,
|
||||
pointDilation
|
||||
pointDilation,
|
||||
imageUnderlay
|
||||
);
|
||||
|
||||
const { currentDimNames } = layoutChoice;
|
||||
@@ -551,6 +572,10 @@ class Graph extends React.Component {
|
||||
pointDilationLabel
|
||||
);
|
||||
|
||||
this.spatialImage = await this.loadTextureFromUrl(
|
||||
"/api/v0.2/spatial/image"
|
||||
);
|
||||
|
||||
const { width, height } = viewport;
|
||||
return {
|
||||
positions,
|
||||
@@ -558,6 +583,8 @@ class Graph extends React.Component {
|
||||
flags,
|
||||
width,
|
||||
height,
|
||||
spatial,
|
||||
imageUnderlay,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -721,6 +748,7 @@ class Graph extends React.Component {
|
||||
flagBuffer,
|
||||
camera,
|
||||
projectionTF,
|
||||
drawSpatialImage,
|
||||
} = this.state;
|
||||
this.renderPoints(
|
||||
regl,
|
||||
@@ -729,12 +757,14 @@ class Graph extends React.Component {
|
||||
pointBuffer,
|
||||
flagBuffer,
|
||||
camera,
|
||||
projectionTF
|
||||
projectionTF,
|
||||
drawSpatialImage
|
||||
);
|
||||
});
|
||||
|
||||
updateReglAndRender(asyncProps, prevAsyncProps) {
|
||||
const { positions, colors, flags, height, width } = asyncProps;
|
||||
const { positions, colors, flags, height, width, imageUnderlay } =
|
||||
asyncProps;
|
||||
this.cachedAsyncProps = asyncProps;
|
||||
const { pointBuffer, colorBuffer, flagBuffer } = this.state;
|
||||
let needToRenderCanvas = false;
|
||||
@@ -754,6 +784,9 @@ class Graph extends React.Component {
|
||||
flagBuffer({ data: flags, dimension: 1 });
|
||||
needToRenderCanvas = true;
|
||||
}
|
||||
if (imageUnderlay !== prevAsyncProps?.imageUnderlay) {
|
||||
needToRenderCanvas = true;
|
||||
}
|
||||
if (needToRenderCanvas) this.renderCanvas();
|
||||
}
|
||||
|
||||
@@ -797,20 +830,25 @@ class Graph extends React.Component {
|
||||
pointBuffer,
|
||||
flagBuffer,
|
||||
camera,
|
||||
projectionTF
|
||||
projectionTF,
|
||||
drawSpatialImage
|
||||
) {
|
||||
const { annoMatrix } = this.props;
|
||||
const { annoMatrix, spatial, imageUnderlay } = this.props;
|
||||
if (!this.reglCanvas || !annoMatrix) return;
|
||||
|
||||
const { schema } = annoMatrix;
|
||||
const cameraTF = camera.view();
|
||||
const projView = mat3.multiply(mat3.create(), projectionTF, cameraTF);
|
||||
const { width, height } = this.reglCanvas;
|
||||
const imW = spatial.data.imageWidth;
|
||||
const imH = spatial.data.imageHeight;
|
||||
|
||||
regl.poll();
|
||||
regl.clear({
|
||||
depth: 1,
|
||||
color: [1, 1, 1, 1],
|
||||
color: [0, 0, 0, 0],
|
||||
});
|
||||
|
||||
drawPoints({
|
||||
distance: camera.distance(),
|
||||
color: colorBuffer,
|
||||
@@ -821,6 +859,19 @@ class Graph extends React.Component {
|
||||
nPoints: schema.dataframe.nObs,
|
||||
minViewportDimension: Math.min(width, height),
|
||||
});
|
||||
if (imageUnderlay?.isActive) {
|
||||
drawSpatialImage({
|
||||
projView,
|
||||
imageWidth: imW,
|
||||
imageHeight: imH,
|
||||
rectCoords: [0, 0, imW, 0, 0, imH, 0, imH, imW, 0, imW, imH],
|
||||
spatialImageAsTexture: regl.texture({
|
||||
data: this.spatialImage,
|
||||
wrapS: "clamp",
|
||||
wrapT: "clamp",
|
||||
}),
|
||||
});
|
||||
}
|
||||
regl._gl.flush();
|
||||
}
|
||||
|
||||
@@ -832,6 +883,8 @@ class Graph extends React.Component {
|
||||
layoutChoice,
|
||||
pointDilation,
|
||||
crossfilter,
|
||||
spatial,
|
||||
imageUnderlay,
|
||||
} = this.props;
|
||||
const { modelTF, projectionTF, camera, viewport, regl } = this.state;
|
||||
const cameraTF = camera?.view()?.slice();
|
||||
@@ -902,6 +955,8 @@ class Graph extends React.Component {
|
||||
pointDilation,
|
||||
crossfilter,
|
||||
viewport,
|
||||
spatial,
|
||||
imageUnderlay,
|
||||
}}
|
||||
>
|
||||
<Async.Pending initial>
|
||||
@@ -951,32 +1006,29 @@ const ErrorLoading = ({ displayName, error, width, height }) => {
|
||||
);
|
||||
};
|
||||
|
||||
const StillLoading = ({ displayName, width, height }) =>
|
||||
const StillLoading = ({ displayName, width, height }) => (
|
||||
/*
|
||||
Render a busy/loading indicator
|
||||
*/
|
||||
(
|
||||
<div
|
||||
style={{
|
||||
position: "fixed",
|
||||
fontWeight: 500,
|
||||
top: height / 2,
|
||||
width,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
position: "fixed",
|
||||
fontWeight: 500,
|
||||
top: height / 2,
|
||||
width,
|
||||
display: "flex",
|
||||
justifyContent: "center",
|
||||
justifyItems: "center",
|
||||
alignItems: "center",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
justifyContent: "center",
|
||||
justifyItems: "center",
|
||||
alignItems: "center",
|
||||
}}
|
||||
>
|
||||
<Button minimal loading intent="primary" />
|
||||
<span style={{ fontStyle: "italic" }}>Loading {displayName}</span>
|
||||
</div>
|
||||
<Button minimal loading intent="primary" />
|
||||
<span style={{ fontStyle: "italic" }}>Loading {displayName}</span>
|
||||
</div>
|
||||
)
|
||||
;
|
||||
|
||||
</div>
|
||||
);
|
||||
export default Graph;
|
||||
|
||||
@@ -28,6 +28,8 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
|
||||
subsetPossible,
|
||||
subsetResetPossible,
|
||||
graphInteractionMode: state.controls.graphInteractionMode,
|
||||
imageUnderlay: state.imageUnderlay,
|
||||
layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
|
||||
clipPercentileMin: Math.round(100 * (annoMatrix?.clipRange?.[0] ?? 0)),
|
||||
clipPercentileMax: Math.round(100 * (annoMatrix?.clipRange?.[1] ?? 1)),
|
||||
userDefinedGenes: state.controls.userDefinedGenes,
|
||||
@@ -206,6 +208,8 @@ class MenuBar extends React.PureComponent {
|
||||
colorAccessor,
|
||||
subsetPossible,
|
||||
subsetResetPossible,
|
||||
imageUnderlay,
|
||||
layoutChoice,
|
||||
} = this.props;
|
||||
const { pendingClipPercentiles } = this.state;
|
||||
|
||||
@@ -268,6 +272,29 @@ class MenuBar extends React.PureComponent {
|
||||
disabled={!isColoredByCategorical}
|
||||
/>
|
||||
</Tooltip>
|
||||
{layoutChoice?.available?.includes(globals.spatialEmbeddingKeyword) && (
|
||||
<ButtonGroup className={styles.menubarButton}>
|
||||
<Tooltip
|
||||
content={"Toggle image"}
|
||||
position="bottom"
|
||||
hoverOpenDelay={globals.tooltipHoverOpenDelay}
|
||||
>
|
||||
<AnchorButton
|
||||
type="button"
|
||||
data-testid="toggle-image-underlay"
|
||||
icon={"media"}
|
||||
intent={imageUnderlay.isActive ? "primary" : "none"}
|
||||
active={imageUnderlay.isActive}
|
||||
onClick={() => {
|
||||
dispatch({
|
||||
type: "toggle image underlay",
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
</ButtonGroup>
|
||||
)}
|
||||
|
||||
<ButtonGroup className={styles.menubarButton}>
|
||||
<Tooltip
|
||||
content={selectionTooltip}
|
||||
|
||||
@@ -2,6 +2,9 @@ import { Colors } from "@blueprintjs/core";
|
||||
import { dispatchNetworkErrorMessageToUser } from "./util/actionHelpers";
|
||||
import ENV_DEFAULT from "../../environment.default.json";
|
||||
|
||||
// visium embedding word, spatial image underlay
|
||||
export const spatialEmbeddingKeyword = "spatial";
|
||||
|
||||
/* overflow category values are created using this string */
|
||||
export const overflowCategoryLabel = ": all other labels";
|
||||
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
const imageUnderlay = (state = { isActive: false }, action) => {
|
||||
switch (action.type) {
|
||||
case "toggle image underlay":
|
||||
return {
|
||||
...state,
|
||||
isActive: !state.isActive,
|
||||
};
|
||||
|
||||
default:
|
||||
return state;
|
||||
}
|
||||
};
|
||||
|
||||
export default imageUnderlay;
|
||||
@@ -11,6 +11,7 @@ import continuousSelection from "./continuousSelection";
|
||||
import graphSelection from "./graphSelection";
|
||||
import colors from "./colors";
|
||||
import differential from "./differential";
|
||||
import spatial from "./spatial";
|
||||
import layoutChoice from "./layoutChoice";
|
||||
import controls from "./controls";
|
||||
import annotations from "./annotations";
|
||||
@@ -19,6 +20,7 @@ import genesetsUI from "./genesetsUI";
|
||||
import autosave from "./autosave";
|
||||
import centroidLabels from "./centroidLabels";
|
||||
import pointDialation from "./pointDilation";
|
||||
import imageUnderlay from "./imageUnderlay";
|
||||
import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
|
||||
|
||||
import undoableConfig from "./undoableConfig";
|
||||
@@ -38,7 +40,9 @@ const Reducer = undoable(
|
||||
["colors", colors],
|
||||
["controls", controls],
|
||||
["differential", differential],
|
||||
["spatial", spatial],
|
||||
["centroidLabels", centroidLabels],
|
||||
["imageUnderlay", imageUnderlay],
|
||||
["pointDilation", pointDialation],
|
||||
["autosave", autosave],
|
||||
]),
|
||||
@@ -51,6 +55,7 @@ const Reducer = undoable(
|
||||
"colors",
|
||||
"controls",
|
||||
"differential",
|
||||
"spatial",
|
||||
"layoutChoice",
|
||||
"centroidLabels",
|
||||
"genesets",
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
const Spatial = (
|
||||
state = {
|
||||
loading: null,
|
||||
error: null,
|
||||
metadata: null,
|
||||
},
|
||||
action
|
||||
) => {
|
||||
switch (action.type) {
|
||||
case "request spatial metadata started":
|
||||
return {
|
||||
...state,
|
||||
loading: true,
|
||||
error: null,
|
||||
};
|
||||
case "request spatial metadata success":
|
||||
return {
|
||||
...state,
|
||||
error: null,
|
||||
loading: false,
|
||||
metadata: action,
|
||||
};
|
||||
case "request spatial metadata error":
|
||||
return {
|
||||
...state,
|
||||
loading: false,
|
||||
error: action.data,
|
||||
};
|
||||
default:
|
||||
return state;
|
||||
}
|
||||
};
|
||||
|
||||
export default Spatial;
|
||||
@@ -52,6 +52,9 @@ const skipOnActions = new Set([
|
||||
"geneset: disable add new genes mode",
|
||||
"geneset: activate rename geneset mode",
|
||||
"geneset: disable rename geneset mode",
|
||||
|
||||
/* spatial */
|
||||
"toggle image underlay",
|
||||
]);
|
||||
|
||||
/*
|
||||
|
||||
@@ -137,13 +137,10 @@ function _getEmbeddingRowOffsets(baseRowIndex, embeddingDf) {
|
||||
- if the embedding contains NaN coordinates, return a rowIndex
|
||||
that contains only the rows with discrete valued coordinates.
|
||||
|
||||
Currently assumes that there will be only two dimensions in the embedding.
|
||||
Currently assumes that there will be onl two dimensions in the embedding.
|
||||
*/
|
||||
// eslint-disable-next-line react/destructuring-assignment -- destructuring fails
|
||||
const X = embeddingDf.icol(0).asArray();
|
||||
// eslint-disable-next-line react/destructuring-assignment -- destructuring fails
|
||||
const Y = embeddingDf.icol(1).asArray();
|
||||
|
||||
const offsets = new Int32Array(X.length);
|
||||
let numOffsets = 0;
|
||||
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/usr/bin/expect -f
|
||||
|
||||
# Mac only! (depends upon `open` command)
|
||||
|
||||
set h5ad [lindex $argv 0]
|
||||
puts "$h5ad"
|
||||
|
||||
spawn cellxgene launch $h5ad
|
||||
|
||||
set timeout 10
|
||||
expect -indices -re "Please go to (http:\/\/localhost:\[0-9\]+)" {
|
||||
set url $expect_out(1,string)
|
||||
exec >@stdout 2>@stderr open $url
|
||||
}
|
||||
|
||||
interact
|
||||
+1
-1
@@ -2,7 +2,7 @@ import logging
|
||||
import sys
|
||||
from server.common.utils.utils import import_plugins
|
||||
|
||||
__version__ = "1.0.1"
|
||||
__version__ = "1.0.0"
|
||||
display_version = "cellxgene v" + __version__
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class AnnotationType(Enum):
|
||||
CELL_TYPE = "cell_type"
|
||||
@@ -190,6 +190,16 @@ class SummarizeVarAPI(Resource):
|
||||
def post(self, data_adaptor):
|
||||
return common_rest.summarize_var_post(request, data_adaptor)
|
||||
|
||||
class SpatialImageAPI(Resource):
|
||||
@rest_get_data_adaptor
|
||||
def get(self, data_adaptor):
|
||||
return common_rest.spatial_image_get(request, data_adaptor)
|
||||
|
||||
class SpatialMetaAPI(Resource):
|
||||
@rest_get_data_adaptor
|
||||
def get(self, data_adaptor):
|
||||
return data_adaptor.get_spatial_metadata()
|
||||
|
||||
|
||||
def get_api_base_resources(bp_base):
|
||||
"""Add resources that are accessed from the api url"""
|
||||
@@ -222,6 +232,9 @@ def get_api_dataroot_resources(bp_dataroot):
|
||||
# Computation routes
|
||||
add_resource(DiffExpObsAPI, "/diffexp/obs")
|
||||
add_resource(LayoutObsAPI, "/layout/obs")
|
||||
# Spatial routes
|
||||
add_resource(SpatialImageAPI, "/spatial/image")
|
||||
add_resource(SpatialMetaAPI, "/spatial/meta")
|
||||
return api
|
||||
|
||||
|
||||
|
||||
@@ -1,231 +0,0 @@
|
||||
import functools
|
||||
import json
|
||||
import os.path
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from subprocess import STDOUT, PIPE
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
import click
|
||||
import pandas as pd
|
||||
from click import BadParameter
|
||||
|
||||
from server.annotate.annotation_types import AnnotationType
|
||||
from server.common.utils.data_locator import DataLocator
|
||||
from server.common.utils.utils import sort_options
|
||||
|
||||
|
||||
def annotate_args(func):
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
@sort_options
|
||||
@click.command(
|
||||
short_help="Annotate H5AD file columns. Run `cellxgene annotation --help` for more information.",
|
||||
options_metavar="<options>",
|
||||
)
|
||||
@click.option(
|
||||
"-i",
|
||||
"--input-h5ad-file",
|
||||
required=True,
|
||||
type=str,
|
||||
help="The input H5AD file containing the missing annotations.",
|
||||
)
|
||||
@click.option(
|
||||
"-m",
|
||||
"--model-url",
|
||||
required=True,
|
||||
help="The URL of the model used to prediction annotated labels. May be a local filesystem directory "
|
||||
"or S3 path (s3://)",
|
||||
)
|
||||
@click.option(
|
||||
"-l",
|
||||
"--counts-layer",
|
||||
help="If specified, raw counts will be read from the AnnData layer of the specified name. If unspecified, "
|
||||
"raw counts will be read from `X` matrix, unless 'raw.X' exists, in which case that will be used.",
|
||||
)
|
||||
@click.option(
|
||||
"-g",
|
||||
"--gene-column-name",
|
||||
help="The name of the `var` column that contains gene identifiers. The values in this column will be used to match "
|
||||
"genes between the query and reference datasets. If not specified, the gene identifiers are expected to exist "
|
||||
"in `var.index`.",
|
||||
)
|
||||
# TODO: Useful if we want to support discoverability of models
|
||||
# @click.option(
|
||||
# "-r",
|
||||
# "--model-repository",
|
||||
# help="The base URL of the model repository. Maybe a local filesystem directory or S3 path (s3://)"
|
||||
# )
|
||||
# TODO: Useful if we want to support other, future annotation types, beyond "Cell Type". Currently hidden
|
||||
@click.option(
|
||||
"-a",
|
||||
"--annotation-type",
|
||||
type=click.Choice([t.value for t in AnnotationType]),
|
||||
default=AnnotationType.CELL_TYPE.value,
|
||||
show_default=True,
|
||||
hidden=True, # Remove if we add support for more annotation types
|
||||
help="The type of annotation to perform. This model to be used will be inferred from the annotation type.",
|
||||
)
|
||||
@click.option(
|
||||
"-c",
|
||||
"--annotation-prefix",
|
||||
type=str,
|
||||
default="cxg",
|
||||
show_default=True,
|
||||
help="An optional prefix used to form the names of: 1) new `obs` annotation columns that will store the predicted "
|
||||
"annotation values and confidence scores, 2) `obsm` embeddings (reference and umap embedding), and "
|
||||
"3) `uns` metadata for the prediction operation",
|
||||
)
|
||||
@click.option(
|
||||
"-n",
|
||||
"--run-name",
|
||||
type=str,
|
||||
help="An optional run name that will be used as a suffix to form the names of new `obs` annotation columns that "
|
||||
"will store the predicted annotation values and confidence scores. This can be used to allow multiple "
|
||||
"annotation predictions to be run on a single AnnData object.",
|
||||
)
|
||||
@click.option(
|
||||
"-u",
|
||||
"--update-h5ad-file",
|
||||
is_flag=True,
|
||||
help="Flag indicating whether to update the input h5ad file with annotation values. This option is mutually "
|
||||
"exclusive with --output-h5ad-file.",
|
||||
)
|
||||
@click.option(
|
||||
"-o",
|
||||
"--output-h5ad-file",
|
||||
help="The output H5AD file that will contain the generated annotation values. This option is mutually "
|
||||
"exclusive with --update-h5ad-file.",
|
||||
)
|
||||
@click.option("--use-model-cache/--no-use-model-cache", default=True)
|
||||
@click.option(
|
||||
"--use-gpu/--no-use-gpu",
|
||||
default=True,
|
||||
help="Whether to use a GPU for annotation operations (highly recommended, if available).",
|
||||
)
|
||||
# TODO: This is a cell type model-specific arg, so not ideal to specify here as a hardcoded option
|
||||
@click.option(
|
||||
"--classifier",
|
||||
default="default",
|
||||
help="For cell type annotation, the classifier level to use. The classifier is model-dependent, so refer to "
|
||||
"documentation for the specified model for valid values.",
|
||||
)
|
||||
# TODO: This is a cell type model-specific arg, so not ideal to specify here as a hardcoded option
|
||||
@click.option(
|
||||
"--organism",
|
||||
type=click.Choice(["Homo sapiens", "Mus musculus"], case_sensitive=True),
|
||||
default="Homo sapiens",
|
||||
help="For cell type annotation, the organism of the dataset. Used to normalize gene names to HGLC conventions when "
|
||||
"an annotation model has been trained using data from different organism.",
|
||||
)
|
||||
@click.option(
|
||||
"--model-cache-dir",
|
||||
default=".models_cache",
|
||||
help="Local directory used to store model files that are retrieved from a remote location. Model files will "
|
||||
"be read from this directory first, if they exist, to avoid repeating large downloads.",
|
||||
)
|
||||
@click.option(
|
||||
"--mlflow-env-manager",
|
||||
type=click.Choice(["virtualenv", "conda", "local"]),
|
||||
default="virtualenv",
|
||||
help="Annotation model prediction will be installed and executed in the specified type of environment. MacOS users "
|
||||
"on Apple Silicon (arm64, M1, M2, etc.) are recommended to use 'conda' to avoid Python package installation "
|
||||
"errors. If 'conda' is specified then cellxgene must also have been installed within a conda environment",
|
||||
)
|
||||
@click.help_option("--help", "-h", help="Show this message and exit.")
|
||||
def annotate(**cli_args):
|
||||
_validate_options(cli_args)
|
||||
|
||||
print(f"Reading query dataset {cli_args['input_h5ad_file']}...")
|
||||
|
||||
annotation_prefix = "_".join(
|
||||
filter(None, [cli_args.get("annotation_prefix"), cli_args.get("annotation_type"), cli_args.get("run_name")])
|
||||
)
|
||||
|
||||
output_h5ad_file = cli_args["input_h5ad_file"] if cli_args["update_h5ad_file"] else cli_args["output_h5ad_file"]
|
||||
|
||||
model_url = cli_args.get("model_url")
|
||||
local_model_path = _retrieve_model(cli_args.get("model_cache_dir"), model_url, cli_args.get("use_model_cache"))
|
||||
|
||||
print(f"Annotating {cli_args.get('input_h5ad_file')} with {cli_args.get('annotation_type')}...")
|
||||
|
||||
if cli_args["annotation_type"] == AnnotationType.CELL_TYPE.value:
|
||||
predict_args = dict(
|
||||
query_dataset_h5ad_path=cli_args.get("input_h5ad_file"),
|
||||
output_h5ad_path=output_h5ad_file,
|
||||
annotation_prefix=annotation_prefix,
|
||||
counts_layer=cli_args.get("counts_layer"),
|
||||
gene_column_name=cli_args.get("gene_column_name"),
|
||||
classifier=cli_args.get("classifier"),
|
||||
organism=cli_args.get("organism"),
|
||||
use_gpu=cli_args.get("use_gpu"),
|
||||
)
|
||||
# Drop args that have values of `None` as these will cause problems when passing into MLflow predict, since it
|
||||
# ultimately gets converted into 1-row Pandas DataFrame (None is interpreted as a float type column!)
|
||||
predict_args = dict([(k, v) for k, v in predict_args.items() if v is not None])
|
||||
|
||||
# Invoke prediction using MLflow cli, as a separate process.
|
||||
# This fully prepares the Python environment that is needed for executing the model.
|
||||
# The Python environment will be reused after it is setup once.
|
||||
with NamedTemporaryFile(buffering=0) as predict_args_file:
|
||||
# write the mlflow predict arguments to a csv file, which will be passed to mlflow cmd
|
||||
pd.DataFrame([json.dumps(predict_args)]).to_csv(predict_args_file, index=None)
|
||||
predict_args_file.seek(0)
|
||||
|
||||
# run mlflow prediction in subprocess
|
||||
predict_cmd = (
|
||||
f"mlflow models predict "
|
||||
f"--env-manager {cli_args['mlflow_env_manager']} "
|
||||
f"--model-uri {local_model_path} "
|
||||
f"--content-type csv --input-path {predict_args_file.name}"
|
||||
)
|
||||
p = subprocess.Popen(
|
||||
args=shlex.split(predict_cmd), stdin=predict_args_file, text=True, bufsize=0, stdout=PIPE, stderr=STDOUT
|
||||
)
|
||||
|
||||
# display mlflow process output as it runs
|
||||
for line in p.stdout:
|
||||
print(line.rstrip())
|
||||
|
||||
p.wait()
|
||||
if p.returncode == 0:
|
||||
print(f"Wrote annotations to {cli_args.get('output_h5ad_file')}")
|
||||
else:
|
||||
print("Annotation failed!")
|
||||
else:
|
||||
raise BadParameter(f"unknown annotation type {cli_args['annotation_type']}")
|
||||
|
||||
|
||||
def _retrieve_model(model_cache_dir, model_url, use_cache=True):
|
||||
local_cache_model_path = os.path.join(model_cache_dir, os.path.splitext(os.path.basename(model_url))[0])
|
||||
if not os.path.exists(local_cache_model_path) or not use_cache:
|
||||
print(f"Retrieving model from {model_url}")
|
||||
# download from remote source
|
||||
with DataLocator(model_url).local_handle() as model_archive_local_path:
|
||||
# unpack archive to local cache dir
|
||||
shutil.unpack_archive(model_archive_local_path, local_cache_model_path)
|
||||
else:
|
||||
print(f"Using cached model at {local_cache_model_path}")
|
||||
|
||||
return local_cache_model_path
|
||||
|
||||
|
||||
def _validate_options(cli_args):
|
||||
# TODO(atolopko): Use cloup library for this logic
|
||||
if cli_args["update_h5ad_file"] and cli_args["output_h5ad_file"]:
|
||||
click.echo("--update_h5ad_file and --output_h5ad_file are mutually exclusive")
|
||||
sys.exit(1)
|
||||
if not (cli_args["update_h5ad_file"] or cli_args["output_h5ad_file"]):
|
||||
click.echo("--update_h5ad_file or --output_h5ad_file must be specified")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
annotate()
|
||||
@@ -1,6 +1,5 @@
|
||||
import click
|
||||
|
||||
from .annotate import annotate
|
||||
from .launch import launch
|
||||
from .prepare import prepare
|
||||
from .upgrade import log_upgrade_check
|
||||
@@ -32,5 +31,4 @@ def cli(upgrade_check):
|
||||
|
||||
|
||||
cli.add_command(launch)
|
||||
cli.add_command(annotate)
|
||||
cli.add_command(prepare)
|
||||
|
||||
@@ -57,7 +57,7 @@ class Annotations(metaclass=ABCMeta):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def write_gene_sets(self, gs, tid, data_adaptor):
|
||||
def write_gene_sets(self, gs, data_adaptor):
|
||||
"""Write the gene sets (gs) to a persistent storage such that it can later be read"""
|
||||
pass
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ from hashlib import blake2b
|
||||
|
||||
import pandas as pd
|
||||
from flask import session
|
||||
from fsspec import AbstractFileSystem
|
||||
|
||||
from server import __version__ as cellxgene_version
|
||||
from server.app.session import get_user_id
|
||||
@@ -63,27 +62,21 @@ class AnnotationsLocalFile(Annotations):
|
||||
self.check_user_annotations_enabled() # raises
|
||||
|
||||
fname = self._get_celllabels_filename(data_adaptor)
|
||||
empty_labels = pd.DataFrame()
|
||||
if fname is None:
|
||||
return empty_labels
|
||||
|
||||
with self.label_lock:
|
||||
locator = DataLocator(fname)
|
||||
if not locator.exists() or locator.size() == 0:
|
||||
return empty_labels
|
||||
|
||||
# return the cached labels if possible
|
||||
if fname == self.last_label_fname:
|
||||
return self.last_labels
|
||||
|
||||
# otherwise, read labels from file
|
||||
with locator.open() as f:
|
||||
labels = pd.read_csv(f, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False)
|
||||
|
||||
# update the cache
|
||||
self.last_label_fname = fname
|
||||
self.last_labels = labels
|
||||
return labels
|
||||
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
|
||||
# returned the cached labels if possible, otherwise read them from the file
|
||||
if fname == self.last_label_fname:
|
||||
return self.last_labels
|
||||
else:
|
||||
labels = pd.read_csv(
|
||||
fname, dtype="category", index_col=0, header=0, comment="#", keep_default_na=False
|
||||
)
|
||||
# update the cache
|
||||
self.last_label_fname = fname
|
||||
self.last_labels = labels
|
||||
return labels
|
||||
else:
|
||||
return pd.DataFrame()
|
||||
|
||||
def write_labels(self, df, data_adaptor):
|
||||
self.check_user_annotations_enabled() # raises
|
||||
@@ -102,12 +95,13 @@ class AnnotationsLocalFile(Annotations):
|
||||
|
||||
fname = self._get_celllabels_filename(data_adaptor)
|
||||
self._backup(fname)
|
||||
locator = DataLocator(fname)
|
||||
with locator.open("w") as f:
|
||||
if not df.empty:
|
||||
if not df.empty:
|
||||
with open(fname, "w", newline="") as f:
|
||||
if header is not None:
|
||||
f.write(header)
|
||||
df.to_csv(f)
|
||||
else:
|
||||
open(fname, "w").close()
|
||||
|
||||
# update the cache
|
||||
self.last_label_fname = fname
|
||||
@@ -115,32 +109,26 @@ class AnnotationsLocalFile(Annotations):
|
||||
|
||||
def read_gene_sets(self, data_adaptor, context=None):
|
||||
fname = self._get_genesets_filename(data_adaptor)
|
||||
empty_gene_sets = {}
|
||||
|
||||
gene_sets = {}
|
||||
tid = None
|
||||
with self.gene_sets_lock:
|
||||
tid = self.last_geneset_tid # inside the critical section
|
||||
if fname is None:
|
||||
return (empty_gene_sets, tid)
|
||||
if fname is not None and os.path.exists(fname) and os.path.getsize(fname) > 0:
|
||||
# return the cached genesets if possible, otherwise read from file and validate them
|
||||
if fname == self.last_geneset_fname:
|
||||
gene_sets = self.last_geneset
|
||||
else:
|
||||
# read
|
||||
gene_sets = read_gene_sets_tidycsv(DataLocator(fname), context)
|
||||
|
||||
locator = DataLocator(fname)
|
||||
if not locator.exists() or locator.size() == 0:
|
||||
return (empty_gene_sets, tid)
|
||||
# validate
|
||||
gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
|
||||
|
||||
# return the cached genesets if possible, otherwise read from file and validate them
|
||||
if fname == self.last_geneset_fname:
|
||||
return (self.last_geneset, tid)
|
||||
# update cache
|
||||
self.last_geneset_fname = fname
|
||||
self.last_geneset = gene_sets
|
||||
|
||||
# read
|
||||
gene_sets = read_gene_sets_tidycsv(locator, context)
|
||||
|
||||
# validate
|
||||
gene_sets = data_adaptor.check_new_gene_sets(gene_sets, context)
|
||||
|
||||
# update cache
|
||||
self.last_geneset_fname = fname
|
||||
self.last_geneset = gene_sets
|
||||
|
||||
return (gene_sets, tid)
|
||||
return (gene_sets, tid)
|
||||
|
||||
def write_gene_sets(self, gene_sets, tid, data_adaptor):
|
||||
self.check_gene_sets_save_enabled() # raises
|
||||
@@ -169,9 +157,9 @@ class AnnotationsLocalFile(Annotations):
|
||||
|
||||
fname = self._get_genesets_filename(data_adaptor)
|
||||
self._backup(fname)
|
||||
locator = DataLocator(fname)
|
||||
with locator.open("w", newline="") as f:
|
||||
f.write(header + self.gene_sets_to_csv(gene_sets))
|
||||
with open(fname, "w", newline="") as f:
|
||||
f.write(header)
|
||||
f.write(self.gene_sets_to_csv(gene_sets))
|
||||
|
||||
# update the cache
|
||||
self.last_geneset_fname = fname
|
||||
@@ -193,7 +181,7 @@ class AnnotationsLocalFile(Annotations):
|
||||
|
||||
output_file = self.label_output_file or self.gene_sets_output_file
|
||||
if output_file:
|
||||
return os.path.dirname(DataLocator(output_file).abspath())
|
||||
return os.path.dirname(os.path.abspath(output_file))
|
||||
|
||||
return os.getcwd()
|
||||
|
||||
@@ -232,37 +220,34 @@ class AnnotationsLocalFile(Annotations):
|
||||
1. fname -> backup_dir/fname-TIME
|
||||
2. delete excess files in backup_dir
|
||||
"""
|
||||
locator = DataLocator(fname)
|
||||
fs: AbstractFileSystem = locator.fs # Handle to underlying fsspec file system
|
||||
|
||||
# Make sure there is work to do
|
||||
if not locator.exists():
|
||||
return
|
||||
|
||||
root, ext = os.path.splitext(locator.abspath())
|
||||
root, ext = os.path.splitext(fname)
|
||||
backup_dir = f"{root}-backups"
|
||||
|
||||
# Make sure there is work to do
|
||||
if not os.path.exists(fname):
|
||||
return
|
||||
|
||||
# Ensure backup_dir exists
|
||||
fs.mkdirs(backup_dir, exist_ok=True)
|
||||
if not os.path.exists(backup_dir):
|
||||
os.mkdir(backup_dir)
|
||||
|
||||
# Save current file to backup_dir
|
||||
fname_base = os.path.basename(fname)
|
||||
fname_base_root, fname_base_ext = os.path.splitext(fname_base)
|
||||
# don't use ISO standard time format, as it contains characters illegal on some filesystems.
|
||||
# don't use ISO standard time format, as it contains characters illegal on some filesytems.
|
||||
nowish = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
|
||||
backup_fname = os.path.join(backup_dir, f"{fname_base_root}-{nowish}{fname_base_ext}")
|
||||
if fs.exists(backup_fname):
|
||||
fs.delete(backup_fname)
|
||||
fs.rename(fname, backup_fname)
|
||||
if os.path.exists(backup_fname):
|
||||
os.remove(backup_fname)
|
||||
os.rename(fname, backup_fname)
|
||||
|
||||
# prune the backup_dir to max number of backup files, keeping the most recent backups
|
||||
backup_path_prefix = DataLocator.strip_protocol(os.path.join(backup_dir, fname_base_root + "-"))
|
||||
backups = list(filter(lambda s: s.startswith(backup_path_prefix), fs.ls(backup_dir)))
|
||||
|
||||
# sorting to drop the oldest
|
||||
excess_backups = list(sorted(backups, reverse=True))[max_backups:]
|
||||
for bu in excess_backups:
|
||||
fs.delete(bu)
|
||||
backups = list(filter(lambda s: s.startswith(fname_base_root), os.listdir(backup_dir)))
|
||||
excess_count = len(backups) - max_backups
|
||||
if excess_count > 0:
|
||||
backups.sort()
|
||||
for bu in backups[0:excess_count]:
|
||||
os.remove(os.path.join(backup_dir, bu))
|
||||
|
||||
def update_parameters(self, parameters, data_adaptor):
|
||||
params = {}
|
||||
|
||||
@@ -4,7 +4,6 @@ from os.path import splitext, isdir
|
||||
from server.common.annotations.local_file_csv import AnnotationsLocalFile
|
||||
from server.common.config.base_config import BaseConfig
|
||||
from server.common.errors import ConfigurationError, AnnotationsError
|
||||
from server.common.utils.data_locator import DataLocator
|
||||
from server.data_common.matrix_loader import MatrixDataLoader
|
||||
|
||||
|
||||
@@ -128,15 +127,11 @@ class DatasetConfig(BaseConfig):
|
||||
if lf_ext and lf_ext != ".csv":
|
||||
raise ConfigurationError(f"genesets file type must be .csv: {genesets_filename}")
|
||||
|
||||
if dirname is not None:
|
||||
if not DataLocator(dirname).islocal():
|
||||
# remote object stores only support objects but not directories, do nothing
|
||||
pass
|
||||
elif not isdir(dirname):
|
||||
try:
|
||||
os.mkdir(dirname)
|
||||
except OSError:
|
||||
raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
|
||||
if dirname is not None and not isdir(dirname):
|
||||
try:
|
||||
os.mkdir(dirname)
|
||||
except OSError:
|
||||
raise ConfigurationError("Unable to create directory specified by --user-generated-data-dir")
|
||||
|
||||
anno_config = {
|
||||
"user-annotations": self.user_annotations__enable,
|
||||
|
||||
+40
-2
@@ -4,8 +4,9 @@ import sys
|
||||
from http import HTTPStatus
|
||||
import zlib
|
||||
import json
|
||||
import numpy as np
|
||||
|
||||
from flask import make_response, jsonify, current_app, abort
|
||||
from flask import make_response, jsonify, current_app, abort, send_file
|
||||
from werkzeug.urls import url_unquote
|
||||
|
||||
from server.common.config.client_config import get_client_config
|
||||
@@ -293,7 +294,7 @@ def layout_obs_get(request, data_adaptor):
|
||||
|
||||
try:
|
||||
return make_response(
|
||||
data_adaptor.layout_to_fbs_matrix(fields), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
|
||||
data_adaptor.layout_to_fbs_matrix(fields, data_adaptor.get_spatial()), HTTPStatus.OK, {"Content-Type": "application/octet-stream"}
|
||||
)
|
||||
except (KeyError, DatasetAccessError) as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
@@ -397,3 +398,40 @@ def summarize_var_post(request, data_adaptor):
|
||||
|
||||
key = request.args.get("key", default=None)
|
||||
return summarize_var_helper(request, data_adaptor, key, request.get_data())
|
||||
|
||||
def spatial_image_get(request, data_adaptor):
|
||||
import io
|
||||
import matplotlib.pyplot
|
||||
|
||||
resolution = "hires"
|
||||
spatial = data_adaptor.get_spatial()
|
||||
|
||||
if len(list(spatial)) == 0:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, "uns does not have spatial information")
|
||||
|
||||
library_id = list(spatial)[0]
|
||||
if len(spatial) > 1:
|
||||
current_app.logger.warning(f"More than one library found under uns.spatial, using library '{library_id}'")
|
||||
|
||||
if "images" not in spatial[library_id]:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, "spatial information does not contain images")
|
||||
|
||||
if resolution not in spatial[library_id]["images"]:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, f"spatial information does not contain requested resolution '{resolution}'")
|
||||
|
||||
response_image = io.BytesIO()
|
||||
img = spatial[library_id]["images"][resolution]
|
||||
matplotlib.pyplot.imsave(response_image, img)
|
||||
response_image.seek(0)
|
||||
|
||||
try:
|
||||
return send_file(response_image, attachment_filename=f"{library_id}-{resolution}.png", mimetype="image/png")
|
||||
except (KeyError, DatasetAccessError) as e:
|
||||
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
|
||||
except PrepareError:
|
||||
return abort_and_log(
|
||||
HTTPStatus.NOT_IMPLEMENTED,
|
||||
f"No spatial image available {request.path}",
|
||||
loglevel=logging.ERROR,
|
||||
include_exc_info=True,
|
||||
)
|
||||
|
||||
@@ -52,10 +52,8 @@ class DataLocator:
|
||||
self.fs = fsspec.filesystem(self.protocol)
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
|
||||
f"path={self.path}, uri_or_path={self.uri_or_path})"
|
||||
)
|
||||
return f"DataLocator(protocol={self.protocol}, cname={self.cname}, "
|
||||
f"path={self.path}, uri_or_path={self.uri_or_path})"
|
||||
|
||||
@staticmethod
|
||||
def _get_protocol_and_path(uri_or_path):
|
||||
@@ -67,10 +65,6 @@ class DataLocator:
|
||||
return protocol, path
|
||||
return None, uri_or_path
|
||||
|
||||
@staticmethod
|
||||
def strip_protocol(uri_or_path):
|
||||
return DataLocator._get_protocol_and_path(uri_or_path)[1]
|
||||
|
||||
def exists(self):
|
||||
return self.fs.exists(self.cname)
|
||||
|
||||
@@ -78,7 +72,7 @@ class DataLocator:
|
||||
return self.fs.size(self.cname)
|
||||
|
||||
def lastmodtime(self):
|
||||
"""return datetime object representing last modification time, or None if unavailable"""
|
||||
""" return datetime object representing last modification time, or None if unavailable """
|
||||
info = self.fs.info(self.cname)
|
||||
if self.islocal() and info is not None:
|
||||
return datetime.fromtimestamp(info["mtime"])
|
||||
@@ -98,8 +92,8 @@ class DataLocator:
|
||||
def isfile(self):
|
||||
return self.fs.isfile(self.cname)
|
||||
|
||||
def open(self, *args, **kwargs):
|
||||
return self.fs.open(self.uri_or_path, *args, **kwargs)
|
||||
def open(self, *args):
|
||||
return self.fs.open(self.uri_or_path, *args)
|
||||
|
||||
def islocal(self):
|
||||
return self.protocol is None or self.protocol == "file"
|
||||
@@ -113,9 +107,10 @@ class DataLocator:
|
||||
# do our best to create a file with the same.
|
||||
ext = os.path.splitext(self.path)
|
||||
suffix = None if ext[1] == "" else ext[1]
|
||||
with tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
|
||||
self.fs.download(self.uri_or_path, tmp.name)
|
||||
with self.open() as src, tempfile.NamedTemporaryFile(prefix="cellxgene_", suffix=suffix, delete=False) as tmp:
|
||||
tmp.write(src.read())
|
||||
tmp.close()
|
||||
src.close()
|
||||
tmp_path = tmp.name
|
||||
return LocalFilePath(tmp_path, delete=True)
|
||||
|
||||
|
||||
@@ -174,14 +174,10 @@ class AnndataAdaptor(DataAdaptor):
|
||||
except MemoryError:
|
||||
raise DatasetAccessError("Out of memory - file is too large for available memory.")
|
||||
except Exception:
|
||||
import traceback
|
||||
message = (
|
||||
raise DatasetAccessError(
|
||||
"File not found or is inaccessible. File must be an .h5ad object. "
|
||||
"Please check your input and try again."
|
||||
)
|
||||
if self.server_config.app__verbose:
|
||||
message += f"\n{traceback.format_exc()}"
|
||||
raise DatasetAccessError(message)
|
||||
)
|
||||
|
||||
def _validate_and_initialize(self):
|
||||
if anndata_version_is_pre_070():
|
||||
@@ -240,14 +236,6 @@ class AnndataAdaptor(DataAdaptor):
|
||||
warnings.warn(
|
||||
f"Anndata data matrix is in {self.data.X.dtype} format not float32. " f"Precision may be truncated."
|
||||
)
|
||||
if self.data.X.dtype < np.float32:
|
||||
if self.data.isbacked:
|
||||
raise DatasetAccessError(f"Data matrix in {self.data.X.dtype} format is not supported in backed mode."
|
||||
" Please reload without --backed, or convert matrix to float32")
|
||||
warnings.warn(
|
||||
f"Anndata data matrix is in unsupported {self.data.X.dtype} format -- will be cast to float32"
|
||||
)
|
||||
self.data.X = self.data.X.astype(np.float32)
|
||||
for ax in Axis:
|
||||
curr_axis = getattr(self.data, str(ax))
|
||||
for ann in curr_axis:
|
||||
@@ -286,6 +274,43 @@ class AnndataAdaptor(DataAdaptor):
|
||||
df = df[fields]
|
||||
return encode_matrix_fbs(df, col_idx=df.columns)
|
||||
|
||||
def get_spatial(self):
|
||||
return self.data.uns["spatial"]
|
||||
|
||||
def get_spatial_metadata(self):
|
||||
spatial = self.get_spatial()
|
||||
|
||||
resolution = "hires"
|
||||
|
||||
if len(list(spatial)) == 0:
|
||||
raise Exception("uns does not have spatial information")
|
||||
|
||||
library_id = list(spatial)[0]
|
||||
|
||||
if "images" not in spatial[library_id]:
|
||||
raise Exception("spatial information does not contain images")
|
||||
|
||||
if resolution not in spatial[library_id]["images"]:
|
||||
raise Exception(f"spatial information does not contain requested resolution '{resolution}'")
|
||||
|
||||
scaleref = spatial[library_id]["scalefactors"][f"tissue_{resolution}_scalef"]
|
||||
(h, w, _) = spatial[library_id]["images"][resolution].shape
|
||||
|
||||
A = self.data.obsm["X_spatial"]
|
||||
min = np.nanmin(A, axis=0)
|
||||
max = np.nanmax(A, axis=0)
|
||||
scale = np.amax(max - min)
|
||||
translate = 0.5 - ((max - min) / scale / 2)
|
||||
|
||||
return {
|
||||
"imageWidth": w,
|
||||
"imageHeight": h,
|
||||
"scaleref": scaleref,
|
||||
"inverseScale": int(scale),
|
||||
"inverseTranslate": translate.tolist(),
|
||||
"inverseMin": min.tolist(),
|
||||
}
|
||||
|
||||
def get_embedding_names(self):
|
||||
"""
|
||||
Return pre-computed embeddings.
|
||||
|
||||
@@ -340,31 +340,57 @@ class DataAdaptor(metaclass=ABCMeta):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def normalize_embedding(embedding):
|
||||
def normalize_embedding(embedding, spatial = None):
|
||||
"""Normalize embedding layout to meet client assumptions.
|
||||
Embedding is an ndarray, shape (n_obs, n)., where n is normally 2
|
||||
Embedding is an ndarray, shape (n_obs, n)., where n is normally 2.
|
||||
Note: if spatial data is available, the normalization will be done
|
||||
according to the size of the underlying image
|
||||
"""
|
||||
|
||||
# scale isotropically
|
||||
try:
|
||||
min = np.nanmin(embedding, axis=0)
|
||||
max = np.nanmax(embedding, axis=0)
|
||||
except RuntimeError:
|
||||
# indicates entire array was NaN, which should propagate
|
||||
min = np.NaN
|
||||
max = np.NaN
|
||||
if spatial is not None:
|
||||
|
||||
scale = np.amax(max - min)
|
||||
normalized_layout = (embedding - min) / scale
|
||||
# TODO: sync with the code in spatial_data_get
|
||||
resolution = "hires"
|
||||
|
||||
# translate to center on both axis
|
||||
translate = 0.5 - ((max - min) / scale / 2)
|
||||
normalized_layout = normalized_layout + translate
|
||||
if len(list(spatial)) == 0:
|
||||
raise Exception("uns does not have spatial information")
|
||||
|
||||
library_id = list(spatial)[0]
|
||||
|
||||
if "images" not in spatial[library_id]:
|
||||
raise Exception("spatial information does not contain images")
|
||||
|
||||
if resolution not in spatial[library_id]["images"]:
|
||||
raise Exception(f"spatial information does not contain requested resolution '{resolution}'")
|
||||
|
||||
scaleref = spatial[library_id]["scalefactors"][f"tissue_{resolution}_scalef"]
|
||||
(h, w, _) = spatial[library_id]["images"][resolution].shape
|
||||
|
||||
A = embedding * scaleref
|
||||
A = np.column_stack([A[:, 0] / w, A[:, 1] / h])
|
||||
normalized_layout = A.astype(dtype=np.float32)
|
||||
|
||||
else:
|
||||
|
||||
# scale isotropically
|
||||
try:
|
||||
min = np.nanmin(embedding, axis=0)
|
||||
max = np.nanmax(embedding, axis=0)
|
||||
except RuntimeError:
|
||||
# indicates entire array was NaN, which should propagate
|
||||
min = np.NaN
|
||||
max = np.NaN
|
||||
|
||||
scale = np.amax(max - min)
|
||||
normalized_layout = (embedding - min) / scale
|
||||
|
||||
# translate to center on both axis
|
||||
translate = 0.5 - ((max - min) / scale / 2)
|
||||
normalized_layout = normalized_layout + translate
|
||||
|
||||
normalized_layout = normalized_layout.astype(dtype=np.float32)
|
||||
return normalized_layout
|
||||
|
||||
def layout_to_fbs_matrix(self, fields):
|
||||
def layout_to_fbs_matrix(self, fields, spatial = None):
|
||||
"""
|
||||
return specified embeddings as a flatbuffer, using the cellxgene matrix fbs encoding.
|
||||
|
||||
@@ -380,7 +406,7 @@ class DataAdaptor(metaclass=ABCMeta):
|
||||
with ServerTiming.time("layout.query"):
|
||||
for ename in embeddings:
|
||||
embedding = self.get_embedding_array(ename, 2)
|
||||
normalized_layout = DataAdaptor.normalize_embedding(embedding)
|
||||
normalized_layout = DataAdaptor.normalize_embedding(embedding, ename == "spatial" and spatial)
|
||||
layout_data.append(pd.DataFrame(normalized_layout, columns=[f"{ename}_0", f"{ename}_1"]))
|
||||
|
||||
with ServerTiming.time("layout.encode"):
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
mlflow
|
||||
scanpy
|
||||
@@ -2,9 +2,9 @@ black
|
||||
bumpversion>=0.5
|
||||
codecov>=2.0.15
|
||||
parameterized>=0.7.0
|
||||
psycopg2-binary>=2.8.5
|
||||
pytest>=3.6.3
|
||||
python-jose>=3.2.0
|
||||
twine>=1.12.1
|
||||
-r requirements.txt
|
||||
-r requirements-prepare.txt
|
||||
-r requirements-annotate.txt
|
||||
|
||||
@@ -14,8 +14,9 @@ flatten-dict>=0.2.0
|
||||
fsspec>=0.4.4,<0.8.0
|
||||
gunicorn>=20.0.4
|
||||
h5py>=3.0.0
|
||||
matplotlib>=3.5.0
|
||||
numba>=0.51.2
|
||||
numpy>=1.17.5,<=1.22
|
||||
numpy>=1.17.5
|
||||
packaging>=20.0
|
||||
pandas>=1.0,!=1.1 # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
|
||||
PyYAML>=5.4 # CVE-2020-14343
|
||||
|
||||
@@ -9,12 +9,9 @@ with open("server/requirements.txt") as fh:
|
||||
with open("server/requirements-prepare.txt") as fh:
|
||||
requirements_prepare = fh.read().splitlines()
|
||||
|
||||
with open("server/requirements-annotate.txt") as fh:
|
||||
requirements_annotate = fh.read().splitlines()
|
||||
|
||||
setup(
|
||||
name="cellxgene",
|
||||
version="1.0.1",
|
||||
version="1.0.0",
|
||||
packages=find_packages(),
|
||||
url="https://github.com/chanzuckerberg/cellxgene",
|
||||
license="MIT",
|
||||
@@ -43,5 +40,5 @@ setup(
|
||||
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
||||
],
|
||||
entry_points={"console_scripts": ["cellxgene = server.cli.cli:cli"]},
|
||||
extras_require=dict(prepare=requirements_prepare, annotate=requirements_annotate),
|
||||
extras_require=dict(prepare=requirements_prepare),
|
||||
)
|
||||
|
||||
Vendored
BIN
Binary file not shown.
@@ -1,5 +0,0 @@
|
||||
from .mlflow_model_fixture import FakeModel
|
||||
|
||||
|
||||
def _load_pyfunc(data_path):
|
||||
return FakeModel()
|
||||
@@ -1,11 +0,0 @@
|
||||
import mlflow
|
||||
|
||||
|
||||
class FakeModel(mlflow.pyfunc.PythonModel):
|
||||
def __init__(self, input_to_output: dict = {}):
|
||||
self.input_to_output = input_to_output
|
||||
|
||||
def predict(self, model_input) -> None:
|
||||
# this stdout output is useful for validating the input in a test, noting that this model will be invoked in a
|
||||
# subprocess, so stdout is one means of communicating information back to the test code
|
||||
print(f"__MODEL_INPUT__={model_input.iloc[0][0]}")
|
||||
@@ -1,111 +0,0 @@
|
||||
import os
|
||||
import shutil
|
||||
import unittest
|
||||
from tempfile import mkstemp, TemporaryDirectory
|
||||
|
||||
import mlflow
|
||||
from click.testing import CliRunner
|
||||
|
||||
from server.cli.annotate import annotate
|
||||
from test.unit.cli.fixtures.mlflow_model_fixture import FakeModel
|
||||
|
||||
|
||||
def write_model(model) -> str:
|
||||
with TemporaryDirectory() as mlflow_model_dir:
|
||||
fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
|
||||
mlflow.pyfunc.save_model(mlflow_model_dir,
|
||||
loader_module='fixtures',
|
||||
code_path=[fixtures_path])
|
||||
return shutil.make_archive(mkstemp()[1], "zip", mlflow_model_dir)
|
||||
|
||||
|
||||
class TestCliAnnotate(unittest.TestCase):
|
||||
def test__annotate__loads_and_runs(self):
|
||||
"""
|
||||
Invokes the `annotate` subcommand of cellxgene CLI, using a CliRunner() programmatic invocation.
|
||||
|
||||
This tests the happy path case:
|
||||
1) Command line options are parsed;
|
||||
2) An MLflow model zip archive can be read in (from local disk), unpacked, and invoked;
|
||||
3) The correct options are passed to the MLflow model.
|
||||
4) The annotate subcommand exits successfully.
|
||||
|
||||
This does not verify model output or predictions (it's a fake MLflow model, after all); it's up to the real model
|
||||
to output its predictions as it wants, but this is specific to the model and so not tested here.
|
||||
|
||||
The CliRunner() invokes the subcommand in a subprocess, and the annotate subcommand itself invokes the MLflow
|
||||
model in yet another subprocess. So while this test can help determine if everything is working, it is not a
|
||||
simple matter to debug in the case of a failure. However, the stdout/stderr of the MLflow process is captured
|
||||
by the CliRunner() subprocess, so errors can be inspected in result.stdout when debugging this test. Hope this
|
||||
helps!
|
||||
"""
|
||||
|
||||
_, query_dataset_file_path = mkstemp()
|
||||
model_file_path = write_model(FakeModel())
|
||||
|
||||
result = CliRunner().invoke(
|
||||
annotate,
|
||||
[
|
||||
"--input-h5ad-file",
|
||||
query_dataset_file_path,
|
||||
"--model-url",
|
||||
model_file_path,
|
||||
"--output-h5ad-file",
|
||||
f"{query_dataset_file_path}.output",
|
||||
# avoid having mflow create conda env or virtualenv when in test env;
|
||||
# this avoids making pip remote requests and is also faster
|
||||
"--mlflow-env-manager", "local"
|
||||
],
|
||||
)
|
||||
|
||||
# to help debugging, show the output from the CliRunner and MLflow stdout
|
||||
if result.exit_code:
|
||||
print(result.stdout)
|
||||
|
||||
self.assertEqual(0, result.exit_code, "runs successfully")
|
||||
|
||||
# The FakeModel will print it inputs to stdout, as "__MODEL_INPUT__={...}", allowing us to assert that it received valid inputs.
|
||||
self.assertIn(
|
||||
"__MODEL_INPUT__={"
|
||||
f'"query_dataset_h5ad_path": "{query_dataset_file_path}", '
|
||||
f'"output_h5ad_path": "{query_dataset_file_path}.output", '
|
||||
'"annotation_prefix": "cxg_cell_type", "classifier": "default", '
|
||||
'"organism": "Homo sapiens", "use_gpu": true}',
|
||||
result.stdout,
|
||||
"inputs passed correctly",
|
||||
)
|
||||
|
||||
def test__annotate__verifies_mutually_exclusive_options(self):
|
||||
required_options = ["--input-h5ad-file", "some.h5ad", "--model-url", "some_url"]
|
||||
result = CliRunner().invoke(
|
||||
annotate,
|
||||
required_options + [],
|
||||
)
|
||||
|
||||
self.assertNotEqual(0, result.exit_code, "aborts with non-success code")
|
||||
self.assertIn(
|
||||
"--update_h5ad_file or --output_h5ad_file must be specified",
|
||||
result.stdout,
|
||||
"error message displayed",
|
||||
)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
annotate, required_options + ["--output-h5ad-file", "some_arg", "--update-h5ad-file"]
|
||||
)
|
||||
|
||||
self.assertNotEqual(0, result.exit_code, "aborts with non-success code")
|
||||
self.assertIn(
|
||||
"--update_h5ad_file and --output_h5ad_file are mutually exclusive",
|
||||
result.stdout,
|
||||
"error message displayed",
|
||||
)
|
||||
|
||||
|
||||
# TODO:
|
||||
# Test annotate cli args more comprehensively
|
||||
# Test server.cli.annotate._validate_options
|
||||
# Test model caching feature works
|
||||
# Test model loading from s3 works (maybe w/just a real model)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -290,7 +290,6 @@ class EndPoints(object):
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data, pbmc3k_colors)
|
||||
|
||||
@unittest.skip('needs fix: https://github.com/chanzuckerberg/cellxgene/issues/2542')
|
||||
def test_static(self):
|
||||
endpoint = "static"
|
||||
file = "assets/favicon.ico"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import unittest
|
||||
|
||||
@@ -36,7 +37,6 @@ Test the anndata adaptor using the pbmc3k data set.
|
||||
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"),
|
||||
(f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"),
|
||||
(f"{FIXTURES_ROOT}/pbmc3k_64.h5ad", False, "auto"), # 64 bit conversion tests
|
||||
(f"{FIXTURES_ROOT}/pbmc3k_16.h5ad", False, "auto"), # 16 bit conversion tests
|
||||
],
|
||||
)
|
||||
class AdaptorTest(unittest.TestCase):
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
import unittest
|
||||
|
||||
from parameterized import parameterized_class
|
||||
|
||||
from server.common.errors import DatasetAccessError
|
||||
from test import FIXTURES_ROOT
|
||||
from test.unit import app_config
|
||||
|
||||
|
||||
@parameterized_class(
|
||||
("data_locator", "backed", "X_approximate_distribution"),
|
||||
[
|
||||
(f"{FIXTURES_ROOT}/pbmc3k_16.h5ad", True, "auto"), # 16 bit conversion tests
|
||||
],
|
||||
)
|
||||
class AdaptorLoadErrorTest(unittest.TestCase):
|
||||
def test_float16_backed_raises_err(self):
|
||||
with self.assertRaises(DatasetAccessError):
|
||||
config = app_config(
|
||||
self.data_locator,
|
||||
backed=self.backed,
|
||||
extra_dataset_config=dict(X_approximate_distribution=self.X_approximate_distribution),
|
||||
)
|
||||
Reference in New Issue
Block a user