fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-15 12:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add user-defined category-label colors (#1402)

Browse Source 
* Add user-defined category-label colors

Fixes https://github.com/chanzuckerberg/cellxgene/issues/1152

As described in https://github.com/chanzuckerberg/cellxgene/issues/1307

* Respond to feedback from @bkmartinjr in nodejs

* Respond to feedback from @bkmartinjr in python

* Add tests to the server module

* Autoformat python, run linter

* Make colors_get error handling specific

* Respond to feedback from @bkmartinjr

* Respond to feedback from @bkmartinjr

* Fix whitespace

* Fix python lint errrors

* Update documentation

* Add --disable-user-colors option to launch and cxgtool.py

* Fix python formatting

* Rename '--disable-user-colors' to '--disable-custom-colors'
This commit is contained in:
Matt Weiden
2020-04-26 22:52:57 -07:00
committed by GitHub
parent 3c191c3f93
commit 546e272a60
91 changed files with 801 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
client/src/actions/index.js
View File

@@ -9,12 +9,13 @@ import {
} from "../util/actionHelpers";
import { PromiseLimit } from "../util/promiseLimit";
import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
/*
return promise to fetch the OBS annotations we need to load. Omit anything
we don't need.
*/
function obsAnnotationFetchAndLoad(dispatch, schema) {
async function obsAnnotationFetchAndLoad(dispatch, schema) {
const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
const index = obsAnnotations.index ?? false;
const columns = (obsAnnotations.columns ?? []).filter(
@@ -23,21 +24,18 @@ function obsAnnotationFetchAndLoad(dispatch, schema) {
const plimit = new PromiseLimit(5);
return Promise.all(
columns.map((col) =>
plimit.add(() => {
const path = `annotations/obs?annotation-name=${encodeURIComponent(
col.name
)}`;
const url = `${globals.API.prefix}${globals.API.version}${path}`;
return doBinaryRequest(url).then((buffer) => {
const df = Universe.matrixFBSToDataframe(buffer);
dispatch({
type: "universe: column load success",
dim: "obsAnnotations",
dataframe: df,
});
});
})
columns.map(col =>
plimit.add(() =>
fetchBinary(`annotations/obs?annotation-name=${encodeURIComponent(col.name)}`)
.then(buffer => Universe.matrixFBSToDataframe(buffer))
.then(df =>
dispatch({
type: "universe: column load success",
dim: "obsAnnotations",
dataframe: df
})
)
)
)
);
}
@@ -45,31 +43,22 @@ function obsAnnotationFetchAndLoad(dispatch, schema) {
/*
return promise fetching VAR annotations we need to load. Only index is currently used.
*/
function varAnnotationFetchAndLoad(dispatch, schema) {
async function varAnnotationFetchAndLoad(dispatch, schema) {
const varAnnotations = schema?.schema?.annotations?.var ?? {};
const index = varAnnotations.index ?? false;
const names = index ? [index] : [];
return Promise.all(
names
.map((name) => {
const path = `annotations/var?annotation-name=${encodeURIComponent(
name
)}`;
const url = `${globals.API.prefix}${globals.API.version}${path}`;
return doBinaryRequest(url);
})
.map((rqst) =>
rqst.then((buffer) => Universe.matrixFBSToDataframe(buffer))
)
.map((resp) =>
resp.then((df) =>
names.map(name =>
fetchBinary(`annotations/var?annotation-name=${encodeURIComponent(name)}`)
.then(buffer => Universe.matrixFBSToDataframe(buffer))
.then(df =>
dispatch({
type: "universe: column load success",
dim: "varAnnotations",
dataframe: df,
dataframe: df
})
)
)
)
);
}
@@ -79,26 +68,35 @@ return promise fetching layout we need
function layoutFetchAndLoad(dispatch, schema) {
const embeddings = schema?.schema?.layout?.obs ?? [];
const embNames = embeddings.map((e) => e.name);
const baseURL = `${globals.API.prefix}${globals.API.version}layout/obs`;
const plimit = new PromiseLimit(5);
return Promise.all(
embNames.map((e) =>
plimit.add(() => {
const url = `${baseURL}?layout-name=${encodeURIComponent(e)}`;
return doBinaryRequest(url).then((buffer) =>
Universe.matrixFBSToDataframe(buffer)
);
})
embNames.map(e =>
plimit.add(() =>
fetchBinary(`layout/obs?layout-name=${encodeURIComponent(e)}`)
.then(buffer => Universe.matrixFBSToDataframe(buffer))
)
)
).then((dfs) => {
const df = Dataframe.Dataframe.empty().withColsFromAll(dfs);
).then(dfs =>
dispatch({
type: "universe: column load success",
dim: "obsLayout",
dataframe: df,
});
});
dataframe: Dataframe.Dataframe.empty().withColsFromAll(dfs)
})
);
}
/*
return promise fetching user-configured colors
*/
async function userColorsFetchAndLoad(dispatch) {
return fetchJson("colors")
.then(response =>
dispatch({
type: "universe: user color load success",
userColors: loadUserColorConfig(response)
})
);
}
/*
@@ -116,13 +114,10 @@ const doInitialDataLoad = () =>
/*
Step 1 - config & schema, all JSON
*/
const requestJson = ["config", "schema"]
.map((r) => `${globals.API.prefix}${globals.API.version}${r}`)
.map((url) => doJsonRequest(url));
const stepOneResults = await Promise.all(requestJson);
const requestJson = ["config", "schema"].map(fetchJson);
const [responseConfig, schema] = await Promise.all(requestJson);
/* set config defaults */
const config = { ...globals.configDefaults, ...stepOneResults[0].config };
const schema = stepOneResults[1];
const config = { ...globals.configDefaults, ...responseConfig.config };
const universe = Universe.createUniverseFromResponse(config, schema);
dispatch({
type: "universe exists, but loading is still in progress",
@@ -137,6 +132,7 @@ const doInitialDataLoad = () =>
Step 2 - load the minimum stuff required to display.
*/
await Promise.all([
userColorsFetchAndLoad(dispatch),
layoutFetchAndLoad(dispatch, schema),
varAnnotationFetchAndLoad(dispatch, schema),
]);
@@ -169,13 +165,6 @@ const setWorldToSelection = () => (dispatch, getState) => {
});
};
// Throws
const dispatchExpressionErrors = (dispatch, res) => {
const msg = `Unexpected HTTP response while fetching expression data ${res.status}, ${res.statusText}`;
dispatchNetworkErrorMessageToUser(msg);
throw new Error(msg);
};
/* double URI encode - needed for query-param filters */
function dubEncURIComponent(s) {
return encodeURIComponent(encodeURIComponent(s));
@@ -196,16 +185,11 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
/* helper for this function only */
const fetchData = async (geneNames) => {
const query = geneNames
.map(
(g) =>
`var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`
)
.map(g => `var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`)
.join("&");
const url = `${globals.API.prefix}${globals.API.version}data/var?${query}`;
return doBinaryRequest(url).then((buffer) =>
// TODO: why convert to an Object and not a Dataframe?
Universe.convertDataFBStoObject(universe, buffer)
);
// TODO: why convert to an Object and not a Dataframe?
return fetchBinary(`data/var?${query}`)
.then(buffer => Universe.convertDataFBStoObject(universe, buffer));
};
/* preload data already in cache */
@@ -367,7 +351,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
*/
const plimit = new PromiseLimit(5);
await Promise.all(
topNGenes.map((gene) =>
topNGenes.map(gene =>
plimit.add(() => _doRequestExpressionData(dispatch, getState, [gene]))
)
);
@@ -447,6 +431,14 @@ const saveObsAnnotations = () => async (dispatch, getState) => {
}
};
function fetchJson(pathAndQuery) {
return doJsonRequest(`${globals.API.prefix}${globals.API.version}${pathAndQuery}`);
}
function fetchBinary(pathAndQuery) {
return doBinaryRequest(`${globals.API.prefix}${globals.API.version}${pathAndQuery}`);
}
export default {
doInitialDataLoad,
requestDifferentialExpression,

2
client/src/components/categorical/category/index.js
View File

@@ -205,7 +205,7 @@ class Category extends React.Component {
return (
<div style={{ marginBottom: 10, marginTop: 4 }}>
<span style={{ fontWeight: 700 }}>
{truncatedString ? truncatedString : metadataField}
{truncatedString || metadataField}
</span>
: {schema.annotations.obsByName[metadataField].categories[0]}
</div>

24
client/src/reducers/colors.js
View File

@@ -27,6 +27,14 @@ const ColorsReducer = (
};
}
case "universe: user color load success": {
const { userColors } = action;
return {
...state,
userColors
};
}
case "reset World to eq Universe": {
/* need to rebuild colors as world may have changed, but don't switch modes */
const { world } = nextSharedState;
@@ -90,7 +98,7 @@ const ColorsReducer = (
case "color by categorical metadata":
case "color by continuous metadata": {
const { world } = prevSharedState;
const { world, colors } = prevSharedState;
/* toggle between this mode and reset */
const resetCurrent =
@@ -99,11 +107,7 @@ const ColorsReducer = (
const colorMode = !resetCurrent ? action.type : null;
const colorAccessor = !resetCurrent ? action.colorAccessor : null;
const { rgb, scale } = ColorHelpers.createColors(
world,
colorMode,
colorAccessor
);
const { rgb, scale } = ColorHelpers.createColors(world, colorMode, colorAccessor, colors.userColors);
return {
...state,
colorMode,
@@ -141,7 +145,7 @@ const ColorsReducer = (
case "annotation: delete label": {
const { world } = nextSharedState;
const { colorMode, colorAccessor } = state;
const { metadataField } = action;
const { metadataField, colors } = action;
if (
colorMode !== "color by categorical metadata" ||
colorAccessor !== metadataField
@@ -149,11 +153,7 @@ const ColorsReducer = (
return state;
/* else, we need to rebuild colors as labels have changed! */
const { rgb, scale } = ColorHelpers.createColors(
world,
colorMode,
colorAccessor
);
const { rgb, scale } = ColorHelpers.createColors(world, colorMode, colorAccessor);
return { ...state, rgb, scale };
}

1
client/src/reducers/undoableConfig.js
View File

@@ -12,6 +12,7 @@ const skipOnActions = new Set([
"interface reset started",
"initial data load start",
"universe: column load success",
"universe: user color load success",
"universe exists, but loading is still in progress",
"configuration load complete",
"increment graph render counter",

49
client/src/util/stateManager/colorHelpers.js
View File

@@ -11,13 +11,16 @@ import { range } from "../range";
/*
create new colors state object. Paramters:
- world - current world object
- mode - color-by mode. One of: null, "color by expression",
"color by continuous metadata", "color by categorical metadata"
-
- colorMode - color-by mode. One of {null, "color by expression", "color by continuous metadata",
"color by categorical metadata"}
- colorAccessor - the obs annotations used for color-by
*/
export function createColors(world, colorMode = null, colorAccessor = null) {
export function createColors(world, colorMode = null, colorAccessor = null, userColors = null) {
switch (colorMode) {
case "color by categorical metadata": {
if (userColors && colorAccessor in userColors) {
return createUserColors(world, colorAccessor, userColors);
}
return createColorsByCategoricalMetadata(world, colorAccessor);
}
case "color by continuous metadata": {
@@ -36,8 +39,29 @@ export function createColors(world, colorMode = null, colorAccessor = null) {
}
}
function createColorsByCategoricalMetadata(world, accessor) {
const { categories } = world.schema.annotations.obsByName[accessor];
export function loadUserColorConfig(userColors) {
const convertedUserColors = {};
Object.keys(userColors).forEach(category => {
const [colors, scaleMap] = Object.keys(userColors[category]).reduce((acc, label, i) => {
const color = parseRGB(userColors[category][label]);
acc[0][label] = color;
acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
return acc;
}, [{}, {}]);
const scale = i => scaleMap[i];
convertedUserColors[category] = { colors, scale };
});
return convertedUserColors;
}
function createUserColors(world, colorAccessor, userColors) {
const { colors, scale } = userColors[colorAccessor];
const rgb = createRgbArray(world, colors, colorAccessor);
return { rgb, scale };
}
function createColorsByCategoricalMetadata(world, colorAccessor) {
const { categories } = world.schema.annotations.obsByName[colorAccessor];
const scale = d3
.scaleSequential(interpolateRainbow)
@@ -49,14 +73,19 @@ function createColorsByCategoricalMetadata(world, accessor) {
return acc;
}, {});
const rgb = createRgbArray(world, colors, colorAccessor);
return { rgb, scale };
}
export function createRgbArray(world, colors, colorAccessor) {
const rgb = new Array(world.nObs);
const df = world.obsAnnotations;
const data = df.col(accessor).asArray();
const data = df.col(colorAccessor).asArray();
for (let i = 0, len = df.length; i < len; i += 1) {
const cat = data[i];
rgb[i] = colors[cat];
const label = data[i];
rgb[i] = colors[label];
}
return { rgb, scale };
return rgb;
}
function createColorsByContinuousMetadata(world, accessor) {

1
docs/_config.yml
View File

@@ -1,5 +1,6 @@
theme: jekyll-theme-minimal
show_downloads: false
url: "https://chanzuckerberg.github.io"
baseurl: "/cellxgene"
logo: cellxgene-logo.png

10
docs/_site/index.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Index | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="Index" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="http://localhost:4000/cellxgene/" />
<meta property="og:url" content="http://localhost:4000/cellxgene/" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebSite","headline":"Index","url":"http://localhost:4000/cellxgene/","name":"cellxgene","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","url":"https://chanzuckerberg.github.io/cellxgene/","headline":"Index","name":"cellxgene","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/annotations.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>annotations | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="annotations" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Creating annotations" />
<meta property="og:description" content="Creating annotations" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/annotations.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/annotations.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"Creating annotations","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"annotations","url":"http://localhost:4000/cellxgene/posts/annotations.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","headline":"annotations","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/contact.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Contact | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="Contact" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Contact" />
<meta property="og:description" content="Contact" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/contact.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/contact.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"Contact","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Contact","url":"http://localhost:4000/cellxgene/posts/contact.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","headline":"Contact","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/contribute.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Code of conduct | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="Code of conduct" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/contribute.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/contribute.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Code of conduct","url":"http://localhost:4000/cellxgene/posts/contribute.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","headline":"Code of conduct","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/demo-data.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>demo-data | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="demo-data" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Demo datasets" />
<meta property="og:description" content="Demo datasets" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/demo-data.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/demo-data.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"Demo datasets","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"demo-data","url":"http://localhost:4000/cellxgene/posts/demo-data.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","headline":"demo-data","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/gallery.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Gallery | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="Gallery" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/gallery.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/gallery.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Gallery","url":"http://localhost:4000/cellxgene/posts/gallery.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","headline":"Gallery","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/hosted.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Hosting cellxgene on the web | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="Hosting cellxgene on the web" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/hosted.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/hosted.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Hosting cellxgene on the web","url":"http://localhost:4000/cellxgene/posts/hosted.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","headline":"Hosting cellxgene on the web","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/install.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Install | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="Install" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/install.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/install.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Install","url":"http://localhost:4000/cellxgene/posts/install.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","headline":"Install","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/launch.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>demo-data | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="demo-data" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Demo datasets" />
<meta property="og:description" content="Demo datasets" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/launch.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/launch.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"Demo datasets","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"demo-data","url":"http://localhost:4000/cellxgene/posts/launch.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","headline":"demo-data","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/methods.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Methods | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="Methods" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
<meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/methods.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/methods.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Methods","url":"http://localhost:4000/cellxgene/posts/methods.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","headline":"Methods","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

64
docs/_site/posts/prepare.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>prepare | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="prepare" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Preparing your data" />
<meta property="og:description" content="Preparing your data" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/prepare.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/prepare.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"Preparing your data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"prepare","url":"http://localhost:4000/cellxgene/posts/prepare.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","headline":"prepare","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->
@@ -102,8 +102,8 @@
<ul>
<li>Expression values (raw or normalized) in <code class="language-plaintext highlighter-rouge">anndata.X</code></li>
<li>At least one embedding (e.g., tSNE, UMAP) in <code class="language-plaintext highlighter-rouge">anndata.obsm</code>, specified with the prefix <code class="language-plaintext highlighter-rouge">X_</code> (e.g., by default scanpy stores UMAP coordinates in <code class="language-plaintext highlighter-rouge">anndata.obsm['X_umap']</code>)</li>
<li>A unique identifier for every cell is available in an <code class="language-plaintext highlighter-rouge">anndata.obs</code> field (you can specify this with the <code class="language-plaintext highlighter-rouge">--obs-names</code> option)</li>
<li>A unique identifier for every gene is available in an <code class="language-plaintext highlighter-rouge">anndata.var</code> field (you can specify which field to use with the <code class="language-plaintext highlighter-rouge">--var-names</code> option)</li>
<li>A unique identifier is required for each cell, which by default will be pulled from the <code class="language-plaintext highlighter-rouge">obs</code> DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with <code class="language-plaintext highlighter-rouge">--obs-names</code></li>
<li>A unique identifier is required for each gene, which by default will be pulled from the <code class="language-plaintext highlighter-rouge">var</code> DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with <code class="language-plaintext highlighter-rouge">--var-names</code></li>
</ul>
<h4 id="what-about-r-objects-from-seurat--bioconductor">What about R objects from seurat / bioconductor!?</h4>
@@ -112,6 +112,32 @@
<h4 id="can-i-use-data-hosted-on-the-web-somewhere">Can I use data hosted on the web somewhere?</h4>
<p>Yes! You can launch from a URL instead of a filepath. The same data format requirements apply. Please see <a href="launch">here</a> for more details.</p>
<h1 id="data-format-options">Data format options</h1>
<h4 id="category-colors">Category colors</h4>
<p><code class="language-plaintext highlighter-rouge">cellxgene</code> will display <a href="https://github.com/chanzuckerberg/cellxgene/issues/1152#issue-564361541">scanpy-style color
information</a>
for category-label pairs. An example of this format is shown below:</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; category = "louvain"
&gt;&gt;&gt; # colors stored in adata.uns must be matplotlib-compatible color information
&gt;&gt;&gt; adata.uns[f"{category}_colors"]
array(['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#bcbd22'], dtype='&lt;U7')
&gt;&gt;&gt; # there must be a matching category in adata.obs
&gt;&gt;&gt; category in adata.obs
True
</code></pre></div></div>
<p>To test that you’ve done this properly, check that for your given <code class="language-plaintext highlighter-rouge">category</code> the number of colors match the number of category values and that the second command below results in a mapping from categories to colors.</p>
<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; len(adata.obs[category].cat.categories) == len(adata.uns[f"{category}_colors"])
True
&gt;&gt;&gt; dict(zip(adata.obs[category].cat.categories, adata.uns[f"{category}_colors"]))
{'CD4 T cells': '#1f77b4', 'CD14+ Monocytes': '#ff7f0e', 'B cells': '#2ca02c', 'CD8 T cells': '#d62728', 'NK cells': '#9467bd', 'FCGR3A+ Monocytes': '#8c564b', 'Dendritic cells': '#e377c2', 'Megakaryocytes': '#bcbd22'}
</code></pre></div></div>
<p>You can disable this feature using the <code class="language-plaintext highlighter-rouge">--disable-custom-colors</code> flag for <code class="language-plaintext highlighter-rouge">cellxgene launch</code>. cellxgene will then chose colors from its standard color palettes.</p>
<h1 id="using-cellxgene-prepare">Using <code class="language-plaintext highlighter-rouge">cellxgene prepare</code></h1>
<p>If your data is in a different format, and/or you still need to perform dimensionality reduction and/or clustering, <code class="language-plaintext highlighter-rouge">cellxgene</code> can do that for you with the <code class="language-plaintext highlighter-rouge">prepare</code> command.</p>
@@ -136,7 +162,7 @@
<p><code class="language-plaintext highlighter-rouge">cellxgene prepare</code> is not meant as a way to formally process or analyze your data. It’s simply a utility for quickly wrangling your data into cellxgene-compatible format and computing a “vanilla” embedding so you can try out <code class="language-plaintext highlighter-rouge">cellxgene</code> and get a general sense of a dataset.</p>
<h2 id="quickstart-for-cellxgene-prepare">Quickstart for <code class="language-plaintext highlighter-rouge">cellxgene prepare</code></h2>
<p>To add <code class="language-plaintext highlighter-rouge">cellxgene prepare</code> to your <a href="install">cellxgene installation</a>, run<br />
<p>To add <code class="language-plaintext highlighter-rouge">cellxgene prepare</code> to your <a href="install">cellxgene installation</a>, run
<code class="language-plaintext highlighter-rouge">pip install cellxgene[prepare]</code></p>
<p>Then run <code class="language-plaintext highlighter-rouge">prepare</code> on your data with:</p>
@@ -162,36 +188,36 @@
<p>Let’s look at what <code class="language-plaintext highlighter-rouge">prepare</code> is doing to our data, and how each step relates to the command above. You can see a walkthrough of what’s going on under the hood for this example in <a href="https://github.com/chanzuckerberg/cellxgene-vignettes/blob/master/dataset-processing/pbmc3k-prepare-example.ipynb">this notebook</a>.</p>
<p><strong>(A) - Compute quality control metrics and store this in our <code class="language-plaintext highlighter-rouge">AnnData</code> object for later inspection</strong> <br />
<strong>(B) - Normalize the expression matrix using a basic preprocessing recipe</strong> <br />
<strong>(auto) - Do some preprocessing to run PCA and compute the neighbor graph</strong><br />
<strong>(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later</strong><br />
<strong>(C) - Compute and store UMAP and tSNE embeddings</strong><br />
<p><strong>(A) - Compute quality control metrics and store this in our <code class="language-plaintext highlighter-rouge">AnnData</code> object for later inspection</strong>
<strong>(B) - Normalize the expression matrix using a basic preprocessing recipe</strong>
<strong>(auto) - Do some preprocessing to run PCA and compute the neighbor graph</strong>
<strong>(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later</strong>
<strong>(C) - Compute and store UMAP and tSNE embeddings</strong>
<strong>(D) - Write results to file</strong></p>
<h2 id="options-for-cellxgene-prepare">Options for cellxgene <code class="language-plaintext highlighter-rouge">prepare</code></h2>
<p><strong>For the most up-to-date and comprehensive list of options, run <code class="language-plaintext highlighter-rouge">cellxgene prepare --help</code></strong></p>
<p><code class="language-plaintext highlighter-rouge">--embedding</code> controls which dimensionality reduction algorithm is applies to your data.<br />
<p><code class="language-plaintext highlighter-rouge">--embedding</code> controls which dimensionality reduction algorithm is applies to your data.
Options are <code class="language-plaintext highlighter-rouge">umap</code> and/or <code class="language-plaintext highlighter-rouge">tsne</code>. Defaults to both.</p>
<p><code class="language-plaintext highlighter-rouge">--recipe</code> controls which normalization steps to apply to your data, based on one of the preprocessing <code class="language-plaintext highlighter-rouge">recipes</code> included with <code class="language-plaintext highlighter-rouge">scanpy</code>.
These recipes include steps like cell filtering and gene selection; see the <code class="language-plaintext highlighter-rouge">scanpy</code> <a href="https://scanpy.readthedocs.io/en/latest/api/index.html#recipes">documentation</a> for more details. <br />
These recipes include steps like cell filtering and gene selection; see the <code class="language-plaintext highlighter-rouge">scanpy</code> <a href="https://scanpy.readthedocs.io/en/latest/api/index.html#recipes">documentation</a> for more details.
Options are <code class="language-plaintext highlighter-rouge">none</code>, <code class="language-plaintext highlighter-rouge">seurat</code>, or <code class="language-plaintext highlighter-rouge">zheng17</code>. Defaults to <code class="language-plaintext highlighter-rouge">none</code>.</p>
<p><code class="language-plaintext highlighter-rouge">--sparse</code> is a flag determines whether to enforce a sparse matrix. For large datasets, <code class="language-plaintext highlighter-rouge">prepare</code> can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with &gt;100k cells). If you want <code class="language-plaintext highlighter-rouge">prepare</code> to run faster we recommend using the <code class="language-plaintext highlighter-rouge">sparse</code> option.<br />
<p><code class="language-plaintext highlighter-rouge">--sparse</code> is a flag determines whether to enforce a sparse matrix. For large datasets, <code class="language-plaintext highlighter-rouge">prepare</code> can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with &gt;100k cells). If you want <code class="language-plaintext highlighter-rouge">prepare</code> to run faster we recommend using the <code class="language-plaintext highlighter-rouge">sparse</code> option.
If this flag is not included, default is <code class="language-plaintext highlighter-rouge">False</code></p>
<p><code class="language-plaintext highlighter-rouge">--skip-qc</code> by default, <code class="language-plaintext highlighter-rouge">cellxgene prepare</code> will compute quality control metrics (saved to <code class="language-plaintext highlighter-rouge">anndata.obs</code> and <code class="language-plaintext highlighter-rouge">anndata.var</code>) as described in the <code class="language-plaintext highlighter-rouge">scanpy</code> <a href="https://scanpy.readthedocs.io/en/stable/api/scanpy.pp.calculate_qc_metrics.html">documentation</a>. Pass this flag if you would like to skip this step.</p>
<p><code class="language-plaintext highlighter-rouge">--make-obs-names-unique</code> / <code class="language-plaintext highlighter-rouge">--make-var-names-unique</code> determine whether to rename <code class="language-plaintext highlighter-rouge">obs</code> (cell) / <code class="language-plaintext highlighter-rouge">var</code> (gene) names, respectively, to be unique.<br />
<p><code class="language-plaintext highlighter-rouge">--make-obs-names-unique</code> / <code class="language-plaintext highlighter-rouge">--make-var-names-unique</code> determine whether to rename <code class="language-plaintext highlighter-rouge">obs</code> (cell) / <code class="language-plaintext highlighter-rouge">var</code> (gene) names, respectively, to be unique.
Default is <code class="language-plaintext highlighter-rouge">True</code>.</p>
<p><code class="language-plaintext highlighter-rouge">--set-obs-names</code> controls which field in <code class="language-plaintext highlighter-rouge">anndata.obs</code> (cell metadata) is used as the <em>index</em> for cells (e.g., a cell ID column).<br />
<p><code class="language-plaintext highlighter-rouge">--set-obs-names</code> controls which field in <code class="language-plaintext highlighter-rouge">anndata.obs</code> (cell metadata) is used as the <em>index</em> for cells (e.g., a cell ID column).
Default is <code class="language-plaintext highlighter-rouge">anndata.obs.names</code></p>
<p><code class="language-plaintext highlighter-rouge">--set-var-names</code> controls which field in <code class="language-plaintext highlighter-rouge">anndata.var</code> (gene metadata) is used as the <em>index</em> for genes.<br />
<p><code class="language-plaintext highlighter-rouge">--set-var-names</code> controls which field in <code class="language-plaintext highlighter-rouge">anndata.var</code> (gene metadata) is used as the <em>index</em> for genes.
Default is <code class="language-plaintext highlighter-rouge">anndata.var.names</code></p>
<p><code class="language-plaintext highlighter-rouge">--output</code> and <code class="language-plaintext highlighter-rouge">--overwrite</code> control where the processed data is saved.</p>

10
docs/_site/posts/roadmap.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>roadmap | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="roadmap" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Roadmap" />
<meta property="og:description" content="Roadmap" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/roadmap.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/roadmap.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"Roadmap","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"roadmap","url":"http://localhost:4000/cellxgene/posts/roadmap.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","headline":"roadmap","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

10
docs/_site/posts/troubleshooting.html
View File

@@ -5,21 +5,21 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Begin Jekyll SEO tag v2.5.0 -->
<!-- Begin Jekyll SEO tag v2.6.1 -->
<title>Troubleshooting | cellxgene</title>
<meta name="generator" content="Jekyll v3.8.5" />
<meta property="og:title" content="Troubleshooting" />
<meta property="og:locale" content="en_US" />
<meta name="description" content="Troubleshooting" />
<meta property="og:description" content="Troubleshooting" />
<link rel="canonical" href="http://localhost:4000/cellxgene/posts/troubleshooting.html" />
<meta property="og:url" content="http://localhost:4000/cellxgene/posts/troubleshooting.html" />
<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
<meta property="og:site_name" content="cellxgene" />
<script type="application/ld+json">
{"description":"Troubleshooting","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Troubleshooting","url":"http://localhost:4000/cellxgene/posts/troubleshooting.html","@context":"http://schema.org"}</script>
{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","headline":"Troubleshooting","@context":"https://schema.org"}</script>
<!-- End Jekyll SEO tag -->
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
<link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
<!--[if lt IE 9]>
<script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
<![endif]-->

70
docs/posts/prepare.md
View File

@@ -5,20 +5,48 @@ description: Preparing your data
---
# Data format requirements
If your data is in `h5ad` file (from the [`anndata`](https://anndata.readthedocs.io/en/latest/index.html) library) and meets the following requirements, you can go straight to `cellxgene launch`:
If your data is in `h5ad` file (from the [`anndata`](https://anndata.readthedocs.io/en/latest/index.html) library) and meets the following requirements, you can go straight to `cellxgene launch`:
- Expression values (raw or normalized) in `anndata.X`
- Expression values (raw or normalized) in `anndata.X`
- At least one embedding (e.g., tSNE, UMAP) in `anndata.obsm`, specified with the prefix `X_` (e.g., by default scanpy stores UMAP coordinates in `anndata.obsm['X_umap']`)
- A unique identifier for every cell is available in an `anndata.obs` field (you can specify this with the `--obs-names` option)
- A unique identifier for every gene is available in an `anndata.var` field (you can specify which field to use with the `--var-names` option)
- A unique identifier is required for each cell, which by default will be pulled from the `obs` DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with `--obs-names`
- A unique identifier is required for each gene, which by default will be pulled from the `var` DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with `--var-names`
#### What about R objects from seurat / bioconductor!?
#### What about R objects from seurat / bioconductor!?
We hear you! We'd also love to be able to ingest these files directly. This isn't currently possible, but in the meantime, you can use [sceasy](https://bioconda.github.io/recipes/r-sceasy/README.html) ([docs](https://cellgeni.readthedocs.io/en/latest/visualisations.html)) to convert to `h5ad`. Seurat also has some [handy conversion tools](https://satijalab.org/seurat/v3.0/conversion_vignette.html) that you can try out.
#### Can I use data hosted on the web somewhere?
#### Can I use data hosted on the web somewhere?
Yes! You can launch from a URL instead of a filepath. The same data format requirements apply. Please see [here](launch) for more details.
# Using `cellxgene prepare`
# Data format options
#### Category colors
`cellxgene` will display [scanpy-style color
information](https://github.com/chanzuckerberg/cellxgene/issues/1152#issue-564361541)
for category-label pairs. An example of this format is shown below:
```
>>> category = "louvain"
>>> # colors stored in adata.uns must be matplotlib-compatible color information
>>> adata.uns[f"{category}_colors"]
array(['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#bcbd22'], dtype='<U7')
>>> # there must be a matching category in adata.obs
>>> category in adata.obs
True
```
To test that you've done this properly, check that for your given `category` the number of colors match the number of category values and that the second command below results in a mapping from categories to colors.
```
>>> len(adata.obs[category].cat.categories) == len(adata.uns[f"{category}_colors"])
True
>>> dict(zip(adata.obs[category].cat.categories, adata.uns[f"{category}_colors"]))
{'CD4 T cells': '#1f77b4', 'CD14+ Monocytes': '#ff7f0e', 'B cells': '#2ca02c', 'CD8 T cells': '#d62728', 'NK cells': '#9467bd', 'FCGR3A+ Monocytes': '#8c564b', 'Dendritic cells': '#e377c2', 'Megakaryocytes': '#bcbd22'}
```
You can disable this feature using the `--disable-custom-colors` flag for `cellxgene launch`. cellxgene will then chose colors from its standard color palettes.
# Using `cellxgene prepare`
If your data is in a different format, and/or you still need to perform dimensionality reduction and/or clustering, `cellxgene` can do that for you with the `prepare` command.
@@ -30,7 +58,7 @@ If your data is in a different format, and/or you still need to perform dimensio
- Handle simple data normalization (from a [recipe](https://www.pydoc.io/pypi/scanpy-0.2.3/autoapi/preprocessing/recipes/index.html))
- Do basic preprocessing to run PCA and compute the neighbor graph
- Reduce dimensionality to generate embeddings
- Reduce dimensionality to generate embeddings
- Infer clusters
You can control which steps to run and their methods (when applicable), via the CLI. The CLI also includes options for computing QC metrics, enforcing matrix sparcity, specifying index names, and plotting output.
@@ -40,10 +68,10 @@ You can control which steps to run and their methods (when applicable), via the
`cellxgene prepare` is not meant as a way to formally process or analyze your data. It's simply a utility for quickly wrangling your data into cellxgene-compatible format and computing a "vanilla" embedding so you can try out `cellxgene` and get a general sense of a dataset.
## Quickstart for `cellxgene prepare`
To add `cellxgene prepare` to your [cellxgene installation](install), run
To add `cellxgene prepare` to your [cellxgene installation](install), run
`pip install cellxgene[prepare]`
Then run `prepare` on your data with:
Then run `prepare` on your data with:
```
cellxgene prepare dataset.h5ad --output=dataset-processed.h5ad
```
@@ -68,36 +96,36 @@ cellxgene prepare pbmc3k-raw.h5ad \
Let's look at what `prepare` is doing to our data, and how each step relates to the command above. You can see a walkthrough of what's going on under the hood for this example in [this notebook](https://github.com/chanzuckerberg/cellxgene-vignettes/blob/master/dataset-processing/pbmc3k-prepare-example.ipynb).
**(A) - Compute quality control metrics and store this in our `AnnData` object for later inspection**
**(B) - Normalize the expression matrix using a basic preprocessing recipe**
**(auto) - Do some preprocessing to run PCA and compute the neighbor graph**
**(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later**
**(C) - Compute and store UMAP and tSNE embeddings**
**(A) - Compute quality control metrics and store this in our `AnnData` object for later inspection**
**(B) - Normalize the expression matrix using a basic preprocessing recipe**
**(auto) - Do some preprocessing to run PCA and compute the neighbor graph**
**(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later**
**(C) - Compute and store UMAP and tSNE embeddings**
**(D) - Write results to file**
## Options for cellxgene `prepare`
**For the most up-to-date and comprehensive list of options, run `cellxgene prepare --help`**
`--embedding` controls which dimensionality reduction algorithm is applies to your data.
`--embedding` controls which dimensionality reduction algorithm is applies to your data.
Options are `umap` and/or `tsne`. Defaults to both.
`--recipe` controls which normalization steps to apply to your data, based on one of the preprocessing `recipes` included with `scanpy`.
These recipes include steps like cell filtering and gene selection; see the `scanpy` [documentation](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes) for more details.
These recipes include steps like cell filtering and gene selection; see the `scanpy` [documentation](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes) for more details.
Options are `none`, `seurat`, or `zheng17`. Defaults to `none`.
`--sparse` is a flag determines whether to enforce a sparse matrix. For large datasets, `prepare` can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want `prepare` to run faster we recommend using the `sparse` option.
`--sparse` is a flag determines whether to enforce a sparse matrix. For large datasets, `prepare` can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want `prepare` to run faster we recommend using the `sparse` option.
If this flag is not included, default is `False`
`--skip-qc` by default, `cellxgene prepare` will compute quality control metrics (saved to `anndata.obs` and `anndata.var`) as described in the `scanpy` [documentation](https://scanpy.readthedocs.io/en/stable/api/scanpy.pp.calculate_qc_metrics.html). Pass this flag if you would like to skip this step.
`--make-obs-names-unique` / `--make-var-names-unique` determine whether to rename `obs` (cell) / `var` (gene) names, respectively, to be unique.
`--make-obs-names-unique` / `--make-var-names-unique` determine whether to rename `obs` (cell) / `var` (gene) names, respectively, to be unique.
Default is `True`.
`--set-obs-names` controls which field in `anndata.obs` (cell metadata) is used as the _index_ for cells (e.g., a cell ID column).
`--set-obs-names` controls which field in `anndata.obs` (cell metadata) is used as the _index_ for cells (e.g., a cell ID column).
Default is `anndata.obs.names`
`--set-var-names` controls which field in `anndata.var` (gene metadata) is used as the _index_ for genes.
`--set-var-names` controls which field in `anndata.var` (gene metadata) is used as the _index_ for genes.
Default is `anndata.var.names`
`--output` and `--overwrite` control where the processed data is saved.

9
server/app/app.py
View File

@@ -207,6 +207,13 @@ class DataVarAPI(Resource):
return common_rest.data_var_get(request, data_adaptor)
class ColorsAPI(Resource):
@cache_control(public=True, max_age=ONE_WEEK)
@rest_get_data_adaptor
def get(self, data_adaptor):
return common_rest.colors_get(data_adaptor)
class DiffExpObsAPI(Resource):
@cache_control(no_store=True)
@rest_get_data_adaptor
@@ -235,6 +242,8 @@ def get_api_resources(bp_api):
api.add_resource(AnnotationsObsAPI, "/annotations/obs")
api.add_resource(AnnotationsVarAPI, "/annotations/var")
api.add_resource(DataVarAPI, "/data/var")
# Display routes
api.add_resource(ColorsAPI, "/colors")
# Computation routes
api.add_resource(DiffExpObsAPI, "/diffexp/obs")
api.add_resource(LayoutObsAPI, "/layout/obs")

11
server/cli/launch.py
View File

@@ -73,6 +73,13 @@ def config_args(func):
show_default=True,
help="Will not display categories with more distinct values than specified.",
)
@click.option(
"--disable-custom-colors",
is_flag=True,
default=False,
show_default=False,
help="Disable user-defined category-label colors drawn from source data file.",
)
@click.option(
"--diffexp-lfc-cutoff",
"-de",
@@ -146,7 +153,7 @@ def dataset_args(func):
"--about",
default=DEFAULT_CONFIG.single_dataset__about,
metavar="<URL>",
help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).",
help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
@@ -311,6 +318,7 @@ def launch(
obs_names,
var_names,
max_category_items,
disable_custom_colors,
diffexp_lfc_cutoff,
title,
scripts,
@@ -381,6 +389,7 @@ def launch(
user_annotations__ontology__enable=experimental_annotations_ontology,
user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
presentation__max_categories=max_category_items,
presentation__custom_colors=not disable_custom_colors,
embeddings__names=embedding,
embeddings__enable_reembedding=experimental_enable_reembedding,
diffexp__enable=not disable_diffexp,

3
server/cli/prepare.py
View File

@@ -73,7 +73,8 @@ def prepare(
(h5ad, loom, or a 10x directory), runs dimensionality reduction,
computes nearest neighbors, computes an embedding, performs clustering,
and saves the results. Includes additional options for naming annotations,
ensuring sparsity, and plotting results."""
ensuring sparsity, and plotting results.
"""
# collect slow imports here to make CLI startup more responsive
click.echo("[cellxgene] Starting CLI...")

3
server/common/app_config.py
View File

@@ -78,6 +78,7 @@ class AppConfig(object):
self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
self.presentation__max_categories = dc["presentation"]["max_categories"]
self.presentation__custom_colors = dc["presentation"]["custom_colors"]
self.embeddings__names = dc["embeddings"]["names"]
self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
@@ -275,6 +276,7 @@ class AppConfig(object):
def handle_presentation(self, context):
self.__check_attr("presentation__max_categories", int)
self.__check_attr("presentation__custom_colors", bool)
def handle_single_dataset(self, context):
self.__check_attr("single_dataset__datapath", (str, type(None)))
@@ -517,6 +519,7 @@ class AppConfig(object):
"annotations_cell_ontology_enabled": False,
"annotations_cell_ontology_obopath": None,
"annotations_cell_ontology_terms": None,
"custom_colors": self.presentation__custom_colors,
"diffexp-may-be-slow": False,
"about_legal_tos": self.server__about_legal_tos,
"about_legal_privacy": self.server__about_legal_privacy,

233
server/common/colors.py Normal file
View File

@@ -0,0 +1,233 @@
import re
from server.common.errors import ColorFormatException
HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
# https://www.w3.org/TR/css-color-4/#named-colors
CSS4_NAMED_COLORS = dict(
aliceblue="#f0f8ff",
antiquewhite="#faebd7",
aqua="#00ffff",
aquamarine="#7fffd4",
azure="#f0ffff",
beige="#f5f5dc",
bisque="#ffe4c4",
black="#000000",
blanchedalmond="#ffebcd",
blue="#0000ff",
blueviolet="#8a2be2",
brown="#a52a2a",
burlywood="#deb887",
cadetblue="#5f9ea0",
chartreuse="#7fff00",
chocolate="#d2691e",
coral="#ff7f50",
cornflowerblue="#6495ed",
cornsilk="#fff8dc",
crimson="#dc143c",
cyan="#00ffff",
darkblue="#00008b",
darkcyan="#008b8b",
darkgoldenrod="#b8860b",
darkgray="#a9a9a9",
darkgreen="#006400",
darkgrey="#a9a9a9",
darkkhaki="#bdb76b",
darkmagenta="#8b008b",
darkolivegreen="#556b2f",
darkorange="#ff8c00",
darkorchid="#9932cc",
darkred="#8b0000",
darksalmon="#e9967a",
darkseagreen="#8fbc8f",
darkslateblue="#483d8b",
darkslategray="#2f4f4f",
darkslategrey="#2f4f4f",
darkturquoise="#00ced1",
darkviolet="#9400d3",
deeppink="#ff1493",
deepskyblue="#00bfff",
dimgray="#696969",
dimgrey="#696969",
dodgerblue="#1e90ff",
firebrick="#b22222",
floralwhite="#fffaf0",
forestgreen="#228b22",
fuchsia="#ff00ff",
gainsboro="#dcdcdc",
ghostwhite="#f8f8ff",
gold="#ffd700",
goldenrod="#daa520",
gray="#808080",
green="#008000",
greenyellow="#adff2f",
grey="#808080",
honeydew="#f0fff0",
hotpink="#ff69b4",
indianred="#cd5c5c",
indigo="#4b0082",
ivory="#fffff0",
khaki="#f0e68c",
lavender="#e6e6fa",
lavenderblush="#fff0f5",
lawngreen="#7cfc00",
lemonchiffon="#fffacd",
lightblue="#add8e6",
lightcoral="#f08080",
lightcyan="#e0ffff",
lightgoldenrodyellow="#fafad2",
lightgray="#d3d3d3",
lightgreen="#90ee90",
lightgrey="#d3d3d3",
lightpink="#ffb6c1",
lightsalmon="#ffa07a",
lightseagreen="#20b2aa",
lightskyblue="#87cefa",
lightslategray="#778899",
lightslategrey="#778899",
lightsteelblue="#b0c4de",
lightyellow="#ffffe0",
lime="#00ff00",
limegreen="#32cd32",
linen="#faf0e6",
magenta="#ff00ff",
maroon="#800000",
mediumaquamarine="#66cdaa",
mediumblue="#0000cd",
mediumorchid="#ba55d3",
mediumpurple="#9370db",
mediumseagreen="#3cb371",
mediumslateblue="#7b68ee",
mediumspringgreen="#00fa9a",
mediumturquoise="#48d1cc",
mediumvioletred="#c71585",
midnightblue="#191970",
mintcream="#f5fffa",
mistyrose="#ffe4e1",
moccasin="#ffe4b5",
navajowhite="#ffdead",
navy="#000080",
oldlace="#fdf5e6",
olive="#808000",
olivedrab="#6b8e23",
orange="#ffa500",
orangered="#ff4500",
orchid="#da70d6",
palegoldenrod="#eee8aa",
palegreen="#98fb98",
paleturquoise="#afeeee",
palevioletred="#db7093",
papayawhip="#ffefd5",
peachpuff="#ffdab9",
peru="#cd853f",
pink="#ffc0cb",
plum="#dda0dd",
powderblue="#b0e0e6",
purple="#800080",
rebeccapurple="#663399",
red="#ff0000",
rosybrown="#bc8f8f",
royalblue="#4169e1",
saddlebrown="#8b4513",
salmon="#fa8072",
sandybrown="#f4a460",
seagreen="#2e8b57",
seashell="#fff5ee",
sienna="#a0522d",
silver="#c0c0c0",
skyblue="#87ceeb",
slateblue="#6a5acd",
slategray="#708090",
slategrey="#708090",
snow="#fffafa",
springgreen="#00ff7f",
steelblue="#4682b4",
tan="#d2b48c",
teal="#008080",
thistle="#d8bfd8",
tomato="#ff6347",
turquoise="#40e0d0",
violet="#ee82ee",
wheat="#f5deb3",
white="#ffffff",
whitesmoke="#f5f5f5",
yellow="#ffff00",
yellowgreen="#9acd32",
)
def convert_color_to_hex_format(unknown):
"""
Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet.
The function accepts for the following formats:
- A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html
- RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0]
- RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255]
- Hex triplet string, as in "#08c0ff"
:param unknown: color info of unknown format
:return: a hex triplet representing that color
"""
try:
if type(unknown) in (list, tuple) and len(unknown) == 3:
if all(0.0 <= ele <= 1.0 for ele in unknown):
tup = tuple(int(ele * 255) for ele in unknown)
elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown):
tup = tuple(unknown)
else:
raise ColorFormatException("Unknown color iterable format!")
return "#%02x%02x%02x" % tup
elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS:
return CSS4_NAMED_COLORS[unknown.lower()]
elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown):
return unknown.lower()
else:
raise ColorFormatException("Unknown color format type!")
except Exception as e:
raise ColorFormatException(e)
def convert_anndata_category_colors_to_cxg_category_colors(data):
"""
Convert color information from anndata files to the cellxgene color data format as described below:
{
"<category_name>": {
"<label_name>": "<color_hex_code>",
...
},
...
}
For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307.
For more on the anndata color data structure, see
https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178.
Handling of malformed data:
- For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot
convert to a hex triplet string, a ColorFormatException is raised
- No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no
adata.obs[f"{category}"] key
:param data: the anndata file
:return: cellxgene color data structure as described above
"""
cxg_colors = dict()
color_key_suffix = "_colors"
for uns_key in data.uns.keys():
# find uns array that describes colors for a category
if not uns_key.endswith(color_key_suffix):
continue
# check to see if we actually have observations for that category
category_name = uns_key[: -len(color_key_suffix)]
if category_name not in data.obs.keys():
continue
# create the cellxgene color entry for this category
cxg_colors[category_name] = dict(
zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
)
return cxg_colors

1
server/common/default_config.py
View File

@@ -19,6 +19,7 @@ server:
presentation:
max_categories: 1000
custom_colors: true
multi_dataset:
dataroot: null

6
server/common/errors.py
View File

@@ -84,3 +84,9 @@ class ComputeError(Exception):
"""
pass
class ColorFormatException(Exception):
"""Raised when color helper functions encounter an unknown color format"""
pass

16
server/common/rest.py
View File

@@ -1,9 +1,11 @@
import sys
from http import HTTPStatus
import copy
import logging
import sys
from http import HTTPStatus
from flask import make_response, jsonify, current_app, abort
from werkzeug.urls import url_unquote
from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
from server.common.errors import (
FilterError,
@@ -12,6 +14,7 @@ from server.common.errors import (
DisabledFeatureError,
ExceedsLimitError,
DatasetAccessError,
ColorFormatException,
)
import json
@@ -222,6 +225,15 @@ def data_var_get(request, data_adaptor):
return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
def colors_get(data_adaptor):
if not data_adaptor.config.presentation__custom_colors:
return make_response(jsonify({}), HTTPStatus.OK)
try:
return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK)
except ColorFormatException as e:
return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True)
def diffexp_obs_post(request, data_adaptor):
if not data_adaptor.config.diffexp__enable:
return abort(HTTPStatus.NOT_IMPLEMENTED)

86
server/converters/cxgtool.py
View File

@@ -4,28 +4,36 @@ into a cellxgene TileDB structure, aka a 'CXG'.
The organization of the TileDB structure is:
the.cxg TileDB Group
|-- obs TileDB array containing cell (row) attributes, one attribute per
| dataframe columm, shape (n_obs,)
|-- var TileDB array containing gene (column) attributes, with one attribute per
| dataframe column, shape (n_var,)
|-- X Main count matrix as a 2D TileDB array, single unnanmed numeric attribute
|-- emb TileDB group, storing optional embeddings (group may be empty)
| |-- <name1> TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
|-- cxg_group_metadata Empty array used only to stash metadata about the overall object.
the.cxg TileDB Group
├─ obs TileDB array containing cell (row) attributes, one attribute per
│ dataframe column, shape (n_obs,)
├─ var TileDB array containing gene (column) attributes, with one attribute per
│ dataframe column, shape (n_obs,)
├─ X Main count matrix as a 2D TileDB array, single unnamed numeric attribute
├─ emb TileDB group, storing optional embeddings (group may be empty)
│ └─ <name1> TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
└─ cxg_group_metadata Empty array used only to stash metadata about the overall object.
└─ cxg_category_colors CXG colors object as described below:
{
"<category_name>": {
"<label_name>": "<color_hex_code>",
...
},
...
}
...
All arrays are defined to have a uint32 domain, zero based. All X counds and embedding
All arrays are defined to have a uint32 domain, zero based. All X counts and embedding
coordinates are coerced to float32, which is ample precision for visualization purposes.
Dataframe (metadata) types are generally preserved, or where that is not possible,
converted to somemthing with equal representative value in the cellxgene application
converted to something with equal representative value in the cellxgene application
(eg, categorical types are converted to string, bools to uint8, etc).
The following objects are also decorated with auxilliary metadata using TileDB
The following objects are also decorated with auxiliary metadata using TileDB
array metadata:
* cxg_group_metadata: minimally, will contain a 'cxg_version' field, which
is a semver string identifing the version number of the CXG layout.
is a semver string identifying the version number of the CXG layout.
It may also contain 'cxg_parameters', a JSON-encoded parameter list
describing CXG-wide dataset parameters.
@@ -40,6 +48,14 @@ including the global data layout, spatial tile size, and the like. The CXG is
self-describing in these areas, and the actual values (eg, tile size) are empirically
derived from benchmarking. They may change in the future.
cxgtool.py will extract color information stored in arrays in the 'uns' anndata
property with the key "{category_name}_colors". For this to work, the following
command must result in a mapping from category names to matplotlib-compatible colors:
```
dict(zip(adata.obs[cat].cat.categories, adata.uns[f"{cat}_colors"]))
```
---
TODO/ISSUES:
@@ -55,10 +71,16 @@ import numpy as np
from os.path import splitext, basename
import json
from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from server.common.errors import ColorFormatException
# the CXG container version number. Must be a semver string.
CXG_VERSION = "0.1"
# log_level must have a default
log_level = 3
def log(level, *args):
global log_level
@@ -72,6 +94,12 @@ def main():
parser.add_argument(
"--backed", action="store_true", help="loaded in file backed mode. Will be slower, but use less memory."
)
parser.add_argument(
"--disable-custom-colors",
action="store_true",
default=False,
help="Do not extract scanpy-compatible category colors from h5ad file.",
)
parser.add_argument(
"--obs-names", help="Name of annotation to use for observations. If not specified, will use the obs index."
)
@@ -99,12 +127,20 @@ def main():
container = out if splitext(out)[1] == ".cxg" else out + ".cxg"
title = args.title if args.title is not None else basefname
write_cxg(adata, container, title, var_names=args.var_names, obs_names=args.obs_names, about=args.about)
write_cxg(
adata,
container,
title,
var_names=args.var_names,
obs_names=args.obs_names,
about=args.about,
extract_colors=not args.disable_custom_colors,
)
log(1, "done")
def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None):
def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None, extract_colors=False):
if not adata.var.index.is_unique:
raise ValueError("Variable index is not unique - unable to convert.")
if not adata.obs.index.is_unique:
@@ -129,7 +165,19 @@ def write_cxg(adata, container, title, var_names=None, obs_names=None, about=Non
log(1, f"\t...group created, with name {container}")
# dataset metadata
save_metadata(container, {"title": title, "about": about})
metadata_dict = dict(cxg_version=CXG_VERSION, cxg_properties=json.dumps({"title": title, "about": about}))
if extract_colors:
try:
metadata_dict["cxg_category_colors"] = json.dumps(
convert_anndata_category_colors_to_cxg_category_colors(adata)
)
except ColorFormatException:
log(
0,
"Warning: failed to extract colors from h5ad file! "
"Fix the h5ad file or rerun with --disable-custom-colors. See help for details.",
)
save_metadata(container, metadata_dict)
log(1, "\t...dataset metadata saved")
# var/gene dataframe
@@ -392,7 +440,7 @@ def save_X(container, adata, ctx):
tiledb.consolidate(X_name, ctx=ctx)
def save_metadata(container, metadata):
def save_metadata(container, metadata_dict):
"""
Save all dataset-wide metadata. This includes:
* CXG version
@@ -407,8 +455,8 @@ def save_metadata(container, metadata):
with tiledb.from_numpy(a_name, np.zeros((1,))) as A:
pass
with tiledb.DenseArray(a_name, mode="w") as A:
A.meta["cxg_version"] = CXG_VERSION
A.meta["cxg_properties"] = json.dumps(metadata)
for k, v in metadata_dict.items():
A.meta[k] = v
def sanitize_keys(keys):

4
server/data_anndata/anndata_adaptor.py
View File

@@ -12,6 +12,7 @@ from server_timing import Timing as ServerTiming
from server.data_common.data_adaptor import DataAdaptor
from server.data_common.fbs.matrix import encode_matrix_fbs
from server.common.utils import series_to_schema
from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
from server.common.constants import Axis, MAX_LAYOUTS
from server.common.errors import PrepareError, DatasetAccessError, FilterError
from server.compute.scanpy import scanpy_umap
@@ -333,6 +334,9 @@ class AnndataAdaptor(DataAdaptor):
lfc_cutoff = self.config.diffexp__lfc_cutoff
return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
def get_colors(self):
return convert_anndata_category_colors_to_cxg_category_colors(self.data)
def get_X_array(self, obs_mask=None, var_mask=None):
if obs_mask is None:
obs_mask = slice(None)

4
server/data_common/data_adaptor.py
View File

@@ -83,6 +83,10 @@ class DataAdaptor(metaclass=ABCMeta):
def query_obs_array(self, term_var):
pass
@abstractmethod
def get_colors(self):
pass
@abstractmethod
def get_obs_index(self):
pass

4
server/data_cxg/cxg_adaptor.py
View File

@@ -194,6 +194,10 @@ class CxgAdaptor(DataAdaptor):
lfc_cutoff = self.config.diffexp__lfc_cutoff
return diffexp_cxg.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
def get_colors(self):
meta = self.open_array("cxg_group_metadata").meta
return json.loads(meta["cxg_category_colors"]) if "cxg_category_colors" in meta else dict()
def get_X_array(self, obs_mask=None, var_mask=None):
obs_items = pack_selector_from_mask(obs_mask)
var_items = pack_selector_from_mask(var_mask)

2
server/eb/app.py
View File

@@ -68,7 +68,7 @@ class WSGIServer(Server):
if len(style_hashes) > 0:
csp["style-src"] = style_hashes
Talisman(app, force_https=app_config.server__force_https, frame_options='DENY', content_security_policy=csp)
Talisman(app, force_https=app_config.server__force_https, frame_options="DENY", content_security_policy=csp)
@staticmethod
def load_csp_hashes(app):

27
server/test/__init__.py
View File

@@ -1,6 +1,6 @@
import shutil
import tempfile
from os import path
from os import path, popen
import pandas as pd
@@ -11,11 +11,14 @@ from server.data_common.fbs.matrix import encode_matrix_fbs
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
tmp_dir = tempfile.mkdtemp()
annotations_file = path.join(tmp_dir, "test_annotations.csv")
if annotations_fixture:
shutil.copyfile(f"test/test_datasets/pbmc3k-annotations.csv", annotations_file)
shutil.copyfile(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-annotations.csv", annotations_file)
args = {
"embeddings__names": ["umap"],
"presentation__max_categories": 100,
@@ -24,7 +27,7 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
"diffexp__lfc_cutoff": 0.01,
}
fname = {
MatrixDataType.H5AD: "../example-dataset/pbmc3k.h5ad",
MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg",
}[ext]
data_locator = DataLocator(fname)
@@ -53,3 +56,21 @@ def skip_if(condition, reason: str):
return wraps
return decorator
def app_config(data_locator, backed=False):
args = {
"embeddings__names": ["umap", "tsne", "pca"],
"presentation__max_categories": 100,
"single_dataset__obs_names": None,
"single_dataset__var_names": None,
"diffexp__lfc_cutoff": 0.01,
"adaptor__anndata_adaptor__backed": backed,
"single_dataset__datapath": data_locator,
"limits__diffexp_cellcount_max": None,
"limits__column_request_max": None,
}
config = AppConfig()
config.update(**args)
config.complete_config()
return config

37
server/test/test_anndata_adaptor.py
View File

@@ -10,10 +10,11 @@ from parameterized import parameterized_class
import numpy as np
import pandas as pd
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.errors import FilterError
from server.common.data_locator import DataLocator
from server.common.app_config import AppConfig
from server.common.errors import FilterError
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.test import PROJECT_ROOT, app_config
from server.test.test_datasets.fixtures import pbmc3k_colors
"""
Test the anndata adaptor using the pbmc3k data set.
@@ -23,30 +24,17 @@ Test the anndata adaptor using the pbmc3k data set.
@parameterized_class(
("data_locator", "backed"),
[
("../example-dataset/pbmc3k.h5ad", False),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
("../example-dataset/pbmc3k.h5ad", True),
("test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
],
)
class AdaptorTest(unittest.TestCase):
def setUp(self):
args = {
"embeddings__names": ["umap", "tsne", "pca"],
"presentation__max_categories": 100,
"single_dataset__obs_names": None,
"single_dataset__var_names": None,
"diffexp__lfc_cutoff": 0.01,
"adaptor__anndata_adaptor__backed": self.backed,
"single_dataset__datapath": self.data_locator,
"limits__diffexp_cellcount_max": None,
"limits__column_request_max": None,
}
config = AppConfig()
config.update(**args)
config.complete_config()
config = app_config(self.data_locator, self.backed)
self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
def test_init(self):
@@ -92,6 +80,9 @@ class AdaptorTest(unittest.TestCase):
self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
def test_get_colors(self):
self.assertEqual(self.data.get_colors(), pbmc3k_colors)
def test_get_schema(self):
with open(path.join(path.dirname(__file__), "schema.json")) as fh:
schema = json.load(fh)

3
server/test/test_anndata_adaptor_data_load.py
View File

@@ -4,6 +4,7 @@ import json
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.data_locator import DataLocator
from server.common.app_config import AppConfig
from server.test import PROJECT_ROOT
class DataLoadAdaptorTest(unittest.TestCase):
@@ -12,7 +13,7 @@ class DataLoadAdaptorTest(unittest.TestCase):
"""
def setUp(self):
self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
config = AppConfig()
config.update(single_dataset__datapath=self.data_file.path)
config.complete_config()

17
server/test/test_api.py
View File

@@ -8,8 +8,10 @@ import pandas as pd
import requests
import server.test.decode_fbs as decode_fbs
from server.test import data_with_tmp_annotations, make_fbs
from server.data_common.matrix_loader import MatrixDataType
from server.test import data_with_tmp_annotations, make_fbs, PROJECT_ROOT
from server.test.test_datasets.fixtures import pbmc3k_colors
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
@@ -255,6 +257,15 @@ class EndPoints(object):
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
def test_colors(self):
endpoint = "colors"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data, pbmc3k_colors)
def test_static(self):
endpoint = "static"
file = "assets/favicon.ico"
@@ -349,7 +360,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
"cellxgene",
"--no-upgrade-check",
"launch",
"../example-dataset/pbmc3k.h5ad",
f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
"--disable-annotations",
"--verbose",
"--port",
@@ -383,7 +394,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
"cellxgene",
"--no-upgrade-check",
"launch",
"test/test_datasets/pbmc3k.cxg",
f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg",
"--disable-annotations",
"--verbose",
"--port",

40
server/test/test_colors.py Normal file
View File

@@ -0,0 +1,40 @@
import unittest
import anndata
from server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors
from server.common.errors import ColorFormatException
from server.test import PROJECT_ROOT
from server.test.test_datasets.fixtures import pbmc3k_colors
class ColorsTest(unittest.TestCase):
""" Test color helper functions """
def test_convert_color_to_hex_format(self):
self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")
self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3")
self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3")
self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3")
self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3")
self.assertEqual(
convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3"
)
for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]:
with self.assertRaises(ColorFormatException):
convert_color_to_hex_format(bad_input)
def test_anndata_colors_to_cxg_colors(self):
# test standard behavior
adata = self._get_h5ad()
self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors)
# test that invalid color formats raise an exception
adata.uns["louvain_colors"][0] = "#NOTCOOL"
with self.assertRaises(ColorFormatException):
convert_anndata_category_colors_to_cxg_category_colors(adata)
# test that colors without a matching obs category are skipped
adata = self._get_h5ad()
del adata.obs["louvain"]
self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {})
def _get_h5ad(self):
return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")

16
server/test/test_cxg_adaptor.py Normal file
View File

@@ -0,0 +1,16 @@
import unittest
from server.common.data_locator import DataLocator
from server.data_cxg.cxg_adaptor import CxgAdaptor
from server.test import PROJECT_ROOT, app_config
from server.test.test_datasets.fixtures import pbmc3k_colors
class TestCxgAdaptor(unittest.TestCase):
def setUp(self):
data_locator = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg"
config = app_config(data_locator)
self.data = CxgAdaptor(DataLocator(data_locator), config)
def test_get_colors(self):
self.assertEqual(self.data.get_colors(), pbmc3k_colors)

40
server/test/test_cxgtool.py Normal file
View File

@@ -0,0 +1,40 @@
import random
import shutil
import string
import unittest
import anndata
from server.common.data_locator import DataLocator
from server.converters.cxgtool import write_cxg
from server.data_cxg.cxg_adaptor import CxgAdaptor
from server.test import PROJECT_ROOT, app_config
from server.test.test_datasets.fixtures import pbmc3k_colors
class TestCxgAdaptor(unittest.TestCase):
def setUp(self) -> None:
self.fixtures = []
def tearDown(self) -> None:
try:
for data_locator in self.fixtures:
print("REMOVING ", data_locator)
shutil.rmtree(data_locator)
except FileNotFoundError:
pass
def test_cxg_category_colors(self):
data = self.convert_pbmc3k(extract_colors=True)
self.assertEqual(data.get_colors(), pbmc3k_colors)
data = self.convert_pbmc3k(extract_colors=False)
self.assertEqual(data.get_colors(), {})
def convert_pbmc3k(self, **kwargs):
random_string = "".join(random.choice(string.ascii_letters) for _ in range(8))
data_locator = f"/tmp/test_{random_string}.cxg"
self.fixtures.append(data_locator)
source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
write_cxg(adata=source_h5ad, container=data_locator, title="pbmc3k", **kwargs)
config = app_config(data_locator)
return CxgAdaptor(DataLocator(data_locator), config)

12
server/test/test_datasets/fixtures.py Normal file
View File

@@ -0,0 +1,12 @@
pbmc3k_colors = {
"louvain": {
"B cells": "#2ca02c",
"CD14+ Monocytes": "#ff7f0e",
"CD4 T cells": "#1f77b4",
"CD8 T cells": "#d62728",
"Dendritic cells": "#e377c2",
"FCGR3A+ Monocytes": "#8c564b",
"Megakaryocytes": "#bcbd22",
"NK cells": "#9467bd",
}
}

BIN
server/test/test_datasets/pbmc3k.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__fragment_metadata.tdb
View File

Binary file not shown.

BIN
server/test/test_datasets/pbmc3k.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__attr.tdb → server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__attr.tdb Normal file → Executable file
View File

Binary file not shown.

BIN
server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__fragment_metadata.tdb Executable file
View File

Binary file not shown.

BIN
server/test/test_datasets/pbmc3k.cxg/X/__array_schema.tdb Normal file → Executable file
View File

Binary file not shown.

0
server/test/test_datasets/pbmc3k.cxg/X/__lock.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/__tiledb_group.tdb Normal file → Executable file
View File

BIN
server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__attr.tdb Executable file
View File

Binary file not shown.

BIN
server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__fragment_metadata.tdb Executable file
View File

Binary file not shown.

BIN
server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__array_schema.tdb Executable file
View File

Binary file not shown.

0
server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__lock.tdb Executable file
View File

BIN
server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__meta/__1587182255768_1587182255768_fa7617ae99f843929911e4bc7b03e3db Executable file
View File

Binary file not shown.

0
server/test/test_datasets/pbmc3k.cxg/emb/__tiledb_group.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__attr.tdb → server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__attr.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__fragment_metadata.tdb → server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__fragment_metadata.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__array_schema.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__lock.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__attr.tdb → server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__attr.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__fragment_metadata.tdb → server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__fragment_metadata.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/pca/__array_schema.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/pca/__lock.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__attr.tdb → server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__attr.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__fragment_metadata.tdb → server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__fragment_metadata.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/tsne/__array_schema.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/tsne/__lock.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__attr.tdb → server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__attr.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__fragment_metadata.tdb → server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__fragment_metadata.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/umap/__array_schema.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/emb/umap/__lock.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/__fragment_metadata.tdb → server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/__fragment_metadata.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain.tdb → server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain_var.tdb → server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain_var.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_counts.tdb → server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_counts.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_genes.tdb → server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_genes.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0.tdb → server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0_var.tdb → server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0_var.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/percent_mito.tdb → server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/percent_mito.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__array_schema.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/obs/__lock.tdb Normal file → Executable file
View File

BIN
server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1576858534024_1576858534024_e856aef5b0e244b1a4b7dde70cd864d0
View File

Binary file not shown.

BIN
server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1587182255787_1587182255787_e19a340576a340db85c37b61b83dbe57 Executable file
View File

Binary file not shown.

0
server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/__fragment_metadata.tdb → server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/__fragment_metadata.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/n_cells.tdb → server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/n_cells.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0.tdb → server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0_var.tdb → server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0_var.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/var/__array_schema.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/var/__lock.tdb Normal file → Executable file
View File

0
server/test/test_datasets/pbmc3k.cxg/var/__meta/__1576858533966_1576858533966_1362646d804b4982b35052a367633436 → server/test/test_datasets/pbmc3k.cxg/var/__meta/__1587182255771_1587182255771_5a3a50f0b360403eab0e427c429f574e Normal file → Executable file
View File

8
server/test/test_diffexp.py
View File

@@ -5,6 +5,8 @@ import server.compute.diffexp_cxg as diffexp_cxg
import server.compute.diffexp_generic as diffexp_generic
import numpy as np
from server.test import PROJECT_ROOT
class DiffExpTest(unittest.TestCase):
"""Tests the diffexp returns the expected results for one test case, using different
@@ -50,7 +52,7 @@ class DiffExpTest(unittest.TestCase):
def test_anndata_default(self):
"""Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
adaptor = self.load_dataset("../example-dataset/pbmc3k.h5ad")
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
@@ -58,7 +60,7 @@ class DiffExpTest(unittest.TestCase):
def test_cxg_default(self):
"""Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)"""
adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg")
adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
@@ -72,7 +74,7 @@ class DiffExpTest(unittest.TestCase):
def test_cxg_generic(self):
"""Test a cxg adaptor with the generic adaptor"""
adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg")
adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
# run it directly

4
server/test/test_matrixcache.py
View File

@@ -7,13 +7,15 @@ import shutil
import os
import time
from server.test import PROJECT_ROOT
class MatrixCacheTest(unittest.TestCase):
def setup(self):
pass
def make_temporay_datasets(self, dirname, num):
source = "test/test_datasets/pbmc3k.cxg"
source = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg"
for i in range(num):
target = os.path.join(dirname, str(i) + ".cxg")
shutil.copytree(source, target)

27
server/test/test_nan_anndata_adaptor.py
View File

@@ -8,39 +8,22 @@ import server.test.decode_fbs as decode_fbs
from server.data_anndata.anndata_adaptor import AnndataAdaptor
from server.common.errors import FilterError
from server.common.data_locator import DataLocator
from server.common.app_config import AppConfig
from server.test import PROJECT_ROOT, app_config
class NaNTest(unittest.TestCase):
def setUp(self):
self.args = {
"embeddings__names": ["umap"],
"presentation__max_categories": 100,
"single_dataset__obs_names": None,
"single_dataset__var_names": None,
"diffexp__lfc_cutoff": 0.01,
"limits__diffexp_cellcount_max": None,
"limits__column_request_max": None,
}
config = AppConfig()
config.update(**self.args)
locator = DataLocator("test/test_datasets/nan.h5ad")
config.update(single_dataset__datapath=locator.path)
config.complete_config()
self.data_locator = DataLocator(f"{PROJECT_ROOT}/server/test/test_datasets/nan.h5ad")
self.config = app_config(self.data_locator.path)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
self.data = AnndataAdaptor(locator, config)
self.data = AnndataAdaptor(self.data_locator, self.config)
self.data._create_schema()
def test_load(self):
with self.assertWarns(UserWarning):
config = AppConfig()
config.update(**self.args)
locator = DataLocator("test/test_datasets/nan.h5ad")
config.update(single_dataset__datapath=locator.path)
config.complete_config()
self.data = AnndataAdaptor(locator, config)
self.data = AnndataAdaptor(self.data_locator, self.config)
def test_init(self):
self.assertEqual(self.data.cell_count, 100)