diff --git a/client/src/actions/index.js b/client/src/actions/index.js index 005eb0b5..b3aa4140 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -9,12 +9,13 @@ import { } from "../util/actionHelpers"; import { PromiseLimit } from "../util/promiseLimit"; import { requestReembed, reembedResetWorldToUniverse } from "./reembed"; +import { loadUserColorConfig } from "../util/stateManager/colorHelpers"; /* return promise to fetch the OBS annotations we need to load. Omit anything we don't need. */ -function obsAnnotationFetchAndLoad(dispatch, schema) { +async function obsAnnotationFetchAndLoad(dispatch, schema) { const obsAnnotations = schema?.schema?.annotations?.obs ?? {}; const index = obsAnnotations.index ?? false; const columns = (obsAnnotations.columns ?? []).filter( @@ -23,21 +24,18 @@ function obsAnnotationFetchAndLoad(dispatch, schema) { const plimit = new PromiseLimit(5); return Promise.all( - columns.map((col) => - plimit.add(() => { - const path = `annotations/obs?annotation-name=${encodeURIComponent( - col.name - )}`; - const url = `${globals.API.prefix}${globals.API.version}${path}`; - return doBinaryRequest(url).then((buffer) => { - const df = Universe.matrixFBSToDataframe(buffer); - dispatch({ - type: "universe: column load success", - dim: "obsAnnotations", - dataframe: df, - }); - }); - }) + columns.map(col => + plimit.add(() => + fetchBinary(`annotations/obs?annotation-name=${encodeURIComponent(col.name)}`) + .then(buffer => Universe.matrixFBSToDataframe(buffer)) + .then(df => + dispatch({ + type: "universe: column load success", + dim: "obsAnnotations", + dataframe: df + }) + ) + ) ) ); } @@ -45,31 +43,22 @@ function obsAnnotationFetchAndLoad(dispatch, schema) { /* return promise fetching VAR annotations we need to load. Only index is currently used. */ -function varAnnotationFetchAndLoad(dispatch, schema) { +async function varAnnotationFetchAndLoad(dispatch, schema) { const varAnnotations = schema?.schema?.annotations?.var ?? {}; const index = varAnnotations.index ?? false; const names = index ? [index] : []; return Promise.all( - names - .map((name) => { - const path = `annotations/var?annotation-name=${encodeURIComponent( - name - )}`; - const url = `${globals.API.prefix}${globals.API.version}${path}`; - return doBinaryRequest(url); - }) - .map((rqst) => - rqst.then((buffer) => Universe.matrixFBSToDataframe(buffer)) - ) - .map((resp) => - resp.then((df) => + names.map(name => + fetchBinary(`annotations/var?annotation-name=${encodeURIComponent(name)}`) + .then(buffer => Universe.matrixFBSToDataframe(buffer)) + .then(df => dispatch({ type: "universe: column load success", dim: "varAnnotations", - dataframe: df, + dataframe: df }) ) - ) + ) ); } @@ -79,26 +68,35 @@ return promise fetching layout we need function layoutFetchAndLoad(dispatch, schema) { const embeddings = schema?.schema?.layout?.obs ?? []; const embNames = embeddings.map((e) => e.name); - const baseURL = `${globals.API.prefix}${globals.API.version}layout/obs`; const plimit = new PromiseLimit(5); return Promise.all( - embNames.map((e) => - plimit.add(() => { - const url = `${baseURL}?layout-name=${encodeURIComponent(e)}`; - return doBinaryRequest(url).then((buffer) => - Universe.matrixFBSToDataframe(buffer) - ); - }) + embNames.map(e => + plimit.add(() => + fetchBinary(`layout/obs?layout-name=${encodeURIComponent(e)}`) + .then(buffer => Universe.matrixFBSToDataframe(buffer)) + ) ) - ).then((dfs) => { - const df = Dataframe.Dataframe.empty().withColsFromAll(dfs); + ).then(dfs => dispatch({ type: "universe: column load success", dim: "obsLayout", - dataframe: df, - }); - }); + dataframe: Dataframe.Dataframe.empty().withColsFromAll(dfs) + }) + ); +} + +/* +return promise fetching user-configured colors +*/ +async function userColorsFetchAndLoad(dispatch) { + return fetchJson("colors") + .then(response => + dispatch({ + type: "universe: user color load success", + userColors: loadUserColorConfig(response) + }) + ); } /* @@ -116,13 +114,10 @@ const doInitialDataLoad = () => /* Step 1 - config & schema, all JSON */ - const requestJson = ["config", "schema"] - .map((r) => `${globals.API.prefix}${globals.API.version}${r}`) - .map((url) => doJsonRequest(url)); - const stepOneResults = await Promise.all(requestJson); + const requestJson = ["config", "schema"].map(fetchJson); + const [responseConfig, schema] = await Promise.all(requestJson); /* set config defaults */ - const config = { ...globals.configDefaults, ...stepOneResults[0].config }; - const schema = stepOneResults[1]; + const config = { ...globals.configDefaults, ...responseConfig.config }; const universe = Universe.createUniverseFromResponse(config, schema); dispatch({ type: "universe exists, but loading is still in progress", @@ -137,6 +132,7 @@ const doInitialDataLoad = () => Step 2 - load the minimum stuff required to display. */ await Promise.all([ + userColorsFetchAndLoad(dispatch), layoutFetchAndLoad(dispatch, schema), varAnnotationFetchAndLoad(dispatch, schema), ]); @@ -169,13 +165,6 @@ const setWorldToSelection = () => (dispatch, getState) => { }); }; -// Throws -const dispatchExpressionErrors = (dispatch, res) => { - const msg = `Unexpected HTTP response while fetching expression data ${res.status}, ${res.statusText}`; - dispatchNetworkErrorMessageToUser(msg); - throw new Error(msg); -}; - /* double URI encode - needed for query-param filters */ function dubEncURIComponent(s) { return encodeURIComponent(encodeURIComponent(s)); @@ -196,16 +185,11 @@ async function _doRequestExpressionData(dispatch, getState, genes) { /* helper for this function only */ const fetchData = async (geneNames) => { const query = geneNames - .map( - (g) => - `var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}` - ) + .map(g => `var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`) .join("&"); - const url = `${globals.API.prefix}${globals.API.version}data/var?${query}`; - return doBinaryRequest(url).then((buffer) => - // TODO: why convert to an Object and not a Dataframe? - Universe.convertDataFBStoObject(universe, buffer) - ); + // TODO: why convert to an Object and not a Dataframe? + return fetchBinary(`data/var?${query}`) + .then(buffer => Universe.convertDataFBStoObject(universe, buffer)); }; /* preload data already in cache */ @@ -367,7 +351,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async ( */ const plimit = new PromiseLimit(5); await Promise.all( - topNGenes.map((gene) => + topNGenes.map(gene => plimit.add(() => _doRequestExpressionData(dispatch, getState, [gene])) ) ); @@ -447,6 +431,14 @@ const saveObsAnnotations = () => async (dispatch, getState) => { } }; +function fetchJson(pathAndQuery) { + return doJsonRequest(`${globals.API.prefix}${globals.API.version}${pathAndQuery}`); +} + +function fetchBinary(pathAndQuery) { + return doBinaryRequest(`${globals.API.prefix}${globals.API.version}${pathAndQuery}`); +} + export default { doInitialDataLoad, requestDifferentialExpression, diff --git a/client/src/components/categorical/category/index.js b/client/src/components/categorical/category/index.js index 7265a4fd..e3c6b239 100644 --- a/client/src/components/categorical/category/index.js +++ b/client/src/components/categorical/category/index.js @@ -205,7 +205,7 @@ class Category extends React.Component { return (
- {truncatedString ? truncatedString : metadataField} + {truncatedString || metadataField} : {schema.annotations.obsByName[metadataField].categories[0]}
diff --git a/client/src/reducers/colors.js b/client/src/reducers/colors.js index a7976319..5c511ddd 100644 --- a/client/src/reducers/colors.js +++ b/client/src/reducers/colors.js @@ -27,6 +27,14 @@ const ColorsReducer = ( }; } + case "universe: user color load success": { + const { userColors } = action; + return { + ...state, + userColors + }; + } + case "reset World to eq Universe": { /* need to rebuild colors as world may have changed, but don't switch modes */ const { world } = nextSharedState; @@ -90,7 +98,7 @@ const ColorsReducer = ( case "color by categorical metadata": case "color by continuous metadata": { - const { world } = prevSharedState; + const { world, colors } = prevSharedState; /* toggle between this mode and reset */ const resetCurrent = @@ -99,11 +107,7 @@ const ColorsReducer = ( const colorMode = !resetCurrent ? action.type : null; const colorAccessor = !resetCurrent ? action.colorAccessor : null; - const { rgb, scale } = ColorHelpers.createColors( - world, - colorMode, - colorAccessor - ); + const { rgb, scale } = ColorHelpers.createColors(world, colorMode, colorAccessor, colors.userColors); return { ...state, colorMode, @@ -141,7 +145,7 @@ const ColorsReducer = ( case "annotation: delete label": { const { world } = nextSharedState; const { colorMode, colorAccessor } = state; - const { metadataField } = action; + const { metadataField, colors } = action; if ( colorMode !== "color by categorical metadata" || colorAccessor !== metadataField @@ -149,11 +153,7 @@ const ColorsReducer = ( return state; /* else, we need to rebuild colors as labels have changed! */ - const { rgb, scale } = ColorHelpers.createColors( - world, - colorMode, - colorAccessor - ); + const { rgb, scale } = ColorHelpers.createColors(world, colorMode, colorAccessor); return { ...state, rgb, scale }; } diff --git a/client/src/reducers/undoableConfig.js b/client/src/reducers/undoableConfig.js index 8c79591b..a6f7ab27 100644 --- a/client/src/reducers/undoableConfig.js +++ b/client/src/reducers/undoableConfig.js @@ -12,6 +12,7 @@ const skipOnActions = new Set([ "interface reset started", "initial data load start", "universe: column load success", + "universe: user color load success", "universe exists, but loading is still in progress", "configuration load complete", "increment graph render counter", diff --git a/client/src/util/stateManager/colorHelpers.js b/client/src/util/stateManager/colorHelpers.js index e6f64552..45c3d6e5 100644 --- a/client/src/util/stateManager/colorHelpers.js +++ b/client/src/util/stateManager/colorHelpers.js @@ -11,13 +11,16 @@ import { range } from "../range"; /* create new colors state object. Paramters: - world - current world object - - mode - color-by mode. One of: null, "color by expression", - "color by continuous metadata", "color by categorical metadata" - - + - colorMode - color-by mode. One of {null, "color by expression", "color by continuous metadata", + "color by categorical metadata"} + - colorAccessor - the obs annotations used for color-by */ -export function createColors(world, colorMode = null, colorAccessor = null) { +export function createColors(world, colorMode = null, colorAccessor = null, userColors = null) { switch (colorMode) { case "color by categorical metadata": { + if (userColors && colorAccessor in userColors) { + return createUserColors(world, colorAccessor, userColors); + } return createColorsByCategoricalMetadata(world, colorAccessor); } case "color by continuous metadata": { @@ -36,8 +39,29 @@ export function createColors(world, colorMode = null, colorAccessor = null) { } } -function createColorsByCategoricalMetadata(world, accessor) { - const { categories } = world.schema.annotations.obsByName[accessor]; +export function loadUserColorConfig(userColors) { + const convertedUserColors = {}; + Object.keys(userColors).forEach(category => { + const [colors, scaleMap] = Object.keys(userColors[category]).reduce((acc, label, i) => { + const color = parseRGB(userColors[category][label]); + acc[0][label] = color; + acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]); + return acc; + }, [{}, {}]); + const scale = i => scaleMap[i]; + convertedUserColors[category] = { colors, scale }; + }); + return convertedUserColors; +} + +function createUserColors(world, colorAccessor, userColors) { + const { colors, scale } = userColors[colorAccessor]; + const rgb = createRgbArray(world, colors, colorAccessor); + return { rgb, scale }; +} + +function createColorsByCategoricalMetadata(world, colorAccessor) { + const { categories } = world.schema.annotations.obsByName[colorAccessor]; const scale = d3 .scaleSequential(interpolateRainbow) @@ -49,14 +73,19 @@ function createColorsByCategoricalMetadata(world, accessor) { return acc; }, {}); + const rgb = createRgbArray(world, colors, colorAccessor); + return { rgb, scale }; +} + +export function createRgbArray(world, colors, colorAccessor) { const rgb = new Array(world.nObs); const df = world.obsAnnotations; - const data = df.col(accessor).asArray(); + const data = df.col(colorAccessor).asArray(); for (let i = 0, len = df.length; i < len; i += 1) { - const cat = data[i]; - rgb[i] = colors[cat]; + const label = data[i]; + rgb[i] = colors[label]; } - return { rgb, scale }; + return rgb; } function createColorsByContinuousMetadata(world, accessor) { diff --git a/docs/_config.yml b/docs/_config.yml index a777d736..dc80f90a 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1,5 +1,6 @@ theme: jekyll-theme-minimal show_downloads: false +url: "https://chanzuckerberg.github.io" baseurl: "/cellxgene" logo: cellxgene-logo.png diff --git a/docs/_site/index.html b/docs/_site/index.html index 64687a05..ce1a61e9 100644 --- a/docs/_site/index.html +++ b/docs/_site/index.html @@ -5,21 +5,21 @@ - + Index | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","url":"https://chanzuckerberg.github.io/cellxgene/","headline":"Index","name":"cellxgene","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/annotations.html b/docs/_site/posts/annotations.html index 1fefc458..af27f5f8 100644 --- a/docs/_site/posts/annotations.html +++ b/docs/_site/posts/annotations.html @@ -5,21 +5,21 @@ - + annotations | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","headline":"annotations","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/contact.html b/docs/_site/posts/contact.html index f5f82e11..18d25b9b 100644 --- a/docs/_site/posts/contact.html +++ b/docs/_site/posts/contact.html @@ -5,21 +5,21 @@ - + Contact | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","headline":"Contact","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/contribute.html b/docs/_site/posts/contribute.html index d98d60b2..19cb1827 100644 --- a/docs/_site/posts/contribute.html +++ b/docs/_site/posts/contribute.html @@ -5,21 +5,21 @@ - + Code of conduct | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","headline":"Code of conduct","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/demo-data.html b/docs/_site/posts/demo-data.html index a8f4447c..8334380c 100644 --- a/docs/_site/posts/demo-data.html +++ b/docs/_site/posts/demo-data.html @@ -5,21 +5,21 @@ - + demo-data | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","headline":"demo-data","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/gallery.html b/docs/_site/posts/gallery.html index fec8f293..21271f02 100644 --- a/docs/_site/posts/gallery.html +++ b/docs/_site/posts/gallery.html @@ -5,21 +5,21 @@ - + Gallery | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","headline":"Gallery","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/hosted.html b/docs/_site/posts/hosted.html index 9b1524d3..0a2a56c9 100644 --- a/docs/_site/posts/hosted.html +++ b/docs/_site/posts/hosted.html @@ -5,21 +5,21 @@ - + Hosting cellxgene on the web | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","headline":"Hosting cellxgene on the web","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/install.html b/docs/_site/posts/install.html index 96eb8cae..f86de462 100644 --- a/docs/_site/posts/install.html +++ b/docs/_site/posts/install.html @@ -5,21 +5,21 @@ - + Install | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","headline":"Install","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/launch.html b/docs/_site/posts/launch.html index 4dd1e622..55d57112 100644 --- a/docs/_site/posts/launch.html +++ b/docs/_site/posts/launch.html @@ -5,21 +5,21 @@ - + demo-data | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","headline":"demo-data","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/methods.html b/docs/_site/posts/methods.html index b6a10d68..57a2b355 100644 --- a/docs/_site/posts/methods.html +++ b/docs/_site/posts/methods.html @@ -5,21 +5,21 @@ - + Methods | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","headline":"Methods","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/prepare.html b/docs/_site/posts/prepare.html index 531ef1e2..114c812c 100644 --- a/docs/_site/posts/prepare.html +++ b/docs/_site/posts/prepare.html @@ -5,21 +5,21 @@ - + prepare | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","headline":"prepare","@context":"https://schema.org"} - + @@ -102,8 +102,8 @@

What about R objects from seurat / bioconductor!?

@@ -112,6 +112,32 @@

Can I use data hosted on the web somewhere?

Yes! You can launch from a URL instead of a filepath. The same data format requirements apply. Please see here for more details.

+

Data format options

+ +

Category colors

+

cellxgene will display scanpy-style color +information +for category-label pairs. An example of this format is shown below:

+ +
>>> category = "louvain"
+>>> # colors stored in adata.uns must be matplotlib-compatible color information
+>>> adata.uns[f"{category}_colors"]
+array(['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#bcbd22'], dtype='<U7')
+>>> # there must be a matching category in adata.obs
+>>> category in adata.obs
+True
+
+ +

To test that you’ve done this properly, check that for your given category the number of colors match the number of category values and that the second command below results in a mapping from categories to colors.

+ +
>>> len(adata.obs[category].cat.categories) == len(adata.uns[f"{category}_colors"])
+True
+>>> dict(zip(adata.obs[category].cat.categories, adata.uns[f"{category}_colors"]))
+{'CD4 T cells': '#1f77b4', 'CD14+ Monocytes': '#ff7f0e', 'B cells': '#2ca02c', 'CD8 T cells': '#d62728', 'NK cells': '#9467bd', 'FCGR3A+ Monocytes': '#8c564b', 'Dendritic cells': '#e377c2', 'Megakaryocytes': '#bcbd22'}
+
+ +

You can disable this feature using the --disable-custom-colors flag for cellxgene launch. cellxgene will then chose colors from its standard color palettes.

+

Using cellxgene prepare

If your data is in a different format, and/or you still need to perform dimensionality reduction and/or clustering, cellxgene can do that for you with the prepare command.

@@ -136,7 +162,7 @@

cellxgene prepare is not meant as a way to formally process or analyze your data. It’s simply a utility for quickly wrangling your data into cellxgene-compatible format and computing a “vanilla” embedding so you can try out cellxgene and get a general sense of a dataset.

Quickstart for cellxgene prepare

-

To add cellxgene prepare to your cellxgene installation, run
+

To add cellxgene prepare to your cellxgene installation, run pip install cellxgene[prepare]

Then run prepare on your data with:

@@ -162,36 +188,36 @@

Let’s look at what prepare is doing to our data, and how each step relates to the command above. You can see a walkthrough of what’s going on under the hood for this example in this notebook.

-

(A) - Compute quality control metrics and store this in our AnnData object for later inspection
-(B) - Normalize the expression matrix using a basic preprocessing recipe
-(auto) - Do some preprocessing to run PCA and compute the neighbor graph
-(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later
-(C) - Compute and store UMAP and tSNE embeddings
+

(A) - Compute quality control metrics and store this in our AnnData object for later inspection +(B) - Normalize the expression matrix using a basic preprocessing recipe +(auto) - Do some preprocessing to run PCA and compute the neighbor graph +(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later +(C) - Compute and store UMAP and tSNE embeddings (D) - Write results to file

Options for cellxgene prepare

For the most up-to-date and comprehensive list of options, run cellxgene prepare --help

-

--embedding controls which dimensionality reduction algorithm is applies to your data.
+

--embedding controls which dimensionality reduction algorithm is applies to your data. Options are umap and/or tsne. Defaults to both.

--recipe controls which normalization steps to apply to your data, based on one of the preprocessing recipes included with scanpy. -These recipes include steps like cell filtering and gene selection; see the scanpy documentation for more details.
+These recipes include steps like cell filtering and gene selection; see the scanpy documentation for more details. Options are none, seurat, or zheng17. Defaults to none.

-

--sparse is a flag determines whether to enforce a sparse matrix. For large datasets, prepare can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want prepare to run faster we recommend using the sparse option.
+

--sparse is a flag determines whether to enforce a sparse matrix. For large datasets, prepare can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want prepare to run faster we recommend using the sparse option. If this flag is not included, default is False

--skip-qc by default, cellxgene prepare will compute quality control metrics (saved to anndata.obs and anndata.var) as described in the scanpy documentation. Pass this flag if you would like to skip this step.

-

--make-obs-names-unique / --make-var-names-unique determine whether to rename obs (cell) / var (gene) names, respectively, to be unique.
+

--make-obs-names-unique / --make-var-names-unique determine whether to rename obs (cell) / var (gene) names, respectively, to be unique. Default is True.

-

--set-obs-names controls which field in anndata.obs (cell metadata) is used as the index for cells (e.g., a cell ID column).
+

--set-obs-names controls which field in anndata.obs (cell metadata) is used as the index for cells (e.g., a cell ID column). Default is anndata.obs.names

-

--set-var-names controls which field in anndata.var (gene metadata) is used as the index for genes.
+

--set-var-names controls which field in anndata.var (gene metadata) is used as the index for genes. Default is anndata.var.names

--output and --overwrite control where the processed data is saved.

diff --git a/docs/_site/posts/roadmap.html b/docs/_site/posts/roadmap.html index a785138c..14e96e5b 100644 --- a/docs/_site/posts/roadmap.html +++ b/docs/_site/posts/roadmap.html @@ -5,21 +5,21 @@ - + roadmap | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","headline":"roadmap","@context":"https://schema.org"} - + diff --git a/docs/_site/posts/troubleshooting.html b/docs/_site/posts/troubleshooting.html index 6eb5240e..0430afc8 100644 --- a/docs/_site/posts/troubleshooting.html +++ b/docs/_site/posts/troubleshooting.html @@ -5,21 +5,21 @@ - + Troubleshooting | cellxgene - - + + +{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","headline":"Troubleshooting","@context":"https://schema.org"} - + diff --git a/docs/posts/prepare.md b/docs/posts/prepare.md index fc9244ee..46483904 100644 --- a/docs/posts/prepare.md +++ b/docs/posts/prepare.md @@ -5,20 +5,48 @@ description: Preparing your data --- # Data format requirements -If your data is in `h5ad` file (from the [`anndata`](https://anndata.readthedocs.io/en/latest/index.html) library) and meets the following requirements, you can go straight to `cellxgene launch`: +If your data is in `h5ad` file (from the [`anndata`](https://anndata.readthedocs.io/en/latest/index.html) library) and meets the following requirements, you can go straight to `cellxgene launch`: -- Expression values (raw or normalized) in `anndata.X` +- Expression values (raw or normalized) in `anndata.X` - At least one embedding (e.g., tSNE, UMAP) in `anndata.obsm`, specified with the prefix `X_` (e.g., by default scanpy stores UMAP coordinates in `anndata.obsm['X_umap']`) -- A unique identifier for every cell is available in an `anndata.obs` field (you can specify this with the `--obs-names` option) -- A unique identifier for every gene is available in an `anndata.var` field (you can specify which field to use with the `--var-names` option) +- A unique identifier is required for each cell, which by default will be pulled from the `obs` DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with `--obs-names` +- A unique identifier is required for each gene, which by default will be pulled from the `var` DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with `--var-names` -#### What about R objects from seurat / bioconductor!? +#### What about R objects from seurat / bioconductor!? We hear you! We'd also love to be able to ingest these files directly. This isn't currently possible, but in the meantime, you can use [sceasy](https://bioconda.github.io/recipes/r-sceasy/README.html) ([docs](https://cellgeni.readthedocs.io/en/latest/visualisations.html)) to convert to `h5ad`. Seurat also has some [handy conversion tools](https://satijalab.org/seurat/v3.0/conversion_vignette.html) that you can try out. -#### Can I use data hosted on the web somewhere? +#### Can I use data hosted on the web somewhere? Yes! You can launch from a URL instead of a filepath. The same data format requirements apply. Please see [here](launch) for more details. -# Using `cellxgene prepare` +# Data format options + +#### Category colors +`cellxgene` will display [scanpy-style color +information](https://github.com/chanzuckerberg/cellxgene/issues/1152#issue-564361541) +for category-label pairs. An example of this format is shown below: + +``` +>>> category = "louvain" +>>> # colors stored in adata.uns must be matplotlib-compatible color information +>>> adata.uns[f"{category}_colors"] +array(['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#bcbd22'], dtype='>> # there must be a matching category in adata.obs +>>> category in adata.obs +True +``` + +To test that you've done this properly, check that for your given `category` the number of colors match the number of category values and that the second command below results in a mapping from categories to colors. + +``` +>>> len(adata.obs[category].cat.categories) == len(adata.uns[f"{category}_colors"]) +True +>>> dict(zip(adata.obs[category].cat.categories, adata.uns[f"{category}_colors"])) +{'CD4 T cells': '#1f77b4', 'CD14+ Monocytes': '#ff7f0e', 'B cells': '#2ca02c', 'CD8 T cells': '#d62728', 'NK cells': '#9467bd', 'FCGR3A+ Monocytes': '#8c564b', 'Dendritic cells': '#e377c2', 'Megakaryocytes': '#bcbd22'} +``` + +You can disable this feature using the `--disable-custom-colors` flag for `cellxgene launch`. cellxgene will then chose colors from its standard color palettes. + +# Using `cellxgene prepare` If your data is in a different format, and/or you still need to perform dimensionality reduction and/or clustering, `cellxgene` can do that for you with the `prepare` command. @@ -30,7 +58,7 @@ If your data is in a different format, and/or you still need to perform dimensio - Handle simple data normalization (from a [recipe](https://www.pydoc.io/pypi/scanpy-0.2.3/autoapi/preprocessing/recipes/index.html)) - Do basic preprocessing to run PCA and compute the neighbor graph -- Reduce dimensionality to generate embeddings +- Reduce dimensionality to generate embeddings - Infer clusters You can control which steps to run and their methods (when applicable), via the CLI. The CLI also includes options for computing QC metrics, enforcing matrix sparcity, specifying index names, and plotting output. @@ -40,10 +68,10 @@ You can control which steps to run and their methods (when applicable), via the `cellxgene prepare` is not meant as a way to formally process or analyze your data. It's simply a utility for quickly wrangling your data into cellxgene-compatible format and computing a "vanilla" embedding so you can try out `cellxgene` and get a general sense of a dataset. ## Quickstart for `cellxgene prepare` -To add `cellxgene prepare` to your [cellxgene installation](install), run +To add `cellxgene prepare` to your [cellxgene installation](install), run `pip install cellxgene[prepare]` -Then run `prepare` on your data with: +Then run `prepare` on your data with: ``` cellxgene prepare dataset.h5ad --output=dataset-processed.h5ad ``` @@ -68,36 +96,36 @@ cellxgene prepare pbmc3k-raw.h5ad \ Let's look at what `prepare` is doing to our data, and how each step relates to the command above. You can see a walkthrough of what's going on under the hood for this example in [this notebook](https://github.com/chanzuckerberg/cellxgene-vignettes/blob/master/dataset-processing/pbmc3k-prepare-example.ipynb). -**(A) - Compute quality control metrics and store this in our `AnnData` object for later inspection** -**(B) - Normalize the expression matrix using a basic preprocessing recipe** -**(auto) - Do some preprocessing to run PCA and compute the neighbor graph** -**(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later** -**(C) - Compute and store UMAP and tSNE embeddings** +**(A) - Compute quality control metrics and store this in our `AnnData` object for later inspection** +**(B) - Normalize the expression matrix using a basic preprocessing recipe** +**(auto) - Do some preprocessing to run PCA and compute the neighbor graph** +**(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later** +**(C) - Compute and store UMAP and tSNE embeddings** **(D) - Write results to file** ## Options for cellxgene `prepare` **For the most up-to-date and comprehensive list of options, run `cellxgene prepare --help`** -`--embedding` controls which dimensionality reduction algorithm is applies to your data. +`--embedding` controls which dimensionality reduction algorithm is applies to your data. Options are `umap` and/or `tsne`. Defaults to both. `--recipe` controls which normalization steps to apply to your data, based on one of the preprocessing `recipes` included with `scanpy`. -These recipes include steps like cell filtering and gene selection; see the `scanpy` [documentation](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes) for more details. +These recipes include steps like cell filtering and gene selection; see the `scanpy` [documentation](https://scanpy.readthedocs.io/en/latest/api/index.html#recipes) for more details. Options are `none`, `seurat`, or `zheng17`. Defaults to `none`. -`--sparse` is a flag determines whether to enforce a sparse matrix. For large datasets, `prepare` can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want `prepare` to run faster we recommend using the `sparse` option. +`--sparse` is a flag determines whether to enforce a sparse matrix. For large datasets, `prepare` can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with >100k cells). If you want `prepare` to run faster we recommend using the `sparse` option. If this flag is not included, default is `False` `--skip-qc` by default, `cellxgene prepare` will compute quality control metrics (saved to `anndata.obs` and `anndata.var`) as described in the `scanpy` [documentation](https://scanpy.readthedocs.io/en/stable/api/scanpy.pp.calculate_qc_metrics.html). Pass this flag if you would like to skip this step. -`--make-obs-names-unique` / `--make-var-names-unique` determine whether to rename `obs` (cell) / `var` (gene) names, respectively, to be unique. +`--make-obs-names-unique` / `--make-var-names-unique` determine whether to rename `obs` (cell) / `var` (gene) names, respectively, to be unique. Default is `True`. -`--set-obs-names` controls which field in `anndata.obs` (cell metadata) is used as the _index_ for cells (e.g., a cell ID column). +`--set-obs-names` controls which field in `anndata.obs` (cell metadata) is used as the _index_ for cells (e.g., a cell ID column). Default is `anndata.obs.names` -`--set-var-names` controls which field in `anndata.var` (gene metadata) is used as the _index_ for genes. +`--set-var-names` controls which field in `anndata.var` (gene metadata) is used as the _index_ for genes. Default is `anndata.var.names` `--output` and `--overwrite` control where the processed data is saved. diff --git a/server/app/app.py b/server/app/app.py index bc9d6990..9d7e33c0 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -207,6 +207,13 @@ class DataVarAPI(Resource): return common_rest.data_var_get(request, data_adaptor) +class ColorsAPI(Resource): + @cache_control(public=True, max_age=ONE_WEEK) + @rest_get_data_adaptor + def get(self, data_adaptor): + return common_rest.colors_get(data_adaptor) + + class DiffExpObsAPI(Resource): @cache_control(no_store=True) @rest_get_data_adaptor @@ -235,6 +242,8 @@ def get_api_resources(bp_api): api.add_resource(AnnotationsObsAPI, "/annotations/obs") api.add_resource(AnnotationsVarAPI, "/annotations/var") api.add_resource(DataVarAPI, "/data/var") + # Display routes + api.add_resource(ColorsAPI, "/colors") # Computation routes api.add_resource(DiffExpObsAPI, "/diffexp/obs") api.add_resource(LayoutObsAPI, "/layout/obs") diff --git a/server/cli/launch.py b/server/cli/launch.py index 53a642ab..a0cf1643 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -73,6 +73,13 @@ def config_args(func): show_default=True, help="Will not display categories with more distinct values than specified.", ) + @click.option( + "--disable-custom-colors", + is_flag=True, + default=False, + show_default=False, + help="Disable user-defined category-label colors drawn from source data file.", + ) @click.option( "--diffexp-lfc-cutoff", "-de", @@ -146,7 +153,7 @@ def dataset_args(func): "--about", default=DEFAULT_CONFIG.single_dataset__about, metavar="", - help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).", + help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).", ) @functools.wraps(func) def wrapper(*args, **kwargs): @@ -311,6 +318,7 @@ def launch( obs_names, var_names, max_category_items, + disable_custom_colors, diffexp_lfc_cutoff, title, scripts, @@ -381,6 +389,7 @@ def launch( user_annotations__ontology__enable=experimental_annotations_ontology, user_annotations__ontology__obo_location=experimental_annotations_ontology_obo, presentation__max_categories=max_category_items, + presentation__custom_colors=not disable_custom_colors, embeddings__names=embedding, embeddings__enable_reembedding=experimental_enable_reembedding, diffexp__enable=not disable_diffexp, diff --git a/server/cli/prepare.py b/server/cli/prepare.py index cc576418..a282c14d 100644 --- a/server/cli/prepare.py +++ b/server/cli/prepare.py @@ -73,7 +73,8 @@ def prepare( (h5ad, loom, or a 10x directory), runs dimensionality reduction, computes nearest neighbors, computes an embedding, performs clustering, and saves the results. Includes additional options for naming annotations, - ensuring sparsity, and plotting results.""" + ensuring sparsity, and plotting results. + """ # collect slow imports here to make CLI startup more responsive click.echo("[cellxgene] Starting CLI...") diff --git a/server/common/app_config.py b/server/common/app_config.py index 57dba4bd..2302f4e6 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -78,6 +78,7 @@ class AppConfig(object): self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"] self.presentation__max_categories = dc["presentation"]["max_categories"] + self.presentation__custom_colors = dc["presentation"]["custom_colors"] self.embeddings__names = dc["embeddings"]["names"] self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"] @@ -275,6 +276,7 @@ class AppConfig(object): def handle_presentation(self, context): self.__check_attr("presentation__max_categories", int) + self.__check_attr("presentation__custom_colors", bool) def handle_single_dataset(self, context): self.__check_attr("single_dataset__datapath", (str, type(None))) @@ -517,6 +519,7 @@ class AppConfig(object): "annotations_cell_ontology_enabled": False, "annotations_cell_ontology_obopath": None, "annotations_cell_ontology_terms": None, + "custom_colors": self.presentation__custom_colors, "diffexp-may-be-slow": False, "about_legal_tos": self.server__about_legal_tos, "about_legal_privacy": self.server__about_legal_privacy, diff --git a/server/common/colors.py b/server/common/colors.py new file mode 100644 index 00000000..17f9eb56 --- /dev/null +++ b/server/common/colors.py @@ -0,0 +1,233 @@ +import re + +from server.common.errors import ColorFormatException + +HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$") + +# https://www.w3.org/TR/css-color-4/#named-colors +CSS4_NAMED_COLORS = dict( + aliceblue="#f0f8ff", + antiquewhite="#faebd7", + aqua="#00ffff", + aquamarine="#7fffd4", + azure="#f0ffff", + beige="#f5f5dc", + bisque="#ffe4c4", + black="#000000", + blanchedalmond="#ffebcd", + blue="#0000ff", + blueviolet="#8a2be2", + brown="#a52a2a", + burlywood="#deb887", + cadetblue="#5f9ea0", + chartreuse="#7fff00", + chocolate="#d2691e", + coral="#ff7f50", + cornflowerblue="#6495ed", + cornsilk="#fff8dc", + crimson="#dc143c", + cyan="#00ffff", + darkblue="#00008b", + darkcyan="#008b8b", + darkgoldenrod="#b8860b", + darkgray="#a9a9a9", + darkgreen="#006400", + darkgrey="#a9a9a9", + darkkhaki="#bdb76b", + darkmagenta="#8b008b", + darkolivegreen="#556b2f", + darkorange="#ff8c00", + darkorchid="#9932cc", + darkred="#8b0000", + darksalmon="#e9967a", + darkseagreen="#8fbc8f", + darkslateblue="#483d8b", + darkslategray="#2f4f4f", + darkslategrey="#2f4f4f", + darkturquoise="#00ced1", + darkviolet="#9400d3", + deeppink="#ff1493", + deepskyblue="#00bfff", + dimgray="#696969", + dimgrey="#696969", + dodgerblue="#1e90ff", + firebrick="#b22222", + floralwhite="#fffaf0", + forestgreen="#228b22", + fuchsia="#ff00ff", + gainsboro="#dcdcdc", + ghostwhite="#f8f8ff", + gold="#ffd700", + goldenrod="#daa520", + gray="#808080", + green="#008000", + greenyellow="#adff2f", + grey="#808080", + honeydew="#f0fff0", + hotpink="#ff69b4", + indianred="#cd5c5c", + indigo="#4b0082", + ivory="#fffff0", + khaki="#f0e68c", + lavender="#e6e6fa", + lavenderblush="#fff0f5", + lawngreen="#7cfc00", + lemonchiffon="#fffacd", + lightblue="#add8e6", + lightcoral="#f08080", + lightcyan="#e0ffff", + lightgoldenrodyellow="#fafad2", + lightgray="#d3d3d3", + lightgreen="#90ee90", + lightgrey="#d3d3d3", + lightpink="#ffb6c1", + lightsalmon="#ffa07a", + lightseagreen="#20b2aa", + lightskyblue="#87cefa", + lightslategray="#778899", + lightslategrey="#778899", + lightsteelblue="#b0c4de", + lightyellow="#ffffe0", + lime="#00ff00", + limegreen="#32cd32", + linen="#faf0e6", + magenta="#ff00ff", + maroon="#800000", + mediumaquamarine="#66cdaa", + mediumblue="#0000cd", + mediumorchid="#ba55d3", + mediumpurple="#9370db", + mediumseagreen="#3cb371", + mediumslateblue="#7b68ee", + mediumspringgreen="#00fa9a", + mediumturquoise="#48d1cc", + mediumvioletred="#c71585", + midnightblue="#191970", + mintcream="#f5fffa", + mistyrose="#ffe4e1", + moccasin="#ffe4b5", + navajowhite="#ffdead", + navy="#000080", + oldlace="#fdf5e6", + olive="#808000", + olivedrab="#6b8e23", + orange="#ffa500", + orangered="#ff4500", + orchid="#da70d6", + palegoldenrod="#eee8aa", + palegreen="#98fb98", + paleturquoise="#afeeee", + palevioletred="#db7093", + papayawhip="#ffefd5", + peachpuff="#ffdab9", + peru="#cd853f", + pink="#ffc0cb", + plum="#dda0dd", + powderblue="#b0e0e6", + purple="#800080", + rebeccapurple="#663399", + red="#ff0000", + rosybrown="#bc8f8f", + royalblue="#4169e1", + saddlebrown="#8b4513", + salmon="#fa8072", + sandybrown="#f4a460", + seagreen="#2e8b57", + seashell="#fff5ee", + sienna="#a0522d", + silver="#c0c0c0", + skyblue="#87ceeb", + slateblue="#6a5acd", + slategray="#708090", + slategrey="#708090", + snow="#fffafa", + springgreen="#00ff7f", + steelblue="#4682b4", + tan="#d2b48c", + teal="#008080", + thistle="#d8bfd8", + tomato="#ff6347", + turquoise="#40e0d0", + violet="#ee82ee", + wheat="#f5deb3", + white="#ffffff", + whitesmoke="#f5f5f5", + yellow="#ffff00", + yellowgreen="#9acd32", +) + + +def convert_color_to_hex_format(unknown): + """ + Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet. + + The function accepts for the following formats: + - A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html + - RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0] + - RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255] + - Hex triplet string, as in "#08c0ff" + + :param unknown: color info of unknown format + :return: a hex triplet representing that color + """ + try: + if type(unknown) in (list, tuple) and len(unknown) == 3: + if all(0.0 <= ele <= 1.0 for ele in unknown): + tup = tuple(int(ele * 255) for ele in unknown) + elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown): + tup = tuple(unknown) + else: + raise ColorFormatException("Unknown color iterable format!") + return "#%02x%02x%02x" % tup + elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS: + return CSS4_NAMED_COLORS[unknown.lower()] + elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown): + return unknown.lower() + else: + raise ColorFormatException("Unknown color format type!") + except Exception as e: + raise ColorFormatException(e) + + +def convert_anndata_category_colors_to_cxg_category_colors(data): + """ + Convert color information from anndata files to the cellxgene color data format as described below: + { + "": { + "": "", + ... + }, + ... + } + + For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307. + + For more on the anndata color data structure, see + https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178. + + Handling of malformed data: + - For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot + convert to a hex triplet string, a ColorFormatException is raised + - No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no + adata.obs[f"{category}"] key + + :param data: the anndata file + :return: cellxgene color data structure as described above + """ + cxg_colors = dict() + color_key_suffix = "_colors" + for uns_key in data.uns.keys(): + # find uns array that describes colors for a category + if not uns_key.endswith(color_key_suffix): + continue + + # check to see if we actually have observations for that category + category_name = uns_key[: -len(color_key_suffix)] + if category_name not in data.obs.keys(): + continue + + # create the cellxgene color entry for this category + cxg_colors[category_name] = dict( + zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]]) + ) + return cxg_colors diff --git a/server/common/default_config.py b/server/common/default_config.py index f378b7d5..64e042c4 100644 --- a/server/common/default_config.py +++ b/server/common/default_config.py @@ -19,6 +19,7 @@ server: presentation: max_categories: 1000 + custom_colors: true multi_dataset: dataroot: null diff --git a/server/common/errors.py b/server/common/errors.py index 53b2bab0..9b31ee6d 100644 --- a/server/common/errors.py +++ b/server/common/errors.py @@ -84,3 +84,9 @@ class ComputeError(Exception): """ pass + + +class ColorFormatException(Exception): + """Raised when color helper functions encounter an unknown color format""" + + pass diff --git a/server/common/rest.py b/server/common/rest.py index faa9f2bc..60a2bedf 100644 --- a/server/common/rest.py +++ b/server/common/rest.py @@ -1,9 +1,11 @@ -import sys -from http import HTTPStatus import copy import logging +import sys +from http import HTTPStatus + from flask import make_response, jsonify, current_app, abort from werkzeug.urls import url_unquote + from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg from server.common.errors import ( FilterError, @@ -12,6 +14,7 @@ from server.common.errors import ( DisabledFeatureError, ExceedsLimitError, DatasetAccessError, + ColorFormatException, ) import json @@ -222,6 +225,15 @@ def data_var_get(request, data_adaptor): return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True) +def colors_get(data_adaptor): + if not data_adaptor.config.presentation__custom_colors: + return make_response(jsonify({}), HTTPStatus.OK) + try: + return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK) + except ColorFormatException as e: + return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True) + + def diffexp_obs_post(request, data_adaptor): if not data_adaptor.config.diffexp__enable: return abort(HTTPStatus.NOT_IMPLEMENTED) diff --git a/server/converters/cxgtool.py b/server/converters/cxgtool.py index d86f2878..0f4a4c9f 100644 --- a/server/converters/cxgtool.py +++ b/server/converters/cxgtool.py @@ -4,28 +4,36 @@ into a cellxgene TileDB structure, aka a 'CXG'. The organization of the TileDB structure is: - the.cxg TileDB Group - |-- obs TileDB array containing cell (row) attributes, one attribute per - | dataframe columm, shape (n_obs,) - |-- var TileDB array containing gene (column) attributes, with one attribute per - | dataframe column, shape (n_var,) - |-- X Main count matrix as a 2D TileDB array, single unnanmed numeric attribute - |-- emb TileDB group, storing optional embeddings (group may be empty) - | |-- TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N) - |-- cxg_group_metadata Empty array used only to stash metadata about the overall object. + the.cxg TileDB Group + ├─ obs TileDB array containing cell (row) attributes, one attribute per + │ dataframe column, shape (n_obs,) + ├─ var TileDB array containing gene (column) attributes, with one attribute per + │ dataframe column, shape (n_obs,) + ├─ X Main count matrix as a 2D TileDB array, single unnamed numeric attribute + ├─ emb TileDB group, storing optional embeddings (group may be empty) + │ └─ TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N) + └─ cxg_group_metadata Empty array used only to stash metadata about the overall object. + └─ cxg_category_colors CXG colors object as described below: + { + "": { + "": "", + ... + }, + ... + } ... -All arrays are defined to have a uint32 domain, zero based. All X counds and embedding +All arrays are defined to have a uint32 domain, zero based. All X counts and embedding coordinates are coerced to float32, which is ample precision for visualization purposes. Dataframe (metadata) types are generally preserved, or where that is not possible, -converted to somemthing with equal representative value in the cellxgene application +converted to something with equal representative value in the cellxgene application (eg, categorical types are converted to string, bools to uint8, etc). -The following objects are also decorated with auxilliary metadata using TileDB +The following objects are also decorated with auxiliary metadata using TileDB array metadata: * cxg_group_metadata: minimally, will contain a 'cxg_version' field, which - is a semver string identifing the version number of the CXG layout. + is a semver string identifying the version number of the CXG layout. It may also contain 'cxg_parameters', a JSON-encoded parameter list describing CXG-wide dataset parameters. @@ -40,6 +48,14 @@ including the global data layout, spatial tile size, and the like. The CXG is self-describing in these areas, and the actual values (eg, tile size) are empirically derived from benchmarking. They may change in the future. +cxgtool.py will extract color information stored in arrays in the 'uns' anndata +property with the key "{category_name}_colors". For this to work, the following +command must result in a mapping from category names to matplotlib-compatible colors: + +``` +dict(zip(adata.obs[cat].cat.categories, adata.uns[f"{cat}_colors"])) +``` + --- TODO/ISSUES: @@ -55,10 +71,16 @@ import numpy as np from os.path import splitext, basename import json +from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors +from server.common.errors import ColorFormatException + # the CXG container version number. Must be a semver string. CXG_VERSION = "0.1" +# log_level must have a default +log_level = 3 + def log(level, *args): global log_level @@ -72,6 +94,12 @@ def main(): parser.add_argument( "--backed", action="store_true", help="loaded in file backed mode. Will be slower, but use less memory." ) + parser.add_argument( + "--disable-custom-colors", + action="store_true", + default=False, + help="Do not extract scanpy-compatible category colors from h5ad file.", + ) parser.add_argument( "--obs-names", help="Name of annotation to use for observations. If not specified, will use the obs index." ) @@ -99,12 +127,20 @@ def main(): container = out if splitext(out)[1] == ".cxg" else out + ".cxg" title = args.title if args.title is not None else basefname - write_cxg(adata, container, title, var_names=args.var_names, obs_names=args.obs_names, about=args.about) + write_cxg( + adata, + container, + title, + var_names=args.var_names, + obs_names=args.obs_names, + about=args.about, + extract_colors=not args.disable_custom_colors, + ) log(1, "done") -def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None): +def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None, extract_colors=False): if not adata.var.index.is_unique: raise ValueError("Variable index is not unique - unable to convert.") if not adata.obs.index.is_unique: @@ -129,7 +165,19 @@ def write_cxg(adata, container, title, var_names=None, obs_names=None, about=Non log(1, f"\t...group created, with name {container}") # dataset metadata - save_metadata(container, {"title": title, "about": about}) + metadata_dict = dict(cxg_version=CXG_VERSION, cxg_properties=json.dumps({"title": title, "about": about})) + if extract_colors: + try: + metadata_dict["cxg_category_colors"] = json.dumps( + convert_anndata_category_colors_to_cxg_category_colors(adata) + ) + except ColorFormatException: + log( + 0, + "Warning: failed to extract colors from h5ad file! " + "Fix the h5ad file or rerun with --disable-custom-colors. See help for details.", + ) + save_metadata(container, metadata_dict) log(1, "\t...dataset metadata saved") # var/gene dataframe @@ -392,7 +440,7 @@ def save_X(container, adata, ctx): tiledb.consolidate(X_name, ctx=ctx) -def save_metadata(container, metadata): +def save_metadata(container, metadata_dict): """ Save all dataset-wide metadata. This includes: * CXG version @@ -407,8 +455,8 @@ def save_metadata(container, metadata): with tiledb.from_numpy(a_name, np.zeros((1,))) as A: pass with tiledb.DenseArray(a_name, mode="w") as A: - A.meta["cxg_version"] = CXG_VERSION - A.meta["cxg_properties"] = json.dumps(metadata) + for k, v in metadata_dict.items(): + A.meta[k] = v def sanitize_keys(keys): diff --git a/server/data_anndata/anndata_adaptor.py b/server/data_anndata/anndata_adaptor.py index 84fe5976..fe7bda4f 100644 --- a/server/data_anndata/anndata_adaptor.py +++ b/server/data_anndata/anndata_adaptor.py @@ -12,6 +12,7 @@ from server_timing import Timing as ServerTiming from server.data_common.data_adaptor import DataAdaptor from server.data_common.fbs.matrix import encode_matrix_fbs from server.common.utils import series_to_schema +from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors from server.common.constants import Axis, MAX_LAYOUTS from server.common.errors import PrepareError, DatasetAccessError, FilterError from server.compute.scanpy import scanpy_umap @@ -333,6 +334,9 @@ class AnndataAdaptor(DataAdaptor): lfc_cutoff = self.config.diffexp__lfc_cutoff return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff) + def get_colors(self): + return convert_anndata_category_colors_to_cxg_category_colors(self.data) + def get_X_array(self, obs_mask=None, var_mask=None): if obs_mask is None: obs_mask = slice(None) diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index 1e92cfb3..7aa5d452 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -83,6 +83,10 @@ class DataAdaptor(metaclass=ABCMeta): def query_obs_array(self, term_var): pass + @abstractmethod + def get_colors(self): + pass + @abstractmethod def get_obs_index(self): pass diff --git a/server/data_cxg/cxg_adaptor.py b/server/data_cxg/cxg_adaptor.py index 3de42eda..5cd1b795 100644 --- a/server/data_cxg/cxg_adaptor.py +++ b/server/data_cxg/cxg_adaptor.py @@ -194,6 +194,10 @@ class CxgAdaptor(DataAdaptor): lfc_cutoff = self.config.diffexp__lfc_cutoff return diffexp_cxg.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff) + def get_colors(self): + meta = self.open_array("cxg_group_metadata").meta + return json.loads(meta["cxg_category_colors"]) if "cxg_category_colors" in meta else dict() + def get_X_array(self, obs_mask=None, var_mask=None): obs_items = pack_selector_from_mask(obs_mask) var_items = pack_selector_from_mask(var_mask) diff --git a/server/eb/app.py b/server/eb/app.py index bc4809f2..9fd27492 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -68,7 +68,7 @@ class WSGIServer(Server): if len(style_hashes) > 0: csp["style-src"] = style_hashes - Talisman(app, force_https=app_config.server__force_https, frame_options='DENY', content_security_policy=csp) + Talisman(app, force_https=app_config.server__force_https, frame_options="DENY", content_security_policy=csp) @staticmethod def load_csp_hashes(app): diff --git a/server/test/__init__.py b/server/test/__init__.py index d096f4c4..fad56894 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -1,6 +1,6 @@ import shutil import tempfile -from os import path +from os import path, popen import pandas as pd @@ -11,11 +11,14 @@ from server.data_common.fbs.matrix import encode_matrix_fbs from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType +PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() + + def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): tmp_dir = tempfile.mkdtemp() annotations_file = path.join(tmp_dir, "test_annotations.csv") if annotations_fixture: - shutil.copyfile(f"test/test_datasets/pbmc3k-annotations.csv", annotations_file) + shutil.copyfile(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-annotations.csv", annotations_file) args = { "embeddings__names": ["umap"], "presentation__max_categories": 100, @@ -24,7 +27,7 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): "diffexp__lfc_cutoff": 0.01, } fname = { - MatrixDataType.H5AD: "../example-dataset/pbmc3k.h5ad", + MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg", }[ext] data_locator = DataLocator(fname) @@ -53,3 +56,21 @@ def skip_if(condition, reason: str): return wraps return decorator + + +def app_config(data_locator, backed=False): + args = { + "embeddings__names": ["umap", "tsne", "pca"], + "presentation__max_categories": 100, + "single_dataset__obs_names": None, + "single_dataset__var_names": None, + "diffexp__lfc_cutoff": 0.01, + "adaptor__anndata_adaptor__backed": backed, + "single_dataset__datapath": data_locator, + "limits__diffexp_cellcount_max": None, + "limits__column_request_max": None, + } + config = AppConfig() + config.update(**args) + config.complete_config() + return config diff --git a/server/test/test_anndata_adaptor.py b/server/test/test_anndata_adaptor.py index f13e8602..1cec773c 100644 --- a/server/test/test_anndata_adaptor.py +++ b/server/test/test_anndata_adaptor.py @@ -10,10 +10,11 @@ from parameterized import parameterized_class import numpy as np import pandas as pd -from server.data_anndata.anndata_adaptor import AnndataAdaptor -from server.common.errors import FilterError from server.common.data_locator import DataLocator -from server.common.app_config import AppConfig +from server.common.errors import FilterError +from server.data_anndata.anndata_adaptor import AnndataAdaptor +from server.test import PROJECT_ROOT, app_config +from server.test.test_datasets.fixtures import pbmc3k_colors """ Test the anndata adaptor using the pbmc3k data set. @@ -23,30 +24,17 @@ Test the anndata adaptor using the pbmc3k data set. @parameterized_class( ("data_locator", "backed"), [ - ("../example-dataset/pbmc3k.h5ad", False), - ("test/test_datasets/pbmc3k-CSC-gz.h5ad", False), - ("test/test_datasets/pbmc3k-CSR-gz.h5ad", False), - ("../example-dataset/pbmc3k.h5ad", True), - ("test/test_datasets/pbmc3k-CSC-gz.h5ad", True), - ("test/test_datasets/pbmc3k-CSR-gz.h5ad", True), + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False), + (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False), + (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False), + (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True), + (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True), + (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True), ], ) class AdaptorTest(unittest.TestCase): def setUp(self): - args = { - "embeddings__names": ["umap", "tsne", "pca"], - "presentation__max_categories": 100, - "single_dataset__obs_names": None, - "single_dataset__var_names": None, - "diffexp__lfc_cutoff": 0.01, - "adaptor__anndata_adaptor__backed": self.backed, - "single_dataset__datapath": self.data_locator, - "limits__diffexp_cellcount_max": None, - "limits__column_request_max": None, - } - config = AppConfig() - config.update(**args) - config.complete_config() + config = app_config(self.data_locator, self.backed) self.data = AnndataAdaptor(DataLocator(self.data_locator), config) def test_init(self): @@ -92,6 +80,9 @@ class AdaptorTest(unittest.TestCase): self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0) self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0) + def test_get_colors(self): + self.assertEqual(self.data.get_colors(), pbmc3k_colors) + def test_get_schema(self): with open(path.join(path.dirname(__file__), "schema.json")) as fh: schema = json.load(fh) diff --git a/server/test/test_anndata_adaptor_data_load.py b/server/test/test_anndata_adaptor_data_load.py index c16bd7f5..f7848b9f 100644 --- a/server/test/test_anndata_adaptor_data_load.py +++ b/server/test/test_anndata_adaptor_data_load.py @@ -4,6 +4,7 @@ import json from server.data_anndata.anndata_adaptor import AnndataAdaptor from server.common.data_locator import DataLocator from server.common.app_config import AppConfig +from server.test import PROJECT_ROOT class DataLoadAdaptorTest(unittest.TestCase): @@ -12,7 +13,7 @@ class DataLoadAdaptorTest(unittest.TestCase): """ def setUp(self): - self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad") + self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") config = AppConfig() config.update(single_dataset__datapath=self.data_file.path) config.complete_config() diff --git a/server/test/test_api.py b/server/test/test_api.py index ad57224b..cd9a4329 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -8,8 +8,10 @@ import pandas as pd import requests import server.test.decode_fbs as decode_fbs -from server.test import data_with_tmp_annotations, make_fbs from server.data_common.matrix_loader import MatrixDataType +from server.test import data_with_tmp_annotations, make_fbs, PROJECT_ROOT +from server.test.test_datasets.fixtures import pbmc3k_colors + BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} @@ -255,6 +257,15 @@ class EndPoints(object): self.assertEqual(df["n_rows"], 2638) self.assertEqual(df["n_cols"], 1) + def test_colors(self): + endpoint = "colors" + url = f"{self.URL_BASE}{endpoint}" + result = self.session.get(url) + self.assertEqual(result.status_code, HTTPStatus.OK) + self.assertEqual(result.headers["Content-Type"], "application/json") + result_data = result.json() + self.assertEqual(result_data, pbmc3k_colors) + def test_static(self): endpoint = "static" file = "assets/favicon.ico" @@ -349,7 +360,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): "cellxgene", "--no-upgrade-check", "launch", - "../example-dataset/pbmc3k.h5ad", + f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", "--disable-annotations", "--verbose", "--port", @@ -383,7 +394,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints): "cellxgene", "--no-upgrade-check", "launch", - "test/test_datasets/pbmc3k.cxg", + f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg", "--disable-annotations", "--verbose", "--port", diff --git a/server/test/test_colors.py b/server/test/test_colors.py new file mode 100644 index 00000000..2970587c --- /dev/null +++ b/server/test/test_colors.py @@ -0,0 +1,40 @@ +import unittest + +import anndata +from server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors +from server.common.errors import ColorFormatException +from server.test import PROJECT_ROOT +from server.test.test_datasets.fixtures import pbmc3k_colors + + +class ColorsTest(unittest.TestCase): + """ Test color helper functions """ + + def test_convert_color_to_hex_format(self): + self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3") + self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3") + self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3") + self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3") + self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3") + self.assertEqual( + convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3" + ) + for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]: + with self.assertRaises(ColorFormatException): + convert_color_to_hex_format(bad_input) + + def test_anndata_colors_to_cxg_colors(self): + # test standard behavior + adata = self._get_h5ad() + self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors) + # test that invalid color formats raise an exception + adata.uns["louvain_colors"][0] = "#NOTCOOL" + with self.assertRaises(ColorFormatException): + convert_anndata_category_colors_to_cxg_category_colors(adata) + # test that colors without a matching obs category are skipped + adata = self._get_h5ad() + del adata.obs["louvain"] + self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {}) + + def _get_h5ad(self): + return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") diff --git a/server/test/test_cxg_adaptor.py b/server/test/test_cxg_adaptor.py new file mode 100644 index 00000000..ef716dfc --- /dev/null +++ b/server/test/test_cxg_adaptor.py @@ -0,0 +1,16 @@ +import unittest + +from server.common.data_locator import DataLocator +from server.data_cxg.cxg_adaptor import CxgAdaptor +from server.test import PROJECT_ROOT, app_config +from server.test.test_datasets.fixtures import pbmc3k_colors + + +class TestCxgAdaptor(unittest.TestCase): + def setUp(self): + data_locator = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg" + config = app_config(data_locator) + self.data = CxgAdaptor(DataLocator(data_locator), config) + + def test_get_colors(self): + self.assertEqual(self.data.get_colors(), pbmc3k_colors) diff --git a/server/test/test_cxgtool.py b/server/test/test_cxgtool.py new file mode 100644 index 00000000..8da8f4c5 --- /dev/null +++ b/server/test/test_cxgtool.py @@ -0,0 +1,40 @@ +import random +import shutil +import string +import unittest + +import anndata + +from server.common.data_locator import DataLocator +from server.converters.cxgtool import write_cxg +from server.data_cxg.cxg_adaptor import CxgAdaptor +from server.test import PROJECT_ROOT, app_config +from server.test.test_datasets.fixtures import pbmc3k_colors + + +class TestCxgAdaptor(unittest.TestCase): + def setUp(self) -> None: + self.fixtures = [] + + def tearDown(self) -> None: + try: + for data_locator in self.fixtures: + print("REMOVING ", data_locator) + shutil.rmtree(data_locator) + except FileNotFoundError: + pass + + def test_cxg_category_colors(self): + data = self.convert_pbmc3k(extract_colors=True) + self.assertEqual(data.get_colors(), pbmc3k_colors) + data = self.convert_pbmc3k(extract_colors=False) + self.assertEqual(data.get_colors(), {}) + + def convert_pbmc3k(self, **kwargs): + random_string = "".join(random.choice(string.ascii_letters) for _ in range(8)) + data_locator = f"/tmp/test_{random_string}.cxg" + self.fixtures.append(data_locator) + source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") + write_cxg(adata=source_h5ad, container=data_locator, title="pbmc3k", **kwargs) + config = app_config(data_locator) + return CxgAdaptor(DataLocator(data_locator), config) diff --git a/server/test/test_datasets/fixtures.py b/server/test/test_datasets/fixtures.py new file mode 100644 index 00000000..e1a1368d --- /dev/null +++ b/server/test/test_datasets/fixtures.py @@ -0,0 +1,12 @@ +pbmc3k_colors = { + "louvain": { + "B cells": "#2ca02c", + "CD14+ Monocytes": "#ff7f0e", + "CD4 T cells": "#1f77b4", + "CD8 T cells": "#d62728", + "Dendritic cells": "#e377c2", + "FCGR3A+ Monocytes": "#8c564b", + "Megakaryocytes": "#bcbd22", + "NK cells": "#9467bd", + } +} diff --git a/server/test/test_datasets/pbmc3k.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__fragment_metadata.tdb b/server/test/test_datasets/pbmc3k.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__fragment_metadata.tdb deleted file mode 100644 index 511a453a..00000000 Binary files a/server/test/test_datasets/pbmc3k.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__fragment_metadata.tdb and /dev/null differ diff --git a/server/test/test_datasets/pbmc3k.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__attr.tdb b/server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__attr.tdb old mode 100644 new mode 100755 similarity index 65% rename from server/test/test_datasets/pbmc3k.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__attr.tdb rename to server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__attr.tdb index 68f06ed1..266ae392 Binary files a/server/test/test_datasets/pbmc3k.cxg/X/__1576858534264_1576858534264_4f12045b32ea45a490bdad087bac4dc3/__attr.tdb and b/server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__attr.tdb differ diff --git a/server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__fragment_metadata.tdb b/server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__fragment_metadata.tdb new file mode 100755 index 00000000..cf8234a2 Binary files /dev/null and b/server/test/test_datasets/pbmc3k.cxg/X/__1587182255882_1587182255882_f7aa6ccb49a944f9ab9e25b11bbfab4f/__fragment_metadata.tdb differ diff --git a/server/test/test_datasets/pbmc3k.cxg/X/__array_schema.tdb b/server/test/test_datasets/pbmc3k.cxg/X/__array_schema.tdb old mode 100644 new mode 100755 index b5c8fd3a..fc22f10f Binary files a/server/test/test_datasets/pbmc3k.cxg/X/__array_schema.tdb and b/server/test/test_datasets/pbmc3k.cxg/X/__array_schema.tdb differ diff --git a/server/test/test_datasets/pbmc3k.cxg/X/__lock.tdb b/server/test/test_datasets/pbmc3k.cxg/X/__lock.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/__tiledb_group.tdb b/server/test/test_datasets/pbmc3k.cxg/__tiledb_group.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__attr.tdb b/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__attr.tdb new file mode 100755 index 00000000..2f481a1d Binary files /dev/null and b/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__attr.tdb differ diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__fragment_metadata.tdb b/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__fragment_metadata.tdb new file mode 100755 index 00000000..a7807b06 Binary files /dev/null and b/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__1587182255763_1587182255763_88cd3d13926f4892af7230837bcc5178/__fragment_metadata.tdb differ diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__array_schema.tdb b/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__array_schema.tdb new file mode 100755 index 00000000..ea9291e1 Binary files /dev/null and b/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__array_schema.tdb differ diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__lock.tdb b/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__lock.tdb new file mode 100755 index 00000000..e69de29b diff --git a/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__meta/__1587182255768_1587182255768_fa7617ae99f843929911e4bc7b03e3db b/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__meta/__1587182255768_1587182255768_fa7617ae99f843929911e4bc7b03e3db new file mode 100755 index 00000000..a608a452 Binary files /dev/null and b/server/test/test_datasets/pbmc3k.cxg/cxg_group_metadata/__meta/__1587182255768_1587182255768_fa7617ae99f843929911e4bc7b03e3db differ diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/__tiledb_group.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/__tiledb_group.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__attr.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__attr.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__attr.tdb rename to server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__fragment_metadata.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__fragment_metadata.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1576858534229_1576858534229_391cdd6b87b649dea76842dfa59ed0d9/__fragment_metadata.tdb rename to server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__1587182255875_1587182255875_e02a92b5d18f45e0a02031568f4633d4/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__array_schema.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__array_schema.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__lock.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/draw_graph_fr/__lock.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__attr.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__attr.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__attr.tdb rename to server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__fragment_metadata.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__fragment_metadata.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/pca/__1576858534121_1576858534121_454903804a694b3b8ccdae56065664ba/__fragment_metadata.tdb rename to server/test/test_datasets/pbmc3k.cxg/emb/pca/__1587182255827_1587182255827_8aeb4df5c2a74918b7aeeb6d34632e24/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/pca/__array_schema.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/pca/__array_schema.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/pca/__lock.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/pca/__lock.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__attr.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__attr.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__attr.tdb rename to server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__fragment_metadata.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__fragment_metadata.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1576858534161_1576858534161_aa4803e7e7be4b23bea35f5d62296f14/__fragment_metadata.tdb rename to server/test/test_datasets/pbmc3k.cxg/emb/tsne/__1587182255846_1587182255846_0b832c3ed3534b458e83c75726f2005f/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__array_schema.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__array_schema.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__lock.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/tsne/__lock.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__attr.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__attr.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__attr.tdb rename to server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__attr.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__fragment_metadata.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__fragment_metadata.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/emb/umap/__1576858534193_1576858534193_67d97bcdd3d1486985f5974b133cb496/__fragment_metadata.tdb rename to server/test/test_datasets/pbmc3k.cxg/emb/umap/__1587182255867_1587182255867_44955ba2220c4ad59bbdbb6b26e21428/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/umap/__array_schema.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/umap/__array_schema.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/emb/umap/__lock.tdb b/server/test/test_datasets/pbmc3k.cxg/emb/umap/__lock.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/__fragment_metadata.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/__fragment_metadata.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/__fragment_metadata.tdb rename to server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain.tdb rename to server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain_var.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain_var.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/louvain_var.tdb rename to server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/louvain_var.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_counts.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_counts.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_counts.tdb rename to server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_counts.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_genes.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_genes.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/n_genes.tdb rename to server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/n_genes.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0.tdb rename to server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0_var.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0_var.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/name_0_var.tdb rename to server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/name_0_var.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/percent_mito.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/percent_mito.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/obs/__1576858534031_1576858534031_1641d0129fe64c78b2d0a6a684ce47ba/percent_mito.tdb rename to server/test/test_datasets/pbmc3k.cxg/obs/__1587182255797_1587182255797_d3d57575169a48eaa00d2f709d0a534c/percent_mito.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__array_schema.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__array_schema.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__lock.tdb b/server/test/test_datasets/pbmc3k.cxg/obs/__lock.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1576858534024_1576858534024_e856aef5b0e244b1a4b7dde70cd864d0 b/server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1576858534024_1576858534024_e856aef5b0e244b1a4b7dde70cd864d0 deleted file mode 100644 index 88398dfe..00000000 Binary files a/server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1576858534024_1576858534024_e856aef5b0e244b1a4b7dde70cd864d0 and /dev/null differ diff --git a/server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1587182255787_1587182255787_e19a340576a340db85c37b61b83dbe57 b/server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1587182255787_1587182255787_e19a340576a340db85c37b61b83dbe57 new file mode 100755 index 00000000..90a5f38c Binary files /dev/null and b/server/test/test_datasets/pbmc3k.cxg/obs/__meta/__1587182255787_1587182255787_e19a340576a340db85c37b61b83dbe57 differ diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/__fragment_metadata.tdb b/server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/__fragment_metadata.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/__fragment_metadata.tdb rename to server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/__fragment_metadata.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/n_cells.tdb b/server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/n_cells.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/n_cells.tdb rename to server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/n_cells.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0.tdb b/server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0.tdb rename to server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0_var.tdb b/server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0_var.tdb old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__1576858533970_1576858533970_d241b2e750eb425a9ee23ed1de686c2a/name_0_var.tdb rename to server/test/test_datasets/pbmc3k.cxg/var/__1587182255773_1587182255773_12e07b1585a64ebe9d40098d67a8e8ad/name_0_var.tdb diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__array_schema.tdb b/server/test/test_datasets/pbmc3k.cxg/var/__array_schema.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__lock.tdb b/server/test/test_datasets/pbmc3k.cxg/var/__lock.tdb old mode 100644 new mode 100755 diff --git a/server/test/test_datasets/pbmc3k.cxg/var/__meta/__1576858533966_1576858533966_1362646d804b4982b35052a367633436 b/server/test/test_datasets/pbmc3k.cxg/var/__meta/__1587182255771_1587182255771_5a3a50f0b360403eab0e427c429f574e old mode 100644 new mode 100755 similarity index 100% rename from server/test/test_datasets/pbmc3k.cxg/var/__meta/__1576858533966_1576858533966_1362646d804b4982b35052a367633436 rename to server/test/test_datasets/pbmc3k.cxg/var/__meta/__1587182255771_1587182255771_5a3a50f0b360403eab0e427c429f574e diff --git a/server/test/test_diffexp.py b/server/test/test_diffexp.py index 11aa1d35..c75731cb 100644 --- a/server/test/test_diffexp.py +++ b/server/test/test_diffexp.py @@ -5,6 +5,8 @@ import server.compute.diffexp_cxg as diffexp_cxg import server.compute.diffexp_generic as diffexp_generic import numpy as np +from server.test import PROJECT_ROOT + class DiffExpTest(unittest.TestCase): """Tests the diffexp returns the expected results for one test case, using different @@ -50,7 +52,7 @@ class DiffExpTest(unittest.TestCase): def test_anndata_default(self): """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)""" - adaptor = self.load_dataset("../example-dataset/pbmc3k.h5ad") + adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad") maskA = self.get_mask(adaptor, 1, 10) maskB = self.get_mask(adaptor, 2, 10) results = adaptor.compute_diffexp_ttest(maskA, maskB, 10) @@ -58,7 +60,7 @@ class DiffExpTest(unittest.TestCase): def test_cxg_default(self): """Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)""" - adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg") + adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg") maskA = self.get_mask(adaptor, 1, 10) maskB = self.get_mask(adaptor, 2, 10) @@ -72,7 +74,7 @@ class DiffExpTest(unittest.TestCase): def test_cxg_generic(self): """Test a cxg adaptor with the generic adaptor""" - adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg") + adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg") maskA = self.get_mask(adaptor, 1, 10) maskB = self.get_mask(adaptor, 2, 10) # run it directly diff --git a/server/test/test_matrixcache.py b/server/test/test_matrixcache.py index b42f09b4..7a1362ae 100644 --- a/server/test/test_matrixcache.py +++ b/server/test/test_matrixcache.py @@ -7,13 +7,15 @@ import shutil import os import time +from server.test import PROJECT_ROOT + class MatrixCacheTest(unittest.TestCase): def setup(self): pass def make_temporay_datasets(self, dirname, num): - source = "test/test_datasets/pbmc3k.cxg" + source = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg" for i in range(num): target = os.path.join(dirname, str(i) + ".cxg") shutil.copytree(source, target) diff --git a/server/test/test_nan_anndata_adaptor.py b/server/test/test_nan_anndata_adaptor.py index 8c37b7d6..6471dd8b 100644 --- a/server/test/test_nan_anndata_adaptor.py +++ b/server/test/test_nan_anndata_adaptor.py @@ -8,39 +8,22 @@ import server.test.decode_fbs as decode_fbs from server.data_anndata.anndata_adaptor import AnndataAdaptor from server.common.errors import FilterError from server.common.data_locator import DataLocator -from server.common.app_config import AppConfig +from server.test import PROJECT_ROOT, app_config class NaNTest(unittest.TestCase): def setUp(self): - self.args = { - "embeddings__names": ["umap"], - "presentation__max_categories": 100, - "single_dataset__obs_names": None, - "single_dataset__var_names": None, - "diffexp__lfc_cutoff": 0.01, - "limits__diffexp_cellcount_max": None, - "limits__column_request_max": None, - } - config = AppConfig() - config.update(**self.args) - locator = DataLocator("test/test_datasets/nan.h5ad") - config.update(single_dataset__datapath=locator.path) - config.complete_config() + self.data_locator = DataLocator(f"{PROJECT_ROOT}/server/test/test_datasets/nan.h5ad") + self.config = app_config(self.data_locator.path) with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) - self.data = AnndataAdaptor(locator, config) + self.data = AnndataAdaptor(self.data_locator, self.config) self.data._create_schema() def test_load(self): with self.assertWarns(UserWarning): - config = AppConfig() - config.update(**self.args) - locator = DataLocator("test/test_datasets/nan.h5ad") - config.update(single_dataset__datapath=locator.path) - config.complete_config() - self.data = AnndataAdaptor(locator, config) + self.data = AnndataAdaptor(self.data_locator, self.config) def test_init(self): self.assertEqual(self.data.cell_count, 100)