fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-29 09:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

add GET routes for expression data (#1387)

Browse Source 
* add GET routes for expression data

* fix comment typo
This commit is contained in:
Bruce Martin
2020-04-13 11:28:35 -07:00
committed by GitHub
parent 98c2a1359b
commit 136093d583
7 changed files with 323 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/actions/index.js
+83 -92
View File
@@ -1,4 +1,3 @@
import _ from "lodash";
import * as globals from "../globals";
import { Universe, MatrixFBS } from "../util/stateManager";
import * as Dataframe from "../util/dataframe";
@@ -6,7 +5,7 @@ import {
catchErrorsWrap,
doJsonRequest,
doBinaryRequest,
dispatchNetworkErrorMessageToUser
dispatchNetworkErrorMessageToUser,
} from "../util/actionHelpers";
import { PromiseLimit } from "../util/promiseLimit";
import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
@@ -19,23 +18,23 @@ function obsAnnotationFetchAndLoad(dispatch, schema) {
const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
const index = obsAnnotations.index ?? false;
const columns = (obsAnnotations.columns ?? []).filter(
col => col.name !== index
(col) => col.name !== index
);
const plimit = new PromiseLimit(4);
const plimit = new PromiseLimit(5);
return Promise.all(
columns.map(col =>
columns.map((col) =>
plimit.add(() => {
const path = `annotations/obs?annotation-name=${encodeURIComponent(
col.name
)}`;
const url = `${globals.API.prefix}${globals.API.version}${path}`;
return doBinaryRequest(url).then(buffer => {
return doBinaryRequest(url).then((buffer) => {
const df = Universe.matrixFBSToDataframe(buffer);
dispatch({
type: "universe: column load success",
dim: "obsAnnotations",
dataframe: df
dataframe: df,
});
});
})
@@ -52,20 +51,22 @@ function varAnnotationFetchAndLoad(dispatch, schema) {
const names = index ? [index] : [];
return Promise.all(
names
.map(name => {
.map((name) => {
const path = `annotations/var?annotation-name=${encodeURIComponent(
name
)}`;
const url = `${globals.API.prefix}${globals.API.version}${path}`;
return doBinaryRequest(url);
})
.map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
.map(resp =>
resp.then(df =>
.map((rqst) =>
rqst.then((buffer) => Universe.matrixFBSToDataframe(buffer))
)
.map((resp) =>
resp.then((df) =>
dispatch({
type: "universe: column load success",
dim: "varAnnotations",
dataframe: df
dataframe: df,
})
)
)
@@ -77,25 +78,25 @@ return promise fetching layout we need
*/
function layoutFetchAndLoad(dispatch, schema) {
const embeddings = schema?.schema?.layout?.obs ?? [];
const embNames = embeddings.map(e => e.name);
const embNames = embeddings.map((e) => e.name);
const baseURL = `${globals.API.prefix}${globals.API.version}layout/obs`;
const plimit = new PromiseLimit(4);
const plimit = new PromiseLimit(5);
return Promise.all(
embNames.map(e =>
embNames.map((e) =>
plimit.add(() => {
const url = `${baseURL}?layout-name=${encodeURIComponent(e)}`;
return doBinaryRequest(url).then(buffer =>
return doBinaryRequest(url).then((buffer) =>
Universe.matrixFBSToDataframe(buffer)
);
})
)
).then(dfs => {
).then((dfs) => {
const df = Dataframe.Dataframe.empty().withColsFromAll(dfs);
dispatch({
type: "universe: column load success",
dim: "obsLayout",
dataframe: df
dataframe: df,
});
});
}
@@ -108,7 +109,7 @@ Bootstrap application with the initial data loading.
* /layout - all default layout
*/
const doInitialDataLoad = () =>
catchErrorsWrap(async dispatch => {
catchErrorsWrap(async (dispatch) => {
dispatch({ type: "initial data load start" });
try {
@@ -116,8 +117,8 @@ const doInitialDataLoad = () =>
Step 1 - config & schema, all JSON
*/
const requestJson = ["config", "schema"]
.map(r => `${globals.API.prefix}${globals.API.version}${r}`)
.map(url => doJsonRequest(url));
.map((r) => `${globals.API.prefix}${globals.API.version}${r}`)
.map((url) => doJsonRequest(url));
const stepOneResults = await Promise.all(requestJson);
/* set config defaults */
const config = { ...globals.configDefaults, ...stepOneResults[0].config };
@@ -125,11 +126,11 @@ const doInitialDataLoad = () =>
const universe = Universe.createUniverseFromResponse(config, schema);
dispatch({
type: "universe exists, but loading is still in progress",
universe
universe,
});
dispatch({
type: "configuration load complete",
config
config,
});
/*
@@ -137,7 +138,7 @@ const doInitialDataLoad = () =>
*/
await Promise.all([
layoutFetchAndLoad(dispatch, schema),
varAnnotationFetchAndLoad(dispatch, schema)
varAnnotationFetchAndLoad(dispatch, schema),
]);
/*
@@ -147,7 +148,7 @@ const doInitialDataLoad = () =>
dispatch({
type: "initial data load complete (universe exists)",
universe
universe,
});
} catch (error) {
dispatch({ type: "initial data load error", error });
@@ -164,7 +165,7 @@ const setWorldToSelection = () => (dispatch, getState) => {
type: "set World to current selection",
universe,
world,
crossfilter
crossfilter,
});
};
@@ -175,6 +176,11 @@ const dispatchExpressionErrors = (dispatch, res) => {
throw new Error(msg);
};
/* double URI encode - needed for query-param filters */
function dubEncURIComponent(s) {
return encodeURIComponent(encodeURIComponent(s));
}
/*
Fetch expression vectors for each gene in genes. This is NOT an action
function, but rather a helper to be called from an action helper that
@@ -188,51 +194,31 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
const varIndexName = universe.schema.annotations.var.index;
/* helper for this function only */
const fetchData = async geneNames => {
const res = await fetch(
`${globals.API.prefix}${globals.API.version}data/var`,
{
method: "PUT",
body: JSON.stringify({
filter: {
var: {
annotation_value: [{ name: varIndexName, values: geneNames }]
}
}
}),
headers: new Headers({
accept: "application/octet-stream",
"Content-Type": "application/json"
}),
credentials: "include"
}
const fetchData = async (geneNames) => {
const query = geneNames
.map(
(g) =>
`var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`
)
.join("&");
const url = `${globals.API.prefix}${globals.API.version}data/var?${query}`;
return doBinaryRequest(url).then((buffer) =>
// TODO: why convert to an Object and not a Dataframe?
Universe.convertDataFBStoObject(universe, buffer)
);
if (
!res.ok ||
res.headers.get("Content-Type") !== "application/octet-stream"
) {
// WILL throw
return dispatchExpressionErrors(dispatch, res);
}
const data = await res.arrayBuffer();
return Universe.convertDataFBStoObject(universe, data);
};
/* preload data already in cache */
let expressionData = _.transform(
genes,
(expData, g) => {
const data = universe.varData.col(g);
if (data) {
expData[g] = data.asArray();
}
},
{}
); // --> { gene: data }
let expressionData = genes.reduce((acc, g) => {
const data = universe.varData.col(g);
if (data) {
acc[g] = data.asArray();
}
return acc;
}, {}); // --> { gene: data }
/* make a list of genes for which we do not have data */
const genesToFetch = _.filter(genes, g => expressionData[g] === undefined);
const genesToFetch = genes.filter((g) => expressionData[g] === undefined);
dispatch({ type: "expression load start" });
@@ -242,7 +228,7 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
const newExpressionData = await fetchData(genesToFetch);
expressionData = {
...expressionData,
...newExpressionData
...newExpressionData,
};
} catch (error) {
dispatch({ type: "expression load error", error });
@@ -264,19 +250,19 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
type: "color by expression",
gene,
data: {
[gene]: world.varData.col(gene).asArray()
}
[gene]: world.varData.col(gene).asArray(),
},
});
} catch (error) {
dispatch({
type: "get single gene expression for coloring error",
error
error,
});
}
};
}
const requestUserDefinedGene = gene => async (dispatch, getState) => {
const requestUserDefinedGene = (gene) => async (dispatch, getState) => {
dispatch({ type: "request user defined gene started" });
try {
await await _doRequestExpressionData(dispatch, getState, [gene]);
@@ -287,13 +273,13 @@ const requestUserDefinedGene = gene => async (dispatch, getState) => {
type: "request user defined gene success",
data: {
genes: [gene],
expression: world.varData.col(gene).asArray()
}
expression: world.varData.col(gene).asArray(),
},
});
} catch (error) {
return dispatch({
type: "request user defined gene error",
error
error,
});
}
};
@@ -315,7 +301,7 @@ const dispatchDiffExpErrors = (dispatch, response) => {
dispatchNetworkErrorMessageToUser(msg);
dispatch({
type: "request differential expression error",
error: new Error(msg)
error: new Error(msg),
});
}
}
@@ -353,15 +339,15 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
method: "POST",
headers: new Headers({
Accept: "application/json",
"Content-Type": "application/json"
"Content-Type": "application/json",
}),
body: JSON.stringify({
mode: "topN",
count: num_genes,
set1: { filter: { obs: { index: set1 } } },
set2: { filter: { obs: { index: set2 } } }
set2: { filter: { obs: { index: set2 } } },
}),
credentials: "include"
credentials: "include",
}
);
@@ -371,7 +357,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
const data = await res.json();
// result is [ [varIdx, ...], ... ]
const topNGenes = _.map(data, r =>
const topNGenes = data.map((r) =>
universe.varAnnotations.at(r[0], varIndexName)
);
@@ -379,17 +365,22 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
Kick off secondary action to fetch all of the expression data for the
topN expressed genes.
*/
await _doRequestExpressionData(dispatch, getState, topNGenes);
const plimit = new PromiseLimit(5);
await Promise.all(
topNGenes.map((gene) =>
plimit.add(() => _doRequestExpressionData(dispatch, getState, [gene]))
)
);
/* then send the success case action through */
return dispatch({
type: "request differential expression success",
data
data,
});
} catch (error) {
return dispatch({
type: "request differential expression error",
error
error,
});
}
};
@@ -399,7 +390,7 @@ const resetWorldToUniverse = () => (dispatch, getState) => {
reembedResetWorldToUniverse(dispatch, getState);
dispatch({
type: "reset World to eq Universe",
universe
universe,
});
};
@@ -409,12 +400,12 @@ const saveObsAnnotations = () => async (dispatch, getState) => {
const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
dispatch({
type: "writable obs annotations - save started"
type: "writable obs annotations - save started",
});
const writableAnnotations = schema.annotations.obs.columns
.filter(s => s.writable)
.map(s => s.name);
.filter((s) => s.writable)
.map((s) => s.name);
const df = obsAnnotations.subset(null, writableAnnotations);
const matrix = MatrixFBS.encodeMatrixFBS(df);
try {
@@ -430,28 +421,28 @@ const saveObsAnnotations = () => async (dispatch, getState) => {
method: "PUT",
body: matrix,
headers: new Headers({
"Content-Type": "application/octet-stream"
"Content-Type": "application/octet-stream",
}),
credentials: "include"
credentials: "include",
}
);
if (res.ok) {
dispatch({
type: "writable obs annotations - save complete",
obsAnnotations
obsAnnotations,
});
} else {
dispatch({
type: "writable obs annotations - save error",
message: `HTTP error ${res.status} - ${res.statusText}`,
res
res,
});
}
} catch (error) {
dispatch({
type: "writable obs annotations - save error",
message: error.toString(),
error
error,
});
}
};
@@ -464,5 +455,5 @@ export default {
requestReembed,
resetWorldToUniverse,
saveObsAnnotations,
setWorldToSelection
setWorldToSelection,
};
client/src/reducers/controls.js
Vendored
+27 -26
View File
@@ -20,8 +20,6 @@ const Controls = (
scatterplotXXaccessor: null, // just easier to read
scatterplotYYaccessor: null,
graphRenderCounter: 0 /* integer as <Component key={graphRenderCounter} - a change in key forces a remount */,
__storedStateForCelllist1__: null /* will need procedural control of brush ie., brush.extent https://bl.ocks.org/micahstubbs/3cda05ca68cba260cb81 */,
__storedStateForCelllist2__: null
},
action,
nextSharedState,
@@ -62,7 +60,7 @@ const Controls = (
universeExists &&
embeddingsExist &&
varAnnotationsIndexExists
)
),
};
}
case "initial data load complete (universe exists)": {
@@ -71,38 +69,42 @@ const Controls = (
...state,
loading: false,
error: null,
resettingInterface: false
resettingInterface: false,
};
}
case "reset World to eq Universe": {
const [ newUserDefinedGenes, newDiffExpGenes ] = subsetAndResetGeneLists(state);
const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
state
);
return {
...state,
resettingInterface: false,
userDefinedGenes: newUserDefinedGenes,
diffexpGenes: newDiffExpGenes
diffexpGenes: newDiffExpGenes,
};
}
case "set World to current selection": {
const [ newUserDefinedGenes, newDiffExpGenes ] = subsetAndResetGeneLists(state);
const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
state
);
return {
...state,
loading: false,
error: null,
userDefinedGenes: newUserDefinedGenes,
diffexpGenes: newDiffExpGenes
diffexpGenes: newDiffExpGenes,
};
}
case "request user defined gene started": {
return {
...state,
userDefinedGenesLoading: true
userDefinedGenesLoading: true,
};
}
case "request user defined gene error": {
return {
...state,
userDefinedGenesLoading: false
userDefinedGenesLoading: false,
};
}
case "request user defined gene success": {
@@ -113,50 +115,49 @@ const Controls = (
return {
...state,
userDefinedGenes: _userDefinedGenes,
userDefinedGenesLoading: false
userDefinedGenesLoading: false,
};
}
case "request differential expression success": {
const { world } = prevSharedState;
const varIndexName = world.schema.annotations.var.index;
const _diffexpGenes = [];
action.data.forEach(d => {
action.data.forEach((d) => {
_diffexpGenes.push(world.varAnnotations.at(d[0], varIndexName));
});
return {
...state,
diffexpGenes: _diffexpGenes
diffexpGenes: _diffexpGenes,
};
}
case "clear differential expression": {
return {
...state,
diffexpGenes: []
diffexpGenes: [],
};
}
case "clear user defined gene": {
const { userDefinedGenes } = state;
const newUserDefinedGenes = _.filter(
userDefinedGenes,
d => d !== action.data
(d) => d !== action.data
);
return {
...state,
userDefinedGenes: newUserDefinedGenes
userDefinedGenes: newUserDefinedGenes,
};
}
case "clear all user defined genes": {
return {
...state,
userDefinedGenes: []
userDefinedGenes: [],
};
}
case "expression load error":
case "initial data load error": {
return {
...state,
loading: false,
error: action.error
error: action.error,
};
}
@@ -166,24 +167,24 @@ const Controls = (
case "change graph interaction mode":
return {
...state,
graphInteractionMode: action.data
graphInteractionMode: action.data,
};
case "change opacity deselected cells in 2d graph background":
return {
...state,
opacityForDeselectedCells: action.data
opacityForDeselectedCells: action.data,
};
case "increment graph render counter": {
const c = state.graphRenderCounter + 1;
return {
...state,
graphRenderCounter: c
graphRenderCounter: c,
};
}
case "interface reset started": {
return {
...state,
resettingInterface: true
resettingInterface: true,
};
}
@@ -193,18 +194,18 @@ const Controls = (
case "set scatterplot x":
return {
...state,
scatterplotXXaccessor: action.data
scatterplotXXaccessor: action.data,
};
case "set scatterplot y":
return {
...state,
scatterplotYYaccessor: action.data
scatterplotYYaccessor: action.data,
};
case "clear scatterplot":
return {
...state,
scatterplotXXaccessor: null,
scatterplotYYaccessor: null
scatterplotYYaccessor: null,
};
default:
client/src/util/actionHelpers.js
+24 -21
View File
@@ -6,7 +6,7 @@ import { postNetworkErrorToast } from "../components/framework/toasters";
dispatch an action error to the user. Currently we use
async toasts.
*/
export const dispatchNetworkErrorMessageToUser = message =>
export const dispatchNetworkErrorMessageToUser = (message) =>
postNetworkErrorToast(message);
/*
@@ -14,7 +14,7 @@ Catch unexpected errors and make sure we don't lose them!
*/
export function catchErrorsWrap(fn, dispatchToUser = false) {
return (dispatch, getState) => {
fn(dispatch, getState).catch(error => {
fn(dispatch, getState).catch((error) => {
console.error(error);
if (dispatchToUser) {
dispatchNetworkErrorMessageToUser(error.message);
@@ -29,30 +29,33 @@ Wrapper to perform async fetch with some modest error handling
and decoding.
*/
const doFetch = async (url, acceptType) => {
const res = await fetch(url, {
method: "get",
headers: new Headers({
Accept: acceptType
}),
credentials: "include"
});
if (res.ok && res.headers.get("Content-Type").includes(acceptType)) {
return res;
try {
const res = await fetch(url, {
method: "get",
headers: new Headers({
Accept: acceptType,
}),
credentials: "include",
});
if (res.ok && res.headers.get("Content-Type").includes(acceptType)) {
return res;
}
// else an error
const msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
dispatchNetworkErrorMessageToUser(msg);
throw new Error(msg);
} catch (e) {
// network error
const msg = "Unexpected HTTP error";
dispatchNetworkErrorMessageToUser(msg);
throw e;
}
// else an error
let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
const body = await res.text();
if (body && body.length > 0) {
msg = `${msg} -- ${body}`;
}
dispatchNetworkErrorMessageToUser(msg);
throw new Error(msg);
};
/*
Wrapper to perform an async fetch and JSON decode response.
*/
export const doJsonRequest = async url => {
export const doJsonRequest = async (url) => {
const res = await doFetch(url, "application/json");
return res.json();
};
@@ -60,7 +63,7 @@ export const doJsonRequest = async url => {
/*
Wrapper to perform an async fetch for binary data.
*/
export const doBinaryRequest = async url => {
export const doBinaryRequest = async (url) => {
const res = await doFetch(url, "application/octet-stream");
return res.arrayBuffer();
};