diff --git a/.travis.yml b/.travis.yml index 0333cbda..92c6e054 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,13 +7,11 @@ cache: pip: true install: - set -eo pipefail - - pip install flake8 httpie + - pip install flake8 - ./bin/build-client - pip install -e . + - pip install -r server/requirements-dev.txt script: - set -eo pipefail - flake8 server/app/ - - pytest -s server/test/test_filter.py server/test/test_scanpy_engine.py - - cellxgene scanpy example-dataset/ & - - for i in {1..90}; do if http :5005/api/v0.1/initialize > /dev/null; then break; else echo "Waiting for server..."; sleep 1; fi; done - - pytest server/test/test_api.py + - pytest -s server/test diff --git a/client/index_template.html b/client/index_template.html index 490167fe..f24eac99 100644 --- a/client/index_template.html +++ b/client/index_template.html @@ -16,9 +16,8 @@ window.CELLXGENE = {}; window.CELLXGENE.API = { prefix: "{{ prefix | safe }}", - version: "v0.1/" + version: "v0.2/" }; -window.CELLXGENE.datasetTitle = "{{ datasetTitle }}"; diff --git a/client/src/actions/index.js b/client/src/actions/index.js index 6034c266..038a5204 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -1,60 +1,30 @@ // jshint esversion: 6 import _ from "lodash"; -import memoize from "memoize-one"; import * as globals from "../globals"; -import store from "../reducers"; -import { Universe } from "../util/stateManager"; +import { Universe, kvCache } from "../util/stateManager"; +import { catchErrorsWrap, doJsonRequest } from "../util/actionHelpers"; -/* -Catch unexpected errors and make sure we don't lose them! -*/ -function catchErrorsWrap(fn) { - return (dispatch, getState) => { - fn(dispatch, getState).catch(error => { - console.error(error); - dispatch({ type: "UNEXPECTED ERROR", error }); - }); - }; -} - -async function doRequestInitialize() { - const res = await fetch( - `${globals.API.prefix}${globals.API.version}initialize`, - { - method: "get", - headers: new Headers({ - "Content-Type": "application/json" - }) - } - ); - return res.json(); -} - -async function doRequestCells(query) { - const res = await fetch( - `${globals.API.prefix}${globals.API.version}cells${query}`, - { - method: "get", - headers: new Headers({ - "Content-Type": "application/json" - }) - } - ); - return res.json(); -} - -function doInitialDataLoad(query = "") { - return catchErrorsWrap(async dispatch => { +const doInitialDataLoad = () => + catchErrorsWrap(async dispatch => { dispatch({ type: "initial data load start" }); + try { - const res = await Promise.all([ - doRequestInitialize(), - doRequestCells(query) - ]); - const universe = Universe.createUniverseFromRESTv01Response( - res[0], - res[1] - ); + const requests = _([ + "config", + "schema", + "annotations/obs", + "annotations/var", + "layout/obs" + ]) + .map(r => `${globals.API.prefix}${globals.API.version}${r}`) + .map(url => doJsonRequest(url)) + .value(); + const results = await Promise.all(requests); + const universe = Universe.createUniverseFromRestV02Response(...results); + dispatch({ + type: "configuration load complete", + config: results[0].config + }); dispatch({ type: "initial data load complete (universe exists)", universe @@ -63,7 +33,6 @@ function doInitialDataLoad(query = "") { dispatch({ type: "initial data load error", error }); } }); -} // XXX TODO - this is the old code for doing a regraph. Preserving it solely // until we port to 0.2 API. The new UX for regraph can't be implemented on @@ -131,66 +100,56 @@ const resetGraph = () => (dispatch, getState) => universe: getState().controls.universe }); -// This code defends against the case where /expression returns a cellname -// never seen before (ie, not returned by /cells). This should not happen -// (see https://github.com/chanzuckerberg/cellxgene-rest-api/issues/34) but -// occasionally does. -// -// XXX TODO - this code is only relevant in v0.1 REST API, and can be retired -// when we port to 0.2. -// -const makeMetadataMap = memoize(metadata => _.keyBy(metadata, "CellName")); -function cleanupExpressionResponse(data) { - const s = store.getState(); - const { universe } = s.controls; - const metadata = makeMetadataMap(universe.obsAnnotations); - let errorFound = false; - data.data.cells = _.filter(data.data.cells, cell => { - if (!errorFound && !metadata[cell.cellname]) { - errorFound = true; - console.error( - "Warning: /expression REST API returned unexpected cell names -- discarding surprises." - ); - } - return metadata[cell.cellname]; - }); - - return data; -} - /* -Fetch [gene, ...] from V0.1 API. Not an action function - just a helper -which implements the new expression data caching. +Fetch expression vectors for each gene in genes. This is NOT an action +function, but rather a helper to be called from an action helper that +needs expression data. + +Transparently utilizes cached data if it is already present. */ async function _doRequestExpressionData(dispatch, getState, genes) { const state = getState(); - /* check cache and only fetch data we do not already have */ const { universe } = state.controls; - const genesToFetch = _.filter(genes, g => !universe.varDataCache[g]); + /* preload data already in cache */ + let expressionData = _.transform(genes, (expData, g) => { + const data = kvCache.get(universe.varDataCache, g); + if (data) { + expData[g] = data; + } + }); // --> { gene: data } + /* make a list of genes for which we do not have data */ + const genesToFetch = _.filter(genes, g => expressionData[g] === undefined); dispatch({ type: "expression load start" }); - let expressionData = {}; // { gene: data } + + /* Fetch data for any genes not in cache */ if (genesToFetch.length) { try { + // XXX: TODO - this could be using /data/var rather than /data/obs, + // as that would simplify the transformation in + // convertExpressionRESTv02ToObject const res = await fetch( - `${globals.API.prefix}${globals.API.version}expression`, + `${globals.API.prefix}${globals.API.version}data/obs`, { - method: "POST", + method: "PUT", body: JSON.stringify({ - genelist: genes + filter: { + var: { + annotation_value: [{ name: "name", values: genesToFetch }] + } + } }), headers: new Headers({ accept: "application/json", + "Accept-Encoding": "gzip, deflate, br", "Content-Type": "application/json" }) } ); - let data = await res.json(); - data = cleanupExpressionResponse(data); - data = Universe.convertExpressionRESTv01ToObject(universe, data); + const data = await res.json(); expressionData = { ...expressionData, - ...data + ...Universe.convertExpressionRESTv02ToObject(universe, data) }; } catch (error) { dispatch({ type: "expression load error", error }); @@ -198,26 +157,24 @@ async function _doRequestExpressionData(dispatch, getState, genes) { } } - // add the cached values - _.forEach(genes, g => { - if (expressionData[g] === undefined) { - expressionData[g] = universe.varDataCache[g]; - } - }); - - return dispatch({ type: "expression load success", expressionData }); + dispatch({ type: "expression load success", expressionData }); + return expressionData; } function requestSingleGeneExpressionCountsForColoringPOST(gene) { return async (dispatch, getState) => { dispatch({ type: "get single gene expression for coloring started" }); try { - await _doRequestExpressionData(dispatch, getState, [gene]); + const expressionData = await _doRequestExpressionData( + dispatch, + getState, + [gene] + ); dispatch({ type: "color by expression", gene, data: { - [gene]: getState().controls.world.varDataCache[gene] + [gene]: expressionData[gene] } }); } catch (error) { @@ -232,65 +189,74 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) { const requestGeneExpressionCountsPOST = genes => async (dispatch, getState) => { dispatch({ type: "get expression started" }); try { - await _doRequestExpressionData(dispatch, getState, genes); + const expressionData = await _doRequestExpressionData( + dispatch, + getState, + genes + ); return dispatch({ type: "get expression success", genes, - data: _.transform( - genes, - (res, gene) => { - res[gene] = getState().controls.world.varDataCache[gene]; - }, - {} - ) + data: expressionData }); } catch (error) { return dispatch({ type: "get expression error", error }); } }; -const requestDifferentialExpression = ( - celllist1, - celllist2, - num_genes = 7 -) => dispatch => { +const requestDifferentialExpression = (set1, set2, num_genes = 10) => async ( + dispatch, + getState +) => { dispatch({ type: "request differential expression started" }); - fetch(`${globals.API.prefix}${globals.API.version}diffexpression`, { - method: "POST", - body: JSON.stringify({ - celllist1, - celllist2, - num_genes - }), - headers: new Headers({ - accept: "application/json", - "Content-Type": "application/json" - }) - }) - .then(res => res.json()) - .then( - data => { - /* - kick off a secondary action to get all expression counts for all cells - now that we know what the top expressed are - */ - dispatch( - requestGeneExpressionCountsPOST( - _.union(data.data.celllist1.topgenes, data.data.celllist2.topgenes) - ) - ); - /* then send the success case action through */ - return dispatch({ - type: "request differential expression success", - data - }); - }, - error => - dispatch({ - type: "request differential expression error", - error + try { + /* + Steps: + 1. get the most differentially expressed genes + 2. get expression data for each + */ + const state = getState(); + const { universe } = state.controls; + const set1ByIndex = _.map(set1, s => universe.obsNameToIndexMap[s]); + const set2ByIndex = _.map(set2, s => universe.obsNameToIndexMap[s]); + const diffExpFetch = await fetch( + `${globals.API.prefix}${globals.API.version}diffexp/obs`, + { + method: "POST", + headers: new Headers({ + Accept: "application/json", + "Accept-Encoding": "gzip, deflate, br", + "Content-Type": "application/json" + }), + body: JSON.stringify({ + mode: "topN", + count: num_genes, + set1: { filter: { obs: { index: set1ByIndex } } }, + set2: { filter: { obs: { index: set2ByIndex } } } }) + } ); + const data = await diffExpFetch.json(); + // result is [ [varIdx, ...], ... ] + const topNGenes = _.map(data, r => universe.varAnnotations[r[0]].name); + + /* + Kick off secondary action to fetch all of the expression data for the + topN expressed genes. + */ + dispatch(requestGeneExpressionCountsPOST(topNGenes)); + + /* then send the success case action through */ + return dispatch({ + type: "request differential expression success", + data + }); + } catch (error) { + return dispatch({ + type: "request differential expression error", + error + }); + } }; export default { diff --git a/client/src/components/expression/diffExpHeatmap.js b/client/src/components/expression/diffExpHeatmap.js index 08b2fde5..6cbf9f32 100644 --- a/client/src/components/expression/diffExpHeatmap.js +++ b/client/src/components/expression/diffExpHeatmap.js @@ -3,15 +3,13 @@ import React from "react"; import _ from "lodash"; import memoize from "memoize-one"; import { connect } from "react-redux"; -import * as globals from "../../globals"; -import styles from "./expression.css"; -import SectionHeader from "../framework/sectionHeader"; -import actions from "../../actions"; import ReactAutocomplete from "react-autocomplete"; /* http://emilebres.github.io/react-virtualized-checkbox/ */ import getContrast from "font-color-contrast"; // https://www.npmjs.com/package/font-color-contrast import FaPaintBrush from "react-icons/lib/fa/paint-brush"; import * as d3 from "d3"; import { interpolateGreys } from "d3-scale-chromatic"; +import * as globals from "../../globals"; +import actions from "../../actions"; class HeatmapSquare extends React.Component { constructor(props) { @@ -22,9 +20,10 @@ class HeatmapSquare extends React.Component { } render() { + const { backgroundColor, text } = this.props; const contrastColor = getContrast( - this.props.backgroundColor - .substring(4, this.props.backgroundColor.length - 1) + backgroundColor + .substring(4, backgroundColor.length - 1) .replace(/ /g, "") .split(",") ); @@ -38,10 +37,10 @@ class HeatmapSquare extends React.Component { flexShrink: 0, fontSize: 12, margin: 0, - backgroundColor: this.props.backgroundColor + backgroundColor }} > - {this.props.text} + {text}

); } @@ -54,13 +53,11 @@ class HeatmapSquare extends React.Component { *********************************** *********************************** **********************************/ -@connect(state => { - return { - scatterplotXXaccessor: state.controls.scatterplotXXaccessor, - scatterplotYYaccessor: state.controls.scatterplotYYaccessor, - colorAccessor: state.controls.colorAccessor - }; -}) +@connect(state => ({ + scatterplotXXaccessor: state.controls.scatterplotXXaccessor, + scatterplotYYaccessor: state.controls.scatterplotYYaccessor, + colorAccessor: state.controls.colorAccessor +})) class HeatmapRow extends React.Component { constructor(props) { super(props); @@ -69,35 +66,44 @@ class HeatmapRow extends React.Component { }; } - handleGeneColorScaleClick(gene) { + handleGeneColorScaleClick() { return () => { - this.props.dispatch( - actions.requestSingleGeneExpressionCountsForColoringPOST( - this.props.gene - ) - ); + const { dispatch, gene } = this.props; + dispatch(actions.requestSingleGeneExpressionCountsForColoringPOST(gene)); }; } - handleSetGeneAsScatterplotX(gene) { + handleSetGeneAsScatterplotX() { return () => { - this.props.dispatch({ + const { dispatch, gene } = this.props; + dispatch({ type: "set scatterplot x", - data: this.props.gene + data: gene }); }; } - handleSetGeneAsScatterplotY(gene) { + handleSetGeneAsScatterplotY() { return () => { - this.props.dispatch({ + const { dispatch, gene } = this.props; + dispatch({ type: "set scatterplot y", - data: this.props.gene + data: gene }); }; } render() { + const { + gene, + aveDiff, + set1exp, + set2exp, + greyColorScale, + scatterplotXXaccessor, + scatterplotYYaccessor, + colorAccessor + } = this.props; return (
- {this.props.gene} + {gene}
- {this.props.aveDiff.toFixed(2)} + {aveDiff.toFixed(2)} X Y @@ -214,12 +207,10 @@ class HeatmapRow extends React.Component { *********************************** **********************************/ -@connect(state => { - return { - differential: state.differential, - world: state.controls.world - }; -}) +@connect(state => ({ + differential: state.differential, + world: state.controls.world +})) class Heatmap extends React.Component { constructor(props) { super(props); @@ -228,24 +219,31 @@ class Heatmap extends React.Component { }; } - getAllGeneNames = memoize(world => - _.map(this.props.world.varAnnotations, "name") - ); + // XXX TODO unused at the moment + // getAllGeneNames = memoize(world => + // _.map(this.props.world.varAnnotations, "name") + // ); render() { - if (!this.props.differential.diffExp) + const { world, differential } = this.props; + if (!differential.diffExp) { return

Select cells & compute differential to see heatmap

; + } - const topGenesForCellSet1 = this.props.differential.diffExp.data.celllist1; - const topGenesForCellSet2 = this.props.differential.diffExp.data.celllist2; - // const allGeneNames = this.getAllGeneNames(this.props.world); + // summarize the information for display. + const topGenes = _.map(differential.diffExp, val => ({ + varIndex: val[0], + geneName: world.varAnnotations[val[0]].name, + avgDiff: val[1], + set1AvgExp: val[4], + set2AvgExp: val[5] + })); + // average expression extent const extent = d3.extent( - _.union( - topGenesForCellSet1.mean_expression_cellset1, - topGenesForCellSet1.mean_expression_cellset2, - topGenesForCellSet2.mean_expression_cellset1, - topGenesForCellSet2.mean_expression_cellset2 + _.concat( + _.map(differential.diffExp, val => val[4]), + _.map(differential.diffExp, val => val[5]) ) ); @@ -269,35 +267,16 @@ class Heatmap extends React.Component {

2

ave diff

- {topGenesForCellSet1.topgenes.map((gene, i) => { + {topGenes.map(g => { + const { geneName, avgDiff, set1AvgExp, set2AvgExp } = g; return ( - ); - })} - {topGenesForCellSet2.topgenes.map((gene, i) => { - return ( - ); })} diff --git a/client/src/components/expression/expressionButtons.js b/client/src/components/expression/expressionButtons.js index a49102c0..7712e608 100644 --- a/client/src/components/expression/expressionButtons.js +++ b/client/src/components/expression/expressionButtons.js @@ -6,14 +6,12 @@ import * as globals from "../../globals"; import actions from "../../actions"; import CellSetButton from "./cellSetButtons"; -@connect(state => { - return { - differential: state.differential, - world: state.controls.world, - crossfilter: _.get(state.controls, "crossfilter", null), - selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null) - }; -}) +@connect(state => ({ + differential: state.differential, + world: state.controls.world, + crossfilter: state.controls.crossfilter, + selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null) +})) class Expression extends React.Component { constructor(props) { super(props); @@ -22,24 +20,27 @@ class Expression extends React.Component { handleClick(gene) { return () => { - this.props.dispatch({ + const { dispatch } = this.props; + dispatch({ type: "color by expression", - gene: gene + gene }); }; } computeDiffExp() { - this.props.dispatch( + const { dispatch, differential } = this.props; + dispatch( actions.requestDifferentialExpression( - this.props.differential.celllist1, - this.props.differential.celllist2 + differential.celllist1, + differential.celllist2 ) ); } render() { - if (!this.props.differential) { + const { differential } = this.props; + if (!differential) { return null; } return ( @@ -47,6 +48,7 @@ class Expression extends React.Component {