fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-25 12:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Dataframe (#576)

Browse Source 
* initial dataframe commit

* initial dataframe port of core app

* rename variables for clarity

* remove unused import

* comment out unused code

* fix array handling bug in crossfilter dimension creation

* allow creation of empty dataframes

* handle non-existent columns

* handle non-existent columns

* revise tests for new dataframe

* comments for clarity

* comments for clarity

* generate bulk add placeholder with real gene names

* fix bug in gene name adding

* more dataframe unit tests

* fix bug - subset from current world, not universe

* put cut and pasted code into a single function

* improve caching of crossfilter

* remove cascading update bug from graph

* more performance work

* improve state handling for scatterplot

* performance optimization of critical path

* add column summarization

* dataframe utils

* add callOnceLazy

* fix tests

* minor updates found during review

* fix misspelling

* remove RESTv02 from function names

* comment cleanup

* cut/icut col parameter defaults to null

* break up large test

* improve tests and comments on dataframe at/has functions
This commit is contained in:
Bruce Martin
2019-02-22 11:31:34 -08:00
committed by GitHub
parent 57c4e9ff33
commit 6b33315cbe
29 changed files with 1798 additions and 611 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/components/brushableHistogram/index.js
+5 -3
View File
@@ -35,9 +35,11 @@ class HistogramBrush extends React.Component {
.scaleLinear()
.range([this.height - this.marginBottom, 0]);
if (obsAnnotations[0][field] !== undefined) {
if (obsAnnotations.col(field)) {
// recalculate expensive stuff
const allValuesForContinuousFieldAsArray = _.map(obsAnnotations, field);
const allValuesForContinuousFieldAsArray = obsAnnotations
.col(field)
.asArray();
histogramCache.x = d3
.scaleLinear()
@@ -149,7 +151,7 @@ class HistogramBrush extends React.Component {
initializeRanges
} = this.props;
if (obsAnnotations[0][field]) {
if (obsAnnotations.col(field)) {
dispatch({
type: "color by continuous metadata",
colorAccessor: field,
client/src/components/categorical/value.js
+5 -1
View File
@@ -65,7 +65,11 @@ class CategoryValue extends React.Component {
})[0].categories;
}
if (colorAccessor && !isColorBy) {
if (
colorAccessor &&
!isColorBy &&
categoricalSelectionState[colorAccessor]
) {
occupancy = countCategoryValues2D(
metadataField,
colorAccessor,
client/src/components/continuous/continuous.js
+1 -3
View File
@@ -10,7 +10,6 @@ import HistogramBrush from "../brushableHistogram";
@connect(state => ({
ranges: _.get(state.controls.world, "summary.obs", null),
metadata: _.get(state.controls.world, "obsAnnotations", null),
colorAccessor: state.controls.colorAccessor,
colorScale: state.controls.colorScale,
selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
@@ -39,7 +38,7 @@ class Continuous extends React.Component {
}
render() {
const { ranges, obsAnnotations, schema } = this.props;
const { ranges, schema } = this.props;
if (schema && !this.continuousChecked) {
this.hasContinuous = _.some(
schema.annotations.obs,
@@ -73,7 +72,6 @@ class Continuous extends React.Component {
field={key}
isObs
zebra={zebra % 2 === 0}
fieldValues={obsAnnotations}
ranges={value.range}
handleColorAction={this.handleColorAction(key).bind(this)}
/>
client/src/components/geneExpression/cellSetButtons.js
+2 -2
View File
@@ -1,8 +1,8 @@
// jshint esversion: 6
import React from "react";
import _ from "lodash";
import { AnchorButton, Tooltip } from "@blueprintjs/core";
import { connect } from "react-redux";
import { World } from "../../util/stateManager";
@connect()
class CellSetButton extends React.Component {
@@ -14,7 +14,7 @@ class CellSetButton extends React.Component {
eitherCellSetOneOrTwo
} = this.props;
const set = _.map(crossfilter.allFiltered(), "name");
const set = World.getSelectedByIndex(crossfilter);
if (!differential.diffExp) {
/* diffexp needs to be cleared before we store a new set */
client/src/components/geneExpression/index.js
+50 -22
View File
@@ -29,8 +29,7 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
return null;
}
/* the fuzzysort wraps the object with other properties, like a score */
const gene = fuzzySortResult.obj;
const text = gene.name;
const geneName = fuzzySortResult.target;
return (
<MenuItem
@@ -39,39 +38,34 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
// Use of annotations in this way is incorrect and dataset specific.
// See https://github.com/chanzuckerberg/cellxgene/issues/483
// label={gene.n_counts}
key={gene.name}
onClick={g => {
key={geneName}
onClick={g =>
/* this fires when user clicks a menu item */
handleClick(g);
}}
text={text}
handleClick(g)
}
text={geneName}
/>
);
};
const filterGenes = (query, genes) => {
const filterGenes = (query, genes) =>
/* fires on load, once, and then for each character typed into the input */
return fuzzysort.go(query, genes, {
key: "name",
fuzzysort.go(query, genes, {
limit: 5,
threshold: -10000 // don't return bad results
});
};
@connect(state => {
const metadata = _.get(state.controls.world, "obsAnnotations", null);
const ranges = _.get(state.controls.world, "summary.obs", null);
const initializeRanges = _.get(state.controls.world, "summary.obs");
return {
ranges,
metadata,
initializeRanges,
userDefinedGenes: state.controls.userDefinedGenes,
userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
world: state.controls.world,
colorAccessor: state.controls.colorAccessor,
allGeneNames: state.controls.allGeneNames,
differential: state.differential
};
})
@@ -84,6 +78,37 @@ class GeneExpression extends React.Component {
};
}
placeholderGeneNames() {
/*
return a string containing gene name suggestions for use as a user hint.
Eg., Apod, Cd74, ...
Will return a max of 3 genes, totalling 15 characters in length.
Randomly selects gene names.
NOTE: the random selection means it will re-render constantly.
*/
const { world } = this.props;
const { varAnnotations } = world;
const geneNames = varAnnotations.col("name").asArray();
if (geneNames.length > 0) {
const placeholder = [];
let len = geneNames.length;
const maxGeneNameCount = 3;
const maxStrLength = 15;
len = len < maxGeneNameCount ? len : maxGeneNameCount;
for (let i = 0, strLen = 0; i < len && strLen < maxStrLength; i += 1) {
const deal = Math.floor(Math.random() * geneNames.length);
const geneName = geneNames[deal];
placeholder.push(geneName);
strLen += geneName.length + 2; // '2' is the length of a comma and space
}
placeholder.push("...");
return placeholder.join(", ");
}
// default - should never happen.
return "Apod, Cd74, ...";
}
handleClick(g) {
const { world, dispatch, userDefinedGenes } = this.props;
const gene = g.target;
@@ -93,7 +118,7 @@ class GeneExpression extends React.Component {
postUserErrorToast(
"That's too many genes, you can have at most 15 user defined genes"
);
} else if (!_.find(world.varAnnotations, { name: gene })) {
} else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
postUserErrorToast("That doesn't appear to be a valid gene name.");
} else {
dispatch(actions.requestUserDefinedGene(gene));
@@ -116,9 +141,13 @@ class GeneExpression extends React.Component {
const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
genes.forEach(gene => {
if (userDefinedGenes.indexOf(gene) !== -1) {
if (gene.length === 0) {
keepAroundErrorToast("Must enter a gene name.");
} else if (userDefinedGenes.indexOf(gene) !== -1) {
keepAroundErrorToast("That gene already exists");
} else if (!_.find(world.varAnnotations, { name: gene })) {
} else if (
world.varAnnotations.col("name").indexOf(gene) === undefined
) {
keepAroundErrorToast(
`${gene} doesn't appear to be a valid gene name.`
);
@@ -214,8 +243,8 @@ class GeneExpression extends React.Component {
itemRenderer={renderGene.bind(this)}
items={
world && world.varAnnotations
? world.varAnnotations
: [{ name: "No genes" }]
? world.varAnnotations.col("name").asArray()
: ["No genes"]
}
popoverProps={{ minimal: true }}
/>
@@ -245,7 +274,7 @@ class GeneExpression extends React.Component {
this.setState({ bulkAdd: e.target.value });
}}
id="text-input-bulk-add"
placeholder="Apod, Cd74, ..."
placeholder={this.placeholderGeneNames()}
value={bulkAdd}
/>
<Button
@@ -290,8 +319,7 @@ class GeneExpression extends React.Component {
<ExpressionButtons />
{differential.diffExp
? _.map(differential.diffExp, (value, index) => {
const annotations = world.varAnnotations[value[0]];
const { name } = annotations;
const name = world.varAnnotations.at(value[0], "name");
const values = world.varDataCache[name];
if (!values) {
return null;
client/src/components/graph/graph.js
+29 -37
View File
@@ -35,7 +35,8 @@ class Graph extends React.Component {
this.graphPaddingRight = globals.leftSidebarWidth;
this.renderCache = {
positions: null,
colors: null
colors: null,
sizes: null
};
this.state = {
svg: null,
@@ -83,12 +84,13 @@ class Graph extends React.Component {
}
componentDidUpdate(prevProps) {
const { renderCache } = this;
const {
world,
crossfilter,
selectionUpdate,
colorRGB,
responsive
responsive,
selectionUpdate
} = this.props;
const {
reglRender,
@@ -109,35 +111,27 @@ class Graph extends React.Component {
if (regl && world) {
/* update the regl state */
const { obsLayout } = world;
const cellCount = crossfilter.size();
const { obsLayout, nObs } = world;
const X = obsLayout.col("X").asArray();
const Y = obsLayout.col("Y").asArray();
// X/Y positions for each point - a cached value that only
// changes if we have loaded entirely new cell data
//
if (
!this.renderCache.positions ||
selectionUpdate !== prevProps.selectionUpdate
) {
if (!this.renderCache.positions) {
this.renderCache.positions = new Float32Array(2 * cellCount);
}
if (!renderCache.positions || world !== prevProps.world) {
renderCache.positions = new Float32Array(2 * nObs);
const glScaleX = scaleLinear([0, 1], [-1, 1]);
const glScaleY = scaleLinear([0, 1], [1, -1]);
const offset = [d3.mean(obsLayout.X) - 0.5, d3.mean(obsLayout.Y) - 0.5];
const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
for (
let i = 0, { positions } = this.renderCache;
i < cellCount;
i += 1
) {
positions[2 * i] = glScaleX(obsLayout.X[i] - offset[0]);
positions[2 * i + 1] = glScaleY(obsLayout.Y[i] - offset[1]);
for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
positions[2 * i] = glScaleX(X[i] - offset[0]);
positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
}
pointBuffer({
data: this.renderCache.positions,
data: renderCache.positions,
dimension: 2
});
@@ -152,30 +146,28 @@ class Graph extends React.Component {
// could have changed for some other reason, but for now color is
// the only metadata that changes client-side. If this is problematic,
// we could add some sort of color-specific indicator to the app state.
if (!this.renderCache.colors || colorRGB !== prevProps.colorRGB) {
if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
const rgb = colorRGB;
if (!this.renderCache.colors) {
this.renderCache.colors = new Float32Array(3 * rgb.length);
if (!renderCache.colors) {
renderCache.colors = new Float32Array(3 * rgb.length);
}
for (let i = 0, { colors } = this.renderCache; i < rgb.length; i += 1) {
for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
colors.set(rgb[i], 3 * i);
}
colorBuffer({ data: this.renderCache.colors, dimension: 3 });
colorBuffer({ data: renderCache.colors, dimension: 3 });
}
// Sizes for each point - this is presumed to change each time the
// component receives new props. Almost always a true assumption, as
// most property upates are due to changes driving a crossfilter
// selection set change.
//
if (!this.renderCache.sizes) {
this.renderCache.sizes = new Float32Array(cellCount);
// Sizes for each point - updates are triggered only when selected
// obs change
if (!renderCache.sizes || selectionUpdate !== prevProps.selectionUpdate) {
if (!renderCache.sizes) {
renderCache.sizes = new Float32Array(nObs);
}
crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
sizeBuffer({ data: renderCache.sizes, dimension: 1 });
}
crossfilter.fillByIsFiltered(this.renderCache.sizes, 4, 0.2);
sizeBuffer({ data: this.renderCache.sizes, dimension: 1 });
this.count = cellCount;
this.count = nObs;
regl._refresh();
this.reglDraw(
client/src/components/scatterplot/scatterplot.js
+53 -49
View File
@@ -65,12 +65,17 @@ class Scatterplot extends React.Component {
super(props);
this.count = 0;
this.axes = false;
this.state = {
svg: null,
minimized: null,
this.renderCache = {
positions: null,
colors: null,
sizes: null,
xScale: null,
yScale: null
};
this.state = {
svg: null,
minimized: null
};
}
componentDidMount() {
@@ -81,6 +86,7 @@ class Scatterplot extends React.Component {
if (svg && expressionX && expressionY) {
scales = Scatterplot.setupScales(expressionX, expressionY);
this.drawAxesSVG(scales.xScale, scales.yScale, svg);
this.renderCache = { ...this.renderCache, ...scales };
}
const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
@@ -113,8 +119,6 @@ class Scatterplot extends React.Component {
pointBuffer,
colorBuffer,
svg,
xScale: scales ? scales.xScale : null,
yScale: scales ? scales.yScale : null,
reglRender,
camera,
drawPoints
@@ -129,12 +133,11 @@ class Scatterplot extends React.Component {
scatterplotYYaccessor,
expressionX,
expressionY,
colorRGB
colorRGB,
selectionUpdate
} = this.props;
const {
reglRender,
xScale,
yScale,
regl,
pointBuffer,
colorBuffer,
@@ -145,17 +148,12 @@ class Scatterplot extends React.Component {
} = this.state;
if (
world &&
svg &&
xScale &&
yScale &&
scatterplotXXaccessor &&
scatterplotYYaccessor &&
(scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
scatterplotYYaccessor !== prevProps.scatterplotYYaccessor || // was CLU now FTH1 etc
!this.axes) // clicked off the tab and back again, rerender
scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
scatterplotYYaccessor !== prevProps.scatterplotYYaccessor // was CLU now FTH1 etc
) {
this.drawAxesSVG(xScale, yScale, svg);
const scales = Scatterplot.setupScales(expressionX, expressionY);
this.drawAxesSVG(scales.xScale, scales.yScale, svg);
this.renderCache = { ...this.renderCache, ...scales };
}
if (reglRender && this.reglRenderState === "rendering") {
@@ -172,35 +170,51 @@ class Scatterplot extends React.Component {
expressionX &&
expressionY &&
scatterplotXXaccessor &&
scatterplotYYaccessor &&
xScale &&
yScale
scatterplotYYaccessor
) {
const { renderCache } = this;
const { xScale, yScale } = this.renderCache;
const cellCount = expressionX.length;
const positionsBuf = new Float32Array(2 * cellCount);
const colorsBuf = new Float32Array(3 * cellCount);
const sizesBuf = new Float32Array(cellCount);
const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
const glScaleY = scaleLinear([0, height], [-1, 1]);
/*
Construct Vectors
*/
for (let i = 0; i < cellCount; i += 1) {
positionsBuf[2 * i] = glScaleX(xScale(expressionX[i]));
positionsBuf[2 * i + 1] = glScaleY(yScale(expressionY[i]));
// Points change when expressionX or expressionY change.
if (
!renderCache.positions ||
expressionX !== prevProps.expressionX ||
expressionY !== prevProps.expressionY
) {
if (!renderCache.positions) {
renderCache.positions = new Float32Array(2 * cellCount);
}
const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
const glScaleY = scaleLinear([0, height], [-1, 1]);
for (let i = 0, { positions } = renderCache; i < cellCount; i += 1) {
positions[2 * i] = glScaleX(xScale(expressionX[i]));
positions[2 * i + 1] = glScaleY(yScale(expressionY[i]));
}
pointBuffer({ data: renderCache.positions, dimension: 2 });
}
for (let i = 0; i < cellCount; i += 1) {
colorsBuf.set(colorRGB[i], 3 * i);
// Colors for each point - change only when props.colorsRGB change.
if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
if (!renderCache.colors) {
renderCache.colors = new Float32Array(3 * cellCount);
}
for (let i = 0, { colors } = renderCache; i < cellCount; i += 1) {
colors.set(colorRGB[i], 3 * i);
}
colorBuffer({ data: renderCache.colors, dimension: 3 });
}
crossfilter.fillByIsFiltered(sizesBuf, 4, 0.2);
// Sizes for each point - updates are triggered only when selected
// obs change
if (!renderCache.sizes || selectionUpdate !== prevProps.selctionUpdate) {
if (!renderCache.sizes) {
renderCache.sizes = new Float32Array(cellCount);
}
crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
sizeBuffer({ data: renderCache.sizes, dimension: 1 });
}
pointBuffer({ data: positionsBuf, dimension: 2 });
colorBuffer({ data: colorsBuf, dimension: 3 });
sizeBuffer({ data: sizesBuf, dimension: 1 });
this.count = cellCount;
regl._refresh();
@@ -213,16 +227,6 @@ class Scatterplot extends React.Component {
camera
);
}
if (
expressionX &&
expressionY &&
(scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
scatterplotYYaccessor !== prevProps.scatterplotYYaccessor)
) {
const scales = Scatterplot.setupScales(expressionX, expressionY);
this.setState(scales);
}
}
static setupScales(expressionX, expressionY) {