fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-27 04:18:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Dataframe, part deux - add varData and summarize() (#608)

Browse Source 
* initial dataframe commit

* initial dataframe port of core app

* rename variables for clarity

* remove unused import

* comment out unused code

* fix array handling bug in crossfilter dimension creation

* allow creation of empty dataframes

* handle non-existent columns

* handle non-existent columns

* revise tests for new dataframe

* comments for clarity

* comments for clarity

* generate bulk add placeholder with real gene names

* fix bug in gene name adding

* more dataframe unit tests

* fix bug - subset from current world, not universe

* put cut and pasted code into a single function

* improve caching of crossfilter

* remove cascading update bug from graph

* more performance work

* improve state handling for scatterplot

* performance optimization of critical path

* add column summarization

* dataframe utils

* add callOnceLazy

* fix tests

* minor updates found during review

* fix misspelling

* remove RESTv02 from function names

* comment cleanup

* cut/icut col parameter defaults to null

* break up large test

* improve tests and comments on dataframe at/has functions

* add Dataframe withCol/dropCol

* expression varData now stored in a dataframe

* dead code cleanup

* use dataframe.summarize()

* test cases for Dataframe.col.summarize

* update test cases for new dataframe summarize

* improve naming

* use new hasCol API

* add comments

* add more Dataframe.withCol tests

* add ability to specify row index in cut operation

* retire subsetVarData function

* correctly handle expression subsetting

* lint and improve comments

* rename cut to subset

* changes based on PR review
This commit is contained in:
Bruce Martin
2019-02-28 08:34:22 -08:00
committed by GitHub
parent 2bae696986
commit ffd6273419
21 changed files with 910 additions and 1140 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/actions/index.js
+13 -10
View File
@@ -1,12 +1,11 @@
// jshint esversion: 6
import _ from "lodash";
import * as globals from "../globals";
import { Universe, kvCache } from "../util/stateManager";
import { Universe } from "../util/stateManager";
import {
catchErrorsWrap,
doJsonRequest,
doBinaryRequest,
rangeEncodeIndices,
dispatchNetworkErrorMessageToUser
} from "../util/actionHelpers";
@@ -130,9 +129,9 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
let expressionData = _.transform(
genes,
(expData, g) => {
const data = kvCache.get(universe.varDataCache, g);
const data = universe.varData.col(g);
if (data) {
expData[g] = data;
expData[g] = data.asArray();
}
},
{}
@@ -170,7 +169,7 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
type: "color by expression",
gene,
data: {
[gene]: kvCache.get(world.varDataCache, gene)
[gene]: world.varData.col(gene).asArray()
}
});
} catch (error) {
@@ -193,7 +192,7 @@ const requestUserDefinedGene = gene => async (dispatch, getState) => {
type: "request user defined gene success",
data: {
genes: [gene],
expression: kvCache.get(world.varDataCache, gene)
expression: world.varData.col(gene).asArray()
}
});
} catch (error) {
@@ -243,12 +242,16 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
const state = getState();
const { universe } = state.controls;
// Legal values are null, Array or TypedArray. Null is initial state.
if (!set1) set1 = [];
if (!set2) set2 = [];
// These lines ensure that we convert any TypedArray to an Array.
// This is necessary because JSON.stringify() does some very strange
// things with TypedArrays (they are marshalled to JSON objects, rather
// than being marshalled as a JSON array).
const aset1 = Array.isArray(set1) ? set1 : Array.from(set1);
const aset2 = Array.isArray(set2) ? set2 : Array.from(set2);
set1 = Array.isArray(set1) ? set1 : Array.from(set1);
set2 = Array.isArray(set2) ? set2 : Array.from(set2);
const res = await fetch(
`${globals.API.prefix}${globals.API.version}diffexp/obs`,
@@ -261,8 +264,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
body: JSON.stringify({
mode: "topN",
count: num_genes,
set1: { filter: { obs: { index: aset1 } } },
set2: { filter: { obs: { index: aset2 } } }
set1: { filter: { obs: { index: set1 } } },
set2: { filter: { obs: { index: set2 } } }
})
}
);
client/src/components/brushableHistogram/index.js
+7 -16
View File
@@ -10,7 +10,6 @@ import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
import { connect } from "react-redux";
import * as d3 from "d3";
import memoize from "memoize-one";
import { kvCache } from "../../util/stateManager";
import * as globals from "../../globals";
import actions from "../../actions";
import finiteExtent from "../../util/finiteExtent";
@@ -21,7 +20,6 @@ import finiteExtent from "../../util/finiteExtent";
scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
crossfilter: state.controls.crossfilter,
differential: state.differential,
initializeRanges: _.get(state.controls.world, "summary.obs"),
colorAccessor: state.controls.colorAccessor,
colorScale: state.controls.colorScale,
obsAnnotations: _.get(state.controls.world, "obsAnnotations", null)
@@ -35,7 +33,7 @@ class HistogramBrush extends React.Component {
.scaleLinear()
.range([this.height - this.marginBottom, 0]);
if (obsAnnotations.col(field)) {
if (obsAnnotations.hasCol(field)) {
// recalculate expensive stuff
const allValuesForContinuousFieldAsArray = obsAnnotations
.col(field)
@@ -52,9 +50,8 @@ class HistogramBrush extends React.Component {
.thresholds(40)(allValuesForContinuousFieldAsArray);
histogramCache.numValues = allValuesForContinuousFieldAsArray.length;
} else if (kvCache.get(world.varDataCache, field)) {
/* it's not in observations, so it's a gene, but let's check to make sure */
const varValues = kvCache.get(world.varDataCache, field);
} else if (world.varData.hasCol(field)) {
const varValues = world.varData.col(field).asArray();
histogramCache.x = d3
.scaleLinear()
@@ -143,21 +140,15 @@ class HistogramBrush extends React.Component {
}
handleColorAction() {
const {
obsAnnotations,
dispatch,
field,
world,
initializeRanges
} = this.props;
const { obsAnnotations, dispatch, field, world, ranges } = this.props;
if (obsAnnotations.col(field)) {
if (obsAnnotations.hasCol(field)) {
dispatch({
type: "color by continuous metadata",
colorAccessor: field,
rangeMaxForColorAccessor: initializeRanges[field].range.max
rangeMaxForColorAccessor: ranges.max
});
} else if (kvCache.get(world.varDataCache, field)) {
} else if (world.varData.hasCol(field)) {
dispatch(actions.requestSingleGeneExpressionCountsForColoringPOST(field));
}
}
client/src/components/continuous/continuous.js
+26 -21
View File
@@ -9,7 +9,7 @@ import * as globals from "../../globals";
import HistogramBrush from "../brushableHistogram";
@connect(state => ({
ranges: _.get(state.controls.world, "summary.obs", null),
obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
colorAccessor: state.controls.colorAccessor,
colorScale: state.controls.colorScale,
selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
@@ -28,17 +28,18 @@ class Continuous extends React.Component {
handleColorAction(key) {
return () => {
const { dispatch, ranges } = this.props;
const { dispatch, obsAnnotations } = this.props;
const summary = obsAnnotations.col(key).summarize();
dispatch({
type: "color by continuous metadata",
colorAccessor: key,
rangeMaxForColorAccessor: ranges[key].range.max
rangeMaxForColorAccessor: summary.max
});
};
}
render() {
const { ranges, schema } = this.props;
const { obsAnnotations, schema } = this.props;
if (schema && !this.continuousChecked) {
this.hasContinuous = _.some(
schema.annotations.obs,
@@ -62,23 +63,27 @@ class Continuous extends React.Component {
Continuous metadata
</p>
) : null}
{_.map(ranges, (value, key) => {
const isColorField = key.includes("color") || key.includes("Color");
zebra += 1;
if (value.range && key !== "name" && !isColorField) {
return (
<HistogramBrush
key={key}
field={key}
isObs
zebra={zebra % 2 === 0}
ranges={value.range}
handleColorAction={this.handleColorAction(key).bind(this)}
/>
);
}
return null;
})}
{obsAnnotations
? _.map(obsAnnotations.colIndex.keys(), key => {
const summary = obsAnnotations.col(key).summarize();
const isColorField =
key.includes("color") || key.includes("Color");
zebra += 1;
if (!summary.categorical && key !== "name" && !isColorField) {
return (
<HistogramBrush
key={key}
field={key}
isObs
zebra={zebra % 2 === 0}
ranges={summary}
handleColorAction={this.handleColorAction(key).bind(this)}
/>
);
}
return null;
})
: null}
</div>
);
}
client/src/components/geneExpression/cellSetButtons.js
+5 -1
View File
@@ -14,7 +14,11 @@ class CellSetButton extends React.Component {
eitherCellSetOneOrTwo
} = this.props;
const set = World.getSelectedByIndex(crossfilter);
// Reducer and components assume that value will be null if
// no selection made. World..getSelectedByIndex() returns a
// zero length TypedArray when nothing is selected.
let set = World.getSelectedByIndex(crossfilter);
if (set.length === 0) set = null;
if (!differential.diffExp) {
/* diffexp needs to be cleared before we store a new set */
client/src/components/geneExpression/index.js
+7 -9
View File
@@ -56,12 +56,8 @@ const filterGenes = (query, genes) =>
});
@connect(state => {
const ranges = _.get(state.controls.world, "summary.obs", null);
const initializeRanges = _.get(state.controls.world, "summary.obs");
return {
ranges,
initializeRanges,
obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
userDefinedGenes: state.controls.userDefinedGenes,
userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
world: state.controls.world,
@@ -293,16 +289,17 @@ class GeneExpression extends React.Component {
) : null}
{world && userDefinedGenes.length > 0
? _.map(userDefinedGenes, (geneName, index) => {
const values = world.varDataCache[geneName];
const values = world.varData.col(geneName);
if (!values) {
return null;
}
const summary = values.summarize();
return (
<HistogramBrush
key={geneName}
field={geneName}
zebra={index % 2 === 0}
ranges={finiteExtent(values)}
ranges={summary}
isUserDefined
/>
);
@@ -322,16 +319,17 @@ class GeneExpression extends React.Component {
{differential.diffExp
? _.map(differential.diffExp, (value, index) => {
const name = world.varAnnotations.at(value[0], "name");
const values = world.varDataCache[name];
const values = world.varData.col(name);
if (!values) {
return null;
}
const summary = values.summarize();
return (
<HistogramBrush
key={name}
field={name}
zebra={index % 2 === 0}
ranges={finiteExtent(values)}
ranges={summary}
isDiffExp
logFoldChange={value[1]}
pval={value[2]}
client/src/components/scatterplot/scatterplot.js
+8 -5
View File
@@ -19,7 +19,6 @@ import _drawPoints from "./drawPointsRegl";
import scaleLinear from "../../util/scaleLinear";
import { margin, width, height } from "./util";
import { kvCache } from "../../util/stateManager";
import finiteExtent from "../../util/finiteExtent";
@connect(state => {
@@ -30,12 +29,16 @@ import finiteExtent from "../../util/finiteExtent";
scatterplotYYaccessor
} = state.controls;
const expressionX =
world && scatterplotXXaccessor
? kvCache.get(world.varDataCache, scatterplotXXaccessor)
world &&
scatterplotXXaccessor &&
world.varData.hasCol(scatterplotXXaccessor)
? world.varData.col(scatterplotXXaccessor).asArray()
: null;
const expressionY =
world && scatterplotYYaccessor
? kvCache.get(world.varDataCache, scatterplotYYaccessor)
world &&
scatterplotYYaccessor &&
world.varData.hasCol(scatterplotYYaccessor)
? world.varData.col(scatterplotYYaccessor).asArray()
: null;
return {
client/src/reducers/controls.js
Vendored
+95 -43
View File
@@ -1,9 +1,8 @@
// jshint esversion: 6
import _ from "lodash";
import { polygonContains } from "d3";
import { World, kvCache, WorldUtil } from "../util/stateManager";
import { World, WorldUtil } from "../util/stateManager";
import parseRGB from "../util/parseRGB";
import Crossfilter from "../util/typedCrossfilter";
import * as globals from "../globals";
@@ -61,19 +60,20 @@ function topNCategories(summary) {
function createCategoricalSelectionState(state, world) {
const res = {};
_.forEach(world.summary.obs, (value, key) => {
if (value.categories) {
_.forEach(world.obsAnnotations.colIndex.keys(), key => {
const summary = world.obsAnnotations.col(key).summarize();
if (summary.categories) {
const isColorField = key.includes("color") || key.includes("Color");
const isSelectableCategory =
!isColorField &&
key !== "name" &&
value.categories.length < state.maxCategoryItems;
summary.categories.length < state.maxCategoryItems;
if (isSelectableCategory) {
const [categoryValues, categoryCounts] = topNCategories(value);
const [categoryValues, categoryCounts] = topNCategories(summary);
const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
const numCategories = categoryIndices.size;
const categorySelected = new Array(numCategories).fill(true);
const isTruncated = categoryValues.length < value.numCategories;
const isTruncated = categoryValues.length < summary.numCategories;
res[key] = {
categoryValues, // array: of natively typed category values
categoryIndices, // map: category value (native type) -> category index
@@ -104,11 +104,10 @@ function selectedValuesForCategory(categorySelectionState) {
build a crossfilter dimension map for all gene expression related dimensions.
*/
function createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter) {
function _createGenesDimMap(genes, nameF) {
function _createGenesDimMap(genes, nameCreator) {
return genes.reduce((acc, gene) => {
acc[nameF(gene)] = World.createVarDimension(
acc[nameCreator(gene)] = World.createVarDataDimension(
world,
world.varDataCache,
crossfilter,
gene
);
@@ -122,6 +121,46 @@ function createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter) {
};
}
function pruneVarDataCache(varData, needed) {
/*
Remove any unneeded columns from the varData dataframe. Will only
prune / remove if the total column count exceeds VarDataCacheLowWatermark
Note: this code leverages the fact that dataframe offsets indicate
the order in which the columns were added. This crudely provides
LRU semantics, so we can delete "older" columns first.
*/
/*
VarDataCacheLowWatermark - this cofig value sets the minimum cache size,
in columns, below which we don't throw away data.
The value should be high enough so we are caching the maximum which will
"typically" be used in the UI (currently: 10 for diffexp, and N for user-
specified genes), and low enough to account for memory use (any single
column size is 4 bytes * numObs, so a column can be multi-megabyte in common
use cases).
*/
const VarDataCacheLowWatermark = 32;
const numOverWatermark = varData.dims[1] - VarDataCacheLowWatermark;
if (numOverWatermark <= 0) return varData;
const { colIndex } = varData;
const all = colIndex.keys();
const unused = _.difference(all, needed);
if (unused.length > 0) {
// sort by offset in the dataframe - ie, psuedo-LRU
unused.sort((a, b) => colIndex.getOffset(a) - colIndex.getOffset(b));
const numToDrop =
unused.length < numOverWatermark ? unused.length : numOverWatermark;
for (let i = 0; i < numToDrop; i += 1) {
varData = varData.dropCol(unused[i]);
}
}
return varData;
}
const Controls = (
state = {
// data loading flag
@@ -295,28 +334,60 @@ const Controls = (
}
case "expression load success": {
const { world, universe } = state;
let universeVarDataCache = universe.varDataCache;
let worldVarDataCache = world.varDataCache;
let universeVarData = universe.varData;
let worldVarData = world.varData;
// Load new expression data into the varData dataframes, if
// not already present.
_.forEach(action.expressionData, (val, key) => {
universeVarDataCache = kvCache.set(universeVarDataCache, key, val);
if (kvCache.get(worldVarDataCache, key) === undefined) {
worldVarDataCache = kvCache.set(
worldVarDataCache,
// If not already in universe.varData, save entire expression column
if (!universeVarData.hasCol(key)) {
universeVarData = universeVarData.withCol(key, val);
}
// If not already in world.varData, save sliced expression column
if (!worldVarData.hasCol(key)) {
// Slice if world !== universe, else just use whole column.
// Use the obsAnnotation index as the cut key, as we keep
// all world dataframes in sync.
let worldValSlice = val;
if (!World.worldEqUniverse(world, universe)) {
worldValSlice = universeVarData
.subset(world.obsAnnotations.rowIndex.keys(), [key], null)
.icol(0)
.asArray();
}
// Now build world's varData dataframe
worldVarData = worldVarData.withCol(
key,
World.subsetVarData(world, universe, val)
worldValSlice,
world.obsAnnotations.rowIndex
);
}
});
// Prune size of varData "cache" if getting out of hand....
const { userDefinedGenes, diffexpGenes } = state;
const allTheGenesWeNeed = _.uniq(
[].concat(
userDefinedGenes,
diffexpGenes,
Object.keys(action.expressionData)
)
);
universeVarData = pruneVarDataCache(universeVarData, allTheGenesWeNeed);
worldVarData = pruneVarDataCache(worldVarData, allTheGenesWeNeed);
return {
...state,
universe: {
...universe,
varDataCache: universeVarDataCache
varData: universeVarData
},
world: {
...world,
varDataCache: worldVarDataCache
varData: worldVarData
}
};
}
@@ -334,17 +405,12 @@ const Controls = (
}
case "request user defined gene success": {
const { world, crossfilter, dimensionMap, userDefinedGenes } = state;
const worldVarDataCache = world.varDataCache;
const _userDefinedGenes = userDefinedGenes.slice();
const gene = action.data.genes[0];
dimensionMap[userDefinedDimensionName(gene)] = World.createVarDimension(
/* "__var__" + */
world,
worldVarDataCache,
crossfilter,
gene
);
dimensionMap[
userDefinedDimensionName(gene)
] = World.createVarDataDimension(world, crossfilter, gene);
return {
...state,
@@ -355,7 +421,6 @@ const Controls = (
}
case "request differential expression success": {
const { world, crossfilter, dimensionMap } = state;
const worldVarDataCache = world.varDataCache;
const _diffexpGenes = [];
action.data.forEach(d => {
@@ -363,10 +428,8 @@ const Controls = (
});
_.forEach(_diffexpGenes, gene => {
dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
/* "__var__" + */
dimensionMap[diffexpDimensionName(gene)] = World.createVarDataDimension(
world,
worldVarDataCache,
crossfilter,
gene
);
@@ -381,9 +444,6 @@ const Controls = (
case "clear differential expression": {
const { world, universe, dimensionMap } = state;
const _dimensionMap = dimensionMap;
const universeVarDataCache = universe.varDataCache;
const worldVarDataCache = world.varDataCache;
_.forEach(action.diffExp, values => {
const name = world.varAnnotations.at(values[0], "name");
// clean up crossfilter dimensions
@@ -394,15 +454,7 @@ const Controls = (
return {
...state,
dimensionMap: _dimensionMap,
diffexpGenes: [],
universe: {
...universe,
varDataCache: universeVarDataCache
},
world: {
...world,
varDataCache: worldVarDataCache
}
diffexpGenes: []
};
}
case "user defined gene": {
client/src/util/dataframe/dataframe.js
+133 -24
View File
@@ -1,4 +1,4 @@
import { IdentityInt32Index } from "./labelIndex";
import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
// weird cross-dependency that we should clean up someday...
import { sort } from "../typedCrossfilter/sort";
import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
@@ -10,7 +10,7 @@ but (currently) without all of the surrounding support functions.
Data is stored in column-major layout, and each column is monomorphic.
It supports:
* Relatively efficient creation, cloning and subsetting ("cut")
* Relatively efficient creation, cloning and subsetting
* Very efficient columnar access (eg, sum down a column), and access
to the underlying column arrays.
* Data access by row/col offset or label. Labels are reasonably well
@@ -76,14 +76,17 @@ class Dataframe {
or a caller-provided index.
All columns and indices must have appropriate dimensionality.
*/
Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex);
const [nRows, nCols] = dims;
if (nRows < 0 || nCols < 0) {
throw new RangeError("Dataframe dimensions must be positive");
}
if (!rowIndex) {
rowIndex = new IdentityInt32Index(nRows);
}
if (!colIndex) {
colIndex = new IdentityInt32Index(nCols);
}
Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex);
this.__columns = Array.from(columnarData);
this.dims = dims;
@@ -94,23 +97,40 @@ class Dataframe {
this.__compile();
}
static __errorChecks(dims, columnarData) {
static __errorChecks(dims, columnarData, rowIndex, colIndex) {
const [nRows, nCols] = dims;
if (nRows < 0 || nCols < 0) {
throw new RangeError("Dataframe dimensions must be positive");
}
/* check for expected types */
if (!Array.isArray(columnarData)) {
throw new TypeError("Dataframe constructor requires array of columns");
}
if (!columnarData.every(c => isArrayOrTypedArray(c))) {
throw new TypeError("Dataframe columns must all be Array or TypedArray");
}
if (!isLabelIndex(rowIndex)) {
throw new TypeError("Dataframe rowIndex is an unsupported type.");
}
if (!isLabelIndex(colIndex)) {
throw new TypeError("Dataframe colIndex is an unsupported type.");
}
/* check for expected dimensionality / size */
if (
nCols !== columnarData.length ||
!columnarData.every(c => c.length === nRows)
) {
throw new RangeError(
"Dataframe dimension does not match column data shape"
"Dataframe dimension does not match provided data shape"
);
}
if (nRows !== rowIndex.size()) {
throw new RangeError(
"Dataframe rowIndex must have same size as underlying data"
);
}
if (nCols !== colIndex.size()) {
throw new RangeError(
"Dataframe colIndex must have same size as underlying data"
);
}
}
@@ -210,6 +230,9 @@ class Dataframe {
}
clone() {
/*
Clone this dataframe
*/
return new this.constructor(
this.dims,
[...this.__columns],
@@ -218,8 +241,57 @@ class Dataframe {
);
}
static empty() {
return new Dataframe([0, 0], []);
withCol(label, colData, withRowIndex = null) {
/*
Create a new DF, which is `this` plus the new column. Example:
const newDf = df.withCol("foo", [1,2,3]);
Dimensionality of new column must match existing dataframe.
Special case: empty dataframe will accept any size column. Example:
const newDf = Dataframe.empty().withCol("foo", [1,2,3]);
If `withRowIndex` specified, the provided index will become the
rowIndex for the newly created dataframe. If not specified,
the rowIndex from `this` will be used (ie, the rowIndex is
unchanged).
*/
let dims;
let rowIndex;
if (this.isEmpty()) {
dims = [colData.length, 1];
rowIndex = null;
} else {
dims = [this.dims[0], this.dims[1] + 1];
({ rowIndex } = this);
}
if (withRowIndex) {
rowIndex = withRowIndex;
}
const columns = [...this.__columns];
columns.push(colData);
const colIndex = this.colIndex.withLabel(label);
return new this.constructor(dims, columns, rowIndex, colIndex);
}
dropCol(label) {
/*
Create a new dataframe, omitting one columns.
const newDf = df.dropCol("colors");
*/
const dims = [this.dims[0], this.dims[1] - 1];
const coffset = this.colIndex.getOffset(label);
const columns = [...this.__columns];
columns.splice(coffset, 1);
const colIndex = this.colIndex.dropLabel(label);
return new this.constructor(dims, columns, this.rowIndex, colIndex);
}
static empty(rowIndex = null, colIndex = null) {
return new Dataframe([0, 0], [], rowIndex, colIndex);
}
static create(dims, columnarData) {
@@ -233,7 +305,7 @@ class Dataframe {
return new Dataframe(dims, columnarData, null, null);
}
__cut(rowOffsets, colOffsets) {
__subset(rowOffsets, colOffsets, withRowIndex) {
const dims = [...this.dims];
const getSortedLabelAndOffsets = (offsets, index) => {
@@ -260,10 +332,11 @@ class Dataframe {
this.colIndex
);
dims[1] = colOffsets.length;
colIndex = this.colIndex.cut(colLabels);
colIndex = this.colIndex.subsetLabels(colLabels);
}
let { rowIndex } = this;
if (withRowIndex) rowIndex = withRowIndex;
if (rowOffsets) {
let rowLabels;
[rowLabels, rowOffsets] = getSortedLabelAndOffsets(
@@ -271,10 +344,10 @@ class Dataframe {
this.rowIndex
);
dims[0] = rowLabels.length;
rowIndex = this.rowIndex.cut(rowLabels);
if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
}
/* cut columns */
/* subset columns */
let columns = this.__columns;
if (colOffsets) {
columns = new Array(colOffsets.length);
@@ -283,7 +356,7 @@ class Dataframe {
}
}
/* cut rows */
/* subset rows */
if (rowOffsets) {
columns = columns.map(col => {
const newCol = new col.constructor(rowOffsets.length);
@@ -296,7 +369,15 @@ class Dataframe {
return new Dataframe(dims, columns, rowIndex, colIndex);
}
cutByList(rowLabels, colLabels = null) {
subset(rowLabels, colLabels = null, withRowIndex = null) {
/*
Subset by row/col labels.
withRowIndex allows assignment of new row index during subset operation.
If withRowIndex === null, it will reset the index to identity (offset)
indexing. if withRowIndex is a label index object, it will be used
for the new dataframe.
*/
const toOffsets = (labels, index) => {
if (!labels) {
return null;
@@ -312,16 +393,29 @@ class Dataframe {
const rowOffsets = toOffsets(rowLabels, this.rowIndex);
const colOffsets = toOffsets(colLabels, this.colIndex);
return this.__cut(rowOffsets, colOffsets);
return this.__subset(rowOffsets, colOffsets, withRowIndex);
}
icutByList(rowOffsets, colOffsets = null) {
return this.__cut(rowOffsets, colOffsets);
}
icutByMask(rowMask, colMask = null) {
isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
/*
Cut on row/column based upon a truthy/falsey array.
Subset by row/col offset.
withRowIndex allows assignment of new row index during subset operation.
If withRowIndex === null, it will reset the index to identity (offset)
indexing. if withRowIndex is a label index object, it will be used
for the new dataframe.
*/
return this.__subset(rowOffsets, colOffsets, withRowIndex);
}
isubsetMask(rowMask, colMask = null, withRowIndex = null) {
/*
Subset on row/column based upon a truthy/falsey array (a mask).
withRowIndex allows assignment of new row index during subset operation.
If withRowIndex === null, it will reset the index to identity (offset)
indexing. if withRowIndex is a label index object, it will be used
for the new dataframe.
*/
const [nRows, nCols] = this.dims;
if (
@@ -348,7 +442,7 @@ class Dataframe {
};
const rowOffsets = toList(rowMask, nRows);
const colOffsets = toList(colMask, nCols);
return this.__cut(rowOffsets, colOffsets);
return this.__subset(rowOffsets, colOffsets, withRowIndex);
}
/**
@@ -431,6 +525,21 @@ class Dataframe {
return c >= 0 && c < nCols && r >= 0 && r < nRows;
}
hasCol(c) {
/*
Test if col label exists - return true/false
*/
return !!this.col(c);
}
isEmpty() {
/*
Return true if this is an empty dataframe, ie, has dimensions [0,0]
*/
const [rows, cols] = this.dims;
return rows === 0 && cols === 0;
}
/****
Functional (map/reduce/etc) data access
client/src/util/dataframe/labelIndex.js
+64 -8
View File
@@ -57,13 +57,13 @@ class IdentityInt32Index {
return i;
}
getMaxOffset() {
size() {
return this.maxOffset;
}
cut(labelArray) {
__promote(labelArray) {
/*
if density of resulting integer
time/space decision - based on the resulting density
*/
const [minLabel, maxLabel] = extent(labelArray);
const labelSpaceSize = maxLabel - minLabel + 1;
@@ -74,6 +74,26 @@ class IdentityInt32Index {
}
return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
}
subsetLabels(labelArray) {
return this.__promote(labelArray);
}
withLabel(label) {
if (label === this.maxOffset) {
return new IdentityInt32Index(label + 1);
}
return this.__promote([...this.keys(), label]);
}
dropLabel(label) {
if (label === this.maxOffset - 1) {
return new IdentityInt32Index(label);
}
const labelArray = [...this.keys()];
labelArray.splice(labelArray.indexOf(label), 1);
return this.__promote(labelArray);
}
}
/* eslint-enable class-methods-use-this */
@@ -121,11 +141,11 @@ class DenseInt32Index {
return this.rindex;
}
getMaxOffset() {
size() {
return this.rindex.length;
}
cut(labelArray) {
__promote(labelArray) {
/*
time/space decision - if we are going to use less than 10% of the
dense index space, switch to a KeyIndex (which is slower, but uses
@@ -140,6 +160,20 @@ class DenseInt32Index {
}
return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
}
subsetLabels(labelArray) {
return this.__promote(labelArray);
}
withLabel(label) {
return this.__promote([...this.keys(), label]);
}
dropLabel(label) {
const labelArray = [...this.keys()];
labelArray.splice(labelArray.indexOf(label), 1);
return this.__promote(labelArray);
}
}
/* eslint-enable class-methods-use-this */
@@ -151,6 +185,9 @@ class KeyIndex {
*/
constructor(labels) {
const index = new Map();
if (labels === undefined) {
labels = [];
}
const rindex = labels;
labels.forEach((v, i) => {
index.set(v, i);
@@ -175,14 +212,33 @@ class KeyIndex {
return this.rindex;
}
getMaxOffset() {
size() {
return this.rindex.length;
}
cut(labelArray) {
subsetLabels(labelArray) {
return new KeyIndex(labelArray);
}
withLabel(label) {
return new KeyIndex([...this.rindex, label]);
}
dropLabel(label) {
const idx = this.rindex.indexOf(label);
const labelArray = [...this.rindex];
labelArray.splice(idx, 1);
return new KeyIndex(labelArray);
}
}
/* eslint-enable class-methods-use-this */
export { DenseInt32Index, IdentityInt32Index, KeyIndex };
function isLabelIndex(i) {
return (
i instanceof IdentityInt32Index ||
i instanceof DenseInt32Index ||
i instanceof KeyIndex
);
}
export { DenseInt32Index, IdentityInt32Index, KeyIndex, isLabelIndex };
client/src/util/stateManager/index.js
-1
View File
@@ -16,5 +16,4 @@ exists to support those concepts.
export * as Universe from "./universe";
export * as World from "./world";
export * as kvCache from "./keyvalcache";
export * as WorldUtil from "./worldUtil";
client/src/util/stateManager/keyvalcache.js
-122
View File
@@ -1,122 +0,0 @@
// jshint esversion: 6
import _ from "lodash";
/*
Very simple key/value cache for use by World & Universe. Cache keys must
be a string, and values are any JS non-primitive value.
* constructor(lowWatermark, minTTL):
- lowWatermark defines the number of cache elements below which
flushing will not occur.
- minTTL defines minimum time in milliseconds that cache entries will live.
A value of -1 disables automatic flushing (flush() can still
be called by external user).
* set() - add a key/val pair.
* get() - get a value or undefined if not present.
* flush(minAgeMs) - flush cache entries in excess of lowWatermark if those
entries are older than minAgeMs.
*/
const cachePrivateKey = "__kvcachekey__";
const defaultLowWatermark = 32;
const defaultMinTTL = 1000;
function create(lowWatermark = defaultLowWatermark, minTTL = defaultMinTTL) {
if (typeof minTTL !== "number" || typeof lowWatermark !== "number") {
throw new TypeError(
"minTTL and lowWatermark parameters must be a primitive number"
);
}
if (lowWatermark < 0 || minTTL < 0) {
throw new RangeError(
"minTTL and lowWatermark parameters must be number greater than zero"
);
}
return {
[cachePrivateKey]: {
lowWatermark,
minTTL
}
};
}
function get(kvcache, key) {
if (key === cachePrivateKey) {
throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
}
const val = kvcache[key];
if (val) {
val[cachePrivateKey] = Date.now();
}
return val;
}
function set(kvcache, key, val) {
if (key === cachePrivateKey) {
throw new RangeError(`key parameter may not have value ${cachePrivateKey}`);
}
const newKvCache = { ...kvcache };
newKvCache[key] = val;
val[cachePrivateKey] = Date.now();
flushInPlace(newKvCache);
return newKvCache;
}
function flush(kvcache) {
const newKvCache = { ...kvcache };
flushInPlace(newKvCache);
return newKvCache;
}
/*
Flush elements from cache IF cache size is greater than lowWatermark, and
those elements are older than minAgeMS
*/
function flushInPlace(kvCache) {
const { lowWatermark, minTTL } = kvCache[cachePrivateKey];
const eol = Date.now() - minTTL;
const allKeys = _(kvCache)
.keys()
.filter(k => k !== cachePrivateKey)
.sortBy([k => kvCache[k][cachePrivateKey]])
.value();
if (allKeys.length > lowWatermark) {
const keysToDelete = _(allKeys)
.slice(0, allKeys.length - lowWatermark)
.filter(k => kvCache[k][cachePrivateKey] <= eol)
.value();
_.forEach(keysToDelete, k => delete kvCache[k]);
}
return kvCache;
}
/*
use to create a cache that is a transformation of another cache.
*/
function map(srcKvCache, cb, createOptions) {
const keysInSrcKvCache = _(srcKvCache)
.keys()
.filter(k => k !== cachePrivateKey)
.value();
const lowWatermark = _.get(
createOptions,
"lowWatermark",
defaultLowWatermark
);
const minTTL = _.get(createOptions, "minTTL", defaultMinTTL);
const newKvCache = create(lowWatermark, minTTL);
_.forEach(keysInSrcKvCache, key => {
const val = cb(get(srcKvCache, key), key);
newKvCache[key] = val;
val[cachePrivateKey] = Date.now();
});
return newKvCache;
}
export { create, get, set, flush, map };
client/src/util/stateManager/summarizeAnnotations.js
-128
View File
@@ -1,128 +0,0 @@
import _ from "lodash";
import finiteExtent from "../finiteExtent";
/*
Build and return obs/var summary using any annotation in the schema
Summary information for each annotation, keyed by annotation name.
Value will be an object, containing summary information.
For continuous annotations (int, float, etc):
<annotation_name>: {
categorical: false,
range {
min: <number>,
max: <number>
}
}
For categorical annotations (boolean, string, category):
<annotation_name>: {
categorical: true,
categories: [ <category1>, <category2>, ... ]
categoryCounts: Map {
<category1>: <number>,
...
},
numCategories: <number>
}
Summarize will be returned for BOTH obs and var annotations.
Example:
{
"Splice_sites_Annotated": {
categorical: false,
range: {
"min": 26,
"max": 1075869
}
},
"Selection": {
categorical: true,
numCategories, 3,
categories: [ "Astrocytes(HEPACAM)", "Endothelial(BSC)", "Unpanned" ],
categoryCounts: Map {
"Astrocytes(HEPACAM)": 714,
"Endothelial(BSC)": 123,
"Unpanned": 665
}
}
}
NOTE: will not summarize the required 'name' annotation, as that is
specified as unique per element.
*/
function _summarizeAnnotations(_schema, df) {
const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
.filter(v => v.name !== "name") // don't summarize name
.keyBy("name")
.mapValues(anno => {
const { name, type } = anno;
const continuous = type === "int32" || type === "float32";
const numRows = df.length;
const col = df.col(name) ? df.col(name).asArray() : null;
if (continuous) {
let min;
let max;
let nan = 0;
let pinf = 0;
let ninf = 0;
if (col) {
for (let r = 0; r < numRows; r += 1) {
const val = Number(col[r]);
if (Number.isFinite(val)) {
if (min === undefined) {
min = val;
max = val;
} else {
min = val < min ? val : min;
max = val > max ? val : max;
}
} else if (Number.isNaN(val)) {
nan += 1;
} else if (val > 0) {
pinf += 1;
} else {
ninf += 1;
}
}
}
return {
categorical: false,
range: { min, max, nan, pinf, ninf }
};
}
/* else categorical */
const categoryCounts = new Map();
if (col) {
for (let r = 0; r < numRows; r += 1) {
const val = col[r];
let curCount = categoryCounts.get(val);
if (curCount === undefined) curCount = 0;
categoryCounts.set(val, curCount + 1);
}
}
return {
categorical: true,
categories: [...categoryCounts.keys()],
categoryCounts,
numCategories: categoryCounts.size
};
})
.value();
return summary;
}
export default function summarizeAnnotations(
schema,
obsAnnotations,
varAnnotations
) {
return {
obs: _summarizeAnnotations(schema.annotations.obs, obsAnnotations),
var: _summarizeAnnotations(schema.annotations.var, varAnnotations)
};
}
client/src/util/stateManager/universe.js
+15 -52
View File
@@ -2,8 +2,6 @@
import _ from "lodash";
import * as kvCache from "./keyvalcache";
import summarizeAnnotations from "./summarizeAnnotations";
import decodeMatrixFBS from "./matrix";
import * as Dataframe from "../dataframe";
@@ -12,15 +10,7 @@ Private helper function - create and return a template Universe
*/
function templateUniverse() {
/* default universe template */
/* varDataCache config - see kvCache for semantics */
const VarDataCacheLowWatermark = 32; // cache element count
const VarDataCacheTTLMs = 1000; // min cache time in MS
return {
api: null,
finalized: false, // XXX: may not be needed
nObs: 0,
nVar: 0,
schema: {},
@@ -28,18 +18,14 @@ function templateUniverse() {
/*
Annotations
*/
obsAnnotations: null,
varAnnotations: null,
obsLayout: null,
summary: null /* derived data summaries. XXX: consider exploding in place */,
obsAnnotations: Dataframe.Dataframe.empty(),
varAnnotations: Dataframe.Dataframe.empty(),
obsLayout: Dataframe.Dataframe.empty(),
/*
Cache of var data (expression), by var annotation name. Data can be
accesses as a POJO, but if you want caching semantics, use the kvCache
API (eg., kvCache.get(), kvCache.set(), ...), which will maintain the
LRU semantics.
Var data columns - subset of all
*/
varDataCache: kvCache.create(VarDataCacheLowWatermark, VarDataCacheTTLMs)
varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
};
}
@@ -51,29 +37,6 @@ These functions are used exclusively by the actions and reducers to
build an internal POJO for use by the rendering components.
*/
/*
generate any client-side transformations or summarization that
is independent of REST API response formats.
*/
function finalize(universe) {
/* A bit of sanity checking! */
const { nObs, nVar } = universe;
if (
nObs !== universe.obsLayout.length ||
nObs !== universe.obsAnnotations.length ||
nVar !== universe.varAnnotations.length
) {
throw new Error("Universe dimensionality mismatch - failed to load");
}
// TODO: add more sanity checks, such as:
// - all annotations in the schema
// - layout has supported number of dimensions
// - ...
universe.finalized = true;
return universe;
}
function AnnotationsFBSToDataframe(arrayBuffer) {
/*
Convert a Matrix FBS to a Dataframe.
@@ -118,7 +81,7 @@ function reconcileSchemaCategoriesWithSummary(universe) {
) {
const categories = _.union(
_.get(s, "categories", []),
_.get(universe.summary.obs[s.name], "categories", [])
_.get(universe.obsAnnotations.col(s.name).summarize(), "categories", [])
);
s.categories = categories;
}
@@ -138,9 +101,6 @@ export function createUniverseFromResponse(
const { schema } = schemaResponse;
const universe = templateUniverse();
/* constants */
universe.api = "0.2";
/* schema related */
universe.schema = schema;
universe.nObs = schema.dataframe.nObs;
@@ -152,14 +112,17 @@ export function createUniverseFromResponse(
/* layout */
universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
universe.summary = summarizeAnnotations(
universe.schema,
universe.obsAnnotations,
universe.varAnnotations
);
/* sanity check */
if (
universe.nObs !== universe.obsLayout.length ||
universe.nObs !== universe.obsAnnotations.length ||
universe.nVar !== universe.varAnnotations.length
) {
throw new Error("Universe dimensionality mismatch - failed to load");
}
reconcileSchemaCategoriesWithSummary(universe);
return finalize(universe);
return universe;
}
export function convertDataFBStoObject(universe, arrayBuffer) {
client/src/util/stateManager/world.js
+28 -75
View File
@@ -1,11 +1,10 @@
// jshint esversion: 6
import _ from "lodash";
import * as kvCache from "./keyvalcache";
import summarizeAnnotations from "./summarizeAnnotations";
import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
import Crossfilter from "../typedCrossfilter";
import { sliceByIndex } from "../typedCrossfilter/util";
import * as Dataframe from "../dataframe";
/*
@@ -38,48 +37,33 @@ Notable keys in the world object:
A dataframe containing the X/Y layout for all obs. Columns are named
'X' and 'Y', and rows are indexed in the same way as obsAnnotation.
* summary: summary of each obsAnnotation column (eg, numeric extent for
continuous data, category counts for categorical metadata)
* varDataCache: expression columns, in a kvCache. TODO: maybe move to a
Dataframe in the future.
* varData: a cache of expression columns, stored in a Dataframe. Cache
managed by controls reducer.
*/
/* varDataCache config - see kvCache for semantics */
const VarDataCacheLowWatermark = 32; // cache element count
const VarDataCacheTTLMs = 1000; // min cache time in MS
function templateWorld() {
return {
/* schema/version related */
api: null,
schema: null,
nObs: 0,
nVar: 0,
/* annotations */
obsAnnotations: null,
varAnnotations: null,
obsAnnotations: Dataframe.Dataframe.empty(),
varAnnotations: Dataframe.Dataframe.empty(),
/* layout of graph. Dataframe. */
obsLayout: null,
obsLayout: Dataframe.Dataframe.empty(),
/* derived data summaries XXX: consider exploding in place */
summary: null,
varDataCache: kvCache.create(
VarDataCacheLowWatermark,
VarDataCacheTTLMs
) /* cache of var data (expression) */
/*
Var data columns - subset of all data (may be empty)
*/
varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
};
}
export function createWorldFromEntireUniverse(universe) {
if (!universe.finalized) {
throw new Error("World can't be created from an partial Universe");
}
const world = templateWorld();
/*
@@ -87,31 +71,21 @@ export function createWorldFromEntireUniverse(universe) {
*/
/* Schema related */
world.api = universe.api;
world.schema = universe.schema;
world.nObs = universe.nObs;
world.nVar = universe.nVar;
/* annotations */
/* annotation dataframes */
world.obsAnnotations = universe.obsAnnotations;
world.varAnnotations = universe.varAnnotations;
/* layout and display characteristics */
/* layout and display characteristics dataframe */
world.obsLayout = universe.obsLayout;
/* derived data & summaries */
world.summary = summarizeAnnotations(
world.schema,
world.obsAnnotations,
world.varAnnotations
);
/* build the varDataCache */
world.varDataCache = kvCache.map(
universe.varDataCache,
val => subsetVarData(world, universe, val),
{ lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs }
);
/*
Var data columns - subset of all
*/
world.varData = universe.varData.clone();
return world;
}
@@ -120,30 +94,24 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
const newWorld = templateWorld();
/* these don't change as only OBS are selected in our current implementation */
newWorld.api = universe.api;
newWorld.nVar = universe.nVar;
newWorld.schema = universe.schema;
newWorld.varAnnotations = universe.varAnnotations;
/* now subset/cut obs */
const mask = crossfilter.allFilteredMask();
newWorld.obsAnnotations = world.obsAnnotations.icutByMask(mask);
newWorld.obsLayout = world.obsLayout.icutByMask(mask);
newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask);
newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
newWorld.nObs = newWorld.obsAnnotations.dims[0];
/* derived data & summaries */
newWorld.summary = summarizeAnnotations(
newWorld.schema,
newWorld.obsAnnotations,
newWorld.varAnnotations
);
/* build the varDataCache */
newWorld.varDataCache = kvCache.map(
universe.varDataCache,
val => subsetVarData(newWorld, universe, val),
{ lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs }
);
/*
Var data columns - subset of all
*/
if (world.varData.isEmpty()) {
newWorld.varData = world.varData.clone();
} else {
newWorld.varData = world.varData.isubsetMask(mask);
}
return newWorld;
}
@@ -183,17 +151,10 @@ function deduceDimensionType(attributes, fieldName) {
when it is no longer needed
(it will not be garbage collected without this call)
*/
export function createVarDimension(
world,
_worldVarDataCache,
crossfilter,
geneName
) {
// return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array);
export function createVarDataDimension(world, crossfilter, name) {
return crossfilter.dimension(
Crossfilter.ScalarDimension,
_worldVarDataCache[geneName],
world.varData.col(name).asArray(),
Float32Array
);
}
@@ -242,14 +203,6 @@ export function worldEqUniverse(world, universe) {
return world.obsAnnotations === universe.obsAnnotations;
}
export function subsetVarData(world, universe, varData) {
// If world === universe, just return the entire varData array
if (worldEqUniverse(world, universe)) {
return varData;
}
return sliceByIndex(varData, world.obsAnnotations.rowIndex.keys());
}
export function getSelectedByIndex(crossfilter) {
/*
return array of obsIndex, containing all selected obs/cells.
client/src/util/typedCrossfilter/index.js
+1 -1
View File
@@ -121,7 +121,7 @@ class TypedCrossfilter {
return res;
}
/* else, Dataframe-like */
return data.icutByMask(this.allFilteredMask());
return data.isubsetMask(this.allFilteredMask());
}
// return Uint8array containing selection state (truthy/falsey) for each record.