fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-22 09:28:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Performance work, plus fix #405 (#406)

Browse Source 
* remove memoization

* update to flash 1.0.2; turn on threading

* stand-alone helper routines for array slicing

* fix issue #405

* reset diffexp state when world changes

* performance work in dimension creation; fix world slicing bug

* update tests to match new state mgmt api

* update flask

* do not make dimensions for useless annotations

* update test to match optimizations
This commit is contained in:
Bruce Martin
2018-11-05 13:57:58 -08:00
committed by GitHub
parent 24a8a23355
commit 2d5bc9d0c9
10 changed files with 246 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/__tests__/util/stateManager/world.test.js
+27 -12
View File
@@ -77,7 +77,8 @@ describe("createWorldFromEntireUniverse", () => {
varDataCache: expect.any(Object),
worldObsIndex: null // indicating full universe
obsIndex: null, // null indicating full universe
obsBackIndex: null
})
);
});
@@ -120,16 +121,21 @@ describe("createWorldFromCurrentSelection", () => {
nObs: universeIndices.length,
obsAnnotations: _.map(universeIndices, i => universe.obsAnnotations[i]),
obsLayout: {
X: _.map(universeIndices, i => universe.obsLayout.X[i]),
Y: _.map(universeIndices, i => universe.obsLayout.Y[i])
X: new Float32Array(
_.map(universeIndices, i => universe.obsLayout.X[i])
),
Y: new Float32Array(
_.map(universeIndices, i => universe.obsLayout.Y[i])
)
},
worldObsIndex: _.transform(
obsBackIndex: _.transform(
universeIndices,
(result, univIdx, worldIdx) => {
result[univIdx] = worldIdx;
},
new Array(universe.nObs).fill(-1)
)
new Uint32Array(universe.nObs).fill(-1)
),
obsIndex: new Uint32Array(universeIndices)
};
expect(world).toMatchObject(
@@ -141,9 +147,13 @@ describe("createWorldFromCurrentSelection", () => {
obsAnnotations: expected.obsAnnotations,
varAnnotations: universe.varAnnotations,
obsLayout: expected.obsLayout,
summary: expect.any(Object) /* we could do better! */,
summary: {
obs: expect.any(Object) /* we could do better! */,
var: expect.any(Object) /* we could do better! */
},
varDataCache: expect.any(Object),
worldObsIndex: expected.worldObsIndex
obsIndex: expected.obsIndex,
obsBackIndex: expected.obsBackIndex
})
);
});
@@ -163,11 +173,15 @@ describe("createObsDimensionMap", () => {
expect(dimensionMap).toBeDefined();
REST.annotationsObs.names.forEach(name => {
const dim = dimensionMap[obsAnnoDimensionName(name)];
const { type } = schemaByObsName[name];
if (type === "string" || type === "boolean" || type === "categorical") {
expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
if (name === "name") {
expect(dim).toBeUndefined();
} else {
expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
const { type } = schemaByObsName[name];
if (type === "string" || type === "boolean" || type === "categorical") {
expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
} else {
expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
}
}
});
expect(dimensionMap[layoutDimensionName("X")]).toBeInstanceOf(
@@ -205,6 +219,7 @@ describe("subsetVarData", () => {
world,
crossfilter
);
expect(newWorld.obsIndex).toMatchObject(new Uint32Array([0, 2]));
/* expect a subset */
const result = World.subsetVarData(newWorld, universe, sourceVarData);
client/__tests__/util/typedCrossfilter/util.test.js
+115
View File
@@ -0,0 +1,115 @@
import {
fillRange,
sliceByIndex,
makeSortIndex,
lowerBound,
lowerBoundIndirect,
upperBoundIndirect
} from "../../../src/util/typedCrossfilter/util";
describe("fillRange", () => {
test("Array", () => {
expect(fillRange(new Array(6))).toMatchObject([0, 1, 2, 3, 4, 5]);
expect(fillRange(new Array(4), 1)).toMatchObject([1, 2, 3, 4]);
expect(fillRange([])).toMatchObject([]);
});
test("Uint32Array", () => {
expect(fillRange(new Uint32Array(6))).toMatchObject(
new Uint32Array([0, 1, 2, 3, 4, 5])
);
expect(fillRange(new Array(4), 1)).toMatchObject(
new Uint32Array([1, 2, 3, 4])
);
expect(fillRange([])).toMatchObject(new Uint32Array([]));
});
});
describe("sliceByIndex", () => {
test("Array", () => {
expect(sliceByIndex([0, 1, 2, 3, 4], [0, 1, 2])).toMatchObject([0, 1, 2]);
expect(sliceByIndex([0, 1, 2, 3, 4], [2, 1, 0])).toMatchObject([2, 1, 0]);
expect(sliceByIndex([0, 1, 2, 3, 4], [])).toMatchObject([]);
expect(sliceByIndex([], [])).toMatchObject([]);
});
test("Uint32Array", () => {
expect(
sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([0, 1, 2]))
).toMatchObject([0, 1, 2]);
expect(
sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([2, 1, 0]))
).toMatchObject([2, 1, 0]);
expect(sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([]))).toMatchObject(
[]
);
expect(sliceByIndex([], new Uint32Array([]))).toMatchObject([]);
expect(
sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([0, 1, 2]))
).toMatchObject(new Uint32Array([0, 1, 2]));
expect(
sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([2, 1, 0]))
).toMatchObject(new Uint32Array([2, 1, 0]));
expect(
sliceByIndex(
new Uint32Array([0, 1, 2, 3, 4]),
new Uint32Array(new Uint32Array([]))
)
).toMatchObject(new Uint32Array([]));
expect(
sliceByIndex(new Uint32Array([]), new Uint32Array([]))
).toMatchObject(new Uint32Array([]));
});
test("Float32Array", () => {
expect(
sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [0, 1, 2])
).toMatchObject(new Float32Array([0, 1, 2]));
expect(
sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [2, 1, 0])
).toMatchObject(new Float32Array([2, 1, 0]));
expect(sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [])).toMatchObject(
new Float32Array([])
);
expect(sliceByIndex(new Float32Array([]), [])).toMatchObject(
new Float32Array([])
);
});
});
describe("makeSortIndex", () => {
test("Array", () => {
expect(makeSortIndex([3, 2, 1, 0])).toMatchObject(
new Uint32Array([3, 2, 1, 0])
);
expect(makeSortIndex([3, 2, 1, 0, 4])).toMatchObject(
new Uint32Array([3, 2, 1, 0, 4])
);
expect(makeSortIndex([])).toMatchObject(new Uint32Array([]));
});
test("Float32Array", () => {
expect(makeSortIndex(new Float32Array([3, 2, 1, 0]))).toMatchObject(
new Uint32Array([3, 2, 1, 0])
);
expect(makeSortIndex(new Float32Array([3, 2, 1, 0, 4]))).toMatchObject(
new Uint32Array([3, 2, 1, 0, 4])
);
expect(makeSortIndex(new Float32Array([]))).toMatchObject(
new Uint32Array([])
);
});
test("Int32Array", () => {
expect(makeSortIndex(new Int32Array([3, 2, 1, 0]))).toMatchObject(
new Uint32Array([3, 2, 1, 0])
);
expect(makeSortIndex(new Int32Array([3, 2, 1, 0, 4]))).toMatchObject(
new Uint32Array([3, 2, 1, 0, 4])
);
expect(makeSortIndex(new Int32Array([]))).toMatchObject(
new Uint32Array([])
);
});
});
client/src/actions/index.js
+6 -10
View File
@@ -149,16 +149,13 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
return async (dispatch, getState) => {
dispatch({ type: "get single gene expression for coloring started" });
try {
const expressionData = await _doRequestExpressionData(
dispatch,
getState,
[gene]
);
await _doRequestExpressionData(dispatch, getState, [gene]);
const { world } = getState().controls;
dispatch({
type: "color by expression",
gene,
data: {
[gene]: expressionData[gene]
[gene]: kvCache.get(world.varDataCache, gene)
}
});
} catch (error) {
@@ -173,16 +170,15 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
const requestUserDefinedGene = gene => async (dispatch, getState) => {
dispatch({ type: "request user defined gene started" });
try {
const data = await await _doRequestExpressionData(dispatch, getState, [
gene
]);
await await _doRequestExpressionData(dispatch, getState, [gene]);
const { world } = getState().controls;
/* then send the success case action through */
return dispatch({
type: "request user defined gene success",
data: {
genes: [gene],
expression: data[gene]
expression: kvCache.get(world.varDataCache, gene)
}
});
} catch (error) {
client/src/reducers/differential.js
+1
View File
@@ -51,6 +51,7 @@ const Differential = (
case "set World to current selection":
return {
...state,
diffExp: null,
celllist1: null,
celllist2: null
};
client/src/util/stateManager/world.js
+50 -57
View File
@@ -4,6 +4,7 @@ import _ from "lodash";
import * as kvCache from "./keyvalcache";
import summarizeAnnotations from "./summarizeAnnotations";
import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
import { sliceByIndex } from "../typedCrossfilter/util";
/*
World is a subset of universe. Most code should use world, and should
@@ -56,7 +57,8 @@ function templateWorld() {
return {
// map from universe obsIndex to world offset.
// Undefined / null indicates identity mapping.
worldObsIndex: null,
obsIndex: null,
obsBackIndex: null,
/* schema/version related */
api: null,
@@ -90,7 +92,12 @@ export function createWorldFromEntireUniverse(universe) {
// map from the universe obsIndex to our world offset.
// undefined/null indicates identity map.
world.worldObsIndex = null;
// In other words obsBackIndex[universeIdx] -> worldIdx
world.obsBackIndex = null;
// Map to the universe index for each element in world.
// Null indicates identity map (aka world === universe)
// In other wrods obsIndex[worldIdx] -> universeIdx
world.obsIndex = null;
/*
public interface follows
@@ -130,42 +137,40 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
const newWorld = templateWorld();
/* these don't change as only OBS are selected in our current implementation */
newWorld.api = world.api;
newWorld.nVar = world.nVar;
newWorld.schema = world.schema;
newWorld.varAnnotations = world.varAnnotations;
newWorld.api = universe.api;
newWorld.nVar = universe.nVar;
newWorld.schema = universe.schema;
newWorld.varAnnotations = universe.varAnnotations;
/*
Subset world from universe based upon world's current selection. Only those
fields which are subset by observation selection/filtering need to be updated.
*/
const numSelected = crossfilter.countFiltered();
/*
Create a world which is based upon current selection
*/
newWorld.nObs = numSelected;
newWorld.obsAnnotations = new Array(numSelected);
newWorld.obsLayout = {
X: new Array(numSelected),
Y: new Array(numSelected)
};
newWorld.worldObsIndex = new Array(universe.nObs);
for (let i = 0, sel = 0; i < world.nObs; i += 1) {
/* build index maps and back maps based upon current selection state */
const obsBackIndex = new Uint32Array(universe.nObs);
obsBackIndex.fill(-1); // default - aka unused
const notSelected = obsBackIndex[0];
let nObs = 0;
for (let i = 0; i < universe.nObs; i += 1) {
if (crossfilter.isElementFiltered(i)) {
newWorld.obsAnnotations[sel] = world.obsAnnotations[i];
newWorld.obsLayout.X[sel] = world.obsLayout.X[i];
newWorld.obsLayout.Y[sel] = world.obsLayout.Y[i];
sel += 1;
obsBackIndex[i] = nObs;
nObs += 1;
}
}
const obsIndex = new Uint32Array(nObs);
for (let i = 0; i < universe.nObs; i += 1) {
const worldIdx = obsBackIndex[i];
if (worldIdx !== notSelected) {
obsIndex[worldIdx] = i;
}
}
// build index to our world offset
newWorld.worldObsIndex.fill(-1); // default - aka unused
for (let i = 0; i < newWorld.nObs; i += 1) {
newWorld.worldObsIndex[newWorld.obsAnnotations[i].__index__] = i;
}
newWorld.nObs = nObs;
newWorld.obsIndex = obsIndex;
newWorld.obsBackIndex = obsBackIndex;
/* now slice */
newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
newWorld.obsLayout = {
X: sliceByIndex(universe.obsLayout.X, obsIndex),
Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
};
/* derived data & summaries */
newWorld.summary = summarizeAnnotations(
@@ -226,14 +231,7 @@ export function createVarDimension(
crossfilter,
geneName
) {
const { worldObsIndex } = world;
const varData = _worldVarDataCache[geneName];
const worldIndex = worldObsIndex ? idx => worldObsIndex[idx] : idx => idx;
return crossfilter.dimension(
r => varData[worldIndex(r.__index__)],
Float32Array
);
return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array);
}
export function createObsDimensionMap(crossfilter, world) {
@@ -241,32 +239,32 @@ export function createObsDimensionMap(crossfilter, world) {
create and return a crossfilter dimension for every obs annotation
for which we have a supported type.
*/
const { schema, obsLayout, worldObsIndex } = world;
const { schema, obsLayout } = world;
const dimensionMap = _.transform(
schema.annotations.obs,
(result, anno) => {
// Create a crossfilter dimension for all obs annotations *except* 'name'
const dimensionMap = _(schema.annotations.obs)
.filter(anno => anno.name !== "name")
.transform((result, anno) => {
const dimType = deduceDimensionType(anno, anno.name);
// XXX if dimtype is a scalar, we may be able to do better?
if (dimType) {
result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
r => r[anno.name],
dimType
);
} // else ignore the annotation
},
{}
);
}, {})
.value();
/*
Add crossfilter dimensions allowing filtering on layout
*/
const worldIndex = worldObsIndex ? idx => worldObsIndex[idx] : idx => idx;
dimensionMap[layoutDimensionName("X")] = crossfilter.dimension(
r => obsLayout.X[worldIndex(r.__index__)],
obsLayout.X,
Float32Array
);
dimensionMap[layoutDimensionName("Y")] = crossfilter.dimension(
r => obsLayout.Y[worldIndex(r.__index__)],
obsLayout.Y,
Float32Array
);
@@ -282,10 +280,5 @@ export function subsetVarData(world, universe, varData) {
if (worldEqUniverse(world, universe)) {
return varData;
}
const newVarData = new Float32Array(world.nObs);
for (let i = 0; i < world.nObs; i += 1) {
newVarData[i] = varData[world.obsAnnotations[i].__index__];
}
return newVarData;
return sliceByIndex(varData, world.obsIndex);
}
client/src/util/typedCrossfilter/index.js
+23 -8
View File
@@ -31,7 +31,7 @@ more complex API. In a few cases, elements of that API were incorporated.
import PositiveIntervals from "./positiveIntervals";
import BitArray from "./bitArray";
import {
fillRange,
makeSortIndex,
lowerBound,
lowerBoundIndirect,
upperBoundIndirect
@@ -126,16 +126,31 @@ class ScalarDimension {
// current selection filter, expressed as PostiveIntervals.
this.currentFilter = [];
// Create value array
const array = this._createValueArray(
value,
new ValueArrayType(this.crossfilter.data.length)
);
// Two modes - caller can provide a pre-created value array,
// or a map function which will create it.
let array;
if (value instanceof ValueArrayType) {
if (value.length !== this.crossfilter.data.length) {
throw new RangeError(
"ScalarDimension values length must equal crossfilter data record count"
);
}
array = value;
} else if (value instanceof Function) {
// Create value array
array = this._createValueArray(
value,
new ValueArrayType(this.crossfilter.data.length)
);
} else {
throw new NotImplementedError(
"dimension value must be function or value array type"
);
}
this.value = array;
// create sort index
this.index = fillRange(new Uint32Array(this.crossfilter.data.length));
this.index.sort((a, b) => array[a] - array[b]);
this.index = makeSortIndex(array);
// groups, if any
this.groups = [];
client/src/util/typedCrossfilter/util.js
+19
View File
@@ -16,6 +16,25 @@ export function fillRange(arr, start = 0) {
return larr;
}
// slice out of one array into another, using an index array
//
export function sliceByIndex(src, index) {
if (index === undefined || index === null) {
return src;
}
const dst = new src.constructor(index.length);
for (let i = 0; i < index.length; i += 1) {
dst[i] = src[index[i]];
}
return dst;
}
export function makeSortIndex(src) {
const index = fillRange(new Uint32Array(src.length));
index.sort((a, b) => src[a] - src[b]);
return index;
}
// Search for `value` in the sorted array `arr`, in the range [first, last).
// Return the first (left most) index where arr[index] >= value.
//
server/app/scanpy_engine/scanpy_engine.py
+3 -8
View File
@@ -5,7 +5,6 @@ from pandas import DataFrame
import scanpy.api as sc
from scipy import stats, sparse
from server.app.app import cache
from server.app.driver.driver import CXGDriver
from server.app.util.constants import Axis, DEFAULT_TOP_N, DiffExpMode
from server.app.util.utils import FilterError, InteractiveError, PrepareError
@@ -96,11 +95,10 @@ class ScanpyEngine(CXGDriver):
@staticmethod
def _load_data(data):
# See https://scanpy.readthedocs.io/en/latest/api/scanpy.api.read.html
# Based upon this advice, setting cache=True parameter
# Based on benchmarking, cache=True has no impact on perf.
# Note: as of current scanpy/anndata release, setting backed='r' will
# result in an error.
return sc.read(data, cache=True)
# result in an error. https://github.com/theislab/anndata/issues/79
return sc.read(data, cache=False)
@staticmethod
def _top_sort(values, sort_order, top_n=None):
@@ -251,7 +249,6 @@ class ScanpyEngine(CXGDriver):
return data
@cache.memoize()
def annotation(self, filter, axis, fields=None):
"""
Gets annotation value for each observation
@@ -274,7 +271,6 @@ class ScanpyEngine(CXGDriver):
}
return result
@cache.memoize()
def data_frame(self, filter, axis):
"""
Retrieves data for each variable for observations in data frame
@@ -366,7 +362,6 @@ class ScanpyEngine(CXGDriver):
# Results need to be returned in var index order
return sorted(result, key=lambda gene: gene[0])
@cache.memoize()
def layout(self, filter, interactive_limit=None):
"""
Computes a n-d layout for cells through dimensionality reduction.
server/cli/launch.py
+1 -1
View File
@@ -96,4 +96,4 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
click.echo('[cellxgene] Type CTRL-C at any time to exit.')
app.run(host=host, debug=debug, port=port)
app.run(host=host, debug=debug, port=port, threaded=True)
server/requirements.txt
+1 -1
View File
@@ -1,6 +1,6 @@
anndata>=0.6.12
click==6.7
Flask==0.12.4
Flask>=1.0.2
Flask-Caching==1.4.0
Flask-Compress==1.4.0
Flask-Cors==3.0.6