diff --git a/client/__tests__/util/stateManager/world.test.js b/client/__tests__/util/stateManager/world.test.js index 6e8c0bf5..4de45b7c 100644 --- a/client/__tests__/util/stateManager/world.test.js +++ b/client/__tests__/util/stateManager/world.test.js @@ -77,7 +77,8 @@ describe("createWorldFromEntireUniverse", () => { varDataCache: expect.any(Object), - worldObsIndex: null // indicating full universe + obsIndex: null, // null indicating full universe + obsBackIndex: null }) ); }); @@ -120,16 +121,21 @@ describe("createWorldFromCurrentSelection", () => { nObs: universeIndices.length, obsAnnotations: _.map(universeIndices, i => universe.obsAnnotations[i]), obsLayout: { - X: _.map(universeIndices, i => universe.obsLayout.X[i]), - Y: _.map(universeIndices, i => universe.obsLayout.Y[i]) + X: new Float32Array( + _.map(universeIndices, i => universe.obsLayout.X[i]) + ), + Y: new Float32Array( + _.map(universeIndices, i => universe.obsLayout.Y[i]) + ) }, - worldObsIndex: _.transform( + obsBackIndex: _.transform( universeIndices, (result, univIdx, worldIdx) => { result[univIdx] = worldIdx; }, - new Array(universe.nObs).fill(-1) - ) + new Uint32Array(universe.nObs).fill(-1) + ), + obsIndex: new Uint32Array(universeIndices) }; expect(world).toMatchObject( @@ -141,9 +147,13 @@ describe("createWorldFromCurrentSelection", () => { obsAnnotations: expected.obsAnnotations, varAnnotations: universe.varAnnotations, obsLayout: expected.obsLayout, - summary: expect.any(Object) /* we could do better! */, + summary: { + obs: expect.any(Object) /* we could do better! */, + var: expect.any(Object) /* we could do better! */ + }, varDataCache: expect.any(Object), - worldObsIndex: expected.worldObsIndex + obsIndex: expected.obsIndex, + obsBackIndex: expected.obsBackIndex }) ); }); @@ -163,11 +173,15 @@ describe("createObsDimensionMap", () => { expect(dimensionMap).toBeDefined(); REST.annotationsObs.names.forEach(name => { const dim = dimensionMap[obsAnnoDimensionName(name)]; - const { type } = schemaByObsName[name]; - if (type === "string" || type === "boolean" || type === "categorical") { - expect(dim).toBeInstanceOf(Crossfilter.EnumDimension); + if (name === "name") { + expect(dim).toBeUndefined(); } else { - expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension); + const { type } = schemaByObsName[name]; + if (type === "string" || type === "boolean" || type === "categorical") { + expect(dim).toBeInstanceOf(Crossfilter.EnumDimension); + } else { + expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension); + } } }); expect(dimensionMap[layoutDimensionName("X")]).toBeInstanceOf( @@ -205,6 +219,7 @@ describe("subsetVarData", () => { world, crossfilter ); + expect(newWorld.obsIndex).toMatchObject(new Uint32Array([0, 2])); /* expect a subset */ const result = World.subsetVarData(newWorld, universe, sourceVarData); diff --git a/client/__tests__/util/typedCrossfilter/util.test.js b/client/__tests__/util/typedCrossfilter/util.test.js new file mode 100644 index 00000000..b002782a --- /dev/null +++ b/client/__tests__/util/typedCrossfilter/util.test.js @@ -0,0 +1,115 @@ +import { + fillRange, + sliceByIndex, + makeSortIndex, + lowerBound, + lowerBoundIndirect, + upperBoundIndirect +} from "../../../src/util/typedCrossfilter/util"; + +describe("fillRange", () => { + test("Array", () => { + expect(fillRange(new Array(6))).toMatchObject([0, 1, 2, 3, 4, 5]); + expect(fillRange(new Array(4), 1)).toMatchObject([1, 2, 3, 4]); + expect(fillRange([])).toMatchObject([]); + }); + + test("Uint32Array", () => { + expect(fillRange(new Uint32Array(6))).toMatchObject( + new Uint32Array([0, 1, 2, 3, 4, 5]) + ); + expect(fillRange(new Array(4), 1)).toMatchObject( + new Uint32Array([1, 2, 3, 4]) + ); + expect(fillRange([])).toMatchObject(new Uint32Array([])); + }); +}); + +describe("sliceByIndex", () => { + test("Array", () => { + expect(sliceByIndex([0, 1, 2, 3, 4], [0, 1, 2])).toMatchObject([0, 1, 2]); + expect(sliceByIndex([0, 1, 2, 3, 4], [2, 1, 0])).toMatchObject([2, 1, 0]); + expect(sliceByIndex([0, 1, 2, 3, 4], [])).toMatchObject([]); + expect(sliceByIndex([], [])).toMatchObject([]); + }); + + test("Uint32Array", () => { + expect( + sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([0, 1, 2])) + ).toMatchObject([0, 1, 2]); + expect( + sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([2, 1, 0])) + ).toMatchObject([2, 1, 0]); + expect(sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([]))).toMatchObject( + [] + ); + expect(sliceByIndex([], new Uint32Array([]))).toMatchObject([]); + + expect( + sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([0, 1, 2])) + ).toMatchObject(new Uint32Array([0, 1, 2])); + expect( + sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([2, 1, 0])) + ).toMatchObject(new Uint32Array([2, 1, 0])); + expect( + sliceByIndex( + new Uint32Array([0, 1, 2, 3, 4]), + new Uint32Array(new Uint32Array([])) + ) + ).toMatchObject(new Uint32Array([])); + expect( + sliceByIndex(new Uint32Array([]), new Uint32Array([])) + ).toMatchObject(new Uint32Array([])); + }); + + test("Float32Array", () => { + expect( + sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [0, 1, 2]) + ).toMatchObject(new Float32Array([0, 1, 2])); + expect( + sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [2, 1, 0]) + ).toMatchObject(new Float32Array([2, 1, 0])); + expect(sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [])).toMatchObject( + new Float32Array([]) + ); + expect(sliceByIndex(new Float32Array([]), [])).toMatchObject( + new Float32Array([]) + ); + }); +}); + +describe("makeSortIndex", () => { + test("Array", () => { + expect(makeSortIndex([3, 2, 1, 0])).toMatchObject( + new Uint32Array([3, 2, 1, 0]) + ); + expect(makeSortIndex([3, 2, 1, 0, 4])).toMatchObject( + new Uint32Array([3, 2, 1, 0, 4]) + ); + expect(makeSortIndex([])).toMatchObject(new Uint32Array([])); + }); + + test("Float32Array", () => { + expect(makeSortIndex(new Float32Array([3, 2, 1, 0]))).toMatchObject( + new Uint32Array([3, 2, 1, 0]) + ); + expect(makeSortIndex(new Float32Array([3, 2, 1, 0, 4]))).toMatchObject( + new Uint32Array([3, 2, 1, 0, 4]) + ); + expect(makeSortIndex(new Float32Array([]))).toMatchObject( + new Uint32Array([]) + ); + }); + + test("Int32Array", () => { + expect(makeSortIndex(new Int32Array([3, 2, 1, 0]))).toMatchObject( + new Uint32Array([3, 2, 1, 0]) + ); + expect(makeSortIndex(new Int32Array([3, 2, 1, 0, 4]))).toMatchObject( + new Uint32Array([3, 2, 1, 0, 4]) + ); + expect(makeSortIndex(new Int32Array([]))).toMatchObject( + new Uint32Array([]) + ); + }); +}); diff --git a/client/src/actions/index.js b/client/src/actions/index.js index 2f1ddf3b..645c2c8c 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -149,16 +149,13 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) { return async (dispatch, getState) => { dispatch({ type: "get single gene expression for coloring started" }); try { - const expressionData = await _doRequestExpressionData( - dispatch, - getState, - [gene] - ); + await _doRequestExpressionData(dispatch, getState, [gene]); + const { world } = getState().controls; dispatch({ type: "color by expression", gene, data: { - [gene]: expressionData[gene] + [gene]: kvCache.get(world.varDataCache, gene) } }); } catch (error) { @@ -173,16 +170,15 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) { const requestUserDefinedGene = gene => async (dispatch, getState) => { dispatch({ type: "request user defined gene started" }); try { - const data = await await _doRequestExpressionData(dispatch, getState, [ - gene - ]); + await await _doRequestExpressionData(dispatch, getState, [gene]); + const { world } = getState().controls; /* then send the success case action through */ return dispatch({ type: "request user defined gene success", data: { genes: [gene], - expression: data[gene] + expression: kvCache.get(world.varDataCache, gene) } }); } catch (error) { diff --git a/client/src/reducers/differential.js b/client/src/reducers/differential.js index 6fa25926..966f8020 100644 --- a/client/src/reducers/differential.js +++ b/client/src/reducers/differential.js @@ -51,6 +51,7 @@ const Differential = ( case "set World to current selection": return { ...state, + diffExp: null, celllist1: null, celllist2: null }; diff --git a/client/src/util/stateManager/world.js b/client/src/util/stateManager/world.js index 881a5177..08cbea44 100644 --- a/client/src/util/stateManager/world.js +++ b/client/src/util/stateManager/world.js @@ -4,6 +4,7 @@ import _ from "lodash"; import * as kvCache from "./keyvalcache"; import summarizeAnnotations from "./summarizeAnnotations"; import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators"; +import { sliceByIndex } from "../typedCrossfilter/util"; /* World is a subset of universe. Most code should use world, and should @@ -56,7 +57,8 @@ function templateWorld() { return { // map from universe obsIndex to world offset. // Undefined / null indicates identity mapping. - worldObsIndex: null, + obsIndex: null, + obsBackIndex: null, /* schema/version related */ api: null, @@ -90,7 +92,12 @@ export function createWorldFromEntireUniverse(universe) { // map from the universe obsIndex to our world offset. // undefined/null indicates identity map. - world.worldObsIndex = null; + // In other words obsBackIndex[universeIdx] -> worldIdx + world.obsBackIndex = null; + // Map to the universe index for each element in world. + // Null indicates identity map (aka world === universe) + // In other wrods obsIndex[worldIdx] -> universeIdx + world.obsIndex = null; /* public interface follows @@ -130,42 +137,40 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) { const newWorld = templateWorld(); /* these don't change as only OBS are selected in our current implementation */ - newWorld.api = world.api; - newWorld.nVar = world.nVar; - newWorld.schema = world.schema; - newWorld.varAnnotations = world.varAnnotations; + newWorld.api = universe.api; + newWorld.nVar = universe.nVar; + newWorld.schema = universe.schema; + newWorld.varAnnotations = universe.varAnnotations; - /* - Subset world from universe based upon world's current selection. Only those - fields which are subset by observation selection/filtering need to be updated. - */ - const numSelected = crossfilter.countFiltered(); - - /* - Create a world which is based upon current selection - */ - newWorld.nObs = numSelected; - newWorld.obsAnnotations = new Array(numSelected); - newWorld.obsLayout = { - X: new Array(numSelected), - Y: new Array(numSelected) - }; - newWorld.worldObsIndex = new Array(universe.nObs); - - for (let i = 0, sel = 0; i < world.nObs; i += 1) { + /* build index maps and back maps based upon current selection state */ + const obsBackIndex = new Uint32Array(universe.nObs); + obsBackIndex.fill(-1); // default - aka unused + const notSelected = obsBackIndex[0]; + let nObs = 0; + for (let i = 0; i < universe.nObs; i += 1) { if (crossfilter.isElementFiltered(i)) { - newWorld.obsAnnotations[sel] = world.obsAnnotations[i]; - newWorld.obsLayout.X[sel] = world.obsLayout.X[i]; - newWorld.obsLayout.Y[sel] = world.obsLayout.Y[i]; - sel += 1; + obsBackIndex[i] = nObs; + nObs += 1; + } + } + const obsIndex = new Uint32Array(nObs); + for (let i = 0; i < universe.nObs; i += 1) { + const worldIdx = obsBackIndex[i]; + if (worldIdx !== notSelected) { + obsIndex[worldIdx] = i; } } - // build index to our world offset - newWorld.worldObsIndex.fill(-1); // default - aka unused - for (let i = 0; i < newWorld.nObs; i += 1) { - newWorld.worldObsIndex[newWorld.obsAnnotations[i].__index__] = i; - } + newWorld.nObs = nObs; + newWorld.obsIndex = obsIndex; + newWorld.obsBackIndex = obsBackIndex; + + /* now slice */ + newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex); + newWorld.obsLayout = { + X: sliceByIndex(universe.obsLayout.X, obsIndex), + Y: sliceByIndex(universe.obsLayout.Y, obsIndex) + }; /* derived data & summaries */ newWorld.summary = summarizeAnnotations( @@ -226,14 +231,7 @@ export function createVarDimension( crossfilter, geneName ) { - const { worldObsIndex } = world; - const varData = _worldVarDataCache[geneName]; - const worldIndex = worldObsIndex ? idx => worldObsIndex[idx] : idx => idx; - - return crossfilter.dimension( - r => varData[worldIndex(r.__index__)], - Float32Array - ); + return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array); } export function createObsDimensionMap(crossfilter, world) { @@ -241,32 +239,32 @@ export function createObsDimensionMap(crossfilter, world) { create and return a crossfilter dimension for every obs annotation for which we have a supported type. */ - const { schema, obsLayout, worldObsIndex } = world; + const { schema, obsLayout } = world; - const dimensionMap = _.transform( - schema.annotations.obs, - (result, anno) => { + // Create a crossfilter dimension for all obs annotations *except* 'name' + const dimensionMap = _(schema.annotations.obs) + .filter(anno => anno.name !== "name") + .transform((result, anno) => { const dimType = deduceDimensionType(anno, anno.name); + // XXX if dimtype is a scalar, we may be able to do better? if (dimType) { result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension( r => r[anno.name], dimType ); } // else ignore the annotation - }, - {} - ); + }, {}) + .value(); /* Add crossfilter dimensions allowing filtering on layout */ - const worldIndex = worldObsIndex ? idx => worldObsIndex[idx] : idx => idx; dimensionMap[layoutDimensionName("X")] = crossfilter.dimension( - r => obsLayout.X[worldIndex(r.__index__)], + obsLayout.X, Float32Array ); dimensionMap[layoutDimensionName("Y")] = crossfilter.dimension( - r => obsLayout.Y[worldIndex(r.__index__)], + obsLayout.Y, Float32Array ); @@ -282,10 +280,5 @@ export function subsetVarData(world, universe, varData) { if (worldEqUniverse(world, universe)) { return varData; } - - const newVarData = new Float32Array(world.nObs); - for (let i = 0; i < world.nObs; i += 1) { - newVarData[i] = varData[world.obsAnnotations[i].__index__]; - } - return newVarData; + return sliceByIndex(varData, world.obsIndex); } diff --git a/client/src/util/typedCrossfilter/index.js b/client/src/util/typedCrossfilter/index.js index e2ce88ee..c5d2029e 100644 --- a/client/src/util/typedCrossfilter/index.js +++ b/client/src/util/typedCrossfilter/index.js @@ -31,7 +31,7 @@ more complex API. In a few cases, elements of that API were incorporated. import PositiveIntervals from "./positiveIntervals"; import BitArray from "./bitArray"; import { - fillRange, + makeSortIndex, lowerBound, lowerBoundIndirect, upperBoundIndirect @@ -126,16 +126,31 @@ class ScalarDimension { // current selection filter, expressed as PostiveIntervals. this.currentFilter = []; - // Create value array - const array = this._createValueArray( - value, - new ValueArrayType(this.crossfilter.data.length) - ); + // Two modes - caller can provide a pre-created value array, + // or a map function which will create it. + let array; + if (value instanceof ValueArrayType) { + if (value.length !== this.crossfilter.data.length) { + throw new RangeError( + "ScalarDimension values length must equal crossfilter data record count" + ); + } + array = value; + } else if (value instanceof Function) { + // Create value array + array = this._createValueArray( + value, + new ValueArrayType(this.crossfilter.data.length) + ); + } else { + throw new NotImplementedError( + "dimension value must be function or value array type" + ); + } this.value = array; // create sort index - this.index = fillRange(new Uint32Array(this.crossfilter.data.length)); - this.index.sort((a, b) => array[a] - array[b]); + this.index = makeSortIndex(array); // groups, if any this.groups = []; diff --git a/client/src/util/typedCrossfilter/util.js b/client/src/util/typedCrossfilter/util.js index 028b6db4..108f1923 100644 --- a/client/src/util/typedCrossfilter/util.js +++ b/client/src/util/typedCrossfilter/util.js @@ -16,6 +16,25 @@ export function fillRange(arr, start = 0) { return larr; } +// slice out of one array into another, using an index array +// +export function sliceByIndex(src, index) { + if (index === undefined || index === null) { + return src; + } + const dst = new src.constructor(index.length); + for (let i = 0; i < index.length; i += 1) { + dst[i] = src[index[i]]; + } + return dst; +} + +export function makeSortIndex(src) { + const index = fillRange(new Uint32Array(src.length)); + index.sort((a, b) => src[a] - src[b]); + return index; +} + // Search for `value` in the sorted array `arr`, in the range [first, last). // Return the first (left most) index where arr[index] >= value. // diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 7d2937a6..3584c114 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -5,7 +5,6 @@ from pandas import DataFrame import scanpy.api as sc from scipy import stats, sparse -from server.app.app import cache from server.app.driver.driver import CXGDriver from server.app.util.constants import Axis, DEFAULT_TOP_N, DiffExpMode from server.app.util.utils import FilterError, InteractiveError, PrepareError @@ -96,11 +95,10 @@ class ScanpyEngine(CXGDriver): @staticmethod def _load_data(data): - # See https://scanpy.readthedocs.io/en/latest/api/scanpy.api.read.html - # Based upon this advice, setting cache=True parameter + # Based on benchmarking, cache=True has no impact on perf. # Note: as of current scanpy/anndata release, setting backed='r' will - # result in an error. - return sc.read(data, cache=True) + # result in an error. https://github.com/theislab/anndata/issues/79 + return sc.read(data, cache=False) @staticmethod def _top_sort(values, sort_order, top_n=None): @@ -251,7 +249,6 @@ class ScanpyEngine(CXGDriver): return data - @cache.memoize() def annotation(self, filter, axis, fields=None): """ Gets annotation value for each observation @@ -274,7 +271,6 @@ class ScanpyEngine(CXGDriver): } return result - @cache.memoize() def data_frame(self, filter, axis): """ Retrieves data for each variable for observations in data frame @@ -366,7 +362,6 @@ class ScanpyEngine(CXGDriver): # Results need to be returned in var index order return sorted(result, key=lambda gene: gene[0]) - @cache.memoize() def layout(self, filter, interactive_limit=None): """ Computes a n-d layout for cells through dimensionality reduction. diff --git a/server/cli/launch.py b/server/cli/launch.py index 7390531c..bce26982 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -96,4 +96,4 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names, click.echo('[cellxgene] Type CTRL-C at any time to exit.') - app.run(host=host, debug=debug, port=port) + app.run(host=host, debug=debug, port=port, threaded=True) diff --git a/server/requirements.txt b/server/requirements.txt index 652e8dbe..6da25340 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,6 +1,6 @@ anndata>=0.6.12 click==6.7 -Flask==0.12.4 +Flask>=1.0.2 Flask-Caching==1.4.0 Flask-Compress==1.4.0 Flask-Cors==3.0.6