diff --git a/client/src/reducers/controls.js b/client/src/reducers/controls.js index 736cdd80..8c3f716e 100644 --- a/client/src/reducers/controls.js +++ b/client/src/reducers/controls.js @@ -2,7 +2,7 @@ import _ from "lodash"; -import { World, WorldUtil } from "../util/stateManager"; +import { World, WorldUtil, ControlsHelper } from "../util/stateManager"; import parseRGB from "../util/parseRGB"; import Crossfilter from "../util/typedCrossfilter"; import * as globals from "../globals"; @@ -13,153 +13,6 @@ import { diffexpDimensionName, makeContinuousDimensionName } from "../util/nameCreators"; -import { fillRange } from "../util/typedCrossfilter/util"; - -/* -Selection state for categoricals are tracked in an Object that -has two main components for each category: -1. mapping of option value to an index -2. array of bool selection state by index -Remember that option values can be ANY js type, except undefined/null. - - { - _category_name_1: { - // map of option value to index - categoryIndices: Map([ - catval1: index, - ... - ]) - - // index->selection true/false state - categorySelected: [ true/false, true/false, ... ] - - // number of options - numCategories: number, - - // isTruncated - true if the options for selection has - // been truncated (ie, was too large to implement) - } - } -*/ -function topNCategories(summary) { - const counts = _.map(summary.categories, cat => - summary.categoryCounts.get(cat) - ); - const sortIndex = fillRange(new Array(summary.numCategories)).sort( - (a, b) => counts[b] - counts[a] - ); - const sortedCategories = _.map(sortIndex, i => summary.categories[i]); - const sortedCounts = _.map(sortIndex, i => counts[i]); - const N = globals.maxCategoricalOptionsToDisplay; - - if (sortedCategories.length < N) { - return [sortedCategories, sortedCounts]; - } - return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)]; -} - -function createCategoricalSelectionState(state, world) { - const res = {}; - _.forEach(world.obsAnnotations.colIndex.keys(), key => { - const summary = world.obsAnnotations.col(key).summarize(); - if (summary.categories) { - const isColorField = key.includes("color") || key.includes("Color"); - const isSelectableCategory = - !isColorField && - key !== "name" && - summary.categories.length < state.maxCategoryItems; - if (isSelectableCategory) { - const [categoryValues, categoryCounts] = topNCategories(summary); - const categoryIndices = new Map(categoryValues.map((v, i) => [v, i])); - const numCategories = categoryIndices.size; - const categorySelected = new Array(numCategories).fill(true); - const isTruncated = categoryValues.length < summary.numCategories; - res[key] = { - categoryValues, // array: of natively typed category values - categoryIndices, // map: category value (native type) -> category index - categorySelected, // array: t/f selection state - numCategories, // number: of categories - isTruncated, // bool: true if list was truncated - categoryCounts // array: cardinality of each category - }; - } - } - }); - return res; -} - -/* -given a categoricalSelectionState, return the list of all category values -where selection state is true (ie, they are selected). -*/ -function selectedValuesForCategory(categorySelectionState) { - const selectedValues = _([...categorySelectionState.categoryIndices]) - .filter(tuple => categorySelectionState.categorySelected[tuple[1]]) - .map(tuple => tuple[0]) - .value(); - return selectedValues; -} - -/* -build a crossfilter dimension map for all gene expression related dimensions. -*/ -function createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter) { - function _createGenesDimMap(genes, nameCreator) { - return genes.reduce((acc, gene) => { - acc[nameCreator(gene)] = World.createVarDataDimension( - world, - crossfilter, - gene - ); - return acc; - }, {}); - } - - return { - ..._createGenesDimMap(userDefinedGenes, userDefinedDimensionName), - ..._createGenesDimMap(diffexpGenes, diffexpDimensionName) - }; -} - -function pruneVarDataCache(varData, needed) { - /* - Remove any unneeded columns from the varData dataframe. Will only - prune / remove if the total column count exceeds VarDataCacheLowWatermark - - Note: this code leverages the fact that dataframe offsets indicate - the order in which the columns were added. This crudely provides - LRU semantics, so we can delete "older" columns first. - */ - - /* - VarDataCacheLowWatermark - this cofig value sets the minimum cache size, - in columns, below which we don't throw away data. - - The value should be high enough so we are caching the maximum which will - "typically" be used in the UI (currently: 10 for diffexp, and N for user- - specified genes), and low enough to account for memory use (any single - column size is 4 bytes * numObs, so a column can be multi-megabyte in common - use cases). - */ - const VarDataCacheLowWatermark = 32; - - const numOverWatermark = varData.dims[1] - VarDataCacheLowWatermark; - if (numOverWatermark <= 0) return varData; - - const { colIndex } = varData; - const all = colIndex.keys(); - const unused = _.difference(all, needed); - if (unused.length > 0) { - // sort by offset in the dataframe - ie, psuedo-LRU - unused.sort((a, b) => colIndex.getOffset(a) - colIndex.getOffset(b)); - const numToDrop = - unused.length < numOverWatermark ? unused.length : numOverWatermark; - for (let i = 0; i < numToDrop; i += 1) { - varData = varData.dropCol(unused[i]); - } - } - return varData; -} const Controls = ( state = { @@ -233,7 +86,7 @@ const Controls = ( const colorRGB = new Array(universe.nObs).fill( parseRGB(globals.defaultCellColor) ); - const categoricalSelectionState = createCategoricalSelectionState( + const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState( state, world ); @@ -269,7 +122,7 @@ const Controls = ( const colorRGB = new Array(universe.nObs).fill( parseRGB(globals.defaultCellColor) ); - const categoricalSelectionState = createCategoricalSelectionState( + const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState( state, world ); @@ -282,7 +135,12 @@ const Controls = ( }); const dimensionMap = { ...fullUniverseCache.dimensionMap, - ...createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter) + ...ControlsHelper.createGenesDimMap( + userDefinedGenes, + diffexpGenes, + world, + crossfilter + ) }; WorldUtil.clearCaches(); @@ -309,14 +167,19 @@ const Controls = ( const colorRGB = new Array(world.nObs).fill( parseRGB(globals.defaultCellColor) ); - const categoricalSelectionState = createCategoricalSelectionState( + const categoricalSelectionState = ControlsHelper.createCategoricalSelectionState( state, world ); const crossfilter = Crossfilter(world.obsAnnotations); const dimensionMap = { ...World.createObsDimensionMap(crossfilter, world), - ...createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter) + ...ControlsHelper.createGenesDimMap( + userDefinedGenes, + diffexpGenes, + world, + crossfilter + ) }; WorldUtil.clearCaches(); @@ -376,8 +239,14 @@ const Controls = ( Object.keys(action.expressionData) ) ); - universeVarData = pruneVarDataCache(universeVarData, allTheGenesWeNeed); - worldVarData = pruneVarDataCache(worldVarData, allTheGenesWeNeed); + universeVarData = ControlsHelper.pruneVarDataCache( + universeVarData, + allTheGenesWeNeed + ); + worldVarData = ControlsHelper.pruneVarDataCache( + worldVarData, + allTheGenesWeNeed + ); return { ...state, @@ -442,7 +311,7 @@ const Controls = ( }; } case "clear differential expression": { - const { world, universe, dimensionMap } = state; + const { world, dimensionMap } = state; const _dimensionMap = dimensionMap; _.forEach(action.diffExp, values => { const name = world.varAnnotations.at(values[0], "name"); @@ -607,7 +476,7 @@ const Controls = ( // update the filter to match all selected options const cat = newCategoricalSelectionState[action.metadataField]; state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum( - selectedValuesForCategory(cat) + ControlsHelper.selectedValuesForCategory(cat) ); return { @@ -631,7 +500,7 @@ const Controls = ( // update the filter to match all selected options const cat = newCategoricalSelectionState[action.metadataField]; state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum( - selectedValuesForCategory(cat) + ControlsHelper.selectedValuesForCategory(cat) ); return { diff --git a/client/src/util/stateManager/controlsHelpers.js b/client/src/util/stateManager/controlsHelpers.js new file mode 100644 index 00000000..dc03ba4b --- /dev/null +++ b/client/src/util/stateManager/controlsHelpers.js @@ -0,0 +1,164 @@ +/* +Helper functions for the controls reducer +*/ + +import _ from "lodash"; + +import * as globals from "../../globals"; +import { fillRange } from "../typedCrossfilter/util"; +import { + userDefinedDimensionName, + diffexpDimensionName +} from "../nameCreators"; +import * as World from "./world"; + +/* +Selection state for categoricals are tracked in an Object that +has two main components for each category: +1. mapping of option value to an index +2. array of bool selection state by index +Remember that option values can be ANY js type, except undefined/null. + + { + _category_name_1: { + // map of option value to index + categoryIndices: Map([ + catval1: index, + ... + ]) + + // index->selection true/false state + categorySelected: [ true/false, true/false, ... ] + + // number of options + numCategories: number, + + // isTruncated - true if the options for selection has + // been truncated (ie, was too large to implement) + } + } +*/ +function topNCategories(summary) { + const counts = _.map(summary.categories, cat => + summary.categoryCounts.get(cat) + ); + const sortIndex = fillRange(new Array(summary.numCategories)).sort( + (a, b) => counts[b] - counts[a] + ); + const sortedCategories = _.map(sortIndex, i => summary.categories[i]); + const sortedCounts = _.map(sortIndex, i => counts[i]); + const N = globals.maxCategoricalOptionsToDisplay; + + if (sortedCategories.length < N) { + return [sortedCategories, sortedCounts]; + } + return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)]; +} + +export function createCategoricalSelectionState(state, world) { + const res = {}; + _.forEach(world.obsAnnotations.colIndex.keys(), key => { + const summary = world.obsAnnotations.col(key).summarize(); + if (summary.categories) { + const isColorField = key.includes("color") || key.includes("Color"); + const isSelectableCategory = + !isColorField && + key !== "name" && + summary.categories.length < state.maxCategoryItems; + if (isSelectableCategory) { + const [categoryValues, categoryCounts] = topNCategories(summary); + const categoryIndices = new Map(categoryValues.map((v, i) => [v, i])); + const numCategories = categoryIndices.size; + const categorySelected = new Array(numCategories).fill(true); + const isTruncated = categoryValues.length < summary.numCategories; + res[key] = { + categoryValues, // array: of natively typed category values + categoryIndices, // map: category value (native type) -> category index + categorySelected, // array: t/f selection state + numCategories, // number: of categories + isTruncated, // bool: true if list was truncated + categoryCounts // array: cardinality of each category + }; + } + } + }); + return res; +} + +/* +given a categoricalSelectionState, return the list of all category values +where selection state is true (ie, they are selected). +*/ +export function selectedValuesForCategory(categorySelectionState) { + const selectedValues = _([...categorySelectionState.categoryIndices]) + .filter(tuple => categorySelectionState.categorySelected[tuple[1]]) + .map(tuple => tuple[0]) + .value(); + return selectedValues; +} + +/* +build a crossfilter dimension map for all gene expression related dimensions. +*/ +export function createGenesDimMap( + userDefinedGenes, + diffexpGenes, + world, + crossfilter +) { + function _createGenesDimMap(genes, nameCreator) { + return genes.reduce((acc, gene) => { + acc[nameCreator(gene)] = World.createVarDataDimension( + world, + crossfilter, + gene + ); + return acc; + }, {}); + } + + return { + ..._createGenesDimMap(userDefinedGenes, userDefinedDimensionName), + ..._createGenesDimMap(diffexpGenes, diffexpDimensionName) + }; +} + +export function pruneVarDataCache(varData, needed) { + /* + Remove any unneeded columns from the varData dataframe. Will only + prune / remove if the total column count exceeds VarDataCacheLowWatermark + + Note: this code leverages the fact that dataframe offsets indicate + the order in which the columns were added. This crudely provides + LRU semantics, so we can delete "older" columns first. + */ + + /* + VarDataCacheLowWatermark - this cofig value sets the minimum cache size, + in columns, below which we don't throw away data. + + The value should be high enough so we are caching the maximum which will + "typically" be used in the UI (currently: 10 for diffexp, and N for user- + specified genes), and low enough to account for memory use (any single + column size is 4 bytes * numObs, so a column can be multi-megabyte in common + use cases). + */ + const VarDataCacheLowWatermark = 32; + + const numOverWatermark = varData.dims[1] - VarDataCacheLowWatermark; + if (numOverWatermark <= 0) return varData; + + const { colIndex } = varData; + const all = colIndex.keys(); + const unused = _.difference(all, needed); + if (unused.length > 0) { + // sort by offset in the dataframe - ie, psuedo-LRU + unused.sort((a, b) => colIndex.getOffset(a) - colIndex.getOffset(b)); + const numToDrop = + unused.length < numOverWatermark ? unused.length : numOverWatermark; + for (let i = 0; i < numToDrop; i += 1) { + varData = varData.dropCol(unused[i]); + } + } + return varData; +} diff --git a/client/src/util/stateManager/index.js b/client/src/util/stateManager/index.js index b3140a2d..8008d5f5 100644 --- a/client/src/util/stateManager/index.js +++ b/client/src/util/stateManager/index.js @@ -17,3 +17,4 @@ exists to support those concepts. export * as Universe from "./universe"; export * as World from "./world"; export * as WorldUtil from "./worldUtil"; +export * as ControlsHelper from "./controlsHelpers"; diff --git a/client/src/util/stateManager/world.js b/client/src/util/stateManager/world.js index 81641c9a..8104a48b 100644 --- a/client/src/util/stateManager/world.js +++ b/client/src/util/stateManager/world.js @@ -3,7 +3,6 @@ import _ from "lodash"; import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators"; import Crossfilter from "../typedCrossfilter"; -import { sliceByIndex } from "../typedCrossfilter/util"; import * as Dataframe from "../dataframe"; /*