From 0c271bc815c7dcc6e46de549663b8fe1e64f377c Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Mon, 22 Oct 2018 09:56:41 -0700 Subject: [PATCH] categorical metadata performance work (#356) * do not create unecessary option state for categories we will not display * performance improvements for categorical metadata handling --- .../__tests__/util/stateManager/world.test.js | 7 +- client/src/actions/index.js | 15 ++- .../src/components/categorical/categorical.js | 17 +++- client/src/components/categorical/category.js | 34 +++++-- client/src/globals.js | 13 +++ client/src/reducers/config.js | 3 +- client/src/reducers/controls.js | 2 +- .../util/stateManager/summarizeAnnotations.js | 94 +++++++++++++++++++ client/src/util/stateManager/universe.js | 8 ++ client/src/util/stateManager/world.js | 87 ++--------------- 10 files changed, 184 insertions(+), 96 deletions(-) create mode 100644 client/src/util/stateManager/summarizeAnnotations.js diff --git a/client/__tests__/util/stateManager/world.test.js b/client/__tests__/util/stateManager/world.test.js index 4cae7872..a90ef3cd 100644 --- a/client/__tests__/util/stateManager/world.test.js +++ b/client/__tests__/util/stateManager/world.test.js @@ -64,10 +64,15 @@ describe("createWorldFromEntireUniverse", () => { summary: expect.objectContaining({ obs: _(REST.schema.schema.annotations.obs) + .filter(v => v.name !== "name") .keyBy("name") .mapValues(() => expect.any(Object)) .value(), - var: {} // TODO: currently unimplemneted + var: _(REST.schema.schema.annotations.var) + .filter(v => v.name !== "name") + .keyBy("name") + .mapValues(() => expect.any(Object)) + .value() }), varDataCache: expect.any(Object), diff --git a/client/src/actions/index.js b/client/src/actions/index.js index 1f538a35..30725123 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -30,10 +30,21 @@ const doInitialDataLoad = () => .map(url => doJsonRequest(url)) .value(); const results = await Promise.all(requests); - const universe = Universe.createUniverseFromRestV02Response(...results); + + /* set config defaults */ + const config = { ...globals.configDefaults, ...results[0].config }; + const [, schema, obsAnno, varAnno, obsLayout] = [...results]; + const universe = Universe.createUniverseFromRestV02Response( + config, + schema, + obsAnno, + varAnno, + obsLayout + ); + dispatch({ type: "configuration load complete", - config: results[0].config + config }); dispatch({ type: "initial data load complete (universe exists)", diff --git a/client/src/components/categorical/categorical.js b/client/src/components/categorical/categorical.js index bddff2dc..595995ec 100644 --- a/client/src/components/categorical/categorical.js +++ b/client/src/components/categorical/categorical.js @@ -23,11 +23,16 @@ const truncateCategories = options => { }; @connect(state => ({ - ranges: _.get(state.controls.world, "summary.obs", null) + ranges: _.get(state.controls.world, "summary.obs", null), + categorySelectionLimit: _.get( + state.config, + "parameters.category-selection-limit", + globals.configDefaults.parameters["category-selection-limit"] + ) })) class Categories extends React.Component { render() { - const { ranges } = this.props; + const { ranges, categorySelectionLimit } = this.props; if (!ranges) return null; return ( @@ -42,7 +47,13 @@ class Categories extends React.Component {

Categorical Metadata

{_.map(ranges, (value, key) => { const isColorField = key.includes("color") || key.includes("Color"); - if (value.options && !isColorField && key !== "name") { + const isSelectableCategory = + value.options && + !isColorField && + key !== "name" && + value.numOptions < categorySelectionLimit; + + if (isSelectableCategory) { const categoryOptions = truncateCategories(value.options); return ( + _.reduce( + values, + (r, v, k) => { + r.total += 1; + if (optsAsBools[k]) { + r.on += 1; + } + return r; + }, + { total: 0, on: 0 } + ); + @connect(state => ({ colorAccessor: state.controls.colorAccessor, categoricalAsBooleansMap: state.controls.categoricalAsBooleansMap @@ -18,22 +32,24 @@ class Category extends React.Component { isChecked: true, isExpanded: false }; + this.countCategories = memoize((values, optsAsBools) => + countCategories(values, optsAsBools) + ); } componentDidUpdate() { - const { categoricalAsBooleansMap, metadataField } = this.props; - - const valuesAsBool = _.values(categoricalAsBooleansMap[metadataField]); - /* count categories toggled on by counting true values */ - const categoriesToggledOn = _.values(valuesAsBool).filter(v => v).length; - - if (categoriesToggledOn === valuesAsBool.length) { + const { categoricalAsBooleansMap, metadataField, values } = this.props; + const categoryCount = this.countCategories( + values, + categoricalAsBooleansMap[metadataField] + ); + if (categoryCount.on === categoryCount.total) { /* everything is on, so not indeterminate */ this.checkbox.indeterminate = false; - } else if (categoriesToggledOn === 0) { + } else if (categoryCount.on === 0) { /* nothing is on, so no */ this.checkbox.indeterminate = false; - } else if (categoriesToggledOn < valuesAsBool.length) { + } else if (categoryCount.on < categoryCount.total) { /* to be explicit... */ this.checkbox.indeterminate = true; } diff --git a/client/src/globals.js b/client/src/globals.js index c997a6cd..beed9b6d 100644 --- a/client/src/globals.js +++ b/client/src/globals.js @@ -28,8 +28,21 @@ export const continuous = [ "Unmapped_short" ]; +/* if a categorical metadata field has more options than this, truncate */ export const maxCategoricalOptionsToDisplay = 100; +/* +these are default values for configuration the CLI may supply. +See the REST API and CLI specs for more info. +*/ +export const configDefaults = { + features: {}, + displayNames: {}, + parameters: { + "category-selection-limit": 1000 + } +}; + /* colors */ export const blue = "#4a90e2"; export const hcaBlue = "#1c7cc7"; diff --git a/client/src/reducers/config.js b/client/src/reducers/config.js index a2dcce3b..4ddf4ec3 100644 --- a/client/src/reducers/config.js +++ b/client/src/reducers/config.js @@ -2,7 +2,8 @@ const Config = ( state = { displayNames: null, - features: null + features: null, + parameters: null }, action ) => { diff --git a/client/src/reducers/controls.js b/client/src/reducers/controls.js index 9de56d8c..1a9d8da5 100644 --- a/client/src/reducers/controls.js +++ b/client/src/reducers/controls.js @@ -16,7 +16,7 @@ import { function createCategoricalAsBooleansMap(world) { const res = {}; _.each(world.summary.obs, (value, key) => { - if (value.options) { + if (value.options && key !== "name") { const optionsAsBooleans = {}; _.each(value.options, (_value, _key) => { optionsAsBooleans[_key] = true; diff --git a/client/src/util/stateManager/summarizeAnnotations.js b/client/src/util/stateManager/summarizeAnnotations.js new file mode 100644 index 00000000..a17d85b1 --- /dev/null +++ b/client/src/util/stateManager/summarizeAnnotations.js @@ -0,0 +1,94 @@ +import _ from "lodash"; + +/* +Build and return obs/var summary using any annotation in the schema + +Summary information for each annotation, keyed by annotation name. +Value will be an object, containing summary information. + +For continuous annotations (int, float, etc): + : { + range { + min: , + max: + } + } + +For categorical annotations (boolean, string, category): + : { + options: { + : , + ... + }, + numOptions: + } + +Summarize will be returned for BOTH obs and var annotations. + +Example: + { + "Splice_sites_Annotated": { + "range": { + "min": 26, + "max": 1075869 + } + }, + "Selection": { + numOptions, 6, + "options": { + "Astrocytes(HEPACAM)": 714, + "Endothelial(BSC)": 123, + "Oligodendrocytes(GC)": 294, + "Neurons(Thy1)": 685, + "Microglia(CD45)": 1108, + "Unpanned": 665 + } + } + } + +NOTE: will not summarize the required 'name' annotation, as that is +specified as unique per element. +*/ +function summarizeDimension(schema, annotations) { + return _(schema) + .filter(v => v.name !== "name") + .keyBy("name") + .mapValues(anno => { + const { name, type } = anno; + const continuous = type === "int32" || type === "float32"; + + if (!continuous) { + const options = _.countBy(annotations, name); + const numOptions = _.size(options); + return { + numOptions, + options + }; + } + + if (continuous) { + let min = Number.POSITIVE_INFINITY; + let max = Number.NEGATIVE_INFINITY; + _.forEach(annotations, obs => { + const val = Number(obs[name]); + min = val < min ? val : min; + max = val > max ? val : max; + }); + return { range: { min, max } }; + } + + throw new Error("incomprehensible schema"); + }) + .value(); +} + +export default function summarizeAnnotations( + schema, + obsAnnotations, + varAnnotations +) { + return { + obs: summarizeDimension(schema.annotations.obs, obsAnnotations), + var: summarizeDimension(schema.annotations.var, varAnnotations) + }; +} diff --git a/client/src/util/stateManager/universe.js b/client/src/util/stateManager/universe.js index 42163b22..a0999ef6 100644 --- a/client/src/util/stateManager/universe.js +++ b/client/src/util/stateManager/universe.js @@ -2,6 +2,7 @@ import _ from "lodash"; import * as kvCache from "./keyvalcache"; +import summarizeAnnotations from "./summarizeAnnotations"; /* Private helper function - create and return a template Universe @@ -28,6 +29,7 @@ function templateUniverse() { varAnnotations: [] /* all var annotations, by var index */, obsNameToIndexMap: {} /* reverse map 'name' to index */, varNameToIndexMap: {} /* reverse map 'name' to index */, + summary: null /* derived data summaries XXX: consider exploding in place */, obsLayout: { X: [], Y: [] } /* xy layout */, @@ -191,6 +193,12 @@ export function createUniverseFromRestV02Response( /* layout */ universe.obsLayout = RESTv02LayoutResponseToInternal(layoutObsResponse); + universe.summary = summarizeAnnotations( + universe.schema, + universe.obsAnnotations, + universe.varAnnotations + ); + return finalize(universe); } diff --git a/client/src/util/stateManager/world.js b/client/src/util/stateManager/world.js index 50f7f8be..3c5157f1 100644 --- a/client/src/util/stateManager/world.js +++ b/client/src/util/stateManager/world.js @@ -2,6 +2,7 @@ import _ from "lodash"; import * as kvCache from "./keyvalcache"; +import summarizeAnnotations from "./summarizeAnnotations"; import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators"; /* @@ -51,83 +52,6 @@ obs/cell. const VarDataCacheLowWatermark = 32; // cache element count const VarDataCacheTTLMs = 1000; // min cache time in MS -function summarizeAnnotations(schema, obsAnnotations) { - /* - Build and return obs/var summary using any annotation in the schema - - Summary information for each annotation, keyed by annotation name. - Value will be an object, containing either 'range' or 'options' object, - depending on the annotation schema type (categorical or continuous). - - Summarize for BOTH obs and var annotations. Result format: - - { - obs: { - annotation_name: { ... }, - ... - }, - var: { - annotation_name: { ... }, - ... - } - } - - Example: - { - "Splice_sites_Annotated": { - "range": { - "min": 26, - "max": 1075869 - } - }, - "Selection": { - "options": { - "Astrocytes(HEPACAM)": 714, - "Endothelial(BSC)": 123, - "Oligodendrocytes(GC)": 294, - "Neurons(Thy1)": 685, - "Microglia(CD45)": 1108, - "Unpanned": 665 - } - } - } - */ - const obsSummary = _(schema.annotations.obs) - .keyBy("name") - .mapValues(anno => { - const { name, type } = anno; - const continuous = type === "int32" || type === "float32"; - - if (!continuous) { - return { - options: _.countBy(obsAnnotations, name) - }; - } - - if (continuous) { - let min = Number.POSITIVE_INFINITY; - let max = Number.NEGATIVE_INFINITY; - _.forEach(obsAnnotations, obs => { - const val = Number(obs[name]); - min = val < min ? val : min; - max = val > max ? val : max; - }); - return { range: { min, max } }; - } - - throw new Error("incomprehensible schema"); - }) - .value(); - - // TODO XXX - not currently used, so skip it - const varSummary = {}; - - return { - obs: obsSummary, - var: varSummary - }; -} - function templateWorld() { return { // map from universe obsIndex to world offset. @@ -186,7 +110,11 @@ export function createWorldFromEntireUniverse(universe) { world.obsLayout = universe.obsLayout; /* derived data & summaries */ - world.summary = summarizeAnnotations(world.schema, world.obsAnnotations); + world.summary = summarizeAnnotations( + world.schema, + world.obsAnnotations, + world.varAnnotations + ); /* build the varDataCache */ world.varDataCache = kvCache.map( @@ -242,7 +170,8 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) { /* derived data & summaries */ newWorld.summary = summarizeAnnotations( newWorld.schema, - newWorld.obsAnnotations + newWorld.obsAnnotations, + newWorld.varAnnotations ); /* build the varDataCache */