diff --git a/client/__tests__/util/stateManager/summarizeAnnotations.test.js b/client/__tests__/util/stateManager/summarizeAnnotations.test.js new file mode 100644 index 00000000..650eb6c8 --- /dev/null +++ b/client/__tests__/util/stateManager/summarizeAnnotations.test.js @@ -0,0 +1,192 @@ +import summarizeAnnotations from "../../../src/util/stateManager/summarizeAnnotations"; + +describe("summarizeAnnotations", () => { + const schema = { + annotations: { + obs: [ + { name: "name", type: "string" }, + { name: "nameString", type: "string" }, + { name: "nameBoolean", type: "boolean" }, + { name: "nameFloat32", type: "float32" }, + { name: "nameInt32", type: "int32" }, + { + name: "nameCategorical", + type: "categorical", + categories: [true, false, 1, 0, 0.00001, 4383.4833, "test", "", "0"] + } + ], + var: [{ name: "name", type: "string" }] + } + }; + + test("empty test", () => { + const summary = summarizeAnnotations(schema, [], []); + expect(summary).toEqual( + expect.objectContaining({ + obs: { + nameString: { + categorical: true, + categories: [], + categoryCounts: new Map(), + numCategories: 0 + }, + nameBoolean: { + categorical: true, + categories: [], + categoryCounts: new Map(), + numCategories: 0 + }, + nameFloat32: { + categorical: false, + range: { + max: Number.NEGATIVE_INFINITY, + min: Number.POSITIVE_INFINITY + } + }, + nameInt32: { + categorical: false, + range: { + max: Number.NEGATIVE_INFINITY, + min: Number.POSITIVE_INFINITY + } + }, + nameCategorical: { + categorical: true, + categories: [], + categoryCounts: new Map(), + numCategories: 0 + } + }, + var: {} + }) + ); + }); + + test("simple test", () => { + const obsAnnotations = [ + { + __index__: 0, + name: "n1", + nameString: "hi", + nameBoolean: true, + nameFloat32: 39.3, + nameInt32: 99, + nameCategorical: 1 + } + ]; + const varAnnotations = []; + + const summary = summarizeAnnotations( + schema, + obsAnnotations, + varAnnotations + ); + + expect(summary).toEqual( + expect.objectContaining({ + obs: { + nameString: { + categorical: true, + categories: ["hi"], + categoryCounts: new Map([["hi", 1]]), + numCategories: 1 + }, + nameBoolean: { + categorical: true, + categories: [true], + categoryCounts: new Map([[true, 1]]), + numCategories: 1 + }, + nameFloat32: { + categorical: false, + range: { min: 39.3, max: 39.3 } + }, + nameInt32: { + categorical: false, + range: { min: 99, max: 99 } + }, + nameCategorical: { + categorical: true, + categories: [1], + categoryCounts: new Map([[1, 1]]), + numCategories: 1 + } + }, + var: {} + }) + ); + }); + + test("multi test", () => { + const obsAnnotations = [ + { + __index__: 0, + name: "n0", + nameString: "hi", + nameBoolean: false, + nameFloat32: 39.3, + nameInt32: 99, + nameCategorical: 1 + }, + { + __index__: 1, + name: "n1", + nameString: "hi", + nameBoolean: true, + nameFloat32: 39.3, + nameInt32: 99, + nameCategorical: false + }, + { + __index__: 2, + name: "n2", + nameString: "bye", + nameBoolean: true, + nameFloat32: 0, + nameInt32: 99, + nameCategorical: "0" + } + ]; + const varAnnotations = []; + + const summary = summarizeAnnotations( + schema, + obsAnnotations, + varAnnotations + ); + + expect(summary).toMatchObject( + expect.objectContaining({ + obs: { + nameString: { + categorical: true, + categories: expect.arrayContaining(["hi", "bye"]), + categoryCounts: new Map([["hi", 2], ["bye", 1]]), + numCategories: 2 + }, + nameBoolean: { + categorical: true, + categories: expect.arrayContaining([true, false]), + categoryCounts: new Map([[true, 2], [false, 1]]), + numCategories: 2 + }, + nameFloat32: { + categorical: false, + range: { min: 0, max: 39.3 } + }, + nameInt32: { + categorical: false, + range: { min: 99, max: 99 } + }, + nameCategorical: { + categorical: true, + categories: expect.arrayContaining([1, false, "0"]), + categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]), + numCategories: 3 + } + }, + var: {} + }) + ); + }); +}); diff --git a/client/__tests__/util/stateManager/worldUtil.test.js b/client/__tests__/util/stateManager/worldUtil.test.js new file mode 100644 index 00000000..b20a7b0a --- /dev/null +++ b/client/__tests__/util/stateManager/worldUtil.test.js @@ -0,0 +1,45 @@ +import { + countCategoryValues2D, + clearCaches +} from "../../../src/util/stateManager/worldUtil"; + +describe("WorldUtil cache management", () => { + test("empty", () => { + const count = countCategoryValues2D("a", "b", []); + expect(count).toMatchObject(new Map()); + }); + + test("simple couts", () => { + const rows = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }]; + const count = countCategoryValues2D("a", "b", rows); + expect(count).toMatchObject( + new Map([ + [0, new Map([[true, 1], [false, 1]])], + [1, new Map([[false, 1]])] + ]) + ); + }); + + test("memo cache clear", () => { + clearCaches(); + const row1 = []; + const row2 = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }]; + const count1 = countCategoryValues2D("a", "b", row1); + const count2 = countCategoryValues2D("a", "b", row1); + const count3 = countCategoryValues2D("a", "b", []); + const count4 = countCategoryValues2D("a", "b", row2); + + clearCaches(); + const count10 = countCategoryValues2D("a", "b", row1); + const count11 = countCategoryValues2D("a", "b", row2); + + expect(count1).toEqual(count2); + expect(count1).toEqual(count3); + expect(count1).toEqual(count10); + expect(count1).not.toBe(count3); + expect(count1).not.toBe(count10); + + expect(count4).toEqual(count11); + expect(count4).not.toBe(count11); + }); +}); diff --git a/client/src/components/categorical/category.js b/client/src/components/categorical/category.js index 3c1cebb4..5d56f350 100644 --- a/client/src/components/categorical/category.js +++ b/client/src/components/categorical/category.js @@ -25,24 +25,22 @@ class Category extends React.Component { const { categoricalSelectionState, metadataField } = this.props; const cat = categoricalSelectionState[metadataField]; const categoryCount = { - // total number of options in this category - totalOptionCount: cat.numOptions, + // total number of categories in this dimension + totalCatCount: cat.numCategories, // number of selected options in this category - selectedOptionCount: _.reduce( - cat.optionSelected, + selectedCatCount: _.reduce( + cat.categorySelected, (res, cond) => (cond ? res + 1 : res), 0 ) }; - if (categoryCount.selectedOptionCount === categoryCount.totalOptionCount) { + if (categoryCount.selectedCatCount === categoryCount.totalCatCount) { /* everything is on, so not indeterminate */ this.checkbox.indeterminate = false; - } else if (categoryCount.selectedOptionCount === 0) { + } else if (categoryCount.selectedCatCount === 0) { /* nothing is on, so no */ this.checkbox.indeterminate = false; - } else if ( - categoryCount.selectedOptionCount < categoryCount.totalOptionCount - ) { + } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) { /* to be explicit... */ this.checkbox.indeterminate = true; } @@ -88,12 +86,12 @@ class Category extends React.Component { const { categoricalSelectionState, metadataField } = this.props; const cat = categoricalSelectionState[metadataField]; - const optTuples = alphabeticallySortedValues([...cat.optionIndex]); + const optTuples = alphabeticallySortedValues([...cat.categoryIndices]); return _.map(optTuples, (tuple, i) => ( )); diff --git a/client/src/components/categorical/value.js b/client/src/components/categorical/value.js index f2a1afb8..444b1b3c 100644 --- a/client/src/components/categorical/value.js +++ b/client/src/components/categorical/value.js @@ -11,20 +11,20 @@ import _ from "lodash"; })) class CategoryValue extends React.Component { toggleOff() { - const { dispatch, metadataField, optionIndex } = this.props; + const { dispatch, metadataField, categoryIndex } = this.props; dispatch({ type: "categorical metadata filter deselect", metadataField, - optionIndex + categoryIndex }); } toggleOn() { - const { dispatch, metadataField, optionIndex } = this.props; + const { dispatch, metadataField, categoryIndex } = this.props; dispatch({ type: "categorical metadata filter select", metadataField, - optionIndex + categoryIndex }); } @@ -32,7 +32,7 @@ class CategoryValue extends React.Component { const { categoricalSelectionState, metadataField, - optionIndex, + categoryIndex, colorAccessor, colorScale, i, @@ -42,10 +42,12 @@ class CategoryValue extends React.Component { if (!categoricalSelectionState) return null; const category = categoricalSelectionState[metadataField]; - const selected = category.optionSelected[optionIndex]; - const count = category.optionCount[optionIndex]; - const value = category.optionValue[optionIndex]; - const displayString = String(category.optionValue[optionIndex]).valueOf(); + const selected = category.categorySelected[categoryIndex]; + const count = category.categoryCounts[categoryIndex]; + const value = category.categoryValues[categoryIndex]; + const displayString = String( + category.categoryValues[categoryIndex] + ).valueOf(); /* this is the color scale, so add swatches below */ const c = metadataField === colorAccessor; diff --git a/client/src/reducers/controls.js b/client/src/reducers/controls.js index 81eac671..b0c84bf2 100644 --- a/client/src/reducers/controls.js +++ b/client/src/reducers/controls.js @@ -1,7 +1,7 @@ // jshint esversion: 6 import _ from "lodash"; -import { World, kvCache } from "../util/stateManager"; +import { World, kvCache, WorldUtil } from "../util/stateManager"; import parseRGB from "../util/parseRGB"; import Crossfilter from "../util/typedCrossfilter"; import * as globals from "../globals"; @@ -24,25 +24,27 @@ Remember that option values can be ANY js type, except undefined/null. { _category_name_1: { // map of option value to index - optionIndex: Map([ - optval1: index, + categoryIndices: Map([ + catval1: index, ... ]) // index->selection true/false state - optionSelected: [ true/false, true/false, ... ] + categorySelected: [ true/false, true/false, ... ] // number of options - numOptions: number, + numCategories: number, // isTruncated - true if the options for selection has // been truncated (ie, was too large to implement) } } */ -function topNoptions(summary) { - const counts = _.map(summary.categories, cat => summary.options[cat]); - const sortIndex = fillRange(new Array(summary.numOptions)).sort( +function topNCategories(summary) { + const counts = _.map(summary.categories, cat => + summary.categoryCounts.get(cat) + ); + const sortIndex = fillRange(new Array(summary.numCategories)).sort( (a, b) => counts[b] - counts[a] ); const sortedCategories = _.map(sortIndex, i => summary.categories[i]); @@ -65,20 +67,18 @@ function createCategoricalSelectionState(state, world) { key !== "name" && value.categories.length < state.maxCategoryItems; if (isSelectableCategory) { - const [optionValue, optionCount] = topNoptions(value); - // const optionCount = Object.values(value.options); - - const optionIndex = new Map(optionValue.map((v, i) => [v, i])); - const numOptions = optionIndex.size; - const optionSelected = new Array(numOptions).fill(true); - const isTruncated = optionValue.length < value.numOptions; + const [categoryValues, categoryCounts] = topNCategories(value); + const categoryIndices = new Map(categoryValues.map((v, i) => [v, i])); + const numCategories = categoryIndices.size; + const categorySelected = new Array(numCategories).fill(true); + const isTruncated = categoryValues.length < value.numCategories; res[key] = { - optionValue, // array: of natively typed option values - optionIndex, // map: option value (native type) -> option index - optionSelected, // array: t/f selection state - numOptions, // number: of options + categoryValues, // array: of natively typed category values + categoryIndices, // map: category value (native type) -> category index + categorySelected, // array: t/f selection state + numCategories, // number: of categories isTruncated, // bool: true if list was truncated - optionCount // array: cardinality of each option + categoryCounts // array: cardinality of each category }; } } @@ -87,12 +87,12 @@ function createCategoricalSelectionState(state, world) { } /* -given a categoricalSelectionState, return the list of all option values +given a categoricalSelectionState, return the list of all category values where selection state is true (ie, they are selected). */ function selectedValuesForCategory(categorySelectionState) { - const selectedValues = _([...categorySelectionState.optionIndex]) - .filter(tuple => categorySelectionState.optionSelected[tuple[1]]) + const selectedValues = _([...categorySelectionState.categoryIndices]) + .filter(tuple => categorySelectionState.categorySelected[tuple[1]]) .map(tuple => tuple[0]) .value(); return selectedValues; @@ -175,6 +175,7 @@ const Controls = ( ); const crossfilter = Crossfilter(world.obsAnnotations); const dimensionMap = World.createObsDimensionMap(crossfilter, world); + WorldUtil.clearCaches(); const worldVarDataCache = world.varDataCache; @@ -247,6 +248,7 @@ const Controls = ( ); const crossfilter = Crossfilter(world.obsAnnotations); const dimensionMap = World.createObsDimensionMap(crossfilter, world); + WorldUtil.clearCaches(); const worldVarDataCache = world.varDataCache; /* var dimensions */ @@ -514,15 +516,15 @@ const Controls = ( Categorical metadata *******************************/ case "categorical metadata filter select": { - const newOptionSelected = Array.from( - state.categoricalSelectionState[action.metadataField].optionSelected + const newCategorySelected = Array.from( + state.categoricalSelectionState[action.metadataField].categorySelected ); - newOptionSelected[action.optionIndex] = true; + newCategorySelected[action.categoryIndex] = true; const newCategoricalSelectionState = { ...state.categoricalSelectionState, [action.metadataField]: { ...state.categoricalSelectionState[action.metadataField], - optionSelected: newOptionSelected + categorySelected: newCategorySelected } }; @@ -538,15 +540,15 @@ const Controls = ( }; } case "categorical metadata filter deselect": { - const newOptionSelected = Array.from( - state.categoricalSelectionState[action.metadataField].optionSelected + const newCategorySelected = Array.from( + state.categoricalSelectionState[action.metadataField].categorySelected ); - newOptionSelected[action.optionIndex] = false; + newCategorySelected[action.categoryIndex] = false; const newCategoricalSelectionState = { ...state.categoricalSelectionState, [action.metadataField]: { ...state.categoricalSelectionState[action.metadataField], - optionSelected: newOptionSelected + categorySelected: newCategorySelected } }; @@ -566,8 +568,9 @@ const Controls = ( ...state.categoricalSelectionState, [action.metadataField]: { ...state.categoricalSelectionState[action.metadataField], - optionSelected: Array.from( - state.categoricalSelectionState[action.metadataField].optionSelected + categorySelected: Array.from( + state.categoricalSelectionState[action.metadataField] + .categorySelected ).fill(false) } }; @@ -584,8 +587,9 @@ const Controls = ( ...state.categoricalSelectionState, [action.metadataField]: { ...state.categoricalSelectionState[action.metadataField], - optionSelected: Array.from( - state.categoricalSelectionState[action.metadataField].optionSelected + categorySelected: Array.from( + state.categoricalSelectionState[action.metadataField] + .categorySelected ).fill(true) } }; diff --git a/client/src/util/stateManager/index.js b/client/src/util/stateManager/index.js index a9e28593..47a6de5f 100644 --- a/client/src/util/stateManager/index.js +++ b/client/src/util/stateManager/index.js @@ -17,3 +17,4 @@ exists to support those concepts. export * as Universe from "./universe"; export * as World from "./world"; export * as kvCache from "./keyvalcache"; +export * as WorldUtil from "./worldUtil"; diff --git a/client/src/util/stateManager/summarizeAnnotations.js b/client/src/util/stateManager/summarizeAnnotations.js index 734b0fe3..016b7dde 100644 --- a/client/src/util/stateManager/summarizeAnnotations.js +++ b/client/src/util/stateManager/summarizeAnnotations.js @@ -8,6 +8,7 @@ Value will be an object, containing summary information. For continuous annotations (int, float, etc): : { + categorical: false, range { min: , max: @@ -15,12 +16,14 @@ For continuous annotations (int, float, etc): } For categorical annotations (boolean, string, category): - : { - options: { - : , + : { + categorical: true, + categories: [ , , ... ] + categoryCounts: Map { + : , ... }, - numOptions: + numCategories: } Summarize will be returned for BOTH obs and var annotations. @@ -28,19 +31,19 @@ Summarize will be returned for BOTH obs and var annotations. Example: { "Splice_sites_Annotated": { - "range": { + categorical: false, + range: { "min": 26, "max": 1075869 } }, "Selection": { - numOptions, 6, - "options": { + categorical: true, + numCategories, 3, + categories: [ "Astrocytes(HEPACAM)", "Endothelial(BSC)", "Unpanned" ], + categoryCounts: Map { "Astrocytes(HEPACAM)": 714, "Endothelial(BSC)": 123, - "Oligodendrocytes(GC)": 294, - "Neurons(Thy1)": 685, - "Microglia(CD45)": 1108, "Unpanned": 665 } } @@ -48,46 +51,46 @@ Example: NOTE: will not summarize the required 'name' annotation, as that is specified as unique per element. - -TODO: XXX - this data structure coerces all metadata categories into a string -(ie, stores values as an Object property in the `options` field). This looses -information (eg, type) for category types which are not strings. Consider an -alterative data structure that does not use the object property for non-string -data types (and does not use _.countBy to summarize). */ -function summarizeDimension(schema, annotations) { - return _(schema) +function _summarizeAnnotations(_schema, annotations) { + const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash .filter(v => v.name !== "name") .keyBy("name") .mapValues(anno => { const { name, type } = anno; const continuous = type === "int32" || type === "float32"; - if (!continuous) { - const categories = _.uniq(_.flatMap(annotations, name)); - const options = _.countBy(annotations, name); - const numOptions = _.size(options); - return { - numOptions, - options, - categories - }; - } - if (continuous) { let min = Number.POSITIVE_INFINITY; let max = Number.NEGATIVE_INFINITY; - _.forEach(annotations, obs => { - const val = Number(obs[name]); + for (let r = 0; r < annotations.length; r += 1) { + const val = Number(annotations[r][name]); min = val < min ? val : min; max = val > max ? val : max; - }); - return { range: { min, max } }; + } + return { + categorical: false, + range: { min, max } + }; } - throw new Error("incomprehensible schema"); + /* else categorical */ + const categoryCounts = new Map(); + for (let r = 0; r < annotations.length; r += 1) { + const val = annotations[r][name]; + let curCount = categoryCounts.get(val); + if (curCount === undefined) curCount = 0; + categoryCounts.set(val, curCount + 1); + } + return { + categorical: true, + categories: [...categoryCounts.keys()], + categoryCounts, + numCategories: categoryCounts.size + }; }) .value(); + return summary; } export default function summarizeAnnotations( @@ -96,7 +99,7 @@ export default function summarizeAnnotations( varAnnotations ) { return { - obs: summarizeDimension(schema.annotations.obs, obsAnnotations), - var: summarizeDimension(schema.annotations.var, varAnnotations) + obs: _summarizeAnnotations(schema.annotations.obs, obsAnnotations), + var: _summarizeAnnotations(schema.annotations.var, varAnnotations) }; } diff --git a/client/src/util/stateManager/worldUtil.js b/client/src/util/stateManager/worldUtil.js new file mode 100644 index 00000000..0fe994cf --- /dev/null +++ b/client/src/util/stateManager/worldUtil.js @@ -0,0 +1,65 @@ +/* eslint-disable import/prefer-default-export */ +import _ from "lodash"; + +/* +Various utility functions operating on World/Universe +*/ + +/* +Count unique category values, binning first by dim1 then by dim2 +Return: + +Map { + dim1_val1: Map { + dim2_val1: number, + dim2_val2: number, + ... + }, + ... +} + +*/ +function _countCategoryValues2D(dim1, dim2, rows) { + const dimMap = new Map(); + for (let r = 0; r < rows.length; r += 1) { + const row = rows[r]; + const val1 = row[dim1]; + const val2 = row[dim2]; + let d2Map = dimMap.get(val1); + if (d2Map === undefined) { + d2Map = new Map(); + dimMap.set(val1, d2Map); + } + let curCount = d2Map.get(val2); + if (curCount === undefined) { + curCount = 0; + } + d2Map.set(val2, curCount + 1); + } + return dimMap; +} + +let __worldUtilMemoId__ = 0; +function _memoizedId(x) { + if (!x.__worldUtilMemoId__) { + __worldUtilMemoId__ += 1; + x.__worldUtilMemoId__ = __worldUtilMemoId__; + } + return x.__worldUtilMemoId__; +} +function _countCategoryValues2DResolver(...args) { + const id = args[0] + args[1] + _memoizedId(args[2]); + return id; +} + +export const countCategoryValues2D = _.memoize( + _countCategoryValues2D, + _countCategoryValues2DResolver +); + +/* +Clear any cached data within WorldUtil caches, eg, memoized functions +*/ +export function clearCaches() { + countCategoryValues2D.cache.clear(); +}