diff --git a/client/__tests__/util/stateManager/summarizeAnnotations.test.js b/client/__tests__/util/stateManager/summarizeAnnotations.test.js
new file mode 100644
index 00000000..650eb6c8
--- /dev/null
+++ b/client/__tests__/util/stateManager/summarizeAnnotations.test.js
@@ -0,0 +1,192 @@
+import summarizeAnnotations from "../../../src/util/stateManager/summarizeAnnotations";
+
+describe("summarizeAnnotations", () => {
+ const schema = {
+ annotations: {
+ obs: [
+ { name: "name", type: "string" },
+ { name: "nameString", type: "string" },
+ { name: "nameBoolean", type: "boolean" },
+ { name: "nameFloat32", type: "float32" },
+ { name: "nameInt32", type: "int32" },
+ {
+ name: "nameCategorical",
+ type: "categorical",
+ categories: [true, false, 1, 0, 0.00001, 4383.4833, "test", "", "0"]
+ }
+ ],
+ var: [{ name: "name", type: "string" }]
+ }
+ };
+
+ test("empty test", () => {
+ const summary = summarizeAnnotations(schema, [], []);
+ expect(summary).toEqual(
+ expect.objectContaining({
+ obs: {
+ nameString: {
+ categorical: true,
+ categories: [],
+ categoryCounts: new Map(),
+ numCategories: 0
+ },
+ nameBoolean: {
+ categorical: true,
+ categories: [],
+ categoryCounts: new Map(),
+ numCategories: 0
+ },
+ nameFloat32: {
+ categorical: false,
+ range: {
+ max: Number.NEGATIVE_INFINITY,
+ min: Number.POSITIVE_INFINITY
+ }
+ },
+ nameInt32: {
+ categorical: false,
+ range: {
+ max: Number.NEGATIVE_INFINITY,
+ min: Number.POSITIVE_INFINITY
+ }
+ },
+ nameCategorical: {
+ categorical: true,
+ categories: [],
+ categoryCounts: new Map(),
+ numCategories: 0
+ }
+ },
+ var: {}
+ })
+ );
+ });
+
+ test("simple test", () => {
+ const obsAnnotations = [
+ {
+ __index__: 0,
+ name: "n1",
+ nameString: "hi",
+ nameBoolean: true,
+ nameFloat32: 39.3,
+ nameInt32: 99,
+ nameCategorical: 1
+ }
+ ];
+ const varAnnotations = [];
+
+ const summary = summarizeAnnotations(
+ schema,
+ obsAnnotations,
+ varAnnotations
+ );
+
+ expect(summary).toEqual(
+ expect.objectContaining({
+ obs: {
+ nameString: {
+ categorical: true,
+ categories: ["hi"],
+ categoryCounts: new Map([["hi", 1]]),
+ numCategories: 1
+ },
+ nameBoolean: {
+ categorical: true,
+ categories: [true],
+ categoryCounts: new Map([[true, 1]]),
+ numCategories: 1
+ },
+ nameFloat32: {
+ categorical: false,
+ range: { min: 39.3, max: 39.3 }
+ },
+ nameInt32: {
+ categorical: false,
+ range: { min: 99, max: 99 }
+ },
+ nameCategorical: {
+ categorical: true,
+ categories: [1],
+ categoryCounts: new Map([[1, 1]]),
+ numCategories: 1
+ }
+ },
+ var: {}
+ })
+ );
+ });
+
+ test("multi test", () => {
+ const obsAnnotations = [
+ {
+ __index__: 0,
+ name: "n0",
+ nameString: "hi",
+ nameBoolean: false,
+ nameFloat32: 39.3,
+ nameInt32: 99,
+ nameCategorical: 1
+ },
+ {
+ __index__: 1,
+ name: "n1",
+ nameString: "hi",
+ nameBoolean: true,
+ nameFloat32: 39.3,
+ nameInt32: 99,
+ nameCategorical: false
+ },
+ {
+ __index__: 2,
+ name: "n2",
+ nameString: "bye",
+ nameBoolean: true,
+ nameFloat32: 0,
+ nameInt32: 99,
+ nameCategorical: "0"
+ }
+ ];
+ const varAnnotations = [];
+
+ const summary = summarizeAnnotations(
+ schema,
+ obsAnnotations,
+ varAnnotations
+ );
+
+ expect(summary).toMatchObject(
+ expect.objectContaining({
+ obs: {
+ nameString: {
+ categorical: true,
+ categories: expect.arrayContaining(["hi", "bye"]),
+ categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+ numCategories: 2
+ },
+ nameBoolean: {
+ categorical: true,
+ categories: expect.arrayContaining([true, false]),
+ categoryCounts: new Map([[true, 2], [false, 1]]),
+ numCategories: 2
+ },
+ nameFloat32: {
+ categorical: false,
+ range: { min: 0, max: 39.3 }
+ },
+ nameInt32: {
+ categorical: false,
+ range: { min: 99, max: 99 }
+ },
+ nameCategorical: {
+ categorical: true,
+ categories: expect.arrayContaining([1, false, "0"]),
+ categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+ numCategories: 3
+ }
+ },
+ var: {}
+ })
+ );
+ });
+});
diff --git a/client/__tests__/util/stateManager/worldUtil.test.js b/client/__tests__/util/stateManager/worldUtil.test.js
new file mode 100644
index 00000000..b20a7b0a
--- /dev/null
+++ b/client/__tests__/util/stateManager/worldUtil.test.js
@@ -0,0 +1,45 @@
+import {
+ countCategoryValues2D,
+ clearCaches
+} from "../../../src/util/stateManager/worldUtil";
+
+describe("WorldUtil cache management", () => {
+ test("empty", () => {
+ const count = countCategoryValues2D("a", "b", []);
+ expect(count).toMatchObject(new Map());
+ });
+
+ test("simple couts", () => {
+ const rows = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
+ const count = countCategoryValues2D("a", "b", rows);
+ expect(count).toMatchObject(
+ new Map([
+ [0, new Map([[true, 1], [false, 1]])],
+ [1, new Map([[false, 1]])]
+ ])
+ );
+ });
+
+ test("memo cache clear", () => {
+ clearCaches();
+ const row1 = [];
+ const row2 = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
+ const count1 = countCategoryValues2D("a", "b", row1);
+ const count2 = countCategoryValues2D("a", "b", row1);
+ const count3 = countCategoryValues2D("a", "b", []);
+ const count4 = countCategoryValues2D("a", "b", row2);
+
+ clearCaches();
+ const count10 = countCategoryValues2D("a", "b", row1);
+ const count11 = countCategoryValues2D("a", "b", row2);
+
+ expect(count1).toEqual(count2);
+ expect(count1).toEqual(count3);
+ expect(count1).toEqual(count10);
+ expect(count1).not.toBe(count3);
+ expect(count1).not.toBe(count10);
+
+ expect(count4).toEqual(count11);
+ expect(count4).not.toBe(count11);
+ });
+});
diff --git a/client/src/components/categorical/category.js b/client/src/components/categorical/category.js
index 3c1cebb4..5d56f350 100644
--- a/client/src/components/categorical/category.js
+++ b/client/src/components/categorical/category.js
@@ -25,24 +25,22 @@ class Category extends React.Component {
const { categoricalSelectionState, metadataField } = this.props;
const cat = categoricalSelectionState[metadataField];
const categoryCount = {
- // total number of options in this category
- totalOptionCount: cat.numOptions,
+ // total number of categories in this dimension
+ totalCatCount: cat.numCategories,
// number of selected options in this category
- selectedOptionCount: _.reduce(
- cat.optionSelected,
+ selectedCatCount: _.reduce(
+ cat.categorySelected,
(res, cond) => (cond ? res + 1 : res),
0
)
};
- if (categoryCount.selectedOptionCount === categoryCount.totalOptionCount) {
+ if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
/* everything is on, so not indeterminate */
this.checkbox.indeterminate = false;
- } else if (categoryCount.selectedOptionCount === 0) {
+ } else if (categoryCount.selectedCatCount === 0) {
/* nothing is on, so no */
this.checkbox.indeterminate = false;
- } else if (
- categoryCount.selectedOptionCount < categoryCount.totalOptionCount
- ) {
+ } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
/* to be explicit... */
this.checkbox.indeterminate = true;
}
@@ -88,12 +86,12 @@ class Category extends React.Component {
const { categoricalSelectionState, metadataField } = this.props;
const cat = categoricalSelectionState[metadataField];
- const optTuples = alphabeticallySortedValues([...cat.optionIndex]);
+ const optTuples = alphabeticallySortedValues([...cat.categoryIndices]);
return _.map(optTuples, (tuple, i) => (
));
diff --git a/client/src/components/categorical/value.js b/client/src/components/categorical/value.js
index f2a1afb8..444b1b3c 100644
--- a/client/src/components/categorical/value.js
+++ b/client/src/components/categorical/value.js
@@ -11,20 +11,20 @@ import _ from "lodash";
}))
class CategoryValue extends React.Component {
toggleOff() {
- const { dispatch, metadataField, optionIndex } = this.props;
+ const { dispatch, metadataField, categoryIndex } = this.props;
dispatch({
type: "categorical metadata filter deselect",
metadataField,
- optionIndex
+ categoryIndex
});
}
toggleOn() {
- const { dispatch, metadataField, optionIndex } = this.props;
+ const { dispatch, metadataField, categoryIndex } = this.props;
dispatch({
type: "categorical metadata filter select",
metadataField,
- optionIndex
+ categoryIndex
});
}
@@ -32,7 +32,7 @@ class CategoryValue extends React.Component {
const {
categoricalSelectionState,
metadataField,
- optionIndex,
+ categoryIndex,
colorAccessor,
colorScale,
i,
@@ -42,10 +42,12 @@ class CategoryValue extends React.Component {
if (!categoricalSelectionState) return null;
const category = categoricalSelectionState[metadataField];
- const selected = category.optionSelected[optionIndex];
- const count = category.optionCount[optionIndex];
- const value = category.optionValue[optionIndex];
- const displayString = String(category.optionValue[optionIndex]).valueOf();
+ const selected = category.categorySelected[categoryIndex];
+ const count = category.categoryCounts[categoryIndex];
+ const value = category.categoryValues[categoryIndex];
+ const displayString = String(
+ category.categoryValues[categoryIndex]
+ ).valueOf();
/* this is the color scale, so add swatches below */
const c = metadataField === colorAccessor;
diff --git a/client/src/reducers/controls.js b/client/src/reducers/controls.js
index 81eac671..b0c84bf2 100644
--- a/client/src/reducers/controls.js
+++ b/client/src/reducers/controls.js
@@ -1,7 +1,7 @@
// jshint esversion: 6
import _ from "lodash";
-import { World, kvCache } from "../util/stateManager";
+import { World, kvCache, WorldUtil } from "../util/stateManager";
import parseRGB from "../util/parseRGB";
import Crossfilter from "../util/typedCrossfilter";
import * as globals from "../globals";
@@ -24,25 +24,27 @@ Remember that option values can be ANY js type, except undefined/null.
{
_category_name_1: {
// map of option value to index
- optionIndex: Map([
- optval1: index,
+ categoryIndices: Map([
+ catval1: index,
...
])
// index->selection true/false state
- optionSelected: [ true/false, true/false, ... ]
+ categorySelected: [ true/false, true/false, ... ]
// number of options
- numOptions: number,
+ numCategories: number,
// isTruncated - true if the options for selection has
// been truncated (ie, was too large to implement)
}
}
*/
-function topNoptions(summary) {
- const counts = _.map(summary.categories, cat => summary.options[cat]);
- const sortIndex = fillRange(new Array(summary.numOptions)).sort(
+function topNCategories(summary) {
+ const counts = _.map(summary.categories, cat =>
+ summary.categoryCounts.get(cat)
+ );
+ const sortIndex = fillRange(new Array(summary.numCategories)).sort(
(a, b) => counts[b] - counts[a]
);
const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
@@ -65,20 +67,18 @@ function createCategoricalSelectionState(state, world) {
key !== "name" &&
value.categories.length < state.maxCategoryItems;
if (isSelectableCategory) {
- const [optionValue, optionCount] = topNoptions(value);
- // const optionCount = Object.values(value.options);
-
- const optionIndex = new Map(optionValue.map((v, i) => [v, i]));
- const numOptions = optionIndex.size;
- const optionSelected = new Array(numOptions).fill(true);
- const isTruncated = optionValue.length < value.numOptions;
+ const [categoryValues, categoryCounts] = topNCategories(value);
+ const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
+ const numCategories = categoryIndices.size;
+ const categorySelected = new Array(numCategories).fill(true);
+ const isTruncated = categoryValues.length < value.numCategories;
res[key] = {
- optionValue, // array: of natively typed option values
- optionIndex, // map: option value (native type) -> option index
- optionSelected, // array: t/f selection state
- numOptions, // number: of options
+ categoryValues, // array: of natively typed category values
+ categoryIndices, // map: category value (native type) -> category index
+ categorySelected, // array: t/f selection state
+ numCategories, // number: of categories
isTruncated, // bool: true if list was truncated
- optionCount // array: cardinality of each option
+ categoryCounts // array: cardinality of each category
};
}
}
@@ -87,12 +87,12 @@ function createCategoricalSelectionState(state, world) {
}
/*
-given a categoricalSelectionState, return the list of all option values
+given a categoricalSelectionState, return the list of all category values
where selection state is true (ie, they are selected).
*/
function selectedValuesForCategory(categorySelectionState) {
- const selectedValues = _([...categorySelectionState.optionIndex])
- .filter(tuple => categorySelectionState.optionSelected[tuple[1]])
+ const selectedValues = _([...categorySelectionState.categoryIndices])
+ .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
.map(tuple => tuple[0])
.value();
return selectedValues;
@@ -175,6 +175,7 @@ const Controls = (
);
const crossfilter = Crossfilter(world.obsAnnotations);
const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+ WorldUtil.clearCaches();
const worldVarDataCache = world.varDataCache;
@@ -247,6 +248,7 @@ const Controls = (
);
const crossfilter = Crossfilter(world.obsAnnotations);
const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+ WorldUtil.clearCaches();
const worldVarDataCache = world.varDataCache;
/* var dimensions */
@@ -514,15 +516,15 @@ const Controls = (
Categorical metadata
*******************************/
case "categorical metadata filter select": {
- const newOptionSelected = Array.from(
- state.categoricalSelectionState[action.metadataField].optionSelected
+ const newCategorySelected = Array.from(
+ state.categoricalSelectionState[action.metadataField].categorySelected
);
- newOptionSelected[action.optionIndex] = true;
+ newCategorySelected[action.categoryIndex] = true;
const newCategoricalSelectionState = {
...state.categoricalSelectionState,
[action.metadataField]: {
...state.categoricalSelectionState[action.metadataField],
- optionSelected: newOptionSelected
+ categorySelected: newCategorySelected
}
};
@@ -538,15 +540,15 @@ const Controls = (
};
}
case "categorical metadata filter deselect": {
- const newOptionSelected = Array.from(
- state.categoricalSelectionState[action.metadataField].optionSelected
+ const newCategorySelected = Array.from(
+ state.categoricalSelectionState[action.metadataField].categorySelected
);
- newOptionSelected[action.optionIndex] = false;
+ newCategorySelected[action.categoryIndex] = false;
const newCategoricalSelectionState = {
...state.categoricalSelectionState,
[action.metadataField]: {
...state.categoricalSelectionState[action.metadataField],
- optionSelected: newOptionSelected
+ categorySelected: newCategorySelected
}
};
@@ -566,8 +568,9 @@ const Controls = (
...state.categoricalSelectionState,
[action.metadataField]: {
...state.categoricalSelectionState[action.metadataField],
- optionSelected: Array.from(
- state.categoricalSelectionState[action.metadataField].optionSelected
+ categorySelected: Array.from(
+ state.categoricalSelectionState[action.metadataField]
+ .categorySelected
).fill(false)
}
};
@@ -584,8 +587,9 @@ const Controls = (
...state.categoricalSelectionState,
[action.metadataField]: {
...state.categoricalSelectionState[action.metadataField],
- optionSelected: Array.from(
- state.categoricalSelectionState[action.metadataField].optionSelected
+ categorySelected: Array.from(
+ state.categoricalSelectionState[action.metadataField]
+ .categorySelected
).fill(true)
}
};
diff --git a/client/src/util/stateManager/index.js b/client/src/util/stateManager/index.js
index a9e28593..47a6de5f 100644
--- a/client/src/util/stateManager/index.js
+++ b/client/src/util/stateManager/index.js
@@ -17,3 +17,4 @@ exists to support those concepts.
export * as Universe from "./universe";
export * as World from "./world";
export * as kvCache from "./keyvalcache";
+export * as WorldUtil from "./worldUtil";
diff --git a/client/src/util/stateManager/summarizeAnnotations.js b/client/src/util/stateManager/summarizeAnnotations.js
index 734b0fe3..016b7dde 100644
--- a/client/src/util/stateManager/summarizeAnnotations.js
+++ b/client/src/util/stateManager/summarizeAnnotations.js
@@ -8,6 +8,7 @@ Value will be an object, containing summary information.
For continuous annotations (int, float, etc):
: {
+ categorical: false,
range {
min: ,
max:
@@ -15,12 +16,14 @@ For continuous annotations (int, float, etc):
}
For categorical annotations (boolean, string, category):
- : {
- options: {
- : ,
+ : {
+ categorical: true,
+ categories: [ , , ... ]
+ categoryCounts: Map {
+ : ,
...
},
- numOptions:
+ numCategories:
}
Summarize will be returned for BOTH obs and var annotations.
@@ -28,19 +31,19 @@ Summarize will be returned for BOTH obs and var annotations.
Example:
{
"Splice_sites_Annotated": {
- "range": {
+ categorical: false,
+ range: {
"min": 26,
"max": 1075869
}
},
"Selection": {
- numOptions, 6,
- "options": {
+ categorical: true,
+ numCategories, 3,
+ categories: [ "Astrocytes(HEPACAM)", "Endothelial(BSC)", "Unpanned" ],
+ categoryCounts: Map {
"Astrocytes(HEPACAM)": 714,
"Endothelial(BSC)": 123,
- "Oligodendrocytes(GC)": 294,
- "Neurons(Thy1)": 685,
- "Microglia(CD45)": 1108,
"Unpanned": 665
}
}
@@ -48,46 +51,46 @@ Example:
NOTE: will not summarize the required 'name' annotation, as that is
specified as unique per element.
-
-TODO: XXX - this data structure coerces all metadata categories into a string
-(ie, stores values as an Object property in the `options` field). This looses
-information (eg, type) for category types which are not strings. Consider an
-alterative data structure that does not use the object property for non-string
-data types (and does not use _.countBy to summarize).
*/
-function summarizeDimension(schema, annotations) {
- return _(schema)
+function _summarizeAnnotations(_schema, annotations) {
+ const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
.filter(v => v.name !== "name")
.keyBy("name")
.mapValues(anno => {
const { name, type } = anno;
const continuous = type === "int32" || type === "float32";
- if (!continuous) {
- const categories = _.uniq(_.flatMap(annotations, name));
- const options = _.countBy(annotations, name);
- const numOptions = _.size(options);
- return {
- numOptions,
- options,
- categories
- };
- }
-
if (continuous) {
let min = Number.POSITIVE_INFINITY;
let max = Number.NEGATIVE_INFINITY;
- _.forEach(annotations, obs => {
- const val = Number(obs[name]);
+ for (let r = 0; r < annotations.length; r += 1) {
+ const val = Number(annotations[r][name]);
min = val < min ? val : min;
max = val > max ? val : max;
- });
- return { range: { min, max } };
+ }
+ return {
+ categorical: false,
+ range: { min, max }
+ };
}
- throw new Error("incomprehensible schema");
+ /* else categorical */
+ const categoryCounts = new Map();
+ for (let r = 0; r < annotations.length; r += 1) {
+ const val = annotations[r][name];
+ let curCount = categoryCounts.get(val);
+ if (curCount === undefined) curCount = 0;
+ categoryCounts.set(val, curCount + 1);
+ }
+ return {
+ categorical: true,
+ categories: [...categoryCounts.keys()],
+ categoryCounts,
+ numCategories: categoryCounts.size
+ };
})
.value();
+ return summary;
}
export default function summarizeAnnotations(
@@ -96,7 +99,7 @@ export default function summarizeAnnotations(
varAnnotations
) {
return {
- obs: summarizeDimension(schema.annotations.obs, obsAnnotations),
- var: summarizeDimension(schema.annotations.var, varAnnotations)
+ obs: _summarizeAnnotations(schema.annotations.obs, obsAnnotations),
+ var: _summarizeAnnotations(schema.annotations.var, varAnnotations)
};
}
diff --git a/client/src/util/stateManager/worldUtil.js b/client/src/util/stateManager/worldUtil.js
new file mode 100644
index 00000000..0fe994cf
--- /dev/null
+++ b/client/src/util/stateManager/worldUtil.js
@@ -0,0 +1,65 @@
+/* eslint-disable import/prefer-default-export */
+import _ from "lodash";
+
+/*
+Various utility functions operating on World/Universe
+*/
+
+/*
+Count unique category values, binning first by dim1 then by dim2
+Return:
+
+Map {
+ dim1_val1: Map {
+ dim2_val1: number,
+ dim2_val2: number,
+ ...
+ },
+ ...
+}
+
+*/
+function _countCategoryValues2D(dim1, dim2, rows) {
+ const dimMap = new Map();
+ for (let r = 0; r < rows.length; r += 1) {
+ const row = rows[r];
+ const val1 = row[dim1];
+ const val2 = row[dim2];
+ let d2Map = dimMap.get(val1);
+ if (d2Map === undefined) {
+ d2Map = new Map();
+ dimMap.set(val1, d2Map);
+ }
+ let curCount = d2Map.get(val2);
+ if (curCount === undefined) {
+ curCount = 0;
+ }
+ d2Map.set(val2, curCount + 1);
+ }
+ return dimMap;
+}
+
+let __worldUtilMemoId__ = 0;
+function _memoizedId(x) {
+ if (!x.__worldUtilMemoId__) {
+ __worldUtilMemoId__ += 1;
+ x.__worldUtilMemoId__ = __worldUtilMemoId__;
+ }
+ return x.__worldUtilMemoId__;
+}
+function _countCategoryValues2DResolver(...args) {
+ const id = args[0] + args[1] + _memoizedId(args[2]);
+ return id;
+}
+
+export const countCategoryValues2D = _.memoize(
+ _countCategoryValues2D,
+ _countCategoryValues2DResolver
+);
+
+/*
+Clear any cached data within WorldUtil caches, eg, memoized functions
+*/
+export function clearCaches() {
+ countCategoryValues2D.cache.clear();
+}