Improved summary counts of annotation values (#478)

* convert annotation summary to a Map

* add 2d annotation count summary

* add memoization on 2D annotation counting

* add tests for annotation summarization

* fix import/exports

* rename WorldOps to WorldUtil

* rename WorldOps to WorldUtil

* add comment

client/__tests__/util/stateManager/summarizeAnnotations.test.js

@@ -0,0 +1,192 @@
+import summarizeAnnotations from "../../../src/util/stateManager/summarizeAnnotations";
+
+describe("summarizeAnnotations", () => {
+  const schema = {
+    annotations: {
+      obs: [
+        { name: "name", type: "string" },
+        { name: "nameString", type: "string" },
+        { name: "nameBoolean", type: "boolean" },
+        { name: "nameFloat32", type: "float32" },
+        { name: "nameInt32", type: "int32" },
+        {
+          name: "nameCategorical",
+          type: "categorical",
+          categories: [true, false, 1, 0, 0.00001, 4383.4833, "test", "", "0"]
+        }
+      ],
+      var: [{ name: "name", type: "string" }]
+    }
+  };
+
+  test("empty test", () => {
+    const summary = summarizeAnnotations(schema, [], []);
+    expect(summary).toEqual(
+      expect.objectContaining({
+        obs: {
+          nameString: {
+            categorical: true,
+            categories: [],
+            categoryCounts: new Map(),
+            numCategories: 0
+          },
+          nameBoolean: {
+            categorical: true,
+            categories: [],
+            categoryCounts: new Map(),
+            numCategories: 0
+          },
+          nameFloat32: {
+            categorical: false,
+            range: {
+              max: Number.NEGATIVE_INFINITY,
+              min: Number.POSITIVE_INFINITY
+            }
+          },
+          nameInt32: {
+            categorical: false,
+            range: {
+              max: Number.NEGATIVE_INFINITY,
+              min: Number.POSITIVE_INFINITY
+            }
+          },
+          nameCategorical: {
+            categorical: true,
+            categories: [],
+            categoryCounts: new Map(),
+            numCategories: 0
+          }
+        },
+        var: {}
+      })
+    );
+  });
+
+  test("simple test", () => {
+    const obsAnnotations = [
+      {
+        __index__: 0,
+        name: "n1",
+        nameString: "hi",
+        nameBoolean: true,
+        nameFloat32: 39.3,
+        nameInt32: 99,
+        nameCategorical: 1
+      }
+    ];
+    const varAnnotations = [];
+
+    const summary = summarizeAnnotations(
+      schema,
+      obsAnnotations,
+      varAnnotations
+    );
+
+    expect(summary).toEqual(
+      expect.objectContaining({
+        obs: {
+          nameString: {
+            categorical: true,
+            categories: ["hi"],
+            categoryCounts: new Map([["hi", 1]]),
+            numCategories: 1
+          },
+          nameBoolean: {
+            categorical: true,
+            categories: [true],
+            categoryCounts: new Map([[true, 1]]),
+            numCategories: 1
+          },
+          nameFloat32: {
+            categorical: false,
+            range: { min: 39.3, max: 39.3 }
+          },
+          nameInt32: {
+            categorical: false,
+            range: { min: 99, max: 99 }
+          },
+          nameCategorical: {
+            categorical: true,
+            categories: [1],
+            categoryCounts: new Map([[1, 1]]),
+            numCategories: 1
+          }
+        },
+        var: {}
+      })
+    );
+  });
+
+  test("multi test", () => {
+    const obsAnnotations = [
+      {
+        __index__: 0,
+        name: "n0",
+        nameString: "hi",
+        nameBoolean: false,
+        nameFloat32: 39.3,
+        nameInt32: 99,
+        nameCategorical: 1
+      },
+      {
+        __index__: 1,
+        name: "n1",
+        nameString: "hi",
+        nameBoolean: true,
+        nameFloat32: 39.3,
+        nameInt32: 99,
+        nameCategorical: false
+      },
+      {
+        __index__: 2,
+        name: "n2",
+        nameString: "bye",
+        nameBoolean: true,
+        nameFloat32: 0,
+        nameInt32: 99,
+        nameCategorical: "0"
+      }
+    ];
+    const varAnnotations = [];
+
+    const summary = summarizeAnnotations(
+      schema,
+      obsAnnotations,
+      varAnnotations
+    );
+
+    expect(summary).toMatchObject(
+      expect.objectContaining({
+        obs: {
+          nameString: {
+            categorical: true,
+            categories: expect.arrayContaining(["hi", "bye"]),
+            categoryCounts: new Map([["hi", 2], ["bye", 1]]),
+            numCategories: 2
+          },
+          nameBoolean: {
+            categorical: true,
+            categories: expect.arrayContaining([true, false]),
+            categoryCounts: new Map([[true, 2], [false, 1]]),
+            numCategories: 2
+          },
+          nameFloat32: {
+            categorical: false,
+            range: { min: 0, max: 39.3 }
+          },
+          nameInt32: {
+            categorical: false,
+            range: { min: 99, max: 99 }
+          },
+          nameCategorical: {
+            categorical: true,
+            categories: expect.arrayContaining([1, false, "0"]),
+            categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
+            numCategories: 3
+          }
+        },
+        var: {}
+      })
+    );
+  });
+});

client/__tests__/util/stateManager/worldUtil.test.js

@@ -0,0 +1,45 @@
+import {
+  countCategoryValues2D,
+  clearCaches
+} from "../../../src/util/stateManager/worldUtil";
+
+describe("WorldUtil cache management", () => {
+  test("empty", () => {
+    const count = countCategoryValues2D("a", "b", []);
+    expect(count).toMatchObject(new Map());
+  });
+
+  test("simple couts", () => {
+    const rows = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
+    const count = countCategoryValues2D("a", "b", rows);
+    expect(count).toMatchObject(
+      new Map([
+        [0, new Map([[true, 1], [false, 1]])],
+        [1, new Map([[false, 1]])]
+      ])
+    );
+  });
+
+  test("memo cache clear", () => {
+    clearCaches();
+    const row1 = [];
+    const row2 = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
+    const count1 = countCategoryValues2D("a", "b", row1);
+    const count2 = countCategoryValues2D("a", "b", row1);
+    const count3 = countCategoryValues2D("a", "b", []);
+    const count4 = countCategoryValues2D("a", "b", row2);
+
+    clearCaches();
+    const count10 = countCategoryValues2D("a", "b", row1);
+    const count11 = countCategoryValues2D("a", "b", row2);
+
+    expect(count1).toEqual(count2);
+    expect(count1).toEqual(count3);
+    expect(count1).toEqual(count10);
+    expect(count1).not.toBe(count3);
+    expect(count1).not.toBe(count10);
+
+    expect(count4).toEqual(count11);
+    expect(count4).not.toBe(count11);
+  });
+});

client/src/components/categorical/category.js

@@ -25,24 +25,22 @@ class Category extends React.Component {
     const { categoricalSelectionState, metadataField } = this.props;
     const cat = categoricalSelectionState[metadataField];
     const categoryCount = {
-      // total number of options in this category
+      // total number of categories in this dimension
-      totalOptionCount: cat.numOptions,
+      totalCatCount: cat.numCategories,
       // number of selected options in this category
-      selectedOptionCount: _.reduce(
+      selectedCatCount: _.reduce(
-        cat.optionSelected,
+        cat.categorySelected,
         (res, cond) => (cond ? res + 1 : res),
         0
       )
     };
-    if (categoryCount.selectedOptionCount === categoryCount.totalOptionCount) {
+    if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
       /* everything is on, so not indeterminate */
       this.checkbox.indeterminate = false;
-    } else if (categoryCount.selectedOptionCount === 0) {
+    } else if (categoryCount.selectedCatCount === 0) {
       /* nothing is on, so no */
       this.checkbox.indeterminate = false;
-    } else if (
+    } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
-      categoryCount.selectedOptionCount < categoryCount.totalOptionCount
-    ) {
       /* to be explicit... */
       this.checkbox.indeterminate = true;
     }
@@ -88,12 +86,12 @@ class Category extends React.Component {
     const { categoricalSelectionState, metadataField } = this.props;
 
     const cat = categoricalSelectionState[metadataField];
-    const optTuples = alphabeticallySortedValues([...cat.optionIndex]);
+    const optTuples = alphabeticallySortedValues([...cat.categoryIndices]);
     return _.map(optTuples, (tuple, i) => (
       <Value
         key={tuple[1]}
         metadataField={metadataField}
-        optionIndex={tuple[1]}
+        categoryIndex={tuple[1]}
         i={i}
       />
     ));

client/src/components/categorical/value.js

@@ -11,20 +11,20 @@ import _ from "lodash";
 }))
 class CategoryValue extends React.Component {
   toggleOff() {
-    const { dispatch, metadataField, optionIndex } = this.props;
+    const { dispatch, metadataField, categoryIndex } = this.props;
     dispatch({
       type: "categorical metadata filter deselect",
       metadataField,
-      optionIndex
+      categoryIndex
     });
   }
 
   toggleOn() {
-    const { dispatch, metadataField, optionIndex } = this.props;
+    const { dispatch, metadataField, categoryIndex } = this.props;
     dispatch({
       type: "categorical metadata filter select",
       metadataField,
-      optionIndex
+      categoryIndex
     });
   }
 
@@ -32,7 +32,7 @@ class CategoryValue extends React.Component {
     const {
       categoricalSelectionState,
       metadataField,
-      optionIndex,
+      categoryIndex,
       colorAccessor,
       colorScale,
       i,
@@ -42,10 +42,12 @@ class CategoryValue extends React.Component {
     if (!categoricalSelectionState) return null;
 
     const category = categoricalSelectionState[metadataField];
-    const selected = category.optionSelected[optionIndex];
+    const selected = category.categorySelected[categoryIndex];
-    const count = category.optionCount[optionIndex];
+    const count = category.categoryCounts[categoryIndex];
-    const value = category.optionValue[optionIndex];
+    const value = category.categoryValues[categoryIndex];
-    const displayString = String(category.optionValue[optionIndex]).valueOf();
+    const displayString = String(
+      category.categoryValues[categoryIndex]
+    ).valueOf();
 
     /* this is the color scale, so add swatches below */
     const c = metadataField === colorAccessor;

client/src/reducers/controls.js

@@ -1,7 +1,7 @@
 // jshint esversion: 6
 
 import _ from "lodash";
-import { World, kvCache } from "../util/stateManager";
+import { World, kvCache, WorldUtil } from "../util/stateManager";
 import parseRGB from "../util/parseRGB";
 import Crossfilter from "../util/typedCrossfilter";
 import * as globals from "../globals";
@@ -24,25 +24,27 @@ Remember that option values can be ANY js type, except undefined/null.
   {
     _category_name_1: {
       // map of option value to index
-      optionIndex: Map([
+      categoryIndices: Map([
-        optval1: index,
+        catval1: index,
         ...
       ])
 
       // index->selection true/false state
-      optionSelected: [ true/false, true/false, ... ]
+      categorySelected: [ true/false, true/false, ... ]
 
       // number of options
-      numOptions: number,
+      numCategories: number,
 
       // isTruncated - true if the options for selection has
       // been truncated (ie, was too large to implement)
     }
   }
 */
-function topNoptions(summary) {
+function topNCategories(summary) {
-  const counts = _.map(summary.categories, cat => summary.options[cat]);
+  const counts = _.map(summary.categories, cat =>
-  const sortIndex = fillRange(new Array(summary.numOptions)).sort(
+    summary.categoryCounts.get(cat)
+  );
+  const sortIndex = fillRange(new Array(summary.numCategories)).sort(
     (a, b) => counts[b] - counts[a]
   );
   const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
@@ -65,20 +67,18 @@ function createCategoricalSelectionState(state, world) {
         key !== "name" &&
         value.categories.length < state.maxCategoryItems;
       if (isSelectableCategory) {
-        const [optionValue, optionCount] = topNoptions(value);
+        const [categoryValues, categoryCounts] = topNCategories(value);
-        // const optionCount = Object.values(value.options);
+        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
-
+        const numCategories = categoryIndices.size;
-        const optionIndex = new Map(optionValue.map((v, i) => [v, i]));
+        const categorySelected = new Array(numCategories).fill(true);
-        const numOptions = optionIndex.size;
+        const isTruncated = categoryValues.length < value.numCategories;
-        const optionSelected = new Array(numOptions).fill(true);
-        const isTruncated = optionValue.length < value.numOptions;
         res[key] = {
-          optionValue, // array: of natively typed option values
+          categoryValues, // array: of natively typed category values
-          optionIndex, // map: option value (native type) -> option index
+          categoryIndices, // map: category value (native type) -> category index
-          optionSelected, // array: t/f selection state
+          categorySelected, // array: t/f selection state
-          numOptions, // number: of options
+          numCategories, // number: of categories
           isTruncated, // bool: true if list was truncated
-          optionCount // array: cardinality of each option
+          categoryCounts // array: cardinality of each category
         };
       }
     }
@@ -87,12 +87,12 @@ function createCategoricalSelectionState(state, world) {
 }
 
 /*
-given a categoricalSelectionState, return the list of all option values
+given a categoricalSelectionState, return the list of all category values
 where selection state is true (ie, they are selected).
 */
 function selectedValuesForCategory(categorySelectionState) {
-  const selectedValues = _([...categorySelectionState.optionIndex])
+  const selectedValues = _([...categorySelectionState.categoryIndices])
-    .filter(tuple => categorySelectionState.optionSelected[tuple[1]])
+    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
     .map(tuple => tuple[0])
     .value();
   return selectedValues;
@@ -175,6 +175,7 @@ const Controls = (
       );
       const crossfilter = Crossfilter(world.obsAnnotations);
       const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+      WorldUtil.clearCaches();
 
       const worldVarDataCache = world.varDataCache;
 
@@ -247,6 +248,7 @@ const Controls = (
       );
       const crossfilter = Crossfilter(world.obsAnnotations);
       const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+      WorldUtil.clearCaches();
 
       const worldVarDataCache = world.varDataCache;
       /* var dimensions */
@@ -514,15 +516,15 @@ const Controls = (
           Categorical metadata
     *******************************/
     case "categorical metadata filter select": {
-      const newOptionSelected = Array.from(
+      const newCategorySelected = Array.from(
-        state.categoricalSelectionState[action.metadataField].optionSelected
+        state.categoricalSelectionState[action.metadataField].categorySelected
       );
-      newOptionSelected[action.optionIndex] = true;
+      newCategorySelected[action.categoryIndex] = true;
       const newCategoricalSelectionState = {
         ...state.categoricalSelectionState,
         [action.metadataField]: {
           ...state.categoricalSelectionState[action.metadataField],
-          optionSelected: newOptionSelected
+          categorySelected: newCategorySelected
         }
       };
 
@@ -538,15 +540,15 @@ const Controls = (
       };
     }
     case "categorical metadata filter deselect": {
-      const newOptionSelected = Array.from(
+      const newCategorySelected = Array.from(
-        state.categoricalSelectionState[action.metadataField].optionSelected
+        state.categoricalSelectionState[action.metadataField].categorySelected
       );
-      newOptionSelected[action.optionIndex] = false;
+      newCategorySelected[action.categoryIndex] = false;
       const newCategoricalSelectionState = {
         ...state.categoricalSelectionState,
         [action.metadataField]: {
           ...state.categoricalSelectionState[action.metadataField],
-          optionSelected: newOptionSelected
+          categorySelected: newCategorySelected
         }
       };
 
@@ -566,8 +568,9 @@ const Controls = (
         ...state.categoricalSelectionState,
         [action.metadataField]: {
           ...state.categoricalSelectionState[action.metadataField],
-          optionSelected: Array.from(
+          categorySelected: Array.from(
-            state.categoricalSelectionState[action.metadataField].optionSelected
+            state.categoricalSelectionState[action.metadataField]
+              .categorySelected
           ).fill(false)
         }
       };
@@ -584,8 +587,9 @@ const Controls = (
         ...state.categoricalSelectionState,
         [action.metadataField]: {
           ...state.categoricalSelectionState[action.metadataField],
-          optionSelected: Array.from(
+          categorySelected: Array.from(
-            state.categoricalSelectionState[action.metadataField].optionSelected
+            state.categoricalSelectionState[action.metadataField]
+              .categorySelected
           ).fill(true)
         }
       };

client/src/util/stateManager/index.js

@@ -17,3 +17,4 @@ exists to support those concepts.
 export * as Universe from "./universe";
 export * as World from "./world";
 export * as kvCache from "./keyvalcache";
+export * as WorldUtil from "./worldUtil";

client/src/util/stateManager/summarizeAnnotations.js

@@ -8,6 +8,7 @@ Value will be an object, containing summary information.
 
 For continuous annotations (int, float, etc):
   <annotation_name>: {
+    categorical: false,
     range {
       min: <number>,
       max: <number>
@@ -15,12 +16,14 @@ For continuous annotations (int, float, etc):
   }
 
 For categorical annotations (boolean, string, category):
-  <annotatoin_name>: {
+  <annotation_name>: {
-    options: {
+    categorical: true,
-      <option1>: <number>,
+    categories: [ <category1>, <category2>, ... ]
+    categoryCounts: Map {
+      <category1>: <number>,
       ...
     },
-    numOptions: <number>
+    numCategories: <number>
   }
 
 Summarize will be returned for BOTH obs and var annotations.
@@ -28,19 +31,19 @@ Summarize will be returned for BOTH obs and var annotations.
 Example:
   {
     "Splice_sites_Annotated": {
-      "range": {
+      categorical: false,
+      range: {
         "min": 26,
         "max": 1075869
       }
     },
     "Selection": {
-      numOptions, 6,
+      categorical: true,
-      "options": {
+      numCategories, 3,
+      categories: [ "Astrocytes(HEPACAM)", "Endothelial(BSC)", "Unpanned" ],
+      categoryCounts: Map {
         "Astrocytes(HEPACAM)": 714,
         "Endothelial(BSC)": 123,
-        "Oligodendrocytes(GC)": 294,
-        "Neurons(Thy1)": 685,
-        "Microglia(CD45)": 1108,
         "Unpanned": 665
       }
     }
@@ -48,46 +51,46 @@ Example:
 
 NOTE: will not summarize the required 'name' annotation, as that is
 specified as unique per element.
-
-TODO: XXX - this data structure coerces all metadata categories into a string
-(ie, stores values as an Object property in the `options` field).   This looses
-information (eg, type) for category types which are not strings.  Consider an
-alterative data structure that does not use the object property for non-string
-data types (and does not use _.countBy to summarize).
 */
-function summarizeDimension(schema, annotations) {
+function _summarizeAnnotations(_schema, annotations) {
-  return _(schema)
+  const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
     .filter(v => v.name !== "name")
     .keyBy("name")
     .mapValues(anno => {
       const { name, type } = anno;
       const continuous = type === "int32" || type === "float32";
 
-      if (!continuous) {
-        const categories = _.uniq(_.flatMap(annotations, name));
-        const options = _.countBy(annotations, name);
-        const numOptions = _.size(options);
-        return {
-          numOptions,
-          options,
-          categories
-        };
-      }
-
       if (continuous) {
         let min = Number.POSITIVE_INFINITY;
         let max = Number.NEGATIVE_INFINITY;
-        _.forEach(annotations, obs => {
+        for (let r = 0; r < annotations.length; r += 1) {
-          const val = Number(obs[name]);
+          const val = Number(annotations[r][name]);
           min = val < min ? val : min;
           max = val > max ? val : max;
-        });
+        }
-        return { range: { min, max } };
+        return {
+          categorical: false,
+          range: { min, max }
+        };
       }
 
-      throw new Error("incomprehensible schema");
+      /* else categorical */
+      const categoryCounts = new Map();
+      for (let r = 0; r < annotations.length; r += 1) {
+        const val = annotations[r][name];
+        let curCount = categoryCounts.get(val);
+        if (curCount === undefined) curCount = 0;
+        categoryCounts.set(val, curCount + 1);
+      }
+      return {
+        categorical: true,
+        categories: [...categoryCounts.keys()],
+        categoryCounts,
+        numCategories: categoryCounts.size
+      };
     })
     .value();
+  return summary;
 }
 
 export default function summarizeAnnotations(
@@ -96,7 +99,7 @@ export default function summarizeAnnotations(
   varAnnotations
 ) {
   return {
-    obs: summarizeDimension(schema.annotations.obs, obsAnnotations),
+    obs: _summarizeAnnotations(schema.annotations.obs, obsAnnotations),
-    var: summarizeDimension(schema.annotations.var, varAnnotations)
+    var: _summarizeAnnotations(schema.annotations.var, varAnnotations)
   };
 }

client/src/util/stateManager/worldUtil.js

@@ -0,0 +1,65 @@
+/* eslint-disable import/prefer-default-export */
+import _ from "lodash";
+
+/*
+Various utility functions operating on World/Universe
+*/
+
+/*
+Count unique category values, binning first by dim1 then by dim2
+Return:
+
+Map {
+   dim1_val1: Map {
+      dim2_val1: number,
+      dim2_val2: number,
+      ...
+   },
+   ...
+}
+
+*/
+function _countCategoryValues2D(dim1, dim2, rows) {
+  const dimMap = new Map();
+  for (let r = 0; r < rows.length; r += 1) {
+    const row = rows[r];
+    const val1 = row[dim1];
+    const val2 = row[dim2];
+    let d2Map = dimMap.get(val1);
+    if (d2Map === undefined) {
+      d2Map = new Map();
+      dimMap.set(val1, d2Map);
+    }
+    let curCount = d2Map.get(val2);
+    if (curCount === undefined) {
+      curCount = 0;
+    }
+    d2Map.set(val2, curCount + 1);
+  }
+  return dimMap;
+}
+
+let __worldUtilMemoId__ = 0;
+function _memoizedId(x) {
+  if (!x.__worldUtilMemoId__) {
+    __worldUtilMemoId__ += 1;
+    x.__worldUtilMemoId__ = __worldUtilMemoId__;
+  }
+  return x.__worldUtilMemoId__;
+}
+function _countCategoryValues2DResolver(...args) {
+  const id = args[0] + args[1] + _memoizedId(args[2]);
+  return id;
+}
+
+export const countCategoryValues2D = _.memoize(
+  _countCategoryValues2D,
+  _countCategoryValues2DResolver
+);
+
+/*
+Clear any cached data within WorldUtil caches, eg, memoized functions
+*/
+export function clearCaches() {
+  countCategoryValues2D.cache.clear();
+}