categorical metadata performance work (#356)

* do not create unecessary option state for categories we will not display

* performance improvements for categorical metadata handling

client/__tests__/util/stateManager/world.test.js

@@ -64,10 +64,15 @@ describe("createWorldFromEntireUniverse", () => {
 
         summary: expect.objectContaining({
           obs: _(REST.schema.schema.annotations.obs)
+            .filter(v => v.name !== "name")
             .keyBy("name")
             .mapValues(() => expect.any(Object))
             .value(),
-          var: {} // TODO: currently unimplemneted
+          var: _(REST.schema.schema.annotations.var)
+            .filter(v => v.name !== "name")
+            .keyBy("name")
+            .mapValues(() => expect.any(Object))
+            .value()
         }),
 
         varDataCache: expect.any(Object),

client/src/actions/index.js

@@ -30,10 +30,21 @@ const doInitialDataLoad = () =>
         .map(url => doJsonRequest(url))
         .value();
       const results = await Promise.all(requests);
-      const universe = Universe.createUniverseFromRestV02Response(...results);
+
+      /* set config defaults */
+      const config = { ...globals.configDefaults, ...results[0].config };
+      const [, schema, obsAnno, varAnno, obsLayout] = [...results];
+      const universe = Universe.createUniverseFromRestV02Response(
+        config,
+        schema,
+        obsAnno,
+        varAnno,
+        obsLayout
+      );
+
       dispatch({
         type: "configuration load complete",
-        config: results[0].config
+        config
       });
       dispatch({
         type: "initial data load complete (universe exists)",

client/src/components/categorical/categorical.js

@@ -23,11 +23,16 @@ const truncateCategories = options => {
 };
 
 @connect(state => ({
-  ranges: _.get(state.controls.world, "summary.obs", null)
+  ranges: _.get(state.controls.world, "summary.obs", null),
+  categorySelectionLimit: _.get(
+    state.config,
+    "parameters.category-selection-limit",
+    globals.configDefaults.parameters["category-selection-limit"]
+  )
 }))
 class Categories extends React.Component {
   render() {
-    const { ranges } = this.props;
+    const { ranges, categorySelectionLimit } = this.props;
     if (!ranges) return null;
 
     return (
@@ -42,7 +47,13 @@ class Categories extends React.Component {
         <p> Categorical Metadata </p>
         {_.map(ranges, (value, key) => {
           const isColorField = key.includes("color") || key.includes("Color");
-          if (value.options && !isColorField && key !== "name") {
+          const isSelectableCategory =
+            value.options &&
+            !isColorField &&
+            key !== "name" &&
+            value.numOptions < categorySelectionLimit;
+
+          if (isSelectableCategory) {
             const categoryOptions = truncateCategories(value.options);
             return (
               <Category

client/src/components/categorical/category.js

@@ -2,11 +2,25 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import { FaChevronRight, FaChevronDown, FaPaintBrush } from "react-icons/fa";
+import memoize from "memoize-one";
 
 import * as globals from "../../globals";
 import Value from "./value";
 import alphabeticallySortedValues from "./util";
 
+const countCategories = (values, optsAsBools) =>
+  _.reduce(
+    values,
+    (r, v, k) => {
+      r.total += 1;
+      if (optsAsBools[k]) {
+        r.on += 1;
+      }
+      return r;
+    },
+    { total: 0, on: 0 }
+  );
+
 @connect(state => ({
   colorAccessor: state.controls.colorAccessor,
   categoricalAsBooleansMap: state.controls.categoricalAsBooleansMap
@@ -18,22 +32,24 @@ class Category extends React.Component {
       isChecked: true,
       isExpanded: false
     };
+    this.countCategories = memoize((values, optsAsBools) =>
+      countCategories(values, optsAsBools)
+    );
   }
 
   componentDidUpdate() {
-    const { categoricalAsBooleansMap, metadataField } = this.props;
+    const { categoricalAsBooleansMap, metadataField, values } = this.props;
-
+    const categoryCount = this.countCategories(
-    const valuesAsBool = _.values(categoricalAsBooleansMap[metadataField]);
+      values,
-    /* count categories toggled on by counting true values */
+      categoricalAsBooleansMap[metadataField]
-    const categoriesToggledOn = _.values(valuesAsBool).filter(v => v).length;
+    );
-
+    if (categoryCount.on === categoryCount.total) {
-    if (categoriesToggledOn === valuesAsBool.length) {
       /* everything is on, so not indeterminate */
       this.checkbox.indeterminate = false;
-    } else if (categoriesToggledOn === 0) {
+    } else if (categoryCount.on === 0) {
       /* nothing is on, so no */
       this.checkbox.indeterminate = false;
-    } else if (categoriesToggledOn < valuesAsBool.length) {
+    } else if (categoryCount.on < categoryCount.total) {
       /* to be explicit... */
       this.checkbox.indeterminate = true;
     }

client/src/globals.js

@@ -28,8 +28,21 @@ export const continuous = [
   "Unmapped_short"
 ];
 
+/* if a categorical metadata field has more options than this, truncate */
 export const maxCategoricalOptionsToDisplay = 100;
 
+/*
+these are default values for configuration  the CLI may supply.
+See the REST API and CLI specs for more info.
+*/
+export const configDefaults = {
+  features: {},
+  displayNames: {},
+  parameters: {
+    "category-selection-limit": 1000
+  }
+};
+
 /* colors */
 export const blue = "#4a90e2";
 export const hcaBlue = "#1c7cc7";

client/src/reducers/config.js

@@ -2,7 +2,8 @@
 const Config = (
   state = {
     displayNames: null,
-    features: null
+    features: null,
+    parameters: null
   },
   action
 ) => {

client/src/reducers/controls.js

@@ -16,7 +16,7 @@ import {
 function createCategoricalAsBooleansMap(world) {
   const res = {};
   _.each(world.summary.obs, (value, key) => {
-    if (value.options) {
+    if (value.options && key !== "name") {
       const optionsAsBooleans = {};
       _.each(value.options, (_value, _key) => {
         optionsAsBooleans[_key] = true;

client/src/util/stateManager/summarizeAnnotations.js

@@ -0,0 +1,94 @@
+import _ from "lodash";
+
+/*
+Build and return obs/var summary using any annotation in the schema
+
+Summary information for each annotation, keyed by annotation name.
+Value will be an object, containing summary information.
+
+For continuous annotations (int, float, etc):
+  <annotation_name>: {
+    range {
+      min: <number>,
+      max: <number>
+    }
+  }
+
+For categorical annotations (boolean, string, category):
+  <annotatoin_name>: {
+    options: {
+      <option1>: <number>,
+      ...
+    },
+    numOptions: <number>
+  }
+
+Summarize will be returned for BOTH obs and var annotations.
+
+Example:
+  {
+    "Splice_sites_Annotated": {
+      "range": {
+        "min": 26,
+        "max": 1075869
+      }
+    },
+    "Selection": {
+      numOptions, 6,
+      "options": {
+        "Astrocytes(HEPACAM)": 714,
+        "Endothelial(BSC)": 123,
+        "Oligodendrocytes(GC)": 294,
+        "Neurons(Thy1)": 685,
+        "Microglia(CD45)": 1108,
+        "Unpanned": 665
+      }
+    }
+  }
+
+NOTE: will not summarize the required 'name' annotation, as that is
+specified as unique per element.
+*/
+function summarizeDimension(schema, annotations) {
+  return _(schema)
+    .filter(v => v.name !== "name")
+    .keyBy("name")
+    .mapValues(anno => {
+      const { name, type } = anno;
+      const continuous = type === "int32" || type === "float32";
+
+      if (!continuous) {
+        const options = _.countBy(annotations, name);
+        const numOptions = _.size(options);
+        return {
+          numOptions,
+          options
+        };
+      }
+
+      if (continuous) {
+        let min = Number.POSITIVE_INFINITY;
+        let max = Number.NEGATIVE_INFINITY;
+        _.forEach(annotations, obs => {
+          const val = Number(obs[name]);
+          min = val < min ? val : min;
+          max = val > max ? val : max;
+        });
+        return { range: { min, max } };
+      }
+
+      throw new Error("incomprehensible schema");
+    })
+    .value();
+}
+
+export default function summarizeAnnotations(
+  schema,
+  obsAnnotations,
+  varAnnotations
+) {
+  return {
+    obs: summarizeDimension(schema.annotations.obs, obsAnnotations),
+    var: summarizeDimension(schema.annotations.var, varAnnotations)
+  };
+}

client/src/util/stateManager/universe.js

@@ -2,6 +2,7 @@
 
 import _ from "lodash";
 import * as kvCache from "./keyvalcache";
+import summarizeAnnotations from "./summarizeAnnotations";
 
 /*
 Private helper function - create and return a template Universe
@@ -28,6 +29,7 @@ function templateUniverse() {
     varAnnotations: [] /* all var annotations, by var index */,
     obsNameToIndexMap: {} /* reverse map 'name' to index */,
     varNameToIndexMap: {} /* reverse map 'name' to index */,
+    summary: null /* derived data summaries XXX: consider exploding in place */,
 
     obsLayout: { X: [], Y: [] } /* xy layout */,
 
@@ -191,6 +193,12 @@ export function createUniverseFromRestV02Response(
   /* layout */
   universe.obsLayout = RESTv02LayoutResponseToInternal(layoutObsResponse);
 
+  universe.summary = summarizeAnnotations(
+    universe.schema,
+    universe.obsAnnotations,
+    universe.varAnnotations
+  );
+
   return finalize(universe);
 }
 

client/src/util/stateManager/world.js

@@ -2,6 +2,7 @@
 
 import _ from "lodash";
 import * as kvCache from "./keyvalcache";
+import summarizeAnnotations from "./summarizeAnnotations";
 import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
 
 /*
@@ -51,83 +52,6 @@ obs/cell.
 const VarDataCacheLowWatermark = 32; // cache element count
 const VarDataCacheTTLMs = 1000; // min cache time in MS
 
-function summarizeAnnotations(schema, obsAnnotations) {
-  /*
-  Build and return obs/var summary using any annotation in the schema
-
-  Summary information for each annotation, keyed by annotation name.
-  Value will be an object, containing either 'range' or 'options' object,
-  depending on the annotation schema type (categorical or continuous).
-
-  Summarize for BOTH obs and var annotations.  Result format:
-
-  {
-    obs: {
-      annotation_name: { ... },
-      ...
-    },
-    var: {
-      annotation_name: { ... },
-      ...
-    }
-  }
-
-  Example:
-    {
-      "Splice_sites_Annotated": {
-        "range": {
-          "min": 26,
-          "max": 1075869
-        }
-      },
-      "Selection": {
-        "options": {
-          "Astrocytes(HEPACAM)": 714,
-          "Endothelial(BSC)": 123,
-          "Oligodendrocytes(GC)": 294,
-          "Neurons(Thy1)": 685,
-          "Microglia(CD45)": 1108,
-          "Unpanned": 665
-        }
-      }
-    }
-  */
-  const obsSummary = _(schema.annotations.obs)
-    .keyBy("name")
-    .mapValues(anno => {
-      const { name, type } = anno;
-      const continuous = type === "int32" || type === "float32";
-
-      if (!continuous) {
-        return {
-          options: _.countBy(obsAnnotations, name)
-        };
-      }
-
-      if (continuous) {
-        let min = Number.POSITIVE_INFINITY;
-        let max = Number.NEGATIVE_INFINITY;
-        _.forEach(obsAnnotations, obs => {
-          const val = Number(obs[name]);
-          min = val < min ? val : min;
-          max = val > max ? val : max;
-        });
-        return { range: { min, max } };
-      }
-
-      throw new Error("incomprehensible schema");
-    })
-    .value();
-
-  // TODO XXX - not currently used, so skip it
-  const varSummary = {};
-
-  return {
-    obs: obsSummary,
-    var: varSummary
-  };
-}
-
 function templateWorld() {
   return {
     // map from universe obsIndex to world offset.
@@ -186,7 +110,11 @@ export function createWorldFromEntireUniverse(universe) {
   world.obsLayout = universe.obsLayout;
 
   /* derived data & summaries */
-  world.summary = summarizeAnnotations(world.schema, world.obsAnnotations);
+  world.summary = summarizeAnnotations(
+    world.schema,
+    world.obsAnnotations,
+    world.varAnnotations
+  );
 
   /* build the varDataCache */
   world.varDataCache = kvCache.map(
@@ -242,7 +170,8 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
   /* derived data & summaries */
   newWorld.summary = summarizeAnnotations(
     newWorld.schema,
-    newWorld.obsAnnotations
+    newWorld.obsAnnotations,
+    newWorld.varAnnotations
   );
 
   /* build the varDataCache */