fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-21 03:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

categorical metadata performance work (#356)

Browse Source 
* do not create unecessary option state for categories we will not display

* performance improvements for categorical metadata handling
This commit is contained in:
Bruce Martin
2018-10-22 09:56:41 -07:00
committed by GitHub
parent efb55a6332
commit 0c271bc815
10 changed files with 184 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/__tests__/util/stateManager/world.test.js
+6 -1
View File
@@ -64,10 +64,15 @@ describe("createWorldFromEntireUniverse", () => {
summary: expect.objectContaining({
obs: _(REST.schema.schema.annotations.obs)
.filter(v => v.name !== "name")
.keyBy("name")
.mapValues(() => expect.any(Object))
.value(),
var: {} // TODO: currently unimplemneted
var: _(REST.schema.schema.annotations.var)
.filter(v => v.name !== "name")
.keyBy("name")
.mapValues(() => expect.any(Object))
.value()
}),
varDataCache: expect.any(Object),
client/src/actions/index.js
+13 -2
View File
@@ -30,10 +30,21 @@ const doInitialDataLoad = () =>
.map(url => doJsonRequest(url))
.value();
const results = await Promise.all(requests);
const universe = Universe.createUniverseFromRestV02Response(...results);
/* set config defaults */
const config = { ...globals.configDefaults, ...results[0].config };
const [, schema, obsAnno, varAnno, obsLayout] = [...results];
const universe = Universe.createUniverseFromRestV02Response(
config,
schema,
obsAnno,
varAnno,
obsLayout
);
dispatch({
type: "configuration load complete",
config: results[0].config
config
});
dispatch({
type: "initial data load complete (universe exists)",
client/src/components/categorical/categorical.js
+14 -3
View File
@@ -23,11 +23,16 @@ const truncateCategories = options => {
};
@connect(state => ({
ranges: _.get(state.controls.world, "summary.obs", null)
ranges: _.get(state.controls.world, "summary.obs", null),
categorySelectionLimit: _.get(
state.config,
"parameters.category-selection-limit",
globals.configDefaults.parameters["category-selection-limit"]
)
}))
class Categories extends React.Component {
render() {
const { ranges } = this.props;
const { ranges, categorySelectionLimit } = this.props;
if (!ranges) return null;
return (
@@ -42,7 +47,13 @@ class Categories extends React.Component {
<p> Categorical Metadata </p>
{_.map(ranges, (value, key) => {
const isColorField = key.includes("color") || key.includes("Color");
if (value.options && !isColorField && key !== "name") {
const isSelectableCategory =
value.options &&
!isColorField &&
key !== "name" &&
value.numOptions < categorySelectionLimit;
if (isSelectableCategory) {
const categoryOptions = truncateCategories(value.options);
return (
<Category
client/src/components/categorical/category.js
+25 -9
View File
@@ -2,11 +2,25 @@ import React from "react";
import _ from "lodash";
import { connect } from "react-redux";
import { FaChevronRight, FaChevronDown, FaPaintBrush } from "react-icons/fa";
import memoize from "memoize-one";
import * as globals from "../../globals";
import Value from "./value";
import alphabeticallySortedValues from "./util";
const countCategories = (values, optsAsBools) =>
_.reduce(
values,
(r, v, k) => {
r.total += 1;
if (optsAsBools[k]) {
r.on += 1;
}
return r;
},
{ total: 0, on: 0 }
);
@connect(state => ({
colorAccessor: state.controls.colorAccessor,
categoricalAsBooleansMap: state.controls.categoricalAsBooleansMap
@@ -18,22 +32,24 @@ class Category extends React.Component {
isChecked: true,
isExpanded: false
};
this.countCategories = memoize((values, optsAsBools) =>
countCategories(values, optsAsBools)
);
}
componentDidUpdate() {
const { categoricalAsBooleansMap, metadataField } = this.props;
const valuesAsBool = _.values(categoricalAsBooleansMap[metadataField]);
/* count categories toggled on by counting true values */
const categoriesToggledOn = _.values(valuesAsBool).filter(v => v).length;
if (categoriesToggledOn === valuesAsBool.length) {
const { categoricalAsBooleansMap, metadataField, values } = this.props;
const categoryCount = this.countCategories(
values,
categoricalAsBooleansMap[metadataField]
);
if (categoryCount.on === categoryCount.total) {
/* everything is on, so not indeterminate */
this.checkbox.indeterminate = false;
} else if (categoriesToggledOn === 0) {
} else if (categoryCount.on === 0) {
/* nothing is on, so no */
this.checkbox.indeterminate = false;
} else if (categoriesToggledOn < valuesAsBool.length) {
} else if (categoryCount.on < categoryCount.total) {
/* to be explicit... */
this.checkbox.indeterminate = true;
}
client/src/globals.js
+13
View File
@@ -28,8 +28,21 @@ export const continuous = [
"Unmapped_short"
];
/* if a categorical metadata field has more options than this, truncate */
export const maxCategoricalOptionsToDisplay = 100;
/*
these are default values for configuration the CLI may supply.
See the REST API and CLI specs for more info.
*/
export const configDefaults = {
features: {},
displayNames: {},
parameters: {
"category-selection-limit": 1000
}
};
/* colors */
export const blue = "#4a90e2";
export const hcaBlue = "#1c7cc7";
client/src/reducers/config.js
+2 -1
View File
@@ -2,7 +2,8 @@
const Config = (
state = {
displayNames: null,
features: null
features: null,
parameters: null
},
action
) => {
client/src/reducers/controls.js
Vendored
+1 -1
View File
@@ -16,7 +16,7 @@ import {
function createCategoricalAsBooleansMap(world) {
const res = {};
_.each(world.summary.obs, (value, key) => {
if (value.options) {
if (value.options && key !== "name") {
const optionsAsBooleans = {};
_.each(value.options, (_value, _key) => {
optionsAsBooleans[_key] = true;
client/src/util/stateManager/summarizeAnnotations.js
+94
View File
@@ -0,0 +1,94 @@
import _ from "lodash";
/*
Build and return obs/var summary using any annotation in the schema
Summary information for each annotation, keyed by annotation name.
Value will be an object, containing summary information.
For continuous annotations (int, float, etc):
<annotation_name>: {
range {
min: <number>,
max: <number>
}
}
For categorical annotations (boolean, string, category):
<annotatoin_name>: {
options: {
<option1>: <number>,
...
},
numOptions: <number>
}
Summarize will be returned for BOTH obs and var annotations.
Example:
{
"Splice_sites_Annotated": {
"range": {
"min": 26,
"max": 1075869
}
},
"Selection": {
numOptions, 6,
"options": {
"Astrocytes(HEPACAM)": 714,
"Endothelial(BSC)": 123,
"Oligodendrocytes(GC)": 294,
"Neurons(Thy1)": 685,
"Microglia(CD45)": 1108,
"Unpanned": 665
}
}
}
NOTE: will not summarize the required 'name' annotation, as that is
specified as unique per element.
*/
function summarizeDimension(schema, annotations) {
return _(schema)
.filter(v => v.name !== "name")
.keyBy("name")
.mapValues(anno => {
const { name, type } = anno;
const continuous = type === "int32" || type === "float32";
if (!continuous) {
const options = _.countBy(annotations, name);
const numOptions = _.size(options);
return {
numOptions,
options
};
}
if (continuous) {
let min = Number.POSITIVE_INFINITY;
let max = Number.NEGATIVE_INFINITY;
_.forEach(annotations, obs => {
const val = Number(obs[name]);
min = val < min ? val : min;
max = val > max ? val : max;
});
return { range: { min, max } };
}
throw new Error("incomprehensible schema");
})
.value();
}
export default function summarizeAnnotations(
schema,
obsAnnotations,
varAnnotations
) {
return {
obs: summarizeDimension(schema.annotations.obs, obsAnnotations),
var: summarizeDimension(schema.annotations.var, varAnnotations)
};
}
client/src/util/stateManager/universe.js
+8
View File
@@ -2,6 +2,7 @@
import _ from "lodash";
import * as kvCache from "./keyvalcache";
import summarizeAnnotations from "./summarizeAnnotations";
/*
Private helper function - create and return a template Universe
@@ -28,6 +29,7 @@ function templateUniverse() {
varAnnotations: [] /* all var annotations, by var index */,
obsNameToIndexMap: {} /* reverse map 'name' to index */,
varNameToIndexMap: {} /* reverse map 'name' to index */,
summary: null /* derived data summaries XXX: consider exploding in place */,
obsLayout: { X: [], Y: [] } /* xy layout */,
@@ -191,6 +193,12 @@ export function createUniverseFromRestV02Response(
/* layout */
universe.obsLayout = RESTv02LayoutResponseToInternal(layoutObsResponse);
universe.summary = summarizeAnnotations(
universe.schema,
universe.obsAnnotations,
universe.varAnnotations
);
return finalize(universe);
}
client/src/util/stateManager/world.js
+8 -79
View File
@@ -2,6 +2,7 @@
import _ from "lodash";
import * as kvCache from "./keyvalcache";
import summarizeAnnotations from "./summarizeAnnotations";
import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
/*
@@ -51,83 +52,6 @@ obs/cell.
const VarDataCacheLowWatermark = 32; // cache element count
const VarDataCacheTTLMs = 1000; // min cache time in MS
function summarizeAnnotations(schema, obsAnnotations) {
/*
Build and return obs/var summary using any annotation in the schema
Summary information for each annotation, keyed by annotation name.
Value will be an object, containing either 'range' or 'options' object,
depending on the annotation schema type (categorical or continuous).
Summarize for BOTH obs and var annotations. Result format:
{
obs: {
annotation_name: { ... },
...
},
var: {
annotation_name: { ... },
...
}
}
Example:
{
"Splice_sites_Annotated": {
"range": {
"min": 26,
"max": 1075869
}
},
"Selection": {
"options": {
"Astrocytes(HEPACAM)": 714,
"Endothelial(BSC)": 123,
"Oligodendrocytes(GC)": 294,
"Neurons(Thy1)": 685,
"Microglia(CD45)": 1108,
"Unpanned": 665
}
}
}
*/
const obsSummary = _(schema.annotations.obs)
.keyBy("name")
.mapValues(anno => {
const { name, type } = anno;
const continuous = type === "int32" || type === "float32";
if (!continuous) {
return {
options: _.countBy(obsAnnotations, name)
};
}
if (continuous) {
let min = Number.POSITIVE_INFINITY;
let max = Number.NEGATIVE_INFINITY;
_.forEach(obsAnnotations, obs => {
const val = Number(obs[name]);
min = val < min ? val : min;
max = val > max ? val : max;
});
return { range: { min, max } };
}
throw new Error("incomprehensible schema");
})
.value();
// TODO XXX - not currently used, so skip it
const varSummary = {};
return {
obs: obsSummary,
var: varSummary
};
}
function templateWorld() {
return {
// map from universe obsIndex to world offset.
@@ -186,7 +110,11 @@ export function createWorldFromEntireUniverse(universe) {
world.obsLayout = universe.obsLayout;
/* derived data & summaries */
world.summary = summarizeAnnotations(world.schema, world.obsAnnotations);
world.summary = summarizeAnnotations(
world.schema,
world.obsAnnotations,
world.varAnnotations
);
/* build the varDataCache */
world.varDataCache = kvCache.map(
@@ -242,7 +170,8 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
/* derived data & summaries */
newWorld.summary = summarizeAnnotations(
newWorld.schema,
newWorld.obsAnnotations
newWorld.obsAnnotations,
newWorld.varAnnotations
);
/* build the varDataCache */