fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-29 04:28:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

categorical metadata performance work (#356)

Browse Source 
* do not create unecessary option state for categories we will not display

* performance improvements for categorical metadata handling
This commit is contained in:
Bruce Martin
2018-10-22 09:56:41 -07:00
committed by GitHub
parent efb55a6332
commit 0c271bc815
10 changed files with 184 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/util/stateManager/summarizeAnnotations.js
+94
View File
@@ -0,0 +1,94 @@
import _ from "lodash";
/*
Build and return obs/var summary using any annotation in the schema
Summary information for each annotation, keyed by annotation name.
Value will be an object, containing summary information.
For continuous annotations (int, float, etc):
<annotation_name>: {
range {
min: <number>,
max: <number>
}
}
For categorical annotations (boolean, string, category):
<annotatoin_name>: {
options: {
<option1>: <number>,
...
},
numOptions: <number>
}
Summarize will be returned for BOTH obs and var annotations.
Example:
{
"Splice_sites_Annotated": {
"range": {
"min": 26,
"max": 1075869
}
},
"Selection": {
numOptions, 6,
"options": {
"Astrocytes(HEPACAM)": 714,
"Endothelial(BSC)": 123,
"Oligodendrocytes(GC)": 294,
"Neurons(Thy1)": 685,
"Microglia(CD45)": 1108,
"Unpanned": 665
}
}
}
NOTE: will not summarize the required 'name' annotation, as that is
specified as unique per element.
*/
function summarizeDimension(schema, annotations) {
return _(schema)
.filter(v => v.name !== "name")
.keyBy("name")
.mapValues(anno => {
const { name, type } = anno;
const continuous = type === "int32" || type === "float32";
if (!continuous) {
const options = _.countBy(annotations, name);
const numOptions = _.size(options);
return {
numOptions,
options
};
}
if (continuous) {
let min = Number.POSITIVE_INFINITY;
let max = Number.NEGATIVE_INFINITY;
_.forEach(annotations, obs => {
const val = Number(obs[name]);
min = val < min ? val : min;
max = val > max ? val : max;
});
return { range: { min, max } };
}
throw new Error("incomprehensible schema");
})
.value();
}
export default function summarizeAnnotations(
schema,
obsAnnotations,
varAnnotations
) {
return {
obs: summarizeDimension(schema.annotations.obs, obsAnnotations),
var: summarizeDimension(schema.annotations.var, varAnnotations)
};
}
client/src/util/stateManager/universe.js
+8
View File
@@ -2,6 +2,7 @@
import _ from "lodash";
import * as kvCache from "./keyvalcache";
import summarizeAnnotations from "./summarizeAnnotations";
/*
Private helper function - create and return a template Universe
@@ -28,6 +29,7 @@ function templateUniverse() {
varAnnotations: [] /* all var annotations, by var index */,
obsNameToIndexMap: {} /* reverse map 'name' to index */,
varNameToIndexMap: {} /* reverse map 'name' to index */,
summary: null /* derived data summaries XXX: consider exploding in place */,
obsLayout: { X: [], Y: [] } /* xy layout */,
@@ -191,6 +193,12 @@ export function createUniverseFromRestV02Response(
/* layout */
universe.obsLayout = RESTv02LayoutResponseToInternal(layoutObsResponse);
universe.summary = summarizeAnnotations(
universe.schema,
universe.obsAnnotations,
universe.varAnnotations
);
return finalize(universe);
}
client/src/util/stateManager/world.js
+8 -79
View File
@@ -2,6 +2,7 @@
import _ from "lodash";
import * as kvCache from "./keyvalcache";
import summarizeAnnotations from "./summarizeAnnotations";
import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
/*
@@ -51,83 +52,6 @@ obs/cell.
const VarDataCacheLowWatermark = 32; // cache element count
const VarDataCacheTTLMs = 1000; // min cache time in MS
function summarizeAnnotations(schema, obsAnnotations) {
/*
Build and return obs/var summary using any annotation in the schema
Summary information for each annotation, keyed by annotation name.
Value will be an object, containing either 'range' or 'options' object,
depending on the annotation schema type (categorical or continuous).
Summarize for BOTH obs and var annotations. Result format:
{
obs: {
annotation_name: { ... },
...
},
var: {
annotation_name: { ... },
...
}
}
Example:
{
"Splice_sites_Annotated": {
"range": {
"min": 26,
"max": 1075869
}
},
"Selection": {
"options": {
"Astrocytes(HEPACAM)": 714,
"Endothelial(BSC)": 123,
"Oligodendrocytes(GC)": 294,
"Neurons(Thy1)": 685,
"Microglia(CD45)": 1108,
"Unpanned": 665
}
}
}
*/
const obsSummary = _(schema.annotations.obs)
.keyBy("name")
.mapValues(anno => {
const { name, type } = anno;
const continuous = type === "int32" || type === "float32";
if (!continuous) {
return {
options: _.countBy(obsAnnotations, name)
};
}
if (continuous) {
let min = Number.POSITIVE_INFINITY;
let max = Number.NEGATIVE_INFINITY;
_.forEach(obsAnnotations, obs => {
const val = Number(obs[name]);
min = val < min ? val : min;
max = val > max ? val : max;
});
return { range: { min, max } };
}
throw new Error("incomprehensible schema");
})
.value();
// TODO XXX - not currently used, so skip it
const varSummary = {};
return {
obs: obsSummary,
var: varSummary
};
}
function templateWorld() {
return {
// map from universe obsIndex to world offset.
@@ -186,7 +110,11 @@ export function createWorldFromEntireUniverse(universe) {
world.obsLayout = universe.obsLayout;
/* derived data & summaries */
world.summary = summarizeAnnotations(world.schema, world.obsAnnotations);
world.summary = summarizeAnnotations(
world.schema,
world.obsAnnotations,
world.varAnnotations
);
/* build the varDataCache */
world.varDataCache = kvCache.map(
@@ -242,7 +170,8 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
/* derived data & summaries */
newWorld.summary = summarizeAnnotations(
newWorld.schema,
newWorld.obsAnnotations
newWorld.obsAnnotations,
newWorld.varAnnotations
);
/* build the varDataCache */