mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-03 01:18:12 +08:00
Redux refactor (#1571)
* refactor categorical controls state * lint * fix race condition in tests * fix typo * add missing update on subset * remove obsolete code * update jest and puppeteer major version; update all minors * update when label changes * remove lint from tests; increase timeouts in e2e tests * initial refactoring to new async annomatrix * refine error handling * fix bad merge * add continuous legend * lint * fix memoization in color table creators * partial implementation of user defined annotations * add new annotations action creator file * first pass at user annotations * additional user annotation bug fixes * user annotation auto-save * unit test cleanup * lint * refactor into multiple files * cleanup * add column GC * fix several bugs in user annotations * remove debug code * no anonymous functions * undo redo cleanup * file cleanup * scatterplot * performance * cleanup * remove old code * render in parallel with load * fix race condition * simply graph rendering * render throttle DRY * fix category label order * fix typo in e2e test setup * re-fix the e2e test setup * be more tolerant of races * anno matrix unit tests * temp disable reembedding * pilot port continuous histo to react-async * name change * lint * fix repaint bug * typo fix * update snap to match new ids * world/universe name cleanup * move annoMatrix to src dir * use private underscore naming convention * fix corner case in all selected * name cleanup * add layout control * init edge case * lint * port scatterplot * fix label indexing bug and improve tests * port category to react-async * fix user annotation labelling while subset * select all of prev layout on layout switch * fix race with crossfilter update * prettier lint * fix misleading comment * fix url composition in loader * first pass at crossfilter tests * lint * lint * fix typo * improved error handling for network errors * fix memoization bug * add memo * refactor for performnce * add missing single-value handling in select exact parser * small bugs discovered by tests * lint * additional crossfilter unit tests * remove extraneous comment * add support for automatic category determination * lint * fix render bug in category * take advantage of schema categories guarantee * lint * do not clear history when resetting * enhanced annomatrix gc * lint * finish renaming to follow conventions; fix clone race bug * lint * add priority based loading to improve initial data load UX * crossfilter cache perf * perf tuning * remove timers * documentation * PR review changes * PR review changes * more PR review edits * improve clarity of comment * more PR review fixes * port centroidLabels to use react-async * remove dead code * pr review updates * oops, remove logging
This commit is contained in:
Bruce Martin
@@ -2,16 +2,13 @@
|
||||
Helper functions for user-editable annotations state management.
|
||||
See also reducers/annotations.js
|
||||
*/
|
||||
import { unassignedCategoryLabel } from "../../globals";
|
||||
import * as SchemaHelpers from "./schemaHelpers";
|
||||
import { obsAnnoDimensionName } from "../nameCreators";
|
||||
|
||||
/*
|
||||
There are a number of state constraints assumed throughout the
|
||||
application:
|
||||
- all obs annotations are in {world|universe}.obsAnnotations,
|
||||
- all obs annotations are in annoMatrix,
|
||||
regardless of whether or not they are user editable.
|
||||
- the {world|universe}.schema is always up to date and matches
|
||||
- the annoMatrix.schema is always up to date and matches
|
||||
the data
|
||||
- the schema flag `writable` correctly indicates whether
|
||||
the annotation is editable/mutable.
|
||||
@@ -31,105 +28,11 @@ export function isContinuousAnnotation(schema, name) {
|
||||
}
|
||||
|
||||
function _isUserAnnotation(schema, name) {
|
||||
return schema.annotations.obsByName[name]?.writable;
|
||||
return schema.annotations.obsByName[name]?.writable || false;
|
||||
}
|
||||
|
||||
export function isUserAnnotation(worldOrUniverse, name) {
|
||||
return _isUserAnnotation(worldOrUniverse.schema, name);
|
||||
}
|
||||
|
||||
export function removeObsAnnoSchema(schema, name) {
|
||||
/*
|
||||
remove named annotation from obs annotation schema
|
||||
*/
|
||||
|
||||
/* only remove if it exists and is a user annotation */
|
||||
if (!_isUserAnnotation(schema, name))
|
||||
throw new Error("removing non-user-defined schema");
|
||||
return SchemaHelpers.removeObsAnnoColumn(schema, name);
|
||||
}
|
||||
|
||||
export function addObsAnnoSchema(schema, name, colSchema) {
|
||||
/*
|
||||
add a categorical type to the obs annotation schema
|
||||
*/
|
||||
|
||||
/* collision detection */
|
||||
if (schema.annotations.obs.columns.some((v) => v.name === name))
|
||||
throw Error("annotations may not contain duplicate category names");
|
||||
if (name !== colSchema.name) throw Error("column schema does not match");
|
||||
return SchemaHelpers.addObsAnnoColumn(schema, name, colSchema);
|
||||
}
|
||||
|
||||
export function dupObsAnnoSchema(schema, sourceName, dupName, defaultSchema) {
|
||||
/*
|
||||
duplicate the obs annotation `sourceName` schema, but with the name `dupName`
|
||||
*/
|
||||
const colSchema = {
|
||||
...schema.annotations.obsByName[sourceName],
|
||||
...defaultSchema,
|
||||
name: dupName,
|
||||
};
|
||||
/* existance check */
|
||||
if (!colSchema) throw Error("source annotation does not exist");
|
||||
/* collision detection */
|
||||
if (schema.annotations.obs.columns.some((v) => v.name === dupName))
|
||||
throw Error("annotations may not contain duplicate category names");
|
||||
return SchemaHelpers.addObsAnnoColumn(schema, dupName, colSchema);
|
||||
}
|
||||
|
||||
export function removeObsAnnoCategory(schema, name, category) {
|
||||
/* don't allow deletion of unassigned category on writable annotations */
|
||||
|
||||
if (!_isUserAnnotation(schema, name))
|
||||
throw new Error("unable to modify read-only schema");
|
||||
if (category === unassignedCategoryLabel)
|
||||
throw new Error("may not remove unassigned category label");
|
||||
|
||||
return SchemaHelpers.removeObsAnnoCategory(schema, name, category);
|
||||
}
|
||||
|
||||
export function addObsAnnoCategory(schema, name, category) {
|
||||
if (!_isUserAnnotation(schema, name))
|
||||
throw new Error("unable to modify read-only schema");
|
||||
|
||||
return SchemaHelpers.addObsAnnoCategory(schema, name, category);
|
||||
}
|
||||
|
||||
export function setLabelByValue(df, colName, fromLabel, toLabel) {
|
||||
/*
|
||||
in the dataframe column `colName`, set any value of `fromLabel` to `toLabel`
|
||||
*/
|
||||
const keys = df.colIndex.labels();
|
||||
const ndf = df.mapColumns((col, colIdx) => {
|
||||
if (colName !== keys[colIdx]) return col;
|
||||
|
||||
/* clone data and return it. */
|
||||
const newCol = col.slice();
|
||||
for (let i = 0, l = newCol.length; i < l; i += 1) {
|
||||
if (newCol[i] === fromLabel) newCol[i] = toLabel;
|
||||
}
|
||||
return newCol;
|
||||
});
|
||||
return ndf;
|
||||
}
|
||||
|
||||
export function setLabelByMask(df, colName, mask, label) {
|
||||
/*
|
||||
in the dataframe column `colName`, set the masked rows to 'label'
|
||||
*/
|
||||
const keys = df.colIndex.labels();
|
||||
const ndf = df.mapColumns((col, colIdx) => {
|
||||
if (colName !== keys[colIdx]) return col;
|
||||
|
||||
/* clone data and return it. */
|
||||
const newCol = col.slice();
|
||||
for (let i = 0, l = newCol.length; i < l; i += 1) {
|
||||
if (mask[i]) newCol[i] = label;
|
||||
}
|
||||
return newCol;
|
||||
});
|
||||
return ndf;
|
||||
export function isUserAnnotation(annoMatrix, name) {
|
||||
return _isUserAnnotation(annoMatrix.schema, name);
|
||||
}
|
||||
|
||||
export function allHaveLabelByMask(df, colName, label, mask) {
|
||||
@@ -148,41 +51,6 @@ export function allHaveLabelByMask(df, colName, label, mask) {
|
||||
return true;
|
||||
}
|
||||
|
||||
export function worldToUniverseMask(worldMask, worldObsAnnotations, nObs) {
|
||||
/*
|
||||
given world seleciton mask, return a selection mask for entire universe
|
||||
that has same selection state.
|
||||
*/
|
||||
const mask = new Uint8Array(nObs);
|
||||
const { rowIndex } = worldObsAnnotations;
|
||||
|
||||
for (let i = 0, l = worldMask.length; i < l; i += 1) {
|
||||
if (worldMask[i]) {
|
||||
const label = rowIndex.getLabel(i);
|
||||
mask[label] = 1;
|
||||
}
|
||||
}
|
||||
return mask;
|
||||
}
|
||||
|
||||
export function createWritableAnnotationDimensions(world, crossfilter) {
|
||||
const { obsAnnotations, schema } = world;
|
||||
const writableAnnotations = schema.annotations.obs.columns
|
||||
.filter((s) => s.writable)
|
||||
.map((s) => s.name);
|
||||
|
||||
crossfilter = writableAnnotations.reduce((xflt, anno) => {
|
||||
const dimName = obsAnnoDimensionName(anno);
|
||||
if (xflt.hasDimension(dimName)) xflt = xflt.delDimension(dimName);
|
||||
return xflt.addDimension(
|
||||
dimName,
|
||||
"enum",
|
||||
obsAnnotations.col(anno).asArray()
|
||||
);
|
||||
}, crossfilter);
|
||||
return crossfilter;
|
||||
}
|
||||
|
||||
const legalCharacters = /^(\w|[ .()-])+$/;
|
||||
export function annotationNameIsErroneous(name) {
|
||||
/*
|
||||
|
||||
@@ -3,46 +3,93 @@ Helper functions for the embedded graph colors
|
||||
*/
|
||||
import * as d3 from "d3";
|
||||
import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
|
||||
import memoize from "memoize-one";
|
||||
import * as globals from "../../globals";
|
||||
import parseRGB from "../parseRGB";
|
||||
import finiteExtent from "../finiteExtent";
|
||||
import { range } from "../range";
|
||||
|
||||
/*
|
||||
create new colors state object. Paramters:
|
||||
- world - current world object
|
||||
- colorMode - color-by mode. One of {null, "color by expression", "color by continuous metadata",
|
||||
"color by categorical metadata"}
|
||||
- colorAccessor - the obs annotations used for color-by
|
||||
given a color mode & accessor, generate an annoMatrix query that will
|
||||
fulfill it
|
||||
*/
|
||||
export function createColors(
|
||||
world,
|
||||
colorMode = null,
|
||||
colorAccessor = null,
|
||||
export function createColorQuery(colorMode, colorByAccessor, schema) {
|
||||
if (!colorMode || !colorByAccessor || !schema) return null;
|
||||
switch (colorMode) {
|
||||
case "color by categorical metadata":
|
||||
case "color by continuous metadata": {
|
||||
return ["obs", colorByAccessor];
|
||||
}
|
||||
case "color by expression": {
|
||||
const varIndex = schema?.annotations?.var?.index;
|
||||
if (!varIndex) return null;
|
||||
return [
|
||||
"X",
|
||||
{
|
||||
field: "var",
|
||||
column: varIndex,
|
||||
value: colorByAccessor,
|
||||
},
|
||||
];
|
||||
}
|
||||
default: {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function _defaultColors(nObs) {
|
||||
const defaultCellColor = parseRGB(globals.defaultCellColor);
|
||||
return {
|
||||
rgb: new Array(nObs).fill(defaultCellColor),
|
||||
scale: undefined,
|
||||
};
|
||||
}
|
||||
const defaultColors = memoize(_defaultColors);
|
||||
|
||||
/*
|
||||
create colors scale and RGB array and return as object. Parameters:
|
||||
* colorMode - categorical, etc.
|
||||
* colorByAccessor - the annotation label name
|
||||
* colorByDataframe - the actual color-by data
|
||||
* schema - the entire schema
|
||||
* userColors - optional user color table
|
||||
Returns:
|
||||
{
|
||||
scale: color scale
|
||||
rgb: cell to color mapping
|
||||
}
|
||||
*/
|
||||
function _createColorTable(
|
||||
colorMode,
|
||||
colorByAccessor,
|
||||
colorByData,
|
||||
schema,
|
||||
userColors = null
|
||||
) {
|
||||
switch (colorMode) {
|
||||
case "color by categorical metadata": {
|
||||
if (userColors && colorAccessor in userColors) {
|
||||
return createUserColors(world, colorAccessor, userColors);
|
||||
const data = colorByData.col(colorByAccessor).asArray();
|
||||
if (userColors && colorByAccessor in userColors) {
|
||||
return createUserColors(data, colorByAccessor, userColors);
|
||||
}
|
||||
return createColorsByCategoricalMetadata(world, colorAccessor);
|
||||
return createColorsByCategoricalMetadata(data, colorByAccessor, schema);
|
||||
}
|
||||
case "color by continuous metadata": {
|
||||
return createColorsByContinuousMetadata(world, colorAccessor);
|
||||
const col = colorByData.col(colorByAccessor);
|
||||
const { min, max } = col.summarize();
|
||||
return createColorsByContinuousMetadata(col.asArray(), min, max);
|
||||
}
|
||||
case "color by expression": {
|
||||
return createColorsByExpression(world, colorAccessor);
|
||||
const col = colorByData.icol(0);
|
||||
const { min, max } = col.summarize();
|
||||
return createColorsByContinuousMetadata(col.asArray(), min, max);
|
||||
}
|
||||
default: {
|
||||
const defaultCellColor = parseRGB(globals.defaultCellColor);
|
||||
return {
|
||||
rgb: new Array(world.nObs).fill(defaultCellColor),
|
||||
scale: undefined,
|
||||
};
|
||||
return defaultColors(schema.dataframe.nObs);
|
||||
}
|
||||
}
|
||||
}
|
||||
export const createColorTable = memoize(_createColorTable);
|
||||
|
||||
export function loadUserColorConfig(userColors) {
|
||||
const convertedUserColors = {};
|
||||
@@ -62,14 +109,15 @@ export function loadUserColorConfig(userColors) {
|
||||
return convertedUserColors;
|
||||
}
|
||||
|
||||
function createUserColors(world, colorAccessor, userColors) {
|
||||
function _createUserColors(data, colorAccessor, userColors) {
|
||||
const { colors, scale } = userColors[colorAccessor];
|
||||
const rgb = createRgbArray(world, colors, colorAccessor);
|
||||
const rgb = createRgbArray(data, colors);
|
||||
return { rgb, scale };
|
||||
}
|
||||
const createUserColors = memoize(_createUserColors);
|
||||
|
||||
function createColorsByCategoricalMetadata(world, colorAccessor) {
|
||||
const { categories } = world.schema.annotations.obsByName[colorAccessor];
|
||||
function _createColorsByCategoricalMetadata(data, colorAccessor, schema) {
|
||||
const { categories } = schema.annotations.obsByName[colorAccessor];
|
||||
|
||||
const scale = d3
|
||||
.scaleSequential(interpolateRainbow)
|
||||
@@ -81,25 +129,24 @@ function createColorsByCategoricalMetadata(world, colorAccessor) {
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const rgb = createRgbArray(world, colors, colorAccessor);
|
||||
const rgb = createRgbArray(data, colors);
|
||||
return { rgb, scale };
|
||||
}
|
||||
const createColorsByCategoricalMetadata = memoize(
|
||||
_createColorsByCategoricalMetadata
|
||||
);
|
||||
|
||||
export function createRgbArray(world, colors, colorAccessor) {
|
||||
const rgb = new Array(world.nObs);
|
||||
const df = world.obsAnnotations;
|
||||
const data = df.col(colorAccessor).asArray();
|
||||
for (let i = 0, len = df.length; i < len; i += 1) {
|
||||
function createRgbArray(data, colors) {
|
||||
const rgb = new Array(data.length);
|
||||
for (let i = 0, len = data.length; i < len; i += 1) {
|
||||
const label = data[i];
|
||||
rgb[i] = colors[label];
|
||||
}
|
||||
return rgb;
|
||||
}
|
||||
|
||||
function createColorsByContinuousMetadata(world, accessor) {
|
||||
function _createColorsByContinuousMetadata(data, min, max) {
|
||||
const colorBins = 100;
|
||||
const col = world.obsAnnotations.col(accessor);
|
||||
const { min, max } = col.summarize();
|
||||
const scale = d3
|
||||
.scaleQuantile()
|
||||
.domain([min, max])
|
||||
@@ -112,9 +159,8 @@ function createColorsByContinuousMetadata(world, accessor) {
|
||||
}
|
||||
|
||||
const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
|
||||
const rgb = new Array(world.nObs);
|
||||
const data = col.asArray();
|
||||
for (let i = 0, len = world.obsAnnotations.length; i < len; i += 1) {
|
||||
const rgb = new Array(data.length);
|
||||
for (let i = 0, len = data.length; i < len; i += 1) {
|
||||
const val = data[i];
|
||||
if (Number.isFinite(val)) {
|
||||
const c = scale(val);
|
||||
@@ -125,56 +171,6 @@ function createColorsByContinuousMetadata(world, accessor) {
|
||||
}
|
||||
return { rgb, scale };
|
||||
}
|
||||
|
||||
function createColorsByExpression(world, accessor) {
|
||||
const expression = world.varData.col(accessor).asArray();
|
||||
const colorBins = 100;
|
||||
const [min, max] = finiteExtent(expression);
|
||||
const scale = d3
|
||||
.scaleQuantile()
|
||||
.domain([min, max])
|
||||
.range(range(colorBins - 1, -1, -1));
|
||||
|
||||
/* pre-create colors - much faster than doing it for each obs */
|
||||
const colors = new Array(colorBins);
|
||||
for (let i = 0; i < colorBins; i += 1) {
|
||||
colors[i] = parseRGB(interpolateCool(i / colorBins));
|
||||
}
|
||||
const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
|
||||
|
||||
const rgb = new Array(world.nObs);
|
||||
for (let i = 0, len = expression.length; i < len; i += 1) {
|
||||
const e = expression[i];
|
||||
if (Number.isFinite(e)) {
|
||||
const c = scale(e);
|
||||
rgb[i] = colors[c];
|
||||
} else {
|
||||
rgb[i] = nonFiniteColor;
|
||||
}
|
||||
}
|
||||
return { rgb, scale };
|
||||
}
|
||||
|
||||
export const resetColors = (world) => {
|
||||
const { rgb, scale } = createColors(world);
|
||||
return {
|
||||
colorMode: null,
|
||||
colorAccessor: null,
|
||||
rgb,
|
||||
scale,
|
||||
};
|
||||
};
|
||||
|
||||
export const checkIfColorByDiffexpAndResetColors = (
|
||||
prevControls,
|
||||
state,
|
||||
prevWorld
|
||||
) => {
|
||||
if (prevControls.diffexpGenes.includes(state.colorAccessor)) {
|
||||
return {
|
||||
...state,
|
||||
...resetColors(prevWorld),
|
||||
};
|
||||
}
|
||||
return null;
|
||||
};
|
||||
export const createColorsByContinuousMetadata = memoize(
|
||||
_createColorsByContinuousMetadata
|
||||
);
|
||||
|
||||
@@ -7,10 +7,6 @@ import _ from "lodash";
|
||||
import * as globals from "../../globals";
|
||||
import { rangeFill as fillRange } from "../range";
|
||||
import fromEntries from "../fromEntries";
|
||||
import {
|
||||
userDefinedDimensionName,
|
||||
diffexpDimensionName,
|
||||
} from "../nameCreators";
|
||||
import { isCategoricalAnnotation } from "./annotationsHelpers";
|
||||
|
||||
/*
|
||||
@@ -40,7 +36,7 @@ Remember that option values can be ANY js type, except undefined/null.
|
||||
}
|
||||
*/
|
||||
function topNCategories(colSchema, summary, N) {
|
||||
/* return top N by occurrences in the data, preserving original category order */
|
||||
/* return top N by occurrences in the data */
|
||||
const { categories } = colSchema;
|
||||
const counts = categories.map((cat) => summary.categoryCounts.get(cat) ?? 0);
|
||||
|
||||
@@ -81,19 +77,16 @@ export function selectableCategoryNames(schema, names) {
|
||||
return names.filter((name) => isSelectableCategoryName(schema, name));
|
||||
}
|
||||
|
||||
export function createCategorySummary(world, name) {
|
||||
export function createCategorySummaryFromDfCol(dfCol, colSchema) {
|
||||
const N = globals.maxCategoricalOptionsToDisplay;
|
||||
const { obsAnnotations, schema } = world;
|
||||
|
||||
const colSchema = schema.annotations.obsByName[name];
|
||||
const { writable: isUserAnno } = colSchema;
|
||||
|
||||
/*
|
||||
Summarize the annotation data currently in world. Must return categoryValues
|
||||
in sorted order, and must include all category values even if they are not
|
||||
actively used in the current world.
|
||||
Summarize the annotation data currently in dataframe column. Must return
|
||||
categoryValues in sorted order, and must include all category values even
|
||||
if they are not actively used in the current annoMatrix view.
|
||||
*/
|
||||
const summary = obsAnnotations.col(name).summarizeCategorical();
|
||||
const summary = dfCol.summarizeCategorical();
|
||||
const [categoryValues, categoryValueCounts] = topNCategories(
|
||||
colSchema,
|
||||
summary,
|
||||
@@ -117,38 +110,6 @@ export function createCategoricalSelection(names) {
|
||||
return fromEntries(names.map((name) => [name, new Map()]));
|
||||
}
|
||||
|
||||
/*
|
||||
build a crossfilter dimensions for all gene expression related dimensions.
|
||||
*/
|
||||
export function createGeneDimensions(
|
||||
userDefinedGenes,
|
||||
diffexpGenes,
|
||||
world,
|
||||
crossfilter
|
||||
) {
|
||||
crossfilter = userDefinedGenes.reduce(
|
||||
(xflt, gene) =>
|
||||
xflt.addDimension(
|
||||
userDefinedDimensionName(gene),
|
||||
"scalar",
|
||||
world.varData.col(gene).asArray(),
|
||||
Float32Array
|
||||
),
|
||||
crossfilter
|
||||
);
|
||||
crossfilter = diffexpGenes.reduce(
|
||||
(xflt, gene) =>
|
||||
xflt.addDimension(
|
||||
diffexpDimensionName(gene),
|
||||
"scalar",
|
||||
world.varData.col(gene).asArray(),
|
||||
Float32Array
|
||||
),
|
||||
crossfilter
|
||||
);
|
||||
return crossfilter;
|
||||
}
|
||||
|
||||
export function pruneVarDataCache(varData, needed) {
|
||||
/*
|
||||
Remove any unneeded columns from the varData dataframe. Will only
|
||||
|
||||
@@ -1,22 +1,10 @@
|
||||
// jshint esversion: 6
|
||||
|
||||
/*
|
||||
Model manager providing an abstraction for the use of the reducer code.
|
||||
This module provides several buckets of functionality:
|
||||
- schema and config driven tranformation of the wire protocol
|
||||
into a format that is easy for the UI code to use.
|
||||
- manage the universe/world abstraction:
|
||||
+ universe: all of the server-provided, read-only data
|
||||
+ world: subset of universe
|
||||
- lazy access and caching of dataframe contents as needed
|
||||
|
||||
This is all VERY tightly integrated with reducers and actions, and
|
||||
exists to support those concepts.
|
||||
*/
|
||||
|
||||
export * as ColorHelpers from "./colorHelpers";
|
||||
export * as Universe from "./universe";
|
||||
export * as World from "./world";
|
||||
export * as ControlsHelpers from "./controlsHelpers";
|
||||
export * as AnnotationsHelpers from "./annotationsHelpers";
|
||||
export * as SchemaHelpers from "./schemaHelpers";
|
||||
|
||||
@@ -13,22 +13,21 @@ import catLabelSort from "../catLabelSort";
|
||||
System wide schema assumptions:
|
||||
- schema and data wil be consistent (eg, for user-created annotations)
|
||||
- schema will be internally self-consistent (eg, index matches columns)
|
||||
- world & universe schema are same - only data is subset
|
||||
*/
|
||||
|
||||
export function indexEntireSchema(schema) {
|
||||
/* Index schema for ease of use */
|
||||
schema.annotations.obsByName = fromEntries(
|
||||
schema.annotations.obs.columns.map((v) => [v.name, v])
|
||||
schema.annotations?.obs?.columns?.map((v) => [v.name, v]) ?? []
|
||||
);
|
||||
schema.annotations.varByName = fromEntries(
|
||||
schema.annotations.var.columns.map((v) => [v.name, v])
|
||||
schema.annotations?.var?.columns?.map((v) => [v.name, v]) ?? []
|
||||
);
|
||||
schema.layout.obsByName = fromEntries(
|
||||
schema.layout.obs.map((v) => [v.name, v])
|
||||
schema.layout?.obs?.map((v) => [v.name, v]) ?? []
|
||||
);
|
||||
schema.layout.varByName = fromEntries(
|
||||
schema.layout.var.map((v) => [v.name, v])
|
||||
schema.layout?.var?.map((v) => [v.name, v]) ?? []
|
||||
);
|
||||
|
||||
return schema;
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
import { unassignedCategoryLabel } from "../../globals";
|
||||
import { decodeMatrixFBS } from "./matrix";
|
||||
import * as Dataframe from "../dataframe";
|
||||
import { isFpTypedArray } from "../typeHelpers";
|
||||
import { indexEntireSchema } from "./schemaHelpers";
|
||||
import catLabelSort from "../catLabelSort";
|
||||
|
||||
/*
|
||||
Private helper function - create and return a template Universe
|
||||
*/
|
||||
function templateUniverse() {
|
||||
/* default universe template */
|
||||
return {
|
||||
nObs: 0,
|
||||
nVar: 0,
|
||||
schema: {},
|
||||
|
||||
/*
|
||||
annotations
|
||||
*/
|
||||
obsAnnotations: Dataframe.Dataframe.empty(),
|
||||
varAnnotations: Dataframe.Dataframe.empty(),
|
||||
/*
|
||||
layout
|
||||
*/
|
||||
obsLayout: Dataframe.Dataframe.empty(),
|
||||
|
||||
/*
|
||||
Var data columns - subset of all
|
||||
*/
|
||||
varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex()),
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
This module implements functions that support storage of "Universe",
|
||||
aka all of the var/obs data and annotations.
|
||||
|
||||
These functions are used exclusively by the actions and reducers to
|
||||
build an internal POJO for use by the rendering components.
|
||||
*/
|
||||
|
||||
export function createUniverseFromResponse(configResponse, schemaResponse) {
|
||||
/*
|
||||
build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
|
||||
*/
|
||||
const { schema } = schemaResponse;
|
||||
const universe = templateUniverse();
|
||||
|
||||
/* schema related */
|
||||
universe.schema = schema;
|
||||
universe.nObs = schema.dataframe.nObs;
|
||||
universe.nVar = schema.dataframe.nVar;
|
||||
|
||||
/* add defaults, as we can't assume back-end will fully populate schema */
|
||||
if (!schema.layout.var) schema.layout.var = [];
|
||||
if (!schema.layout.obs) schema.layout.obs = [];
|
||||
indexEntireSchema(universe.schema);
|
||||
normalizeEntireSchema(universe.schema);
|
||||
|
||||
return universe;
|
||||
}
|
||||
|
||||
function normalizeSchemaCategory(colSchema, col = undefined) {
|
||||
const { type, writable } = colSchema;
|
||||
if (type === "string" || type === "boolean" || type === "categorical") {
|
||||
let categories = [
|
||||
...new Set([
|
||||
...(colSchema.categories ?? []),
|
||||
...(col?.summarize?.().categories ?? []),
|
||||
]),
|
||||
];
|
||||
if (writable && categories.indexOf(unassignedCategoryLabel) === -1) {
|
||||
categories = categories.concat(unassignedCategoryLabel);
|
||||
}
|
||||
colSchema.categories = categories;
|
||||
} else if (writable) {
|
||||
throw new Error(
|
||||
"Writable continuous obs annotations are not supported - failed to load"
|
||||
);
|
||||
}
|
||||
|
||||
if (colSchema.categories) {
|
||||
colSchema.categories = catLabelSort(writable, colSchema.categories);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeEntireSchema(schema) {
|
||||
// currently only needed for obsAnnotations
|
||||
schema.annotations.obs.columns.forEach((colSchema) =>
|
||||
normalizeSchemaCategory(colSchema)
|
||||
);
|
||||
}
|
||||
|
||||
export function addObsAnnotations(universe, df) {
|
||||
const obsAnnotations = universe.obsAnnotations.withColsFromAll(df);
|
||||
if (universe.nObs !== obsAnnotations.length) {
|
||||
throw new Error("Universe dimensionality mismatch - failed to load");
|
||||
}
|
||||
|
||||
// for all of the new data, reconcile with schema and sort categories.
|
||||
const dfs = Array.isArray(df) ? df : [df];
|
||||
const keys = dfs.map((d) => d.colIndex.labels()).flat();
|
||||
const { schema } = universe;
|
||||
keys.forEach((k) => {
|
||||
const colSchema = schema.annotations.obsByName[k];
|
||||
const col = obsAnnotations.col(k);
|
||||
normalizeSchemaCategory(colSchema, col);
|
||||
});
|
||||
|
||||
return { obsAnnotations, schema };
|
||||
}
|
||||
|
||||
export function addVarAnnotations(universe, df) {
|
||||
const varAnnotations = universe.varAnnotations.withColsFromAll(df);
|
||||
if (universe.nVar !== varAnnotations.length) {
|
||||
throw new Error("Universe dimensionality mismatch - failed to load");
|
||||
}
|
||||
return { varAnnotations };
|
||||
}
|
||||
|
||||
export function addObsLayout(universe, df) {
|
||||
const obsLayout = universe.obsLayout.withColsFromAll(df);
|
||||
if (universe.nObs !== obsLayout.length) {
|
||||
throw new Error("Universe dimensionality mismatch - failed to load");
|
||||
}
|
||||
return { obsLayout };
|
||||
}
|
||||
|
||||
export function convertDataFBStoObject(universe, arrayBuffer) {
|
||||
/*
|
||||
/data/var returns a flatbuffer (FBS) as described by cellxgene/fbs/matrix.fbs
|
||||
|
||||
This routine converts the binary wire encoding into a JS object:
|
||||
|
||||
{
|
||||
gene: Float32Array,
|
||||
...
|
||||
}
|
||||
*/
|
||||
const fbs = decodeMatrixFBS(arrayBuffer);
|
||||
const { colIdx, columns } = fbs;
|
||||
const result = {};
|
||||
|
||||
if (!columns.every(isFpTypedArray)) {
|
||||
// We have strong assumptions that all var data is float
|
||||
throw new Error("Unexpected non-floating point response from server.");
|
||||
}
|
||||
|
||||
const varIndexName = universe.schema.annotations.var.index;
|
||||
for (let c = 0; c < colIdx.length; c += 1) {
|
||||
const varName = universe.varAnnotations.at(colIdx[c], varIndexName);
|
||||
result[varName] = columns[c];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -1,335 +0,0 @@
|
||||
import clip from "../clip";
|
||||
import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
|
||||
import * as Dataframe from "../dataframe";
|
||||
import { isContinuousAnnotation } from "./annotationsHelpers";
|
||||
|
||||
/*
|
||||
|
||||
World is a subset of universe. Most code should use world, and should
|
||||
(generally) not use Universe. World contains any per-obs or per-var data
|
||||
that must be consistent acorss the app when we view/manipulate subsets
|
||||
of Universe.
|
||||
|
||||
Private API indicated by leading underscore in key name (eg, _foo). Anything else
|
||||
is public.
|
||||
|
||||
Notable keys in the world object:
|
||||
|
||||
* nObs, nVar: dimensions
|
||||
|
||||
* schema: data schema from the server
|
||||
|
||||
* clipQuantiles: the quantiles used to clip all data in world.
|
||||
|
||||
* obsAnnotations:
|
||||
|
||||
Dataframe containing obs annotations. Columns are indexed by annotation
|
||||
name (eg, 'tissue type'), and rows are indexed by the REST API obsIndex
|
||||
(ie, the offset into the underlying server-side dataframe).
|
||||
|
||||
This indexing means that you can access data by _either_ the server's
|
||||
obxIndex, or the offset into the client-side column array . Be careful
|
||||
to know which you want and are using.
|
||||
|
||||
* obsLayout:
|
||||
|
||||
A dataframe containing the X/Y layout for all obs. Columns are named
|
||||
'X' and 'Y', and rows are indexed in the same way as obsAnnotation.
|
||||
|
||||
* varData: a cache of expression columns, stored in a Dataframe. Cache
|
||||
managed by controls reducer.
|
||||
|
||||
* unclipped: will contain unclipped variants of all potentiall clipped
|
||||
dataframes (obsAnnotations, varData).
|
||||
|
||||
*/
|
||||
|
||||
function templateWorld() {
|
||||
const obsAnnotations = Dataframe.Dataframe.empty();
|
||||
const varAnnotations = Dataframe.Dataframe.empty();
|
||||
const obsLayout = Dataframe.Dataframe.empty();
|
||||
const varData = Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex());
|
||||
return {
|
||||
/* schema/version related */
|
||||
schema: null,
|
||||
nObs: 0,
|
||||
nVar: 0,
|
||||
clipQuantiles: { min: 0, max: 1 },
|
||||
|
||||
/* annotations */
|
||||
obsAnnotations,
|
||||
varAnnotations,
|
||||
|
||||
/* layout of graph. Dataframe. */
|
||||
obsLayout,
|
||||
|
||||
/* Var data columns - subset of all data (may be empty) */
|
||||
varData,
|
||||
|
||||
/* unclipped dataframes - subset, but not value clipped */
|
||||
unclipped: {
|
||||
obsAnnotations,
|
||||
varData,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function clipDataframe(
|
||||
df,
|
||||
lowerQuantile,
|
||||
upperQuantile,
|
||||
quantileF,
|
||||
clipPredicate = () => true,
|
||||
value = Number.NaN
|
||||
) {
|
||||
/*
|
||||
For all columns in the dataframe, clip all values above or below specified
|
||||
quantiles to `value` if clipPredicate returns True for that column (if it
|
||||
returns false, skip the column entirely).
|
||||
|
||||
Returns a clipped copy - does not mutate original.
|
||||
|
||||
clipPredicate must have signature: (dataframe, colIndex, colLabel) => boolean
|
||||
True signifies that the column should be clipped; false indicates that the
|
||||
column should be left intact/unchanged.
|
||||
|
||||
quantileF must have signature: (label, qval) => number
|
||||
*/
|
||||
if (lowerQuantile < 0) lowerQuantile = 0;
|
||||
if (upperQuantile > 1) upperQuantile = 1;
|
||||
if (lowerQuantile === 0 && upperQuantile === 1) return df;
|
||||
|
||||
const keys = df.colIndex.labels();
|
||||
return df.mapColumns((col, colIdx) => {
|
||||
const colLabel = keys[colIdx];
|
||||
if (!clipPredicate(df, colIdx, colLabel)) return col;
|
||||
|
||||
const colMin = quantileF(colLabel, lowerQuantile);
|
||||
const colMax = quantileF(colLabel, upperQuantile);
|
||||
const newCol = clip(col.slice(), colMin, colMax, value);
|
||||
return newCol;
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
Create World with contents eq entire universe. Commonly used to initialize World.
|
||||
*/
|
||||
export function createWorldFromEntireUniverse(universe) {
|
||||
const world = templateWorld();
|
||||
|
||||
/* Schema related */
|
||||
world.schema = universe.schema;
|
||||
world.nObs = universe.nObs;
|
||||
world.nVar = universe.nVar;
|
||||
world.clipQuantiles = { min: 0, max: 1 };
|
||||
|
||||
/* dataframes: annotations and layout */
|
||||
world.obsAnnotations = universe.obsAnnotations.clone();
|
||||
world.varAnnotations = universe.varAnnotations.clone();
|
||||
world.obsLayout = universe.obsLayout.clone();
|
||||
|
||||
/* Var dataframe - contains a subset of all var columns */
|
||||
world.varData = universe.varData.clone();
|
||||
|
||||
/* save unclipped copies of potentially clipped dataframes */
|
||||
world.unclipped = {
|
||||
obsAnnotations: world.obsAnnotations.clone(),
|
||||
varData: world.varData.clone(),
|
||||
};
|
||||
|
||||
return world;
|
||||
}
|
||||
|
||||
/*
|
||||
clip dataframes based on quantiles.
|
||||
|
||||
This is an in-place operation on the world object provided as an argument.
|
||||
The values in world.unclipped are clipped and assigned to world.obsAnnotations
|
||||
and world.varData.
|
||||
*/
|
||||
function setClippedDataframes(world) {
|
||||
const { schema } = world;
|
||||
const isContinuousObsAnnotation = (df, idx, label) =>
|
||||
isContinuousAnnotation(schema, label);
|
||||
const obsQuantile = (label, q) =>
|
||||
world.unclipped.obsAnnotations.col(label).summarize().percentiles[100 * q];
|
||||
world.obsAnnotations = clipDataframe(
|
||||
world.unclipped.obsAnnotations,
|
||||
world.clipQuantiles.min,
|
||||
world.clipQuantiles.max,
|
||||
obsQuantile,
|
||||
isContinuousObsAnnotation
|
||||
);
|
||||
|
||||
const varDataQuantile = (label, q) =>
|
||||
world.unclipped.varData.col(label).summarize().percentiles[100 * q];
|
||||
world.varData = clipDataframe(
|
||||
world.unclipped.varData,
|
||||
world.clipQuantiles.min,
|
||||
world.clipQuantiles.max,
|
||||
varDataQuantile,
|
||||
() => true
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
Subset the current world based upon the current selection, maintaining any existing
|
||||
clip. Returns new world. Parameters:
|
||||
* universe
|
||||
* world - the current world
|
||||
* crossfilter - the selection state
|
||||
*/
|
||||
export function createWorldBySelection(universe, world, crossfilter) {
|
||||
const newWorld = { ...world, obsLayout: null, unclipped: {}, varData: null };
|
||||
|
||||
/* subset unclipped dataframes based upon current selection */
|
||||
const mask = crossfilter.allSelectedMask();
|
||||
newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
|
||||
newWorld.unclipped.obsAnnotations = world.unclipped.obsAnnotations.isubsetMask(
|
||||
mask
|
||||
);
|
||||
if (world.unclipped.varData.isEmpty()) {
|
||||
newWorld.unclipped.varData = world.unclipped.varData.clone();
|
||||
} else {
|
||||
newWorld.unclipped.varData = world.unclipped.varData.isubsetMask(mask);
|
||||
}
|
||||
/* subsetting changings dimension size */
|
||||
newWorld.nObs = newWorld.unclipped.obsAnnotations.dims[0];
|
||||
|
||||
/* and now clip */
|
||||
setClippedDataframes(newWorld);
|
||||
return newWorld;
|
||||
}
|
||||
|
||||
/*
|
||||
Change clip quantiles on the current world, returning a new world.
|
||||
Parameters:
|
||||
* universe
|
||||
* world - current world
|
||||
* clipQuantiles - new clip
|
||||
*/
|
||||
export function createWorldWithNewClip(
|
||||
universe,
|
||||
world,
|
||||
crossfilter,
|
||||
clipQuantiles
|
||||
) {
|
||||
const newWorld = { ...world, obsAnnotation: null, varData: null };
|
||||
newWorld.clipQuantiles = clipQuantiles;
|
||||
newWorld.obsLayout = world.obsLayout.clone();
|
||||
newWorld.unclipped = {
|
||||
obsAnnotations: world.unclipped.obsAnnotations.clone(),
|
||||
varData: world.unclipped.varData.clone(),
|
||||
};
|
||||
|
||||
/* and now clip */
|
||||
setClippedDataframes(newWorld);
|
||||
return newWorld;
|
||||
}
|
||||
|
||||
/*
|
||||
Deduce the correct crossfilter dimension type from a metadata
|
||||
schema description.
|
||||
*/
|
||||
function deduceDimensionType(attributes, fieldName) {
|
||||
let dimensionType;
|
||||
const { type } = attributes;
|
||||
if (type === "string" || type === "categorical" || type === "boolean") {
|
||||
dimensionType = "enum";
|
||||
} else if (type === "int32") {
|
||||
dimensionType = Int32Array;
|
||||
} else if (type === "float32") {
|
||||
dimensionType = Float32Array;
|
||||
} else {
|
||||
/*
|
||||
Currently not supporting boolean and categorical types.
|
||||
*/
|
||||
console.error(
|
||||
`Warning - REST API returned unknown metadata schema (${type}) for field ${fieldName}.`
|
||||
);
|
||||
// skip it - we don't know what to do with this type
|
||||
}
|
||||
return dimensionType;
|
||||
}
|
||||
|
||||
function addObsDimension(crossfilter, world, anno) {
|
||||
/*
|
||||
add single dimension to the crosfilter
|
||||
*/
|
||||
const { obsAnnotations } = world;
|
||||
if (obsAnnotations.hasCol(anno.name)) {
|
||||
const dimType = deduceDimensionType(anno, anno.name);
|
||||
const colData = obsAnnotations.col(anno.name).asArray();
|
||||
const name = obsAnnoDimensionName(anno.name);
|
||||
if (dimType === "enum") {
|
||||
return crossfilter.addDimension(name, "enum", colData);
|
||||
}
|
||||
if (dimType) {
|
||||
return crossfilter.addDimension(name, "scalar", colData, dimType);
|
||||
}
|
||||
}
|
||||
return crossfilter;
|
||||
}
|
||||
|
||||
export function addObsDimensions(crossfilter, world) {
|
||||
/*
|
||||
Add to crossfilter any dimension present in world.obsAnnotations
|
||||
but not yet in the crossfilter
|
||||
*/
|
||||
const schema = world.schema.annotations.obsByName;
|
||||
const dimsWeNeed = world.obsAnnotations.colIndex.labels();
|
||||
crossfilter = dimsWeNeed.reduce((xfltr, name) => {
|
||||
const dimName = obsAnnoDimensionName(name);
|
||||
if (xfltr.hasDimension(dimName)) return xfltr;
|
||||
return addObsDimension(xfltr, world, schema[name]);
|
||||
}, crossfilter);
|
||||
return crossfilter;
|
||||
}
|
||||
|
||||
export function createObsDimensions(crossfilter, world, XYdimNames) {
|
||||
/*
|
||||
create and return a crossfilter with a dimension for every obs annotation
|
||||
for which we have a supported type, *except* for the index column, indicated
|
||||
by schema.annotations.obs.index.
|
||||
*/
|
||||
const { schema, obsLayout } = world;
|
||||
const indexName = schema.annotations.obs.index;
|
||||
const annoList = schema.annotations.obs.columns.filter(
|
||||
(anno) => anno.name !== indexName
|
||||
);
|
||||
crossfilter = annoList.reduce((xfltr, anno) => {
|
||||
return addObsDimension(xfltr, world, anno);
|
||||
}, crossfilter);
|
||||
|
||||
return crossfilter.addDimension(
|
||||
layoutDimensionName("XY"),
|
||||
"spatial",
|
||||
obsLayout.col(XYdimNames[0]).asArray(),
|
||||
obsLayout.col(XYdimNames[1]).asArray()
|
||||
);
|
||||
}
|
||||
|
||||
export function worldEqUniverse(world, universe) {
|
||||
return (
|
||||
world.obsAnnotations === universe.obsAnnotations ||
|
||||
world.obsAnnotations.rowIndex === universe.obsAnnotations.rowIndex
|
||||
);
|
||||
}
|
||||
|
||||
export function getSelectedByIndex(crossfilter) {
|
||||
/*
|
||||
return array of obsIndex, containing all selected obs/cells.
|
||||
*/
|
||||
const selected = crossfilter.allSelectedMask(); // array of bool-ish
|
||||
const keys = crossfilter.data.rowIndex.labels(); // row keys, aka universe rowIndex
|
||||
|
||||
const set = new Int32Array(selected.length);
|
||||
let numElems = 0;
|
||||
for (let i = 0, l = selected.length; i < l; i += 1) {
|
||||
if (selected[i]) {
|
||||
set[numElems] = keys[i];
|
||||
numElems += 1;
|
||||
}
|
||||
}
|
||||
return new Int32Array(set.buffer, 0, numElems);
|
||||
}
|
||||
Reference in New Issue
Block a user