fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-25 10:48:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Clip continuous values based on percentile cutoffs (#672)

Browse Source 
* Add numeric inputs for percentiles

* Define initial values for percentile cutoffs in world reducer

* add percentil to crossfilter dimensions

* worldEqUniverse now handles cloned worlds

* add Dataframe.mapColumns

* Wire up handlers for percentile inputs

* World reducer and stateManager know about continuousPercentileMin/Max

* Create world as universe clone (not pointer) to avoid clobbering vals

* Define basic actions for setting continuousPercentileMin/Max

* Under the hood, deal with percentiles between 0 and 1

* Move percentile inputs to visualization settings menu

* Fix padding for undo/redo buttons

* Trigger world rebuild from percentile actions

* BROKEN - pseudocode for clamping dataframe by percentiles upon world rebuild

* fix error handling on clip quantiles; start world clipping implementation

* more unclipped reorg

* rename crossfilter.percentile to quantile

* simplify schema access

* update continuous legend when scale changes

* update color cache when clip changes

* clip obs annotations and var data when clip quantile changes

* use own fromEntries

* fix tests

* stable non-finite float sort/search

* clarify comments

* fix syntax typo

* use new stand-alone clip

* clip expresssion data

* add select tests for non-finite scalars

* basic styles

* clip UI now requires explicit commit

* reset enable/disable accounts for clip percentiles

* better error messages

* fix bug in undo interaction with programatic min brush selection

* small refactoring

* support clipping of int data

* do not perform unnecessary summarizations

* improve caching of dataframe compiled columns

* add percentile precompute to Dataframe.summarize

* use Dataframe.summarize for clip percentiles

* remove obsolete quantile code from corssfilter

* histogram scale and label Y axis, add unclipped X range labels

* layout tweaks

* scatterplot now updates when clip changes

* improve comments

* remove debugging comment

* rework clip number entry validation for usability

* ui tweaks to histogram colors and layout

* enable undo/redo for clip user action

* refine UI on clip value entry

* api cleanup

* update confusing comment

* clarify purpose of isValidDigitKeyEvent

* fix misleading comment

* apply appropriate button-group classes; do not mix span and div

* variable name and comment changes suggested in PR review

* rename sort to sortArray; remove unused and dead code path

* naming changes suggested in PR review

* code review improvements for clarity

* more small changes from PR review

* lint fixes for PR review

* fix spelling error

* clarify that function performs in-place modification of world

* add comment to clarify intent of range operation

* fix bad indents in comments

* clean up __columnsAccessor comments and code

* improve comments around clipPredicate

* field name consistency

* improve comment on quantiles params
This commit is contained in:
Sidney Bell
2019-04-30 16:20:10 -07:00
committed by Bruce Martin
parent 9f10d8095a
commit a08e19bbd0
34 changed files with 1842 additions and 551 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/util/clip.js
+25
View File
@@ -0,0 +1,25 @@
/*
clip - clip all values in a Array or TypedArray, IN PLACE.
Values in array are clipped if less than `lower` or greater than `upper`.
If `setTo` is undefined, values less than `lower` will be set to `lower`,
and values greater than `upper` will be set to `upper`.
If `setTo` is not undefined, values outside the [lower, upper] range will be set to
`setTo`.
*/
export default function clip(arr, lower, upper, setTo) {
const lowerSet = setTo === undefined ? lower : setTo;
const upperSet = setTo === undefined ? upper : setTo;
for (let i = 0, l = arr.length; i < l; i += 1) {
const v = arr[i];
if (v < lower) {
arr[i] = lowerSet;
} else if (v > upper) {
arr[i] = upperSet;
}
}
return arr;
}
client/src/util/dataframe/dataframe.js
+150 -95
View File
@@ -1,6 +1,6 @@
import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
// weird cross-dependency that we should clean up someday...
import { sort } from "../typedCrossfilter/sort";
import { sortArray } from "../typedCrossfilter/sort";
import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
import { summarizeContinuous, summarizeCategorical } from "./summarize";
@@ -63,7 +63,13 @@ class Dataframe {
Constructors & factories
**/
constructor(dims, columnarData, rowIndex = null, colIndex = null) {
constructor(
dims,
columnarData,
rowIndex = null,
colIndex = null,
__columnsAccessor = [] // private interface
) {
/*
The base constructor is relatively hard to use - as an alternative,
see factory methods and clone/slice, below.
@@ -74,6 +80,9 @@ class Dataframe {
or TypedArray of length nRows.
* rowIndex/colIndex - null (create default index using offsets as key),
or a caller-provided index.
* __columnsAccessor - private interface, do not specify. Used internally
to improve caching of column accessors when possible (eg, clone(),
dropCol(), withCol()).
All columns and indices must have appropriate dimensionality.
*/
const [nRows, nCols] = dims;
@@ -94,7 +103,7 @@ class Dataframe {
this.rowIndex = rowIndex;
this.colIndex = colIndex;
this.__compile();
this.__compile(__columnsAccessor);
}
static __errorChecks(dims, columnarData, rowIndex, colIndex) {
@@ -135,97 +144,107 @@ class Dataframe {
}
}
__compile() {
static __compileColumn(column, getOffset, getLabel) {
/*
Each column accessor is a function which will lookup data by
index (ie, is equivalent to dataframe.get(row, col), where 'col'
is fixed.
In addition, each column accessor has several functions:
asArray() -- return the entire column as a native Array or TypedArray.
Crucially, this native array only supports label indexing.
Example:
const arr = df.col('a').asArray();
has(rlabel) -- return boolean indicating of the row label
is contained within the column. Example:
const isInColumn = df.col('a').includes(99)
For the default offset indexing, this is identical to:
const isInColumn = (99 > 0) && (99 < df.nRows);
ihas(roffset) -- same as has(), but accepts a row offset
instead of a row label.
indexOf(value) -- return the label (not offset) of the first instance of
'value' in the column. If you want the offset, just use the builtin JS
indexOf() function, available on both Array and TypedArray.
iget(offset) -- return the value at 'offset'
*/
const { length } = column;
/* get value by row label */
const get = function get(rlabel) {
return column[getOffset(rlabel)];
};
/* get value by row offset */
const iget = function iget(roffset) {
return column[roffset];
};
/* full column array access */
const asArray = function asArray() {
return column;
};
/* test for row label inclusion in column */
const has = function has(rlabel) {
const offset = getOffset(rlabel);
return offset >= 0 && offset < length;
};
const ihas = function ihas(offset) {
return offset >= 0 && offset < length;
};
/*
return first label (index) at which the value is found in this column,
or undefined if not found.
NOTE: not found return is DIFFERENT than the default Array.indexOf as
-1 is a plausible Dataframe row/col label.
*/
const indexOf = function indexOf(value) {
const offset = column.indexOf(value);
if (offset === -1) {
return undefined;
}
return getLabel(offset);
};
/*
Summarize the column data. Lazy eval;
*/
const summarize = callOnceLazy(() =>
isTypedArray(column)
? summarizeContinuous(column)
: summarizeCategorical(column)
);
get.summarize = summarize;
get.asArray = asArray;
get.has = has;
get.ihas = ihas;
get.indexOf = indexOf;
get.iget = iget;
return get;
}
__compile(accessors) {
/*
Compile data accessors for each column.
Each column accessor is a function which will lookup data by
index (ie, is equivalent to dataframe.get(row, col), where 'col'
is fixed.
In addition, each column accessor has several functions:
asArray() -- return the entire column as a native Array or TypedArray.
Crucially, this native array only supports label indexing.
Example:
const arr = df.col('a').asArray();
has(rlabel) -- return boolean indicating of the row label
is contained within the column. Example:
const isInColumn = df.col('a').includes(99)
For the default offset indexing, this is identical to:
const isInColumn = (99 > 0) && (99 < df.nRows);
ihas(roffset) -- same as has(), but accepts a row offset
instead of a row label.
indexOf(value) -- return the label (not offset) of the first instance of
'value' in the column. If you want the offset, just use the builtin JS
indexOf() function, available on both Array and TypedArray.
iget(offset) -- return the value at 'offset'
Use an existing accessor if provided, else compile a new one.
*/
const { getOffset, getLabel } = this.rowIndex;
this.__columnsAccessor = this.__columns.map(column => {
const { length } = column;
/* get value by row label */
const get = function get(rlabel) {
return column[getOffset(rlabel)];
};
/* get value by row offset */
const iget = function iget(roffset) {
return column[roffset];
};
/* full column array access */
const asArray = function asArray() {
return column;
};
/* test for row label inclusion in column */
const has = function has(rlabel) {
const offset = getOffset(rlabel);
return offset >= 0 && offset < length;
};
const ihas = function ihas(offset) {
return offset >= 0 && offset < length;
};
/*
return first label (index) at which the value is found in this column,
or undefined if not found.
NOTE: not found return is DIFFERENT than the default Array.indexOf as
-1 is a plausible Dataframe row/col label.
*/
const indexOf = function indexOf(value) {
const offset = column.indexOf(value);
if (offset === -1) {
return undefined;
}
return getLabel(offset);
};
/*
Summarize the column data. Lazy eval;
*/
const summarize = callOnceLazy(() =>
isTypedArray(column)
? summarizeContinuous(column)
: summarizeCategorical(column)
);
get.summarize = summarize;
get.asArray = asArray;
get.has = has;
get.ihas = ihas;
get.indexOf = indexOf;
get.iget = iget;
return get;
this.__columnsAccessor = this.__columns.map((column, idx) => {
if (accessors[idx]) {
return accessors[idx];
}
return Dataframe.__compileColumn(column, getOffset, getLabel);
});
}
@@ -237,7 +256,8 @@ class Dataframe {
this.dims,
[...this.__columns],
this.rowIndex,
this.colIndex
this.colIndex,
[...this.__columnsAccessor]
);
}
@@ -273,7 +293,14 @@ class Dataframe {
const columns = [...this.__columns];
columns.push(colData);
const colIndex = this.colIndex.withLabel(label);
return new this.constructor(dims, columns, rowIndex, colIndex);
const columnsAccessor = [...this.__columnsAccessor];
return new this.constructor(
dims,
columns,
rowIndex,
colIndex,
columnsAccessor
);
}
dropCol(label) {
@@ -287,7 +314,15 @@ class Dataframe {
const columns = [...this.__columns];
columns.splice(coffset, 1);
const colIndex = this.colIndex.dropLabel(label);
return new this.constructor(dims, columns, this.rowIndex, colIndex);
const columnsAccessor = [...this.__columnsAccessor];
columnsAccessor.splice(coffset, 1);
return new this.constructor(
dims,
columns,
this.rowIndex,
colIndex,
columnsAccessor
);
}
static empty(rowIndex = null, colIndex = null) {
@@ -316,7 +351,7 @@ class Dataframe {
if (!offsets) {
return [null, null];
}
const sortedOffsets = sort(offsets);
const sortedOffsets = sortArray(offsets);
const sortedLabels = new Array(sortedOffsets.length);
for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
sortedLabels[i] = index.getLabel(sortedOffsets[i]);
@@ -543,14 +578,34 @@ class Dataframe {
/****
Functional (map/reduce/etc) data access
XXX: not yet implemented, as there is no clear use case. Can easily
TODO: most are not yet implemented, as there is no clear use case. Can easily
add these as useful.
****/
mapColumns(callback) {
/*
map all columns in the dataframe, returning a new dataframe comprised of the
return values, with the same index as the original dataframe.
callback MUST not modify the column, but instead return a mutated copy.
*/
const columns = this.__columns.map(callback);
const columnsAccessor = columns.map((c, idx) =>
this.__columns[idx] === c ? this.__columnsAccessor[idx] : undefined
);
return new this.constructor(
this.dims,
columns,
this.rowIndex,
this.colIndex,
columnsAccessor
);
}
/*
Map & reduce of column or row
XXX TODO remainder of map/reduce functions: mapCol, mapRow, reduceRow, ...
TODO remainder of map/reduce functions: mapCol, mapRow, reduceRow, ...
*/
/* comment out until we have a use for this
client/src/util/dataframe/summarize.js
+42 -17
View File
@@ -1,32 +1,56 @@
/*
Private dataframe support functions
TODO / XXX: for scalar/continuous data, this uses a naive method
of computing quantiles. Would be good to switch from sort to
partition at some point.
*/
import quantile from "../quantile";
import { sortArray } from "../typedCrossfilter/sort";
// [ 0, 0.01, 0.02, ..., 1.0]
const centileNames = new Array(101).fill(0).map((v, idx) => idx / 100);
export function summarizeContinuous(col) {
let min;
let max;
let nan = 0;
let pinf = 0;
let ninf = 0;
let percentiles;
if (col) {
for (let r = 0, l = col.length; r < l; r += 1) {
const val = Number(col[r]);
if (Number.isFinite(val)) {
if (min === undefined) {
min = val;
max = val;
} else {
min = val < min ? val : min;
max = val > max ? val : max;
}
} else if (Number.isNaN(val)) {
nan += 1;
} else if (val > 0) {
pinf += 1;
} else {
ninf += 1;
// -Inf < finite < Inf < NaN
const sortedCol = sortArray(new col.constructor(col));
// count non-finites, which are at each end of sorted data
for (let i = sortedCol.length - 1; i >= 0; i -= 1) {
if (!Number.isNaN(sortedCol[i])) {
nan = sortedCol.length - i - 1;
break;
}
}
for (let i = 0, l = sortedCol.length; i < l; i += 1) {
if (sortedCol[i] !== Number.NEGATIVE_INFINITY) {
ninf = i;
break;
}
}
for (let i = sortedCol.length - nan - 1; i >= 0; i -= 1) {
if (sortedCol[i] !== Number.POSITIVE_INFINITY) {
pinf = sortedCol.length - i - nan - 1;
break;
}
}
// compute percentiles on finite data ONLY
const sortedColFiniteOnly = sortedCol.slice(
ninf,
sortedCol.length - nan - pinf
);
percentiles = quantile(centileNames, sortedColFiniteOnly, true);
min = percentiles[0];
max = percentiles[100];
}
return {
categorical: false,
@@ -34,7 +58,8 @@ export function summarizeContinuous(col) {
max,
nan,
pinf,
ninf
ninf,
percentiles
};
}
client/src/util/dataframe/util.js
+1 -10
View File
@@ -2,16 +2,7 @@
Private utility code for dataframe
*/
export function isTypedArray(x) {
return (
ArrayBuffer.isView(x) &&
Object.prototype.toString.call(x) !== "[object DataView]"
);
}
export function isArrayOrTypedArray(x) {
return Array.isArray(x) || isTypedArray(x);
}
export { isTypedArray, isArrayOrTypedArray } from "../typeHelpers";
export function callOnceLazy(f) {
let value;
client/src/util/fromEntries.js
+13
View File
@@ -0,0 +1,13 @@
export default function fromEntries(arr) {
/*
Similar to Object.fromEntries, but only handles array.
This could be replaced with the standard fucnction once it
is widely available. As of 3/20/2019, it has not yet
been released in the Chrome stable channel.
*/
const obj = {};
for (let i = 0, l = arr.length; i < l; i += 1) {
obj[arr[i][0]] = arr[i][1];
}
return obj;
}
client/src/util/quantile.js
+29
View File
@@ -0,0 +1,29 @@
/*
quantiles - calculate quantiles for the typed array.
Currently interpolates to 'lower' value.
Arguments:
* quantArr - array of quantiles to compute, where values: 0 <= value <= 1.0
* tarr - a typed array
* sorted - option bool. If false (default), will assume array is not sorted.
If true, will assume it is sorted.
*/
import { sortArray } from "./typedCrossfilter/sort";
export default function quantile(quantArr, tarr, sorted = false) {
/*
start with the naive (sort) implementation. Later, use a faster partition
*/
const arr = sorted ? tarr : sortArray(new tarr.constructor(tarr)); // copy
const len = arr.length;
return quantArr.map(q => {
if (q === 1) {
return arr[len - 1];
}
return arr[Math.floor(q * len)];
});
}
client/src/util/stateManager/universe.js
+64 -2
View File
@@ -4,6 +4,8 @@ import _ from "lodash";
import decodeMatrixFBS from "./matrix";
import * as Dataframe from "../dataframe";
import fromEntries from "../fromEntries";
import { isFpTypedArray } from "../typeHelpers";
/*
Private helper function - create and return a template Universe
@@ -37,14 +39,57 @@ These functions are used exclusively by the actions and reducers to
build an internal POJO for use by the rendering components.
*/
function promoteTypedArray(o) {
/*
Decide what internal data type to use for the data returned from
the server.
TODO - future optimization: not all int32/uint32 data series require
promotion to float64. We COULD simply look at the data to decide.
*/
if (isFpTypedArray(o) || Array.isArray(o)) return o;
let TyepdArrayCtor;
switch (o.constructor) {
case Int8Array:
case Uint8Array:
case Uint8ClampedArray:
case Int16Array:
case Uint16Array:
TyepdArrayCtor = Float32Array;
break;
case Int32Array:
case Uint32Array:
TyepdArrayCtor = Float64Array;
break;
default:
throw new Error("Unexpected data type returned from server.");
}
if (o.constructor === TyepdArrayCtor) return o;
return new TyepdArrayCtor(o);
}
function AnnotationsFBSToDataframe(arrayBuffer) {
/*
Convert a Matrix FBS to a Dataframe.
The application has strong assumptions that all scalar data will be
stored as a float32 or float64 (regardless of underlying data types).
For example, clipping of value ranges (eg, user-selected percentiles)
All float data from the server is left as is. All non-float is promoted
to an appropriate float.
*/
const fbs = decodeMatrixFBS(arrayBuffer);
const fbs = decodeMatrixFBS(arrayBuffer, true); // leave in place
const columns = fbs.columns.map(c => {
if (isFpTypedArray(c) || Array.isArray(c)) return c;
return promoteTypedArray(c);
});
const df = new Dataframe.Dataframe(
[fbs.nRows, fbs.nCols],
fbs.columns,
columns,
null,
new Dataframe.KeyIndex(fbs.colIdx)
);
@@ -53,6 +98,10 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
function LayoutFBSToDataframe(arrayBuffer) {
const fbs = decodeMatrixFBS(arrayBuffer, true);
if (fbs.columns.length !== 2 || !fbs.columns.every(isFpTypedArray)) {
// We have strong assumptions about the shape & type of layout data.
throw new Error("Unexpected layout data type returned from server");
}
const df = new Dataframe.Dataframe(
[fbs.nRows, fbs.nCols],
fbs.columns,
@@ -122,6 +171,14 @@ export function createUniverseFromResponse(
}
reconcileSchemaCategoriesWithSummary(universe);
/* Index schema for ease of use */
universe.schema.annotations.obsByName = fromEntries(
universe.schema.annotations.obs.map(v => [v.name, v])
);
universe.schema.annotations.varByName = fromEntries(
universe.schema.annotations.var.map(v => [v.name, v])
);
return universe;
}
@@ -140,6 +197,11 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
const { colIdx, columns } = fbs;
const result = {};
if (!columns.every(isFpTypedArray)) {
// We have strong assumptions that all var data is float
throw new Error("Unexpected non-floating point response from server.");
}
for (let c = 0; c < colIdx.length; c += 1) {
const varName = universe.varAnnotations.at(colIdx[c], "name");
result[varName] = columns[c];
client/src/util/stateManager/world.js
+164 -38
View File
@@ -1,7 +1,14 @@
// jshint esversion: 6
import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
import clip from "../clip";
import {
layoutDimensionName,
obsAnnoDimensionName,
diffexpDimensionName,
userDefinedDimensionName
} from "../nameCreators";
import * as Dataframe from "../dataframe";
import ImmutableTypedCrossfilter from "../typedCrossfilter/crossfilter";
/*
@@ -19,6 +26,8 @@ Notable keys in the world object:
* schema: data schema from the server
* clipQuantiles: the quantiles used to clip all data in world.
* obsAnnotations:
Dataframe containing obs annotations. Columns are indexed by annotation
@@ -37,78 +46,192 @@ Notable keys in the world object:
* varData: a cache of expression columns, stored in a Dataframe. Cache
managed by controls reducer.
* unclipped: will contain unclipped variants of all potentiall clipped
dataframes (obsAnnotations, varData).
*/
function templateWorld() {
const obsAnnotations = Dataframe.Dataframe.empty();
const varAnnotations = Dataframe.Dataframe.empty();
const obsLayout = Dataframe.Dataframe.empty();
const varData = Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex());
return {
/* schema/version related */
schema: null,
nObs: 0,
nVar: 0,
clipQuantiles: { min: 0, max: 1 },
/* annotations */
obsAnnotations: Dataframe.Dataframe.empty(),
varAnnotations: Dataframe.Dataframe.empty(),
obsAnnotations,
varAnnotations,
/* layout of graph. Dataframe. */
obsLayout: Dataframe.Dataframe.empty(),
obsLayout,
/*
Var data columns - subset of all data (may be empty)
*/
varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
/* Var data columns - subset of all data (may be empty) */
varData,
/* unclipped dataframes - subset, but not value clipped */
unclipped: {
obsAnnotations,
varData
}
};
}
function clipDataframe(
df,
lowerQuantile,
upperQuantile,
quantileF,
clipPredicate = () => true,
value = Number.NaN
) {
/*
For all columns in the dataframe, clip all values above or below specified
quantiles to `value` if clipPredicate returns True for that column (if it
returns false, skip the column entirely).
Returns a clipped copy - does not mutate original.
clipPredicate must have signature: (dataframe, colIndex, colLabel) => boolean
True signifies that the column should be clipped; false indicates that the
column should be left intact/unchanged.
quantileF must have signature: (label, qval) => number
*/
if (lowerQuantile < 0) lowerQuantile = 0;
if (upperQuantile > 1) upperQuantile = 1;
if (lowerQuantile === 0 && upperQuantile === 1) return df;
const keys = df.colIndex.keys();
return df.mapColumns((col, colIdx) => {
const colLabel = keys[colIdx];
if (!clipPredicate(df, colIdx, colLabel)) return col;
const colMin = quantileF(colLabel, lowerQuantile);
const colMax = quantileF(colLabel, upperQuantile);
const newCol = clip(col.slice(), colMin, colMax, value);
return newCol;
});
}
/*
Create World with contents eq entire universe. Commonly used to initialize World.
If clipQuantiles
*/
export function createWorldFromEntireUniverse(universe) {
const world = templateWorld();
/*
public interface follows
*/
/* Schema related */
world.schema = universe.schema;
world.nObs = universe.nObs;
world.nVar = universe.nVar;
world.clipQuantiles = { min: 0, max: 1 };
/* annotation dataframes */
world.obsAnnotations = universe.obsAnnotations;
world.varAnnotations = universe.varAnnotations;
/* dataframes: annotations and layout */
world.obsAnnotations = universe.obsAnnotations.clone();
world.varAnnotations = universe.varAnnotations.clone();
world.obsLayout = universe.obsLayout.clone();
/* layout and display characteristics dataframe */
world.obsLayout = universe.obsLayout;
/*
Var data columns - subset of all
*/
/* Var dataframe - contains a subset of all var columns */
world.varData = universe.varData.clone();
/* save unclipped copies of potentially clipped dataframes */
world.unclipped = {
obsAnnotations: world.obsAnnotations.clone(),
varData: world.varData.clone()
};
return world;
}
export function createWorldFromCurrentSelection(universe, world, crossfilter) {
const newWorld = templateWorld();
/*
clip dataframes based on quantiles.
/* these don't change as only OBS are selected in our current implementation */
newWorld.nVar = universe.nVar;
newWorld.schema = universe.schema;
newWorld.varAnnotations = universe.varAnnotations;
This is an in-place operation on the world object provided as an argument.
The values in world.unclipped are clipped and assigned to world.obsAnnotations
and world.varData.
*/
function setClippedDataframes(world) {
const { schema } = world;
const isContinuousObsAnnotation = (df, idx, label) =>
deduceDimensionType(schema.annotations.obsByName[label], label) !== "enum";
const obsQuantile = (label, q) =>
world.unclipped.obsAnnotations.col(label).summarize().percentiles[100 * q];
world.obsAnnotations = clipDataframe(
world.unclipped.obsAnnotations,
world.clipQuantiles.min,
world.clipQuantiles.max,
obsQuantile,
isContinuousObsAnnotation
);
/* now subset/cut obs */
const varDataQuantile = (label, q) =>
world.unclipped.varData.col(label).summarize().percentiles[100 * q];
world.varData = clipDataframe(
world.unclipped.varData,
world.clipQuantiles.min,
world.clipQuantiles.max,
varDataQuantile,
() => true
);
}
/*
Subset the current world based upon the current selection, maintaining any existing
clip. Returns new world. Parameters:
* unvierse
* world - the current world
* crossfilter - the selection state
*/
export function createWorldBySelection(universe, world, crossfilter) {
const newWorld = { ...world, obsLayout: null, unclipped: {}, varData: null };
/* subset unclipped dataframes based upon current selection */
const mask = crossfilter.allSelectedMask();
newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask);
newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
newWorld.nObs = newWorld.obsAnnotations.dims[0];
/*
Var data columns - subset of all
*/
if (world.varData.isEmpty()) {
newWorld.varData = world.varData.clone();
newWorld.unclipped.obsAnnotations = world.unclipped.obsAnnotations.isubsetMask(
mask
);
if (world.unclipped.varData.isEmpty()) {
newWorld.unclipped.varData = world.unclipped.varData.clone();
} else {
newWorld.varData = world.varData.isubsetMask(mask);
newWorld.unclipped.varData = world.unclipped.varData.isubsetMask(mask);
}
/* subsetting changings dimension size */
newWorld.nObs = newWorld.unclipped.obsAnnotations.dims[0];
/* and now clip */
setClippedDataframes(newWorld);
return newWorld;
}
/*
Change clip quantiles on the current world, returning a new world.
Parameters:
* universe
* world - current world
* clipQuantiles - new clip
*/
export function createWorldWithNewClip(
universe,
world,
crossfilter,
clipQuantiles
) {
const newWorld = { ...world, obsAnnotation: null, varData: null };
newWorld.clipQuantiles = clipQuantiles;
newWorld.obsLayout = world.obsLayout.clone();
newWorld.unclipped = {
obsAnnotations: world.unclipped.obsAnnotations.clone(),
varData: world.unclipped.varData.clone()
};
/* and now clip */
setClippedDataframes(newWorld);
return newWorld;
}
@@ -166,7 +289,10 @@ export function createObsDimensions(crossfilter, world) {
}
export function worldEqUniverse(world, universe) {
return world.obsAnnotations === universe.obsAnnotations;
return (
world.obsAnnotations === universe.obsAnnotations ||
world.obsAnnotations.rowIndex === universe.obsAnnotations.rowIndex
);
}
export function getSelectedByIndex(crossfilter) {
client/src/util/typeHelpers.js
+29
View File
@@ -0,0 +1,29 @@
/*
Various type and schema related helper functions.
*/
/*
Utility function to test for a typed array
*/
export function isTypedArray(x) {
return (
ArrayBuffer.isView(x) &&
Object.prototype.toString.call(x) !== "[object DataView]"
);
}
/*
Test for float typed array, ie, Float32TypedArray or Float64TypedArray
*/
export function isFpTypedArray(x) {
let constructor;
const isFloatArray =
x &&
({ constructor } = x) &&
(constructor === Float32Array || constructor === Float64Array);
return isFloatArray;
}
export function isArrayOrTypedArray(x) {
return Array.isArray(x) || isTypedArray(x);
}
client/src/util/typedCrossfilter/crossfilter.js
+11 -7
View File
@@ -2,13 +2,13 @@ import { polygonContains } from "d3";
import PositiveIntervals from "./positiveIntervals";
import BitArray from "./bitArray";
import { sort } from "./sort";
import {
makeSortIndex,
sortArray,
lowerBound,
lowerBoundIndirect,
upperBoundIndirect
} from "./util";
} from "./sort";
import { makeSortIndex } from "./util";
class NotImplementedError extends Error {
constructor(...params) {
@@ -61,6 +61,10 @@ export default class ImmutableTypedCrossfilter {
return Object.keys(this.dimensions);
}
hasDimension(name) {
return !!this.dimensions[name];
}
addDimension(name, type, ...rest) {
/*
Add a new dimension to this crossfilter, of type DimensionType.
@@ -284,15 +288,15 @@ class _ImmutableBaseDimension {
this.name = name;
}
/* eslint-disable class-methods-use-this */
select(spec) {
const { mode } = spec;
if (mode === undefined) {
throw new Error("select spec does not contain 'mode'");
}
throw new Error(`select mode ${mode} not implemented`);
throw new Error(
`select mode ${mode} not implemented by dimension ${this.name}`
);
}
/* eslint-enable class-methods-use-this */
}
class ImmutableScalarDimension extends _ImmutableBaseDimension {
@@ -414,7 +418,7 @@ class ImmutableEnumDimension extends ImmutableScalarDimension {
for (let i = 0; i < len; i += 1) {
s.add(mapf(i, data));
}
const enumIndex = sort(Array.from(s));
const enumIndex = sortArray(Array.from(s));
this.enumIndex = enumIndex;
// create dimension value array
client/src/util/typedCrossfilter/sort.js
+324 -9
View File
@@ -1,5 +1,29 @@
const SmallArray = 32;
import { isTypedArray, isFpTypedArray } from "../typeHelpers";
/* eslint no-bitwise: "off" */
/*
** fast sort and search, with separate code paths for floats (NaN ordering),
** indirect and direct search/sort.
*/
/*
Comparators for float sort. -Infinity < finite < Infinity < NaN
*/
function lt(a, b) {
if (Number.isNaN(b)) return !Number.isNaN(a);
return a < b;
}
function gt(a, b) {
if (Number.isNaN(a)) return !Number.isNaN(b);
return a > b;
}
/*
insertion sort, used for small arrays (controlled by SMALL_ARRAY constant)
*/
const SMALL_ARRAY = 32;
function insertionsort(a, lo, hi) {
for (let i = lo + 1; i < hi + 1; i += 1) {
const x = a[i];
@@ -12,6 +36,18 @@ function insertionsort(a, lo, hi) {
return a;
}
function insertionsortFloats(a, lo, hi) {
for (let i = lo + 1; i < hi + 1; i += 1) {
const x = a[i];
let j;
for (j = i; j > lo && gt(a[j - 1], x); j -= 1) {
a[j] = a[j - 1];
}
a[j] = x;
}
return a;
}
function insertionsortIndirect(a, s, lo, hi) {
for (let i = lo + 1; i < hi + 1; i += 1) {
const x = a[i];
@@ -25,8 +61,24 @@ function insertionsortIndirect(a, s, lo, hi) {
return a;
}
function insertionsortFloatsIndirect(a, s, lo, hi) {
for (let i = lo + 1; i < hi + 1; i += 1) {
const x = a[i];
const t = s[x];
let j;
for (j = i; j > lo && gt(s[a[j - 1]], t); j -= 1) {
a[j] = a[j - 1];
}
a[j] = x;
}
return a;
}
/*
Quicksort - used for larger arrays
*/
function quicksort(a, lo, hi) {
if (hi - lo < SmallArray) {
if (hi - lo < SMALL_ARRAY) {
return insertionsort(a, lo, hi);
}
if (lo < hi) {
@@ -55,8 +107,38 @@ function quicksort(a, lo, hi) {
return a;
}
function quicksortFloats(a, lo, hi) {
if (hi - lo < SMALL_ARRAY) {
return insertionsortFloats(a, lo, hi);
}
if (lo < hi) {
// partition
const mid = Math.floor((lo + hi) / 2);
const p = a[mid];
let i = lo - 1;
let j = hi + 1;
while (i < j) {
do {
i += 1;
} while (lt(a[i], p));
do {
j -= 1;
} while (gt(a[j], p));
if (i < j) {
const tmp = a[i];
a[i] = a[j];
a[j] = tmp;
}
}
// sort
quicksortFloats(a, lo, j);
quicksortFloats(a, j + 1, hi);
}
return a;
}
function quicksortIndirect(a, s, lo, hi) {
if (hi - lo < SmallArray) {
if (hi - lo < SMALL_ARRAY) {
return insertionsortIndirect(a, s, lo, hi);
}
if (lo < hi) {
@@ -86,15 +168,248 @@ function quicksortIndirect(a, s, lo, hi) {
return a;
}
// Convenience wrappers
export function sort(arr, comparator = undefined) {
if (comparator !== undefined) {
// XXX for now
return arr.sort(arr, comparator);
function quicksortFloatsIndirect(a, s, lo, hi) {
if (hi - lo < SMALL_ARRAY) {
return insertionsortFloatsIndirect(a, s, lo, hi);
}
return quicksort(arr, 0, arr.length - 1);
if (lo < hi) {
// partition
const mid = Math.floor((lo + hi) / 2);
const p = a[mid];
const t = s[p];
let i = lo - 1;
let j = hi + 1;
while (i < j) {
do {
i += 1;
} while (lt(s[a[i]], t));
do {
j -= 1;
} while (gt(s[a[j]], t));
if (i < j) {
const tmp = a[i];
a[i] = a[j];
a[j] = tmp;
}
}
// sort
quicksortFloatsIndirect(a, s, lo, j);
quicksortFloatsIndirect(a, s, j + 1, hi);
}
return a;
}
/*
Convenience wrappers, handling optimization paths and default
handlers for NaN comparisons. Sorts in place.
*/
export function sortArray(arr) {
if (Array.isArray(arr)) {
return quicksort(arr, 0, arr.length - 1);
}
if (isTypedArray(arr)) {
if (isFpTypedArray(arr)) {
return quicksortFloats(arr, 0, arr.length - 1);
}
return quicksort(arr, 0, arr.length - 1);
}
/* else unsupported */
throw new Error("sortArray received unsupported object type");
}
export function sortIndex(index, source) {
if (isFpTypedArray(source))
return quicksortFloatsIndirect(index, source, 0, index.length - 1);
return quicksortIndirect(index, source, 0, index.length - 1);
}
// Search for `value` in the sorted array `arr`, in the range [first, last).
// Return the first (left most) index where arr[index] >= value.
//
// In other words, return array index I where:
// arr[i] < value for all tarr[lo:I]
// arr[i] >= value for all tarr[I:last]
//
// The same semantics/behavior as:
// C++: lower_bound()
// Python: bisect.bisect_left()
//
function lowerBoundNonFloat(valueArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (valueArray[middle] < value) {
lfirst = middle + 1;
} else {
llast = middle;
}
}
return lfirst;
}
// lowerBound, but with NaN handling
//
// If the underlying array is a Float32Array or Float64Array, will enforce
// the ordering -Infinity < finite < Infinity < NaN.
//
function lowerBoundFloat(valueArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (lt(valueArray[middle], value)) {
lfirst = middle + 1;
} else {
llast = middle;
}
}
return lfirst;
}
export function lowerBound(valueArray, value, first, last) {
if (isFpTypedArray(valueArray)) {
return lowerBoundFloat(valueArray, value, first, last);
}
return lowerBoundNonFloat(valueArray, value, first, last);
}
// Inlined performance optimization - used to indirect through a sort map.
//
function lowerBoundNonFloatIndirect(
valueArray,
indexArray,
value,
first,
last
) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (valueArray[indexArray[middle]] < value) {
lfirst = middle + 1;
} else {
llast = middle;
}
}
return lfirst;
}
function lowerBoundFloatIndirect(valueArray, indexArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (lt(valueArray[indexArray[middle]], value)) {
lfirst = middle + 1;
} else {
llast = middle;
}
}
return lfirst;
}
export function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
if (isFpTypedArray(valueArray)) {
return lowerBoundFloatIndirect(valueArray, indexArray, value, first, last);
}
return lowerBoundNonFloatIndirect(valueArray, indexArray, value, first, last);
}
// Search for `value in the sorted array `arr`, in the range [first, last).
// Return the first value where arr[index] > value.
//
// In other words, return array index I, where:
// arr[i] <= value for all tarr[lo:I]
// arr[i] > value for all tarr[I:last]
//
// The same semantics/behavior as:
// C++: upper_bound()
// Python: bisect.bisect_right()
//
function upperBoundNonFloat(valueArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (valueArray[middle] > value) {
llast = middle;
} else {
lfirst = middle + 1;
}
}
return lfirst;
}
function upperBoundFloat(valueArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (gt(valueArray[middle], value)) {
llast = middle;
} else {
lfirst = middle + 1;
}
}
return lfirst;
}
export function upperBound(valueArray, value, first, last) {
if (isFpTypedArray(valueArray)) {
return upperBoundFloat(valueArray, value, first, last);
}
return upperBoundNonFloat(valueArray, value, first, last);
}
// Inline performance optimization
//
function upperBoundNonFloatIndirect(
valueArray,
indexArray,
value,
first,
last
) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (valueArray[indexArray[middle]] > value) {
llast = middle;
} else {
lfirst = middle + 1;
}
}
return lfirst;
}
function upperBoundFloatIndirect(valueArray, indexArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (gt(valueArray[indexArray[middle]], value)) {
llast = middle;
} else {
lfirst = middle + 1;
}
}
return lfirst;
}
export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
if (isFpTypedArray(valueArray)) {
return upperBoundFloatIndirect(valueArray, indexArray, value, first, last);
}
return upperBoundNonFloatIndirect(valueArray, indexArray, value, first, last);
}
client/src/util/typedCrossfilter/util.js
-91
View File
@@ -1,5 +1,4 @@
// jshint esversion: 6
/* eslint no-bitwise: "off" */
import { sortIndex } from "./sort";
@@ -36,93 +35,3 @@ export function makeSortIndex(src) {
sortIndex(index, src);
return index;
}
// Search for `value` in the sorted array `arr`, in the range [first, last).
// Return the first (left most) index where arr[index] >= value.
//
// In other words, return array index I where:
// arr[i] < value for all tarr[lo:I]
// arr[i] >= value for all tarr[I:last]
//
// The same semantics/behavior as:
// C++: lower_bound()
// Python: bisect.bisect_left()
//
// XXX: it is likely that there would be minimal performance hit from creating
// a factory version of lowerBound that takes an accessor (rather than having
// a special-cased version for lining the indirection).
//
export function lowerBound(valueArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (valueArray[middle] < value) {
lfirst = middle + 1;
} else {
llast = middle;
}
}
return lfirst;
}
// Inlined performance optimization - used to indirect through a sort map.
//
export function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (valueArray[indexArray[middle]] < value) {
lfirst = middle + 1;
} else {
llast = middle;
}
}
return lfirst;
}
// Search for `value in the sorted array `arr`, in the range [first, last).
// Return the first value where arr[index] > value.
//
// In other words, return array index I, where:
// arr[i] <= value for all tarr[lo:I]
// arr[i] > value for all tarr[I:last]
//
// The same semantics/behavior as:
// C++: upper_bound()
// Python: bisect.bisect_right()
//
export function upperBound(valueArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (valueArray[middle] > value) {
llast = middle;
} else {
lfirst = middle + 1;
}
}
return lfirst;
}
// Inline performance optimization
//
export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
let lfirst = first;
let llast = last;
// this is just a binary search
while (lfirst < llast) {
const middle = (lfirst + llast) >>> 1;
if (valueArray[indexArray[middle]] > value) {
llast = middle;
} else {
lfirst = middle + 1;
}
}
return lfirst;
}