fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-25 10:58:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

TS Revert (1) (#2402)

Browse Source 
* revert all commits to before Typescript migration

* update compat workflow to match latest deps (#2335)

* update compat workflow to match latest deps

* attempt to debug

* attempt to debug

* remove debugging code

* typo

* update deps to match desktop (#2340)

* fix: don't run lint with `--fix` on push tests (#2273)

* fix: don't run lint with `--fix` on push tests

* npx

Co-authored-by: maniarathi <mani.arathi@gmail.com>
Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>

* rename X_approx_distribution to X_approximate_distribution (#2337)

* Correctly handle non-finite numbers in heuristic determination of X distribution (#2342)

* handle non-finites explicitly

* improve and test edge case handling for distribution estimation

* revert debugging changes

* code readability

* clean up type inferencing (#2332)

* unit tests for 64 bit conversion

* clean up type handling

* type inference tests

* more type inference fixes

* use schema to determine user intent for data typing

* stop using deprecated API

* fbs type encoding test

* add missing test

* add more tests

* correctly infer X type for CXG adaptor

* lint

* fix typo

* ts migration

* cleanup from PR review

* lint

* PR review changes

* remove unused packages from client (#2359)

* remove unused packages from client

* add missing peer dep

* fix: disable FE auth testing on compatibility tests (#2377)

* update: release process (#2277)

Co-authored-by: maniarathi <mani.arathi@gmail.com>

* fix: remove spaces in param setup (#2380)

* delete deploy workflow (#2396)

* undo reformatting which now does not pass lint

* fix snapshots which changed due to npm dep changes

* add missing quoting to snapshot

* another snapshot typo fix

* TS Revert (2) - replay PR #2347 and #2354 (#2403)

* replay edits from PR 2347

* TS Revert (3) - replay edits in PR #2327 (#2404)

* replay edits in PR 2327

* TS Revert (4) - replay PR #2355 (#2405)

* replay edits in PR 2355

* add additional babel config

* reformat with new prettier config

Co-authored-by: Severiano Badajoz <sbadajoz@chanzuckerberg.com>
Co-authored-by: maniarathi <mani.arathi@gmail.com>
Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>
This commit is contained in:
Bruce Martin
2021-08-23 15:01:36 -07:00
committed by GitHub
co-authored by maniarathi Madison Dunitz Severiano Badajoz
parent 295590a7c6
commit eaae6df5e3
253 changed files with 11175 additions and 22271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/util/dataframe/cache.ts → client/src/util/dataframe/cache.js
+3 -4
View File
@@ -10,18 +10,17 @@ objects.
*/
import { memoize } from "./util";
import Dataframe from "./dataframe";
function hashDataframe(df: Dataframe): string {
function hashDataframe(df) {
if (df.isEmpty()) return "";
return df.__columnsAccessor.map((c) => c.__id).join(",");
}
function noop(df: Dataframe): Dataframe {
function noop(df) {
return df;
}
const dataframeMemo = (capacity = 100): ((df: Dataframe) => Dataframe) =>
const dataframeMemo = (capacity = 100) =>
memoize(noop, hashDataframe, capacity);
export default dataframeMemo;
client/src/util/dataframe/dataframe.ts → client/src/util/dataframe/dataframe.js
+143 -252
View File
@@ -1,44 +1,30 @@
import { callOnceLazy, memoize, __getMemoId } from "./util";
import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
// weird cross-dependency that we should clean up someday...
import {
isTypedArray,
isAnyArray,
AnyArray,
GenericArrayConstructor,
} from "../../common/types/arraytypes";
import { IdentityInt32Index, LabelIndex, isLabelIndex } from "./labelIndex";
isArrayOrTypedArray,
callOnceLazy,
memoize,
__getMemoId,
} from "./util";
import {
summarizeContinuous as _summarizeContinuous,
summarizeContinuous,
summarizeCategorical as _summarizeCategorical,
} from "./summarize";
import {
histogramCategorical as _histogramCategorical,
histogramCategoricalBy as _histogramCategoricalBy,
hashCategorical,
hashCategoricalBy,
histogramContinuous as _histogramContinuous,
histogramContinuousBy as _histogramContinuousBy,
histogramContinuous,
hashContinuous,
hashContinuousBy,
} from "./histogram";
import {
DataframeValue,
DataframeValueArray,
DataframeColumn,
OffsetType,
OffsetArray,
LabelType,
LabelArray,
ContinuousHistogram,
ContinuousHistogramBy,
} from "./types";
/*
Dataframe is an immutable 2D matrix similar to Python Pandas Dataframe,
Dataframe is an immutable 2D matrix similiar to Python Pandas Dataframe,
but (currently) without all of the surrounding support functions.
Data is stored in column-major layout, and each column is monomorphic.
It supports:
* Relatively efficient create, clone and subset operations
* Relatively efficient creation, cloning and subsetting
* Very efficient columnar access (eg, sum down a column), and access
to the underlying column arrays.
* Data access by row/col offset or label. Labels are reasonably well
@@ -46,10 +32,10 @@ It supports:
It does not currently support:
* Views on matrix subset - for currently known access patterns,
it is more efficient to copy on subsetting, optimizing for access
it is more effiicent to copy on subsetting, optimizing for access
speed over memory use.
* JS iterators - they are too slow. Use explicit iteration over
offset or labels.
offest or labels.
Important assumptions embedded in the API:
* Columns are implicitly categorical if they are a JS Array and numeric
@@ -86,55 +72,24 @@ dominant pattern in cellxgene.
Dataframe
**/
interface DataframeConstructor {
new (...args: ConstructorParameters<typeof Dataframe>): Dataframe;
}
export type MapColumnsCallbackFn = (
data: DataframeValueArray,
idx: number,
df: Dataframe
) => DataframeValueArray;
/** @internal */
function raiseIsNotContinuous<R = void>(): R {
throw TypeError("Column is not a continuous data type.");
}
class Dataframe {
/** @internal */
__columns: DataframeValueArray[];
/** @internal */
__columnsAccessor: DataframeColumn[] = [];
__id: string;
colIndex: LabelIndex;
dims: [number, number];
length: number;
rowIndex: LabelIndex;
/**
Constructors & factories
**/
constructor(
dims: [number, number],
columnarData: DataframeValueArray[],
rowIndex?: LabelIndex | null,
colIndex?: LabelIndex | null,
__columnsAccessor: (DataframeColumn | null)[] = [] // private interface
dims,
columnarData,
rowIndex = null,
colIndex = null,
__columnsAccessor = [] // private interface
) {
/*
The base constructor is relatively hard to use - as an alternative,
see factory methods and clone/slice, below.
Parameters:
* dims - 2D array describing intended dimensionality: [nRows,nCols].
* dims - 2D array describing intendend dimensionality: [nRows,nCols].
* columnarData - JS array, nCols in length, containing array
or TypedArray of length nRows.
* rowIndex/colIndex - null (create default index using offsets as key),
@@ -167,20 +122,14 @@ class Dataframe {
Object.freeze(this);
}
/** @internal */
static __errorChecks(
dims: [number, number],
columnarData: AnyArray[],
rowIndex: LabelIndex,
colIndex: LabelIndex
): void | never {
static __errorChecks(dims, columnarData, rowIndex, colIndex) {
const [nRows, nCols] = dims;
/* check for expected types */
if (!Array.isArray(columnarData)) {
throw new TypeError("Dataframe constructor requires array of columns");
}
if (!columnarData.every((c) => isAnyArray(c))) {
if (!columnarData.every((c) => isArrayOrTypedArray(c))) {
throw new TypeError("Dataframe columns must all be Array or TypedArray");
}
if (!isLabelIndex(rowIndex)) {
@@ -211,12 +160,7 @@ class Dataframe {
}
}
/** @internal */
static __compileColumn(
column: DataframeValueArray,
getRowOffset: (label: LabelType) => OffsetType | -1,
getRowLabel: (offset: number) => LabelType | undefined
): DataframeColumn {
static __compileColumn(column, getRowByOffset, getRowByLabel) {
/*
Each column accessor is a function which will lookup data by
index (ie, is equivalent to dataframe.get(row, col), where 'col'
@@ -249,17 +193,14 @@ class Dataframe {
*/
const { length } = column;
const __id = __getMemoId();
const isContinuous = isTypedArray(column);
/* get value by row label */
const get = function get(rlabel: LabelType): DataframeValue | undefined {
const idx = getRowOffset(rlabel);
if (idx === -1) return undefined;
return column[idx];
const get = function get(rlabel) {
return column[getRowByOffset(rlabel)];
};
/* get value by row offset */
const iget = function iget(roffset: OffsetType) {
const iget = function iget(roffset) {
return column[roffset];
};
@@ -269,12 +210,12 @@ class Dataframe {
};
/* test for row label inclusion in column */
const has = function has(rlabel: LabelType) {
const offset = getRowOffset(rlabel);
const has = function has(rlabel) {
const offset = getRowByOffset(rlabel);
return offset >= 0 && offset < length;
};
const ihas = function ihas(offset: OffsetType) {
const ihas = function ihas(offset) {
return offset >= 0 && offset < length;
};
@@ -285,88 +226,76 @@ class Dataframe {
NOTE: not found return is DIFFERENT than the default Array.indexOf as
-1 is a plausible Dataframe row/col label.
*/
const _indexOf = function _indexOf(value: DataframeValue) {
let offset: number;
if (isTypedArray(column)) offset = column.indexOf(value as number);
else offset = column.indexOf(value);
const indexOf = function indexOf(value) {
const offset = column.indexOf(value);
if (offset === -1) {
return undefined;
}
return getRowLabel(offset);
return getRowByLabel(offset);
};
/*
Summarize the column data. Lazy eval, memoized
*/
get.summarizeCategorical = callOnceLazy(() =>
const summarizeCategorical = callOnceLazy(() =>
_summarizeCategorical(column)
);
get.summarizeContinuous = isContinuous
? callOnceLazy(() => _summarizeContinuous(column))
: raiseIsNotContinuous;
const summarize = callOnceLazy(() =>
isTypedArray(column)
? summarizeContinuous(column)
: summarizeCategorical(column)
);
/*
Create histogram bins for this column. Memoized.
*/
get.histogramContinuous = isContinuous
? (bins: number, domain: [number, number]): ContinuousHistogram =>
memoize(_histogramContinuous, hashContinuous)(get, bins, domain)
: raiseIsNotContinuous;
get.histogramContinuousBy = isContinuous
? (
bins: number,
domain: [number, number],
by: DataframeColumn
): ContinuousHistogramBy =>
memoize(_histogramContinuousBy, hashContinuousBy)(
get,
bins,
domain,
by
)
: raiseIsNotContinuous;
get.histogramCategorical = () =>
memoize(_histogramCategorical, hashCategorical)(get);
get.histogramCategoricalBy = (by: DataframeColumn) =>
memoize(_histogramCategoricalBy, hashCategoricalBy)(get, by);
const _memoHistoCat = memoize(_histogramCategorical, hashCategorical);
const histogramCategorical = (by) => _memoHistoCat(get, by);
let histogram = null;
if (isTypedArray(column)) {
const mFn = memoize(histogramContinuous, hashContinuous);
histogram = (bins, domain, by) => mFn(get, bins, domain, by);
} else {
histogram = histogramCategorical;
}
get.summarize = summarize;
get.summarizeCategorical = summarizeCategorical;
get.histogram = histogram;
get.histogramCategorical = histogramCategorical;
get.asArray = asArray;
get.has = has;
get.ihas = ihas;
get.indexOf = _indexOf;
get.indexOf = indexOf;
get.iget = iget;
get.__id = __id;
get.isContinuous = isContinuous;
Object.freeze(get);
return get;
}
/** @internal */
__compile(accessors: (DataframeColumn | null)[]): void {
__compile(accessors) {
/*
Compile data accessors for each column.
Use an existing accessor if provided, else compile a new one.
*/
const getRowOffset = this.rowIndex.getOffset.bind(this.rowIndex);
const getRowLabel = this.rowIndex.getLabel.bind(this.rowIndex);
this.__columnsAccessor = this.__columns.map(
(column, idx): DataframeColumn => {
if (accessors[idx]) {
return accessors[idx] as DataframeColumn;
}
return Dataframe.__compileColumn(column, getRowOffset, getRowLabel);
const getRowByOffset = this.rowIndex.getOffset.bind(this.rowIndex);
const getRowByLabel = this.rowIndex.getLabel.bind(this.rowIndex);
this.__columnsAccessor = this.__columns.map((column, idx) => {
if (accessors[idx]) {
return accessors[idx];
}
);
return Dataframe.__compileColumn(column, getRowByOffset, getRowByLabel);
});
Object.freeze(this.__columnsAccessor);
}
clone(): Dataframe {
clone() {
/*
Clone this dataframe
*/
return new (this.constructor as DataframeConstructor)(
return new this.constructor(
this.dims,
[...this.__columns],
this.rowIndex,
@@ -375,11 +304,7 @@ class Dataframe {
);
}
withCol(
label: LabelType,
colData: DataframeValueArray,
withRowIndex?: LabelIndex
): Dataframe {
withCol(label, colData, withRowIndex = null) {
/*
Create a new DF, which is `this` plus the new column. Example:
const newDf = df.withCol("foo", [1,2,3]);
@@ -394,10 +319,11 @@ class Dataframe {
the rowIndex from `this` will be used (ie, the rowIndex is
unchanged).
*/
let dims: [number, number];
let rowIndex: LabelIndex | null = null;
let dims;
let rowIndex;
if (this.isEmpty()) {
dims = [colData.length, 1];
rowIndex = null;
} else {
dims = [this.dims[0], this.dims[1] + 1];
({ rowIndex } = this);
@@ -411,7 +337,7 @@ class Dataframe {
columns.push(colData);
const colIndex = this.colIndex.withLabel(label);
const columnsAccessor = [...this.__columnsAccessor];
return new (this.constructor as DataframeConstructor)(
return new this.constructor(
dims,
columns,
rowIndex,
@@ -420,10 +346,7 @@ class Dataframe {
);
}
withColsFrom(
dataframe: Dataframe,
labels?: Record<string | number, LabelType> | LabelType[]
): Dataframe {
withColsFrom(dataframe, labels) {
/*
return a new dataframe containing all columns from both `this` and the
provided dataframe argument.
@@ -449,8 +372,8 @@ class Dataframe {
*/
// resolve the source and dest label names.
let srcLabels: LabelArray;
let dstLabels: LabelArray;
let srcLabels;
let dstLabels;
if (!labels) {
// combine all columns
dstLabels = dataframe.colIndex.labels();
@@ -485,9 +408,9 @@ class Dataframe {
return dataframe;
}
// otherwise, build a new dataframe combining columns from both
const srcOffsets = Array.from(dataframe.colIndex.getOffsets(srcLabels));
if (srcOffsets.some((i) => i === -1)) throw RangeError("Unknown label.");
// otherwise, bulid a new dataframe combining columns from both
const srcOffsets = srcLabels.map((l) => dataframe.colIndex.getOffset(l));
// check for label collisions
if (dstLabels.some(this.hasCol, this)) {
@@ -495,12 +418,9 @@ class Dataframe {
}
// const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
const dims: [number, number] = [
this.dims[0],
this.dims[1] + srcOffsets.length,
];
const dims = [this.dims[0], this.dims[1] + srcOffsets.length];
const { rowIndex } = this;
const columns: DataframeValueArray[] = [
const columns = [
...this.__columns,
...srcOffsets.map((i) => dataframe.__columns[i]),
];
@@ -510,7 +430,7 @@ class Dataframe {
...srcOffsets.map((i) => dataframe.__columnsAccessor[i]),
];
return new (this.constructor as DataframeConstructor)(
return new this.constructor(
dims,
columns,
rowIndex,
@@ -519,12 +439,12 @@ class Dataframe {
);
}
withColsFromAll(dataframes: Dataframe[] = []): Dataframe {
withColsFromAll(dataframes = []) {
dataframes = Array.isArray(dataframes) ? dataframes : [dataframes];
return dataframes.reduce((acc, df) => acc.withColsFrom(df), this);
}
dropCol(label: LabelType): Dataframe {
dropCol(label) {
/*
Create a new dataframe, omitting one columns.
@@ -543,15 +463,14 @@ class Dataframe {
return Dataframe.empty();
}
const dims: [number, number] = [this.dims[0], this.dims[1] - 1];
const dims = [this.dims[0], this.dims[1] - 1];
const coffset = this.colIndex.getOffset(label);
if (coffset === -1) throw new RangeError("Unknown label.");
const columns = [...this.__columns];
columns.splice(coffset, 1);
const colIndex = this.colIndex.dropLabel(label);
const columnsAccessor = [...this.__columnsAccessor];
columnsAccessor.splice(coffset, 1);
return new (this.constructor as DataframeConstructor)(
return new this.constructor(
dims,
columns,
this.rowIndex,
@@ -560,12 +479,11 @@ class Dataframe {
);
}
renameCol(oldLabel: LabelType, newLabel: LabelType): Dataframe {
renameCol(oldLabel, newLabel) {
/*
Accelerator for dropping a column and then adding it again with a new label
*/
const coffset = this.colIndex.getOffset(oldLabel);
if (coffset === -1) throw new RangeError("Unknown label.");
const colIndex = this.colIndex.dropLabel(oldLabel).withLabel(newLabel);
const columns = [...this.__columns];
@@ -576,7 +494,7 @@ class Dataframe {
columnsAccessor.push(columnsAccessor[coffset]);
columnsAccessor.splice(coffset, 1);
return new (this.constructor as DataframeConstructor)(
return new this.constructor(
this.dims,
columns,
this.rowIndex,
@@ -585,21 +503,18 @@ class Dataframe {
);
}
replaceColData(label: LabelType, newColData: DataframeValueArray): Dataframe {
replaceColData(label, newColData) {
/*
Accelerator for dropping a column then adding it again with same
label and different values.
*/
const coffset = this.colIndex.getOffset(label);
if (coffset === -1) throw RangeError("Unknown column label.");
const columns = [...this.__columns];
columns[coffset] = newColData;
const columnsAccessor: (DataframeColumn | null)[] = [
...this.__columnsAccessor,
];
const columnsAccessor = [...this.__columnsAccessor];
columnsAccessor[coffset] = null;
return new (this.constructor as DataframeConstructor)(
return new this.constructor(
this.dims,
columns,
this.rowIndex,
@@ -608,8 +523,8 @@ class Dataframe {
);
}
static empty(rowIndex?: LabelIndex, colIndex?: LabelIndex): Dataframe {
const dims: [number, number] = [
static empty(rowIndex = null, colIndex = null) {
const dims = [
rowIndex ? rowIndex.size() : 0,
colIndex ? colIndex.size() : 0,
];
@@ -617,10 +532,7 @@ class Dataframe {
return new Dataframe(dims, new Array(dims[1]), rowIndex, colIndex);
}
static create(
dims: [number, number],
columnarData: DataframeValueArray[]
): Dataframe {
static create(dims, columnarData) {
/*
Create a dataframe from raw columnar data. All column arrays
must have the same length. Identity indexing will be used.
@@ -628,15 +540,11 @@ class Dataframe {
Example:
const df = Dataframe.create([2,2], [new Uint32Array(2), new Float32Array(2)]);
*/
return new Dataframe(dims, columnarData);
return new Dataframe(dims, columnarData, null, null);
}
/** @internal */
__subset(
newRowIndex: LabelIndex | null,
newColIndex: LabelIndex | null
): Dataframe {
const dims: [number, number] = [...this.dims];
__subset(newRowIndex, newColIndex) {
const dims = [...this.dims];
/* subset columns */
let { __columns, colIndex, __columnsAccessor } = this;
@@ -645,10 +553,8 @@ class Dataframe {
__columns = new Array(colOffsets.length);
__columnsAccessor = new Array(colOffsets.length);
for (let i = 0, l = colOffsets.length; i < l; i += 1) {
const colOffset = colOffsets[i];
if (colOffset === -1) throw new RangeError("Unexpected column offset.");
__columns[i] = this.__columns[colOffset];
__columnsAccessor[i] = this.__columnsAccessor[colOffset];
__columns[i] = this.__columns[colOffsets[i]];
__columnsAccessor[i] = this.__columnsAccessor[colOffsets[i]];
}
colIndex = newColIndex;
dims[1] = colOffsets.length;
@@ -658,13 +564,9 @@ class Dataframe {
if (newRowIndex) {
const rowOffsets = this.rowIndex.getOffsets(newRowIndex.labels());
__columns = __columns.map((col) => {
const newCol = new (col.constructor as GenericArrayConstructor<
typeof col
>)(rowOffsets.length);
const newCol = new col.constructor(rowOffsets.length);
for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
const rowOffset = rowOffsets[i];
if (rowOffset === -1) throw new RangeError("Unexpected row offset.");
newCol[i] = col[rowOffset];
newCol[i] = col[rowOffsets[i]];
}
return newCol;
});
@@ -683,11 +585,7 @@ class Dataframe {
);
}
subset(
rowLabels: LabelArray | null,
colLabels: LabelArray | null,
withRowIndex?: LabelIndex | null
): Dataframe {
subset(rowLabels, colLabels = null, withRowIndex = null) {
/*
Subset by row/col labels.
@@ -709,11 +607,7 @@ class Dataframe {
return this.__subset(rowIndex, colIndex);
}
isubset(
rowOffsets: OffsetArray | null,
colOffsets: OffsetArray | null,
withRowIndex?: LabelIndex | null
): Dataframe {
isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
/*
Subset by row/col offset.
@@ -722,14 +616,14 @@ class Dataframe {
indexing. If withRowIndex is a label index object, it will be used
for the new dataframe.
*/
let rowIndex: LabelIndex | null = null;
let rowIndex = null;
if (withRowIndex) {
rowIndex = withRowIndex;
} else if (rowOffsets) {
rowIndex = this.rowIndex.isubset(rowOffsets);
}
let colIndex: LabelIndex | null = null;
let colIndex = null;
if (colOffsets) {
colIndex = this.colIndex.isubset(colOffsets);
}
@@ -737,11 +631,7 @@ class Dataframe {
return this.__subset(rowIndex, colIndex);
}
isubsetMask(
rowMask: Uint8Array | boolean[] | null,
colMask: Uint8Array | boolean[] | null,
withRowIndex?: LabelIndex | null
): Dataframe {
isubsetMask(rowMask, colMask = null, withRowIndex = null) {
/*
Subset on row/column based upon a truthy/falsey array (a mask).
@@ -759,10 +649,7 @@ class Dataframe {
}
/* convert masks to lists - method wastes space, but is fast */
const toList = (
mask: Uint8Array | boolean[] | null | undefined,
maxSize: number
) => {
const toList = (mask, maxSize) => {
if (!mask) {
return null;
}
@@ -785,12 +672,12 @@ class Dataframe {
Data access with row/col.
**/
columns(): DataframeColumn[] {
columns() {
/* return all column accessors as an array, in offset order */
return [...this.__columnsAccessor];
}
col(columnLabel: LabelType): DataframeColumn {
col(columnLabel) {
/*
Return accessor bound to a column. Allows random row access
based upon the row indexing. Returns undefined if the
@@ -807,44 +694,36 @@ class Dataframe {
See __compile() for the functions available in a column accessor.
*/
const coff = this.colIndex.getOffset(columnLabel);
if (coff === -1) throw RangeError("Unknown label.");
return this.__columnsAccessor[coff];
}
icol(columnOffset: OffsetType): DataframeColumn {
icol(columnOffset) {
/*
Return column accessor by offset.
*/
if (
Number.isInteger(columnOffset) &&
columnOffset >= 0 &&
columnOffset < this.__columnsAccessor.length
) {
return this.__columnsAccessor[columnOffset];
}
throw new RangeError("Unknown offset.");
return Number.isInteger(columnOffset)
? this.__columnsAccessor[columnOffset]
: undefined;
}
at(r: LabelType, c: LabelType): DataframeValue {
at(r, c) {
/*
Access a single value, for a row/col label pair.
For performance reasons, there are no bounds or existence
For performance reasons, there are no bounds or existance
checks on labels, and no defined behavior when these are supplied.
May return undefined, throw an Error, or do something else for
non-existent labels. If you want predictable out-of-bounds
non-existant labels. If you want predictable out-of-bounds
behavior, use has(), eg,
const myVal = df.has(r,l) ? df.at(r,l) : undefined;
*/
const coff = this.colIndex.getOffset(c);
const roff = this.rowIndex.getOffset(r);
if (coff === undefined || roff === undefined)
throw new RangeError("Unknown row or column label.");
return this.__columns[coff][roff];
}
iat(r: OffsetType, c: OffsetType): DataframeValue {
iat(r, c) {
/*
Access a single value, for a row/col offset (integer) position.
@@ -854,12 +733,10 @@ class Dataframe {
const myVal = df.ihas(r, c) ? df.iat(r, c) : undefined;
*/
if (c >= 0 && c < this.dims[1] && r >= 0 && r < this.dims[0])
return this.__columns[c][r];
throw new RangeError("Unknown row or column index.");
return this.__columns[c][r];
}
has(r: LabelType, c: LabelType): boolean {
has(r, c) {
/*
Test if row/col labels exist in the dataframe - returns true/false
*/
@@ -869,7 +746,7 @@ class Dataframe {
return coff >= 0 && coff < nCols && roff >= 0 && roff < nRows;
}
ihas(r: number, c: number): boolean {
ihas(r, c) {
/*
Test if row/col offset (integer) position exists in the
dataframe - returns true/false
@@ -885,23 +762,14 @@ class Dataframe {
);
}
hasCol(c: LabelType): boolean {
hasCol(c) {
/*
Test if col label exists - return true/false
*/
const coff = this.colIndex.getOffset(c);
return coff !== -1;
return !!this.col(c);
}
ihasCol(i: number): boolean {
/*
Test if col offset exists - return true/false
*/
const [, nCols] = this.dims;
return i >= 0 && i < nCols;
}
isEmpty(): boolean {
isEmpty() {
/*
Return true if this is an empty dataframe, ie, has dimensions [0,0]
*/
@@ -916,7 +784,7 @@ class Dataframe {
add these as useful.
****/
mapColumns(callback: MapColumnsCallbackFn): Dataframe {
mapColumns(callback) {
/*
map all columns in the dataframe, returning a new dataframe comprised of the
return values, with the same index as the original dataframe.
@@ -926,10 +794,10 @@ class Dataframe {
const columns = this.__columns.map((colData, colIdx) =>
callback(colData, colIdx, this)
);
const columnsAccessor: (DataframeColumn | null)[] = columns.map((c, idx) =>
this.__columns[idx] === c ? this.__columnsAccessor[idx] : null
const columnsAccessor = columns.map((c, idx) =>
this.__columns[idx] === c ? this.__columnsAccessor[idx] : undefined
);
return new (this.constructor as DataframeConstructor)(
return new this.constructor(
this.dims,
columns,
this.rowIndex,
@@ -937,6 +805,29 @@ class Dataframe {
columnsAccessor
);
}
/*
Map & reduce of column or row
TODO remainder of map/reduce functions: mapCol, mapRow, reduceRow, ...
*/
/* comment out until we have a use for this
reduceCol(clabel, callback, initialValue) {
const coff = this.colIndex.getOffset(clabel);
const column = this.__columns[coff];
let start = 0;
let acc = initialValue;
if (initialValue === undefined) {
acc = column[0];
start = 1;
}
for (let i = start, l = column.length; i < l; i += 1) {
acc = callback(acc, column[i]);
}
return acc;
}
*/
}
export default Dataframe;
client/src/util/dataframe/histogram.ts → client/src/util/dataframe/histogram.js
+39 -51
View File
@@ -1,27 +1,15 @@
/*
Dataframe histogram
*/
import { NumberArray } from "../../common/types/arraytypes";
import {
DataframeColumn,
ContinuousHistogram,
ContinuousHistogramBy,
CategoricalHistogram,
CategoricalHistogramBy,
} from "./types";
import { isTypedArray } from "./util";
export function histogramContinuous(
column: DataframeColumn,
bins: number,
domain: [number, number]
): ContinuousHistogram {
function _histogramContinuous(column, bins, min, max) {
const valBins = new Array(bins).fill(0);
if (!column) {
return valBins;
}
const [min, max] = domain;
const binWidth = (max - min) / bins;
const colArray: NumberArray = column.asArray() as NumberArray;
const colArray = column.asArray();
for (let r = 0, len = colArray.length; r < len; r += 1) {
const val = colArray[r];
if (val <= max && val >= min) {
@@ -33,20 +21,14 @@ export function histogramContinuous(
return valBins;
}
export function histogramContinuousBy(
column: DataframeColumn,
bins: number,
domain: [number, number],
by: DataframeColumn
): ContinuousHistogramBy {
function _histogramContinuousBy(column, bins, min, max, by) {
const byMap = new Map();
if (!column || !by) {
return byMap;
}
const [min, max] = domain;
const binWidth = (max - min) / bins;
const byArray = by.asArray();
const colArray = column.asArray() as NumberArray;
const colArray = column.asArray();
for (let r = 0, len = colArray.length; r < len; r += 1) {
const byBin = byArray[r];
let valBins = byMap.get(byBin);
@@ -64,9 +46,7 @@ export function histogramContinuousBy(
return byMap;
}
export function histogramCategorical(
column: DataframeColumn
): CategoricalHistogram {
function _histogramCategorical(column) {
const valMap = new Map();
if (!column) {
return valMap;
@@ -83,10 +63,7 @@ export function histogramCategorical(
return valMap;
}
export function histogramCategoricalBy(
column: DataframeColumn,
by: DataframeColumn
): CategoricalHistogramBy {
function _histogramCategoricalBy(column, by) {
const byMap = new Map();
if (!column || !by) {
return byMap;
@@ -110,38 +87,49 @@ export function histogramCategoricalBy(
return byMap;
}
/*
Count category occupancy. Optional group-by category.
*/
export function histogramCategorical(column, by) {
if (by && isTypedArray(by)) {
throw new Error("Group by column must be categorical");
}
return by
? _histogramCategoricalBy(column, by)
: _histogramCategorical(column);
}
/*
Memoization hash for histogramCategorical()
*/
export function hashCategorical(column: DataframeColumn): string {
export function hashCategorical(column, by) {
if (by) {
return `${column.__id}:${by.__id}`;
}
return `${column.__id}:`;
}
export function hashCategoricalBy(
column: DataframeColumn,
by: DataframeColumn
): string {
return `${column.__id}:${by.__id}`;
/*
Bin counts for continuous/scalar values, with optional group-by category.
Values outside domain are ignored.
*/
export function histogramContinuous(column, bins = 40, domain = [0, 1], by) {
if (by && isTypedArray(by)) {
throw new Error("Group by column must be categorical");
}
const [min, max] = domain;
return by
? _histogramContinuousBy(column, bins, min, max, by)
: _histogramContinuous(column, bins, min, max);
}
/*
Memoization hash for histogramContinuous
*/
export function hashContinuous(
column: DataframeColumn,
bins: number,
domain: [number, number]
): string {
export function hashContinuous(column, bins = "", domain = [0, 0], by) {
const [min, max] = domain;
if (by) {
return `${column.__id}:${bins}:${min}:${max}:${by.__id}`;
}
return `${column.__id}::${bins}:${min}:${max}`;
}
export function hashContinuousBy(
column: DataframeColumn,
bins: number,
domain: [number, number],
by: DataframeColumn
): string {
const [min, max] = domain;
return `${column.__id}:${bins}:${min}:${max}:${by.__id}`;
}
client/src/util/dataframe/index.js
+8
View File
@@ -0,0 +1,8 @@
export { default as Dataframe } from "./dataframe";
export {
DenseInt32Index,
IdentityInt32Index,
KeyIndex,
isLabelIndex,
} from "./labelIndex";
export { default as dataframeMemo } from "./cache";
client/src/util/dataframe/index.ts
-21
View File
@@ -1,21 +0,0 @@
export { default as Dataframe } from "./dataframe";
export {
DenseInt32Index,
IdentityInt32Index,
KeyIndex,
isLabelIndex,
} from "./labelIndex";
export { default as dataframeMemo } from "./cache";
export type {
LabelType,
DataframeValue,
DataframeValueArray,
DataframeColumn,
ContinuousHistogram,
ContinuousHistogramBy,
CategoricalHistogram,
CategoricalHistogramBy,
ContinuousColumnSummary,
CategoricalColumnSummary,
} from "./types";
export type { LabelIndex } from "./labelIndex";
client/src/util/dataframe/labelIndex.js
+396
View File
@@ -0,0 +1,396 @@
/* eslint-disable max-classes-per-file -- Classes are interrelated*/
/**
Label indexing - map a label to & from an integer offset. See Dataframe
for how this is used.
**/
import { rangeFill as fillRange } from "../range";
import { __getMemoId } from "./util";
/*
Private utility functions
*/
function extent(tarr) {
let min = 0x7fffffff;
let max = ~min; // eslint-disable-line no-bitwise -- Establishes 0 of same size
for (let i = 0, l = tarr.length; i < l; i += 1) {
const v = tarr[i];
if (v < min) {
min = v;
}
if (v > max) {
max = v;
}
}
return [min, max];
}
class IdentityInt32Index {
/*
identity/noop index, with small assumptions that labels are int32
*/
constructor(maxOffset) {
this.maxOffset = maxOffset;
}
get __id() {
return `IdentityInt32Index_${this.maxOffset}`;
}
labels() {
// memoize
const k = fillRange(new Int32Array(this.maxOffset));
this.labels = function labels() {
return k;
};
return k;
}
getOffset(i) {
// label to offset
return Number.isInteger(i) && i >= 0 && i < this.maxOffset ? i : undefined;
}
getOffsets(arr) {
// labels to offsets
return arr.map((i) => this.getOffset(i));
}
getLabel(i) {
// offset to label
return Number.isInteger(i) && i >= 0 && i < this.maxOffset ? i : undefined;
}
getLabels(arr) {
// offsets to labels
return arr.map((i) => this.getLabel(i));
}
size() {
return this.maxOffset;
}
__promote(labelArray) {
/*
time/space decision - based on the resulting density
*/
const [minLabel, maxLabel] = extent(labelArray);
if (minLabel === 0 && maxLabel === labelArray.length - 1)
return new IdentityInt32Index(labelArray.length);
const labelSpaceSize = maxLabel - minLabel + 1;
const density = labelSpaceSize / this.maxOffset;
/* 0.1 is a magic number, that needs testing to optimize */
if (density < 0.1) {
return new KeyIndex(labelArray);
}
return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
}
subset(labels) {
/* validate subset */
const { maxOffset } = this;
for (let i = 0, l = labels.length; i < l; i += 1) {
const label = labels[i];
if (!Number.isInteger(label) || label < 0 || label >= maxOffset)
throw new RangeError(`offset or label: ${label}`);
}
return this.__promote(labels);
}
/* identity index - labels are offsets */
isubset(offsets) {
return this.subset(offsets);
}
/* identity index - labels are offsets */
isubsetMask(mask) {
let count = 0;
if (mask.length !== this.maxOffset) {
throw new RangeError("mask has invalid length for index");
}
let labels = new Int32Array(mask.length);
for (let i = 0, l = mask.length; i < l; i += 1) {
if (mask[i]) {
labels[count] = i;
count += 1;
}
}
labels = labels.slice(0, count);
return this.subset(labels);
}
withLabel(label) {
if (label === this.maxOffset) {
return new IdentityInt32Index(label + 1);
}
return this.__promote([...this.labels(), label]);
}
withLabels(labels) {
return this.__promote([...this.labels(), ...labels]);
}
dropLabel(label) {
if (label === this.maxOffset - 1) {
return new IdentityInt32Index(label);
}
const labelArray = [...this.labels()];
labelArray.splice(labelArray.indexOf(label), 1);
return this.__promote(labelArray);
}
}
class DenseInt32Index {
/*
DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
for both forward and reverse indexing. This means that the min/max range
of the forward index labels must be known a priori (so that the index
array can be pre-allocated).
*/
constructor(labels, labelRange = null) {
if (labels.constructor !== Int32Array) {
labels = new Int32Array(labels);
}
if (!labelRange) {
labelRange = extent(labels);
}
const [minLabel, maxLabel] = labelRange;
const labelSpaceSize = maxLabel - minLabel + 1;
const index = new Int32Array(labelSpaceSize).fill(-1);
for (let i = 0, l = labels.length; i < l; i += 1) {
const label = labels[i];
index[label - minLabel] = i;
}
this.minLabel = minLabel;
this.rindex = labels;
this.index = index;
this.__id = __getMemoId();
this.__compile();
}
__compile() {
const { minLabel, index, rindex } = this;
this.getOffset = function getOffset(l) {
if (!Number.isInteger(l)) return undefined;
const offset = index[l - minLabel];
return offset === -1 ? undefined : offset;
};
this.getOffsets = function getOffsets(arr) {
return arr.map((i) => this.getOffset(i));
};
this.getLabel = function getLabel(i) {
return Number.isInteger(i) ? rindex[i] : undefined;
};
this.getLabels = function getLabels(arr) {
return arr.map((i) => this.getLabel(i));
};
}
labels() {
return this.rindex;
}
size() {
return this.rindex.length;
}
__promote(labelArray) {
/*
time/space decision - if we are going to use less than 10% of the
dense index space, switch to a KeyIndex (which is slower, but uses
less memory for sparse label spaces).
*/
const [minLabel, maxLabel] = extent(labelArray);
const labelSpaceSize = maxLabel - minLabel + 1;
const density = labelSpaceSize / this.rindex.length;
/* 0.1 is a magic number, that needs testing to optimize */
if (density < 0.1) {
return new KeyIndex(labelArray);
}
return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
}
subset(labels) {
/* validate subset */
for (let i = 0, l = labels.length; i < l; i += 1) {
const label = labels[i];
const offset = this.getOffset(label);
if (offset === undefined || offset === -1)
throw new RangeError(`unknown label: ${label}`);
}
return this.__promote(labels);
}
isubset(offsets) {
/* validate subset */
const { rindex } = this;
const maxOffset = rindex.length;
const labels = new Int32Array(offsets.length);
for (let i = 0, l = offsets.length; i < l; i += 1) {
const offset = offsets[i];
if (offset < 0 || offset >= maxOffset)
throw new RangeError(`out of bounds offset: ${offset}`);
labels[i] = rindex[offset];
}
return this.__promote(labels);
}
isubsetMask(mask) {
const { rindex } = this;
if (mask.length !== rindex.length)
throw new RangeError("mask has invalid length for index");
let count = 0;
let labels = new Int32Array(mask.length);
for (let i = 0, l = mask.length; i < l; i += 1) {
if (mask[i]) {
labels[count] = rindex[i];
count += 1;
}
}
labels = labels.slice(0, count);
return this.__promote(labels);
}
withLabel(label) {
return this.__promote([...this.labels(), label]);
}
withLabels(labels) {
return this.__promote([...this.labels(), ...labels]);
}
dropLabel(label) {
const labelArray = [...this.labels()];
labelArray.splice(labelArray.indexOf(label), 1);
return this.__promote(labelArray);
}
}
class KeyIndex {
/*
KeyIndex indexes arbitrary JS primitive types, and uses a Map()
as its core data structure.
*/
constructor(labels) {
const index = new Map();
if (labels === undefined) {
labels = [];
}
const rindex = labels;
labels.forEach((v, i) => {
index.set(v, i);
});
if (index.size !== rindex.length) {
/* if true, there was a duplicate in the keys */
throw new Error("duplicate label provided to KeyIndex");
}
this.index = index;
this.rindex = rindex;
this.__id = __getMemoId();
this.__compile();
}
__compile() {
const { index, rindex } = this;
this.getOffset = function getOffset(k) {
return index.get(k);
};
this.getOffsets = function getOffsets(arr) {
return arr.map((l) => this.getOffset(l));
};
this.getLabel = function getLabel(i) {
return Number.isInteger(i) ? rindex[i] : undefined;
};
this.getLabels = function getLabels(arr) {
return arr.map((i) => this.getLabel(i));
};
}
labels() {
return this.rindex;
}
size() {
return this.rindex.length;
}
subset(labels) {
/* validate subset */
for (let i = 0, l = labels.length; i < l; i += 1) {
const label = labels[i];
const offset = this.getOffset(label);
if (offset === undefined || offset === -1) {
throw new RangeError(`unknown label: ${label}`);
}
}
return new KeyIndex(labels);
}
isubset(offsets) {
const { rindex } = this;
const maxOffset = rindex.length;
const labels = new Array(offsets.length);
for (let i = 0, l = offsets.length; i < l; i += 1) {
const offset = offsets[i];
if (offset < 0 || offset >= maxOffset)
throw new RangeError(`out of bounds offset: ${offset}`);
labels[i] = rindex[offset];
}
return new KeyIndex(labels);
}
isubsetMask(mask) {
const { rindex } = this;
if (mask.length !== rindex.length)
throw new RangeError("mask has invalid length for index");
let labels = new Array(mask.length);
let count = 0;
for (let i = 0, l = mask.length; i < l; i += 1) {
if (mask[i]) {
labels[count] = rindex[i];
count += 1;
}
}
labels = labels.slice(0, count);
return new KeyIndex(labels);
}
withLabel(label) {
return new KeyIndex([...this.rindex, label]);
}
withLabels(labels) {
return new KeyIndex([...this.rindex, ...labels]);
}
dropLabel(label) {
const idx = this.rindex.indexOf(label);
const labelArray = [...this.rindex];
labelArray.splice(idx, 1);
return new KeyIndex(labelArray);
}
}
function isLabelIndex(i) {
return (
i instanceof IdentityInt32Index ||
i instanceof DenseInt32Index ||
i instanceof KeyIndex
);
}
export { DenseInt32Index, IdentityInt32Index, KeyIndex, isLabelIndex };
/* eslint-enable max-classes-per-file -- enable*/
client/src/util/dataframe/labelIndex.ts
-495
View File
@@ -1,495 +0,0 @@
/* eslint-disable max-classes-per-file -- Classes are interrelated*/
/**
Label indexing - map a label to & from an integer offset. See Dataframe
for how this is used.
**/
import { rangeFill as fillRange } from "../range";
import { __getMemoId } from "./util";
import { OffsetArray, LabelType, LabelArray, GenericLabelArray } from "./types";
export abstract class LabelIndexBase {
readonly __id: string; // memoization helper
constructor(id: string) {
this.__id = id;
}
abstract labels(): LabelArray;
/**
* Look up the offset for the label.
*
* @param label - label to look up
* @returns - offset number or -1 if not found.
*/
abstract getOffset(label: LabelType): number;
getOffsets(labels: LabelArray): Int32Array {
// labels to offsets
const result = new Int32Array(labels.length);
for (let i = 0; i < labels.length; i += 1) {
result[i] = this.getOffset(labels[i]);
}
return result;
}
/**
* Look up the label for the offset.
*
* @param offset - offset to look up
* @returns - label or undefined if not found.
*/
abstract getLabel(offset: number): LabelType | undefined;
getLabels(offsets: OffsetArray): (LabelType | undefined)[] {
// offsets to labels
const result = new Array(offsets.length);
for (let i = 0; i < offsets.length; i += 1) {
result[i] = this.getLabel(offsets[i]);
}
return result;
}
abstract size(): number;
abstract subset(labels: LabelArray): LabelIndexBase;
abstract isubset(offsets: OffsetArray): LabelIndexBase;
abstract isubsetMask(mask: Uint8Array | boolean[]): LabelIndexBase;
abstract withLabel(label: LabelType): LabelIndexBase;
abstract withLabels(labels: LabelArray): LabelIndexBase;
abstract dropLabel(label: LabelType): LabelIndexBase;
}
export class IdentityInt32Index extends LabelIndexBase {
readonly maxOffset: number;
/*
identity/noop index, with small assumptions that labels are int32
*/
constructor(maxOffset: number) {
super(`IdentityInt32Index_${maxOffset}`);
this.maxOffset = maxOffset;
}
labels(): LabelArray {
// memoize
const k = fillRange(new Int32Array(this.maxOffset));
this.labels = function labels() {
return k;
};
return k;
}
getOffset(label: LabelType): number {
// label to offset
return Number.isInteger(label) && label >= 0 && label < this.maxOffset
? (label as number)
: -1;
}
getLabel(offset: number): number | undefined {
// offset to label
return Number.isInteger(offset) && offset >= 0 && offset < this.maxOffset
? offset
: undefined;
}
size(): number {
return this.maxOffset;
}
/** @internal */
__promote(labelArray: LabelArray, allInts: boolean): LabelIndexBase {
/*
time/space decision - based on the resulting density
*/
if (labelArray.length === 0) return new KeyIndex(Array.from(labelArray));
if (allInts) {
const [minLabel, maxLabel] = extent(
labelArray as GenericLabelArray<number> // safe, as allInts is true
);
if (minLabel === 0 && maxLabel === labelArray.length - 1)
return new IdentityInt32Index(labelArray.length);
const labelSpaceSize = maxLabel - minLabel + 1;
const density = labelSpaceSize / this.maxOffset;
/* 0.1 is a magic number which needs testing to optimize */
if (density < 0.1) {
return new KeyIndex(Array.from(labelArray));
}
return new DenseInt32Index(labelArray as GenericLabelArray<number>, [
minLabel,
maxLabel,
]);
}
return new KeyIndex(Array.from(labelArray));
}
subset(labels: LabelArray): LabelIndexBase {
/* validate subset */
const { maxOffset } = this;
for (let i = 0, l = labels.length; i < l; i += 1) {
const label = labels[i];
if (!Number.isInteger(label) || label < 0 || label >= maxOffset)
throw new RangeError(`label: ${label}`);
}
return this.__promote(labels, true);
}
/* identity index - labels are offsets */
isubset(offsets: OffsetArray): LabelIndexBase {
/* validate isubset */
const { maxOffset } = this;
for (let i = 0, l = offsets.length; i < l; i += 1) {
const offset = offsets[i];
if (!Number.isInteger(offset) || offset < 0 || offset >= maxOffset)
throw new RangeError(`offset: ${offset}`);
}
if (!(offsets instanceof Int32Array)) {
offsets = new Int32Array(offsets);
}
return this.__promote(offsets, true);
}
/* identity index - labels are offsets */
isubsetMask(mask: Uint8Array | boolean[]): LabelIndexBase {
let count = 0;
if (mask.length !== this.maxOffset) {
throw new RangeError("mask has invalid length for index");
}
let labels = new Int32Array(mask.length);
for (let i = 0, l = mask.length; i < l; i += 1) {
if (mask[i]) {
labels[count] = i;
count += 1;
}
}
labels = labels.slice(0, count);
return this.subset(labels);
}
withLabel(label: LabelType): LabelIndexBase {
if (label === this.maxOffset) {
return new IdentityInt32Index(label + 1);
}
return this.__promote([...this.labels(), label], Number.isInteger(label));
}
withLabels(labels: LabelArray): LabelIndexBase {
return this.__promote(
[...this.labels(), ...labels],
labels.every(Number.isInteger)
);
}
dropLabel(label: LabelType): LabelIndexBase {
if (!Number.isInteger(label) || label < 0 || label > this.maxOffset - 1)
throw new RangeError("Invalid label.");
if (label === this.maxOffset - 1) {
return new IdentityInt32Index(label);
}
const labelArray = [...this.labels()];
labelArray.splice(labelArray.indexOf(label as number), 1);
return this.__promote(labelArray, true);
}
}
export class DenseInt32Index extends LabelIndexBase {
getLabel: (offset: number) => number | undefined;
getOffset: (label: LabelType) => number;
index: Int32Array;
minLabel: number;
rindex: Int32Array;
/*
DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
for both forward and reverse indexing. This means that the min/max range
of the forward index labels must be known a priori (so that the index
array can be pre-allocated).
*/
constructor(
labels: GenericLabelArray<number>,
labelRange?: [number, number]
) {
super(__getMemoId());
const int32Labels =
labels instanceof Int32Array ? labels : new Int32Array(labels);
if (!labelRange) {
labelRange = extent(int32Labels);
}
const [minLabel, maxLabel] = labelRange;
const labelSpaceSize = maxLabel - minLabel + 1;
const index = new Int32Array(labelSpaceSize).fill(-1);
for (let i = 0, l = labels.length; i < l; i += 1) {
const label = labels[i];
index[label - minLabel] = i;
}
this.minLabel = minLabel;
this.rindex = int32Labels;
this.index = index;
this.getOffset = function getOffset(label: LabelType) {
if (!Number.isInteger(label)) return -1;
const lblIdx: number = <number>label - minLabel;
if (lblIdx < 0 || lblIdx >= index.length) return -1;
const offset = index[lblIdx];
return offset;
};
this.getLabel = function getLabel(offset: number) {
return Number.isInteger(offset) ? labels[offset] : undefined;
};
}
labels(): LabelArray {
return this.rindex;
}
size(): number {
return this.rindex.length;
}
/** @internal */
__promote(labelArray: LabelArray, allInts: boolean): LabelIndexBase {
/*
time/space decision - if we are going to use less than 10% of the
dense index space, switch to a KeyIndex (which is slower, but uses
less memory for sparse label spaces).
*/
if (labelArray.length === 0) return new KeyIndex(Array.from(labelArray));
if (allInts) {
if (!(labelArray instanceof Int32Array)) {
labelArray = new Int32Array(labelArray as number[]);
}
const [minLabel, maxLabel] = extent(
labelArray as GenericLabelArray<number> // safe, as allInts is true
);
const labelSpaceSize = maxLabel - minLabel + 1;
const density = labelSpaceSize / this.rindex.length;
/* 0.1 is a magic number, that needs testing to optimize */
if (density < 0.1) {
return new KeyIndex(Array.from(labelArray));
}
return new DenseInt32Index(labelArray as GenericLabelArray<number>, [
minLabel,
maxLabel,
]);
}
return new KeyIndex(Array.from(labelArray));
}
subset(labels: LabelArray): LabelIndexBase {
/* validate subset */
for (let i = 0, l = labels.length; i < l; i += 1) {
const label = labels[i]; // if not a number, getOffset will error
const offset = this.getOffset(label as number);
if (offset === -1) throw new RangeError(`unknown label: ${label}`);
}
return this.__promote(labels as GenericLabelArray<number>, true);
}
isubset(offsets: OffsetArray): LabelIndexBase {
/* validate subset */
const { rindex } = this;
const maxOffset = rindex.length;
const labels = new Int32Array(offsets.length);
for (let i = 0, l = offsets.length; i < l; i += 1) {
const offset = offsets[i];
if (offset < 0 || offset >= maxOffset)
throw new RangeError(`out of bounds offset: ${offset}`);
labels[i] = rindex[offset];
}
return this.__promote(labels, true);
}
isubsetMask(mask: Uint8Array | boolean[]): LabelIndexBase {
const { rindex } = this;
if (mask.length !== rindex.length)
throw new RangeError("mask has invalid length for index");
let count = 0;
let labels = new Int32Array(mask.length);
for (let i = 0, l = mask.length; i < l; i += 1) {
if (mask[i]) {
labels[count] = rindex[i];
count += 1;
}
}
labels = labels.slice(0, count);
return this.__promote(labels, true);
}
withLabel(label: LabelType): LabelIndexBase {
return this.__promote([...this.labels(), label], Number.isInteger(label));
}
withLabels(labels: LabelArray): LabelIndexBase {
return this.__promote(
[...this.labels(), ...labels],
labels.every(Number.isInteger)
);
}
dropLabel(label: LabelType): LabelIndexBase {
if (!Number.isInteger(label)) throw new RangeError("Invalid label.");
const labelArray = [...this.labels()];
labelArray.splice(labelArray.indexOf(label as number), 1);
return this.__promote(
new Int32Array(labelArray as GenericLabelArray<number>),
true
);
}
}
export class KeyIndex extends LabelIndexBase {
getLabel: (offset: number) => LabelType | undefined;
getOffset: (label: LabelType) => number | -1;
index: Map<string | number, number>;
rindex: (string | number)[];
/*
KeyIndex indexes arbitrary JS primitive types, and uses a Map()
as its core data structure.
*/
constructor(labels: Array<string | number>) {
super(__getMemoId());
const index = new Map<string | number, number>();
if (labels === undefined) {
labels = [];
}
if (!Array.isArray(labels)) {
labels = Array.from(labels);
}
const rindex = labels;
labels.forEach((v, i) => {
index.set(v, i);
});
if (index.size !== rindex.length) {
/* if true, there was a duplicate in the keys */
throw new Error("duplicate label provided to KeyIndex");
}
this.index = index;
this.rindex = rindex;
this.getOffset = function getOffset(label: LabelType) {
const offset = index.get(label);
if (offset === undefined) return -1;
return offset;
};
this.getLabel = function getLabel(offset: number) {
return Number.isInteger(offset) ? rindex[offset] : undefined;
};
}
labels(): LabelArray {
return this.rindex;
}
size(): number {
return this.rindex.length;
}
subset(labels: (string | number)[]): LabelIndexBase {
/* validate subset */
for (let i = 0, l = labels.length; i < l; i += 1) {
const label = labels[i];
const offset = this.getOffset(label);
if (offset === undefined || offset === -1) {
throw new RangeError(`unknown label: ${label}`);
}
}
return new KeyIndex(labels);
}
isubset(offsets: OffsetArray): LabelIndexBase {
const { rindex } = this;
const maxOffset = rindex.length;
const labels = new Array(offsets.length);
for (let i = 0, l = offsets.length; i < l; i += 1) {
const offset = offsets[i];
if (offset < 0 || offset >= maxOffset)
throw new RangeError(`out of bounds offset: ${offset}`);
labels[i] = rindex[offset];
}
return new KeyIndex(labels);
}
isubsetMask(mask: Uint8Array | boolean[]): LabelIndexBase {
const { rindex } = this;
if (mask.length !== rindex.length)
throw new RangeError("mask has invalid length for index");
let labels = new Array(mask.length);
let count = 0;
for (let i = 0, l = mask.length; i < l; i += 1) {
if (mask[i]) {
labels[count] = rindex[i];
count += 1;
}
}
labels = labels.slice(0, count);
return new KeyIndex(labels);
}
withLabel(label: LabelType): LabelIndexBase {
return new KeyIndex([...this.rindex, label]);
}
withLabels(labels: LabelArray): LabelIndexBase {
return new KeyIndex([...this.rindex, ...labels]);
}
dropLabel(label: LabelType): LabelIndexBase {
const idx = this.rindex.indexOf(label);
const labelArray = [...this.rindex];
labelArray.splice(idx, 1);
return new KeyIndex(labelArray);
}
}
export type LabelIndex = LabelIndexBase;
export function isLabelIndex(i: unknown): i is LabelIndex {
return (
i instanceof LabelIndexBase ||
i instanceof IdentityInt32Index ||
i instanceof DenseInt32Index ||
i instanceof KeyIndex
);
}
/** @internal */
function extent(tarr: GenericLabelArray<number>): [number, number] {
let min = 0x7fffffff;
let max = ~min; // eslint-disable-line no-bitwise -- Establishes 0 of same size
for (let i = 0, l = tarr.length; i < l; i += 1) {
const v = tarr[i];
if (v < min) {
min = v;
}
if (v > max) {
max = v;
}
}
return [min, max];
}
/* eslint-enable max-classes-per-file -- enable*/
client/src/util/dataframe/summarize.js
+85
View File
@@ -0,0 +1,85 @@
/*
Private dataframe support functions
TODO / XXX: for scalar/continuous data, this uses a naive method
of computing quantiles. Would be good to switch from sort to
partition at some point.
*/
import quantile from "../quantile";
import { sortArray } from "../typedCrossfilter/sort";
// [ 0, 0.01, 0.02, ..., 1.0]
const centileNames = new Array(101).fill(0).map((v, idx) => idx / 100);
export function summarizeContinuous(col) {
let min;
let max;
let nan = 0;
let pinf = 0;
let ninf = 0;
let percentiles;
if (col) {
// -Inf < finite < Inf < NaN
const sortedCol = sortArray(new col.constructor(col));
// count non-finites, which are at each end of sorted data
for (let i = sortedCol.length - 1; i >= 0; i -= 1) {
if (!Number.isNaN(sortedCol[i])) {
nan = sortedCol.length - i - 1;
break;
}
}
for (let i = 0, l = sortedCol.length; i < l; i += 1) {
if (sortedCol[i] !== Number.NEGATIVE_INFINITY) {
ninf = i;
break;
}
}
for (let i = sortedCol.length - nan - 1; i >= 0; i -= 1) {
if (sortedCol[i] !== Number.POSITIVE_INFINITY) {
pinf = sortedCol.length - i - nan - 1;
break;
}
}
// compute percentiles on finite data ONLY
const sortedColFiniteOnly = sortedCol.slice(
ninf,
sortedCol.length - nan - pinf
);
percentiles = quantile(centileNames, sortedColFiniteOnly, true);
min = percentiles[0];
max = percentiles[100];
}
return {
categorical: false,
min,
max,
nan,
pinf,
ninf,
percentiles,
};
}
export function summarizeCategorical(col) {
const categoryCounts = new Map();
if (col) {
for (let r = 0, l = col.length; r < l; r += 1) {
const val = col[r];
let curCount = categoryCounts.get(val);
if (curCount === undefined) curCount = 0;
categoryCounts.set(val, curCount + 1);
}
}
const sortedCategoryByCounts = new Map(
[...categoryCounts.entries()].sort((a, b) => b[1] - a[1])
);
return {
categorical: true,
categories: [...sortedCategoryByCounts.keys()],
categoryCounts: sortedCategoryByCounts,
numCategories: sortedCategoryByCounts.size,
};
}
client/src/util/dataframe/summarize.ts
-88
View File
@@ -1,88 +0,0 @@
/*
Private dataframe support functions
TODO / XXX: for scalar/continuous data, this uses a naive method
of computing quantiles. Would be good to switch from sort to
partition at some point.
*/
import {
AnyArray,
GenericArrayConstructor,
} from "../../common/types/arraytypes";
import { ContinuousColumnSummary, CategoricalColumnSummary } from "./types";
import quantile from "../quantile";
import { sortArray } from "../typedCrossfilter/sort";
// [ 0, 0.01, 0.02, ..., 1.0]
const centileNames = new Array(101).fill(0).map((_v, idx) => idx / 100);
export function summarizeContinuous(col: AnyArray): ContinuousColumnSummary {
let nan = 0;
let pinf = 0;
let ninf = 0;
// -Inf < finite < Inf < NaN
const sortedCol = sortArray(
new (col.constructor as GenericArrayConstructor<typeof col>)(col)
);
// count non-finites, which are at each end of sorted data
for (let i = sortedCol.length - 1; i >= 0; i -= 1) {
if (!Number.isNaN(sortedCol[i])) {
nan = sortedCol.length - i - 1;
break;
}
}
for (let i = 0, l = sortedCol.length; i < l; i += 1) {
if (sortedCol[i] !== Number.NEGATIVE_INFINITY) {
ninf = i;
break;
}
}
for (let i = sortedCol.length - nan - 1; i >= 0; i -= 1) {
if (sortedCol[i] !== Number.POSITIVE_INFINITY) {
pinf = sortedCol.length - i - nan - 1;
break;
}
}
// compute percentiles on finite data ONLY
const sortedColFiniteOnly = sortedCol.slice(
ninf,
sortedCol.length - nan - pinf
);
const percentiles = quantile(centileNames, sortedColFiniteOnly, true);
const min = percentiles[0];
const max = percentiles[100];
return {
categorical: false,
min,
max,
nan,
pinf,
ninf,
percentiles,
};
}
export function summarizeCategorical(col: AnyArray): CategoricalColumnSummary {
const categoryCounts = new Map();
if (col) {
for (let r = 0, l = col.length; r < l; r += 1) {
const val = col[r];
let curCount = categoryCounts.get(val);
if (curCount === undefined) curCount = 0;
categoryCounts.set(val, curCount + 1);
}
}
const sortedCategoryByCounts = new Map(
[...categoryCounts.entries()].sort((a, b) => b[1] - a[1])
);
return {
categorical: true,
categories: [...sortedCategoryByCounts.keys()],
categoryCounts: sortedCategoryByCounts,
numCategories: sortedCategoryByCounts.size,
};
}
client/src/util/dataframe/types.ts
-139
View File
@@ -1,139 +0,0 @@
import { TypedArray } from "../../common/types/arraytypes";
export type LabelType = number | string;
type CommonProps<A, B> = {
[K in keyof A & keyof B]: A[K] | B[K];
};
export type GenericLabelArray<T> = CommonProps<Array<T>, Int32Array>;
export type LabelArray = GenericLabelArray<number | string>;
export type OffsetType = number;
export type OffsetArray =
| Int8Array
| Uint8Array
| Int16Array
| Uint16Array
| Int32Array
| Uint32Array
| number[];
export type ContinuousColumnSummary = {
categorical: false;
min: number;
max: number;
nan: number;
pinf: number;
ninf: number;
percentiles: number[];
};
export type CategoricalColumnSummary = {
categorical: true;
categories: (number | string | boolean)[];
categoryCounts: Map<number | string | boolean, number>;
numCategories: number;
};
export type ColumnSummary = ContinuousColumnSummary | CategoricalColumnSummary;
export type ContinuousHistogram = number[];
export type ContinuousHistogramBy = Map<DataframeValue, ContinuousHistogram>;
export type CategoricalHistogram = Map<DataframeValue, number>;
export type CategoricalHistogramBy = Map<DataframeValue, CategoricalHistogram>;
export type DataframeValue = number | string | boolean;
export type DataframeValueArray = DataframeValue[] | TypedArray;
export type DataframeColumnGetter = (
label: LabelType
) => DataframeValue | undefined;
/**
* Interface representing a Dataframe column. Eg, returned by
* Dataframe.col().
*/
export interface DataframeColumn extends DataframeColumnGetter {
/**
* __id is unique per Dataframe and DataframeColumn, and is used as a memoization key.
*/
readonly __id: string;
/**
* Boolean indicating if the underlying data supports continuous operations, eg,
* summarizeContinuous.
*/
isContinuous: boolean;
/**
* Return underlying column data as an array-like object.
*/
asArray: () => DataframeValueArray;
/**
* Continuous data summary. Will throw if !isContinuous.
*/
summarizeContinuous: () => ContinuousColumnSummary;
/**
* Categorical data summary.
*/
summarizeCategorical: () => CategoricalColumnSummary;
/**
* Continuous bin/histogram. Will throw if !isContinuous.
* @param bins - array of bin boundary fractions, in range [0., 1.]
* @param domain - data domain [min, max]
*/
histogramContinuous: (
bins: number,
domain: [number, number]
) => ContinuousHistogram;
/**
* Continuous bin/histogram, grouped by another categorical column. Will throw if !isContinuous.
* @param bins - array of bin boundary fractions, in range [0., 1.]
* @param domain - data domain [min, max]
* @param by - group by categorical column
*/
histogramContinuousBy: (
bins: number,
domain: [number, number],
by: DataframeColumn
) => ContinuousHistogramBy;
/**
* Categorical bin/histogram.
*/
histogramCategorical: () => CategoricalHistogram;
/**
* Categorical bin/histogram grouped by another column.
* @param by - group by categorical column
*/
histogramCategoricalBy: (by: DataframeColumn) => CategoricalHistogramBy;
/**
* Return true if the column contains the row label.
*/
has: (rlabel: LabelType) => boolean;
/**
* Return true if the column contains the row offset. Identical to
* (offset >= 0 && offset < dataframe.length)
*/
ihas: (offset: OffsetType) => boolean;
/**
* Return index of the value, as a _label_. Returns undefined if
* not present. *NOTE*: unlike Array.indexOf, does not return an
* offset.
*/
indexOf: (value: DataframeValue) => LabelType | undefined;
/**
* Return the value at the given offset, or undefined if not present.
*/
iget: (offset: OffsetType) => DataframeValue | undefined;
}
client/src/util/dataframe/util.ts → client/src/util/dataframe/util.js
+12 -20
View File
@@ -2,19 +2,18 @@
Private utility code for dataframe
*/
export function callOnceLazy<
// eslint-disable-next-line @typescript-eslint/no-explicit-any --- a legitimate use of any.
T extends (...args: any[]) => any = (...args: any[]) => any
>(fn: T): (...args: Parameters<T>) => ReturnType<T> {
export { isTypedArray, isArrayOrTypedArray } from "../typeHelpers";
export function callOnceLazy(f) {
/*
call function once, and save the result, regardless of arguments (this is not
the same as typical memoization).
*/
let value: ReturnType<T>;
let value;
let calledOnce = false;
const result = function result(...args: Parameters<T>): ReturnType<T> {
const result = function result(...args) {
if (!calledOnce) {
value = fn(...args);
value = f(...args);
calledOnce = true;
}
return value;
@@ -22,14 +21,7 @@ export function callOnceLazy<
return result;
}
export function memoize<
// eslint-disable-next-line @typescript-eslint/no-explicit-any --- a legitimate use of any.
T extends (...args: any[]) => any = (...args: any[]) => any
>(
fn: T,
hashFn: (...args: Parameters<T>) => string,
maxResultsCached = -1
): (...args: Parameters<T>) => ReturnType<T> {
export function memoize(fn, hashFn, maxResultsCached = -1) {
/*
function memoization, with user-provided hash. hashFn must return a
key which will be unique as a Map key (ie, obeys "sameValueZero" algorithm
@@ -37,7 +29,7 @@ export function memoize<
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map#Key_equality
*/
const cache = new Map();
const wrap = function wrap(...args: Parameters<T>): ReturnType<T> {
const wrap = function wrap(...args) {
const key = hashFn(...args);
if (cache.has(key)) {
return cache.get(key);
@@ -62,11 +54,11 @@ export function memoize<
}
/**
*memoization helpers - just a global counter.
*/
memoization helpers - just a global counter.
**/
let __DataframeMemoId__ = 0;
export function __getMemoId(): string {
export function __getMemoId() {
const id = __DataframeMemoId__;
__DataframeMemoId__ += 1;
return id.toString();
return id;
}