fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-25 07:48:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

TS Revert (1) (#2402)

Browse Source 
* revert all commits to before Typescript migration

* update compat workflow to match latest deps (#2335)

* update compat workflow to match latest deps

* attempt to debug

* attempt to debug

* remove debugging code

* typo

* update deps to match desktop (#2340)

* fix: don't run lint with `--fix` on push tests (#2273)

* fix: don't run lint with `--fix` on push tests

* npx

Co-authored-by: maniarathi <mani.arathi@gmail.com>
Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>

* rename X_approx_distribution to X_approximate_distribution (#2337)

* Correctly handle non-finite numbers in heuristic determination of X distribution (#2342)

* handle non-finites explicitly

* improve and test edge case handling for distribution estimation

* revert debugging changes

* code readability

* clean up type inferencing (#2332)

* unit tests for 64 bit conversion

* clean up type handling

* type inference tests

* more type inference fixes

* use schema to determine user intent for data typing

* stop using deprecated API

* fbs type encoding test

* add missing test

* add more tests

* correctly infer X type for CXG adaptor

* lint

* fix typo

* ts migration

* cleanup from PR review

* lint

* PR review changes

* remove unused packages from client (#2359)

* remove unused packages from client

* add missing peer dep

* fix: disable FE auth testing on compatibility tests (#2377)

* update: release process (#2277)

Co-authored-by: maniarathi <mani.arathi@gmail.com>

* fix: remove spaces in param setup (#2380)

* delete deploy workflow (#2396)

* undo reformatting which now does not pass lint

* fix snapshots which changed due to npm dep changes

* add missing quoting to snapshot

* another snapshot typo fix

* TS Revert (2) - replay PR #2347 and #2354 (#2403)

* replay edits from PR 2347

* TS Revert (3) - replay edits in PR #2327 (#2404)

* replay edits in PR 2327

* TS Revert (4) - replay PR #2355 (#2405)

* replay edits in PR 2355

* add additional babel config

* reformat with new prettier config

Co-authored-by: Severiano Badajoz <sbadajoz@chanzuckerberg.com>
Co-authored-by: maniarathi <mani.arathi@gmail.com>
Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>
This commit is contained in:
Bruce Martin
2021-08-23 15:01:36 -07:00
committed by GitHub
co-authored by maniarathi Madison Dunitz Severiano Badajoz
parent 295590a7c6
commit eaae6df5e3
253 changed files with 11175 additions and 22271 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/src/annoMatrix/annoMatrix.ts → client/src/annoMatrix/annoMatrix.js
+209 -295
View File
@@ -2,9 +2,6 @@ import {
Dataframe,
IdentityInt32Index,
dataframeMemo,
LabelType,
DataframeValue,
DataframeValueArray,
} from "../util/dataframe";
import {
_getColumnDimensionNames,
@@ -13,71 +10,13 @@ import {
_getWritableColumns,
} from "./schema";
import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
import {
_whereCacheGet,
_whereCacheMerge,
WhereCache,
WhereCacheColumnLabels,
} from "./whereCache";
import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
import _shallowClone from "./clone";
import { _queryValidate, _queryCacheKey, Query } from "./query";
import { GCHints } from "../common/types/entities";
import {
AnnotationColumnSchema,
Category,
Field,
EmbeddingSchema,
Schema,
ArraySchema,
RawSchema,
} from "../common/types/schema";
import { LabelArray } from "../util/dataframe/types";
import { LabelIndexBase } from "../util/dataframe/labelIndex";
import { _queryValidate, _queryCacheKey } from "./query";
const _dataframeCache = dataframeMemo(128);
interface Cache {
[Field.obs]: Dataframe;
[Field.var]: Dataframe;
[Field.emb]: Dataframe;
[Field.X]: Dataframe;
}
interface PendingLoad {
[Field.obs]: { [key: string]: Promise<void> };
[Field.var]: { [key: string]: Promise<void> };
[Field.emb]: { [key: string]: Promise<void> };
[Field.X]: { [key: string]: Promise<void> };
}
export interface UserFlags {
isUserSubsetView?: boolean;
isEmbSubsetView?: boolean;
}
export default abstract class AnnoMatrix {
public isView: boolean;
public nObs: number;
public nVar: number;
public rowIndex: LabelIndexBase;
public schema: Schema;
public userFlags: UserFlags;
public viewOf: AnnoMatrix;
public _cache: Cache;
private _pendingLoad: PendingLoad;
private _whereCache: WhereCache;
private _gcInfo: Map<string, number>;
export default class AnnoMatrix {
/*
Abstract base class for all AnnoMatrix objects. This class provides a proxy
to the annotated matrix data authoritatively served by the server/back-end.
@@ -108,19 +47,14 @@ export default abstract class AnnoMatrix {
subset(annoMatrix, rowLabels) -> annoMatrix
etc.
*/
static fields(): Field[] {
static fields() {
/*
return the fields present in the AnnoMatrix instance.
*/
return [Field.obs, Field.var, Field.emb, Field.X];
return ["obs", "var", "emb", "X"];
}
constructor(
schema: RawSchema,
nObs: number,
nVar: number,
rowIndex: LabelIndexBase | null = null
) {
constructor(schema, nObs, nVar, rowIndex = null) {
/*
Private constructor - this is an abstract base class. Do not use.
*/
@@ -136,7 +70,7 @@ export default abstract class AnnoMatrix {
* rowIndex - a rowIndex shared by all data on this view (ie, the list of cells).
The row index labels are as defined by the base dataset from the server.
* isView - true if this is a view, false if not.
* viewOf - pointer to parent annomatrix if a view, self if not a view.
* viewOf - pointer to parent annomatrix if a view, undefined/null if not a view.
* userFlags - container for any additional state a user of this API wants to hang
off of an annoMatrix, and have propagated by the (shallow) cloning protocol.
*/
@@ -145,17 +79,17 @@ export default abstract class AnnoMatrix {
this.nVar = nVar;
this.rowIndex = rowIndex || new IdentityInt32Index(nObs);
this.isView = false;
this.viewOf = this;
this.viewOf = undefined;
this.userFlags = {};
/*
Private instance variables.
Private instance variables.
These are caches - lazily loaded. The only guarantee is that if they
are loaded, they will conform to the schema & dimensionality constraints.
These are caches - lazily loaded. The only guarantee is that if they
are loaded, they will conform to the schema & dimensionality constraints.
Do NOT use directly - instead, use the fetch() and preload() API.
*/
Do NOT use directly - instead, use the fetch() and preload() API.
*/
this._cache = {
obs: Dataframe.empty(this.rowIndex),
var: Dataframe.empty(this.rowIndex),
@@ -168,14 +102,14 @@ export default abstract class AnnoMatrix {
emb: {},
X: {},
};
this._whereCache = {} as WhereCache;
this._whereCache = {};
this._gcInfo = new Map();
}
/**
** Schema helper/accessors
**/
getMatrixColumns(field: Field): string[] {
getMatrixColumns(field) {
/*
Return array of column names in the field. ONLY supported on the
obs, var and emb fields. X currently unimplemented and will throw.
@@ -188,7 +122,7 @@ export default abstract class AnnoMatrix {
}
// eslint-disable-next-line class-methods-use-this -- need to be able to call this on instances
getMatrixFields(): Field[] {
getMatrixFields() {
/*
Return array of fields in this annoMatrix. Currently hard-wired to
return: ["X", "obs", "var", "emb"].
@@ -198,7 +132,7 @@ export default abstract class AnnoMatrix {
return AnnoMatrix.fields();
}
getColumnSchema(field: Field, col: LabelType): ArraySchema {
getColumnSchema(field, col) {
/*
Return the schema for the field & column ,eg,
@@ -210,7 +144,7 @@ export default abstract class AnnoMatrix {
return _getColumnSchema(this.schema, field, col);
}
getColumnDimensions(field: Field, col: LabelType): LabelArray | undefined {
getColumnDimensions(field, col) {
/*
Return the dimensions on this field / column. For most fields, which are 1D,
this just return the column name. Multi-dimensional columns, such as embeddings,
@@ -228,19 +162,19 @@ export default abstract class AnnoMatrix {
/**
** General utility methods
**/
base(): AnnoMatrix {
base() {
/*
return the base of view, or `this` if not a view.
*/
let annoMatrix = this._getViewOf();
while (annoMatrix.isView) annoMatrix = annoMatrix._getViewOf();
let annoMatrix = this;
while (annoMatrix.isView) annoMatrix = annoMatrix.viewOf;
return annoMatrix;
}
/**
** Load / read interfaces
**/
fetch(field: Field, q: Query | Query[]): Promise<Dataframe> {
fetch(field, q) {
/*
Return the given query on a single matrix field as a single dataframe.
Currently supports ONLY full column query.
@@ -269,7 +203,7 @@ export default abstract class AnnoMatrix {
1. Fetch the "n_genes" column the "obs":
const df = await fetch("obs", "n_genes")
console.log("Largest number of genes is: ", df.summarizeContinuous().max);
console.log("Largest number of genes is: ", df.summarize().max);
2. Fetch two separate columns from obs. Returns a single dataframe containing
the columns:
@@ -297,7 +231,7 @@ export default abstract class AnnoMatrix {
return this._fetch(field, q);
}
prefetch(field: Field, q: Query): void {
prefetch(field, q) {
/*
Start a data fetch & cache fill. Identical to fetch() except it does
not return a value.
@@ -306,6 +240,7 @@ export default abstract class AnnoMatrix {
overall component rendering latency.
*/
this._fetch(field, q);
return undefined;
}
/**
@@ -326,172 +261,176 @@ export default abstract class AnnoMatrix {
** The actual implementation is in the sub-classes, which MUST override these.
**/
/*
Add a new category value (aka "label") to a writable obs column, and return the new AnnoMatrix.
Typical use is to add a new user-created label to a user-created obs categorical
annotation.
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
addObsAnnoCategory(col, category) {
/*
Add a new category value (aka "label") to a writable obs column, and return the new AnnoMatrix.
Typical use is to add a new user-created label to a user-created obs categorical
annotation.
Will throw column does not exist or is not writable.
Will throw column does not exist or is not writable.
Example:
Example:
addObsAnnoCategory("my cell type", "left toenail") -> AnnoMatrix
*/
abstract addObsAnnoCategory(col: LabelType, category: string): AnnoMatrix;
/*
Remove a category value from an obs column, reassign any obs having that value
to the 'unassignedCategory' value, and return a promise for a new AnnoMatrix.
Typical use is to remove a user-created label from a user-created obs categorical
annotation.
Will throw column does not exist or is not writable.
An `unassignedCategory` value must be provided, for assignment to any obs/cells
that had the now-delete category label as their value.
Example:
await removeObsAnnoCategory("my-tissue-type", "right earlobe", "unassigned") -> AnnoMatrix
NOTE: method is async as it may need to fetch data to provide the reassignment.
*/
abstract removeObsAnnoCategory(
col: LabelType,
category: Category,
unassignedCategory: string
): Promise<AnnoMatrix>;
/*
Drop an entire writable column, eg a user-created obs annotation. Typical use
is to provide the "Delete Category" implementation. Returns the new AnnoMatrix.
Will throw if not a writable annotation.
Will throw column does not exist or is not writable.
Example:
dropObsColumn("old annotations") -> AnnoMatrix
*/
abstract dropObsColumn(col: LabelType): AnnoMatrix;
/*
Add a new writable OBS annotation column, with the caller-specified schema, initial value
type and value.
Value may be any one of:
* an array of values
* a primitive type, including null or undefined.
If an array, length must be the same as 'this.nObs', and constructor must equal 'Ctor'.
If a primitive, 'Ctor' will be used to create the initial value, which will be filled
with 'value'.
Throws if the name specified in 'colSchema' duplicates an existing obs column.
Returns a new AnnoMatrix.
Examples:
addObsColumn(
{ name: "foo", type: "categorical", categories: "unassigned" },
Array,
"unassigned"
) -> AnnoMatrix
*/
abstract addObsColumn<T extends DataframeValueArray>(
colSchema: AnnotationColumnSchema,
Ctor: new (n: number) => T,
value: T
): AnnoMatrix;
/*
Rename the obs column 'oldCol' to have name 'newCol' and returns new AnnoMatrix.
Will throw column does not exist or is not writable, or if 'newCol' is not unique.
Example:
renameObsColumn('cell type', 'old cell type') -> AnnoMatrix.
*/
abstract renameObsColumn(oldCol: LabelType, newCol: LabelType): AnnoMatrix;
/*
Set all obs with label in array 'obsLabels' to have 'value'. Typical use would be
to set a group of cells to have a label on a user-created categorical annotation
(eg set all selected cells to have a label).
NOTE: async method, as it may need to fetch.
Will throw column does not exist or is not writable.
Example:
await setObsColmnValues("flavor", [383, 400], "tasty") -> AnnoMtarix
*/
abstract setObsColumnValues(
col: LabelType,
obsLabels: Int32Array,
value: DataframeValue
): Promise<AnnoMatrix>;
/*
Set by value - all elements in the column with value 'oldValue' are set to 'newValue'.
Async method - returns a promise for a new AnnoMatrix.
Typical use would be to set all labels of one value to another.
Will throw column does not exist or is not writable.
Example:
await resetObsColumnValues("my notes", "good", "not-good") -> AnnoMatrix
addObsAnnoCategory("my cell type", "left toenail") -> AnnoMatrix
*/
abstract resetObsColumnValues<T extends DataframeValue>(
col: LabelType,
oldValue: T,
newValue: T
): Promise<AnnoMatrix>;
_subclassResponsibility();
}
/*
Add a new obs embedding to the AnnoMatrix, with provided schema.
Returns a new annomatrix.
Typical use will be to add a re-embedding that the server has calculated.
Will throw if the column schema is invalid (eg, duplicate name).
*/
abstract addEmbedding(colSchema: EmbeddingSchema): AnnoMatrix;
getCacheKeys(
field: Field,
query: Query
): WhereCacheColumnLabels | [undefined] {
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
async removeObsAnnoCategory(col, category, unassignedCategory) {
/*
Return cache keys for columns associated with this query. May return
[unknown] if no keys are known (ie, nothing is or was cached).
*/
Remove a category value from an obs column, reassign any obs having that value
to the 'unassignedCategory' value, and return a promise for a new AnnoMatrix.
Typical use is to remove a user-created label from a user-created obs categorical
annotation.
Will throw column does not exist or is not writable.
An `unassignedCategory` value must be provided, for assignment to any obs/cells
that had the now-delete category label as their value.
Example:
await removeObsAnnoCategory("my-tissue-type", "right earlobe", "unassigned") -> AnnoMatrix
NOTE: method is async as it may need to fetch data to provide the reassignment.
*/
_subclassResponsibility();
}
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
dropObsColumn(col) {
/*
Drop an entire writable column, eg a user-created obs annotation. Typical use
is to provide the "Delete Category" implementation. Returns the new AnnoMatrix.
Will throw if not a writable annotation.
Will throw column does not exist or is not writable.
Example:
dropObsColumn("old annotations") -> AnnoMatrix
*/
_subclassResponsibility();
}
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
addObsColumn(colSchema, Ctor, value) {
/*
Add a new writable OBS annotation column, with the caller-specified schema, initial value
type and value.
Value may be any one of:
* an array of values
* a primitive type, including null or undefined.
If an array, length must be the same as 'this.nObs', and constructor must equal 'Ctor'.
If a primitive, 'Ctor' will be used to create the initial value, which will be filled
with 'value'.
Throws if the name specified in 'colSchema' duplicates an existing obs column.
Returns a new AnnoMatrix.
Examples:
addObsColumn(
{ name: "foo", type: "categorical", categories: "unassigned" },
Array,
"unassigned"
) -> AnnoMatrix
*/
_subclassResponsibility();
}
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
renameObsColumn(oldCol, newCol) {
/*
Rename the obs column 'oldCol' to have name 'newCol' and returns new AnnoMatrix.
Will throw column does not exist or is not writable, or if 'newCol' is not unique.
Example:
renameObsColumn('cell type', 'old cell type') -> AnnoMatrix.
*/
_subclassResponsibility();
}
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
async setObsColumnValues(col, obsLabels, value) {
/*
Set all obs with label in array 'obsLabels' to have 'value'. Typical use would be
to set a group of cells to have a label on a user-created categorical anntoation
(eg set all selected cells to have a label).
NOTE: async method, as it may need to fetch.
Will throw column does not exist or is not writable.
Example:
await setObsColmnValues("flavor", [383, 400], "tasty") -> AnnoMtarix
*/
_subclassResponsibility();
}
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
async resetObsColumnValues(col, oldValue, newValue) {
/*
Set by value - all elements in the column with value 'oldValue' are set to 'newValue'.
Async method - returns a promise for a new AnnoMatrix.
Typical use would be to set all labels of one value to another.
Will throw column does not exist or is not writable.
Example:
await resetObsColumnValues("my notes", "good", "not-good") -> AnnoMatrix
*/
_subclassResponsibility();
}
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
addEmbedding(colSchema) {
/*
Add a new obs embedding to the AnnoMatrix, with provided schema.
Returns a new annomatrix.
Typical use will be to add a re-embedding that the server has calculated.
Will throw if the column schema is invalid (eg, duplicate name).
*/
_subclassResponsibility();
}
getCacheKeys(field, query) {
/*
Return cache keys for columns associated with this query. May return
[unknown] if no keys are known (ie, nothing is or was cached).
*/
return _whereCacheGet(this._whereCache, this.schema, field, query);
}
/**
** Private interfaces below.
**/
_resolveCachedQueries(field: Field, queries: Query[]): LabelArray {
_resolveCachedQueries(field, queries) {
return queries
.map((query: Query) =>
// @ts-expect-error --- TODO revisit:
// `filter`: This expression is not callable.
.map((query) =>
_whereCacheGet(this._whereCache, this.schema, field, query).filter(
(cacheKey?: LabelType) =>
(cacheKey) =>
cacheKey !== undefined && this._cache[field].hasCol(cacheKey)
)
)
.flat();
}
async _fetch(field: Field, q: Query | Query[]): Promise<Dataframe> {
if (!AnnoMatrix.fields().includes(field)) return Dataframe.empty();
async _fetch(field, q) {
if (!AnnoMatrix.fields().includes(field)) return undefined;
const queries = Array.isArray(q) ? q : [q];
queries.forEach(_queryValidate);
@@ -502,7 +441,7 @@ Return cache keys for columns associated with this query. May return
/* find any query not already cached */
const uncachedQueries = queries.filter((query) =>
_whereCacheGet(this._whereCache, this.schema, field, query).some(
(cacheKey?: LabelType) =>
(cacheKey) =>
cacheKey === undefined || !this._cache[field].hasCol(cacheKey)
)
);
@@ -511,19 +450,15 @@ Return cache keys for columns associated with this query. May return
if (uncachedQueries.length > 0) {
await Promise.all(
uncachedQueries.map((query) =>
this._getPendingLoad(
field,
query,
async (_field: Field, _query: Query): Promise<void> => {
/* fetch, then index. _doLoad is subclass interface */
const [whereCacheUpdate, df] = await this._doLoad(_field, _query);
this._cache[_field] = this._cache[_field].withColsFrom(df);
this._whereCache = _whereCacheMerge(
this._whereCache,
whereCacheUpdate
);
}
)
this._getPendingLoad(field, query, async (_field, _query) => {
/* fetch, then index. _doLoad is subclass interface */
const [whereCacheUpdate, df] = await this._doLoad(_field, _query);
this._cache[_field] = this._cache[_field].withColsFrom(df);
this._whereCache = _whereCacheMerge(
this._whereCache,
whereCacheUpdate
);
})
)
);
}
@@ -537,11 +472,7 @@ Return cache keys for columns associated with this query. May return
return response;
}
async _getPendingLoad(
field: Field,
query: Query,
fetchFn: (_field: Field, _query: Query) => Promise<void>
): Promise<void> {
async _getPendingLoad(field, query, fetchFn) {
/*
Given a query on a field, ensure that we only have a single outstanding
fetch at any given time. If multiple requests occur while a fetch is
@@ -562,22 +493,9 @@ Return cache keys for columns associated with this query. May return
return this._pendingLoad[field][key];
}
abstract _doLoad(
field: Field,
query: Query
): Promise<[WhereCache | null, Dataframe]>;
/**
* Determines viewOf for this annoMatrix.
*
* @internal
* @returns - parent annoMatrix if this annoMatrix is a view, otherwise this annoMatrix if it's not a view.
*/
_getViewOf(): AnnoMatrix {
if (this.isView) {
return this.viewOf;
}
return this;
// eslint-disable-next-line class-methods-use-this -- make sure subclass implements
async _doLoad() {
_subclassResponsibility();
}
/**
@@ -609,21 +527,20 @@ Return cache keys for columns associated with this query. May return
To be effective, the GC callback needs to be invoked from the undo/redo code,
as much of the cache is pinned by that data structure.
*/
_gcField(field: Field, isHot: boolean, pinnedColumns: LabelArray): void {
const maxColumns = isHot ? 256 : 10;
_gcField(field, isHot, pinnedColumns) {
const maxColumns = isHot ? 256 : 10; // maybe to aggressive?
const cache = this._cache[field];
if (cache.colIndex.size() < maxColumns) return; // trivial rejection
const candidates = cache.colIndex
.labels()
// @ts-expect-error --- TODO revisit:
// `col`: Argument of type 'LabelType' is not assignable to parameter of type 'number'. Type 'string' is not assignable to type 'number'.
.filter((col: LabelType) => !pinnedColumns.includes(col));
.filter((col) => !pinnedColumns.includes(col));
const excessCount = candidates.length + pinnedColumns.length - maxColumns;
if (excessCount > 0) {
const { _gcInfo } = this;
candidates.sort((a: LabelType, b: LabelType) => {
candidates.sort((a, b) => {
let atime = _gcInfo.get(_columnCacheKey(field, a));
if (atime === undefined) atime = 0;
@@ -640,49 +557,41 @@ Return cache keys for columns associated with this query. May return
// ", "
// )}]`
// );
// @ts-expect-error --- TODO revisit:
// `reduce`: This expression is not callable.
this._cache[field] = toDrop.reduce(
(df: Dataframe, col: LabelType) => df.dropCol(col),
(df, col) => df.dropCol(col),
this._cache[field]
);
toDrop.forEach((col: LabelType) =>
_gcInfo.delete(_columnCacheKey(field, col))
);
toDrop.forEach((col) => _gcInfo.delete(_columnCacheKey(field, col)));
}
}
_gcFetchCleanup(field: Field, pinnedColumns: LabelArray): void {
_gcFetchCleanup(field, pinnedColumns) {
/*
Called during data load/fetch. By definition, this is 'hot', so we
only want to gc X.
*/
if (field === Field.X) {
if (field === "X") {
this._gcField(
field,
true,
// @ts-expect-error --- TODO revisit:
// Property 'concat' does not exist on type 'LabelArray'.
pinnedColumns.concat(_getWritableColumns(this.schema, field))
);
}
}
_gc(hints: GCHints): void {
_gc(hints) {
/*
Called from middleware, or elsewhere. isHot is true if we are in the active store,
or false if we are in some other context (eg, history state).
*/
const { isHot } = hints;
const candidateFields = isHot
? [Field.X]
: [Field.X, Field.emb, Field.var, Field.obs];
const candidateFields = isHot ? ["X"] : ["X", "emb", "var", "obs"];
candidateFields.forEach((field) =>
this._gcField(field, isHot, _getWritableColumns(this.schema, field))
);
}
_gcUpdateStats(field: Field, dataframe: Dataframe): void {
_gcUpdateStats(field, dataframe) {
/*
called each time a query is performed, allowing the gc to update any bookkeeping
information. Currently, this is just a simple last-fetched timestamp, stored
@@ -691,7 +600,7 @@ Return cache keys for columns associated with this query. May return
const cols = dataframe.colIndex.labels();
const { _gcInfo } = this;
const now = Date.now();
cols.forEach((c: LabelType) => {
cols.forEach((c) => {
_gcInfo.set(_columnCacheKey(field, c), now);
});
}
@@ -708,7 +617,7 @@ Return cache keys for columns associated with this query. May return
Do not override _clone();
**/
_cloneDeeper(clone: AnnoMatrix): AnnoMatrix {
_cloneDeeper(clone) {
clone._cache = _shallowClone(this._cache);
clone._gcInfo = new Map();
clone._pendingLoad = {
@@ -720,7 +629,7 @@ Return cache keys for columns associated with this query. May return
return clone;
}
_clone(): AnnoMatrix {
_clone() {
const clone = _shallowClone(this);
this._cloneDeeper(clone);
Object.seal(clone);
@@ -731,6 +640,11 @@ Return cache keys for columns associated with this query. May return
/*
private utility functions below
*/
function _columnCacheKey(field: Field, column: LabelType): string {
function _columnCacheKey(field, column) {
return `${field}/${column}`;
}
function _subclassResponsibility() {
/* protect against bugs in subclass */
throw new Error("subclass failed to implement required method");
}
client/src/annoMatrix/clone.ts → client/src/annoMatrix/clone.js
+1 -2
View File
@@ -1,7 +1,6 @@
/*
Shallow clone an object, correctly handling prototype
*/
export default function _shallowClone<T>(orig: T): T {
export default function _shallowClone(orig) {
return Object.assign(Object.create(Object.getPrototypeOf(orig)), orig);
}
client/src/annoMatrix/crossfilter.ts → client/src/annoMatrix/crossfilter.js
+36 -119
View File
@@ -9,94 +9,56 @@ AnnoMatrix stay in sync:
*/
import Crossfilter from "../util/typedCrossfilter";
import { _getColumnSchema } from "./schema";
import {
AnnotationColumnSchema,
Field,
EmbeddingSchema,
} from "../common/types/schema";
import AnnoMatrix from "./annoMatrix";
import {
Dataframe,
DataframeValue,
DataframeValueArray,
LabelType,
} from "../util/dataframe";
import { Query } from "./query";
import { TypedArray } from "../common/types/arraytypes";
import { LabelArray } from "../util/dataframe/types";
type ObsDimensionParams =
| [string, DataframeValueArray, DataframeValueArray]
| [string, DataframeValueArray]
| [string, DataframeValueArray, Int32ArrayConstructor]
| [string, DataframeValueArray, Float32ArrayConstructor];
function _dimensionNameFromDf(field: Field, df: Dataframe): string {
function _dimensionNameFromDf(field, df) {
const colNames = df.colIndex.labels();
return _dimensionName(field, colNames);
}
function _dimensionName(
field: Field,
colNames: LabelType | LabelArray
): string {
function _dimensionName(field, colNames) {
if (!Array.isArray(colNames)) return `${field}/${colNames}`;
return `${field}/${colNames.join(":")}`;
}
export default class AnnoMatrixObsCrossfilter {
annoMatrix: AnnoMatrix;
obsCrossfilter: Crossfilter;
constructor(
annoMatrix: AnnoMatrix,
_obsCrossfilter: Crossfilter | null = null
) {
constructor(annoMatrix, _obsCrossfilter = null) {
this.annoMatrix = annoMatrix;
this.obsCrossfilter =
_obsCrossfilter || new Crossfilter(annoMatrix._cache.obs);
this.obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
}
size(): number {
size() {
return this.obsCrossfilter.size();
}
/**
Managing the associated annoMatrix. These wrappers are necessary to
Managing the associated annoMatrix. These wrappers are necessary to
make coordinated changes to BOTH the crossfilter and annoMatrix, and
ensure that all state stays synchronized.
See API documentation in annoMatrix.js.
**/
addObsColumn<T extends DataframeValueArray>(
colSchema: AnnotationColumnSchema,
Ctor: new (n: number) => T,
value: T
): AnnoMatrixObsCrossfilter {
addObsColumn(colSchema, Ctor, value) {
const annoMatrix = this.annoMatrix.addObsColumn(colSchema, Ctor, value);
const obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
}
dropObsColumn(col: LabelType): AnnoMatrixObsCrossfilter {
dropObsColumn(col) {
const annoMatrix = this.annoMatrix.dropObsColumn(col);
let { obsCrossfilter } = this;
const dimName = _dimensionName(Field.obs, col);
const dimName = _dimensionName("obs", col);
if (obsCrossfilter.hasDimension(dimName)) {
obsCrossfilter = obsCrossfilter.delDimension(dimName);
}
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
}
renameObsColumn(
oldCol: LabelType,
newCol: LabelType
): AnnoMatrixObsCrossfilter {
renameObsColumn(oldCol, newCol) {
const annoMatrix = this.annoMatrix.renameObsColumn(oldCol, newCol);
const oldDimName = _dimensionName(Field.obs, oldCol);
const newDimName = _dimensionName(Field.obs, newCol);
const oldDimName = _dimensionName("obs", oldCol);
const newDimName = _dimensionName("obs", newCol);
let { obsCrossfilter } = this;
if (obsCrossfilter.hasDimension(oldDimName)) {
obsCrossfilter = obsCrossfilter.renameDimension(oldDimName, newDimName);
@@ -104,12 +66,9 @@ export default class AnnoMatrixObsCrossfilter {
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
}
addObsAnnoCategory(
col: LabelType,
category: string
): AnnoMatrixObsCrossfilter {
addObsAnnoCategory(col, category) {
const annoMatrix = this.annoMatrix.addObsAnnoCategory(col, category);
const dimName = _dimensionName(Field.obs, col);
const dimName = _dimensionName("obs", col);
let { obsCrossfilter } = this;
if (obsCrossfilter.hasDimension(dimName)) {
obsCrossfilter = obsCrossfilter.delDimension(dimName);
@@ -117,17 +76,13 @@ export default class AnnoMatrixObsCrossfilter {
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
}
async removeObsAnnoCategory(
col: LabelType,
category: string,
unassignedCategory: string
): Promise<AnnoMatrixObsCrossfilter> {
async removeObsAnnoCategory(col, category, unassignedCategory) {
const annoMatrix = await this.annoMatrix.removeObsAnnoCategory(
col,
category,
unassignedCategory
);
const dimName = _dimensionName(Field.obs, col);
const dimName = _dimensionName("obs", col);
let { obsCrossfilter } = this;
if (obsCrossfilter.hasDimension(dimName)) {
obsCrossfilter = obsCrossfilter.delDimension(dimName);
@@ -135,17 +90,13 @@ export default class AnnoMatrixObsCrossfilter {
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
}
async setObsColumnValues(
col: LabelType,
rowLabels: Int32Array,
value: DataframeValue
): Promise<AnnoMatrixObsCrossfilter> {
async setObsColumnValues(col, rowLabels, value) {
const annoMatrix = await this.annoMatrix.setObsColumnValues(
col,
rowLabels,
value
);
const dimName = _dimensionName(Field.obs, col);
const dimName = _dimensionName("obs", col);
let { obsCrossfilter } = this;
if (obsCrossfilter.hasDimension(dimName)) {
obsCrossfilter = obsCrossfilter.delDimension(dimName);
@@ -153,17 +104,13 @@ export default class AnnoMatrixObsCrossfilter {
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
}
async resetObsColumnValues<T extends DataframeValue>(
col: LabelType,
oldValue: T,
newValue: T
): Promise<AnnoMatrixObsCrossfilter> {
async resetObsColumnValues(col, oldValue, newValue) {
const annoMatrix = await this.annoMatrix.resetObsColumnValues(
col,
oldValue,
newValue
);
const dimName = _dimensionName(Field.obs, col);
const dimName = _dimensionName("obs", col);
let { obsCrossfilter } = this;
if (obsCrossfilter.hasDimension(dimName)) {
obsCrossfilter = obsCrossfilter.delDimension(dimName);
@@ -171,25 +118,23 @@ export default class AnnoMatrixObsCrossfilter {
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
}
addEmbedding(colSchema: EmbeddingSchema): AnnoMatrixObsCrossfilter {
addEmbedding(colSchema) {
const annoMatrix = this.annoMatrix.addEmbedding(colSchema);
return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
}
/**
* Drop the crossfilter dimension. Do not change the annoMatrix. Useful when we
* want to stop tracking the selection state, but aren't sure we want to blow the
* want to stop trackin the selection state, but aren't sure we want to blow the
* annomatrix cache.
*/
dropDimension(field: Field, query: Query): AnnoMatrixObsCrossfilter {
dropDimension(field, query) {
const { annoMatrix } = this;
let { obsCrossfilter } = this;
const keys = annoMatrix
.getCacheKeys(field, query)
// @ts-expect-error ts-migrate --- suppressing TS defect (https://github.com/microsoft/TypeScript/issues/44373).
// Compiler is complaining that expression is not callable on array union types. Remove suppression once fixed.
.filter((k?: string | number) => k !== undefined);
const dimName = _dimensionName(field, keys as string[]);
.filter((k) => k !== undefined);
const dimName = _dimensionName(field, keys);
if (obsCrossfilter.hasDimension(dimName)) {
obsCrossfilter = obsCrossfilter.delDimension(dimName);
}
@@ -201,12 +146,7 @@ export default class AnnoMatrixObsCrossfilter {
are just wrappers to lazy create indices.
**/
async select(
field: Field,
query: Query,
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- TODO revisit: waiting for typings from util/typedCrossfilter
spec: any
): Promise<AnnoMatrixObsCrossfilter> {
async select(field, query, spec) {
const { annoMatrix } = this;
let { obsCrossfilter } = this;
@@ -219,9 +159,7 @@ export default class AnnoMatrixObsCrossfilter {
// grab the data, so we can grab the index.
const df = await annoMatrix.fetch(field, query);
if (!df) {
throw new Error("Dataframe cannot be `undefined`");
}
const dimName = _dimensionNameFromDf(field, df);
if (!obsCrossfilter.hasDimension(dimName)) {
// lazy index generation - add dimension when first used
@@ -238,26 +176,23 @@ export default class AnnoMatrixObsCrossfilter {
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
}
selectAll(): AnnoMatrixObsCrossfilter {
selectAll() {
/*
Select all on any dimension in this field.
*/
const { annoMatrix } = this;
const currentDims = this.obsCrossfilter.dimensionNames();
const obsCrossfilter = currentDims.reduce(
(xfltr, dim) => xfltr.select(dim, { mode: "all" }),
this.obsCrossfilter
);
const obsCrossfilter = currentDims.reduce((xfltr, dim) => xfltr.select(dim, { mode: "all" }), this.obsCrossfilter);
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
}
countSelected(): number {
countSelected() {
/* if no data yet indexed in the crossfilter, just say everything is selected */
if (this.obsCrossfilter.size() === 0) return this.annoMatrix.nObs;
return this.obsCrossfilter.countSelected();
}
allSelectedMask(): Uint8Array {
allSelectedMask() {
/* if no data yet indexed in the crossfilter, just say everything is selected */
if (
this.obsCrossfilter.size() === 0 ||
@@ -269,7 +204,7 @@ export default class AnnoMatrixObsCrossfilter {
return this.obsCrossfilter.allSelectedMask();
}
allSelectedLabels(): LabelArray {
allSelectedLabels() {
/* if no data yet indexed in the crossfilter, just say everything is selected */
if (
this.obsCrossfilter.size() === 0 ||
@@ -283,18 +218,12 @@ export default class AnnoMatrixObsCrossfilter {
return index.labels();
}
fillByIsSelected<A extends TypedArray>(
array: A,
selectedValue: A[0],
deselectedValue: A[0]
): A {
fillByIsSelected(array, selectedValue, deselectedValue) {
/* if no data yet indexed in the crossfilter, just say everything is selected */
if (
this.obsCrossfilter.size() === 0 ||
this.obsCrossfilter.dimensionNames().length === 0
) {
// @ts-expect-error ts-migrate --- TODO revisit:
// Type 'Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array' is not assignable to type 'A'...
return array.fill(selectedValue);
}
return this.obsCrossfilter.fillByIsSelected(
@@ -308,35 +237,25 @@ export default class AnnoMatrixObsCrossfilter {
** Private below
**/
_addObsCrossfilterDimension(
annoMatrix: AnnoMatrix,
obsCrossfilter: Crossfilter,
field: Field,
df: Dataframe
): Crossfilter {
_addObsCrossfilterDimension(annoMatrix, obsCrossfilter, field, df) {
if (field === "var") return obsCrossfilter;
const dimName = _dimensionNameFromDf(field, df);
const dimParams = this._getObsDimensionParams(field, df);
obsCrossfilter = obsCrossfilter.setData(annoMatrix._cache.obs);
// @ts-expect-error ts-migrate --- TODO revisit:
// `...dimParams`: A spread argument must either have a tuple type or be passed to a rest parameter.
obsCrossfilter = obsCrossfilter.addDimension(dimName, ...dimParams);
return obsCrossfilter;
}
_getColumnBaseType(field: Field, col: LabelType): string {
_getColumnBaseType(field, col) {
/* Look up the primitive type for this field/col */
const colSchema = _getColumnSchema(this.annoMatrix.schema, field, col);
return colSchema.type;
}
_getObsDimensionParams(
field: Field,
df: Dataframe
): ObsDimensionParams | undefined {
_getObsDimensionParams(field, df) {
/* return the crossfilter dimensiontype type and params for this field/dataframe */
if (field === Field.emb) {
if (field === "emb") {
/* assumed to be 2D */
return ["spatial", df.icol(0).asArray(), df.icol(1).asArray()];
}
@@ -344,8 +263,6 @@ export default class AnnoMatrixObsCrossfilter {
/* assumed to be 1D */
const col = df.icol(0);
const colName = df.colIndex.getLabel(0);
// @ts-expect-error --- TODO revisit:
// `colName` Argument of type 'LabelType | undefined' is not assignable to parameter of type 'LabelType'. Type 'undefined' is not assignable to type 'LabelType'.
const type = this._getColumnBaseType(field, colName);
if (type === "string" || type === "categorical" || type === "boolean") {
return ["enum", col.asArray()];
client/src/annoMatrix/fetchHelpers.js
+25
View File
@@ -0,0 +1,25 @@
export { doBinaryRequest, doFetch } from "../util/actionHelpers";
/* double URI encode - needed for query-param filters */
export function _dubEncURIComp(s) {
return encodeURIComponent(encodeURIComponent(s));
}
/* currently unused, consider deleting */
export function _fetchResult(promise) {
let _status = "pending";
const res = promise.then(
(r) => {
_status = "success";
return r;
},
(e) => {
_status = "error";
throw e;
}
);
res.status = () => _status;
return res;
}
client/src/annoMatrix/fetchHelpers.ts
-28
View File
@@ -1,28 +0,0 @@
export { doBinaryRequest, doFetch } from "../util/actionHelpers";
/* double URI encode - needed for query-param filters */
export function _dubEncURIComp(s: string | number | boolean): string {
return encodeURIComponent(encodeURIComponent(s));
}
/* currently unused, consider deleting */
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
export function _fetchResult(promise: any) {
let _status = "pending";
const res = promise.then(
// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
(r: any) => {
_status = "success";
return r;
},
// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
(e: any) => {
_status = "error";
throw e;
}
);
res.status = () => _status;
return res;
}
client/src/annoMatrix/index.ts → client/src/annoMatrix/index.js
View File
client/src/annoMatrix/loader.ts → client/src/annoMatrix/loader.js
+51 -133
View File
@@ -2,47 +2,31 @@ import { doBinaryRequest, doFetch } from "./fetchHelpers";
import { matrixFBSToDataframe } from "../util/stateManager/matrix";
import { _getColumnSchema } from "./schema";
import {
addObsAnnoCategory,
addObsAnnoColumn,
addObsLayout,
removeObsAnnoCategory,
removeObsAnnoColumn,
addObsAnnoCategory,
removeObsAnnoCategory,
addObsLayout,
} from "../util/stateManager/schemaHelpers";
import { isAnyArray } from "../common/types/arraytypes";
import { _whereCacheCreate, WhereCache } from "./whereCache";
import { isArrayOrTypedArray } from "../util/typeHelpers";
import { _whereCacheCreate } from "./whereCache";
import AnnoMatrix from "./annoMatrix";
import PromiseLimit from "../util/promiseLimit";
import {
_expectComplexQuery,
_expectSimpleQuery,
_hashStringValues,
_urlEncodeComplexQuery,
_expectComplexQuery,
_urlEncodeLabelQuery,
ComplexQuery,
Query,
_urlEncodeComplexQuery,
_hashStringValues,
} from "./query";
import {
normalizeResponse,
normalizeWritableCategoricalSchema,
} from "./normalize";
import {
AnnotationColumnSchema,
Field,
EmbeddingSchema,
RawSchema,
} from "../common/types/schema";
import {
Dataframe,
DataframeValue,
DataframeValueArray,
LabelType,
} from "../util/dataframe";
const promiseThrottle = new PromiseLimit(5);
export default class AnnoMatrixLoader extends AnnoMatrix {
baseURL: string;
/*
AnnoMatrix implementation which proxies to HTTP server using the CXG REST API.
Used as the base (non-view) instance.
@@ -53,7 +37,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
new AnnoMatrixLoader(serverBaseURL, schema) -> instance
*/
constructor(baseURL: string, schema: RawSchema) {
constructor(baseURL, schema) {
const { nObs, nVar } = schema.dataframe;
super(schema, nObs, nVar);
@@ -68,36 +52,24 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
/**
** Public. API described in base class.
**/
addObsAnnoCategory(col: LabelType, category: string): AnnoMatrix {
addObsAnnoCategory(col, category) {
/*
Add a new category (aka label) to the schema for an obs column.
*/
const colSchema = _getColumnSchema(
this.schema,
Field.obs,
col
) as AnnotationColumnSchema;
_writableObsCategoryTypeCheck(colSchema); // throws on error
const colSchema = _getColumnSchema(this.schema, "obs", col);
_writableCategoryTypeCheck(colSchema); // throws on error
const newAnnoMatrix = this._clone();
newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, category);
return newAnnoMatrix;
}
async removeObsAnnoCategory(
col: LabelType,
category: string,
unassignedCategory: string
): Promise<AnnoMatrix> {
async removeObsAnnoCategory(col, category, unassignedCategory) {
/*
Remove a single "category" (aka "label") from the data & schema of an obs column.
*/
const colSchema = _getColumnSchema(
this.schema,
Field.obs,
col
) as AnnotationColumnSchema;
_writableObsCategoryTypeCheck(colSchema); // throws on error
const colSchema = _getColumnSchema(this.schema, "obs", col);
_writableCategoryTypeCheck(colSchema); // throws on error
const newAnnoMatrix = await this.resetObsColumnValues(
col,
@@ -112,16 +84,12 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
return newAnnoMatrix;
}
dropObsColumn(col: LabelType): AnnoMatrix {
dropObsColumn(col) {
/*
drop column from field
*/
const colSchema = _getColumnSchema(
this.schema,
Field.obs,
col
) as AnnotationColumnSchema;
_writableObsCheck(colSchema); // throws on error
const colSchema = _getColumnSchema(this.schema, "obs", col);
_writableCheck(colSchema); // throws on error
const newAnnoMatrix = this._clone();
newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
@@ -129,11 +97,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
return newAnnoMatrix;
}
addObsColumn<T extends DataframeValueArray>(
colSchema: AnnotationColumnSchema,
Ctor: new (n: number) => T,
value: T
): AnnoMatrix {
addObsColumn(colSchema, Ctor, value) {
/*
add a column to field, initializing with value. Value may
be one of:
@@ -144,7 +108,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
colSchema.writable = true;
const colName = colSchema.name;
if (
_getColumnSchema(this.schema, Field.obs, colName) ||
_getColumnSchema(this.schema, "obs", colName) ||
this._cache.obs.hasCol(colName)
) {
throw new Error("column already exists");
@@ -152,7 +116,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
const newAnnoMatrix = this._clone();
let data;
if (isAnyArray(value)) {
if (isArrayOrTypedArray(value)) {
if (value.constructor !== Ctor)
throw new Error("Mismatched value array type");
if (value.length !== this.nObs)
@@ -170,50 +134,35 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
return newAnnoMatrix;
}
renameObsColumn(oldCol: LabelType, newCol: LabelType): AnnoMatrix {
renameObsColumn(oldCol, newCol) {
/*
Rename the obs oldColName to newColName. oldCol must be writable.
*/
const oldColSchema = _getColumnSchema(
this.schema,
Field.obs,
oldCol
) as AnnotationColumnSchema;
_writableObsCheck(oldColSchema);
const oldColSchema = _getColumnSchema(this.schema, "obs", oldCol);
_writableCheck(oldColSchema); // throws on error
const value = this._cache.obs.hasCol(oldCol)
? this._cache.obs.col(oldCol).asArray()
: undefined;
return this.dropObsColumn(oldCol).addObsColumn(
{
...oldColSchema,
// @ts-expect-error ts-migrate --- TODO revisit:
// `name`: Type 'LabelType' is not assignable to type 'string'. Type 'number' is not assignable to type 'string'.
name: newCol,
},
// @ts-expect-error ts-migrate --- TODO revisit:
// `value`: Object is possibly 'undefined'.
value.constructor,
value
);
}
async setObsColumnValues(
col: LabelType,
rowLabels: Int32Array,
value: DataframeValue
): Promise<AnnoMatrix> {
async setObsColumnValues(col, rowLabels, value) {
/*
Set all rows identified by rowLabels to value.
*/
const colSchema = _getColumnSchema(
this.schema,
Field.obs,
col
) as AnnotationColumnSchema;
_writableObsCategoryTypeCheck(colSchema); // throws on error
const colSchema = _getColumnSchema(this.schema, "obs", col);
_writableCategoryTypeCheck(colSchema); // throws on error
// ensure that we have the data in cache before we manipulate it
await this.fetch(Field.obs, col);
await this.fetch("obs", col);
if (!this._cache.obs.hasCol(col))
throw new Error("Internal error - user annotation data missing");
@@ -221,7 +170,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
const data = this._cache.obs.col(col).asArray().slice();
for (let i = 0, len = rowIndices.length; i < len; i += 1) {
const idx = rowIndices[i];
if (idx === -1) throw new Error("Unknown row label");
if (idx === undefined) throw new Error("Unknown row label");
data[idx] = value;
}
@@ -234,29 +183,19 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
return newAnnoMatrix;
}
async resetObsColumnValues<T extends DataframeValue>(
col: LabelType,
oldValue: T,
newValue: T
): Promise<AnnoMatrix> {
async resetObsColumnValues(col, oldValue, newValue) {
/*
Set all rows with value 'oldValue' to 'newValue'.
*/
const colSchema = _getColumnSchema(
this.schema,
Field.obs,
col
) as AnnotationColumnSchema;
_writableObsCategoryTypeCheck(colSchema); // throws on error
const colSchema = _getColumnSchema(this.schema, "obs", col);
_writableCategoryTypeCheck(colSchema); // throws on error
// @ts-expect-error ts-migrate --- TODO revisit:
// `colSchema.categories`: Object is possibly 'undefined'.
if (!colSchema.categories.includes(oldValue)) {
throw new Error("unknown category");
}
// ensure that we have the data in cache before we manipulate it
await this.fetch(Field.obs, col);
await this.fetch("obs", col);
if (!this._cache.obs.hasCol(col))
throw new Error("Internal error - user annotation data missing");
@@ -274,12 +213,12 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
return newAnnoMatrix;
}
addEmbedding(colSchema: EmbeddingSchema): AnnoMatrix {
addEmbedding(colSchema) {
/*
add new layout to the obs embeddings
*/
const { name: colName } = colSchema;
if (_getColumnSchema(this.schema, Field.emb, colName)) {
if (_getColumnSchema(this.schema, "emb", colName)) {
throw new Error("column already exists");
}
@@ -291,10 +230,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
/**
** Private below
**/
async _doLoad(
field: Field,
query: Query
): Promise<[WhereCache | null, Dataframe]> {
async _doLoad(field, query) {
/*
_doLoad - evaluates the query against the field. Returns:
* whereCache update: column query map mapping the query to the column labels
@@ -321,9 +257,8 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
default:
throw new Error("Unknown field name");
}
const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
// @ts-expect-error --- TODO revisit:
// `buffer`: Argument of type 'unknown' is not assignable to parameter of type 'ArrayBuffer | ArrayBuffer[]'. Type 'unknown' is not assignable to type 'ArrayBuffer[]'.
let result = matrixFBSToDataframe(buffer);
if (!result || result.isEmpty()) throw Error("Unknown field/col");
@@ -333,7 +268,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
result.colIndex.labels()
);
result = normalizeResponse(field, this.schema, result);
result = normalizeResponse(field, query, this.schema, result);
return [whereCacheUpdate, result];
}
@@ -343,26 +278,20 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
Utility functions below
*/
function _writableObsCheck(obsColSchema: AnnotationColumnSchema): void {
if (!obsColSchema?.writable) {
function _writableCheck(colSchema) {
if (!colSchema?.writable) {
throw new Error("Unknown or readonly obs column");
}
}
function _writableObsCategoryTypeCheck(
obsColSchema: AnnotationColumnSchema
): void {
_writableObsCheck(obsColSchema);
if (obsColSchema.type !== "categorical") {
function _writableCategoryTypeCheck(colSchema) {
_writableCheck(colSchema);
if (colSchema.type !== "categorical") {
throw new Error("column must be categorical");
}
}
function _embLoader(
baseURL: string,
_field: Field,
query: Query
): () => Promise<ArrayBuffer> {
function _embLoader(baseURL, _field, query) {
_expectSimpleQuery(query);
const urlBase = `${baseURL}layout/obs`;
@@ -371,11 +300,7 @@ function _embLoader(
return () => doBinaryRequest(url);
}
function _obsOrVarLoader(
baseURL: string,
field: Field,
query: Query
): () => Promise<ArrayBuffer> {
function _obsOrVarLoader(baseURL, field, query) {
_expectSimpleQuery(query);
const urlBase = `${baseURL}annotations/${field}`;
@@ -384,26 +309,19 @@ function _obsOrVarLoader(
return () => doBinaryRequest(url);
}
function _XLoader(
baseURL: string,
_field: Field,
query: Query
): () => Promise<ArrayBuffer> {
function _XLoader(baseURL, field, query) {
_expectComplexQuery(query);
// Casting here as query is validated to be complex in _expectComplexQuery above.
const complexQuery = query as ComplexQuery;
if ("where" in complexQuery) {
if (query.where) {
const urlBase = `${baseURL}data/var`;
const urlQuery = _urlEncodeComplexQuery(complexQuery);
const urlQuery = _urlEncodeComplexQuery(query);
const url = `${urlBase}?${urlQuery}`;
return () => doBinaryRequest(url);
}
if ("summarize" in complexQuery) {
if (query.summarize) {
const urlBase = `${baseURL}summarize/var`;
const urlQuery = _urlEncodeComplexQuery(complexQuery);
const urlQuery = _urlEncodeComplexQuery(query);
if (urlBase.length + urlQuery.length < 2000) {
const url = `${urlBase}?${urlQuery}`;
client/src/annoMatrix/middleware.ts → client/src/annoMatrix/middleware.js
+15 -30
View File
@@ -11,24 +11,16 @@ Undoable metareducer and the AnnoMatrix private API. It would be helpful
to make the Undoable interface better factored.
*/
import { Action, Dispatch, MiddlewareAPI } from "redux";
import AnnoMatrix from "./annoMatrix";
import { GCHints } from "../common/types/entities";
const annoMatrixGC =
(store: MiddlewareAPI) =>
// GC middleware doesn't add any extra types to dispatch; it just executes GC and continues.
(next: Dispatch) =>
(action: Action): Action => {
if (_itIsTimeForGC()) {
_doGC(store);
}
return next(action);
};
const annoMatrixGC = (store) => (next) => (action) => {
if (_itIsTimeForGC()) {
_doGC(store);
}
return next(action);
};
let lastGCTime = 0;
const InterGCDelayMS = 30 * 1000; // 30 seconds
function _itIsTimeForGC(): boolean {
function _itIsTimeForGC() {
/*
we don't want to run GC on every dispatch, so throttle it a bit.
@@ -42,22 +34,17 @@ function _itIsTimeForGC(): boolean {
return false;
}
function _doGC(store: MiddlewareAPI): void {
function _doGC(store) {
const state = store.getState();
// these should probably be a function imported from undoable.js, etc, as
// they have overly intimate knowledge of our reducers.
// they have overly intimiate knowledge of our reducers.
const undoablePast = state["@@undoable/past"];
const undoableFuture = state["@@undoable/future"];
const undoableStack = undoablePast
.concat(undoableFuture)
// eslint-disable-next-line @typescript-eslint/no-explicit-any --- TODO revisit: waiting for typings from /reducers
.flatMap((snapshot: any) =>
snapshot
// eslint-disable-next-line @typescript-eslint/no-explicit-any --- TODO revisit: waiting for typings from /reducers
.filter((v: any) => v[0] === "annoMatrix")
// eslint-disable-next-line @typescript-eslint/no-explicit-any --- TODO revisit: waiting for typings from /reducers
.map((v: any) => v[1])
.flatMap((snapshot) =>
snapshot.filter((v) => v[0] === "annoMatrix").map((v) => v[1])
);
const currentAnnoMatrix = state.annoMatrix;
@@ -65,17 +52,15 @@ function _doGC(store: MiddlewareAPI): void {
We want to identify those matrixes currently "hot", ie, linked from the current annoMatrix,
as our current gc algo is more aggressive with those not hot.
*/
const allAnnoMatrices = new Map<AnnoMatrix, GCHints>(
undoableStack.map((m: AnnoMatrix) => [m, { isHot: false }])
const allAnnoMatrices = new Map(
undoableStack.map((m) => [m, { isHot: false }])
);
let am = currentAnnoMatrix;
while (am?.isView) {
while (am) {
allAnnoMatrices.set(am, { isHot: true });
am = am.viewOf;
}
allAnnoMatrices.forEach((hints, annoMatrix: AnnoMatrix) =>
annoMatrix._gc(hints)
);
allAnnoMatrices.forEach((hints, annoMatrix) => annoMatrix._gc(hints));
}
export default annoMatrixGC;
client/src/annoMatrix/normalize.ts → client/src/annoMatrix/normalize.js
+9 -31
View File
@@ -5,19 +5,8 @@ import {
overflowCategoryLabel,
globalConfig,
} from "../globals";
import { Dataframe, LabelType, DataframeColumn } from "../util/dataframe";
import {
AnnotationColumnSchema,
ArraySchema,
Field,
Schema,
} from "../common/types/schema";
export function normalizeResponse(
field: Field,
schema: Schema,
response: Dataframe
): Dataframe {
export function normalizeResponse(field, query, schema, response) {
/**
* There are a number of assumptions in the front-end about data typing and data
* characteristics. This routine will normalize a server response dataframe
@@ -42,15 +31,11 @@ export function normalizeResponse(
*/
// currently no data or schema normalization necessary for X or emb
if (field !== Field.obs && field !== Field.var) return response;
if (field !== "obs" && field !== "var") return response;
const colLabels = response.colIndex.labels();
for (const colLabel of colLabels) {
const colSchema = _getColumnSchema(
schema,
field,
colLabel
) as AnnotationColumnSchema;
const colSchema = _getColumnSchema(schema, field, colLabel);
const isIndex = _isIndex(schema, field, colLabel);
const { type, writable } = colSchema;
@@ -74,7 +59,7 @@ export function normalizeResponse(
return response;
}
function castColumnToBoolean(df: Dataframe, label: LabelType): Dataframe {
function castColumnToBoolean(df, label) {
const colData = df.col(label).asArray();
const newColData = new Array(colData.length);
for (let i = 0; i < colData.length; i += 1) newColData[i] = !!colData[i];
@@ -82,16 +67,13 @@ function castColumnToBoolean(df: Dataframe, label: LabelType): Dataframe {
return df;
}
export function normalizeWritableCategoricalSchema(
colSchema: AnnotationColumnSchema, // eslint-disable-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
col: DataframeColumn
): ArraySchema {
export function normalizeWritableCategoricalSchema(colSchema, col) {
/*
Ensure all enum writable / categorical schema have a categories array, that
the categories array contains all unique values in the data array, AND that
the array is UI sorted.
*/
const categorySet = new Set<string>(
const categorySet = new Set(
col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
);
if (!categorySet.has(unassignedCategoryLabel)) {
@@ -101,19 +83,15 @@ export function normalizeWritableCategoricalSchema(
return colSchema;
}
export function normalizeCategorical(
df: Dataframe,
colLabel: LabelType,
colSchema: AnnotationColumnSchema
): Dataframe {
export function normalizeCategorical(df, colLabel, colSchema) {
/*
If writable, ensure schema matches data and we have an unassigned label
If not writable, ensure schema matches data and that we consolidate labels in excess
of "top N" into an overflow labels.
*/
const { writable } = colSchema;
const col = df.col(colLabel);
if (writable) {
// writable (aka user) annotations
normalizeWritableCategoricalSchema(colSchema, col);
@@ -125,7 +103,7 @@ export function normalizeCategorical(
// consolidate all categories from data and schema into a single list
const colDataSummary = col.summarizeCategorical();
const allCategories = new Set<string>(
const allCategories = new Set(
colDataSummary.categories.concat(colSchema.categories ?? [])
);
client/src/annoMatrix/query.ts → client/src/annoMatrix/query.js
+25 -51
View File
@@ -1,52 +1,21 @@
import sha1 from "sha1";
import { _dubEncURIComp } from "./fetchHelpers";
import { Field } from "../common/types/schema";
import { LabelType } from "../util/dataframe";
/**
* Query utilities, mostly for debugging support and validation.
*/
export type ComplexQuery = SummarizeQuery | WhereQuery;
export type Query = LabelType | ComplexQuery;
interface SummarizeQuery {
summarize: SummarizeQueryTerm;
}
interface SummarizeQueryTerm {
column: string;
field: string;
method: string;
values: string[];
}
interface WhereQuery {
where: WhereQueryTerm;
}
interface WhereQueryTerm {
column: string;
field: string;
value: string;
}
export function _expectSimpleQuery(query: Query): void {
if (typeof query === "object") throw new Error("expected simple query");
}
/**
* Normalize & error check the query.
* @param {Query} query - the query
* @returns {Query} - the normalized query
* @param {object | string} query - the query
* @returns {object | string} - the normalized query
*/
export function _queryValidate(query: Query): Query {
export function _queryValidate(query) {
if (typeof query !== "object") return query;
if ("where" in query && "summarize" in query)
if (query.where && query.summarize)
throw new Error("query may not specify both where and summarize");
if ("where" in query) {
if (query.where) {
const {
field: queryField,
column: queryColumn,
@@ -56,7 +25,7 @@ export function _queryValidate(query: Query): Query {
throw new Error("Incomplete where query");
return query;
}
if ("summarize" in query) {
if (query.summarize) {
const {
field: queryField,
column: queryColumn,
@@ -71,7 +40,11 @@ export function _queryValidate(query: Query): Query {
throw new Error("query must specify one of where or summarize");
}
export function _expectComplexQuery(query: Query): void {
export function _expectSimpleQuery(query) {
if (typeof query === "object") throw new Error("expected simple query");
}
export function _expectComplexQuery(query) {
if (typeof query !== "object") throw new Error("expected complex query");
}
@@ -80,12 +53,12 @@ export function _expectComplexQuery(query: Query): void {
*
* @param {string} field
* @param {string|object} query
* @returns {string} the key
* @returns the key
*/
export function _queryCacheKey(field: Field, query: Query): string {
export function _queryCacheKey(field, query) {
if (typeof query === "object") {
// complex query
if ("where" in query) {
if (query.where) {
const {
field: queryField,
column: queryColumn,
@@ -93,7 +66,7 @@ export function _queryCacheKey(field: Field, query: Query): string {
} = query.where;
return `${field}/${queryField}/${queryColumn}/${queryValue}`;
}
if ("summarize" in query) {
if (query.summarize) {
const {
method,
field: queryField,
@@ -111,34 +84,34 @@ export function _queryCacheKey(field: Field, query: Query): string {
return `${field}/${query}`;
}
function _urlEncodeWhereQuery(q: WhereQueryTerm): string {
function _urlEncodeWhereQuery(q) {
const { field: queryField, column: queryColumn, value: queryValue } = q;
return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
queryColumn
)}=${_dubEncURIComp(queryValue)}`;
}
function _urlEncodeSummarizeQuery(q: SummarizeQueryTerm): string {
function _urlEncodeSummarizeQuery(q) {
const { method, field, column, values } = q;
const filter = values
.map((value: string) => _urlEncodeWhereQuery({ field, column, value }))
.map((value) => _urlEncodeWhereQuery({ field, column, value }))
.join("&");
return `method=${method}&${filter}`;
}
export function _urlEncodeComplexQuery(q: ComplexQuery): string {
export function _urlEncodeComplexQuery(q) {
if (typeof q === "object") {
if ("where" in q) {
if (q.where) {
return _urlEncodeWhereQuery(q.where);
}
if ("summarize" in q) {
if (q.summarize) {
return _urlEncodeSummarizeQuery(q.summarize);
}
}
throw new Error("Unrecognized complex query type");
}
export function _urlEncodeLabelQuery(colKey: string, q: Query): string {
export function _urlEncodeLabelQuery(colKey, q) {
if (!colKey) throw new Error("Unsupported query by name");
if (typeof q !== "string") throw new Error("Query must be a simple label.");
return `${colKey}=${encodeURIComponent(q)}`;
@@ -147,6 +120,7 @@ export function _urlEncodeLabelQuery(colKey: string, q: Query): string {
/**
* Generate the column key the server will send us for this query.
*/
export function _hashStringValues(arrayOfString: string[]): string {
return sha1(arrayOfString.join(""));
export function _hashStringValues(arrayOfString) {
const hash = sha1(arrayOfString.join(""));
return hash;
}
client/src/annoMatrix/schema.ts → client/src/annoMatrix/schema.js
+18 -41
View File
@@ -1,54 +1,34 @@
/*
Private helper functions related to schema
*/
import {
AnnotationColumnSchema,
ArraySchema,
Field,
Schema,
} from "../common/types/schema";
import { LabelArray, LabelType } from "../util/dataframe/types";
export function _getColumnSchema(
schema: Schema,
field: Field,
col: LabelType
): ArraySchema {
export function _getColumnSchema(schema, field, col) {
/* look up the column definition */
switch (field) {
case Field.obs:
case "obs":
if (typeof col === "object")
throw new Error("unable to get column schema by query");
return schema.annotations.obsByName[col];
case Field.var:
case "var":
if (typeof col === "object")
throw new Error("unable to get column schema by query");
return schema.annotations.varByName[col];
case Field.emb:
case "emb":
if (typeof col === "object")
throw new Error("unable to get column schema by query");
return schema.layout.obsByName[col];
case Field.X:
case "X":
return schema.dataframe;
default:
throw new Error(`unknown field name: ${field}`);
}
}
export function _isIndex(
schema: Schema,
field: Field.obs | Field.var,
col: LabelType
): boolean {
export function _isIndex(schema, field, col) {
const index = schema.annotations?.[field].index;
return !!(index && index === col);
return index && index === col;
}
export function _getColumnDimensionNames(
schema: Schema,
field: Field,
col: LabelType
): LabelArray | undefined {
export function _getColumnDimensionNames(schema, field, col) {
/*
field/col may be an alias for multiple columns. Currently used to map ND
values to 1D dataframe columns for embeddings/layout. Signified by the presence
@@ -58,33 +38,30 @@ export function _getColumnDimensionNames(
if (!colSchema) {
return undefined;
}
if ("dims" in colSchema) {
return colSchema.dims;
}
return [col];
return colSchema.dims || [col];
}
export function _schemaColumns(schema: Schema, field: Field): string[] {
export function _schemaColumns(schema, field) {
switch (field) {
case Field.obs:
case "obs":
return Object.keys(schema.annotations.obsByName);
case Field.var:
case "var":
return Object.keys(schema.annotations.varByName);
case Field.emb:
case "emb":
return Object.keys(schema.layout.obsByName);
default:
throw new Error(`unknown field name: ${field}`);
}
}
export function _getWritableColumns(schema: Schema, field: Field): string[] {
if (field !== Field.obs) return [];
export function _getWritableColumns(schema, field) {
if (field !== "obs") return [];
return schema.annotations.obs.columns
.filter((v: AnnotationColumnSchema) => v.writable)
.map((v: AnnotationColumnSchema) => v.name);
.filter((v) => v.writable)
.map((v) => v.name);
}
export function _isContinuousType(schema: ArraySchema): boolean {
export function _isContinuousType(schema) {
const { type } = schema;
return !(type === "string" || type === "boolean" || type === "categorical");
}
client/src/annoMatrix/viewCreators.ts → client/src/annoMatrix/viewCreators.js
+9 -29
View File
@@ -4,18 +4,8 @@ instances of AnnoMatrix, implementing common UI functions.
*/
import { AnnoMatrixRowSubsetView, AnnoMatrixClipView } from "./views";
import AnnoMatrix from "./annoMatrix";
import {
DenseInt32Index,
IdentityInt32Index,
KeyIndex,
} from "../util/dataframe";
import { OffsetArray } from "../util/dataframe/types";
export function isubsetMask(
annoMatrix: AnnoMatrix,
obsMask: Uint8Array
): AnnoMatrixRowSubsetView {
export function isubsetMask(annoMatrix, obsMask) {
/*
Subset annomatrix to contain the rows which have truish value in the mask.
Maks length must equal annoMatrix.nObs (row count).
@@ -23,10 +13,7 @@ export function isubsetMask(
return isubset(annoMatrix, _maskToList(obsMask));
}
export function isubset(
annoMatrix: AnnoMatrix,
obsOffsets: OffsetArray
): AnnoMatrixRowSubsetView {
export function isubset(annoMatrix, obsOffsets) {
/*
Subset annomatrix to contain the positions contained in the obsOffsets array
@@ -38,10 +25,7 @@ export function isubset(
return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
}
export function subset(
annoMatrix: AnnoMatrix,
obsLabels: Int32Array
): AnnoMatrixRowSubsetView {
export function subset(annoMatrix, obsLabels) {
/*
subset based on labels
*/
@@ -49,21 +33,14 @@ export function subset(
return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
}
export function subsetByIndex(
annoMatrix: AnnoMatrix,
obsIndex: DenseInt32Index | IdentityInt32Index | KeyIndex
): AnnoMatrixRowSubsetView {
export function subsetByIndex(annoMatrix, obsIndex) {
/*
subset based upon the new obs index.
*/
return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
}
export function clip(
annoMatrix: AnnoMatrix,
qmin: number,
qmax: number
): AnnoMatrix {
export function clip(annoMatrix, qmin, qmax) {
/*
Create a view that clips all continuous data to the [min, max] range.
The matrix shape does not change, but the continuous values outside the
@@ -76,8 +53,11 @@ export function clip(
Private utility functions below
*/
function _maskToList(mask: Uint8Array): OffsetArray {
function _maskToList(mask) {
/* convert masks to lists - method wastes space, but is fast */
if (!mask) {
return null;
}
const list = new Int32Array(mask.length);
let elems = 0;
for (let i = 0, l = mask.length; i < l; i += 1) {
client/src/annoMatrix/views.ts → client/src/annoMatrix/views.js
+33 -104
View File
@@ -5,51 +5,25 @@ Views on the annomatrix. all API here is defined in viewCreators.js and annoMat
*/
import clip from "../util/clip";
import AnnoMatrix from "./annoMatrix";
import { _whereCacheCreate, WhereCache } from "./whereCache";
import { _whereCacheCreate } from "./whereCache";
import { _isContinuousType, _getColumnSchema } from "./schema";
import {
Dataframe,
DataframeValue,
DataframeValueArray,
LabelType,
} from "../util/dataframe";
import { Query } from "./query";
import {
AnnotationColumnSchema,
ArraySchema,
Field,
EmbeddingSchema,
} from "../common/types/schema";
import { LabelIndexBase } from "../util/dataframe/labelIndex";
type MapFn = (
field: Field,
colLabel: LabelType,
colSchema: ArraySchema,
colData: DataframeValueArray,
df: Dataframe
) => DataframeValueArray;
abstract class AnnoMatrixView extends AnnoMatrix {
constructor(viewOf: AnnoMatrix, rowIndex: LabelIndexBase | null = null) {
class AnnoMatrixView extends AnnoMatrix {
constructor(viewOf, rowIndex = null) {
const nObs = rowIndex ? rowIndex.size() : viewOf.nObs;
super(viewOf.schema, nObs, viewOf.nVar, rowIndex || viewOf.rowIndex);
this.viewOf = viewOf;
this.isView = true;
}
addObsAnnoCategory(col: LabelType, category: string): AnnoMatrix {
addObsAnnoCategory(col, category) {
const newAnnoMatrix = this._clone();
newAnnoMatrix.viewOf = this.viewOf.addObsAnnoCategory(col, category);
newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
return newAnnoMatrix;
}
async removeObsAnnoCategory(
col: LabelType,
category: string,
unassignedCategory: string
): Promise<AnnoMatrix> {
async removeObsAnnoCategory(col, category, unassignedCategory) {
const newAnnoMatrix = this._clone();
newAnnoMatrix.viewOf = await this.viewOf.removeObsAnnoCategory(
col,
@@ -60,7 +34,7 @@ abstract class AnnoMatrixView extends AnnoMatrix {
return newAnnoMatrix;
}
dropObsColumn(col: LabelType): AnnoMatrix {
dropObsColumn(col) {
const newAnnoMatrix = this._clone();
newAnnoMatrix.viewOf = this.viewOf.dropObsColumn(col);
newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
@@ -68,29 +42,21 @@ abstract class AnnoMatrixView extends AnnoMatrix {
return newAnnoMatrix;
}
addObsColumn<T extends DataframeValueArray>(
colSchema: AnnotationColumnSchema,
Ctor: new (n: number) => T,
value: T
): AnnoMatrix {
addObsColumn(colSchema, Ctor, value) {
const newAnnoMatrix = this._clone();
newAnnoMatrix.viewOf = this.viewOf.addObsColumn(colSchema, Ctor, value);
newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
return newAnnoMatrix;
}
renameObsColumn(oldCol: LabelType, newCol: LabelType): AnnoMatrix {
renameObsColumn(oldCol, newCol) {
const newAnnoMatrix = this._clone();
newAnnoMatrix.viewOf = this.viewOf.renameObsColumn(oldCol, newCol);
newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
return newAnnoMatrix;
}
async setObsColumnValues(
col: LabelType,
rowLabels: Int32Array,
value: DataframeValue
): Promise<AnnoMatrix> {
async setObsColumnValues(col, rowLabels, value) {
const newAnnoMatrix = this._clone();
newAnnoMatrix.viewOf = await this.viewOf.setObsColumnValues(
col,
@@ -102,11 +68,7 @@ abstract class AnnoMatrixView extends AnnoMatrix {
return newAnnoMatrix;
}
async resetObsColumnValues<T extends DataframeValue>(
col: LabelType,
oldValue: T,
newValue: T
): Promise<AnnoMatrix> {
async resetObsColumnValues(col, oldValue, newValue) {
const newAnnoMatrix = this._clone();
newAnnoMatrix.viewOf = await this.viewOf.resetObsColumnValues(
col,
@@ -118,7 +80,7 @@ abstract class AnnoMatrixView extends AnnoMatrix {
return newAnnoMatrix;
}
addEmbedding(colSchema: EmbeddingSchema): AnnoMatrix {
addEmbedding(colSchema) {
const newAnnoMatrix = this._clone();
newAnnoMatrix.viewOf = this.viewOf.addEmbedding(colSchema);
newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
@@ -127,32 +89,21 @@ abstract class AnnoMatrixView extends AnnoMatrix {
}
class AnnoMatrixMapView extends AnnoMatrixView {
mapFn: MapFn;
/*
A view which knows how to transform its data.
*/
constructor(viewOf: AnnoMatrix, mapFn: MapFn) {
A view which knows how to transform its data.
*/
constructor(viewOf, mapFn) {
super(viewOf);
this.mapFn = mapFn;
}
async _doLoad(
field: Field,
query: Query
): Promise<[WhereCache | null, Dataframe]> {
async _doLoad(field, query) {
const df = await this.viewOf._fetch(field, query);
const dfMapped = df.mapColumns(
(colData: DataframeValueArray, colIdx: number) => {
const colLabel = df.colIndex.getLabel(colIdx);
// @ts-expect-error ts-migrate --- TODO revisit:
// `colLabel`: Argument of type 'LabelType | undefined' is not assignable to parameter of type 'LabelType'.
const colSchema = _getColumnSchema(this.schema, field, colLabel);
// @ts-expect-error ts-migrate --- TODO revisit:
// `colLabel`: Argument of type 'LabelType | undefined' is not assignable to parameter of type 'LabelType'.
return this.mapFn(field, colLabel, colSchema, colData, df);
}
);
const dfMapped = df.mapColumns((colData, colIdx) => {
const colLabel = df.colIndex.getLabel(colIdx);
const colSchema = _getColumnSchema(this.schema, field, colLabel);
return this.mapFn(field, colLabel, colSchema, colData, df);
});
const whereCacheUpdate = _whereCacheCreate(
field,
query,
@@ -163,23 +114,12 @@ class AnnoMatrixMapView extends AnnoMatrixView {
}
export class AnnoMatrixClipView extends AnnoMatrixMapView {
clipRange: [number, number];
isClipped: boolean;
/*
A view which is a clipped transformation of its parent
*/
constructor(viewOf: AnnoMatrix, qmin: number, qmax: number) {
super(
viewOf,
(
field: Field,
colLabel: LabelType,
colSchema: ArraySchema,
colData: DataframeValueArray,
df: Dataframe
) => _clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax)
A view which is a clipped transformation of its parent
*/
constructor(viewOf, qmin, qmax) {
super(viewOf, (field, colLabel, colSchema, colData, df) =>
_clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax)
);
this.isClipped = true;
this.clipRange = [qmin, qmax];
@@ -189,21 +129,18 @@ export class AnnoMatrixClipView extends AnnoMatrixMapView {
export class AnnoMatrixRowSubsetView extends AnnoMatrixView {
/*
A view which is a subset of total rows.
*/
constructor(viewOf: AnnoMatrix, rowIndex: LabelIndexBase) {
A view which is a subset of total rows.
*/
constructor(viewOf, rowIndex) {
super(viewOf, rowIndex);
Object.seal(this);
}
async _doLoad(
field: Field,
query: Query
): Promise<[WhereCache | null, Dataframe]> {
async _doLoad(field, query) {
const df = await this.viewOf._fetch(field, query);
// don't try to row-subset the var dimension.
if (field === Field.var) {
if (field === "var") {
return [null, df];
}
@@ -221,23 +158,15 @@ export class AnnoMatrixRowSubsetView extends AnnoMatrixView {
Utility functions below
*/
function _clipAnnoMatrix(
field: Field,
colLabel: LabelType,
colSchema: ArraySchema,
colData: DataframeValueArray,
df: Dataframe,
qmin: number,
qmax: number
): DataframeValueArray {
function _clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax) {
/* only clip obs and var scalar columns */
if (field !== Field.obs && field !== Field.X) return colData;
if (field !== "obs" && field !== "X") return colData;
if (!_isContinuousType(colSchema)) return colData;
if (qmin < 0) qmin = 0;
if (qmax > 1) qmax = 1;
if (qmin === 0 && qmax === 1) return colData;
const quantiles = df.col(colLabel).summarizeContinuous().percentiles;
const quantiles = df.col(colLabel).summarize().percentiles;
const lower = quantiles[100 * qmin];
const upper = quantiles[100 * qmax];
const clippedData = clip(colData.slice(), lower, upper, Number.NaN);
client/src/annoMatrix/whereCache.ts → client/src/annoMatrix/whereCache.js
+12 -45
View File
@@ -2,7 +2,7 @@
Private support functions.
This implements a query resolver cache, mapping a query onto the column labels
resolved by that query. These labels are then used to manage the actual data cache,
resolved by that query. These labels are then used to manage the acutal data cache,
which stores data by the resolved label.
There are three query forms:
@@ -49,33 +49,9 @@ creates a cache entry of:
}
*/
import { _getColumnDimensionNames } from "./schema";
import { _hashStringValues, Query } from "./query";
import { Field, Schema } from "../common/types/schema";
import { LabelArray } from "../util/dataframe/types";
import { _hashStringValues } from "./query";
export interface WhereCache {
summarize?: {
[key: string]: {
[key: string]: WhereCacheTerms;
};
};
where?: {
[key: string]: WhereCacheTerms;
};
}
export type WhereCacheColumnLabels = LabelArray;
interface WhereCacheTerms {
[key: string]: Map<string, Map<string, WhereCacheColumnLabels>>;
}
export function _whereCacheGet(
whereCache: WhereCache,
schema: Schema,
field: Field,
query: Query
): WhereCacheColumnLabels | [undefined] {
export function _whereCacheGet(whereCache, schema, field, query) {
/*
query will either be an where query (object) or a column name (string).
@@ -83,7 +59,7 @@ export function _whereCacheGet(
*/
if (typeof query === "object") {
if ("where" in query) {
if (query.where) {
const {
field: queryField,
column: queryColumn,
@@ -92,7 +68,7 @@ export function _whereCacheGet(
const columnMap = whereCache?.where?.[field]?.[queryField];
return columnMap?.get(queryColumn)?.get(queryValue) ?? [undefined];
}
if ("summarize" in query) {
if (query.summarize) {
const {
method,
field: queryField,
@@ -109,17 +85,13 @@ export function _whereCacheGet(
return _getColumnDimensionNames(schema, field, query) ?? [undefined];
}
export function _whereCacheCreate(
field: Field,
query: Query,
columnLabels: LabelArray
): WhereCache | null {
export function _whereCacheCreate(field, query, columnLabels) {
/*
Create a new whereCache
*/
if (typeof query !== "object") return null;
if ("where" in query) {
if (query.where) {
const {
field: queryField,
column: queryColumn,
@@ -135,7 +107,7 @@ export function _whereCacheCreate(
},
};
}
if ("summarize" in query) {
if (query.summarize) {
const {
method,
field: queryField,
@@ -159,25 +131,20 @@ export function _whereCacheCreate(
return {};
}
function __mergeQueries(dst: WhereCacheTerms, src: WhereCacheTerms) {
function __mergeQueries(dst, src) {
for (const [queryField, columnMap] of Object.entries(src)) {
dst[queryField] = dst[queryField] || new Map();
for (const [queryColumn, valueMap] of columnMap) {
if (!dst[queryField].has(queryColumn))
dst[queryField].set(queryColumn, new Map());
for (const [queryValue, columnLabels] of valueMap) {
// @ts-expect-error ts-migrate --- TODO revisit:
// `dst[queryField].get(queryColumn)` Object is possibly 'undefined'.
dst[queryField].get(queryColumn).set(queryValue, columnLabels);
}
}
}
}
function __whereCacheMerge(
dst: WhereCache,
src: WhereCache | null
): WhereCache {
function __whereCacheMerge(dst, src) {
/*
merge src into dst (modifies dst)
*/
@@ -204,6 +171,6 @@ function __whereCacheMerge(
return dst;
}
export function _whereCacheMerge(...caches: (WhereCache | null)[]): WhereCache {
return caches.reduce(__whereCacheMerge, {} as WhereCache);
export function _whereCacheMerge(...caches) {
return caches.reduce(__whereCacheMerge, {});
}