mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-03 13:18:12 +08:00
Redux refactor (#1571)
* refactor categorical controls state * lint * fix race condition in tests * fix typo * add missing update on subset * remove obsolete code * update jest and puppeteer major version; update all minors * update when label changes * remove lint from tests; increase timeouts in e2e tests * initial refactoring to new async annomatrix * refine error handling * fix bad merge * add continuous legend * lint * fix memoization in color table creators * partial implementation of user defined annotations * add new annotations action creator file * first pass at user annotations * additional user annotation bug fixes * user annotation auto-save * unit test cleanup * lint * refactor into multiple files * cleanup * add column GC * fix several bugs in user annotations * remove debug code * no anonymous functions * undo redo cleanup * file cleanup * scatterplot * performance * cleanup * remove old code * render in parallel with load * fix race condition * simply graph rendering * render throttle DRY * fix category label order * fix typo in e2e test setup * re-fix the e2e test setup * be more tolerant of races * anno matrix unit tests * temp disable reembedding * pilot port continuous histo to react-async * name change * lint * fix repaint bug * typo fix * update snap to match new ids * world/universe name cleanup * move annoMatrix to src dir * use private underscore naming convention * fix corner case in all selected * name cleanup * add layout control * init edge case * lint * port scatterplot * fix label indexing bug and improve tests * port category to react-async * fix user annotation labelling while subset * select all of prev layout on layout switch * fix race with crossfilter update * prettier lint * fix misleading comment * fix url composition in loader * first pass at crossfilter tests * lint * lint * fix typo * improved error handling for network errors * fix memoization bug * add memo * refactor for performnce * add missing single-value handling in select exact parser * small bugs discovered by tests * lint * additional crossfilter unit tests * remove extraneous comment * add support for automatic category determination * lint * fix render bug in category * take advantage of schema categories guarantee * lint * do not clear history when resetting * enhanced annomatrix gc * lint * finish renaming to follow conventions; fix clone race bug * lint * add priority based loading to improve initial data load UX * crossfilter cache perf * perf tuning * remove timers * documentation * PR review changes * PR review changes * more PR review edits * improve clarity of comment * more PR review fixes * port centroidLabels to use react-async * remove dead code * pr review updates * oops, remove logging
This commit is contained in:
Bruce Martin
@@ -0,0 +1,634 @@
|
||||
import { Dataframe, IdentityInt32Index } from "../util/dataframe";
|
||||
import {
|
||||
_getColumnDimensionNames,
|
||||
_getColumnSchema,
|
||||
_schemaColumns,
|
||||
_getWritableColumns,
|
||||
} from "./schema";
|
||||
import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
|
||||
import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
|
||||
import _shallowClone from "./clone";
|
||||
|
||||
export default class AnnoMatrix {
|
||||
/*
|
||||
Abstract base class for all AnnoMatrix objects. This class provides a proxy
|
||||
to the annotated matrix data authoritatively served by the server/back-end.
|
||||
|
||||
AnnoMatrix instances are immutable, meaning that their schema and dimensionality
|
||||
will not change, and simple object equality can be used to detect structural
|
||||
changes. The actual data is cached, and not guaranteed to be present -- any
|
||||
request to access data must be resolved by a fetch() call, which is async, and
|
||||
may involve a server round-trip.
|
||||
|
||||
Guarantees made by the immutabilty, ie, any of these can be detected by
|
||||
simple annoMatrix compare:
|
||||
* schema is the same, including all fields and columns
|
||||
* dimensionality is the same (nObs, nVar)
|
||||
* data mapping/transformation, such as clipping, are the same
|
||||
|
||||
AnnoMatrixes also "stack" like filters, allowing for the construction of
|
||||
views which transform the data in some manner.
|
||||
|
||||
The bootstrap class is AnnoMatrixLoader, which is the caching server proxy, and
|
||||
is bootstrapped with a API URL:
|
||||
new AnnoMatirx(url, schema) -> annoMatrix
|
||||
|
||||
There are various "views", such as AnnoMatrixRowSubsetView, which provide
|
||||
the same interface but with a transformed view of the server data. Utilities in
|
||||
viewCreators.js can be used to create these views:
|
||||
clip(annoMatrix, min, max) -> annoMatrix
|
||||
subset(annoMatrix, rowLabels) -> annoMatrix
|
||||
etc.
|
||||
*/
|
||||
static fields() {
|
||||
/*
|
||||
return the fields present in the AnnoMatrix instance.
|
||||
*/
|
||||
return ["obs", "var", "emb", "X"];
|
||||
}
|
||||
|
||||
constructor(schema, nObs, nVar, rowIndex = null) {
|
||||
/*
|
||||
Private constructor - this is an abstract base class. Do not use.
|
||||
*/
|
||||
|
||||
/*
|
||||
Public instance fields:
|
||||
* schema - the matrix schema. IMPORTANT: always the entire schema, for the
|
||||
base (unfiltered, unclipped, unsubset) annotated matrix, as the server
|
||||
presents it.
|
||||
* nObs, nVar - size of each dimension. These will accurately reflect the
|
||||
size of the current annoMatrix view. For example, if you subset the view,
|
||||
the nObs will be smaller.
|
||||
* rowIndex - a rowIndex shared by all data on this view (ie, the list of cells).
|
||||
The row index labels are as defined by the base dataset from the server.
|
||||
* isView - true if this is a view, false if not.
|
||||
* viewOf - pointer to parent annomatrix if a view, undefined/null if not a view.
|
||||
*/
|
||||
this.schema = indexEntireSchema(schema);
|
||||
this.nObs = nObs;
|
||||
this.nVar = nVar;
|
||||
this.rowIndex = rowIndex || new IdentityInt32Index(nObs);
|
||||
this.isView = false;
|
||||
this.viewOf = undefined;
|
||||
|
||||
/*
|
||||
Private instance variables.
|
||||
|
||||
These are caches - lazily loaded. The only guarantee is that if they
|
||||
are loaded, they will conform to the schema & dimensionality constraints.
|
||||
|
||||
Do NOT use directly - instead, use the fetch() and preload() API.
|
||||
*/
|
||||
this._cache = {
|
||||
obs: Dataframe.empty(this.rowIndex),
|
||||
var: Dataframe.empty(this.rowIndex),
|
||||
emb: Dataframe.empty(this.rowIndex),
|
||||
X: Dataframe.empty(this.rowIndex),
|
||||
};
|
||||
this._pendingLoad = {
|
||||
obs: {},
|
||||
var: {},
|
||||
emb: {},
|
||||
X: {},
|
||||
};
|
||||
this._whereCache = {};
|
||||
this._gcInfo = new Map();
|
||||
}
|
||||
|
||||
/**
|
||||
** Schema helper/accessors
|
||||
**/
|
||||
getMatrixColumns(field) {
|
||||
/*
|
||||
Return array of column names in the field. ONLY supported on the
|
||||
obs, var and emb fields. X currently unimplemented and will throw.
|
||||
|
||||
For exmaple:
|
||||
|
||||
annoMatrix.getMatrixColumns("obs") -> ["louvain", "n_genes"]
|
||||
*/
|
||||
return _schemaColumns(this.schema, field);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this -- need to be able to call this on instances
|
||||
getMatrixFields() {
|
||||
/*
|
||||
Return array of fields in this annoMatrix. Currently hard-wired to
|
||||
return: ["X", "obs", "var", "emb"].
|
||||
|
||||
These are the fields from data may be requested.
|
||||
*/
|
||||
return AnnoMatrix.fields();
|
||||
}
|
||||
|
||||
getColumnSchema(field, col) {
|
||||
/*
|
||||
Return the schema for the field & column ,eg,
|
||||
|
||||
anonMatrix.getColumnSchema("obs", "n_genes") -> { type: "int32", name: "n_genes" }
|
||||
|
||||
This is identical to the information in the annoMatrix.schema
|
||||
instance variable.
|
||||
*/
|
||||
return _getColumnSchema(this.schema, field, col);
|
||||
}
|
||||
|
||||
getColumnDimensions(field, col) {
|
||||
/*
|
||||
Return the dimensions on this field / column. For most fields, which are 1D,
|
||||
this just return the column name. Multi-dimensional columns, such as embeddings,
|
||||
will return >1 name.
|
||||
|
||||
Examples:
|
||||
|
||||
getColumnDimensions("obs", "louvain") -> ["louvain"]
|
||||
getColumnDimensions("emb", "umap") -> ["umap_0", "umap_1"]
|
||||
|
||||
*/
|
||||
return _getColumnDimensionNames(this.schema, field, col);
|
||||
}
|
||||
|
||||
/**
|
||||
** General utility methods
|
||||
**/
|
||||
base() {
|
||||
/*
|
||||
return the base of view, or `this` if not a view.
|
||||
*/
|
||||
let annoMatrix = this;
|
||||
while (annoMatrix.isView) annoMatrix = annoMatrix.viewOf;
|
||||
return annoMatrix;
|
||||
}
|
||||
|
||||
/**
|
||||
** Load / read interfaces
|
||||
**/
|
||||
fetch(field, q) {
|
||||
/*
|
||||
Return the given query on a single matrix field as a single dataframe.
|
||||
Currently supports ONLY full column query.
|
||||
|
||||
Returns a Promise for the query result, which will resolve to a dataframe.
|
||||
|
||||
Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'. Value
|
||||
represents the underlying object upon which the query is occuring.
|
||||
|
||||
Query is one of:
|
||||
* a string, representing a single column name from the field, eg,
|
||||
"n_genes"
|
||||
* an object, containing an "value" query (see below).
|
||||
* an array, containing one or more of the above.
|
||||
|
||||
Columns may have more than one dimension, and all will be fetched
|
||||
and returned together. This is most commonly seen in an embedding,
|
||||
which usually has two dimensions.
|
||||
|
||||
A value query allows for fetching based upon the value in another
|
||||
field/column, similar to a join. Currently only supported on the var
|
||||
dimension, allowing query of X columns by var value (eg, gene name)
|
||||
|
||||
The query filter is a single value filter:
|
||||
{ "field name": [
|
||||
{name: "column name", values: [ list of values ]}
|
||||
]}
|
||||
One and only one value filter is allowed in a value query.
|
||||
|
||||
Examples:
|
||||
|
||||
1. Fetch the "n_genes" column the "obs":
|
||||
|
||||
const df = await fetch("obs", "n_genes")
|
||||
console.log("Largest number of genes is: ", df.summarize().max);
|
||||
|
||||
2. Fetch two separate columns from obs. Returns a single dataframe containing
|
||||
the columns:
|
||||
|
||||
const df = await fetch("obs", ["n_genes", "louvain"])
|
||||
console.log("Cell 0 has category: ", df.at(0, "louvain"));
|
||||
|
||||
3. Fetch an entire X (expression counts) column that has a var annotation
|
||||
value "TYMP" in the var index.
|
||||
|
||||
fetch("X", {
|
||||
where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
|
||||
})
|
||||
|
||||
In AnnData & Pandas DataFrame API, this is equivalent to:
|
||||
adata.X[:, adata.var.index.get_loc("SUMO3")]
|
||||
|
||||
The value query is a recodification and subset of the server REST API
|
||||
value filter JSON. Range queries and multiple filters are not currently
|
||||
supported.
|
||||
|
||||
*/
|
||||
return this._fetch(field, q);
|
||||
}
|
||||
|
||||
prefetch(field, q) {
|
||||
/*
|
||||
Start a data fetch & cache fill. Identical to fetch() except it does
|
||||
not return a value.
|
||||
|
||||
Primary use is to being a cache load as early as is possible, reducing
|
||||
overall component rendering latency.
|
||||
*/
|
||||
this._fetch(field, q);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
** Save / mutate interfaces - manipulation of "writable" OBS annotations.
|
||||
**
|
||||
** These are all present to support client-side creation of OBS annotations, aka
|
||||
** "user annotations".
|
||||
**
|
||||
** They implement common manipulations to the AnnoMatrix, maintaining the
|
||||
** norma guarantees around correctness of public API, eg,
|
||||
** - schema will be correct, including the "writable" attribute
|
||||
** - fetch() will return the latest data, even from views
|
||||
** - immutability guranteeds
|
||||
**
|
||||
** As most of these interfaces mutate the annoMatrix, they return a new
|
||||
** annoMatrix
|
||||
**
|
||||
** The actual implementation is in the sub-classes, which MUST override these.
|
||||
**/
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
|
||||
addObsAnnoCategory(col, category) {
|
||||
/*
|
||||
Add a new category value (aka "label") to a writable obs column, and return the new AnnoMatrix.
|
||||
Typical use is to add a new user-created label to a user-created obs categorical
|
||||
annotation.
|
||||
|
||||
Will throw column does not exist or is not writable.
|
||||
|
||||
Example:
|
||||
|
||||
addObsAnnoCategory("my cell type", "left toenail") -> AnnoMatrix
|
||||
|
||||
*/
|
||||
_subclassResponsibility();
|
||||
}
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
|
||||
async removeObsAnnoCategory(col, category, unassignedCategory) {
|
||||
/*
|
||||
Remove a category value from an obs column, reassign any obs having that value
|
||||
to the 'unassignedCategory' value, and return a promise for a new AnnoMatrix.
|
||||
Typical use is to remove a user-created label from a user-created obs categorical
|
||||
annotation.
|
||||
|
||||
Will throw column does not exist or is not writable.
|
||||
|
||||
An `unassignedCategory` value must be provided, for assignment to any obs/cells
|
||||
that had the now-delete category label as their value.
|
||||
|
||||
Example:
|
||||
await removeObsAnnoCategory("my-tissue-type", "right earlobe", "unassigned") -> AnnoMatrix
|
||||
|
||||
NOTE: method is async as it may need to fetch data to provide the reassignment.
|
||||
*/
|
||||
_subclassResponsibility();
|
||||
}
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
|
||||
dropObsColumn(col) {
|
||||
/*
|
||||
Drop an entire writable column, eg a user-created obs annotation. Typical use
|
||||
is to provide the "Delete Category" implementation. Returns the new AnnoMatrix.
|
||||
Will throw if not a writable annotation.
|
||||
|
||||
Will throw column does not exist or is not writable.
|
||||
|
||||
Example:
|
||||
|
||||
dropObsColumn("old annotations") -> AnnoMatrix
|
||||
*/
|
||||
_subclassResponsibility();
|
||||
}
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
|
||||
addObsColumn(colSchema, Ctor, value) {
|
||||
/*
|
||||
Add a new writable OBS annotation column, with the caller-specified schema, initial value
|
||||
type and value.
|
||||
|
||||
Value may be any one of:
|
||||
* an array of values
|
||||
* a primitive type, including null or undefined.
|
||||
If an array, length must be the same as 'this.nObs', and constructor must equal 'Ctor'.
|
||||
If a primitive, 'Ctor' will be used to create the initial value, which will be filled
|
||||
with 'value'.
|
||||
|
||||
Throws if the name specified in 'colSchema' duplicates an existing obs column.
|
||||
|
||||
Returns a new AnnoMatrix.
|
||||
|
||||
Examples:
|
||||
|
||||
addObsColumn(
|
||||
{ name: "foo", type: "categorical", categories: "unassigned" },
|
||||
Array,
|
||||
"unassigned"
|
||||
) -> AnnoMatrix
|
||||
|
||||
*/
|
||||
_subclassResponsibility();
|
||||
}
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
|
||||
renameObsColumn(oldCol, newCol) {
|
||||
/*
|
||||
Rename the obs column 'oldCol' to have name 'newCol' and returns new AnnoMatrix.
|
||||
|
||||
Will throw column does not exist or is not writable, or if 'newCol' is not unique.
|
||||
|
||||
Example:
|
||||
|
||||
renameObsColumn('cell type', 'old cell type') -> AnnoMatrix.
|
||||
|
||||
*/
|
||||
_subclassResponsibility();
|
||||
}
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
|
||||
async setObsColumnValues(col, obsLabels, value) {
|
||||
/*
|
||||
Set all obs with label in array 'obsLabels' to have 'value'. Typical use would be
|
||||
to set a group of cells to have a label on a user-created categorical anntoation
|
||||
(eg set all selected cells to have a label).
|
||||
|
||||
NOTE: async method, as it may need to fetch.
|
||||
|
||||
Will throw column does not exist or is not writable.
|
||||
|
||||
Example:
|
||||
await setObsColmnValues("flavor", [383, 400], "tasty") -> AnnoMtarix
|
||||
|
||||
*/
|
||||
_subclassResponsibility();
|
||||
}
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
|
||||
async resetObsColumnValues(col, oldValue, newValue) {
|
||||
/*
|
||||
Set by value - all elements in the column with value 'oldValue' are set to 'newValue'.
|
||||
Async method - returns a promise for a new AnnoMatrix.
|
||||
|
||||
Typical use would be to set all labels of one value to another.
|
||||
|
||||
Will throw column does not exist or is not writable.
|
||||
|
||||
Example:
|
||||
await resetObsColumnValues("my notes", "good", "not-good") -> AnnoMatrix
|
||||
|
||||
*/
|
||||
_subclassResponsibility();
|
||||
}
|
||||
|
||||
/**
|
||||
** Private interfaces below.
|
||||
**/
|
||||
_resolveCachedQueries(field, queries) {
|
||||
return queries
|
||||
.map((query) =>
|
||||
_whereCacheGet(this._whereCache, this.schema, field, query).filter(
|
||||
(cacheKey) =>
|
||||
cacheKey !== undefined && this._cache[field].hasCol(cacheKey)
|
||||
)
|
||||
)
|
||||
.flat();
|
||||
}
|
||||
|
||||
async _fetch(field, q) {
|
||||
if (!AnnoMatrix.fields().includes(field)) return undefined;
|
||||
const queries = Array.isArray(q) ? q : [q];
|
||||
|
||||
/* find cached columns we need, and GC the rest */
|
||||
const cachedColumns = this._resolveCachedQueries(field, queries);
|
||||
this._gcFetchCleanup(field, cachedColumns);
|
||||
|
||||
/* find any query not already cached */
|
||||
const uncachedQueries = queries.filter((query) =>
|
||||
_whereCacheGet(this._whereCache, this.schema, field, query).some(
|
||||
(cacheKey) =>
|
||||
cacheKey === undefined || !this._cache[field].hasCol(cacheKey)
|
||||
)
|
||||
);
|
||||
|
||||
/* load uncached queries */
|
||||
if (uncachedQueries.length > 0) {
|
||||
await Promise.all(
|
||||
uncachedQueries.map((query) =>
|
||||
this._getPendingLoad(field, query, async (_field, _query) => {
|
||||
/* fetch, then index. _doLoad is subclass interface */
|
||||
const [whereCacheUpdate, df] = await this._doLoad(_field, _query);
|
||||
this._cache[_field] = this._cache[_field].withColsFrom(df);
|
||||
this._whereCache = _whereCacheMerge(
|
||||
this._whereCache,
|
||||
whereCacheUpdate
|
||||
);
|
||||
})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
/* everything we need is in the cache, so just cherry-pick requested columns */
|
||||
const requestedCacheKeys = this._resolveCachedQueries(field, queries);
|
||||
const response = this._cache[field].subset(null, requestedCacheKeys);
|
||||
this._gcUpdateStats(field, response);
|
||||
return response;
|
||||
}
|
||||
|
||||
async _getPendingLoad(field, query, fetchFn) {
|
||||
/*
|
||||
Given a query on a field, ensure that we only have a single outstanding
|
||||
fetch at any given time. If multiple requests occur while a fetch is
|
||||
outstanding, just wait for the original.
|
||||
|
||||
This is implemented by returning a promise that will await the singular
|
||||
fetch promise.
|
||||
*/
|
||||
const key = _queryCacheKey(field, query);
|
||||
if (!this._pendingLoad[field][key]) {
|
||||
this._pendingLoad[field][key] = fetchFn(field, query);
|
||||
try {
|
||||
await this._pendingLoad[field][key];
|
||||
} finally {
|
||||
delete this._pendingLoad[field][key];
|
||||
}
|
||||
}
|
||||
return this._pendingLoad[field][key];
|
||||
}
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this -- make sure subclass implements
|
||||
async _doLoad() {
|
||||
_subclassResponsibility();
|
||||
}
|
||||
|
||||
/**
|
||||
** Garbage collection of annomatrix cache to manage memory use.
|
||||
**/
|
||||
|
||||
/*
|
||||
These callbacks implement a GC policy for the cache. Background:
|
||||
|
||||
* For the Loader (base) annomatrix, re-filling the cache is expensive as
|
||||
it requires an HTTP fetch.
|
||||
* user-defined / writable columns must not be GC'ed as they may be
|
||||
still pending a save/commit.
|
||||
* For views, cost is less and (roughly) proportional with nObs
|
||||
* obs, var and emb do not grow without bounds, and are needed constantly
|
||||
for rendering.
|
||||
a) There is no upside to GC'ing these in the base (loader)
|
||||
b) The undo/redo cache can hold a large number in views, which is worht GC'ing
|
||||
* X is often much larger than memory, and the UI allows add/del from
|
||||
this. Most of the GC potential is here in both the base and views.
|
||||
|
||||
Current policy:
|
||||
* if in active use ("hot") do not GC obs, var or emb.
|
||||
* never, ever GC writable obs columns
|
||||
* For base/loader set a numeric limit on maximum X column count
|
||||
* For views, apply a fixed limit to the number of columns cached in any field.
|
||||
Limit will be lower if not hot.
|
||||
|
||||
To be effective, the GC callback needs to be invoked from the undo/redo code,
|
||||
as much of the cache is pinned by that data structure.
|
||||
*/
|
||||
_gcField(field, isHot, pinnedColumns) {
|
||||
const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
|
||||
|
||||
const cache = this._cache[field];
|
||||
if (cache.colIndex.size() < maxColumns) return; // trivial rejection
|
||||
|
||||
const candidates = cache.colIndex
|
||||
.labels()
|
||||
.filter((col) => !pinnedColumns.includes(col));
|
||||
|
||||
const excessCount = candidates.length + pinnedColumns.length - maxColumns;
|
||||
if (excessCount > 0) {
|
||||
const { _gcInfo } = this;
|
||||
candidates.sort((a, b) => {
|
||||
let atime = _gcInfo.get(_columnCacheKey(field, a));
|
||||
if (atime === undefined) atime = 0;
|
||||
|
||||
let btime = _gcInfo.get(_columnCacheKey(field, b));
|
||||
if (btime === undefined) btime = 0;
|
||||
|
||||
return atime - btime;
|
||||
});
|
||||
|
||||
const toDrop = candidates.slice(0, excessCount);
|
||||
// helpful debugging - please leave in place.
|
||||
// console.log(
|
||||
// `GC: dropping from ${field} hot:${isHot}, columns [${toDrop.join(
|
||||
// ", "
|
||||
// )}]`
|
||||
// );
|
||||
this._cache[field] = toDrop.reduce(
|
||||
(df, col) => df.dropCol(col),
|
||||
this._cache[field]
|
||||
);
|
||||
toDrop.forEach((col) => _gcInfo.delete(_columnCacheKey(field, col)));
|
||||
}
|
||||
}
|
||||
|
||||
_gcFetchCleanup(field, pinnedColumns) {
|
||||
/*
|
||||
Called during data load/fetch. By definition, this is 'hot', so we
|
||||
only want to gc X.
|
||||
*/
|
||||
if (field === "X") {
|
||||
this._gcField(
|
||||
field,
|
||||
true,
|
||||
pinnedColumns.concat(_getWritableColumns(this.schema, field))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
_gc(hints) {
|
||||
/*
|
||||
Called from middleware, or elsewhere. isHot is true if we are in the active store,
|
||||
or false if we are in some other context (eg, history state).
|
||||
*/
|
||||
const { isHot } = hints;
|
||||
const candidateFields = isHot ? ["X"] : ["X", "emb", "var", "obs"];
|
||||
candidateFields.forEach((field) =>
|
||||
this._gcField(field, isHot, _getWritableColumns(this.schema, field))
|
||||
);
|
||||
}
|
||||
|
||||
_gcUpdateStats(field, dataframe) {
|
||||
/*
|
||||
called each time a query is performed, allowing the gc to update any bookkeeping
|
||||
information. Currently, this is just a simple last-fetched timestamp, stored
|
||||
in a Map.
|
||||
|
||||
Map objects preserve order of insertion. This is leveraged as a cheap way to
|
||||
do LRU, by removing and re-inserting keys. IMPORTANT: the cleanup code assumes
|
||||
the map insertion order is least-recently-used first.
|
||||
*/
|
||||
const cols = dataframe.colIndex.labels();
|
||||
const { _gcInfo } = this;
|
||||
const now = Date.now();
|
||||
cols.forEach((c) => {
|
||||
// gcInfo.delete(c);
|
||||
_gcInfo.set(_columnCacheKey(field, c), now);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
Cloning sublcass protocol - we rely in cloning to preserve immutable
|
||||
symantics while not causing races or other side effects in internal
|
||||
cache management.
|
||||
|
||||
Subclasses must override _cloneDeeper() if they have state which requires
|
||||
something other than a shallow copy. Overrides MUST call super()._cloneDeepr(),
|
||||
and return its result (after any required modification). _cloneDeeper()
|
||||
will be called on the OLD object, with the NEW object as an argument.
|
||||
|
||||
Do not override _clone();
|
||||
**/
|
||||
_cloneDeeper(clone) {
|
||||
clone._cache = _shallowClone(this._cache);
|
||||
clone._gcInfo = new Map();
|
||||
clone._pendingLoad = {
|
||||
obs: {},
|
||||
var: {},
|
||||
emb: {},
|
||||
X: {},
|
||||
};
|
||||
return clone;
|
||||
}
|
||||
|
||||
_clone() {
|
||||
const clone = _shallowClone(this);
|
||||
this._cloneDeeper(clone);
|
||||
Object.seal(clone);
|
||||
return clone;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
private utility functions below
|
||||
*/
|
||||
|
||||
function _queryCacheKey(field, query) {
|
||||
if (typeof query === "object") {
|
||||
const { field: queryField, column: queryColumn, value: queryValue } = query;
|
||||
return `${field}/${queryField}/${queryColumn}/${queryValue}`;
|
||||
}
|
||||
return `${field}/${query}`;
|
||||
}
|
||||
|
||||
function _columnCacheKey(field, column) {
|
||||
return `${field}/${column}`;
|
||||
}
|
||||
|
||||
function _subclassResponsibility() {
|
||||
/* protect against bugs in subclass */
|
||||
throw new Error("subclass failed to implement required method");
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
/*
|
||||
Shallow clone an object, correctly handling prototype
|
||||
*/
|
||||
export default function _shallowClone(orig) {
|
||||
return Object.assign(Object.create(Object.getPrototypeOf(orig)), orig);
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
/*
|
||||
Row crossfilter proxy for an AnnoMatrix. This wraps Crossfilter,
|
||||
providing a number of services, and ensuring that the crossfilter and
|
||||
AnnoMatrix stay in sync:
|
||||
- on-demand index creation as data is loaded
|
||||
- transparently mapping between queries and crossfilter index names.
|
||||
- for mutation of the matrix by user annotations, maintain synchronization
|
||||
between Crossfilter and AnnoMatrix.
|
||||
*/
|
||||
import Crossfilter from "../util/typedCrossfilter";
|
||||
import { _getColumnSchema } from "./schema";
|
||||
|
||||
function _dimensionNameFromDf(field, df) {
|
||||
const colNames = df.colIndex.labels();
|
||||
return _dimensionName(field, colNames);
|
||||
}
|
||||
|
||||
function _dimensionName(field, colNames) {
|
||||
if (!Array.isArray(colNames)) return `${field}/${colNames}`;
|
||||
return `${field}/${colNames.join(":")}`;
|
||||
}
|
||||
|
||||
export default class AnnoMatrixObsCrossfilter {
|
||||
constructor(annoMatrix, _obsCrossfilter = null) {
|
||||
this.annoMatrix = annoMatrix;
|
||||
this.obsCrossfilter =
|
||||
_obsCrossfilter || new Crossfilter(annoMatrix._cache.obs);
|
||||
this.obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
|
||||
}
|
||||
|
||||
size() {
|
||||
return this.obsCrossfilter.size();
|
||||
}
|
||||
|
||||
/**
|
||||
Managing the associated annoMatrix. These wrappers are necessary to
|
||||
make coordinated changes to BOTH the crossfilter and annoMatrix, and
|
||||
ensure that all state stays synchronized.
|
||||
|
||||
See API documentation in annoMatrix.js.
|
||||
**/
|
||||
addObsColumn(colSchema, Ctor, value) {
|
||||
const annoMatrix = this.annoMatrix.addObsColumn(colSchema, Ctor, value);
|
||||
const obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
|
||||
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
|
||||
}
|
||||
|
||||
dropObsColumn(col) {
|
||||
const annoMatrix = this.annoMatrix.dropObsColumn(col);
|
||||
let { obsCrossfilter } = this;
|
||||
const dimName = _dimensionName("obs", col);
|
||||
if (obsCrossfilter.hasDimension(dimName)) {
|
||||
obsCrossfilter = obsCrossfilter.delDimension(dimName);
|
||||
}
|
||||
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
|
||||
}
|
||||
|
||||
renameObsColumn(oldCol, newCol) {
|
||||
const annoMatrix = this.annoMatrix.renameObsColumn(oldCol, newCol);
|
||||
const oldDimName = _dimensionName("obs", oldCol);
|
||||
const newDimName = _dimensionName("obs", newCol);
|
||||
let { obsCrossfilter } = this;
|
||||
if (obsCrossfilter.hasDimension(oldDimName)) {
|
||||
obsCrossfilter = obsCrossfilter.renameDimension(oldDimName, newDimName);
|
||||
}
|
||||
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
|
||||
}
|
||||
|
||||
addObsAnnoCategory(col, category) {
|
||||
const annoMatrix = this.annoMatrix.addObsAnnoCategory(col, category);
|
||||
const dimName = _dimensionName("obs", col);
|
||||
let { obsCrossfilter } = this;
|
||||
if (obsCrossfilter.hasDimension(dimName)) {
|
||||
obsCrossfilter = obsCrossfilter.delDimension(dimName);
|
||||
}
|
||||
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
|
||||
}
|
||||
|
||||
async removeObsAnnoCategory(col, category, unassignedCategory) {
|
||||
const annoMatrix = await this.annoMatrix.removeObsAnnoCategory(
|
||||
col,
|
||||
category,
|
||||
unassignedCategory
|
||||
);
|
||||
const dimName = _dimensionName("obs", col);
|
||||
let { obsCrossfilter } = this;
|
||||
if (obsCrossfilter.hasDimension(dimName)) {
|
||||
obsCrossfilter = obsCrossfilter.delDimension(dimName);
|
||||
}
|
||||
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
|
||||
}
|
||||
|
||||
async setObsColumnValues(col, rowLabels, value) {
|
||||
const annoMatrix = await this.annoMatrix.setObsColumnValues(
|
||||
col,
|
||||
rowLabels,
|
||||
value
|
||||
);
|
||||
const dimName = _dimensionName("obs", col);
|
||||
let { obsCrossfilter } = this;
|
||||
if (obsCrossfilter.hasDimension(dimName)) {
|
||||
obsCrossfilter = obsCrossfilter.delDimension(dimName);
|
||||
}
|
||||
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
|
||||
}
|
||||
|
||||
async resetObsColumnValues(col, oldValue, newValue) {
|
||||
const annoMatrix = await this.annoMatrix.resetObsColumnValues(
|
||||
col,
|
||||
oldValue,
|
||||
newValue
|
||||
);
|
||||
const dimName = _dimensionName("obs", col);
|
||||
let { obsCrossfilter } = this;
|
||||
if (obsCrossfilter.hasDimension(dimName)) {
|
||||
obsCrossfilter = obsCrossfilter.delDimension(dimName);
|
||||
}
|
||||
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
|
||||
}
|
||||
|
||||
/**
|
||||
Selection state - API is identical to ImmutableTypedCrossfilter, as these
|
||||
are just wrappers to lazy create indices.
|
||||
**/
|
||||
|
||||
async select(field, query, spec) {
|
||||
const { annoMatrix } = this;
|
||||
let { obsCrossfilter } = this;
|
||||
|
||||
if (!annoMatrix?._cache?.[field]) {
|
||||
throw new Error("Unknown field name");
|
||||
}
|
||||
if (field === "var") {
|
||||
throw new Error("unable to obsSelect upon the var dimension");
|
||||
}
|
||||
|
||||
// grab the data, so we can grab the index.
|
||||
const df = await annoMatrix.fetch(field, query);
|
||||
|
||||
const dimName = _dimensionNameFromDf(field, df);
|
||||
if (!obsCrossfilter.hasDimension(dimName)) {
|
||||
// lazy index generation - add dimension when first used
|
||||
obsCrossfilter = this._addObsCrossfilterDimension(
|
||||
annoMatrix,
|
||||
obsCrossfilter,
|
||||
field,
|
||||
df
|
||||
);
|
||||
}
|
||||
|
||||
// select
|
||||
obsCrossfilter = obsCrossfilter.select(dimName, spec);
|
||||
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
|
||||
}
|
||||
|
||||
selectAll() {
|
||||
/*
|
||||
Select all on any dimension in this field.
|
||||
*/
|
||||
const { annoMatrix } = this;
|
||||
const currentDims = this.obsCrossfilter.dimensionNames();
|
||||
const obsCrossfilter = currentDims.reduce((xfltr, dim) => {
|
||||
return xfltr.select(dim, { mode: "all" });
|
||||
}, this.obsCrossfilter);
|
||||
return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
|
||||
}
|
||||
|
||||
countSelected() {
|
||||
/* if no data yet indexed in the crossfilter, just say everything is selected */
|
||||
if (this.obsCrossfilter.size() === 0) return this.annoMatrix.nObs;
|
||||
return this.obsCrossfilter.countSelected();
|
||||
}
|
||||
|
||||
allSelectedMask() {
|
||||
/* if no data yet indexed in the crossfilter, just say everything is selected */
|
||||
if (
|
||||
this.obsCrossfilter.size() === 0 ||
|
||||
this.obsCrossfilter.dimensionNames().length === 0
|
||||
) {
|
||||
/* fake the mask */
|
||||
return new Uint8Array(this.annoMatrix.nObs).fill(1);
|
||||
}
|
||||
return this.obsCrossfilter.allSelectedMask();
|
||||
}
|
||||
|
||||
allSelectedLabels() {
|
||||
/* if no data yet indexed in the crossfilter, just say everything is selected */
|
||||
if (
|
||||
this.obsCrossfilter.size() === 0 ||
|
||||
this.obsCrossfilter.dimensionNames().length === 0
|
||||
) {
|
||||
return this.annoMatrix.rowIndex.labels();
|
||||
}
|
||||
|
||||
const mask = this.obsCrossfilter.allSelectedMask();
|
||||
const index = this.annoMatrix.rowIndex.isubsetMask(mask);
|
||||
return index.labels();
|
||||
}
|
||||
|
||||
fillByIsSelected(array, selectedValue, deselectedValue) {
|
||||
/* if no data yet indexed in the crossfilter, just say everything is selected */
|
||||
if (
|
||||
this.obsCrossfilter.size() === 0 ||
|
||||
this.obsCrossfilter.dimensionNames().length === 0
|
||||
) {
|
||||
return array.fill(selectedValue);
|
||||
}
|
||||
return this.obsCrossfilter.fillByIsSelected(
|
||||
array,
|
||||
selectedValue,
|
||||
deselectedValue
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
** Private below
|
||||
**/
|
||||
|
||||
_addObsCrossfilterDimension(annoMatrix, obsCrossfilter, field, df) {
|
||||
if (field === "var") return obsCrossfilter;
|
||||
const dimName = _dimensionNameFromDf(field, df);
|
||||
const dimParams = this._getObsDimensionParams(field, df);
|
||||
obsCrossfilter = obsCrossfilter.setData(annoMatrix._cache.obs);
|
||||
obsCrossfilter = obsCrossfilter.addDimension(dimName, ...dimParams);
|
||||
return obsCrossfilter;
|
||||
}
|
||||
|
||||
_getColumnBaseType(field, col) {
|
||||
/* Look up the primitive type for this field/col */
|
||||
const colSchema = _getColumnSchema(this.annoMatrix.schema, field, col);
|
||||
return colSchema.type;
|
||||
}
|
||||
|
||||
_getObsDimensionParams(field, df) {
|
||||
/* return the crossfilter dimensiontype type and params for this field/dataframe */
|
||||
|
||||
if (field === "emb") {
|
||||
/* assumed to be 2D */
|
||||
return ["spatial", df.icol(0).asArray(), df.icol(1).asArray()];
|
||||
}
|
||||
|
||||
/* assumed to be 1D */
|
||||
const col = df.icol(0);
|
||||
const colName = df.colIndex.getLabel(0);
|
||||
const type = this._getColumnBaseType(field, colName);
|
||||
if (type === "string" || type === "categorical" || type === "boolean") {
|
||||
return ["enum", col.asArray()];
|
||||
}
|
||||
if (type === "int32") {
|
||||
return ["scalar", col.asArray(), Int32Array];
|
||||
}
|
||||
if (type === "float32") {
|
||||
return ["scalar", col.asArray(), Float32Array];
|
||||
}
|
||||
// Currently not supporting boolean and categorical types.
|
||||
console.error(
|
||||
`Warning - unknown metadata schema (${type}) for field ${field} ${colName}.`
|
||||
);
|
||||
// skip it - we don't know what to do with this type
|
||||
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
export { doBinaryRequest } from "../util/actionHelpers";
|
||||
|
||||
/* double URI encode - needed for query-param filters */
|
||||
export function _dubEncURIComp(s) {
|
||||
return encodeURIComponent(encodeURIComponent(s));
|
||||
}
|
||||
|
||||
/* currently unused, consider deleting */
|
||||
export function _fetchResult(promise) {
|
||||
let _status = "pending";
|
||||
const res = promise.then(
|
||||
(r) => {
|
||||
_status = "success";
|
||||
return r;
|
||||
},
|
||||
(e) => {
|
||||
_status = "error";
|
||||
throw e;
|
||||
}
|
||||
);
|
||||
|
||||
res.status = () => {
|
||||
return _status;
|
||||
};
|
||||
|
||||
return res;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
/*
|
||||
AnnoMatrix -- Annotated Matrix exported interface
|
||||
|
||||
Public API is defined in:
|
||||
|
||||
annoMatrix.js
|
||||
viewCreators.js
|
||||
crossfilter.js
|
||||
|
||||
*/
|
||||
|
||||
export { default as AnnoMatrixLoader } from "./loader";
|
||||
export * from "./viewCreators";
|
||||
export { default as AnnoMatrixObsCrossfilter } from "./crossfilter";
|
||||
export { default as gcMiddleware } from "./middleware";
|
||||
@@ -0,0 +1,287 @@
|
||||
import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
|
||||
import { matrixFBSToDataframe } from "../util/stateManager/matrix";
|
||||
import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
|
||||
import {
|
||||
addObsAnnoColumn,
|
||||
removeObsAnnoColumn,
|
||||
addObsAnnoCategory,
|
||||
removeObsAnnoCategory,
|
||||
} from "../util/stateManager/schemaHelpers";
|
||||
import { isArrayOrTypedArray } from "../util/typeHelpers";
|
||||
import { _whereCacheCreate } from "./whereCache";
|
||||
import AnnoMatrix from "./annoMatrix";
|
||||
import PromiseLimit from "../util/promiseLimit";
|
||||
|
||||
const promiseThrottle = new PromiseLimit(5);
|
||||
|
||||
export default class AnnoMatrixLoader extends AnnoMatrix {
|
||||
/*
|
||||
AnnoMatrix implementation which proxies to HTTP server using the CXG REST API.
|
||||
Used as the base (non-view) instance.
|
||||
|
||||
Public API is same as AnnoMatrix class (refer there for API description),
|
||||
with the addition of the constructor which bootstraps:
|
||||
|
||||
new AnnoMatrixLoader(serverBaseURL, schema) -> instance
|
||||
|
||||
*/
|
||||
constructor(baseURL, schema) {
|
||||
const { nObs, nVar } = schema.dataframe;
|
||||
super(schema, nObs, nVar);
|
||||
|
||||
if (baseURL[baseURL.length - 1] !== "/") {
|
||||
// must have trailing slash
|
||||
baseURL += "/";
|
||||
}
|
||||
this.baseURL = baseURL;
|
||||
Object.seal(this);
|
||||
}
|
||||
|
||||
/**
|
||||
** Public. API described in base class.
|
||||
**/
|
||||
addObsAnnoCategory(col, category) {
|
||||
/*
|
||||
Add a new category (aka label) to the schema for an obs column.
|
||||
*/
|
||||
const colSchema = _getColumnSchema(this.schema, "obs", col);
|
||||
_writableCategoryTypeCheck(colSchema); // throws on error
|
||||
|
||||
const o = this._clone();
|
||||
o.schema = addObsAnnoCategory(this.schema, col, category);
|
||||
return o;
|
||||
}
|
||||
|
||||
async removeObsAnnoCategory(col, category, unassignedCategory) {
|
||||
/*
|
||||
Remove a single "category" (aka "label") from the data & schema of an obs column.
|
||||
*/
|
||||
const colSchema = _getColumnSchema(this.schema, "obs", col);
|
||||
_writableCategoryTypeCheck(colSchema); // throws on error
|
||||
|
||||
const o = await this.resetObsColumnValues(
|
||||
col,
|
||||
category,
|
||||
unassignedCategory
|
||||
);
|
||||
o.schema = removeObsAnnoCategory(o.schema, col, category);
|
||||
return o;
|
||||
}
|
||||
|
||||
dropObsColumn(col) {
|
||||
/*
|
||||
drop column from field
|
||||
*/
|
||||
const colSchema = _getColumnSchema(this.schema, "obs", col);
|
||||
_writableCheck(colSchema); // throws on error
|
||||
|
||||
const o = this._clone();
|
||||
o._cache.obs = this._cache.obs.dropCol(col);
|
||||
o.schema = removeObsAnnoColumn(this.schema, col);
|
||||
return o;
|
||||
}
|
||||
|
||||
addObsColumn(colSchema, Ctor, value) {
|
||||
/*
|
||||
add a column to field, initializing with value. Value may
|
||||
be one of:
|
||||
* an array of values
|
||||
* a primitive type, including null or undefined.
|
||||
If an array, it must be of same size as nObs and same type as Ctor
|
||||
*/
|
||||
colSchema.writable = true;
|
||||
const col = colSchema.name;
|
||||
if (
|
||||
_getColumnSchema(this.schema, "obs", col) ||
|
||||
this._cache.obs.hasCol(col)
|
||||
) {
|
||||
throw new Error("column already exists");
|
||||
}
|
||||
|
||||
const o = this._clone();
|
||||
let data;
|
||||
if (isArrayOrTypedArray(value)) {
|
||||
if (value.constructor !== Ctor)
|
||||
throw new Error("Mismatched value array type");
|
||||
if (value.length !== this.nObs)
|
||||
throw new Error("Value array has incorrect length");
|
||||
data = value.slice();
|
||||
} else {
|
||||
data = new Ctor(this.nObs).fill(value);
|
||||
}
|
||||
o._cache.obs = this._cache.obs.withCol(col, data);
|
||||
o.schema = addObsAnnoColumn(this.schema, col, {
|
||||
...colSchema,
|
||||
writable: true,
|
||||
});
|
||||
return o;
|
||||
}
|
||||
|
||||
renameObsColumn(oldCol, newCol) {
|
||||
/*
|
||||
Rename the obs oldColName to newColName. oldCol must be writable.
|
||||
*/
|
||||
const oldColSchema = _getColumnSchema(this.schema, "obs", oldCol);
|
||||
_writableCheck(oldColSchema); // throws on error
|
||||
|
||||
const value = this._cache.obs.hasCol(oldCol)
|
||||
? this._cache.obs.col(oldCol).asArray()
|
||||
: undefined;
|
||||
return this.dropObsColumn(oldCol).addObsColumn(
|
||||
{
|
||||
...oldColSchema,
|
||||
name: newCol,
|
||||
},
|
||||
value.constructor,
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
async setObsColumnValues(col, rowLabels, value) {
|
||||
/*
|
||||
Set all rows identified by rowLabels to value.
|
||||
*/
|
||||
const colSchema = _getColumnSchema(this.schema, "obs", col);
|
||||
_writableCategoryTypeCheck(colSchema); // throws on error
|
||||
|
||||
// ensure that we have the data in cache before we manipulate it
|
||||
await this.fetch("obs", col);
|
||||
if (!this._cache.obs.hasCol(col))
|
||||
throw new Error("Internal error - user annotation data missing");
|
||||
|
||||
const rowIndices = this.rowIndex.getOffsets(rowLabels);
|
||||
const data = this._cache.obs.col(col).asArray().slice();
|
||||
for (let i = 0, len = rowIndices.length; i < len; i += 1) {
|
||||
const idx = rowIndices[i];
|
||||
if (idx === undefined) throw new Error("Unknown row label");
|
||||
data[idx] = value;
|
||||
}
|
||||
|
||||
const o = this._clone();
|
||||
o._cache.obs = this._cache.obs.replaceColData(col, data);
|
||||
const { categories } = colSchema;
|
||||
if (!categories?.includes(value)) {
|
||||
o.schema = addObsAnnoCategory(this.schema, col, value);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
async resetObsColumnValues(col, oldValue, newValue) {
|
||||
/*
|
||||
Set all rows with value 'oldValue' to 'newValue'.
|
||||
*/
|
||||
const colSchema = _getColumnSchema(this.schema, "obs", col);
|
||||
_writableCategoryTypeCheck(colSchema); // throws on error
|
||||
|
||||
if (!colSchema.categories.includes(oldValue)) {
|
||||
throw new Error("unknown category");
|
||||
}
|
||||
|
||||
// ensure that we have the data in cache before we manipulate it
|
||||
await this.fetch("obs", col);
|
||||
if (!this._cache.obs.hasCol(col))
|
||||
throw new Error("Internal error - user annotation data missing");
|
||||
|
||||
const data = this._cache.obs.col(col).asArray().slice();
|
||||
for (let i = 0, l = data.length; i < l; i += 1) {
|
||||
if (data[i] === oldValue) data[i] = newValue;
|
||||
}
|
||||
|
||||
const o = this._clone();
|
||||
o._cache.obs = this._cache.obs.replaceColData(col, data);
|
||||
const { categories } = colSchema;
|
||||
if (!categories?.includes(newValue)) {
|
||||
o.schema = addObsAnnoCategory(this.schema, col, newValue);
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
/**
|
||||
** Private below
|
||||
**/
|
||||
async _doLoad(field, query) {
|
||||
/*
|
||||
_doLoad - evaluates the query against the field. Returns:
|
||||
* whereCache update: column query map mapping the query to the column labels
|
||||
* Dataframe containing the new colums (one per dimension)
|
||||
*/
|
||||
let urlQuery;
|
||||
let urlBase;
|
||||
let priority = 10; // default fetch priority
|
||||
|
||||
switch (field) {
|
||||
case "obs":
|
||||
case "var": {
|
||||
urlBase = `${this.baseURL}annotations/${field}`;
|
||||
urlQuery = _encodeQuery("annotation-name", query);
|
||||
break;
|
||||
}
|
||||
case "X": {
|
||||
urlBase = `${this.baseURL}data/var`;
|
||||
urlQuery = _encodeQuery(undefined, query);
|
||||
break;
|
||||
}
|
||||
case "emb": {
|
||||
urlBase = `${this.baseURL}layout/obs`;
|
||||
urlQuery = _encodeQuery("layout-name", query);
|
||||
priority = 0; // high prio load for embeddings
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error("Unknown field name");
|
||||
}
|
||||
|
||||
const url = `${urlBase}?${urlQuery}`;
|
||||
const buffer = await promiseThrottle.priorityAdd(
|
||||
priority,
|
||||
doBinaryRequest,
|
||||
url
|
||||
);
|
||||
const result = matrixFBSToDataframe(buffer);
|
||||
if (!result || result.isEmpty()) throw Error("Unknown field/col");
|
||||
|
||||
const whereCacheUpdate = _whereCacheCreate(
|
||||
field,
|
||||
query,
|
||||
result.colIndex.labels()
|
||||
);
|
||||
|
||||
if (field === "obs") {
|
||||
/* cough, cough - see comment on method */
|
||||
_normalizeCategoricalSchema(
|
||||
this.schema.annotations.obsByName[query],
|
||||
result.col(query)
|
||||
);
|
||||
}
|
||||
|
||||
return [whereCacheUpdate, result];
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Utility functions below
|
||||
*/
|
||||
|
||||
function _encodeQuery(colKey, q) {
|
||||
if (typeof q === "object") {
|
||||
const { field: queryField, column: queryColumn, value: queryValue } = q;
|
||||
return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
|
||||
queryColumn
|
||||
)}=${_dubEncURIComp(queryValue)}`;
|
||||
}
|
||||
if (!colKey) throw new Error("Unsupported query by name");
|
||||
return `${colKey}=${encodeURIComponent(q)}`;
|
||||
}
|
||||
|
||||
function _writableCheck(colSchema) {
|
||||
if (!colSchema?.writable) {
|
||||
throw new Error("Unknown or readonly obs column");
|
||||
}
|
||||
}
|
||||
|
||||
function _writableCategoryTypeCheck(colSchema) {
|
||||
_writableCheck(colSchema);
|
||||
if (colSchema.type !== "categorical") {
|
||||
throw new Error("column must be categorical");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
Garbage collection / cache management support
|
||||
|
||||
Middleware that knows how to pull annoMatrix from the undoable state,
|
||||
and pass it along to the AnnoMatrix class for possible cache GC.
|
||||
|
||||
Private interface.
|
||||
|
||||
Future work item: this middleware knows internal details of both the
|
||||
Undoable metareducer and the AnnoMatrix private API. It would be helpful
|
||||
to make the Undoable interface better factored.
|
||||
*/
|
||||
|
||||
const annoMatrixGC = (store) => (next) => (action) => {
|
||||
if (_itIsTimeForGC()) {
|
||||
_doGC(store);
|
||||
}
|
||||
return next(action);
|
||||
};
|
||||
|
||||
let lastGCTime = 0;
|
||||
const InterGCDelayMS = 30 * 1000; // 30 seconds
|
||||
function _itIsTimeForGC() {
|
||||
/*
|
||||
we don't want to run GC on every dispatch, so throttle it a bit.
|
||||
|
||||
Runs every InterGCDelay period
|
||||
*/
|
||||
const now = Date.now();
|
||||
if (now - lastGCTime > InterGCDelayMS) {
|
||||
lastGCTime = now;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function _doGC(store) {
|
||||
const state = store.getState();
|
||||
|
||||
// these should probably be a function imported from undoable.js, etc, as
|
||||
// they have overly intimiate knowledge of our reducers.
|
||||
const undoablePast = state["@@undoable/past"];
|
||||
const undoableFuture = state["@@undoable/future"];
|
||||
const undoableStack = undoablePast
|
||||
.concat(undoableFuture)
|
||||
.flatMap((snapshot) =>
|
||||
snapshot.filter((v) => v[0] === "annoMatrix").map((v) => v[1])
|
||||
);
|
||||
const currentAnnoMatrix = state.annoMatrix;
|
||||
|
||||
/*
|
||||
We want to identify those matrixes currently "hot", ie, linked from the current annoMatrix,
|
||||
as our current gc algo is more aggressive with those not hot.
|
||||
*/
|
||||
const allAnnoMatrices = new Map(
|
||||
undoableStack.map((m) => [m, { isHot: false }])
|
||||
);
|
||||
let am = currentAnnoMatrix;
|
||||
while (am) {
|
||||
allAnnoMatrices.set(am, { isHot: true });
|
||||
am = am.viewOf;
|
||||
}
|
||||
allAnnoMatrices.forEach((hints, annoMatrix) => annoMatrix._gc(hints));
|
||||
}
|
||||
|
||||
export default annoMatrixGC;
|
||||
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
Private helper functions related to schema
|
||||
*/
|
||||
import catLabelSort from "../util/catLabelSort";
|
||||
import { unassignedCategoryLabel } from "../globals";
|
||||
|
||||
export function _getColumnSchema(schema, field, col) {
|
||||
/* look up the column definition */
|
||||
switch (field) {
|
||||
case "obs":
|
||||
if (typeof col === "object")
|
||||
throw new Error("unable to get column schema by query");
|
||||
return schema.annotations.obsByName[col];
|
||||
case "var":
|
||||
if (typeof col === "object")
|
||||
throw new Error("unable to get column schema by query");
|
||||
return schema.annotations.varByName[col];
|
||||
case "emb":
|
||||
if (typeof col === "object")
|
||||
throw new Error("unable to get column schema by query");
|
||||
return schema.layout.obsByName[col];
|
||||
case "X":
|
||||
return schema.dataframe;
|
||||
default:
|
||||
throw new Error(`unknown field name: ${field}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function _getColumnDimensionNames(schema, field, col) {
|
||||
/*
|
||||
field/col may be an alias for multiple columns. Currently used to map ND
|
||||
values to 1D dataframe columns for embeddings/layout. Signfied by the presence
|
||||
of the "dims" value in the schema.
|
||||
*/
|
||||
const colSchema = _getColumnSchema(schema, field, col);
|
||||
if (!colSchema) {
|
||||
return undefined;
|
||||
}
|
||||
return colSchema.dims || [col];
|
||||
}
|
||||
|
||||
export function _schemaColumns(schema, field) {
|
||||
switch (field) {
|
||||
case "obs":
|
||||
return Object.keys(schema.annotations.obsByName);
|
||||
case "var":
|
||||
return Object.keys(schema.annotations.varByName);
|
||||
case "emb":
|
||||
return Object.keys(schema.layout.obsByName);
|
||||
default:
|
||||
throw new Error(`unknown field name: ${field}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function _getWritableColumns(schema, field) {
|
||||
if (field !== "obs") return [];
|
||||
return schema.annotations.obs.columns
|
||||
.filter((v) => v.writable)
|
||||
.map((v) => v.name);
|
||||
}
|
||||
|
||||
export function _isContinuousType(schema) {
|
||||
const { type } = schema;
|
||||
return !(type === "string" || type === "boolean" || type === "categorical");
|
||||
}
|
||||
|
||||
export function _normalizeCategoricalSchema(colSchema, col) {
|
||||
const { type, writable } = colSchema;
|
||||
if (type === "string" || type === "boolean" || type === "categorical") {
|
||||
const categorySet = new Set(
|
||||
col.summarize().categories.concat(colSchema.categories ?? [])
|
||||
);
|
||||
if (writable && !categorySet.has(unassignedCategoryLabel)) {
|
||||
categorySet.add(unassignedCategoryLabel);
|
||||
}
|
||||
colSchema.categories = Array.from(categorySet);
|
||||
}
|
||||
|
||||
if (colSchema.categories) {
|
||||
colSchema.categories = catLabelSort(writable, colSchema.categories);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
View creators. These are helper functions which create new views from existing
|
||||
instances of AnnoMatrix, implementing common UI functions.
|
||||
*/
|
||||
|
||||
import { AnnoMatrixRowSubsetView, AnnoMatrixClipView } from "./views";
|
||||
|
||||
export function isubsetMask(annoMatrix, obsMask) {
|
||||
/*
|
||||
Subset annomatrix to contain the rows which have truish value in the mask.
|
||||
Maks length must equal annoMatrix.nObs (row count).
|
||||
*/
|
||||
return isubset(annoMatrix, _maskToList(obsMask));
|
||||
}
|
||||
|
||||
export function isubset(annoMatrix, obsOffsets) {
|
||||
/*
|
||||
Subset annomatrix to contain the positions contained in the obsOffsets array
|
||||
|
||||
Example:
|
||||
|
||||
isubset(annoMatrix, [0, 1]) -> annoMatrix with only the first two rows
|
||||
*/
|
||||
const obsIndex = annoMatrix.rowIndex.isubset(obsOffsets);
|
||||
return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
|
||||
}
|
||||
|
||||
export function subset(annoMatrix, obsLabels) {
|
||||
/*
|
||||
subset based on labels
|
||||
*/
|
||||
const obsIndex = annoMatrix.rowIndex.subset(obsLabels);
|
||||
return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
|
||||
}
|
||||
|
||||
export function clip(annoMatrix, qmin, qmax) {
|
||||
/*
|
||||
Create a view that clips all continuous data to the [min, max] range.
|
||||
The matrix shape does not change, but the continuous values outside the
|
||||
specified range will become a NaN.
|
||||
*/
|
||||
return new AnnoMatrixClipView(annoMatrix, qmin, qmax);
|
||||
}
|
||||
|
||||
/*
|
||||
Private utility functions below
|
||||
*/
|
||||
|
||||
function _maskToList(mask) {
|
||||
/* convert masks to lists - method wastes space, but is fast */
|
||||
if (!mask) {
|
||||
return null;
|
||||
}
|
||||
const list = new Int32Array(mask.length);
|
||||
let elems = 0;
|
||||
for (let i = 0, l = mask.length; i < l; i += 1) {
|
||||
if (mask[i]) {
|
||||
list[elems] = i;
|
||||
elems += 1;
|
||||
}
|
||||
}
|
||||
return new Int32Array(list.buffer, 0, elems);
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/* eslint-disable max-classes-per-file -- Classes are interrelated*/
|
||||
|
||||
/*
|
||||
Views on the annomatrix. all API here is defined in viewCreators.js and annoMatrix.js.
|
||||
*/
|
||||
import clip from "../util/clip";
|
||||
import AnnoMatrix from "./annoMatrix";
|
||||
import { _whereCacheCreate } from "./whereCache";
|
||||
import { _isContinuousType, _getColumnSchema } from "./schema";
|
||||
|
||||
class AnnoMatrixView extends AnnoMatrix {
|
||||
constructor(viewOf, rowIndex = null) {
|
||||
const nObs = rowIndex ? rowIndex.size() : viewOf.nObs;
|
||||
super(viewOf.schema, nObs, viewOf.nVar, rowIndex || viewOf.rowIndex);
|
||||
this.viewOf = viewOf;
|
||||
this.isView = true;
|
||||
}
|
||||
|
||||
addObsAnnoCategory(col, category) {
|
||||
const o = this._clone();
|
||||
o.viewOf = this.viewOf.addObsAnnoCategory(col, category);
|
||||
o.schema = o.viewOf.schema;
|
||||
return o;
|
||||
}
|
||||
|
||||
async removeObsAnnoCategory(col, category, unassignedCategory) {
|
||||
const o = this._clone();
|
||||
o.viewOf = await this.viewOf.removeObsAnnoCategory(
|
||||
col,
|
||||
category,
|
||||
unassignedCategory
|
||||
);
|
||||
o.schema = o.viewOf.schema;
|
||||
return o;
|
||||
}
|
||||
|
||||
dropObsColumn(col) {
|
||||
const o = this._clone();
|
||||
o.viewOf = this.viewOf.dropObsColumn(col);
|
||||
o._cache.obs = this._cache.obs.dropCol(col);
|
||||
o.schema = o.viewOf.schema;
|
||||
return o;
|
||||
}
|
||||
|
||||
addObsColumn(colSchema, Ctor, value) {
|
||||
const o = this._clone();
|
||||
o.viewOf = this.viewOf.addObsColumn(colSchema, Ctor, value);
|
||||
o.schema = o.viewOf.schema;
|
||||
return o;
|
||||
}
|
||||
|
||||
renameObsColumn(oldCol, newCol) {
|
||||
const o = this._clone();
|
||||
o.viewOf = this.viewOf.renameObsColumn(oldCol, newCol);
|
||||
o.schema = o.viewOf.schema;
|
||||
return o;
|
||||
}
|
||||
|
||||
async setObsColumnValues(col, rowLabels, value) {
|
||||
const o = this._clone();
|
||||
o.viewOf = await this.viewOf.setObsColumnValues(col, rowLabels, value);
|
||||
o._cache.obs = this._cache.obs.dropCol(col);
|
||||
o.schema = o.viewOf.schema;
|
||||
return o;
|
||||
}
|
||||
|
||||
async resetObsColumnValues(col, oldValue, newValue) {
|
||||
const o = this._clone();
|
||||
o.viewOf = await this.viewOf.resetObsColumnValues(col, oldValue, newValue);
|
||||
o._cache.obs = this._cache.obs.dropCol(col);
|
||||
o.schema = o.viewOf.schema;
|
||||
return o;
|
||||
}
|
||||
}
|
||||
|
||||
class AnnoMatrixMapView extends AnnoMatrixView {
|
||||
/*
|
||||
A view which knows how to transform its data.
|
||||
*/
|
||||
constructor(viewOf, mapFn) {
|
||||
super(viewOf);
|
||||
this.mapFn = mapFn;
|
||||
}
|
||||
|
||||
async _doLoad(field, query) {
|
||||
const df = await this.viewOf._fetch(field, query);
|
||||
const dfMapped = df.mapColumns((colData, colIdx) => {
|
||||
const colLabel = df.colIndex.getLabel(colIdx);
|
||||
const colSchema = _getColumnSchema(this.schema, field, colLabel);
|
||||
return this.mapFn(field, colLabel, colSchema, colData, df);
|
||||
});
|
||||
const whereCacheUpdate = _whereCacheCreate(
|
||||
field,
|
||||
query,
|
||||
dfMapped.colIndex.labels()
|
||||
);
|
||||
return [whereCacheUpdate, dfMapped];
|
||||
}
|
||||
}
|
||||
|
||||
export class AnnoMatrixClipView extends AnnoMatrixMapView {
|
||||
/*
|
||||
A view which is a clipped transformation of its parent
|
||||
*/
|
||||
constructor(viewOf, qmin, qmax) {
|
||||
super(viewOf, (field, colLabel, colSchema, colData, df) =>
|
||||
_clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax)
|
||||
);
|
||||
this.isClipped = true;
|
||||
this.clipRange = [qmin, qmax];
|
||||
Object.seal(this);
|
||||
}
|
||||
}
|
||||
|
||||
export class AnnoMatrixRowSubsetView extends AnnoMatrixView {
|
||||
/*
|
||||
A view which is a subset of total rows.
|
||||
*/
|
||||
constructor(viewOf, rowIndex) {
|
||||
super(viewOf, rowIndex);
|
||||
Object.seal(this);
|
||||
}
|
||||
|
||||
async _doLoad(field, query) {
|
||||
const df = await this.viewOf._fetch(field, query);
|
||||
|
||||
// don't try to row-subset the var dimension.
|
||||
if (field === "var") {
|
||||
return [null, df];
|
||||
}
|
||||
|
||||
const dfSubset = df.subset(null, null, this.rowIndex);
|
||||
const whereCacheUpdate = _whereCacheCreate(
|
||||
field,
|
||||
query,
|
||||
dfSubset.colIndex.labels()
|
||||
);
|
||||
return [whereCacheUpdate, dfSubset];
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
Utility functions below
|
||||
*/
|
||||
|
||||
function _clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax) {
|
||||
/* only clip obs and var scalar columns */
|
||||
if (field !== "obs" && field !== "X") return colData;
|
||||
if (!_isContinuousType(colSchema)) return colData;
|
||||
if (qmin < 0) qmin = 0;
|
||||
if (qmax > 1) qmax = 1;
|
||||
if (qmin === 0 && qmax === 1) return colData;
|
||||
|
||||
const quantiles = df.col(colLabel).summarize().percentiles;
|
||||
const lower = quantiles[100 * qmin];
|
||||
const upper = quantiles[100 * qmax];
|
||||
const clippedData = clip(colData.slice(), lower, upper, Number.NaN);
|
||||
return clippedData;
|
||||
}
|
||||
|
||||
/* eslint-enable max-classes-per-file -- enable*/
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
Private support functions.
|
||||
|
||||
Support for a "where" query, eg,
|
||||
|
||||
{ where: { field: "var", column: "gene", value: "FOXP2" }}
|
||||
|
||||
These evaluate to a given column label.
|
||||
|
||||
The "where cache" is a map that saves evaluated queries and points
|
||||
to the column label they resolve to.
|
||||
|
||||
Data structure, using X as the example field being queried, and var as
|
||||
the index.
|
||||
|
||||
{
|
||||
X: {
|
||||
var: Map(
|
||||
column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
|
||||
)
|
||||
}
|
||||
}
|
||||
*/
|
||||
import { _getColumnDimensionNames } from "./schema";
|
||||
|
||||
export function _whereCacheGet(whereCache, schema, field, query) {
|
||||
/*
|
||||
query will either be an where query (object) or a column name (string).
|
||||
|
||||
Return array of column labels or undefined.
|
||||
*/
|
||||
|
||||
if (typeof query === "object") {
|
||||
const { field: queryField, column: queryColumn, value: queryValue } = query;
|
||||
|
||||
const columnMap = whereCache?.[field]?.[queryField];
|
||||
if (columnMap === undefined) return [undefined];
|
||||
|
||||
const valueMap = columnMap.get(queryColumn);
|
||||
if (valueMap === undefined) return [undefined];
|
||||
|
||||
const columnLabels = valueMap.get(queryValue);
|
||||
return columnLabels === undefined ? [undefined] : columnLabels;
|
||||
}
|
||||
|
||||
const colDims = _getColumnDimensionNames(schema, field, query);
|
||||
return colDims === undefined ? [undefined] : colDims;
|
||||
}
|
||||
|
||||
export function _whereCacheCreate(field, query, columnLabels) {
|
||||
/*
|
||||
Create a new whereCache
|
||||
*/
|
||||
if (typeof query !== "object") return null;
|
||||
|
||||
const { field: queryField, column: queryColumn, value: queryValue } = query;
|
||||
const whereCache = {
|
||||
[field]: {
|
||||
[queryField]: new Map([
|
||||
[queryColumn, new Map([[queryValue, columnLabels]])],
|
||||
]),
|
||||
},
|
||||
};
|
||||
return whereCache;
|
||||
}
|
||||
|
||||
function __whereCacheMerge(dst, src) {
|
||||
/*
|
||||
merge src into dst (modifies dst)
|
||||
*/
|
||||
if (!dst) dst = {};
|
||||
if (!src || typeof src !== "object") return dst;
|
||||
Object.entries(src).forEach(([field, query]) => {
|
||||
if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
|
||||
Object.entries(query).forEach(([queryField, columnMap]) => {
|
||||
if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
|
||||
dst[field][queryField] = new Map();
|
||||
columnMap.forEach((valueMap, queryColumn) => {
|
||||
if (!dst[field][queryField].has(queryColumn))
|
||||
dst[field][queryField].set(queryColumn, new Map());
|
||||
valueMap.forEach((columnLabels, queryValue) => {
|
||||
dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
return dst;
|
||||
}
|
||||
|
||||
export function _whereCacheMerge(...caches) {
|
||||
return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
|
||||
}
|
||||
Reference in New Issue
Block a user