mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-01 23:28:12 +08:00
gene set summary progress (#2127)
* revert removal of cache control headers * checkpoint work on revising summary route * add summary query support to annoMatrix * summarize route cleanup * add mising file * clean up summarize route * add summary histogram * update deps * lint * more lint * lint * manage crossfiler during gene set state changes * remove obsolete debugging code * correctly perform async watch in histogram * better error handling
This commit is contained in:
Bruce Martin
@@ -1,27 +1,55 @@
|
||||
/*
|
||||
Private support functions.
|
||||
|
||||
Support for a "where" query, eg,
|
||||
This implements a query resolver cache, mapping a query onto the column labels
|
||||
resolved by that query. These labels are then used to manage the acutal data cache,
|
||||
which stores data by the resolved label.
|
||||
|
||||
{ where: { field: "var", column: "gene", value: "FOXP2" }}
|
||||
There are three query forms:
|
||||
* primitive (string, number) - which is just reference the column label of same value
|
||||
* where query (object) - eg, { where: { field: "var", column: "gene", value: "FOXP2" }}
|
||||
* summary query (object) - eg, { summarize: { method: "mean", field: "var", column: "gene", values: ["FOXP2", "GNE", "F5"]}}
|
||||
|
||||
These evaluate to a given column label.
|
||||
These queries all resolve to one or more column labels on a field. This
|
||||
cache maintains a record of this, allowing direct access to the data caches
|
||||
without a server round-trip.
|
||||
|
||||
The "where cache" is a map that saves evaluated queries and points
|
||||
to the column label they resolve to.
|
||||
The data structure for where queries, the following query against X as an example:
|
||||
{ where: { field: "var", column: "column_label_in_var", value: "value_in_var_column" } }
|
||||
results in the following cached entry:
|
||||
{
|
||||
where: {
|
||||
X: {
|
||||
var: Map(
|
||||
column_label_in_var => Map(
|
||||
value_in_var_column => [column_label_in_X, ...]
|
||||
)
|
||||
)
|
||||
}
|
||||
},
|
||||
summarize: {},
|
||||
}
|
||||
|
||||
Data structure, using X as the example field being queried, and var as
|
||||
the index.
|
||||
|
||||
{
|
||||
X: {
|
||||
var: Map(
|
||||
column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
|
||||
)
|
||||
}
|
||||
And for summarize queries, for the following summary on X:
|
||||
{ summarize: { method: "mean", field: "var", column: "gene", values: ["G1", "G2"]}}
|
||||
creates a cache entry of:
|
||||
{
|
||||
where: {},
|
||||
summarize: {
|
||||
X: {
|
||||
mean: {
|
||||
var: Map(
|
||||
"gene" => Map(
|
||||
"G1,G2" => [summary_column_label, ...]
|
||||
)
|
||||
)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
*/
|
||||
import { _getColumnDimensionNames } from "./schema";
|
||||
import { _hashStringValues } from "./query";
|
||||
|
||||
export function _whereCacheGet(whereCache, schema, field, query) {
|
||||
/*
|
||||
@@ -31,20 +59,30 @@ export function _whereCacheGet(whereCache, schema, field, query) {
|
||||
*/
|
||||
|
||||
if (typeof query === "object") {
|
||||
const { field: queryField, column: queryColumn, value: queryValue } = query;
|
||||
|
||||
const columnMap = whereCache?.[field]?.[queryField];
|
||||
if (columnMap === undefined) return [undefined];
|
||||
|
||||
const valueMap = columnMap.get(queryColumn);
|
||||
if (valueMap === undefined) return [undefined];
|
||||
|
||||
const columnLabels = valueMap.get(queryValue);
|
||||
return columnLabels === undefined ? [undefined] : columnLabels;
|
||||
if (query.where) {
|
||||
const {
|
||||
field: queryField,
|
||||
column: queryColumn,
|
||||
value: queryValue,
|
||||
} = query.where;
|
||||
const columnMap = whereCache?.where?.[field]?.[queryField];
|
||||
return columnMap?.get(queryColumn)?.get(queryValue) ?? [undefined];
|
||||
}
|
||||
if (query.summarize) {
|
||||
const {
|
||||
method,
|
||||
field: queryField,
|
||||
column: queryColumn,
|
||||
values: queryValues,
|
||||
} = query.summarize;
|
||||
const columnMap = whereCache?.summarize?.[field]?.[method]?.[queryField];
|
||||
const queryValueHash = _hashStringValues(queryValues);
|
||||
return columnMap?.get(queryColumn)?.get(queryValueHash) ?? [undefined];
|
||||
}
|
||||
return [undefined];
|
||||
}
|
||||
|
||||
const colDims = _getColumnDimensionNames(schema, field, query);
|
||||
return colDims === undefined ? [undefined] : colDims;
|
||||
return _getColumnDimensionNames(schema, field, query) ?? [undefined];
|
||||
}
|
||||
|
||||
export function _whereCacheCreate(field, query, columnLabels) {
|
||||
@@ -53,40 +91,86 @@ export function _whereCacheCreate(field, query, columnLabels) {
|
||||
*/
|
||||
if (typeof query !== "object") return null;
|
||||
|
||||
const { field: queryField, column: queryColumn, value: queryValue } = query;
|
||||
const whereCache = {
|
||||
[field]: {
|
||||
[queryField]: new Map([
|
||||
[queryColumn, new Map([[queryValue, columnLabels]])],
|
||||
]),
|
||||
},
|
||||
};
|
||||
return whereCache;
|
||||
if (query.where) {
|
||||
const {
|
||||
field: queryField,
|
||||
column: queryColumn,
|
||||
value: queryValue,
|
||||
} = query.where;
|
||||
return {
|
||||
where: {
|
||||
[field]: {
|
||||
[queryField]: new Map([
|
||||
[queryColumn, new Map([[queryValue, columnLabels]])],
|
||||
]),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (query.summarize) {
|
||||
const {
|
||||
method,
|
||||
field: queryField,
|
||||
column: queryColumn,
|
||||
values: queryValues,
|
||||
} = query.summarize;
|
||||
const queryValueHash = _hashStringValues(queryValues);
|
||||
return {
|
||||
summarize: {
|
||||
[field]: {
|
||||
[method]: {
|
||||
[queryField]: new Map([
|
||||
[queryColumn, new Map([[queryValueHash, columnLabels]])],
|
||||
]),
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
// oops, not sure what that query is!
|
||||
return {};
|
||||
}
|
||||
|
||||
function __mergeQueries(dst, src) {
|
||||
for (const [queryField, columnMap] of Object.entries(src)) {
|
||||
dst[queryField] = dst[queryField] || new Map();
|
||||
for (const [queryColumn, valueMap] of columnMap) {
|
||||
if (!dst[queryField].has(queryColumn))
|
||||
dst[queryField].set(queryColumn, new Map());
|
||||
for (const [queryValue, columnLabels] of valueMap) {
|
||||
dst[queryField].get(queryColumn).set(queryValue, columnLabels);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function __whereCacheMerge(dst, src) {
|
||||
/*
|
||||
merge src into dst (modifies dst)
|
||||
*/
|
||||
if (!dst) dst = {};
|
||||
if (!src || typeof src !== "object") return dst;
|
||||
Object.entries(src).forEach(([field, query]) => {
|
||||
if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
|
||||
Object.entries(query).forEach(([queryField, columnMap]) => {
|
||||
if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
|
||||
dst[field][queryField] = new Map();
|
||||
columnMap.forEach((valueMap, queryColumn) => {
|
||||
if (!dst[field][queryField].has(queryColumn))
|
||||
dst[field][queryField].set(queryColumn, new Map());
|
||||
valueMap.forEach((columnLabels, queryValue) => {
|
||||
dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
if (src.where) {
|
||||
dst.where = dst.where || {};
|
||||
for (const [field, query] of Object.entries(src.where)) {
|
||||
dst.where[field] = dst.where[field] || {};
|
||||
__mergeQueries(dst.where[field], query);
|
||||
}
|
||||
}
|
||||
if (src.summarize) {
|
||||
dst.summarize = dst.summarize || {};
|
||||
for (const [field, method] of Object.entries(src.summarize)) {
|
||||
dst.summarize[field] = dst.summarize[field] || {};
|
||||
for (const [methodName, query] of Object.entries(method)) {
|
||||
dst.summarize[field][methodName] =
|
||||
dst.summarize[field][methodName] || {};
|
||||
__mergeQueries(dst.summarize[field][methodName], query);
|
||||
}
|
||||
}
|
||||
}
|
||||
return dst;
|
||||
}
|
||||
|
||||
export function _whereCacheMerge(...caches) {
|
||||
return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
|
||||
return caches.reduce(__whereCacheMerge, {});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user