mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-27 14:38:12 +08:00
Clean up max-category front-end limit (#2347)
* remove topN category truncation from component rendering layer * clean up category item limit implementation * name change for clarity * fix snapshot * comments
This commit is contained in:
Bruce Martin
@@ -1,6 +1,6 @@
|
||||
import { doBinaryRequest, doFetch } from "./fetchHelpers";
|
||||
import { matrixFBSToDataframe } from "../util/stateManager/matrix";
|
||||
import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
|
||||
import { _getColumnSchema } from "./schema";
|
||||
import {
|
||||
addObsAnnoColumn,
|
||||
removeObsAnnoColumn,
|
||||
@@ -19,6 +19,10 @@ import {
|
||||
_urlEncodeComplexQuery,
|
||||
_hashStringValues,
|
||||
} from "./query";
|
||||
import {
|
||||
normalizeResponse,
|
||||
normalizeWritableCategoricalSchema,
|
||||
} from "./normalize";
|
||||
|
||||
const promiseThrottle = new PromiseLimit(5);
|
||||
|
||||
@@ -141,7 +145,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
|
||||
newAnnoMatrix._cache.obs = (this as any)._cache.obs.withCol(colName, data);
|
||||
_normalizeCategoricalSchema(
|
||||
normalizeWritableCategoricalSchema(
|
||||
colSchema,
|
||||
newAnnoMatrix._cache.obs.col(colName)
|
||||
);
|
||||
@@ -306,61 +310,12 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
|
||||
result.colIndex.labels()
|
||||
);
|
||||
|
||||
result = _responseTypeNormalization(field, query, this.schema, result);
|
||||
result = normalizeResponse(field, query, this.schema, result);
|
||||
|
||||
return [whereCacheUpdate, result];
|
||||
}
|
||||
}
|
||||
|
||||
// @ts-expect-error ts-migrate(7006)
|
||||
function _responseTypeNormalization(field, query, schema, response) {
|
||||
/*
|
||||
Schema-driven type normalization - there are a number of assumptions the front-end
|
||||
makes about data typing, eg, that the schema will contain all categories in a categorical
|
||||
column, that booleans are a JS array of true/false, etc.
|
||||
|
||||
The OTA format does not follow precisely the same conventions. This routine implements
|
||||
the front-end conventions given the schema and an OTA data column.
|
||||
*/
|
||||
if (field === "obs" || field === "var") {
|
||||
response = _booleanCast(field, query, schema, response);
|
||||
}
|
||||
if (field === "obs") {
|
||||
/*
|
||||
Note: this must be performed after any possible changes to string, boolean or categorical
|
||||
columns. This routine relies on having access to any casts or other data transformations
|
||||
made in this routine, above, in order to correctly determine schema updates.
|
||||
*/
|
||||
_normalizeCategoricalSchema(
|
||||
schema.annotations.obsByName[query],
|
||||
response.col(query)
|
||||
);
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
// @ts-expect-error ts-migrate(7006)
|
||||
function _booleanCast(field, query, schema, response) {
|
||||
/*
|
||||
Boolean columns may be transmitted as an int [0/1] or bool [true/false].
|
||||
Force to JS Array of bool.
|
||||
*/
|
||||
if (field === "obs" || field === "var") {
|
||||
// @ts-expect-error ts-migrate(7006)
|
||||
response = response.mapColumns((colData, colIdx) => {
|
||||
const colLabel = response.colIndex.getLabel(colIdx);
|
||||
const colSchema = _getColumnSchema(schema, field, colLabel);
|
||||
if (colSchema?.type === "boolean") {
|
||||
const nColData = new Array(colData.length);
|
||||
for (let i = 0; i < colData.length; i += 1) nColData[i] = !!colData[i];
|
||||
return nColData;
|
||||
}
|
||||
return colData;
|
||||
});
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
/*
|
||||
Utility functions below
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import { _getColumnSchema, _isIndex } from "./schema";
|
||||
import catLabelSort from "../util/catLabelSort";
|
||||
import {
|
||||
unassignedCategoryLabel,
|
||||
overflowCategoryLabel,
|
||||
globalConfig,
|
||||
} from "../globals";
|
||||
import { Dataframe } from "../util/dataframe";
|
||||
|
||||
// @ts-expect-error ts-migrate(7006)
|
||||
export function normalizeResponse(field, query, schema, response) {
|
||||
/**
|
||||
* There are a number of assumptions in the front-end about data typing and data
|
||||
* characteristics. This routine will normalize a server response dataframe
|
||||
* to match front-end expectations and UI conventions. This includes cast/transform
|
||||
* of the data and schema updates.
|
||||
*
|
||||
* This consolidates all assumptions into one location, for ease of update.
|
||||
*
|
||||
* Currently, this includes normalization for obs/var columns only:
|
||||
*
|
||||
* - Dataframe columns in var/obs that are declared type: boolean may be sent by
|
||||
* the server in a variety of formats (eg uint8, etc). Cast to JS Array[boolean]
|
||||
*
|
||||
* - "Categorical" columns may not have all categories represented in the server-provided
|
||||
* schema (for valid reasons, eg, floating point rounding differences). For all
|
||||
* types we treat as categorical in the UI (string, boolean, categorical), update
|
||||
* the schema to contain all categories as a convenience.
|
||||
*
|
||||
* - "Categorical" columns (ie, string, boolean, categorical) may contain an excess
|
||||
* of category values (aka labels). Consolidate any excess into an "all other"
|
||||
* category.
|
||||
*/
|
||||
|
||||
// currently no data or schema normalization necessary for X or emb
|
||||
if (field !== "obs" && field !== "var") return response;
|
||||
|
||||
const colLabels = response.colIndex.labels();
|
||||
for (const colLabel of colLabels) {
|
||||
const colSchema = _getColumnSchema(schema, field, colLabel);
|
||||
const isIndex = _isIndex(schema, field, colLabel);
|
||||
const { type, writable } = colSchema;
|
||||
|
||||
// Boolean data -- cast entire array to Array[bool]
|
||||
if (type === "boolean") {
|
||||
response = castColumnToBoolean(response, colLabel);
|
||||
}
|
||||
|
||||
// Types that are categorical in UI (string, boolean, categorical) OR are writable
|
||||
// are introspected to ensure the schema `categories` field and data values match,
|
||||
// and that we do not have an excess of category values (for non-writable columns)
|
||||
const isEnumType =
|
||||
type === "boolean" ||
|
||||
type === "string" ||
|
||||
type === "categorical" ||
|
||||
writable;
|
||||
if (!isIndex && isEnumType) {
|
||||
response = normalizeCategorical(response, colLabel, colSchema);
|
||||
}
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
function castColumnToBoolean(df: Dataframe, label: any): Dataframe {
|
||||
const colData = df.col(label).asArray();
|
||||
const newColData = new Array(colData.length);
|
||||
for (let i = 0; i < colData.length; i += 1) newColData[i] = !!colData[i];
|
||||
df = df.replaceColData(label, newColData);
|
||||
return df;
|
||||
}
|
||||
|
||||
export function normalizeWritableCategoricalSchema(colSchema: any, col: any) {
|
||||
/*
|
||||
Ensure all enum writable / categorical schema have a categories array, that
|
||||
the categories array contains all unique values in the data array, AND that
|
||||
the array is UI sorted.
|
||||
*/
|
||||
const categorySet = new Set(
|
||||
col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
|
||||
);
|
||||
if (!categorySet.has(unassignedCategoryLabel)) {
|
||||
categorySet.add(unassignedCategoryLabel);
|
||||
}
|
||||
colSchema.categories = catLabelSort(true, Array.from(categorySet));
|
||||
return colSchema;
|
||||
}
|
||||
|
||||
export function normalizeCategorical(
|
||||
df: Dataframe,
|
||||
colLabel: any,
|
||||
colSchema: any
|
||||
) {
|
||||
/*
|
||||
If writable, ensure schema matches data and we have an unassigned label
|
||||
|
||||
If not writable, ensure schema matches data and that we consolidate labels in excess
|
||||
of "top N" into an overflow labels.
|
||||
*/
|
||||
const { writable } = colSchema;
|
||||
const col = df.col(colLabel);
|
||||
|
||||
if (writable) {
|
||||
// writable (aka user) annotations
|
||||
normalizeWritableCategoricalSchema(colSchema, col);
|
||||
return df;
|
||||
}
|
||||
|
||||
// else read-only, categorical columns
|
||||
const TopN = globalConfig.maxCategoricalOptionsToDisplay;
|
||||
|
||||
// consolidate all categories from data and schema into a single list
|
||||
const colDataSummary = col.summarizeCategorical();
|
||||
const allCategories = new Set(
|
||||
colDataSummary.categories.concat(colSchema.categories ?? [])
|
||||
);
|
||||
|
||||
// if no overflow, just UI sort schema categories and return
|
||||
if (allCategories.size <= TopN) {
|
||||
colSchema.categories = catLabelSort(writable, [...allCategories.keys()]);
|
||||
return df;
|
||||
}
|
||||
|
||||
// Otherwise, pick top N categories by count and rewrite data
|
||||
|
||||
// choose unique overflow category label
|
||||
let overflowCatName = `${colLabel}${overflowCategoryLabel}`;
|
||||
while (allCategories.has(overflowCatName)) {
|
||||
overflowCatName += "_";
|
||||
}
|
||||
|
||||
// pick top N category labels and add overflow label
|
||||
const topNCategories = new Set(
|
||||
[...colDataSummary.categoryCounts.keys()].slice(0, TopN)
|
||||
);
|
||||
topNCategories.add(overflowCatName);
|
||||
|
||||
// rewrite data - consolidate all excess labels into overflow label
|
||||
const newColData = Array.from(col.asArray());
|
||||
for (let i = 0; i < newColData.length; i += 1) {
|
||||
if (!topNCategories.has(newColData[i])) {
|
||||
newColData[i] = overflowCatName;
|
||||
}
|
||||
}
|
||||
|
||||
// replace data in dataframe
|
||||
df = df.replaceColData(colLabel, newColData);
|
||||
|
||||
// Update schema with categories, in UI sort order. Ensure overflow label is at end
|
||||
// of list for display purposes.
|
||||
const revisedCategories = df.col(colLabel).summarizeCategorical().categories;
|
||||
revisedCategories.push(
|
||||
revisedCategories.splice(revisedCategories.indexOf(overflowCatName), 1)[0]
|
||||
);
|
||||
colSchema.categories = catLabelSort(writable, revisedCategories);
|
||||
|
||||
return df;
|
||||
}
|
||||
@@ -1,10 +1,6 @@
|
||||
/*
|
||||
Private helper functions related to schema
|
||||
*/
|
||||
import catLabelSort from "../util/catLabelSort";
|
||||
import { unassignedCategoryLabel } from "../globals";
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
|
||||
export function _getColumnSchema(schema: any, field: any, col: any) {
|
||||
/* look up the column definition */
|
||||
switch (field) {
|
||||
@@ -27,7 +23,11 @@ export function _getColumnSchema(schema: any, field: any, col: any) {
|
||||
}
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
|
||||
export function _isIndex(schema: any, field: any, col: any): bool {
|
||||
const index = schema.annotations?.[field].index;
|
||||
return index && index === col;
|
||||
}
|
||||
|
||||
export function _getColumnDimensionNames(schema: any, field: any, col: any) {
|
||||
/*
|
||||
field/col may be an alias for multiple columns. Currently used to map ND
|
||||
@@ -75,36 +75,3 @@ export function _isContinuousType(schema) {
|
||||
const { type } = schema;
|
||||
return !(type === "string" || type === "boolean" || type === "categorical");
|
||||
}
|
||||
|
||||
// @ts-expect-error ts-migrate(7006) FIXME: Parameter 'colSchema' implicitly has an 'any' type... Remove this comment to see the full error message
|
||||
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS.
|
||||
export function _normalizeCategoricalSchema(colSchema, col) {
|
||||
/*
|
||||
Ensure all enum schema types have a categories array, that
|
||||
the categories array contains all unique values in the data
|
||||
array, AND that the array is sorted.
|
||||
|
||||
Note that the back-end will not always set this hint, so we
|
||||
must assume it may be incorrect and/or missing.
|
||||
*/
|
||||
const { type, writable } = colSchema;
|
||||
if (
|
||||
type === "string" ||
|
||||
type === "boolean" ||
|
||||
type === "categorical" ||
|
||||
writable
|
||||
) {
|
||||
const categorySet = new Set(
|
||||
col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
|
||||
);
|
||||
if (writable && !categorySet.has(unassignedCategoryLabel)) {
|
||||
categorySet.add(unassignedCategoryLabel);
|
||||
}
|
||||
colSchema.categories = Array.from(categorySet);
|
||||
}
|
||||
|
||||
if (colSchema.categories) {
|
||||
colSchema.categories = catLabelSort(writable, colSchema.categories);
|
||||
}
|
||||
return colSchema;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user