diff --git a/client/__tests__/e2e/__snapshots__/e2e.test.ts.snap b/client/__tests__/e2e/__snapshots__/e2e.test.ts.snap index bcbca82f..bc3ebbf0 100644 --- a/client/__tests__/e2e/__snapshots__/e2e.test.ts.snap +++ b/client/__tests__/e2e/__snapshots__/e2e.test.ts.snap @@ -2,4 +2,4 @@ exports[`did launch page launched 1`] = `"pbmc3kc3k"`; -exports[`metadata loads categories and values from dataset appear 1`] = `"
louvainvain
tint
"`; +exports[`metadata loads categories and values from dataset appear 1`] = `"
louvainvain
tint
"`; diff --git a/client/__tests__/util/stateManager/controlsHelpers.test.ts b/client/__tests__/util/stateManager/controlsHelpers.test.ts index d4e856ba..1ccb3c0c 100644 --- a/client/__tests__/util/stateManager/controlsHelpers.test.ts +++ b/client/__tests__/util/stateManager/controlsHelpers.test.ts @@ -2,7 +2,7 @@ test controls helpers */ // TODO #2227 test: improve test coverage on control helper functions -// (`topNCategories()`, `isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, ) +// (`isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, ) describe("controls helpers", () => { test("placeholder", () => {}); diff --git a/client/src/actions/index.ts b/client/src/actions/index.ts index c32fa151..60800dae 100644 --- a/client/src/actions/index.ts +++ b/client/src/actions/index.ts @@ -12,6 +12,16 @@ import * as viewActions from "./viewStack"; import * as embActions from "./embedding"; import * as genesetActions from "./geneset"; +function setGlobalConfig(config: any) { + /** + * Set any global run-time config not _exclusively_ managed by the config reducer. + * This should only set fields defined in globals.globalConfig. + */ + globals.globalConfig.maxCategoricalOptionsToDisplay = + config?.parameters?.["max-category-items"] ?? + globals.globalConfig.maxCategoricalOptionsToDisplay; +} + /* return promise fetching user-configured colors */ @@ -33,6 +43,9 @@ async function schemaFetch() { async function configFetch(dispatch: any) { return fetchJson("config").then((response) => { const config = { ...globals.configDefaults, ...response.config }; + + setGlobalConfig(config); + dispatch({ type: "configuration load complete", config, diff --git a/client/src/annoMatrix/loader.ts b/client/src/annoMatrix/loader.ts index b557d37d..02d5e6d0 100644 --- a/client/src/annoMatrix/loader.ts +++ b/client/src/annoMatrix/loader.ts @@ -1,6 +1,6 @@ import { doBinaryRequest, doFetch } from "./fetchHelpers"; import { matrixFBSToDataframe } from "../util/stateManager/matrix"; -import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema"; +import { _getColumnSchema } from "./schema"; import { addObsAnnoColumn, removeObsAnnoColumn, @@ -19,6 +19,10 @@ import { _urlEncodeComplexQuery, _hashStringValues, } from "./query"; +import { + normalizeResponse, + normalizeWritableCategoricalSchema, +} from "./normalize"; const promiseThrottle = new PromiseLimit(5); @@ -141,7 +145,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix { } // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS. newAnnoMatrix._cache.obs = (this as any)._cache.obs.withCol(colName, data); - _normalizeCategoricalSchema( + normalizeWritableCategoricalSchema( colSchema, newAnnoMatrix._cache.obs.col(colName) ); @@ -306,61 +310,12 @@ export default class AnnoMatrixLoader extends AnnoMatrix { result.colIndex.labels() ); - result = _responseTypeNormalization(field, query, this.schema, result); + result = normalizeResponse(field, query, this.schema, result); return [whereCacheUpdate, result]; } } -// @ts-expect-error ts-migrate(7006) -function _responseTypeNormalization(field, query, schema, response) { - /* - Schema-driven type normalization - there are a number of assumptions the front-end - makes about data typing, eg, that the schema will contain all categories in a categorical - column, that booleans are a JS array of true/false, etc. - - The OTA format does not follow precisely the same conventions. This routine implements - the front-end conventions given the schema and an OTA data column. - */ - if (field === "obs" || field === "var") { - response = _booleanCast(field, query, schema, response); - } - if (field === "obs") { - /* - Note: this must be performed after any possible changes to string, boolean or categorical - columns. This routine relies on having access to any casts or other data transformations - made in this routine, above, in order to correctly determine schema updates. - */ - _normalizeCategoricalSchema( - schema.annotations.obsByName[query], - response.col(query) - ); - } - return response; -} - -// @ts-expect-error ts-migrate(7006) -function _booleanCast(field, query, schema, response) { - /* - Boolean columns may be transmitted as an int [0/1] or bool [true/false]. - Force to JS Array of bool. - */ - if (field === "obs" || field === "var") { - // @ts-expect-error ts-migrate(7006) - response = response.mapColumns((colData, colIdx) => { - const colLabel = response.colIndex.getLabel(colIdx); - const colSchema = _getColumnSchema(schema, field, colLabel); - if (colSchema?.type === "boolean") { - const nColData = new Array(colData.length); - for (let i = 0; i < colData.length; i += 1) nColData[i] = !!colData[i]; - return nColData; - } - return colData; - }); - } - return response; -} - /* Utility functions below */ diff --git a/client/src/annoMatrix/normalize.ts b/client/src/annoMatrix/normalize.ts new file mode 100644 index 00000000..36c00526 --- /dev/null +++ b/client/src/annoMatrix/normalize.ts @@ -0,0 +1,157 @@ +import { _getColumnSchema, _isIndex } from "./schema"; +import catLabelSort from "../util/catLabelSort"; +import { + unassignedCategoryLabel, + overflowCategoryLabel, + globalConfig, +} from "../globals"; +import { Dataframe } from "../util/dataframe"; + +// @ts-expect-error ts-migrate(7006) +export function normalizeResponse(field, query, schema, response) { + /** + * There are a number of assumptions in the front-end about data typing and data + * characteristics. This routine will normalize a server response dataframe + * to match front-end expectations and UI conventions. This includes cast/transform + * of the data and schema updates. + * + * This consolidates all assumptions into one location, for ease of update. + * + * Currently, this includes normalization for obs/var columns only: + * + * - Dataframe columns in var/obs that are declared type: boolean may be sent by + * the server in a variety of formats (eg uint8, etc). Cast to JS Array[boolean] + * + * - "Categorical" columns may not have all categories represented in the server-provided + * schema (for valid reasons, eg, floating point rounding differences). For all + * types we treat as categorical in the UI (string, boolean, categorical), update + * the schema to contain all categories as a convenience. + * + * - "Categorical" columns (ie, string, boolean, categorical) may contain an excess + * of category values (aka labels). Consolidate any excess into an "all other" + * category. + */ + + // currently no data or schema normalization necessary for X or emb + if (field !== "obs" && field !== "var") return response; + + const colLabels = response.colIndex.labels(); + for (const colLabel of colLabels) { + const colSchema = _getColumnSchema(schema, field, colLabel); + const isIndex = _isIndex(schema, field, colLabel); + const { type, writable } = colSchema; + + // Boolean data -- cast entire array to Array[bool] + if (type === "boolean") { + response = castColumnToBoolean(response, colLabel); + } + + // Types that are categorical in UI (string, boolean, categorical) OR are writable + // are introspected to ensure the schema `categories` field and data values match, + // and that we do not have an excess of category values (for non-writable columns) + const isEnumType = + type === "boolean" || + type === "string" || + type === "categorical" || + writable; + if (!isIndex && isEnumType) { + response = normalizeCategorical(response, colLabel, colSchema); + } + } + return response; +} + +function castColumnToBoolean(df: Dataframe, label: any): Dataframe { + const colData = df.col(label).asArray(); + const newColData = new Array(colData.length); + for (let i = 0; i < colData.length; i += 1) newColData[i] = !!colData[i]; + df = df.replaceColData(label, newColData); + return df; +} + +export function normalizeWritableCategoricalSchema(colSchema: any, col: any) { + /* + Ensure all enum writable / categorical schema have a categories array, that + the categories array contains all unique values in the data array, AND that + the array is UI sorted. + */ + const categorySet = new Set( + col.summarizeCategorical().categories.concat(colSchema.categories ?? []) + ); + if (!categorySet.has(unassignedCategoryLabel)) { + categorySet.add(unassignedCategoryLabel); + } + colSchema.categories = catLabelSort(true, Array.from(categorySet)); + return colSchema; +} + +export function normalizeCategorical( + df: Dataframe, + colLabel: any, + colSchema: any +) { + /* + If writable, ensure schema matches data and we have an unassigned label + + If not writable, ensure schema matches data and that we consolidate labels in excess + of "top N" into an overflow labels. + */ + const { writable } = colSchema; + const col = df.col(colLabel); + + if (writable) { + // writable (aka user) annotations + normalizeWritableCategoricalSchema(colSchema, col); + return df; + } + + // else read-only, categorical columns + const TopN = globalConfig.maxCategoricalOptionsToDisplay; + + // consolidate all categories from data and schema into a single list + const colDataSummary = col.summarizeCategorical(); + const allCategories = new Set( + colDataSummary.categories.concat(colSchema.categories ?? []) + ); + + // if no overflow, just UI sort schema categories and return + if (allCategories.size <= TopN) { + colSchema.categories = catLabelSort(writable, [...allCategories.keys()]); + return df; + } + + // Otherwise, pick top N categories by count and rewrite data + + // choose unique overflow category label + let overflowCatName = `${colLabel}${overflowCategoryLabel}`; + while (allCategories.has(overflowCatName)) { + overflowCatName += "_"; + } + + // pick top N category labels and add overflow label + const topNCategories = new Set( + [...colDataSummary.categoryCounts.keys()].slice(0, TopN) + ); + topNCategories.add(overflowCatName); + + // rewrite data - consolidate all excess labels into overflow label + const newColData = Array.from(col.asArray()); + for (let i = 0; i < newColData.length; i += 1) { + if (!topNCategories.has(newColData[i])) { + newColData[i] = overflowCatName; + } + } + + // replace data in dataframe + df = df.replaceColData(colLabel, newColData); + + // Update schema with categories, in UI sort order. Ensure overflow label is at end + // of list for display purposes. + const revisedCategories = df.col(colLabel).summarizeCategorical().categories; + revisedCategories.push( + revisedCategories.splice(revisedCategories.indexOf(overflowCatName), 1)[0] + ); + colSchema.categories = catLabelSort(writable, revisedCategories); + + return df; +} diff --git a/client/src/annoMatrix/schema.ts b/client/src/annoMatrix/schema.ts index 5b6b07e8..daacd98b 100644 --- a/client/src/annoMatrix/schema.ts +++ b/client/src/annoMatrix/schema.ts @@ -1,10 +1,6 @@ /* Private helper functions related to schema */ -import catLabelSort from "../util/catLabelSort"; -import { unassignedCategoryLabel } from "../globals"; - -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS. export function _getColumnSchema(schema: any, field: any, col: any) { /* look up the column definition */ switch (field) { @@ -27,7 +23,11 @@ export function _getColumnSchema(schema: any, field: any, col: any) { } } -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS. +export function _isIndex(schema: any, field: any, col: any): bool { + const index = schema.annotations?.[field].index; + return index && index === col; +} + export function _getColumnDimensionNames(schema: any, field: any, col: any) { /* field/col may be an alias for multiple columns. Currently used to map ND @@ -75,36 +75,3 @@ export function _isContinuousType(schema) { const { type } = schema; return !(type === "string" || type === "boolean" || type === "categorical"); } - -// @ts-expect-error ts-migrate(7006) FIXME: Parameter 'colSchema' implicitly has an 'any' type... Remove this comment to see the full error message -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS. -export function _normalizeCategoricalSchema(colSchema, col) { - /* - Ensure all enum schema types have a categories array, that - the categories array contains all unique values in the data - array, AND that the array is sorted. - - Note that the back-end will not always set this hint, so we - must assume it may be incorrect and/or missing. - */ - const { type, writable } = colSchema; - if ( - type === "string" || - type === "boolean" || - type === "categorical" || - writable - ) { - const categorySet = new Set( - col.summarizeCategorical().categories.concat(colSchema.categories ?? []) - ); - if (writable && !categorySet.has(unassignedCategoryLabel)) { - categorySet.add(unassignedCategoryLabel); - } - colSchema.categories = Array.from(categorySet); - } - - if (colSchema.categories) { - colSchema.categories = catLabelSort(writable, colSchema.categories); - } - return colSchema; -} diff --git a/client/src/components/categorical/category/index.tsx b/client/src/components/categorical/category/index.tsx index f3df3f93..5d07d700 100644 --- a/client/src/components/categorical/category/index.tsx +++ b/client/src/components/categorical/category/index.tsx @@ -310,7 +310,6 @@ class Category extends React.PureComponent { // @ts-expect-error ts-migrate(2339) FIXME: Property 'handleCategoryToggleAllClick' does not e... Remove this comment to see the full error message handleCategoryToggleAllClick, } = asyncProps; - const isTruncated = !!categorySummary?.isTruncated; const selectionState = this.getSelectionState(categorySummary); return ( @@ -562,8 +553,6 @@ const CategoryRender = React.memo( checkboxID, // @ts-expect-error ts-migrate(2339) FIXME: Property 'isUserAnno' does not exist on type '{ ch... Remove this comment to see the full error message isUserAnno, - // @ts-expect-error ts-migrate(2339) FIXME: Property 'isTruncated' does not exist on type '{ c... Remove this comment to see the full error message - isTruncated, // @ts-expect-error ts-migrate(2339) FIXME: Property 'isColorAccessor' does not exist on type ... Remove this comment to see the full error message isColorAccessor, // @ts-expect-error ts-migrate(2339) FIXME: Property 'isExpanded' does not exist on type '{ ch... Remove this comment to see the full error message @@ -625,7 +614,6 @@ const CategoryRender = React.memo( metadataField={metadataField} checkboxID={checkboxID} isUserAnno={isUserAnno} - isTruncated={isTruncated} isExpanded={isExpanded} isColorAccessor={isColorAccessor} selectionState={selectionState} @@ -652,11 +640,6 @@ const CategoryRender = React.memo( ) : null } -
- {isExpanded && isTruncated ? ( -

... truncated list ...

- ) : null} -
); } diff --git a/client/src/globals.ts b/client/src/globals.ts index b2ca73c7..d23867f6 100644 --- a/client/src/globals.ts +++ b/client/src/globals.ts @@ -2,14 +2,14 @@ import { Colors } from "@blueprintjs/core"; import { dispatchNetworkErrorMessageToUser } from "./util/actionHelpers"; import ENV_DEFAULT from "../../environment.default.json"; -/* if a categorical metadata field has more options than this, truncate */ -export const maxCategoricalOptionsToDisplay = 200; +/* overflow category values are created using this string */ +export const overflowCategoryLabel = ": all other labels"; /* default "unassigned" value for user-created categorical metadata */ export const unassignedCategoryLabel = "unassigned"; /* -these are default values for configuration the CLI may supply. +these are default values for configuration the CLI may supply. See the REST API and CLI specs for more info. */ export const configDefaults = { @@ -22,6 +22,18 @@ export const configDefaults = { links: {}, }; +/* +Most configuration is stored in the reducer. A handful of values +are global and stored here. They are typically set by the config +action handler, which pull the information from the backend/CLI. + +All should be set here to their default value. +*/ +export const globalConfig = { + /* if a categorical metadata field has more options than this, truncate */ + maxCategoricalOptionsToDisplay: 200, +}; + /* colors */ export const blue = Colors.BLUE3; export const linkBlue = Colors.BLUE5; diff --git a/client/src/util/catLabelSort.ts b/client/src/util/catLabelSort.ts index ff328f70..bcad2912 100644 --- a/client/src/util/catLabelSort.ts +++ b/client/src/util/catLabelSort.ts @@ -18,15 +18,11 @@ function caseInsensitiveCompare(a: any, b: any) { return textA < textB ? -1 : textA > textB ? 1 : 0; } -// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS. -const catLabelSort = (isUserAnno: any, values: any) => { +const catLabelSort = (isUserAnno: boolean, values: any[]): any[] => { /* this sort could be memoized for perf */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS. - const strings: any = []; - // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS. - const ints: any = []; - // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS. + const strings: string[] = []; + const ints: number[] = []; const unassignedOrNaN: any = []; // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS. @@ -43,11 +39,10 @@ const catLabelSort = (isUserAnno: any, values: any) => { }); strings.sort(caseInsensitiveCompare); - // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'a' implicitly has an 'any' type. ints.sort((a, b) => +a - +b); unassignedOrNaN.sort(caseInsensitiveCompare); - return ints.concat(strings, unassignedOrNaN); + return (ints).concat(strings, unassignedOrNaN); }; export default catLabelSort; diff --git a/client/src/util/dataframe/summarize.ts b/client/src/util/dataframe/summarize.ts index 07a7eb9d..f275522a 100644 --- a/client/src/util/dataframe/summarize.ts +++ b/client/src/util/dataframe/summarize.ts @@ -76,10 +76,13 @@ export function summarizeCategorical(col: any) { categoryCounts.set(val, curCount + 1); } } + const sortedCategoryByCounts = new Map( + [...categoryCounts.entries()].sort((a, b) => b[1] - a[1]) + ); return { categorical: true, - categories: [...categoryCounts.keys()], - categoryCounts, - numCategories: categoryCounts.size, + categories: [...sortedCategoryByCounts.keys()], + categoryCounts: sortedCategoryByCounts, + numCategories: sortedCategoryByCounts.size, }; } diff --git a/client/src/util/stateManager/controlsHelpers.ts b/client/src/util/stateManager/controlsHelpers.ts index 20ed7ff1..8891b9f6 100644 --- a/client/src/util/stateManager/controlsHelpers.ts +++ b/client/src/util/stateManager/controlsHelpers.ts @@ -4,8 +4,6 @@ Helper functions for the controls reducer import difference from "lodash.difference"; -import * as globals from "../../globals"; -import { rangeFill as fillRange } from "../range"; import fromEntries from "../fromEntries"; import { isCategoricalAnnotation } from "./annotationsHelpers"; @@ -29,41 +27,9 @@ Remember that option values can be ANY js type, except undefined/null. // number of options numCategoryValues: number, - - // isTruncated - true if the options for selection has - // been truncated (ie, was too large to implement) } } */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS. -function topNCategories(colSchema: any, summary: any, N: any) { - /* return top N categories by occurrences in the data */ - const { categories: allCategories } = colSchema; - const counts = allCategories.map( - // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS. - (cat: any) => summary.categoryCounts.get(cat) ?? 0 - ); - - if (allCategories.length <= N) { - return [allCategories, allCategories, counts]; - } - - const sortIndex = fillRange(new Array(allCategories.length)).sort( - // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS. - (a: any, b: any) => counts[b] - counts[a] - ); - const topNindices = new Set(sortIndex.slice(0, N)); - - const _topNCategories = []; - const topNCounts = []; - for (let i = 0; i < allCategories.length; i += 1) { - if (topNindices.has(i)) { - _topNCategories.push(allCategories[i]); - topNCounts.push(counts[i]); - } - } - return [allCategories, _topNCategories, topNCounts]; -} // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS. export function isSelectableCategoryName(schema: any, name: any) { @@ -93,7 +59,6 @@ export function selectableCategoryNames(schema: any, names: any) { // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS. export function createCategorySummaryFromDfCol(dfCol: any, colSchema: any) { - const N = globals.maxCategoricalOptionsToDisplay; const { writable: isUserAnno } = colSchema; /* @@ -102,24 +67,22 @@ export function createCategorySummaryFromDfCol(dfCol: any, colSchema: any) { if they are not actively used in the current annoMatrix view. */ const summary = dfCol.summarizeCategorical(); - const [ - allCategoryValues, - categoryValues, - categoryValueCounts, - ] = topNCategories(colSchema, summary, N); + const { categories: allCategoryValues } = colSchema; + const categoryValues = allCategoryValues; + const categoryValueCounts = allCategoryValues.map( + (cat: any) => summary.categoryCounts.get(cat) ?? 0 + ); const categoryValueIndices = new Map( // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS. categoryValues.map((v: any, i: any) => [v, i]) ); const numCategoryValues = categoryValueIndices.size; - const isTruncated = categoryValues.length < summary.numCategories; return { allCategoryValues, // array: of natively typed category values (all of them) categoryValues, // array: of natively typed category values (top N only) categoryValueIndices, // map: category value (native type) -> category index (top N only) numCategoryValues, // number: of values in the category (top N) - isTruncated, // bool: true if list was truncated (ie, if topN != all) categoryValueCounts, // array: cardinality of each category, (top N) isUserAnno, // bool };