Clean up max-category front-end limit (#2347)

* remove topN category truncation from component rendering layer

* clean up category item limit implementation

* name change for clarity

* fix snapshot

* comments

client/__tests__/e2e/__snapshots__/e2e.test.ts.snap

@@ -2,4 +2,4 @@
 
 exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
-exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;

client/__tests__/util/stateManager/controlsHelpers.test.ts

@@ -2,7 +2,7 @@
 test controls helpers
 */
 // TODO #2227 test: improve test coverage on control helper functions
-// (`topNCategories()`, `isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, )
+// (`isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, )
 
 describe("controls helpers", () => {
   test("placeholder", () => {});

client/src/actions/index.ts

@@ -12,6 +12,16 @@ import * as viewActions from "./viewStack";
 import * as embActions from "./embedding";
 import * as genesetActions from "./geneset";
 
+function setGlobalConfig(config: any) {
+  /**
+   * Set any global run-time config not _exclusively_ managed by the config reducer.
+   * This should only set fields defined in globals.globalConfig.
+   */
+  globals.globalConfig.maxCategoricalOptionsToDisplay =
+    config?.parameters?.["max-category-items"] ??
+    globals.globalConfig.maxCategoricalOptionsToDisplay;
+}
+
 /*
 return promise fetching user-configured colors
 */
@@ -33,6 +43,9 @@ async function schemaFetch() {
 async function configFetch(dispatch: any) {
   return fetchJson("config").then((response) => {
     const config = { ...globals.configDefaults, ...response.config };
+
+    setGlobalConfig(config);
+
     dispatch({
       type: "configuration load complete",
       config,

client/src/annoMatrix/loader.ts

@@ -1,6 +1,6 @@
 import { doBinaryRequest, doFetch } from "./fetchHelpers";
 import { matrixFBSToDataframe } from "../util/stateManager/matrix";
-import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
+import { _getColumnSchema } from "./schema";
 import {
   addObsAnnoColumn,
   removeObsAnnoColumn,
@@ -19,6 +19,10 @@ import {
   _urlEncodeComplexQuery,
   _hashStringValues,
 } from "./query";
+import {
+  normalizeResponse,
+  normalizeWritableCategoricalSchema,
+} from "./normalize";
 
 const promiseThrottle = new PromiseLimit(5);
 
@@ -141,7 +145,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     }
     // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
     newAnnoMatrix._cache.obs = (this as any)._cache.obs.withCol(colName, data);
-    _normalizeCategoricalSchema(
+    normalizeWritableCategoricalSchema(
       colSchema,
       newAnnoMatrix._cache.obs.col(colName)
     );
@@ -306,61 +310,12 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       result.colIndex.labels()
     );
 
-    result = _responseTypeNormalization(field, query, this.schema, result);
+    result = normalizeResponse(field, query, this.schema, result);
 
     return [whereCacheUpdate, result];
   }
 }
 
-// @ts-expect-error ts-migrate(7006)
-function _responseTypeNormalization(field, query, schema, response) {
-  /*
-  Schema-driven type normalization - there are a number of assumptions the front-end
-  makes about data typing, eg, that the schema will contain all categories in a categorical
-  column, that booleans are a JS array of true/false, etc.
-
-  The OTA format does not follow precisely the same conventions. This routine implements
-  the front-end conventions given the schema and an OTA data column.
-  */
-  if (field === "obs" || field === "var") {
-    response = _booleanCast(field, query, schema, response);
-  }
-  if (field === "obs") {
-    /* 
-    Note: this must be performed after any possible changes to string, boolean or categorical 
-    columns. This routine relies on having access to any casts or other data transformations
-    made in this routine, above, in order to correctly determine schema updates.
-    */
-    _normalizeCategoricalSchema(
-      schema.annotations.obsByName[query],
-      response.col(query)
-    );
-  }
-  return response;
-}
-
-// @ts-expect-error ts-migrate(7006)
-function _booleanCast(field, query, schema, response) {
-  /*
-  Boolean columns may be transmitted as an int [0/1] or bool [true/false].
-  Force to JS Array of bool.
-  */
-  if (field === "obs" || field === "var") {
-    // @ts-expect-error ts-migrate(7006)
-    response = response.mapColumns((colData, colIdx) => {
-      const colLabel = response.colIndex.getLabel(colIdx);
-      const colSchema = _getColumnSchema(schema, field, colLabel);
-      if (colSchema?.type === "boolean") {
-        const nColData = new Array(colData.length);
-        for (let i = 0; i < colData.length; i += 1) nColData[i] = !!colData[i];
-        return nColData;
-      }
-      return colData;
-    });
-  }
-  return response;
-}
-
 /*
 Utility functions below
 */

client/src/annoMatrix/normalize.ts

@@ -0,0 +1,157 @@
+import { _getColumnSchema, _isIndex } from "./schema";
+import catLabelSort from "../util/catLabelSort";
+import {
+  unassignedCategoryLabel,
+  overflowCategoryLabel,
+  globalConfig,
+} from "../globals";
+import { Dataframe } from "../util/dataframe";
+
+// @ts-expect-error ts-migrate(7006)
+export function normalizeResponse(field, query, schema, response) {
+  /**
+   * There are a number of assumptions in the front-end about data typing and data
+   * characteristics. This routine will normalize a server response dataframe
+   * to match front-end expectations and UI conventions. This includes cast/transform
+   * of the data and schema updates.
+   *
+   * This consolidates all assumptions into one location, for ease of update.
+   *
+   * Currently, this includes normalization for obs/var columns only:
+   *
+   * - Dataframe columns in var/obs that are declared type: boolean may be sent by
+   *   the server in a variety of formats (eg uint8, etc).  Cast to JS Array[boolean]
+   *
+   * - "Categorical" columns may not have all categories represented in the server-provided
+   *   schema (for valid reasons, eg, floating point rounding differences).  For all
+   *   types we treat as categorical in the UI (string, boolean, categorical), update
+   *   the schema to contain all categories as a convenience.
+   *
+   * - "Categorical" columns (ie, string, boolean, categorical) may contain an excess
+   *   of category values (aka labels).  Consolidate any excess into an "all other"
+   *   category.
+   */
+
+  // currently no data or schema normalization necessary for X or emb
+  if (field !== "obs" && field !== "var") return response;
+
+  const colLabels = response.colIndex.labels();
+  for (const colLabel of colLabels) {
+    const colSchema = _getColumnSchema(schema, field, colLabel);
+    const isIndex = _isIndex(schema, field, colLabel);
+    const { type, writable } = colSchema;
+
+    // Boolean data -- cast entire array to Array[bool]
+    if (type === "boolean") {
+      response = castColumnToBoolean(response, colLabel);
+    }
+
+    // Types that are categorical in UI (string, boolean, categorical) OR are writable
+    // are introspected to ensure the schema `categories` field and data values match,
+    // and that we do not have an excess of category values (for non-writable columns)
+    const isEnumType =
+      type === "boolean" ||
+      type === "string" ||
+      type === "categorical" ||
+      writable;
+    if (!isIndex && isEnumType) {
+      response = normalizeCategorical(response, colLabel, colSchema);
+    }
+  }
+  return response;
+}
+
+function castColumnToBoolean(df: Dataframe, label: any): Dataframe {
+  const colData = df.col(label).asArray();
+  const newColData = new Array(colData.length);
+  for (let i = 0; i < colData.length; i += 1) newColData[i] = !!colData[i];
+  df = df.replaceColData(label, newColData);
+  return df;
+}
+
+export function normalizeWritableCategoricalSchema(colSchema: any, col: any) {
+  /*
+  Ensure all enum writable / categorical schema have a categories array, that
+  the categories array contains all unique values in the data array, AND that 
+  the array is UI sorted.
+  */
+  const categorySet = new Set(
+    col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
+  );
+  if (!categorySet.has(unassignedCategoryLabel)) {
+    categorySet.add(unassignedCategoryLabel);
+  }
+  colSchema.categories = catLabelSort(true, Array.from(categorySet));
+  return colSchema;
+}
+
+export function normalizeCategorical(
+  df: Dataframe,
+  colLabel: any,
+  colSchema: any
+) {
+  /*
+  If writable, ensure schema matches data and we have an unassigned label
+
+  If not writable, ensure schema matches data and that we consolidate labels in excess
+  of "top N" into an overflow labels.
+  */
+  const { writable } = colSchema;
+  const col = df.col(colLabel);
+
+  if (writable) {
+    // writable (aka user) annotations
+    normalizeWritableCategoricalSchema(colSchema, col);
+    return df;
+  }
+
+  // else read-only, categorical columns
+  const TopN = globalConfig.maxCategoricalOptionsToDisplay;
+
+  // consolidate all categories from data and schema into a single list
+  const colDataSummary = col.summarizeCategorical();
+  const allCategories = new Set(
+    colDataSummary.categories.concat(colSchema.categories ?? [])
+  );
+
+  // if no overflow, just UI sort schema categories and return
+  if (allCategories.size <= TopN) {
+    colSchema.categories = catLabelSort(writable, [...allCategories.keys()]);
+    return df;
+  }
+
+  // Otherwise, pick top N categories by count and rewrite data
+
+  // choose unique overflow category label
+  let overflowCatName = `${colLabel}${overflowCategoryLabel}`;
+  while (allCategories.has(overflowCatName)) {
+    overflowCatName += "_";
+  }
+
+  // pick top N category labels and add overflow label
+  const topNCategories = new Set(
+    [...colDataSummary.categoryCounts.keys()].slice(0, TopN)
+  );
+  topNCategories.add(overflowCatName);
+
+  // rewrite data - consolidate all excess labels into overflow label
+  const newColData = Array.from(col.asArray());
+  for (let i = 0; i < newColData.length; i += 1) {
+    if (!topNCategories.has(newColData[i])) {
+      newColData[i] = overflowCatName;
+    }
+  }
+
+  // replace data in dataframe
+  df = df.replaceColData(colLabel, newColData);
+
+  // Update schema with categories, in UI sort order. Ensure overflow label is at end
+  // of list for display purposes.
+  const revisedCategories = df.col(colLabel).summarizeCategorical().categories;
+  revisedCategories.push(
+    revisedCategories.splice(revisedCategories.indexOf(overflowCatName), 1)[0]
+  );
+  colSchema.categories = catLabelSort(writable, revisedCategories);
+
+  return df;
+}

client/src/annoMatrix/schema.ts

@@ -1,10 +1,6 @@
 /*
 Private helper functions related to schema
 */
-import catLabelSort from "../util/catLabelSort";
-import { unassignedCategoryLabel } from "../globals";
-
-// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
 export function _getColumnSchema(schema: any, field: any, col: any) {
   /* look up the column definition */
   switch (field) {
@@ -27,7 +23,11 @@ export function _getColumnSchema(schema: any, field: any, col: any) {
   }
 }
 
-// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
+export function _isIndex(schema: any, field: any, col: any): bool {
+  const index = schema.annotations?.[field].index;
+  return index && index === col;
+}
+
 export function _getColumnDimensionNames(schema: any, field: any, col: any) {
   /*
 		field/col may be an alias for multiple columns. Currently used to map ND 
@@ -75,36 +75,3 @@ export function _isContinuousType(schema) {
   const { type } = schema;
   return !(type === "string" || type === "boolean" || type === "categorical");
 }
-
-// @ts-expect-error ts-migrate(7006) FIXME: Parameter 'colSchema' implicitly has an 'any' type... Remove this comment to see the full error message
-// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS.
-export function _normalizeCategoricalSchema(colSchema, col) {
-  /*
-  Ensure all enum schema types have a categories array, that
-  the categories array contains all unique values in the data
-  array, AND that the array is sorted.
-
-  Note that the back-end will not always set this hint, so we
-  must assume it may be incorrect and/or missing.
-  */
-  const { type, writable } = colSchema;
-  if (
-    type === "string" ||
-    type === "boolean" ||
-    type === "categorical" ||
-    writable
-  ) {
-    const categorySet = new Set(
-      col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
-    );
-    if (writable && !categorySet.has(unassignedCategoryLabel)) {
-      categorySet.add(unassignedCategoryLabel);
-    }
-    colSchema.categories = Array.from(categorySet);
-  }
-
-  if (colSchema.categories) {
-    colSchema.categories = catLabelSort(writable, colSchema.categories);
-  }
-  return colSchema;
-}

client/src/components/categorical/category/index.tsx

@@ -310,7 +310,6 @@ class Category extends React.PureComponent {
                 // @ts-expect-error ts-migrate(2339) FIXME: Property 'handleCategoryToggleAllClick' does not e... Remove this comment to see the full error message
                 handleCategoryToggleAllClick,
               } = asyncProps;
-              const isTruncated = !!categorySummary?.isTruncated;
               const selectionState = this.getSelectionState(categorySummary);
               return (
                 <CategoryRender
@@ -318,7 +317,6 @@ class Category extends React.PureComponent {
                   metadataField={metadataField}
                   checkboxID={checkboxID}
                   isUserAnno={isUserAnno}
-                  isTruncated={isTruncated}
                   isExpanded={isExpanded}
                   isColorAccessor={isColorAccessor}
                   selectionState={selectionState}
@@ -422,8 +420,6 @@ const CategoryHeader = React.memo(
     checkboxID,
     // @ts-expect-error ts-migrate(2339) FIXME: Property 'isUserAnno' does not exist on type '{ ch... Remove this comment to see the full error message
     isUserAnno,
-    // @ts-expect-error ts-migrate(2339) FIXME: Property 'isTruncated' does not exist on type '{ c... Remove this comment to see the full error message
-    isTruncated,
     // @ts-expect-error ts-migrate(2339) FIXME: Property 'isColorAccessor' does not exist on type ... Remove this comment to see the full error message
     isColorAccessor,
     // @ts-expect-error ts-migrate(2339) FIXME: Property 'isExpanded' does not exist on type '{ ch... Remove this comment to see the full error message
@@ -525,11 +521,7 @@ const CategoryHeader = React.memo(
           />
 
           <Tooltip
-            content={
+            content="Use as color scale"
-              isTruncated
-                ? `Coloring by ${metadataField} is disabled, as it exceeds the limit of ${globals.maxCategoricalOptionsToDisplay} labels`
-                : "Use as color scale"
-            }
             position={Position.LEFT}
             usePortal
             hoverOpenDelay={globals.tooltipHoverOpenDelay}
@@ -544,7 +536,6 @@ const CategoryHeader = React.memo(
               onClick={onColorChangeClick}
               active={isColorAccessor}
               intent={isColorAccessor ? "primary" : "none"}
-              disabled={isTruncated}
               icon="tint"
             />
           </Tooltip>
@@ -562,8 +553,6 @@ const CategoryRender = React.memo(
     checkboxID,
     // @ts-expect-error ts-migrate(2339) FIXME: Property 'isUserAnno' does not exist on type '{ ch... Remove this comment to see the full error message
     isUserAnno,
-    // @ts-expect-error ts-migrate(2339) FIXME: Property 'isTruncated' does not exist on type '{ c... Remove this comment to see the full error message
-    isTruncated,
     // @ts-expect-error ts-migrate(2339) FIXME: Property 'isColorAccessor' does not exist on type ... Remove this comment to see the full error message
     isColorAccessor,
     // @ts-expect-error ts-migrate(2339) FIXME: Property 'isExpanded' does not exist on type '{ ch... Remove this comment to see the full error message
@@ -625,7 +614,6 @@ const CategoryRender = React.memo(
             metadataField={metadataField}
             checkboxID={checkboxID}
             isUserAnno={isUserAnno}
-            isTruncated={isTruncated}
             isExpanded={isExpanded}
             isColorAccessor={isColorAccessor}
             selectionState={selectionState}
@@ -652,11 +640,6 @@ const CategoryRender = React.memo(
             ) : null
           }
         </div>
-        <div>
-          {isExpanded && isTruncated ? (
-            <p style={{ paddingLeft: 15 }}>... truncated list ...</p>
-          ) : null}
-        </div>
       </div>
     );
   }

client/src/globals.ts

@@ -2,14 +2,14 @@ import { Colors } from "@blueprintjs/core";
 import { dispatchNetworkErrorMessageToUser } from "./util/actionHelpers";
 import ENV_DEFAULT from "../../environment.default.json";
 
-/* if a categorical metadata field has more options than this, truncate */
+/* overflow category values are created  using this string */
-export const maxCategoricalOptionsToDisplay = 200;
+export const overflowCategoryLabel = ": all other labels";
 
 /* default "unassigned" value for user-created categorical metadata */
 export const unassignedCategoryLabel = "unassigned";
 
 /*
-these are default values for configuration  the CLI may supply.
+these are default values for configuration the CLI may supply.
 See the REST API and CLI specs for more info.
 */
 export const configDefaults = {
@@ -22,6 +22,18 @@ export const configDefaults = {
   links: {},
 };
 
+/*
+Most configuration is stored in the reducer.  A handful of values
+are global and stored here.  They are typically set by the config
+action handler, which pull the information from the backend/CLI.
+
+All should be set here to their default value.
+*/
+export const globalConfig = {
+  /* if a categorical metadata field has more options than this, truncate */
+  maxCategoricalOptionsToDisplay: 200,
+};
+
 /* colors */
 export const blue = Colors.BLUE3;
 export const linkBlue = Colors.BLUE5;

client/src/util/catLabelSort.ts

@@ -18,15 +18,11 @@ function caseInsensitiveCompare(a: any, b: any) {
   return textA < textB ? -1 : textA > textB ? 1 : 0;
 }
 
-// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
+const catLabelSort = (isUserAnno: boolean, values: any[]): any[] => {
-const catLabelSort = (isUserAnno: any, values: any) => {
   /* this sort could be memoized for perf */
 
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  const strings: string[] = [];
-  const strings: any = [];
+  const ints: number[] = [];
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
-  const ints: any = [];
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
   const unassignedOrNaN: any = [];
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
@@ -43,11 +39,10 @@ const catLabelSort = (isUserAnno: any, values: any) => {
   });
 
   strings.sort(caseInsensitiveCompare);
-  // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'a' implicitly has an 'any' type.
   ints.sort((a, b) => +a - +b);
   unassignedOrNaN.sort(caseInsensitiveCompare);
 
-  return ints.concat(strings, unassignedOrNaN);
+  return (<any>ints).concat(strings, unassignedOrNaN);
 };
 
 export default catLabelSort;

client/src/util/dataframe/summarize.ts

@@ -76,10 +76,13 @@ export function summarizeCategorical(col: any) {
       categoryCounts.set(val, curCount + 1);
     }
   }
+  const sortedCategoryByCounts = new Map(
+    [...categoryCounts.entries()].sort((a, b) => b[1] - a[1])
+  );
   return {
     categorical: true,
-    categories: [...categoryCounts.keys()],
+    categories: [...sortedCategoryByCounts.keys()],
-    categoryCounts,
+    categoryCounts: sortedCategoryByCounts,
-    numCategories: categoryCounts.size,
+    numCategories: sortedCategoryByCounts.size,
   };
 }

client/src/util/stateManager/controlsHelpers.ts

@@ -4,8 +4,6 @@ Helper functions for the controls reducer
 
 import difference from "lodash.difference";
 
-import * as globals from "../../globals";
-import { rangeFill as fillRange } from "../range";
 import fromEntries from "../fromEntries";
 import { isCategoricalAnnotation } from "./annotationsHelpers";
 
@@ -29,41 +27,9 @@ Remember that option values can be ANY js type, except undefined/null.
 
       // number of options
       numCategoryValues: number,
-
-      // isTruncated - true if the options for selection has
-      // been truncated (ie, was too large to implement)
     }
   }
 */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
-function topNCategories(colSchema: any, summary: any, N: any) {
-  /* return top N categories by occurrences in the data */
-  const { categories: allCategories } = colSchema;
-  const counts = allCategories.map(
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
-    (cat: any) => summary.categoryCounts.get(cat) ?? 0
-  );
-
-  if (allCategories.length <= N) {
-    return [allCategories, allCategories, counts];
-  }
-
-  const sortIndex = fillRange(new Array(allCategories.length)).sort(
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
-    (a: any, b: any) => counts[b] - counts[a]
-  );
-  const topNindices = new Set(sortIndex.slice(0, N));
-
-  const _topNCategories = [];
-  const topNCounts = [];
-  for (let i = 0; i < allCategories.length; i += 1) {
-    if (topNindices.has(i)) {
-      _topNCategories.push(allCategories[i]);
-      topNCounts.push(counts[i]);
-    }
-  }
-  return [allCategories, _topNCategories, topNCounts];
-}
 
 // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
 export function isSelectableCategoryName(schema: any, name: any) {
@@ -93,7 +59,6 @@ export function selectableCategoryNames(schema: any, names: any) {
 
 // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
 export function createCategorySummaryFromDfCol(dfCol: any, colSchema: any) {
-  const N = globals.maxCategoricalOptionsToDisplay;
   const { writable: isUserAnno } = colSchema;
 
   /*
@@ -102,24 +67,22 @@ export function createCategorySummaryFromDfCol(dfCol: any, colSchema: any) {
   if they are not actively used in the current annoMatrix view.
   */
   const summary = dfCol.summarizeCategorical();
-  const [
+  const { categories: allCategoryValues } = colSchema;
-    allCategoryValues,
+  const categoryValues = allCategoryValues;
-    categoryValues,
+  const categoryValueCounts = allCategoryValues.map(
-    categoryValueCounts,
+    (cat: any) => summary.categoryCounts.get(cat) ?? 0
-  ] = topNCategories(colSchema, summary, N);
+  );
   const categoryValueIndices = new Map(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
     categoryValues.map((v: any, i: any) => [v, i])
   );
   const numCategoryValues = categoryValueIndices.size;
-  const isTruncated = categoryValues.length < summary.numCategories;
 
   return {
     allCategoryValues, // array: of natively typed category values (all of them)
     categoryValues, // array: of natively typed category values (top N only)
     categoryValueIndices, // map: category value (native type) -> category index (top N only)
     numCategoryValues, // number: of values in the category (top N)
-    isTruncated, // bool: true if list was truncated (ie, if topN != all)
     categoryValueCounts, // array: cardinality of each category, (top N)
     isUserAnno, // bool
   };