clean up type inferencing (#2332)

* unit tests for 64 bit conversion

* clean up type handling

* type inference tests

* more type inference fixes

* use schema to determine user intent for data typing

* stop using deprecated API

* fbs type encoding test

* add missing test

* add more tests

* correctly infer X type for CXG adaptor

* lint

* fix typo

* ts migration

* cleanup from PR review

* lint

* PR review changes

client/src/annoMatrix/loader.ts

@@ -272,7 +272,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     }
 
     const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
-    const result = matrixFBSToDataframe(buffer);
+    let result = matrixFBSToDataframe(buffer);
     if (!result || result.isEmpty()) throw Error("Unknown field/col");
 
     const whereCacheUpdate = _whereCacheCreate(
@@ -281,18 +281,61 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       result.colIndex.labels()
     );
 
-    if (field === "obs") {
-      /* cough, cough - see comment on the function called */
-      _normalizeCategoricalSchema(
-        this.schema.annotations.obsByName[query],
-        result.col(query)
-      );
-    }
+    result = _responseTypeNormalization(field, query, this.schema, result);
 
     return [whereCacheUpdate, result];
   }
 }
 
+// @ts-expect-error ts-migrate(7006)
+function _responseTypeNormalization(field, query, schema, response) {
+  /*
+  Schema-driven type normalization - there are a number of assumptions the front-end
+  makes about data typing, eg, that the schema will contain all categories in a categorical
+  column, that booleans are a JS array of true/false, etc.
+
+  The OTA format does not follow precisely the same conventions. This routine implements
+  the front-end conventions given the schema and an OTA data column.
+  */
+  if (field === "obs" || field === "var") {
+    response = _booleanCast(field, query, schema, response);
+  }
+  if (field === "obs") {
+    /* 
+    Note: this must be performed after any possible changes to string, boolean or categorical 
+    columns. This routine relies on having access to any casts or other data transformations
+    made in this routine, above, in order to correctly determine schema updates.
+    */
+    _normalizeCategoricalSchema(
+      schema.annotations.obsByName[query],
+      response.col(query)
+    );
+  }
+  return response;
+}
+
+// @ts-expect-error ts-migrate(7006)
+function _booleanCast(field, query, schema, response) {
+  /*
+  Boolean columns may be transmitted as an int [0/1] or bool [true/false].
+  Force to JS Array of bool.
+  */
+  if (field === "obs" || field === "var") {
+    // @ts-expect-error ts-migrate(7006)
+    response = response.mapColumns((colData, colIdx) => {
+      const colLabel = response.colIndex.getLabel(colIdx);
+      const colSchema = _getColumnSchema(schema, field, colLabel);
+      if (colSchema?.type === "boolean") {
+        const nColData = new Array(colData.length);
+        for (let i = 0; i < colData.length; i += 1) nColData[i] = !!colData[i];
+        return nColData;
+      }
+      return colData;
+    });
+  }
+  return response;
+}
+
 /*
 Utility functions below
 */

client/src/annoMatrix/schema.ts

@@ -73,6 +73,14 @@ export function _isContinuousType(schema) {
 
 // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'colSchema' implicitly has an 'any' type... Remove this comment to see the full error message
 export function _normalizeCategoricalSchema(colSchema, col) {
+  /*
+  Ensure all enum schema types have a categories array, that
+  the categories array contains all unique values in the data
+  array, AND that the array is sorted.
+
+  Note that the back-end will not always set this hint, so we
+  must assume it may be incorrect and/or missing.
+  */
   const { type, writable } = colSchema;
   if (
     type === "string" ||