Merge branch 'main' into colinmegill/#632

Co-authored-by: maniarathi <mani.arathi@gmail.com>

.github/workflows/compatibility_tests.yml

@@ -9,6 +9,7 @@ on:
 
 env:
   JEST_ENV: prod
+  CXG_AUTH_TYPE: none
 
 jobs:
   docker-build:
@@ -28,8 +29,8 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
-        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5]
+        python-version: [3.6, 3.7, 3.8]
+        anndata-version: [0.7.6]
         test-suite: [smoke-test, smoke-test-annotations]
     steps:
     - uses: actions/checkout@v2
@@ -47,8 +48,6 @@ jobs:
         make pydist install-dist
         # 3. install anndata
         pip install anndata==${{ matrix.anndata-version }}
-        # workaround for anndata 0.6.22.post1 bug
-        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
     - name: Tests
       run: make unit-test ${{ matrix.test-suite }}
 

.github/workflows/push_tests.yml

@@ -40,7 +40,7 @@ jobs:
       - name: Lint src with eslint
         working-directory: ./client
         run: |
-          make lint
+          npx eslint src __tests__
 
   unit-test:
     runs-on: ubuntu-latest

.github/workflows/sastisfaction.yml

@@ -0,0 +1,27 @@
+name: Run SASTisfaction
+on:
+  - pull_request
+
+jobs:
+  sastisfaction:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
+        with:
+          repository: chanzuckerberg/sastisfaction
+          ref: main
+          path: .github/actions/sastisfaction
+          ssh-key: ${{ secrets.SASTISFACTION_READ_KEY }}
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker pull
+        run: docker pull ghcr.io/chanzuckerberg/sastisfaction:main
+      - name: Run SASTisfaction
+        uses: ./.github/actions/sastisfaction
+        with:
+          snowflake_private_key: ${{ secrets.SASTISFACTION_RSA_KEY }}

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2020 Chan Zuckerberg Initiative
+Copyright (c) 2017-2021 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

backend/common/compute/diffexp_generic.py

@@ -1,5 +1,6 @@
 import numpy as np
 from scipy import sparse, stats
+from backend.common.constants import XApproximateDistribution
 
 
 def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
@@ -7,7 +8,7 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     Return differential expression statistics for top N variables.
 
     Algorithm:
-    - compute log fold change (log2(meanA/meanB))
+    - compute fold change
     - compute Welch's t-test statistic and pvalue (w/ Bonferroni correction)
     - return top N abs(logfoldchange) where lfc > diffexp_lfc_cutoff
 
@@ -26,21 +27,24 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     :param top_n: number of variables to return stats for
     :param diffexp_lfc_cutoff: minimum
     absolute value returning [ varindex, logfoldchange, pval, pval_adj ] for top N genes
-    :return:  for top N genes, {"positive": for top N genes, [ varindex, logfoldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, logfoldchange, pval, pval_adj ]}
+    :return:  for top N genes, {"positive": for top N genes, [ varindex, foldchange, pval, pval_adj ], "negative": for top N genes, [ varindex, foldchange, pval, pval_adj ]}
     """
 
+    X_approximate_distribution = adaptor.get_X_approximate_distribution()
     dataA = adaptor.get_X_array(maskA, None)
     dataB = adaptor.get_X_array(maskB, None)
 
     # mean, variance, N - calculate for both selections
-    meanA, vA, nA = mean_var_n(dataA)
-    meanB, vB, nB = mean_var_n(dataB)
+    meanA, vA, nA = mean_var_n(dataA, X_approximate_distribution)
+    meanB, vB, nB = mean_var_n(dataB, X_approximate_distribution)
     res = diffexp_ttest_from_mean_var(meanA, vA, nA, meanB, vB, nB, top_n, diffexp_lfc_cutoff)
 
     return res
 
 
 def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp_lfc_cutoff):
+    # IMPORTANT NOTE: this code assumes the data is normally distributed and/or already logged.
+
     n_var = meanA.shape[0]
     top_n = min(top_n, n_var)
 
@@ -64,15 +68,15 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj = pvals * n_var
     pvals_adj[pvals_adj > 1] = 1  # cap adjusted p-value at 1
 
-    # logfoldchanges: log2(meanA / meanB)
-    logfoldchanges = np.log2(np.abs((meanA + 1e-9) / (meanB + 1e-9)))
+    # log fold change. The data is normally distributed/logged, so just subtract the means.
+    logfoldchanges = meanA - meanB
 
     stats_to_sort = tscores
     # find all with lfc > cutoff
     lfc_above_cutoff_idx = np.nonzero(np.abs(logfoldchanges) > diffexp_lfc_cutoff)[0]
 
     # derive sort order
-    if lfc_above_cutoff_idx.shape[0] > top_n*2:
+    if lfc_above_cutoff_idx.shape[0] > top_n * 2:
         # partition top N
         rel_t_partition = np.argpartition(stats_to_sort[lfc_above_cutoff_idx], (top_n, -top_n))
         rel_t_partition_top_n = np.concatenate((rel_t_partition[-top_n:], rel_t_partition[:top_n]))
@@ -95,16 +99,21 @@ def diffexp_ttest_from_mean_var(meanA, varA, nA, meanB, varB, nB, top_n, diffexp
     pvals_adj_top_n = pvals_adj[sort_order]
 
     # varIndex, logfoldchange, pval, pval_adj
-    result = {"positive": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
-                           range(top_n)],
-              "negative": [[sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in
-                           range(-1, -1 - top_n, -1)], }
+    result = {
+        "positive": [
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]] for i in range(top_n)
+        ],
+        "negative": [
+            [sort_order[i], logfoldchanges_top_n[i], pvals_top_n[i], pvals_adj_top_n[i]]
+            for i in range(-1, -1 - top_n, -1)
+        ],
+    }
 
     return result
 
 
 # Convenience function which handles sparse data
-def mean_var_n(X):
+def mean_var_n(X, X_approximate_distribution=XApproximateDistribution.NORMAL):
     """
     Two-pass variance calculation.  Numerically (more) stable
     than naive methods (and same method used by numpy.var())
@@ -122,16 +131,27 @@ def mean_var_n(X):
     with np.errstate(divide="call", invalid="call", call=fp_err_set):
         n = X.shape[0]
         if sparse.issparse(X):
+            if X_approximate_distribution == XApproximateDistribution.COUNT:
+                X = X.log1p()
             mean = X.mean(axis=0).A1
             dfm = X - mean
             sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
             v = sumsq / (n - 1)
         else:
+            if X_approximate_distribution == XApproximateDistribution.COUNT:
+                X = np.log1p(X)
             mean = X.mean(axis=0)
             dfm = X - mean
             sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
             v = sumsq / (n - 1)
 
+    # AnnData does not guarantee that operations on a view of X will
+    # return an ndarray, so force the cast if it wasn't done for us.
+    if type(mean) is not np.ndarray:
+        mean = mean.toarray()
+    if type(v) is not np.ndarray:
+        v = v.toarray()
+
     if fp_err_occurred:
         mean[np.isfinite(mean) == False] = 0  # noqa: E712
         v[np.isfinite(v) == False] = 0  # noqa: E712

backend/common/compute/estimate_distribution.py

@@ -0,0 +1,87 @@
+import numba
+import concurrent.futures
+import numpy as np
+from scipy import sparse
+from backend.common.constants import XApproximateDistribution
+
+
+@numba.njit(error_model="numpy", nogil=True)
+def min_max(arr: np.ndarray):
+    """Return (min, max) values for the ndarray."""
+
+    # initialize to first finite value in array.  Normally,
+    # this will exit on the first value.
+    for i in range(arr.size):
+        min_val = max_val = arr[i]
+        if np.isfinite(min_val):
+            break
+
+    # now find min/max, unrolled by two
+    odd = arr.size % 2
+    unrolled_loop_limit = arr.size - 1 if odd else arr.size
+    i = 0
+    while i < unrolled_loop_limit:
+        x = arr[i]
+        y = arr[i + 1]
+
+        # ignore non-finites
+        x = x if np.isfinite(x) else min_val
+        y = y if np.isfinite(y) else min_val
+
+        if x > y:
+            x, y = y, x
+        min_val = min(x, min_val)
+        max_val = max(y, max_val)
+        i += 2
+
+    # handle the tail if any
+    if odd:
+        x = arr[arr.size - 1]
+
+        # ignore non-finites
+        x = x if np.isfinite(x) else min_val
+
+        min_val = min(x, min_val)
+        max_val = max(x, max_val)
+
+    return min_val, max_val
+
+
+def estimate_approximate_distribution(X) -> XApproximateDistribution:
+    """
+    Estimate the distribution (normal, count) of the X matrix.
+
+    Currently this is based upon the assumption that scRNA-seq data is
+    exponentially distributed in its raw (count) form, and when logged,
+    any (max-min) range in excess of 24 is implies tens of millions of
+    observations of a single feature and so is extremely unlikely.
+    """
+    if X.dtype.kind not in ["i", "u", "f"]:
+        raise TypeError(f"Unsupported matrix dtype: {X.dtype.name}")
+
+    if X.size == 0:
+        # default for empty array
+        return XApproximateDistribution.NORMAL
+
+    if sparse.isspmatrix_csc(X) or sparse.isspmatrix_csr(X):
+        Xdata = X.data
+    elif type(X) is np.ndarray:
+        Xdata = X.reshape(
+            X.size,
+        )
+    else:
+        raise TypeError(f"Unsupported matrix format: {str(type(X))}")
+
+    CHUNKSIZE = 1 << 24
+    if Xdata.size > CHUNKSIZE:
+        min_val = max_val = Xdata[0]
+        with concurrent.futures.ThreadPoolExecutor() as tp:
+            for (_min, _max) in tp.map(min_max, [Xdata[i : i + CHUNKSIZE] for i in range(0, Xdata.size, CHUNKSIZE)]):
+                min_val = min(_min, min_val)
+                max_val = max(_max, max_val)
+
+    else:
+        min_val, max_val = min_max(Xdata)
+
+    excess_range = (max_val - min_val) > 24
+    return XApproximateDistribution.COUNT if excess_range else XApproximateDistribution.NORMAL

backend/common/constants.py

@@ -24,6 +24,11 @@ class DiffExpMode(AugmentedEnum):
     VAR_FILTER = "varFilter"
 
 
+class XApproximateDistribution(AugmentedEnum):
+    NORMAL = "normal"
+    COUNT = "count"
+
+
 JSON_NaN_to_num_warning_msg = "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
 REACTIVE_LIMIT = 1_000_000
 

backend/common/errors.py

@@ -51,7 +51,6 @@ define_request_exception(
     default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
 )
 
-define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
 define_exception("ConfigurationError", "Raised when checking configuration errors")
 define_exception("PrepareError", "Raised when data is misprepared")
 define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")

backend/common/fbs/matrix.py

@@ -5,6 +5,8 @@ import pandas as pd
 from flatbuffers import Builder
 from scipy import sparse
 
+from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array
+
 import backend.common.fbs.NetEncoding.Column as Column
 import backend.common.fbs.NetEncoding.Float32Array as Float32Array
 import backend.common.fbs.NetEncoding.Float64Array as Float64Array
@@ -14,6 +16,7 @@ import backend.common.fbs.NetEncoding.Matrix as Matrix
 import backend.common.fbs.NetEncoding.TypedArray as TypedArray
 import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array
 
+
 # Serialization helper
 def serialize_column(builder, typed_arr):
     """ Serialize NetEncoding.Column """
@@ -84,7 +87,7 @@ def serialize_typed_array(builder, source_array, encoding_info):
 def column_encoding(arr):
     column_encoding_type_map = {
         # array protocol string:  ( array_type, as_type )
-        np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64),
+        np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
         np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
         np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
         np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
@@ -98,7 +101,8 @@ def column_encoding(arr):
     }
     column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
 
-    return column_encoding_type_map.get(arr.dtype.str, column_encoding_default)
+    encoding_dtype = np.dtype(get_encoding_dtype_of_array(arr))
+    return column_encoding_type_map.get(encoding_dtype.str, column_encoding_default)
 
 
 def index_encoding(arr):
@@ -198,7 +202,7 @@ def deserialize_typed_array(tarr):
     arr.Init(u.Bytes, u.Pos)
     narr = arr.DataAsNumpy()
     if u_type == TypedArray.TypedArray.JSONEncodedArray:
-        narr = json.loads(narr.tostring().decode("utf-8"))
+        narr = json.loads(narr.tobytes().decode("utf-8"))
     return narr
 
 

backend/common/utils/type_conversion_utils.py

@@ -1,8 +1,42 @@
+from typing import Union, Tuple
 import logging
 
 import numpy as np
 import pandas as pd
 
+"""
+These routines drive all type inference for the schema generation and the
+FBS (REST OTA) encoding. They are also used for CXG generation.
+
+
+H5AD Type                       REST              REST
+(ndarray, Series, Index)        FBS encoding      schema type       ERROR/exceptions
+----------------------------    --------------    ---------------   ----------------------
+bool_/bool                      uint8             boolean
+(u)int8, (u)int16, int32        int32             int32
+uint32, (u)int64                int32             int32             CHECKS value bounds
+float16, float32, float64       float32           float32[0]
+
+categorical[T is numeric[4]]:
+    hasna = False               T                 categorical[1]
+    hasna = True                float32           categorical[1]    CHECKS value bounds
+
+categorical[T not numeric]      JSON/str          categorical[1,2]
+
+(other object)                  JSON/str          string
+
+(all other)                                                         Always an ERROR[3]
+
+
+Notes:
+[0] IEEE format, includes non-finite numbers (NaN, Inf, ...)
+[1] with NO categories enumerated (client side does it to handle rounding)
+[2] NA (undefined) categories are assigned a JSON null value
+[3] Includes all other numpy types:  datetime, complex, etc.
+[4] means float, int, uint (dtype.kind in ['i','u','f'])
+
+"""
+
 
 def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
     dtypes_by_column_name = {}
@@ -17,133 +51,132 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
     return dtypes_by_column_name, schema_type_hints_by_column_name
 
 
-def get_dtype_of_array(array: pd.Series):
-    return get_dtype_and_schema_of_array(array)[0]
+def get_encoding_dtype_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> np.dtype:
+    return _get_type_info(array)[0]
 
 
-def get_schema_type_hint_of_array(array: pd.Series):
-    return get_dtype_and_schema_of_array(array)[1]
+def get_schema_type_hint_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> dict:
+    return _get_type_info(array)[1]
 
 
-def get_dtype_and_schema_of_array(array: pd.Series):
-    return (
-        get_dtype_from_dtype(array.dtype, array_values=array),
-        get_schema_type_hint_from_dtype(array.dtype, array_values=array),
-    )
+def get_dtype_and_schema_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """Return tuple (encoding_dtype, schema_type_hint)"""
+    return _get_type_info(array)
 
 
-def get_dtype_from_dtype(dtype, array_values=None):
+def get_schema_type_hint_from_dtype(dtype) -> dict:
+    res = _get_type_info_from_dtype(dtype)
+    if res is None:
+        raise TypeError(f"Annotations of type {dtype} are unsupported.")
+    else:
+        return res[1]
+
+
+def _get_type_info_from_dtype(dtype) -> Union[Tuple[np.dtype, dict], None]:
     """
-    Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
-    for 64 bit values which will get downcast to 32 bit.
+    Best-effort to determine encoding type and schema hint from a dtype.
+    If this is not possible, or the type is unsupported, return None.
+
+    This should be a subset of the cases which are supported by
+    _get_type_info().  The latter should be preferred if the array (values)
+    are available for typing.
     """
+    if dtype.kind == "b":
+        return (np.uint8, {"type": "boolean"})
 
-    dtype_name = dtype.name
-    dtype_kind = dtype.kind
-
-    if dtype_name == "bool":
-        return np.uint8
-    if dtype_name == "object" and dtype_kind == "O":
-        return str
-    if dtype_name == "category":
-        return get_dtype_from_dtype(dtype.categories.dtype, array_values)
-
-    if can_cast_to_int32(dtype, array_values):
-        return np.int32
-    if can_cast_to_float32(dtype, array_values):
-        return np.float32
-    if not can_cast_to_float32(dtype, array_values):
-        return np.float64
-
-    raise TypeError(f"Annotations of type {dtype} are unsupported.")
-
-
-def get_schema_type_hint_from_dtype(dtype, array_values=None):
-    """
-    Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
-    and will be downcast to 32 bit.
-    """
-
-    dtype_name = dtype.name
-    dtype_kind = dtype.kind
-
-    if dtype == np.float32 or dtype == np.int32:
-        return {"type": dtype_name}
-    if dtype_name == "bool":
-        return {"type": "boolean"}
-    if dtype_name == "object" and dtype_kind == "O":
-        return {"type": "string"}
-    if dtype_name == "category":
-        return {"type": "categorical", "categories": dtype.categories.tolist()}
-
-    if can_cast_to_int32(dtype, array_values):
-        return {"type": "int32"}
-    if can_cast_to_float32(dtype, array_values):
-        return {"type": "float32"}
-    if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
-        return {"type": "float64"}
-
-    raise TypeError(f"Annotations of type {dtype} are unsupported.")
-
-
-def can_cast_to_float32(dtype, array_values):
-    """
-    Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
-    a float.
-
-    We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
-    Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
-    """
-
-    if dtype.kind == "f":
-        if not np.can_cast(dtype, np.float32):
-            logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
-
-        return True
-
-    if dtype.kind == "O" and array_values.hasnans:
-        return True
-
-    return False
-
-
-def can_cast_to_int32(dtype, array_values=None):
-    """
-    A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
-    the higher precision type has values that are entirely within the range of the downcast type.
-    """
-
-    # Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
-    # return False.
-    if array_values.hasnans:
-        return False
-
-    # If the array is categorical, then we need to order the array values so that functions min and max that occur
-    # later, can function. They do not function on unordered categories.
-    ordered_array_values = array_values
-    if array_values.dtype.name == "category" and not array_values.cat.ordered:
-        ordered_array_values = array_values.cat.as_ordered()
+    if dtype.kind == "U":
+        return (np.dtype(str), {"type": "string"})
 
     if dtype.kind in ["i", "u"]:
         if np.can_cast(dtype, np.int32):
-            return True
-        ii32 = np.iinfo(np.int32)
-        if (
-            not ordered_array_values.empty
-            and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
-            or ordered_array_values.empty
-        ):
-            return True
+            return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(dtype)
+        return (np.float32, {"type": "float32"})
+
+    if dtype.kind == "O" and not dtype.name == "category":
+        return (np.dtype(str), {"type": "string"})
+
+    return None
+
+
+def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
+    """
+    Determine encoding type and schema hint from an array.  This allows more
+    flexible casting than may be possible by using just the dtype, as it can
+    account for category types and array values.
+    """
+    if (
+        not isinstance(array, np.ndarray)
+        and not isinstance(array, pd.Series)
+        and not isinstance(array, pd.Index)
+        and not hasattr(array, "dtype")
+    ):
+        raise TypeError("Unsupported data type.")
+
+    dtype = array.dtype
+
+    res = _get_type_info_from_dtype(dtype)
+    if res is not None:
+        return res
+
+    if dtype.kind == "O":
+        if dtype.name == "category":
+            # Sometimes CategoricalDType can be encoded as int or float without further fuss.
+            # Do not specify the categories in the schema - let the client-side figure it out
+            # on its own.  Utilize Series.to_numpy() to do casting that handles categorical
+            # NA/NaN (missing or undefined) categories.
+            if dtype.categories.dtype.kind in ["f", "i", "u"]:
+                return (
+                    _get_type_info(array.to_numpy())[0],
+                    {"type": "categorical"},
+                )
+            else:
+                return (np.dtype(str), {"type": "categorical", "categories": dtype.categories.to_list()})
+
+        # all other extension types are str-encoded
+        return (np.dtype(str), {"type": "string"})
+
+    if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
+        return (np.int32, {"type": "int32"})
+
+    if dtype.kind == "f":
+        _float64_warning(array.dtype)
+        return (np.float32, {"type": "float32"})
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def _float64_warning(dtype):
+    """
+    Warn the user if we are down-casting a float64 to float32, and may potentially lose information.
+    """
+    if dtype.kind == "f" and not np.can_cast(dtype, np.float32):
+        logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
+
+
+def _can_cast_array_values_to_int32(array: Union[np.ndarray, pd.Series, pd.Index]) -> bool:
+    """
+    Return true if the (U)INT array values can be safely cast to int32.  We allow size reducing
+    casts (ie, int64 to int32) if no actual values require the larger size (ie, actual values
+    can be represented by the smaller type).
+    """
+    assert array.dtype.kind in ["u", "i"]
+
+    if np.can_cast(array.dtype, np.int32):
+        return True
+
+    if array.size == 0:
+        return True
+
+    int32_machine_limits = np.iinfo(np.int32)
+    if array.min() >= int32_machine_limits.min and array.max() <= int32_machine_limits.max:
+        return True
+
     return False
 
 
-def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
-    if series_to_convert.hasnans and dtype == np.int32:
-        logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
-
-    return series_to_convert.to_numpy(dtype)
-
-
 def convert_string_to_value(value: str):
     """convert a string to value with the most appropriate type"""
     if value.lower() == "true":

backend/czi_hosted/app/app.py

@@ -312,11 +312,6 @@ class LayoutObsAPI(DatasetResource):
     def get(self, data_adaptor):
         return common_rest.layout_obs_get(request, data_adaptor)
 
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.layout_obs_put(request, data_adaptor)
-
 
 class GenesetsAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)

backend/czi_hosted/cli/launch.py

@@ -44,20 +44,6 @@ def annotation_args(func):
         help="Directory of where to save output annotations; filename will be specified in the application. "
         "Incompatible with --annotations-file.",
     )
-    @click.option(
-        "--experimental-annotations-ontology",
-        is_flag=True,
-        default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__enable,
-        show_default=True,
-        help="When creating annotations, optionally autocomplete names from ontology terms.",
-    )
-    @click.option(
-        "--experimental-annotations-ontology-obo",
-        default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__obo_location,
-        show_default=True,
-        metavar="<path or url>",
-        help="Location of OBO file defining cell annotation autosuggest terms.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -104,14 +90,6 @@ def config_args(func):
         metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
     )
-    @click.option(
-        "--experimental-enable-reembedding",
-        is_flag=True,
-        default=DEFAULT_CONFIG.default_dataset_config.embeddings__enable_reembedding,
-        show_default=False,
-        hidden=True,
-        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -328,9 +306,6 @@ def launch(
     annotations_dir,
     backed,
     disable_diffexp,
-    experimental_annotations_ontology,
-    experimental_annotations_ontology_obo,
-    experimental_enable_reembedding,
     config_file,
     dump_default_config,
 ):
@@ -389,12 +364,9 @@ def launch(
             user_annotations__enable=not disable_annotations,
             user_annotations__local_file_csv__file=annotations_file,
             user_annotations__local_file_csv__directory=annotations_dir,
-            user_annotations__ontology__enable=experimental_annotations_ontology,
-            user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
             presentation__max_categories=max_category_items,
             presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
-            embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
         )

backend/czi_hosted/common/annotations/annotations.py

@@ -1,10 +1,8 @@
-import fastobo
-import fsspec
 import os
 
 from flask import current_app, has_request_context
 
-from backend.common.errors import OntologyLoadFailure, DisabledFeatureError
+from backend.common.errors import DisabledFeatureError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
 from backend.common.genesets import write_gene_sets_tidycsv, read_gene_sets_tidycsv, validate_gene_sets
 from backend.common.utils.data_locator import DataLocator
@@ -12,14 +10,9 @@ from backend.common.utils.utils import path_join
 
 
 class Annotations:
-    """ baseclass for annotations, including ontologies and genesets """
-
-    """ our default ontology is the PURL for the Cell Ontology.
-    See http://www.obofoundry.org/ontology/cl.html """
-    DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
+    """baseclass for annotations and genesets"""
 
     def __init__(self, config={}):
-        self.ontology_data = None
         self.config = config
 
     def user_annotations_enabled(self):
@@ -29,27 +22,6 @@ class Annotations:
         if not self.user_annotations_enabled():
             raise DisabledFeatureError("User annotations are disabled.")
 
-    def load_ontology(self, path):
-        """Load and parse ontologies - currently support OBO files only."""
-        if path is None:
-            path = self.DefaultOnotology
-
-        try:
-            with fsspec.open(path) as f:
-                obo = fastobo.iter(f)
-                terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
-                names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
-                self.ontology_data = names
-
-        except FileNotFoundError as e:
-            raise OntologyLoadFailure("Unable to find OBO ontology path") from e
-
-        except SyntaxError as e:
-            raise OntologyLoadFailure(f"{path}:{e.lineno}:{e.offset} OBO syntax error, unable to read ontology") from e
-
-        except Exception as e:
-            raise OntologyLoadFailure(f"{path}:Error loading OBO file") from e
-
     def get_schema(self, data_adaptor):
         schema = []
         labels = self.read_labels(data_adaptor)
@@ -126,7 +98,7 @@ class Annotations:
 
 
 def dataset_uri_to_geneset_uri(data_uri_or_path):
-    """ given a dataset URI, return the associated gene set URI """
+    """given a dataset URI, return the associated gene set URI"""
     data_basename = os.path.basename(data_uri_or_path)
     base, ext = os.path.splitext(data_basename)
     if ext is not None:  # strip extension, if any

backend/czi_hosted/common/annotations/hosted_tiledb.py

@@ -10,7 +10,7 @@ from flask import current_app
 from backend.czi_hosted.common.annotations.annotations import Annotations
 from backend.common.errors import AnnotationCategoryNameError
 from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
-from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
+from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_encoding_dtype_of_array
 from backend.czi_hosted.db.cellxgene_orm import Annotation
 
 
@@ -143,7 +143,7 @@ class AnnotationsHostedTileDB(Annotations):
             # convert to tiledb datatypes
 
             for col in df:
-                df[col] = df[col].astype(get_dtype_of_array(df[col]))
+                df[col] = df[col].astype(get_encoding_dtype_of_array(df[col]))
             tiledb.from_pandas(uri, df, sparse=True)
         else:
             uri = ""
@@ -164,10 +164,4 @@ class AnnotationsHostedTileDB(Annotations):
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = False
 
-        if self.ontology_data:
-            params["annotations_cell_ontology_enabled"] = True
-            params["annotations_cell_ontology_terms"] = self.ontology_data
-        else:
-            params["annotations_cell_ontology_enabled"] = False
-
         parameters.update(params)

backend/czi_hosted/common/annotations/local_file_csv.py

@@ -115,7 +115,7 @@ class AnnotationsLocalFile(Annotations):
         return os.getcwd()
 
     def _get_filename(self, data_adaptor):
-        """ return the current annotation file name """
+        """return the current annotation file name"""
         if self.output_file:
             return self.output_file
 
@@ -175,12 +175,6 @@ class AnnotationsLocalFile(Annotations):
         params["annotations"] = True
         params["user_annotation_collection_name_enabled"] = True
 
-        if self.ontology_data:
-            params["annotations_cell_ontology_enabled"] = True
-            params["annotations_cell_ontology_terms"] = self.ontology_data
-        else:
-            params["annotations_cell_ontology_enabled"] = False
-
         if self.output_file is not None:
             # user has hard-wired the name of the annotation data collection
             fname = os.path.basename(self.output_file)

backend/czi_hosted/common/config/client_config.py

@@ -40,16 +40,12 @@ def get_client_config(app_config, data_adaptor):
         "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
         "backed": server_config.adaptor__anndata_adaptor__backed,
         "disable-diffexp": not dataset_config.diffexp__enable,
-        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,
         "annotations_genesets": True,  # feature flag
         "annotations_genesets_readonly": True,
         "annotations_genesets_summary_methods": ["mean"],
-        "annotations_cell_ontology_enabled": False,
-        "annotations_cell_ontology_obopath": None,
-        "annotations_cell_ontology_terms": None,
         "custom_colors": dataset_config.presentation__custom_colors,
         "diffexp-may-be-slow": False,
         "about_legal_tos": dataset_config.app__about_legal_tos,

backend/czi_hosted/common/config/dataset_config.py

@@ -5,9 +5,7 @@ from backend.czi_hosted.common.annotations.annotations import Annotations
 from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
 from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.czi_hosted.common.config.base_config import BaseConfig
-from backend.common.errors import ConfigurationError, OntologyLoadFailure
-from backend.czi_hosted.compute.scanpy import get_scanpy_module
-from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from backend.common.errors import ConfigurationError
 from backend.czi_hosted.db.db_utils import DbUtils
 
 
@@ -33,10 +31,6 @@ class DatasetConfig(BaseConfig):
                 "directory"
             ]
             self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
-            self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
-            self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
-                "obo_location"
-            ]
             self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
                 "hosted_tiledb_array"
             ]["db_uri"]
@@ -45,17 +39,18 @@ class DatasetConfig(BaseConfig):
             ]["hosted_file_directory"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
-            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
 
             self.diffexp__enable = default_config["diffexp"]["enable"]
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
             self.diffexp__top_n = default_config["diffexp"]["top_n"]
 
+            self.X_approximate_distribution = default_config["X_approximate_distribution"]
+
         except KeyError as e:
             raise ConfigurationError(f"Unexpected config: {str(e)}")
 
         # Create the default annotation, which supports gene set reading without
-        # further configuration. Depending on configuration options, `complete_config` 
+        # further configuration. Depending on configuration options, `complete_config`
         # may create a more specialized annotation object and replace this default.
         self.user_annotations = Annotations()
 
@@ -65,6 +60,7 @@ class DatasetConfig(BaseConfig):
         self.handle_user_annotations(context)
         self.handle_embeddings()
         self.handle_diffexp(context)
+        self.handle_X_approximate_distribution()
 
     def handle_app(self):
         self.validate_correct_type_of_configuration_attribute("app__scripts", list)
@@ -101,10 +97,6 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__local_file_csv__file", (type(None), str)
         )
-        self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
-        self.validate_correct_type_of_configuration_attribute(
-            "user_annotations__ontology__obo_location", (type(None), str)
-        )
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
         )
@@ -125,11 +117,6 @@ class DatasetConfig(BaseConfig):
                 self.handle_hosted_tiledb_annotations()
             else:
                 raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
-            if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
-                try:
-                    self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
-                except OntologyLoadFailure as e:
-                    raise ConfigurationError("Unable to load ontology terms\n" + str(e))
         else:
             self.check_annotation_config_vars_not_set(context)
 
@@ -197,33 +184,8 @@ class DatasetConfig(BaseConfig):
                     "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
                 )
 
-        if self.user_annotations__ontology__enable:
-            context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
-        if self.user_annotations__ontology__obo_location is not None:
-            context["messagefn"](
-                "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
-            )
-
     def handle_embeddings(self):
         self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                matrix_data_loader = MatrixDataLoader(
-                    server_config.single_dataset__datapath, app_config=self.app_config
-                )
-                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
-                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                get_scanpy_module()
-            except NotImplementedError:
-                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
 
     def handle_diffexp(self, context):
         self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
@@ -240,3 +202,10 @@ class DatasetConfig(BaseConfig):
                         "CAUTION: due to the size of your dataset, "
                         "running differential expression may take longer or fail."
                     )
+
+    def handle_X_approximate_distribution(self):
+        self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
+        if self.X_approximate_distribution not in ["normal", "count"]:
+            raise ConfigurationError(
+                "X_approximate_distribution has unknown value -- must be 'normal' or 'count'."
+            )

backend/czi_hosted/common/rest.py

@@ -265,7 +265,9 @@ def diffexp_obs_post(request, data_adaptor):
 
         set1_filter = args.get("set1", {"filter": {}})["filter"]
         set2_filter = args.get("set2", {"filter": {}})["filter"]
-        count = args.get("count", None)
+        # TODO(#1281): When we simplify the config, we should actually use the config to determine this number,
+        #  this will also require an update in the client
+        count = 15
 
         if set1_filter is None or set2_filter is None or count is None:
             return abort_and_log(HTTPStatus.BAD_REQUEST, "missing required parameter")
@@ -312,25 +314,6 @@ def layout_obs_get(request, data_adaptor):
         )
 
 
-def layout_obs_put(request, data_adaptor):
-    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    args = request.get_json()
-    filter = args["filter"] if args else None
-    if not filter:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
-    method = args["method"] if args else "umap"
-
-    try:
-        schema = data_adaptor.compute_embedding(method, filter)
-        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
-    except NotImplementedError as e:
-        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
-    except (ValueError, DisabledFeatureError, FilterError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
 def genesets_get(request, data_adaptor):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
     if preferred_mimetype not in ("application/json", "text/csv"):

backend/czi_hosted/common/utils/cxg_generation_utils.py

@@ -3,7 +3,7 @@ import json
 import numpy as np
 import tiledb
 
-from backend.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
+from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array, get_dtype_and_schema_of_array
 
 
 def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
@@ -47,7 +47,7 @@ def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, ind
             ]
         )
         attrs = [
-            tiledb.Attr(name=column, dtype=get_dtype_of_array(dataframe[column]), filters=tiledb_filter)
+            tiledb.Attr(name=column, dtype=get_encoding_dtype_of_array(dataframe[column]), filters=tiledb_filter)
             for column in dataframe
         ]
         domain = tiledb.Domain(

backend/czi_hosted/compute/scanpy.py

@@ -1,53 +0,0 @@
-import importlib
-import numpy as np
-
-"""
-Wrapper for various scanpy modules.  Will raise NotImplementedError if the scanpy
-module is not installed/available
-"""
-
-
-def get_scanpy_module():
-    try:
-        sc = importlib.import_module("scanpy")
-        # Future: we could enforce versions here, eg, lookat sc.__version__
-        return sc
-    except ModuleNotFoundError as e:
-        raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
-    except Exception as e:
-        # will capture other ImportError corner cases
-        raise NotImplementedError() from e
-
-
-def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
-    """
-    Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
-    as an ndarray of shape (len(obs_mask), N), where N>=2.
-
-    Do NOT mutate adata.
-    """
-
-    # backed mode is incompatible with the current implementation
-    if adata.isbacked:
-        raise NotImplementedError("Backed mode is incompatible with re-embedding")
-
-    # safely get scanpy module, which may not be present.
-    sc = get_scanpy_module()
-
-    # https://github.com/theislab/anndata/issues/311
-    obs_mask = slice(None) if obs_mask is None else obs_mask
-    adata = adata[obs_mask, :].copy()
-
-    for k in list(adata.obsm.keys()):
-        del adata.obsm[k]
-    for k in list(adata.uns.keys()):
-        del adata.uns[k]
-
-    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
-    sc.pp.neighbors(adata, **neighbors_options)
-    sc.tl.umap(adata, **umap_options)
-
-    umap = adata.obsm["X_umap"]
-    result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
-    result[obs_mask] = umap
-    return result

backend/czi_hosted/data_anndata/anndata_adaptor.py

@@ -1,20 +1,17 @@
 import warnings
-from datetime import datetime
 
 import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
-from server_timing import Timing as ServerTiming
 
 import backend.common.compute.diffexp_generic as diffexp_generic
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from backend.common.constants import Axis, MAX_LAYOUTS
+from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
 from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError, ConfigurationError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.czi_hosted.compute.scanpy import scanpy_umap
 from backend.czi_hosted.data_common.data_adaptor import DataAdaptor
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -31,6 +28,7 @@ class AnndataAdaptor(DataAdaptor):
     def __init__(self, data_locator, app_config=None, dataset_config=None):
         super().__init__(data_locator, app_config, dataset_config)
         self.data = None
+        self.X_approximate_distribution = None
         self._load_data(data_locator)
         self._validate_and_initialize()
 
@@ -68,11 +66,11 @@ class AnndataAdaptor(DataAdaptor):
 
     @staticmethod
     def _create_unique_column_name(df, col_name_prefix):
-        """ given the columns of a dataframe, and a name prefix, return a column name which
-            does not exist in the dataframe, AND which is prefixed by `prefix`
+        """given the columns of a dataframe, and a name prefix, return a column name which
+        does not exist in the dataframe, AND which is prefixed by `prefix`
 
-            The approach is to append a numeric suffix, starting at zero and increasing by
-            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
+        The approach is to append a numeric suffix, starting at zero and increasing by
+        one, until an unused name is found (eg, prefix_0, prefix_1, ...).
         """
         suffix = 0
         while f"{col_name_prefix}{suffix}" in df:
@@ -126,7 +124,11 @@ class AnndataAdaptor(DataAdaptor):
 
     def _create_schema(self):
         self.schema = {
-            "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
+            "dataframe": {
+                "nObs": self.cell_count,
+                "nVar": self.gene_count,
+                **get_schema_type_hint_of_array(self.data.X),
+            },
             "annotations": {
                 "obs": {"index": self.parameters.get("obs_names"), "columns": []},
                 "var": {"index": self.parameters.get("var_names"), "columns": []},
@@ -193,16 +195,20 @@ class AnndataAdaptor(DataAdaptor):
         self.gene_count = self.data.shape[1]
         self._create_schema()
 
+        if self.dataset_config.X_approximate_distribution == "auto":
+            raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
+        self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
+
         # heuristic
         n_values = self.data.shape[0] * self.data.shape[1]
         if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
             self.parameters.update({"diffexp_may_be_slow": True})
 
     def _is_valid_layout(self, arr):
-        """ return True if this layout data is a valid array for front-end presentation:
-            * ndarray, dtype float/int/uint
-            * with shape (n_obs, >= 2)
-            * with all values finite or NaN (no +Inf or -Inf)
+        """return True if this layout data is a valid array for front-end presentation:
+        * ndarray, dtype float/int/uint
+        * with shape (n_obs, >= 2)
+        * with all values finite or NaN (no +Inf or -Inf)
         """
         is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
         is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
@@ -301,28 +307,6 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
-    def compute_embedding(self, method, obsFilter):
-        if Axis.VAR in obsFilter:
-            raise FilterError("Observation filters may not contain variable conditions")
-        if method != "umap":
-            raise NotImplementedError(f"re-embedding method {method} is not available.")
-        try:
-            shape = self.get_shape()
-            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        with ServerTiming.time("layout.compute"):
-            X_umap = scanpy_umap(self.data, obs_mask)
-
-        # Server picks reemedding name, which must not collide with any other
-        # embedding name generated by this backend.
-        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
-        dims = [f"{name}_0", f"{name}_1"]
-        layout_schema = {"name": name, "type": "float32", "dims": dims}
-        self.schema["layout"]["obs"].append(layout_schema)
-        self.data.obsm[f"X_{name}"] = X_umap
-        return layout_schema
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n
@@ -334,13 +318,22 @@ class AnndataAdaptor(DataAdaptor):
         return convert_anndata_category_colors_to_cxg_category_colors(self.data)
 
     def get_X_array(self, obs_mask=None, var_mask=None):
+        # H5Py does not support boolean indexing (masks), so convert to integer indexing
+        # when backed (ie, when AnnData is using H5Py indexing)
         if obs_mask is None:
             obs_mask = slice(None)
+        elif self.data.isbacked and obs_mask.dtype == bool:
+            obs_mask = obs_mask.nonzero()[0]
         if var_mask is None:
             var_mask = slice(None)
+        elif self.data.isbacked and var_mask.dtype == bool:
+            var_mask = var_mask.nonzero()[0]
         X = self.data.X[obs_mask, var_mask]
         return X
 
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        return self.X_approximate_distribution
+
     def get_shape(self):
         return self.data.shape
 

backend/czi_hosted/data_common/data_adaptor.py

@@ -7,8 +7,14 @@ from scipy import sparse
 from server_timing import Timing as ServerTiming
 
 from backend.czi_hosted.common.config.app_config import AppConfig
-from backend.common.constants import Axis
-from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod, DatasetAccessError
+from backend.common.constants import Axis, XApproximateDistribution
+from backend.common.errors import (
+    FilterError,
+    JSONEncodingValueError,
+    ExceedsLimitError,
+    UnsupportedSummaryMethod,
+    DatasetAccessError,
+)
 from backend.common.utils.utils import jsonify_numpy
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -71,17 +77,17 @@ class DataAdaptor(metaclass=ABCMeta):
         """return an numpy array for the given pre-computed embedding name."""
         pass
 
-    @abstractmethod
-    def compute_embedding(self, method, filter):
-        """compute a new embedding on the specified obs subset, and return the embedding schema. """
-        pass
-
     @abstractmethod
     def get_X_array(self, obs_mask=None, var_mask=None):
         """return the X array, possibly filtered by obs_mask or var_mask.
         the return type is either ndarray or scipy.sparse.spmatrix."""
         pass
 
+    @abstractmethod
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        """return the approximate distribution of the X matrix."""
+        pass
+
     @abstractmethod
     def get_shape(self):
         pass
@@ -163,7 +169,7 @@ class DataAdaptor(metaclass=ABCMeta):
         mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0]: i[1]] = True
+                mask[i[0] : i[1]] = True
             else:
                 mask[i] = True
         return mask
@@ -321,12 +327,13 @@ class DataAdaptor(metaclass=ABCMeta):
             top_n = self.dataset_config.diffexp__top_n
 
         if self.server_config.exceeds_limit(
-                "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 
         result = self.compute_diffexp_ttest(
-            maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff)
+            maskA=obs_mask_A, maskB=obs_mask_B, top_n=top_n, lfc_cutoff=self.dataset_config.diffexp__lfc_cutoff
+        )
 
         try:
             return jsonify_numpy(result)

backend/czi_hosted/data_cxg/cxg_adaptor.py

@@ -8,7 +8,7 @@ import pandas as pd
 import tiledb
 from server_timing import Timing as ServerTiming
 
-from backend.common.constants import Axis
+from backend.common.constants import Axis, XApproximateDistribution
 from backend.common.errors import DatasetAccessError, ConfigurationError
 from backend.czi_hosted.common.immutable_kvcache import ImmutableKVCache
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
@@ -37,6 +37,7 @@ class CxgAdaptor(DataAdaptor):
         self.lsuri_results = ImmutableKVCache(lambda key: self._lsuri(uri=key, tiledb_ctx=self.tiledb_ctx))
         self.arrays = ImmutableKVCache(lambda key: self._open_array(uri=key, tiledb_ctx=self.tiledb_ctx))
         self.schema = None
+        self.X_approximate_distribution = None
 
         self._validate_and_initialize()
 
@@ -175,6 +176,10 @@ class CxgAdaptor(DataAdaptor):
         if cxg_version not in ["0.0", "0.1", "0.2.0"]:
             raise DatasetAccessError(f"cxg matrix is not valid: {self.url}")
 
+        if self.dataset_config.X_approximate_distribution == "auto":
+            raise ConfigurationError("X-approximate-distribution 'auto' mode unsupported.")
+        self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
+
         self.title = title
         self.about = about
         self.cxg_version = cxg_version
@@ -199,16 +204,14 @@ class CxgAdaptor(DataAdaptor):
         array = self.open_array(f"emb/{ename}")
         return array[:, 0:dims]
 
-    def compute_embedding(self, method, filter):
-        raise NotImplementedError("CXG does not yet support re-embedding")
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n
         if lfc_cutoff is None:
             lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
         return diffexp_cxg.diffexp_ttest(
-            adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff)
+            adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff
+        )
 
     def get_colors(self):
         if self.cxg_version == "0.0":
@@ -284,6 +287,9 @@ class CxgAdaptor(DataAdaptor):
                 data = X.multi_index[obs_items, var_items][""]
             return data
 
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        return self.X_approximate_distribution
+
     def get_shape(self):
         X = self.open_array("X")
         return X.shape
@@ -352,7 +358,7 @@ class CxgAdaptor(DataAdaptor):
         shape = self.get_shape()
         dtype = self.get_X_array_dtype()
 
-        dataframe = {"nObs": shape[0], "nVar": shape[1], "type": dtype.name}
+        dataframe = {"nObs": shape[0], "nVar": shape[1], **get_schema_type_hint_from_dtype(dtype)}
 
         annotations = {}
         for ax in ("obs", "var"):

backend/czi_hosted/default_config.py

@@ -194,19 +194,17 @@ dataset:
     local_file_csv:
       directory: null
       file: null
-    ontology:
-      enable: false
-      obo_location: null
 
   embeddings:
     names : []
-    enable_reembedding: false
 
   diffexp:
     enable: true
     lfc_cutoff: 0.01
     top_n: 10
 
+  X_approximate_distribution: normal # currently fixed config
+
 external:
   # You can retrieve configuration parameters from this config file, the environment,
   # the AWS secrets manager, or from the "cellxgene launch" command line arguments.

backend/czi_hosted/eb/app.py

@@ -164,7 +164,6 @@ try:
         app_config.update_server_config(multi_dataset__dataroot=dataroot)
 
     # overwrite configuration for the eb app
-    app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
     app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
 
     # complete config

backend/czi_hosted/requirements-dev.txt

@@ -8,4 +8,5 @@ pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
 -r requirements.txt
+-r requirements-prepare.txt
 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability

backend/czi_hosted/requirements-prepare.txt

@@ -1,2 +1,4 @@
 python-igraph
 louvain>=0.6
+scanpy
+umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/czi_hosted/requirements.txt

@@ -1,10 +1,9 @@
-anndata>=0.7.0
+anndata>=0.7.6 # we need to_memory(), added in 0.7.6
 boto3>=1.12.18
 click>=7.1.2
-fastobo>=0.6.1
 Flask>=1.0.2,<2.0.0 # Flask 2.0 is not compatible with the latest version of Flask-RESTful (0.3.8)
 Flask-Compress>=1.4.0
-Flask-Cors>=3.0.6
+Flask-Cors>=3.0.9  # CVE-2020-25032
 Flask-RESTful>=0.3.6
 flask-server-timing>=0.1.2
 flask-talisman>=0.7.0
@@ -12,16 +11,14 @@ flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires mi
 flatten-dict>=0.2.0
 fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
-h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5
-numba>=0.49.1,<0.53.0
-numpy>=1.15.0
+h5py>=3.0.0
+numba>=0.51.2
+numpy>=1.17.5
 packaging>=20.0
 pandas>=1.0,!=1.1  # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
-PyYAML>=5.3
-scipy>=1.0
+PyYAML>=5.4  # CVE-2020-14343
+scipy>=1.4
 requests>=2.22.0
 tiledb>=0.5.9,>=0.6.2,!=0.7.2, !=0.8.6
 s3fs==0.4.2
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
 sqlalchemy>=1.3.18
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/app/app.py

@@ -187,11 +187,6 @@ class LayoutObsAPI(Resource):
     def get(self, data_adaptor):
         return common_rest.layout_obs_get(request, data_adaptor)
 
-    @cache_control(no_store=True)
-    @rest_get_data_adaptor
-    def put(self, data_adaptor):
-        return common_rest.layout_obs_put(request, data_adaptor)
-
 
 class GenesetsAPI(Resource):
     @cache_control(public=True, max_age=ONE_WEEK)

backend/server/cli/launch.py

@@ -44,20 +44,6 @@ def annotation_args(func):
         help="Directory of where to save output annotations; filename will be specified in the application. "
         "Incompatible with --annotations-file and --gene-sets-file.",
     )
-    @click.option(
-        "--experimental-annotations-ontology",
-        is_flag=True,
-        default=DEFAULT_CONFIG.dataset_config.user_annotations__ontology__enable,
-        show_default=True,
-        help="When creating annotations, optionally autocomplete names from ontology terms.",
-    )
-    @click.option(
-        "--experimental-annotations-ontology-obo",
-        default=DEFAULT_CONFIG.dataset_config.user_annotations__ontology__obo_location,
-        show_default=True,
-        metavar="<path or url>",
-        help="Location of OBO file defining cell annotation autosuggest terms.",
-    )
     @click.option(
         "--disable-gene-sets-save",
         is_flag=True,
@@ -121,14 +107,6 @@ def config_args(func):
         metavar="<text>",
         help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
     )
-    @click.option(
-        "--experimental-enable-reembedding",
-        is_flag=True,
-        default=DEFAULT_CONFIG.dataset_config.embeddings__enable_reembedding,
-        show_default=False,
-        hidden=True,
-        help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
-    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -172,6 +150,14 @@ def dataset_args(func):
         metavar="<URL>",
         help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
     )
+    @click.option(
+        "--X-approximate-distribution",
+        default=DEFAULT_CONFIG.dataset_config.X_approximate_distribution,
+        show_default=True,
+        type=click.Choice(["auto", "normal", "count"], case_sensitive=False),
+        help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic "
+        "to determine the approximate distribution.  Mode 'auto' is incompatible with --backed.",
+    )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
         return func(*args, **kwargs)
@@ -338,11 +324,9 @@ def launch(
     disable_gene_sets_save,
     backed,
     disable_diffexp,
-    experimental_annotations_ontology,
-    experimental_annotations_ontology_obo,
-    experimental_enable_reembedding,
     config_file,
     dump_default_config,
+    x_approximate_distribution,
 ):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.
@@ -396,14 +380,12 @@ def launch(
             user_annotations__local_file_csv__directory=user_generated_data_dir,
             user_annotations__local_file_csv__gene_sets_file=gene_sets_file,
             user_annotations__gene_sets__readonly=disable_gene_sets_save,
-            user_annotations__ontology__enable=experimental_annotations_ontology,
-            user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
             presentation__max_categories=max_category_items,
             presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
-            embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,
             diffexp__lfc_cutoff=diffexp_lfc_cutoff,
+            X_approximate_distribution=x_approximate_distribution,
         )
 
         diff = cli_config.server_config.changes_from_default()

backend/server/common/annotations/annotations.py

@@ -1,22 +1,14 @@
 from abc import ABCMeta, abstractmethod
 
-import fastobo
-import fsspec
-
-from backend.common.errors import OntologyLoadFailure, DisabledFeatureError
+from backend.common.errors import DisabledFeatureError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
 from backend.common.genesets import write_gene_sets_tidycsv
 
 
 class Annotations(metaclass=ABCMeta):
-    """ baseclass for annotations, including ontologies and gene sets"""
-
-    """ our default ontology is the PURL for the Cell Ontology.
-    See http://www.obofoundry.org/ontology/cl.html """
-    DefaultOnotology = "http://purl.obolibrary.org/obo/cl.obo"
+    """baseclass for annotations and gene sets"""
 
     def __init__(self, config={}):
-        self.ontology_data = None
         self.config = config
 
     def user_annotations_enabled(self):
@@ -33,27 +25,6 @@ class Annotations(metaclass=ABCMeta):
         if not self.gene_sets_save_enabled():
             raise DisabledFeatureError("User gene sets save is disabled.")
 
-    def load_ontology(self, path):
-        """Load and parse ontologies - currently support OBO files only."""
-        if path is None:
-            path = self.DefaultOnotology
-
-        try:
-            with fsspec.open(path) as f:
-                obo = fastobo.iter(f)
-                terms = filter(lambda stanza: type(stanza) is fastobo.term.TermFrame, obo)
-                names = [tag.name for term in terms for tag in term if type(tag) is fastobo.term.NameClause]
-                self.ontology_data = names
-
-        except FileNotFoundError as e:
-            raise OntologyLoadFailure("Unable to find OBO ontology path") from e
-
-        except SyntaxError as e:
-            raise OntologyLoadFailure("Syntax error loading OBO ontology") from e
-
-        except Exception as e:
-            raise OntologyLoadFailure("Error loading OBO file") from e
-
     def get_schema(self, data_adaptor):
         schema = []
         labels = self.read_labels(data_adaptor)
@@ -82,7 +53,7 @@ class Annotations(metaclass=ABCMeta):
 
     @abstractmethod
     def read_gene_sets(self, data_adaptor):
-        """Return the gene sets from persistent storage """
+        """Return the gene sets from persistent storage"""
         pass
 
     @abstractmethod

backend/server/common/annotations/local_file_csv.py

@@ -193,14 +193,14 @@ class AnnotationsLocalFile(Annotations):
         return os.getcwd()
 
     def _get_celllabels_filename(self, data_adaptor):
-        """ return the current annotation file name """
+        """return the current annotation file name"""
         if self.label_output_file:
             return self.label_output_file
 
         return self._get_filename(data_adaptor, "cell-labels")
 
     def _get_genesets_filename(self, data_adaptor):
-        """ return the current gene sets file name """
+        """return the current gene sets file name"""
         if self.gene_sets_output_file:
             return self.gene_sets_output_file
 
@@ -263,12 +263,6 @@ class AnnotationsLocalFile(Annotations):
         params["annotations_genesets_name_is_read_only"] = self.gene_sets_output_file is not None
         params["user_annotation_collection_name_enabled"] = True
 
-        if self.ontology_data:
-            params["annotations_cell_ontology_enabled"] = True
-            params["annotations_cell_ontology_terms"] = self.ontology_data
-        else:
-            params["annotations_cell_ontology_enabled"] = False
-
         if self.label_output_file is not None:
             # user has hard-wired the name of the annotation cell label data collection
             fname = os.path.basename(self.label_output_file)

backend/server/common/config/client_config.py

@@ -40,16 +40,12 @@ def get_client_config(app_config, data_adaptor):
         "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
         "backed": server_config.adaptor__anndata_adaptor__backed,
         "disable-diffexp": not dataset_config.diffexp__enable,
-        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
         "annotations": False,
         "annotations_file": None,
         "annotations_dir": None,
         "annotations_genesets": True,  # feature flag
         "annotations_genesets_readonly": dataset_config.user_annotations__gene_sets__readonly,
         "annotations_genesets_summary_methods": ["mean"],
-        "annotations_cell_ontology_enabled": False,
-        "annotations_cell_ontology_obopath": None,
-        "annotations_cell_ontology_terms": None,
         "custom_colors": dataset_config.presentation__custom_colors,
         "diffexp-may-be-slow": False,
     }

backend/server/common/config/dataset_config.py

@@ -3,8 +3,7 @@ from os.path import splitext, isdir
 
 from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.server.common.config.base_config import BaseConfig
-from backend.common.errors import ConfigurationError, OntologyLoadFailure, AnnotationsError
-from backend.server.compute.scanpy import get_scanpy_module
+from backend.common.errors import ConfigurationError, AnnotationsError
 from backend.server.data_common.matrix_loader import MatrixDataLoader
 
 
@@ -28,22 +27,19 @@ class DatasetConfig(BaseConfig):
                 "directory"
             ]
             self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
-            self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
-            self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
-                "obo_location"
-            ]
             self.user_annotations__gene_sets__readonly = default_config["user_annotations"]["gene_sets"]["readonly"]
             self.user_annotations__local_file_csv__gene_sets_file = default_config["user_annotations"][
                 "local_file_csv"
             ]["gene_sets_file"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
-            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
 
             self.diffexp__enable = default_config["diffexp"]["enable"]
             self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
             self.diffexp__top_n = default_config["diffexp"]["top_n"]
 
+            self.X_approximate_distribution = default_config["X_approximate_distribution"]
+
         except KeyError as e:
             raise ConfigurationError(f"Unexpected config: {str(e)}")
 
@@ -56,6 +52,7 @@ class DatasetConfig(BaseConfig):
         self.handle_user_annotations(context)
         self.handle_embeddings()
         self.handle_diffexp(context)
+        self.handle_X_approximate_distribution()
 
     def get_data_adaptor(self):
         server_config = self.app_config.server_config
@@ -101,10 +98,6 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__local_file_csv__gene_sets_file", (type(None), str)
         )
-        self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
-        self.validate_correct_type_of_configuration_attribute(
-            "user_annotations__ontology__obo_location", (type(None), str)
-        )
         self.validate_correct_type_of_configuration_attribute("user_annotations__gene_sets__readonly", bool)
 
         if self.user_annotations__enable or not self.user_annotations__gene_sets__readonly:
@@ -122,13 +115,6 @@ class DatasetConfig(BaseConfig):
         else:
             raise ConfigurationError('The only annotation type support is "local_file_csv"')
 
-        if self.user_annotations__enable:
-            if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
-                try:
-                    self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
-                except OntologyLoadFailure as e:
-                    raise ConfigurationError("Unable to load ontology terms\n" + str(e))
-
         self.check_annotation_config_vars_not_set(context)
 
     def handle_local_file_csv_annotations(self, context):
@@ -183,32 +169,11 @@ class DatasetConfig(BaseConfig):
             if not self.user_annotations__enable:
                 if filename is not None:
                     context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
-                if self.user_annotations__ontology__enable:
-                    context["messagefn"](
-                        "Warning: --experimental-annotations-ontology ignored as annotations are disabled."
-                    )
-                if self.user_annotations__ontology__obo_location is not None:
-                    context["messagefn"](
-                        "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
-                    )
                 if dirname is not None:
                     context["messagefn"]("Warning: --user-generated-data-dir ignored as annotations are disabled.")
 
     def handle_embeddings(self):
         self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
-        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                get_scanpy_module()
-            except NotImplementedError:
-                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
 
     def handle_diffexp(self, context):
         self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
@@ -220,3 +185,10 @@ class DatasetConfig(BaseConfig):
             context["messagefn"](
                 "CAUTION: due to the size of your dataset, " "running differential expression may take longer or fail."
             )
+
+    def handle_X_approximate_distribution(self):
+        self.validate_correct_type_of_configuration_attribute("X_approximate_distribution", str)
+        if self.X_approximate_distribution not in ["auto", "normal", "count"]:
+            raise ConfigurationError(
+                "X_approximate_distribution has unknown value -- must be 'auto', 'normal' or 'count'."
+            )

backend/server/common/rest.py

@@ -311,25 +311,6 @@ def layout_obs_get(request, data_adaptor):
         )
 
 
-def layout_obs_put(request, data_adaptor):
-    if not data_adaptor.dataset_config.embeddings__enable_reembedding:
-        return abort(HTTPStatus.NOT_IMPLEMENTED)
-
-    args = request.get_json()
-    filter = args["filter"] if args else None
-    if not filter:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, "obs filter is required")
-    method = args["method"] if args else "umap"
-
-    try:
-        schema = data_adaptor.compute_embedding(method, filter)
-        return make_response(jsonify(schema), HTTPStatus.OK, {"Content-Type": "application/json"})
-    except NotImplementedError as e:
-        return abort_and_log(HTTPStatus.NOT_IMPLEMENTED, str(e))
-    except (ValueError, DisabledFeatureError, FilterError) as e:
-        return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
-
-
 def genesets_get(request, data_adaptor):
     preferred_mimetype = request.accept_mimetypes.best_match(["application/json", "text/csv"])
     if preferred_mimetype not in ("application/json", "text/csv"):

backend/server/compute/scanpy.py

@@ -1,53 +0,0 @@
-import importlib
-import numpy as np
-
-"""
-Wrapper for various scanpy modules.  Will raise NotImplementedError if the scanpy
-module is not installed/available
-"""
-
-
-def get_scanpy_module():
-    try:
-        sc = importlib.import_module("scanpy")
-        # Future: we could enforce versions here, eg, lookat sc.__version__
-        return sc
-    except ModuleNotFoundError as e:
-        raise NotImplementedError("Please install scanpy to enable UMAP re-embedding") from e
-    except Exception as e:
-        # will capture other ImportError corner cases
-        raise NotImplementedError() from e
-
-
-def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap_options={}):
-    """
-    Given adata and an obs mask, return a new embedding for adata[obs_mask, :]
-    as an ndarray of shape (len(obs_mask), N), where N>=2.
-
-    Do NOT mutate adata.
-    """
-
-    # backed mode is incompatible with the current implementation
-    if adata.isbacked:
-        raise NotImplementedError("Backed mode is incompatible with re-embedding")
-
-    # safely get scanpy module, which may not be present.
-    sc = get_scanpy_module()
-
-    # https://github.com/theislab/anndata/issues/311
-    obs_mask = slice(None) if obs_mask is None else obs_mask
-    adata = adata[obs_mask, :].copy()
-
-    for k in list(adata.obsm.keys()):
-        del adata.obsm[k]
-    for k in list(adata.uns.keys()):
-        del adata.uns[k]
-
-    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
-    sc.pp.neighbors(adata, **neighbors_options)
-    sc.tl.umap(adata, **umap_options)
-
-    umap = adata.obsm["X_umap"]
-    result = np.full((obs_mask.shape[0], umap.shape[1]), np.NaN)
-    result[obs_mask] = umap
-    return result

backend/server/data_anndata/anndata_adaptor.py

@@ -1,20 +1,18 @@
 import warnings
-from datetime import datetime
 
 import anndata
 import numpy as np
 from packaging import version
 from pandas.core.dtypes.dtypes import CategoricalDtype
 from scipy import sparse
-from server_timing import Timing as ServerTiming
 
 import backend.common.compute.diffexp_generic as diffexp_generic
+import backend.common.compute.estimate_distribution as estimate_distribution
 from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
-from backend.common.constants import Axis, MAX_LAYOUTS
+from backend.common.constants import Axis, MAX_LAYOUTS, XApproximateDistribution
 from backend.server.common.corpora import corpora_get_props_from_anndata
-from backend.common.errors import PrepareError, DatasetAccessError, FilterError
+from backend.common.errors import PrepareError, DatasetAccessError
 from backend.common.utils.type_conversion_utils import get_schema_type_hint_of_array
-from backend.server.compute.scanpy import scanpy_umap
 from backend.server.data_common.data_adaptor import DataAdaptor
 from backend.common.fbs.matrix import encode_matrix_fbs
 
@@ -31,6 +29,7 @@ class AnndataAdaptor(DataAdaptor):
     def __init__(self, data_locator, app_config=None, dataset_config=None):
         super().__init__(data_locator, app_config, dataset_config)
         self.data = None
+        self.X_approximate_distribution = None
         self._load_data(data_locator)
         self._validate_and_initialize()
 
@@ -126,7 +125,11 @@ class AnndataAdaptor(DataAdaptor):
 
     def _create_schema(self):
         self.schema = {
-            "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
+            "dataframe": {
+                "nObs": self.cell_count,
+                "nVar": self.gene_count,
+                **get_schema_type_hint_of_array(self.data.X),
+            },
             "annotations": {
                 "obs": {"index": self.parameters.get("obs_names"), "columns": []},
                 "var": {"index": self.parameters.get("var_names"), "columns": []},
@@ -193,6 +196,13 @@ class AnndataAdaptor(DataAdaptor):
         self.gene_count = self.data.shape[1]
         self._create_schema()
 
+        if self.dataset_config.X_approximate_distribution == "auto":
+            """Lazy evaluate the heuristic if we are backed."""
+            if not self.data.isbacked:
+                self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
+        else:
+            self.X_approximate_distribution = self.dataset_config.X_approximate_distribution
+
         # heuristic
         n_values = self.data.shape[0] * self.data.shape[1]
         if (n_values > 1e8 and self.server_config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8):
@@ -301,28 +311,6 @@ class AnndataAdaptor(DataAdaptor):
         full_embedding = self.data.obsm[f"X_{ename}"]
         return full_embedding[:, 0:dims]
 
-    def compute_embedding(self, method, obsFilter):
-        if Axis.VAR in obsFilter:
-            raise FilterError("Observation filters may not contain variable conditions")
-        if method != "umap":
-            raise NotImplementedError(f"re-embedding method {method} is not available.")
-        try:
-            shape = self.get_shape()
-            obs_mask = self._axis_filter_to_mask(Axis.OBS, obsFilter["obs"], shape[0])
-        except (KeyError, IndexError):
-            raise FilterError("Error parsing filter")
-        with ServerTiming.time("layout.compute"):
-            X_umap = scanpy_umap(self.data, obs_mask)
-
-        # Server picks reemedding name, which must not collide with any other
-        # embedding name generated by this backend.
-        name = f"reembed:{method}_{datetime.now().isoformat(timespec='milliseconds')}"
-        dims = [f"{name}_0", f"{name}_1"]
-        layout_schema = {"name": name, "type": "float32", "dims": dims}
-        self.schema["layout"]["obs"].append(layout_schema)
-        self.data.obsm[f"X_{name}"] = X_umap
-        return layout_schema
-
     def compute_diffexp_ttest(self, maskA, maskB, top_n=None, lfc_cutoff=None):
         if top_n is None:
             top_n = self.dataset_config.diffexp__top_n
@@ -334,13 +322,29 @@ class AnndataAdaptor(DataAdaptor):
         return convert_anndata_category_colors_to_cxg_category_colors(self.data)
 
     def get_X_array(self, obs_mask=None, var_mask=None):
+        # H5Py does not support boolean indexing (masks), so convert to integer indexing
+        # when backed (ie, when AnnData is using H5Py indexing)
         if obs_mask is None:
             obs_mask = slice(None)
+        elif self.data.isbacked and obs_mask.dtype == bool:
+            obs_mask = obs_mask.nonzero()[0]
         if var_mask is None:
             var_mask = slice(None)
+        elif self.data.isbacked and var_mask.dtype == bool:
+            var_mask = var_mask.nonzero()[0]
         X = self.data.X[obs_mask, var_mask]
         return X
 
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        """return the approximate distribution of the X matrix."""
+        if self.X_approximate_distribution is None:
+            """Not yet evaluated."""
+            assert self.dataset_config.X_approximate_distribution == "auto"
+            self.data = self.data.to_memory()  # loads data
+            self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
+
+        return self.X_approximate_distribution
+
     def get_shape(self):
         return self.data.shape
 

backend/server/data_common/data_adaptor.py

@@ -6,7 +6,7 @@ from scipy import sparse
 from server_timing import Timing as ServerTiming
 
 from backend.server.common.config.app_config import AppConfig
-from backend.common.constants import Axis
+from backend.common.constants import Axis, XApproximateDistribution
 from backend.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError, UnsupportedSummaryMethod
 from backend.common.utils.utils import jsonify_numpy
 from backend.common.fbs.matrix import encode_matrix_fbs
@@ -66,17 +66,16 @@ class DataAdaptor(metaclass=ABCMeta):
         """return an numpy array for the given pre-computed embedding name."""
         pass
 
-    @abstractmethod
-    def compute_embedding(self, method, filter):
-        """compute a new embedding on the specified obs subset, and return the embedding schema."""
-        pass
-
     @abstractmethod
     def get_X_array(self, obs_mask=None, var_mask=None):
         """return the X array, possibly filtered by obs_mask or var_mask.
         the return type is either ndarray or scipy.sparse.spmatrix."""
         pass
 
+    def get_X_approximate_distribution(self) -> XApproximateDistribution:
+        """return the approximate distribution of the X matrix."""
+        return XApproximateDistribution.NORMAL
+
     @abstractmethod
     def get_shape(self):
         pass

backend/server/default_config.py

@@ -66,21 +66,19 @@ dataset:
       directory: null
       file: null            # annotations file name
       gene_sets_file: null  # gene sets file name
-    ontology:
-      enable: false
-      obo_location: null
     gene_sets:
       readonly: false       # gene sets CRUD enabled/disabled
 
   embeddings:
     names : []
-    enable_reembedding: false
 
   diffexp:
     enable: true
     lfc_cutoff: 0.01
     top_n: 10
 
+  X_approximate_distribution: auto
+
 external:
   # You can retrieve configuration parameters from this config file, the environment,
   # the AWS secrets manager, or from the "cellxgene launch" command line arguments.

backend/server/requirements-dev.txt

@@ -8,3 +8,4 @@ pytest>=3.6.3
 python-jose>=3.2.0
 twine>=1.12.1
 -r requirements.txt
+-r requirements-prepare.txt

backend/server/requirements-prepare.txt

@@ -1,2 +1,4 @@
 python-igraph>=0.8
 louvain>=0.6
+scanpy
+umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/server/requirements.txt

@@ -1,7 +1,6 @@
-anndata>=0.7.0
+anndata>=0.7.6 # we need to_memory(), added in 0.7.6
 boto3>=1.12.18
 click>=7.1.2
-fastobo>=0.6.1
 Flask>=1.0.2,<2.0.0 # Flask 2.0 is not compatible with the latest version of Flask-RESTful (0.3.8)
 Flask-Compress>=1.4.0
 Flask-Cors>=3.0.9  # CVE-2020-25032
@@ -12,9 +11,9 @@ flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires mi
 flatten-dict>=0.2.0
 fsspec>=0.4.4,<0.8.0
 gunicorn>=20.0.4
-h5py<3.0.0 # h5py>=3.0.0 had a breaking change; there is a fix in anndata>=0.7.5
+h5py>=3.0.0
 jinja2>=2.11.3  # Flask sub-dependency. Added due to CVE-2020-28493
-numba>=0.51.2,<0.53.0
+numba>=0.51.2
 numpy>=1.17.5
 packaging>=20.0
 pandas>=1.0,!=1.1  # pandas 1.1 breaks tests, https://github.com/pandas-dev/pandas/issues/35446
@@ -22,5 +21,3 @@ PyYAML>=5.4  # CVE-2020-14343
 scipy>=1.4
 requests>=2.22.0
 s3fs==0.4.2
-scanpy==1.4.6 # Until we move to anndata 0.7.4 scanpy needs to be pinned here
-umap-learn<0.5.0 # The pinned version scanpy is not compatible with latest umap-learn

backend/test/fixtures/czi_hosted_dataset_config_outline.py

@@ -22,16 +22,14 @@ dataset:
     local_file_csv:
       directory: {local_file_csv_directory}
       file: {local_file_csv_file}
-    ontology:
-      enable: {ontology_enabled}
-      obo_location: {obo_location}
 
   embeddings:
     names: {embedding_names}
-    enable_reembedding: {enable_reembedding}
 
   diffexp:
     enable: {enable_difexp}
     lfc_cutoff: {lfc_cutoff}
     top_n: {top_n}
+
+  X_approximate_distribution: {X_approximate_distribution}
 """

backend/test/fixtures/dataset_config_outline.py

@@ -17,18 +17,16 @@ dataset:
       directory: {local_file_csv_directory}
       file: {local_file_csv_file}
       gene_sets_file: {local_file_csv_gene_sets_file}
-    ontology:
-      enable: {ontology_enabled}
-      obo_location: {obo_location}
     gene_sets:
       readonly: {gene_sets_readonly}
 
   embeddings:
     names: {embedding_names}
-    enable_reembedding: {enable_reembedding}
 
   diffexp:
     enable: {enable_difexp}
     lfc_cutoff: {lfc_cutoff}
     top_n: {top_n}
+
+  X_approximate_distribution: {X_approximate_distribution}
 """

backend/test/fixtures/pbmc3k-genesets.csv

@@ -16,6 +16,6 @@ summary test,,F5,
 summary test,,PIGU,
 geneset_to_delete,,,
 geneset_to_edit,,,
-fill_this_geneset,,RER1,
+fill_this_geneset,,,
 empty_this_geneset,,SIK1,
 brush_this_gene,,SIK1,

backend/test/test_common/unit/fbs/test_matrix.py

@@ -2,16 +2,20 @@ import unittest
 import pandas as pd
 import numpy as np
 from scipy import sparse
+from parameterized import parameterized_class
+import json
 
-import backend.test.decode_fbs as decode_fbs
+from backend.test import decode_fbs
 from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
+from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe
+import backend.common.fbs as fbs
 
 
 class FbsTests(unittest.TestCase):
-    """Test Case for Matrix FBS data encode/decode """
+    """Test Case for Matrix FBS data encode/decode"""
 
     def test_encode_boundary(self):
-        """ test various boundary checks """
+        """test various boundary checks"""
 
         # row indexing is unsupported
         with self.assertRaises(ValueError):
@@ -46,7 +50,7 @@ class FbsTests(unittest.TestCase):
                 "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
             }
         )
-        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None))
         fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
         self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
 
@@ -80,3 +84,133 @@ class FbsTests(unittest.TestCase):
                 self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
             else:
                 self.assertEqual(dfSrc[c], dfDst[c])
+
+
+"""
+Test type consistency between FBS encoding and the underlying schema hint.
+
+Basic assertion:  the FBS type returned by encode_matrix_fbs() will be consistent
+with the schema hint returned by type_conversion_utils (which is in turn used
+to create the client schema).
+
+The following test cases are all dicts which contain the following keys:
+    - dataframe - the dataframe used as input for encode_matrix_fbs
+    - expected_fbs_types - upon success, dict of FBS column types expected (eg, Float32Array)
+    - expected_schema_hints - upon success, dict of schema hint
+All are keyed by column name.
+"""
+
+# simple tests that we convert all ints to int32
+int_dtypes = [np.dtype(d) for d in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]]
+int_test_cases = [
+    {
+        "dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in int_dtypes}),
+        "expected_fbs_types": dict(
+            [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Int32Array) for dtype in int_dtypes]
+        ),
+        "expected_schema_hints": dict([(dtype.name, {"type": "int32"}) for dtype in int_dtypes]),
+    }
+]
+
+# simple tests that we convert all floats to float32
+float_dtypes = [np.dtype(d) for d in [np.float16, np.float32, np.float64]]
+float_test_cases = [
+    {
+        "dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in float_dtypes}),
+        "expected_fbs_types": dict(
+            [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Float32Array) for dtype in float_dtypes]
+        ),
+        "expected_schema_hints": dict([(dtype.name, {"type": "float32"}) for dtype in float_dtypes]),
+    }
+]
+
+# boolean - should be encoded as an uint32
+bool_dtypes = [np.dtype(d) for d in [np.bool_, bool]]
+bool_test_cases = [
+    {
+        "dataframe": pd.DataFrame({dtype.name: np.ones((10,), dtype=dtype) for dtype in bool_dtypes}),
+        "expected_fbs_types": dict(
+            [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Uint32Array) for dtype in bool_dtypes]
+        ),
+        "expected_schema_hints": dict([(dtype.name, {"type": "boolean"}) for dtype in bool_dtypes]),
+    }
+]
+
+cat_test_cases = [
+    {
+        "dataframe": pd.DataFrame({"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category")}),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray},
+        "expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "b", "c"]}},
+    },
+    {
+        "dataframe": pd.DataFrame(
+            {"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category").cat.remove_categories("b")}
+        ),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray},
+        "expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "c"]}},
+    },
+    {
+        "dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category")}),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Int32Array},
+        "expected_schema_hints": {"a": {"type": "categorical"}},
+    },
+    {
+        "dataframe": pd.DataFrame(
+            {"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category").cat.remove_categories(2)}
+        ),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
+        "expected_schema_hints": {"a": {"type": "categorical"}},
+    },
+    {
+        "dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category")}),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
+        "expected_schema_hints": {"a": {"type": "categorical"}},
+    },
+    {
+        "dataframe": pd.DataFrame(
+            {"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category").cat.remove_categories(2)}
+        ),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
+        "expected_schema_hints": {"a": {"type": "categorical"}},
+    },
+]
+
+
+test_cases = [
+    *int_test_cases,
+    *float_test_cases,
+    *bool_test_cases,
+    *cat_test_cases,
+]
+
+
+@parameterized_class(test_cases)
+class TestTypeConversionConsistency(unittest.TestCase):
+    def test_type_conversion_consistency(self):
+        self.assertEqual(self.dataframe.shape[1], len(self.expected_fbs_types))
+        self.assertEqual(self.dataframe.shape[1], len(self.expected_schema_hints))
+
+        buf = encode_matrix_fbs(matrix=self.dataframe, col_idx=self.dataframe.columns)
+        encoding_dtypes, schema_hints = get_dtypes_and_schemas_of_dataframe(self.dataframe)
+
+        # check schema hints
+        # print(schema_hints)
+        # print(self.expected_schema_hints)
+        self.assertEqual(schema_hints, self.expected_schema_hints)
+
+        # inspect the FBS types
+        matrix = fbs.NetEncoding.Matrix.Matrix.GetRootAsMatrix(buf, 0)
+        columns_length = matrix.ColumnsLength()
+        self.assertEqual(columns_length, self.dataframe.shape[1])
+
+        self.assertEqual(matrix.ColIndexType(), fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray)
+        col_labels_arr = fbs.NetEncoding.JSONEncodedArray.JSONEncodedArray()
+        col_labels_arr.Init(matrix.ColIndex().Bytes, matrix.ColIndex().Pos)
+        col_index_labels = json.loads(col_labels_arr.DataAsNumpy().tobytes().decode("utf-8"))
+        self.assertEqual(len(col_index_labels), self.dataframe.shape[1])
+
+        for col_idx in range(0, columns_length):
+            col_label = col_index_labels[col_idx]
+            col = matrix.Columns(col_idx)
+            col_type = col.UType()
+            self.assertEqual(self.expected_fbs_types[col_label], col_type)

backend/test/test_common/unit/utils/test_type_conversion_utils.py

@@ -1,179 +1,22 @@
 import unittest
-from time import time
-from unittest.mock import patch
+import logging
+from parameterized import parameterized_class
 
 import numpy as np
+import pandas as pd
 from pandas import Series, DataFrame
+from scipy import sparse
 
 from backend.common.utils.type_conversion_utils import (
-    can_cast_to_float32,
-    can_cast_to_int32,
-    get_dtype_of_array,
+    get_encoding_dtype_of_array,
     get_schema_type_hint_of_array,
     get_dtypes_and_schemas_of_dataframe,
-    convert_pandas_series_to_numpy,
+    get_dtype_and_schema_of_array,
+    get_schema_type_hint_from_dtype,
 )
 
 
 class TestTypeConversionUtils(unittest.TestCase):
-    def test__can_cast_to_float32__string_is_false(self):
-        array_to_convert = Series(data=["1", "2", "3"], dtype=str)
-
-        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
-
-        self.assertFalse(can_cast)
-
-    def test__can_cast_to_float32__float64_is_true_warning_outputted(self):
-        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
-
-        with self.assertLogs(level="WARN") as logger:
-            can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
-            self.assertIn("may lose precision", logger.output[0])
-
-        self.assertTrue(can_cast)
-
-    @patch("logging.warning")
-    def test__can_cast_to_float32__float32_is_false(self, mock_log_warning):
-        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32))
-
-        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
-
-        self.assertTrue(can_cast)
-        assert not mock_log_warning.called
-
-    def test__can_cast_to_float32__categorical_float64_is_false(self):
-        array_to_convert = Series(data=[1.1, 2.2, 3.3], dtype="category")
-
-        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
-
-        self.assertFalse(can_cast)
-
-    def test__can_cast_to_float32__categorical_int64_with_nans_is_true(self):
-        array_to_convert = Series(data=[1, 2, np.NaN], dtype="category")
-
-        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
-
-        self.assertTrue(can_cast)
-
-    def test__can_cast_to_float_32__float_32_with_nans_is_true(self):
-        array_to_convert = Series(data=[1, 2, np.NaN], dtype=np.dtype(np.float32))
-
-        can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
-
-        self.assertTrue(can_cast)
-
-    def test__can_cast_to_int32__string_is_false(self):
-        array_to_convert = Series(data=["1", "2", "3"], dtype=str)
-
-        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
-
-        self.assertFalse(can_cast)
-
-    def test__can_cast_to_int32__int64_is_true(self):
-        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64))
-
-        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
-
-        self.assertTrue(can_cast)
-
-    def test__can_cast_to_int32__int16_is_true(self):
-        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16))
-
-        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
-
-        self.assertTrue(can_cast)
-
-    def test__can_cast_to_int32__int64_with_large_value_is_false(self):
-        array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64))
-
-        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
-
-        self.assertFalse(can_cast)
-
-    def test__can_cast_to_int32__int64_with_nans_is_false(self):
-        array_to_convert = Series(data=[np.NaN, "2", "3"], dtype="category")
-
-        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
-
-        self.assertFalse(can_cast)
-
-    def test__get_dtype_of_array__supported_dtypes_return_as_expected(self):
-        types = [np.float32, np.int32, np.bool_, str]
-        expected_dtypes = [np.float32, np.int32, np.uint8, str]
-
-        for test_type_index in range(len(types)):
-            with self.subTest(
-                f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index
-            ):
-                array = Series(data=[], dtype=types[test_type_index])
-                self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
-
-    def test__get_dtype_of_array__categories_return_as_expected(self):
-        array = Series(data=["a", "b", "c"], dtype="category")
-        expected_dtype = str
-
-        actual_dtype = get_dtype_of_array(array)
-
-        self.assertEqual(expected_dtype, actual_dtype)
-
-    def test__get_dtype_of_array__unordered_integer_categories_return_as_expected(self):
-        array = Series(data=[2, 3, 1, 3, 1, 2], dtype="category")
-        expected_dtype = np.int32
-
-        actual_dtype = get_dtype_of_array(array)
-
-        self.assertEqual(expected_dtype, actual_dtype)
-
-    def test__get_dtype_of_array__castable_dtypes_return_as_expected(self):
-        types = [np.float64, np.int64]
-        expected_dtypes = [np.float32, np.int32]
-
-        for test_type_index in range(len(types)):
-            with self.subTest(
-                f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index
-            ):
-                array = Series(data=[], dtype=types[test_type_index])
-                self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
-
-    def test__get_dtype_of_array__unsupported_type_raises_exception(self):
-        unsupported_array = Series(list([time() for _ in range(2)]), dtype="datetime64[ns]")
-
-        with self.assertRaises(TypeError) as exception_context:
-            get_dtype_of_array(unsupported_array)
-
-        self.assertIn("unsupported", str(exception_context.exception))
-
-    def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self):
-        types = [np.float32, np.int32, np.bool_, str]
-        expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}]
-
-        for test_type_index in range(len(types)):
-            with self.subTest(
-                f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index
-            ):
-                array = Series(data=[], dtype=types[test_type_index])
-                self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
-
-    def test__get_schema_type_hint_of_array__categories_return_as_expected(self):
-        array = Series(data=["a", "b", "b"], dtype="category")
-        expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]}
-
-        actual_schema_hint = get_schema_type_hint_of_array(array)
-
-        self.assertEqual(expected_schema_hint, actual_schema_hint)
-
-    def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self):
-        types = [np.float64, np.int64]
-        expected_schema_hints = [{"type": "float32"}, {"type": "int32"}]
-
-        for test_type_index in range(len(types)):
-            with self.subTest(
-                f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
-                i=test_type_index,
-            ):
-                array = Series(data=[], dtype=types[test_type_index])
-                self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
-
     def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self):
         float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
         category_array = Series(data=["a", "b", "b"], dtype="category")
@@ -190,28 +33,292 @@ class TestTypeConversionUtils(unittest.TestCase):
         self.assertEqual(expected_data_types_dict, actual_dataframe_data_types)
         self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints)
 
-    def test__convert_pandas_series_to_numpy__categorical_float64_to_float64_with_nans(self):
-        expected_float_array = np.array([1.1, 2.2, np.NaN], dtype=np.float64)
-        float_series = Series(data=[1.1, 2.2, np.NaN], dtype="category")
+    def test__get_schema_type_hint_from_dtype(self):
+        self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(np.bool_)), {"type": "boolean"})
 
-        actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64)
+        for dtype in [np.int8, np.int8, np.int16, np.uint16, np.int32]:
+            self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "int32"})
+        for dtype in [np.uint32, np.int64, np.uint64]:
+            with self.assertRaises(TypeError):
+                get_schema_type_hint_from_dtype(np.dtype(dtype))
 
-        np.testing.assert_equal(expected_float_array, actual_float_array)
+        for dtype in [np.float16, np.float32, np.float64]:
+            self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
 
-    def test__convert_pandas_series_to_numpy__float64_to_float64(self):
-        expected_float_array = np.array([1.1, 2.2], dtype=np.float64)
-        float_series = Series(data=[1.1, 2.2], dtype=np.dtype(np.float64))
+        for dtype in [np.dtype(object), np.dtype(str)]:
+            self.assertEqual(get_schema_type_hint_from_dtype(dtype), {"type": "string"})
 
-        actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64)
 
-        np.testing.assert_equal(expected_float_array, actual_float_array)
+# Credit: https://stackoverflow.com/questions/35871815/python-3-unit-testing-assert-logger-not-called/64774103#64774103
+class AssertNoLog:
+    def assertNoLogs(self, logger, level):
+        """functions as a context manager.  To be introduced in python 3.10"""
 
-    def test__convert_pandas_series_to_numpy__int64_to_int32_with_nans_throws_error(self):
-        int_series = Series(data=[1, 2, np.NaN], dtype="category")
+        class AssertNoLogsContext(unittest.TestCase):
+            def __init__(self, logger, level):
+                self.logger = logger
+                self.level = level
+                self.context = self.assertLogs(logger, level)
 
-        with self.assertLogs(level="ERROR") as logger:
-            convert_pandas_series_to_numpy(int_series, np.int32)
+            def __enter__(self):
+                """enter self.assertLogs as context manager, and log something"""
+                self.initial_logmsg = "sole message"
+                self.cm = self.context.__enter__()
+                self.logger.log(self.level, self.initial_logmsg)
+                return self.cm
 
-            self.assertIn(
-                "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0]
-            )
+            def __exit__(self, exc_type, exc_val, exc_tb):
+                """cleanup logs, and then check nothing extra was logged"""
+                # assertLogs.__exit__ should never fail because of initial msg
+                self.context.__exit__(exc_type, exc_val, exc_tb)
+                if len(self.cm.output) > 1:
+                    """override any exception passed to __exit__"""
+                    self.context._raiseFailure(
+                        "logs of level {} or higher triggered on {} : {}".format(
+                            logging.getLevelName(self.level), self.logger.name, self.cm.output[1:]
+                        )
+                    )
+
+        return AssertNoLogsContext(logger, level)
+
+
+"""
+See table of expected cases in type_conversion_utils.py.
+
+This probes all edge cases. Each case is a dict containing keys:
+    - data - the array to be introspected
+    - throws - if not None, the expected Error (eg, TypeError)
+    - expected_encoding_dtype - upon success
+    - expected_schema_hint - upon success
+    - logs - if not None, specify expected log output
+"""
+
+bool_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.uint8,
+        "expected_schema_hint": {"type": "boolean"},
+    }
+    for data in [
+        np.array([0, 1, 0, 1], dtype=np.bool_),
+        pd.Series(np.array([0, 1, 0, 1], dtype=np.bool_)),
+        # pd.Index with bools doesn't really make any sense...and becomes dtype=object
+    ]
+]
+
+int_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.int32,
+        "expected_schema_hint": {"type": "int32"},
+    }
+    for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]
+    for data in [
+        np.arange(0, 1000, dtype=dtype),
+        pd.Series(np.arange(0, 1000, dtype=dtype)),
+        pd.Index(np.arange(0, 1000, dtype=dtype)),
+        sparse.csr_matrix((10, 100), dtype=dtype),
+    ]
+]
+
+float_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.float32,
+        "expected_schema_hint": {"type": "float32"},
+        "logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"},
+    }
+    for dtype in [np.float16, np.float32, np.float64]
+    for data in [
+        np.arange(-128, 1000, dtype=dtype),
+        pd.Series(np.arange(-128, 1000, dtype=dtype)),
+        pd.Index(np.arange(-129, 1000, dtype=dtype)),
+        np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
+        np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
+        sparse.csr_matrix((10, 100), dtype=dtype),
+    ]
+]
+
+
+numeric_ERR_cases = [
+    {
+        "data": data,
+        "throws": TypeError,
+    }
+    for data in [
+        np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64),
+        np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64),
+        np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32),
+    ]
+]
+
+
+string_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.dtype(str),
+        "expected_schema_hint": {"type": "string"},
+    }
+    for data in [
+        np.array(["a", "b", "c"]),
+        np.array(["a", "b", "c"], dtype="object"),
+        pd.Series(["a", "b", "c"]),
+        pd.Index(["a", "b", "c"]),
+        np.array(["a", [], {}, None, True, False, 383.2], dtype="object"),
+    ]
+]
+
+category_nonnumeric_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.dtype(str),
+        "expected_schema_hint": {"type": "categorical", "categories": data.dtype.categories.to_list()},
+    }
+    for data in [
+        pd.Series(["a", "b", "c"], dtype="category"),
+        pd.Series(["a", "b", "c", 0, 1, 2], dtype="category"),
+        pd.Series(["a", "b", "c"], dtype="category").cat.remove_categories(["b"]),
+        pd.Series(["a", "b", "c", 0, 1, 2], dtype="category").cat.remove_categories(["b", 0]),
+    ]
+]
+
+category_numeric_OK_cases = [
+    # numeric, no NA/NaN, int
+    *[
+        {
+            "data": data,
+            "expected_encoding_dtype": np.int32,
+            "expected_schema_hint": {"type": "categorical"},
+        }
+        for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]
+        for data in [
+            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
+        ]
+    ],
+    # numeric, no NA/NaN, float
+    *[
+        {
+            "data": data,
+            "expected_encoding_dtype": np.float32,
+            "expected_schema_hint": {"type": "categorical"},
+            "logs": {"level": logging.WARNING, "output": "may lose precision"},
+        }
+        for dtype in [np.float16, np.float32, np.float64]
+        for data in [
+            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
+            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
+            pd.Categorical(np.array([0, 1, 2], dtype=dtype)),
+        ]
+    ],
+    # numeric, has NA-induced cast to float32
+    *[
+        {
+            "data": data,
+            "expected_encoding_dtype": np.float32,
+            "expected_schema_hint": {"type": "categorical"},
+            "logs": {"level": logging.WARNING, "output": "may lose precision"},
+        }
+        for dtype in [
+            np.int8,
+            np.uint8,
+            np.int16,
+            np.uint16,
+            np.int32,
+            np.uint32,
+            np.int64,
+            np.uint64,
+            np.float16,
+            np.float32,
+            np.float64,
+        ]
+        for data in [
+            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
+            pd.Categorical(np.array([0, 1, 2], dtype=dtype), categories=np.array([0, 1], dtype=dtype)),
+        ]
+    ],
+]
+
+category_ERR_cases = [
+    # catch expected categorical exceptions for Int64(etc) that have large values
+    {
+        "data": data,
+        "throws": TypeError,
+    }
+    for data in [
+        pd.Categorical(np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64)),
+        pd.Categorical(np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64)),
+        pd.Categorical(np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32)),
+    ]
+]
+
+object_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.dtype(str),
+        "expected_schema_hint": {"type": "string"},
+    }
+    for data in [
+        np.array(["a", True, 1, [], {}], dtype="object"),
+        pd.Series(["a", True, 1, [], {}], dtype="object"),
+        pd.Index(["a", True, 1, [], {}], dtype="object"),
+    ]
+]
+
+err_cases = [
+    {"data": np.array, "throws": TypeError}
+    for data in [
+        np.ones((10,), dtype=np.complex64),
+        np.ones((10,), dtype=np.complex128),
+        np.array([b"foobar"], dtype=np.bytes_),
+        np.ones((10,), dtype=np.void),
+        np.arange("2005-02", "2005-03", dtype="datetime64[D]"),
+        np.arange("2005-02", "2005-03", dtype="datetime64[D]") - np.datetime64("2008-01-01"),
+        [],
+        {},
+    ]
+]
+
+test_cases = [
+    *bool_OK_cases,
+    *int_OK_cases,
+    *float_OK_cases,
+    *numeric_ERR_cases,
+    *string_OK_cases,
+    *category_nonnumeric_OK_cases,
+    *category_numeric_OK_cases,
+    *category_ERR_cases,
+    *object_OK_cases,
+    *err_cases,
+]
+
+
+@parameterized_class(test_cases)
+class TestTypeInference(unittest.TestCase, AssertNoLog):
+    def test_type_inference(self):
+        throws = getattr(self, "throws", None)
+        if throws:
+            with self.assertRaises(throws):
+                get_dtype_and_schema_of_array(self.data)
+            with self.assertRaises(throws):
+                get_encoding_dtype_of_array(self.data)
+            with self.assertRaises(throws):
+                get_schema_type_hint_of_array(self.data)
+
+        else:
+            logs = getattr(self, "logs", None)
+            if logs is not None:
+                with self.assertLogs(level=logs["level"]) as logger:
+                    encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
+                    self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
+                    self.assertEqual(schema_hint, self.expected_schema_hint)
+                    self.assertIn(logs["output"], logger.output[0])
+
+            else:
+                with self.assertNoLogs(logging.getLogger(), logging.WARNING):
+                    encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
+                    self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
+                    self.assertEqual(schema_hint, self.expected_schema_hint)
+
+            # also test the other public API
+            self.assertEqual(get_encoding_dtype_of_array(self.data), self.expected_encoding_dtype)
+            self.assertEqual(get_schema_type_hint_of_array(self.data), self.expected_schema_hint)

backend/test/test_czi_hosted/unit/__init__.py

@@ -170,7 +170,6 @@ class BaseTest(unittest.TestCase):
                 multi_dataset__index=True,
                 multi_dataset__allowed_matrix_types=["cxg"]
             )
-            app_config.update_default_dataset_config(embeddings__enable_reembedding=False, )
         app_config.complete_config(logging.info)
 
         app = TestServer(app_config).app

backend/test/test_czi_hosted/unit/common/config/__init__.py

@@ -124,16 +124,14 @@ class ConfigTests(BaseTest):
         hosted_file_directory="null",
         local_file_csv_directory="null",
         local_file_csv_file="null",
-        ontology_enabled="false",
-        obo_location="null",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
         environment=None,
         aws_secrets_manager_region=None,
         aws_secrets_manager_secrets=[],
+        X_approximate_distribution="normal",
         config_file_name="app_config.yml",
     ):
         random_num = random.randrange(999999)
@@ -193,13 +191,11 @@ class ConfigTests(BaseTest):
             hosted_file_directory=hosted_file_directory,
             local_file_csv_directory=local_file_csv_directory,
             local_file_csv_file=local_file_csv_file,
-            ontology_enabled=ontology_enabled,
-            obo_location=obo_location,
             embedding_names=embedding_names,
-            enable_reembedding=enable_reembedding,
             enable_difexp=enable_difexp,
             lfc_cutoff=lfc_cutoff,
             top_n=top_n,
+            X_approximate_distribution=X_approximate_distribution,
             config_file_name=f"temp_dataset_config_{random_num}.yml",
         )
         external_config = self.custom_external_config(
@@ -231,13 +227,11 @@ class ConfigTests(BaseTest):
         hosted_file_directory="null",
         local_file_csv_directory="null",
         local_file_csv_file="null",
-        ontology_enabled="false",
-        obo_location="null",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
+        X_approximate_distribution="normal",
         config_file_name="dataset_config.yml",
     ):
         configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)

backend/test/test_czi_hosted/unit/common/config/test_base_config.py

@@ -36,7 +36,6 @@ class BaseConfigTest(ConfigTests):
         mapping = config.default_dataset_config.create_mapping(config.default_config)
         self.assertIsNotNone(mapping["server__app__verbose"])
         self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
-        self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
         self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
 
     def test_changes_from_default_returns_list_of_nondefault_config_values(self):

backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py

@@ -44,13 +44,12 @@ class TestDatasetConfig(ConfigTests):
         self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
         self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
         self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
-        self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location)
 
     @patch("backend.czi_hosted.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
     def test_complete_config_checks_all_attr(self, mock_check_attrs):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.dataset_config.complete_config(self.context)
-        self.assertEqual(mock_check_attrs.call_count, 21)
+        self.assertEqual(mock_check_attrs.call_count, 19)
 
     def test_app_sets_script_vars(self):
         config = self.get_config(scripts=["path/to/script"])
@@ -131,20 +130,6 @@ class TestDatasetConfig(ConfigTests):
         cwd = os.getcwd()
         self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
 
-    def test_handle_embeddings__checks_data_file_types(self):
-        file_name = self.custom_app_config(
-            embedding_names=["name1", "name2"],
-            enable_reembedding="true",
-            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
-            anndata_backed="true",
-            config_file_name=self.config_file_name,
-        )
-        config = AppConfig()
-        config.update_from_config_file(file_name)
-        config.server_config.complete_config(self.context)
-        with self.assertRaises(ConfigurationError):
-            config.default_dataset_config.handle_embeddings()
-
     def test_handle_diffexp__raises_warning_for_large_datasets(self):
         config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
         config.server_config.complete_config(self.context)

backend/test/test_czi_hosted/unit/common/test_api.py

@@ -69,35 +69,6 @@ class EndPoints(BaseTest):
         self.assertIsNone(df["row_idx"])
         self.assertEqual(len(df["columns"]), df["n_cols"])
 
-    def test_put_layout_fbs(self):
-        # first check that re-embedding is turned on
-        self.app.auth.get_user_id = lambda : "123"
-        result = self.client.get(f"{self.TEST_URL_BASE}config")
-        config_data = json.loads(result.data)
-        re_embed = config_data["config"]["parameters"]["enable-reembedding"]
-        if not re_embed:
-            return
-        # attempt to reembed with umap over 100 cells.
-        endpoint = "layout/obs"
-        url = f"{self.TEST_URL_BASE}{endpoint}"
-        data = {}
-        data["filter"] = {}
-        data["filter"]["obs"] = {}
-        data["filter"]["obs"]["index"] = list(range(100))
-        data["method"] = "umap"
-        result = self.client.put(url, json=data)
-
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data = result.json()
-        self.assertIsInstance(result_data, dict)
-        self.assertEqual(result_data["type"], "float32")
-        self.assertTrue(result_data["name"].startswith("reembed:umap_"))
-        self.assertIsInstance(result_data["dims"], list)
-        self.assertEqual(len(result_data["dims"]), 2)
-        dims = result_data["dims"]
-        self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
-        self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
-
     def test_bad_filter(self):
         endpoint = "data/var"
         url = f"{self.TEST_URL_BASE}{endpoint}"
@@ -148,6 +119,8 @@ class EndPoints(BaseTest):
         result = self.client.get(url, headers=header)
         self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
 
+# TEMP: Testing count 15 to match hardcoded values for diffexp
+# TODO(#1281): Switch back to dynamic values
     def test_diff_exp(self):
         endpoint = "diffexp/obs"
         url = f"{self.TEST_URL_BASE}{endpoint}"
@@ -155,21 +128,21 @@ class EndPoints(BaseTest):
             "mode": "topN",
             "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
             "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
-            "count": 7,
+            "count": 15,
         }
         result = self.client.post(url, json=params)
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = json.loads(result.data)
-        self.assertEqual(len(result_data['positive']), 7)
-        self.assertEqual(len(result_data['negative']), 7)
+        self.assertEqual(len(result_data['positive']), 15)
+        self.assertEqual(len(result_data['negative']), 15)
 
     def test_diff_exp_indices(self):
         endpoint = "diffexp/obs"
         url = f"{self.TEST_URL_BASE}{endpoint}"
         params = {
             "mode": "topN",
-            "count": 10,
+            "count": 15,
             "set1": {"filter": {"obs": {"index": [[0, 500]]}}},
             "set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
         }
@@ -177,8 +150,8 @@ class EndPoints(BaseTest):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = json.loads(result.data)
-        self.assertEqual(len(result_data['positive']), 10)
-        self.assertEqual(len(result_data['negative']), 10)
+        self.assertEqual(len(result_data['positive']), 15)
+        self.assertEqual(len(result_data['negative']), 15)
 
     def test_get_annotations_var_fbs(self):
         endpoint = "annotations/var"
@@ -421,7 +394,7 @@ class EndPointsCxg(EndPoints):
     @classmethod
     def setUpClass(cls):
         app_config = AppConfig()
-        app_config.update_default_dataset_config(embeddings__enable_reembedding=True, user_annotations__enable=False)
+        app_config.update_default_dataset_config(user_annotations__enable=False)
 
     def test_get_genesets_json(self):
         self.app.auth.is_user_authenticated = lambda: True
@@ -471,7 +444,7 @@ class EndPointsCxg(EndPoints):
                     {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_delete'},
                     {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_edit'},
                     {
-                        'genes': [{'gene_description': '', 'gene_symbol': 'RER1'}],
+                        'genes': [],
                         'geneset_description': '',
                         'geneset_name': 'fill_this_geneset'
                     },
@@ -512,7 +485,7 @@ summary test,,F5,\r
 summary test,,PIGU,\r
 geneset_to_delete,,,\r
 geneset_to_edit,,,\r
-fill_this_geneset,,RER1,\r
+fill_this_geneset,,,\r
 empty_this_geneset,,SIK1,\r
 brush_this_gene,,SIK1,\r
 """

backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py

@@ -20,6 +20,7 @@ class DiffExpTest(unittest.TestCase):
     adaptor types and different algorithms."""
 
     def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        extra_dataset_config["X_approximate_distribution"] = "normal"  # hardwired for now
         config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
         loader = MatrixDataLoader(path)
         adaptor = loader.open(config)
@@ -46,28 +47,28 @@ class DiffExpTest(unittest.TestCase):
         """Checks the results for a specific set of rows selections"""
 
         positive_expects = [
-            [1712, -0.5525154, 0.0051788902660723345, 1.0],
-            [1575, 1.0317602, 0.007830310753043345, 1.0],
-            [693, 0.4703904, 0.008715846769131548, 1.0],
-            [916, 0.9567287, 0.009080596532247588, 1.0],
-            [77, 0.02665649, 0.010070392939027756, 1.0],
-            [782, -1.0981874, 0.010161745218916036, 1.0],
-            [913, 0.5683986, 0.010782030711612685, 1.0],
-            [910, 0.83164597, 0.014596411069229197, 1.0],
-            [1727, 0.4127781, 0.015168372104237176, 1.0],
-            [1443, -0.8241895, 0.015337080567465522, 1.0]
+            [1712, 0.24104056, 0.0051788902660723345, 1.0],
+            [1575, 0.2615018, 0.007830310753043345, 1.0],
+            [693, 0.23106655, 0.008715846769131548, 1.0],
+            [916, 0.2395215, 0.009080596532247588, 1.0],
+            [77, 0.22927025, 0.010070392939027756, 1.0],
+            [782, 0.20581803, 0.010161745218916036, 1.0],
+            [913, 0.23841085, 0.010782030711612685, 1.0],
+            [910, 0.21493295, 0.014596411069229197, 1.0],
+            [1727, 0.21911663, 0.015168372104237176, 1.0],
+            [1443, 0.19814226, 0.015337080567465522, 1.0],
         ]
         negative_expects = [
-            [956, 0.016060986, 0.0008649321884808977, 1.0],
-            [1124, 0.96602094, 0.0011717216548271284, 1.0],
-            [1809, 1.1110606, 0.0019304405196777848, 1.0],
-            [1754, 0.5201581, 0.005691734062127954, 1.0],
-            [948, 1.6390722, 0.006622111055981219, 1.0],
-            [1810, 0.78618884, 0.007055917428377063, 1.0],
-            [779, 1.5241305, 0.007202934422407284, 1.0],
-            [576, 0.97873515, 0.008272092578813124, 1.0],
-            [538, 0.89114505, 0.01062259019889307, 1.0],
-            [436, 0.3119122, 0.01127515110543434, 1.0]
+            [956, -0.29662406, 0.0008649321884808977, 1.0],
+            [1124, -0.2607333, 0.0011717216548271284, 1.0],
+            [1809, -0.24854594, 0.0019304405196777848, 1.0],
+            [1754, -0.24683577, 0.005691734062127954, 1.0],
+            [948, -0.18708363, 0.006622111055981219, 1.0],
+            [1810, -0.2172082, 0.007055917428377063, 1.0],
+            [779, -0.21150622, 0.007202934422407284, 1.0],
+            [576, -0.19008157, 0.008272092578813124, 1.0],
+            [538, -0.21803819, 0.01062259019889307, 1.0],
+            [436, -0.2100364, 0.01127515110543434, 1.0],
         ]
 
         self.compare_diffexp_results(results['positive'], positive_expects)

backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py

@@ -196,39 +196,3 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(data["n_rows"], 2638)
         self.assertEqual(data["n_cols"], 3)
         self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
-
-    def test_compute_embedding(self):
-        filter = {"obs": {"index": [[0, 100]]}}
-
-        # Verify that we correctly handle the case where we lack scanpy
-        import unittest.mock
-
-        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-
-        # if we happen to have scanpy, test the full API, else punt
-        import importlib
-
-        scanpy_spec = importlib.util.find_spec("scanpy")
-        if scanpy_spec is None:
-            print("Skipping compute_embedding test as ScanPy not installed")
-            return
-
-        # this feature is unsupported in backed mode, and we expect an error
-        if self.data.data.isbacked:
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-            return
-
-        schema = self.data.compute_embedding("umap", filter)
-
-        self.assertIsInstance(schema["name"], str)
-        name = schema["name"]
-        self.assertEqual(schema["type"], "float32")
-        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
-
-        emb = self.data.data.obsm[f"X_{name}"]
-        self.assertEqual(emb.shape, (2638, 2))
-        self.assertTrue(np.isfinite(emb[0:100]).all())
-        self.assertTrue(np.isnan(emb[100:]).all())

backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py

@@ -46,7 +46,7 @@ class FbsTests(unittest.TestCase):
                 "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
             }
         )
-        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None))
         fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
         self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
 

backend/test/test_server/unit/common/config/__init__.py

@@ -95,17 +95,15 @@ class ConfigTests(unittest.TestCase):
         local_file_csv_directory="null",
         local_file_csv_file="null",
         local_file_csv_gene_sets_file="null",
-        ontology_enabled="false",
-        obo_location="null",
         gene_sets_readonly="false",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
         environment=None,
         aws_secrets_manager_region=None,
         aws_secrets_manager_secrets=[],
+        X_approximate_distribution="auto",
         config_file_name="app_config.yml",
     ):
         random_num = random.randrange(999999)
@@ -148,14 +146,12 @@ class ConfigTests(unittest.TestCase):
             local_file_csv_directory=local_file_csv_directory,
             local_file_csv_file=local_file_csv_file,
             local_file_csv_gene_sets_file=local_file_csv_gene_sets_file,
-            ontology_enabled=ontology_enabled,
-            obo_location=obo_location,
             gene_sets_readonly=gene_sets_readonly,
             embedding_names=embedding_names,
-            enable_reembedding=enable_reembedding,
             enable_difexp=enable_difexp,
             lfc_cutoff=lfc_cutoff,
             top_n=top_n,
+            X_approximate_distribution=X_approximate_distribution,
             config_file_name=f"temp_dataset_config_{random_num}.yml",
         )
         external_config = self.custom_external_config(
@@ -186,14 +182,12 @@ class ConfigTests(unittest.TestCase):
         local_file_csv_directory="null",
         local_file_csv_file="null",
         local_file_csv_gene_sets_file="null",
-        ontology_enabled="false",
-        obo_location="null",
         gene_sets_readonly="false",
         embedding_names=[],
-        enable_reembedding="false",
         enable_difexp="true",
         lfc_cutoff=0.01,
         top_n=10,
+        X_approximate_distribution="auto",
         config_file_name="dataset_config.yml",
     ):
         configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)

backend/test/test_server/unit/common/config/test_app_config.py

@@ -131,9 +131,8 @@ class AppConfigTest(ConfigTests):
 
         # test simple value in default dataset
         config.update_single_config_from_path_and_value(
-            ["dataset", "user_annotations", "ontology", "obo_location"], "dummy_location",
+            ["dataset", "user_annotations"], "dummy_location",
         )
-        self.assertEqual(config.dataset_config.user_annotations__ontology__obo_location, "dummy_location")
 
         # error checking
         bad_paths = [

backend/test/test_server/unit/common/config/test_base_config.py

@@ -36,7 +36,6 @@ class BaseConfigTest(ConfigTests):
         mapping = config.dataset_config.create_mapping(config.default_config)
         self.assertIsNotNone(mapping["server__app__verbose"])
         self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
-        self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
 
     def test_changes_from_default_returns_list_of_nondefault_config_values(self):
         config = self.get_config(verbose="true", lfc_cutoff=0.05)

backend/test/test_server/unit/common/config/test_dataset_config.py

@@ -7,7 +7,7 @@ from unittest.mock import patch
 from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
 from backend.server.common.config.app_config import AppConfig
 from backend.server.common.config.base_config import BaseConfig
-from backend.test import FIXTURES_ROOT, H5AD_FIXTURE
+from backend.test import H5AD_FIXTURE
 
 from backend.common.errors import ConfigurationError
 from backend.test.test_server.unit.common.config import ConfigTests
@@ -40,14 +40,13 @@ class TestDatasetConfig(ConfigTests):
         self.assertEqual(config.dataset_config.presentation__max_categories, 1000)
         self.assertEqual(config.dataset_config.user_annotations__type, "local_file_csv")
         self.assertEqual(config.dataset_config.diffexp__lfc_cutoff, 0.01)
-        self.assertIsNone(config.dataset_config.user_annotations__ontology__obo_location)
 
     @patch("backend.server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
     def test_complete_config_checks_all_attr(self, mock_check_attrs):
         mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
         self.dataset_config.complete_config(self.context)
         self.assertIsNotNone(self.config.server_config.data_adaptor)
-        self.assertEqual(mock_check_attrs.call_count, 19)
+        self.assertEqual(mock_check_attrs.call_count, 17)
 
     def test_app_sets_script_vars(self):
         config = self.get_config(scripts=["path/to/script"])
@@ -107,20 +106,6 @@ class TestDatasetConfig(ConfigTests):
         cwd = os.getcwd()
         self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)
 
-    def test_handle_embeddings__checks_data_file_types(self):
-        file_name = self.custom_app_config(
-            embedding_names=["name1", "name2"],
-            enable_reembedding="true",
-            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
-            anndata_backed="true",
-            config_file_name=self.config_file_name,
-        )
-        config = AppConfig()
-        config.update_from_config_file(file_name)
-        config.server_config.complete_config(self.context)
-        with self.assertRaises(ConfigurationError):
-            config.dataset_config.handle_embeddings()
-
     def test_handle_diffexp__raises_warning_for_large_datasets(self):
         config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
         config.server_config.complete_config(self.context)

backend/test/test_server/unit/common/test_api.py

@@ -6,9 +6,13 @@ from http import HTTPStatus
 import tempfile
 from os import path
 import hashlib
+from os.path import basename, splitext
 
 import pandas as pd
 import requests
+import numpy as np
+
+from parameterized import parameterized_class
 
 import backend.test.decode_fbs as decode_fbs
 from backend.server.data_common.matrix_loader import MatrixDataType
@@ -44,6 +48,14 @@ class EndPoints(object):
             len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5
         )
 
+        # Check that all schema types are legal
+        legal_types = ["boolean", "string", "categorical", "float32", "int32"]
+        self.assertEqual(result_data["schema"]["dataframe"]["type"], "float32")
+        for column in result_data["schema"]["annotations"]["obs"]["columns"]:
+            self.assertIn(column["type"], legal_types)
+        for column in result_data["schema"]["annotations"]["var"]["columns"]:
+            self.assertIn(column["type"], legal_types)
+
     def test_config(self):
         endpoint = "config"
         url = f"{self.URL_BASE}{endpoint}"
@@ -52,7 +64,12 @@ class EndPoints(object):
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertIn("library_versions", result_data["config"])
-        self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
+
+        if hasattr(self, "data_locator"):
+            title = splitext(basename(self.data_locator))[0]
+        else:
+            title = "pbmc3k"
+        self.assertEqual(result_data["config"]["displayNames"]["dataset"], title)
         self.assertIsNotNone(result_data["config"]["parameters"])
 
     def test_get_layout_fbs(self):
@@ -72,34 +89,8 @@ class EndPoints(object):
         )
         self.assertIsNone(df["row_idx"])
         self.assertEqual(len(df["columns"]), df["n_cols"])
-
-    def test_put_layout_fbs(self):
-        # first check that re-embedding is turned on
-        result = self.session.get(f"{self.URL_BASE}config")
-        config_data = result.json()
-        re_embed = config_data["config"]["parameters"]["enable-reembedding"]
-        if not re_embed:
-            return
-        # attempt to reembed with umap over 100 cells.
-        endpoint = "layout/obs"
-        url = f"{self.URL_BASE}{endpoint}"
-        data = {}
-        data["filter"] = {}
-        data["filter"]["obs"] = {}
-        data["filter"]["obs"]["index"] = list(range(100))
-        data["method"] = "umap"
-        result = self.session.put(url, json=data)
-
-        self.assertEqual(result.status_code, HTTPStatus.OK)
-        result_data = result.json()
-        self.assertIsInstance(result_data, dict)
-        self.assertEqual(result_data["type"], "float32")
-        self.assertTrue(result_data["name"].startswith("reembed:umap_"))
-        self.assertIsInstance(result_data["dims"], list)
-        self.assertEqual(len(result_data["dims"]), 2)
-        dims = result_data["dims"]
-        self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
-        self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
+        for column in df["columns"]:
+            self.assertEqual(column.dtype, np.float32)
 
     def test_bad_filter(self):
         endpoint = "data/var"
@@ -126,6 +117,9 @@ class EndPoints(object):
             [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
             + (["cluster-test"] if self.ANNOTATIONS_ENABLED else []),
         )
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
 
     def test_get_annotations_obs_keys_fbs(self):
         endpoint = "annotations/obs"
@@ -165,6 +159,9 @@ class EndPoints(object):
         self.assertEqual(len(df["columns"]), df["n_cols"])
         var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
         self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
 
     def test_get_annotations_var_keys_fbs(self):
         endpoint = "annotations/var"
@@ -235,6 +232,9 @@ class EndPoints(object):
         self.assertIsNone(df["row_idx"])
         self.assertEqual(len(df["columns"]), df["n_cols"])
         self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
 
     def test_data_get_filter_fbs(self):
         index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
@@ -248,6 +248,9 @@ class EndPoints(object):
         df = decode_fbs.decode_matrix_FBS(result.content)
         self.assertEqual(df["n_rows"], 2638)
         self.assertEqual(df["n_cols"], 1)
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
 
     def test_data_get_unknown_filter_fbs(self):
         index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
@@ -274,6 +277,9 @@ class EndPoints(object):
         df = decode_fbs.decode_matrix_FBS(result.content)
         self.assertEqual(df["n_rows"], 2638)
         self.assertEqual(df["n_cols"], 1)
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
 
     def test_colors(self):
         endpoint = "colors"
@@ -375,6 +381,14 @@ class EndPointsAnnotations(EndPoints):
         self.assertTrue(matching_columns[0]["writable"])
 
 
+@parameterized_class(
+    [
+        {"data_locator": f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"},
+        {"data_locator": f"{FIXTURES_ROOT}/pbmc3k_64.h5ad"},
+        {"data_locator": f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"},
+        {"data_locator": f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad"},
+    ]
+)
 class EndPointsAnndata(unittest.TestCase, EndPoints):
     """Test Case for endpoints"""
 
@@ -383,13 +397,15 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
 
     @classmethod
     def setUpClass(cls):
+        if cls == EndPointsAnndata:
+            raise unittest.SkipTest("`parameterized_class` bug")
+
         cls._setupClass(
             cls,
             [
-                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+                cls.data_locator,
                 "--disable-annotations",
                 "--disable-gene-sets-save",
-                "--experimental-enable-reembedding",
             ],
         )
 
@@ -414,8 +430,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
-        self.assertEqual(len(result_data['positive']), 7)
-        self.assertEqual(len(result_data['negative']), 7)
+        self.assertEqual(len(result_data["positive"]), 7)
+        self.assertEqual(len(result_data["negative"]), 7)
 
     def test_diff_exp_indices(self):
         endpoint = "diffexp/obs"
@@ -430,8 +446,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
-        self.assertEqual(len(result_data['positive']), 10)
-        self.assertEqual(len(result_data['negative']), 10)
+        self.assertEqual(len(result_data["positive"]), 10)
+        self.assertEqual(len(result_data["negative"]), 10)
 
     def test_get_summaryvar(self):
         index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
@@ -590,23 +606,23 @@ class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
                         "geneset_description": "",
                         "geneset_name": "summary test",
                     },
-                    {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_delete'},
-                    {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_edit'},
+                    {"genes": [], "geneset_description": "", "geneset_name": "geneset_to_delete"},
+                    {"genes": [], "geneset_description": "", "geneset_name": "geneset_to_edit"},
                     {
-                        'genes': [{'gene_description': '', 'gene_symbol': 'RER1'}],
-                        'geneset_description': '',
-                        'geneset_name': 'fill_this_geneset'
+                        "genes": [],
+                        "geneset_description": "",
+                        "geneset_name": "fill_this_geneset",
                     },
                     {
-                        'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
-                        'geneset_description': '',
-                        'geneset_name': 'empty_this_geneset'
+                        "genes": [{"gene_description": "", "gene_symbol": "SIK1"}],
+                        "geneset_description": "",
+                        "geneset_name": "empty_this_geneset",
                     },
                     {
-                        'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
-                        'geneset_description': '',
-                        'geneset_name': 'brush_this_gene'
-                    }
+                        "genes": [{"gene_description": "", "gene_symbol": "SIK1"}],
+                        "geneset_description": "",
+                        "geneset_name": "brush_this_gene",
+                    },
                 ],
                 "tid": 0,
             },
@@ -635,7 +651,7 @@ summary test,,F5,\r
 summary test,,PIGU,\r
 geneset_to_delete,,,\r
 geneset_to_edit,,,\r
-fill_this_geneset,,RER1,\r
+fill_this_geneset,,,\r
 empty_this_geneset,,SIK1,\r
 brush_this_gene,,SIK1,\r
 """,
@@ -709,7 +725,7 @@ brush_this_gene,,SIK1,\r
         self.assertEqual(result.json(), test3)
 
     def test_put_genesets_malformed(self):
-        """ test malformed submissions that we expect the backend to catch/tolerate """
+        """test malformed submissions that we expect the backend to catch/tolerate"""
         endpoint = "genesets"
         url = f"{self.URL_BASE}{endpoint}"
 
@@ -719,7 +735,7 @@ brush_this_gene,,SIK1,\r
         tid = original_data["tid"]
 
         def test_case(test, expected_code, original_data):
-            """ check for expected error AND that no change was made to the original state """
+            """check for expected error AND that no change was made to the original state"""
             result = self.session.put(url, json=test)
             self.assertEqual(result.status_code, expected_code)
             result = self.session.get(url, headers={"Accept": "application/json"})

backend/test/test_server/unit/compute/test_diffexp_h5ad.py

@@ -38,28 +38,28 @@ class DiffExpTest(unittest.TestCase):
         """Checks the results for a specific set of rows selections"""
 
         positive_expects = [
-            [1712, -0.5525154, 0.0051788902660723345, 1.0],
-            [1575, 1.0317602, 0.007830310753043345, 1.0],
-            [693, 0.4703904, 0.008715846769131548, 1.0],
-            [916, 0.9567287, 0.009080596532247588, 1.0],
-            [77, 0.02665649, 0.010070392939027756, 1.0],
-            [782, -1.0981874, 0.010161745218916036, 1.0],
-            [913, 0.5683986, 0.010782030711612685, 1.0],
-            [910, 0.83164597, 0.014596411069229197, 1.0],
-            [1727, 0.4127781, 0.015168372104237176, 1.0],
-            [1443, -0.8241895, 0.015337080567465522, 1.0]
+            [1712, 0.24104056, 0.0051788902660723345, 1.0],
+            [1575, 0.2615018, 0.007830310753043345, 1.0],
+            [693, 0.23106655, 0.008715846769131548, 1.0],
+            [916, 0.2395215, 0.009080596532247588, 1.0],
+            [77, 0.22927025, 0.010070392939027756, 1.0],
+            [782, 0.20581803, 0.010161745218916036, 1.0],
+            [913, 0.23841085, 0.010782030711612685, 1.0],
+            [910, 0.21493295, 0.014596411069229197, 1.0],
+            [1727, 0.21911663, 0.015168372104237176, 1.0],
+            [1443, 0.19814226, 0.015337080567465522, 1.0],
         ]
         negative_expects = [
-            [956, 0.016060986, 0.0008649321884808977, 1.0],
-            [1124, 0.96602094, 0.0011717216548271284, 1.0],
-            [1809, 1.1110606, 0.0019304405196777848, 1.0],
-            [1754, 0.5201581, 0.005691734062127954, 1.0],
-            [948, 1.6390722, 0.006622111055981219, 1.0],
-            [1810, 0.78618884, 0.007055917428377063, 1.0],
-            [779, 1.5241305, 0.007202934422407284, 1.0],
-            [576, 0.97873515, 0.008272092578813124, 1.0],
-            [538, 0.89114505, 0.01062259019889307, 1.0],
-            [436, 0.3119122, 0.01127515110543434, 1.0]
+            [956, -0.29662406, 0.0008649321884808977, 1.0],
+            [1124, -0.2607333, 0.0011717216548271284, 1.0],
+            [1809, -0.24854594, 0.0019304405196777848, 1.0],
+            [1754, -0.24683577, 0.005691734062127954, 1.0],
+            [948, -0.18708363, 0.006622111055981219, 1.0],
+            [1810, -0.2172082, 0.007055917428377063, 1.0],
+            [779, -0.21150622, 0.007202934422407284, 1.0],
+            [576, -0.19008157, 0.008272092578813124, 1.0],
+            [538, -0.21803819, 0.01062259019889307, 1.0],
+            [436, -0.2100364, 0.01127515110543434, 1.0],
         ]
 
         self.compare_diffexp_results(results["positive"], positive_expects)

backend/test/test_server/unit/compute/test_est_dist.py

@@ -0,0 +1,120 @@
+import unittest
+import numpy as np
+from scipy import sparse
+from backend.common.compute.estimate_distribution import estimate_approximate_distribution
+from backend.common.constants import XApproximateDistribution
+from backend.server.data_common.matrix_loader import MatrixDataLoader
+from backend.test.test_server.unit import app_config
+from backend.test import PROJECT_ROOT
+
+
+class EstDistTest(unittest.TestCase):
+    """Tests the diffexp returns the expected results for one test case, using the h5ad
+    adaptor types and different algorithms."""
+
+    def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
+        loader = MatrixDataLoader(path)
+        adaptor = loader.open(config)
+        return adaptor
+
+    def test_adaptestimate_approximate_distribution(self):
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        self.assertEqual(adaptor.get_X_approximate_distribution(), XApproximateDistribution.NORMAL)
+
+    def test_estimate_approximate_distribution(self):
+        raw = np.random.exponential(scale=1000, size=(100, 40))
+
+        # empty
+        self.assertEqual(estimate_approximate_distribution(np.zeros((0,))), XApproximateDistribution.NORMAL)
+
+        # ndarray
+        self.assertEqual(estimate_approximate_distribution(raw), XApproximateDistribution.COUNT)
+        self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproximateDistribution.NORMAL)
+
+        # csr_matrix
+        self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproximateDistribution.COUNT)
+        self.assertEqual(
+            estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproximateDistribution.NORMAL
+        )
+
+        # csc_matrix
+        self.assertEqual(estimate_approximate_distribution(sparse.csc_matrix(raw)), XApproximateDistribution.COUNT)
+        self.assertEqual(
+            estimate_approximate_distribution(sparse.csc_matrix(np.log1p(raw))), XApproximateDistribution.NORMAL
+        )
+
+        # BIG (ie, trigger MT)
+        big = np.random.exponential(scale=100, size=(1_000_000, 100))
+        self.assertEqual(estimate_approximate_distribution(big), XApproximateDistribution.COUNT)
+        self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproximateDistribution.NORMAL)
+
+    def test_unsupported_throws(self):
+        # dtypes and matrix formats we do not support
+        with self.assertRaises(TypeError):
+            estimate_approximate_distribution(np.array(["a", "b"]))
+        with self.assertRaises(TypeError):
+            estimate_approximate_distribution(sparse.coo_matrix(np.array([[0, 1, 2], [3, 0, 2]])))
+
+    def test_nonfinites(self):
+        def put(arr, ind, vals):
+            # like np.put, but creates and returns a modified copy of original array
+            a = arr.copy()
+            np.put(a, ind, vals)
+            return a
+
+        # non-finites
+        self.assertEqual(estimate_approximate_distribution(np.array([np.nan])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.PINF])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.NINF])), XApproximateDistribution.NORMAL)
+        self.assertEqual(
+            estimate_approximate_distribution(np.array([np.PINF, np.NINF, 0])), XApproximateDistribution.NORMAL
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(np.array([np.nan, np.PINF, np.NINF])), XApproximateDistribution.NORMAL
+        )
+
+        raw = np.random.exponential(scale=1000, size=(50, 3))
+        logged = np.log1p(raw)
+
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [1], [np.nan])),
+            XApproximateDistribution.COUNT,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [1], [np.PINF])),
+            XApproximateDistribution.COUNT,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [1], [np.NINF])),
+            XApproximateDistribution.COUNT,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
+            XApproximateDistribution.COUNT,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [0, 1], [np.nan, np.nan])),
+            XApproximateDistribution.COUNT,
+        )
+
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [1], [np.nan])),
+            XApproximateDistribution.NORMAL,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [1], [np.PINF])),
+            XApproximateDistribution.NORMAL,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [1], [np.NINF])),
+            XApproximateDistribution.NORMAL,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
+            XApproximateDistribution.NORMAL,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [0, 1], [np.nan, np.nan])),
+            XApproximateDistribution.NORMAL,
+        )

backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py

@@ -22,19 +22,30 @@ Test the anndata adaptor using the pbmc3k data set.
 
 
 @parameterized_class(
-    ("data_locator", "backed"),
+    ("data_locator", "backed", "X_approximate_distribution"),
     [
-        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
-        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False),
-        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False),
-        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
-        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True),
-        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "auto"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "auto"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False, "auto"),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "auto"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "auto"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "auto"),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False, "normal"),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k_64.h5ad", False, "auto"),  # 64 bit conversion tests
     ],
 )
 class AdaptorTest(unittest.TestCase):
     def setUp(self):
-        config = app_config(self.data_locator, self.backed)
+        config = app_config(
+            self.data_locator,
+            self.backed,
+            extra_dataset_config=dict(X_approximate_distribution=self.X_approximate_distribution),
+        )
         self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
 
     def test_init(self):
@@ -90,7 +101,8 @@ class AdaptorTest(unittest.TestCase):
 
     def test_schema_produces_error(self):
         self.data.data.obs["time"] = pd.Series(
-            list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
+            list([time.time() for i in range(self.data.cell_count)]),
+            dtype="datetime64[ns]",
         )
         with pytest.raises(TypeError):
             self.data._create_schema()
@@ -107,7 +119,7 @@ class AdaptorTest(unittest.TestCase):
         self.assertTrue((Y >= 0).all() and (Y <= 1).all())
 
     def test_layout_fields(self):
-        """ X_pca, X_tsne, X_umap are available """
+        """X_pca, X_tsne, X_umap are available"""
         fbs = self.data.layout_to_fbs_matrix(["pca"])
         layout = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(layout["n_cols"], 2)
@@ -127,7 +139,8 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(annotations["n_cols"], 5)
         obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
         self.assertEqual(
-            annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
+            annotations["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
         )
 
         fbs = self.data.annotation_to_fbs_matrix("var")
@@ -153,12 +166,12 @@ class AdaptorTest(unittest.TestCase):
         f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
         f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
-        self.assertEqual(len(result['positive']), 10)
-        self.assertEqual(len(result['negative']), 10)
+        self.assertEqual(len(result["positive"]), 10)
+        self.assertEqual(len(result["negative"]), 10)
 
         result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
-        self.assertEqual(len(result['positive']), 20)
-        self.assertEqual(len(result['negative']), 20)
+        self.assertEqual(len(result["positive"]), 20)
+        self.assertEqual(len(result["negative"]), 20)
 
     def test_data_frame(self):
         f1 = {"var": {"index": [[0, 10]]}}
@@ -198,39 +211,3 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(data["n_rows"], 2638)
         self.assertEqual(data["n_cols"], 3)
         self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
-
-    def test_compute_embedding(self):
-        filter = {"obs": {"index": [[0, 100]]}}
-
-        # Verify that we correctly handle the case where we lack scanpy
-        import unittest.mock
-
-        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-
-        # if we happen to have scanpy, test the full API, else punt
-        import importlib
-
-        scanpy_spec = importlib.util.find_spec("scanpy")
-        if scanpy_spec is None:
-            print("Skipping compute_embedding test as ScanPy not installed")
-            return
-
-        # this feature is unsupported in backed mode, and we expect an error
-        if self.data.data.isbacked:
-            with self.assertRaises(NotImplementedError):
-                self.data.compute_embedding("umap", filter)
-            return
-
-        schema = self.data.compute_embedding("umap", filter)
-
-        self.assertIsInstance(schema["name"], str)
-        name = schema["name"]
-        self.assertEqual(schema["type"], "float32")
-        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
-
-        emb = self.data.data.obsm[f"X_{name}"]
-        self.assertEqual(emb.shape, (2638, 2))
-        self.assertTrue(np.isfinite(emb[0:100]).all())
-        self.assertTrue(np.isnan(emb[100:]).all())

client/Makefile

@@ -1,11 +1,13 @@
 include ../common.mk
 
 ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../backend/test/fixtures/pbmc3k-annotations.csv)
-GENE_SETS := $(if $(GENE_SETS),$(GENE_SETS),../backend/test/fixtures/pbmc3k-genesets.csv) 
+GENE_SETS := $(if $(GENE_SETS),$(GENE_SETS),../backend/test/fixtures/pbmc3k-genesets.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
 GENE_SETS_FILENAME := $(shell basename $(GENE_SETS))
 
-CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)
+CXG_CONFIG := $(if $(CXG_CONFIG),$(CXG_CONFIG),./__tests__/e2e/test_config.yaml)
+
+CXG_AUTH_TYPE := $(if $(CXG_AUTH_TYPE),$(CXG_AUTH_TYPE),"test")
 
 
 # Packaging
@@ -38,7 +40,7 @@ smoke-test:
 	start_server_and_test \
 		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
-		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false'
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE=$(CXG_AUTH_TYPE) npm run e2e -- --verbose false'
 
 # start an instance of cellxgene and run the end-to-end annotations tests
 .PHONY: smoke-test-annotations

client/__tests__/e2e/__snapshots__/e2e.test.ts.snap

@@ -2,4 +2,4 @@
 
 exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
-exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;

client/__tests__/e2e/__snapshots__/e2eAnnotations.test.ts.snap

@@ -2,15 +2,15 @@
 
 exports[`annotations stacked bar graph renders 1`] = `
 Array [
-  "<div class=\\"categorical__value___2m6V7\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover2-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" tabindex=\\"-1\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 15px; height: 15px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
-  "<div class=\\"categorical__value___2m6V7\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover2-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" tabindex=\\"-1\\" aria-label=\\"unassigned\\" class=\\"\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 15px; height: 15px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2m6V7\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover2-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" tabindex=\\"-1\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 15px; height: 15px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2m6V7\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover2-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" tabindex=\\"-1\\" aria-label=\\"unassigned\\" class=\\"\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 15px; height: 15px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
 ]
 `;
 
 exports[`annotations stacked bar graph renders 2`] = `
 Array [
-  "<div class=\\"categorical__value___2m6V7\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover2-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" tabindex=\\"-1\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 15px; height: 15px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
-  "<div class=\\"categorical__value___2m6V7\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover2-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" tabindex=\\"-1\\" aria-label=\\"unassigned\\" class=\\"\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 15px; height: 15px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2m6V7\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover2-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" tabindex=\\"-1\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 15px; height: 15px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2m6V7\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover2-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" tabindex=\\"-1\\" aria-label=\\"unassigned\\" class=\\"\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 15px; height: 15px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
 ]
 `;
 

client/__tests__/e2e/cellxgeneActions.js

@@ -1,516 +0,0 @@
-/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
-import { strict as assert } from "assert";
-import {
-  clearInputAndTypeInto,
-  clickOn,
-  getAllByClass,
-  getOneElementInnerText,
-  typeInto,
-  waitByID,
-  waitByClass,
-  waitForAllByIds,
-  clickOnUntil,
-  getTestClass,
-  getTestId,
-  isElementPresent,
-  goToPage,
-} from "./puppeteerUtils";
-
-import { appUrlBase, TEST_EMAIL, TEST_PASSWORD } from "./config";
-
-export async function drag(testId, start, end, lasso = false) {
-  const layout = await waitByID(testId);
-  const elBox = await layout.boxModel();
-  const x1 = elBox.content[0].x + start.x;
-  const x2 = elBox.content[0].x + end.x;
-  const y1 = elBox.content[0].y + start.y;
-  const y2 = elBox.content[0].y + end.y;
-  await page.mouse.move(x1, y1);
-  await page.mouse.down();
-  if (lasso) {
-    await page.mouse.move(x2, y1);
-    await page.mouse.move(x2, y2);
-    await page.mouse.move(x1, y2);
-    await page.mouse.move(x1, y1);
-  } else {
-    await page.mouse.move(x2, y2);
-  }
-  await page.mouse.up();
-}
-
-export async function clickOnCoordinate(testId, coord) {
-  const layout = await expect(page).toMatchElement(getTestId(testId));
-  const elBox = await layout.boxModel();
-
-  if (!elBox) {
-    throw Error("Layout's boxModel is not available!");
-  }
-
-  const x = elBox.content[0].x + coord.x;
-  const y = elBox.content[0].y + coord.y;
-  await page.mouse.click(x, y);
-}
-
-export async function getAllHistograms(testclass, testIds) {
-  const histTestIds = testIds.map((tid) => `histogram-${tid}`);
-
-  // these load asynchronously, so we need to wait for each histogram individually,
-  // and they may be quite slow in some cases.
-  await waitForAllByIds(histTestIds, { timeout: 4 * 60 * 1000 });
-
-  const allHistograms = await getAllByClass(testclass);
-
-  const testIDs = await Promise.all(
-    allHistograms.map((hist) => {
-      return page.evaluate((elem) => {
-        return elem.dataset.testid;
-      }, hist);
-    })
-  );
-
-  return testIDs.map((id) => id.replace(/^histogram-/, ""));
-}
-
-export async function getAllCategoriesAndCounts(category) {
-  // these load asynchronously, so we have to wait for the specific category.
-  await waitByID(`category-${category}`);
-
-  return page.$$eval(
-    `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
-    (rows) =>
-      Object.fromEntries(
-        rows.map((row) => {
-          const cat = row
-            .querySelector("[data-testclass='categorical-value']")
-            .getAttribute("aria-label");
-
-          const count = row.querySelector(
-            "[data-testclass='categorical-value-count']"
-          ).innerText;
-
-          return [cat, count];
-        })
-      )
-  );
-}
-
-export async function getCellSetCount(num) {
-  await clickOn(`cellset-button-${num}`);
-  return getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
-}
-
-export async function resetCategory(category) {
-  const checkboxId = `${category}:category-select`;
-  await waitByID(checkboxId);
-  const checkedPseudoclass = await page.$eval(
-    `[data-testid='${checkboxId}']`,
-    (el) => el.matches(":checked")
-  );
-  if (!checkedPseudoclass) await clickOn(checkboxId);
-
-  const categoryRow = await waitByID(`${category}:category-expand`);
-
-  const isExpanded = await categoryRow.$(
-    "[data-testclass='category-expand-is-expanded']"
-  );
-
-  if (isExpanded) await clickOn(`${category}:category-expand`);
-}
-
-export async function calcCoordinate(testId, xAsPercent, yAsPercent) {
-  const el = await waitByID(testId);
-  const size = await el.boxModel();
-  return {
-    x: Math.floor(size.width * xAsPercent),
-    y: Math.floor(size.height * yAsPercent),
-  };
-}
-
-export async function calcDragCoordinates(testId, coordinateAsPercent) {
-  return {
-    start: await calcCoordinate(
-      testId,
-      coordinateAsPercent.x1,
-      coordinateAsPercent.y1
-    ),
-    end: await calcCoordinate(
-      testId,
-      coordinateAsPercent.x2,
-      coordinateAsPercent.y2
-    ),
-  };
-}
-
-export async function selectCategory(category, values, reset = true) {
-  if (reset) await resetCategory(category);
-
-  await clickOn(`${category}:category-expand`);
-  await clickOn(`${category}:category-select`);
-
-  for (const value of values) {
-    await clickOn(`categorical-value-select-${category}-${value}`);
-  }
-}
-
-export async function expandCategory(category) {
-  const expand = await waitByID(`${category}:category-expand`);
-  const notExpanded = await expand.$(
-    "[data-testclass='category-expand-is-not-expanded']"
-  );
-  if (notExpanded) await clickOn(`${category}:category-expand`);
-}
-
-export async function clip(min = 0, max = 100) {
-  await clickOn("visualization-settings");
-  await clearInputAndTypeInto("clip-min-input", min);
-  await clearInputAndTypeInto("clip-max-input", max);
-  await clickOn("clip-commit");
-}
-
-export async function createCategory(categoryName) {
-  await clickOnUntil("open-annotation-dialog", async () => {
-    await expect(page).toMatchElement(getTestId("new-category-name"));
-  });
-
-  await typeInto("new-category-name", categoryName);
-  await clickOn("submit-category");
-}
-
-/* 
-
-  GENESET 
-
-*/
-
-export async function colorByGeneset(genesetName) {
-  await clickOn(`${genesetName}:colorby-entire-geneset`);
-}
-
-export async function colorByGene(gene) {
-  await clickOn(`colorby-${gene}`);
-}
-
-export async function assertColorLegendLabel(label) {
-  const handle = await waitByID("continuous_legend_color_by_label");
-
-  const result = await handle.evaluate((node) => {
-    return node.getAttribute("aria-label");
-  });
-
-  return expect(result).toBe(label);
-}
-
-export async function expandGeneset(genesetName) {
-  const expand = await waitByID(`${genesetName}:geneset-expand`);
-  const notExpanded = await expand.$(
-    "[data-testclass='geneset-expand-is-not-expanded']"
-  );
-  if (notExpanded) await clickOn(`${genesetName}:geneset-expand`);
-}
-
-export async function createGeneset(genesetName) {
-  await clickOnUntil("open-create-geneset-dialog", async () => {
-    await expect(page).toMatchElement(getTestId("create-geneset-input"));
-  });
-
-  await typeInto("create-geneset-input", genesetName);
-  await clickOn("submit-geneset");
-  await waitByClass("autosave-complete");
-}
-
-export async function editGenesetName(genesetName, editText) {
-  const editButton = `${genesetName}:edit-genesetName-mode`;
-  const submitButton = `${genesetName}:submit-geneset`;
-  await clickOnUntil(`${genesetName}:see-actions`, async () => {
-    await expect(page).toMatchElement(getTestId(editButton));
-  });
-  await clickOn(editButton);
-  await typeInto("rename-geneset-modal", editText);
-  await clickOn(submitButton);
-}
-
-export async function deleteGeneset(genesetName) {
-  const targetId = `${genesetName}:delete-geneset`;
-
-  await clickOnUntil(`${genesetName}:see-actions`, async () => {
-    await expect(page).toMatchElement(getTestId(targetId));
-  });
-
-  await clickOn(targetId);
-
-  await assertGenesetDoesNotExist(genesetName);
-  await waitByClass("autosave-complete");
-}
-
-export async function assertGenesetDoesNotExist(genesetName) {
-  const result = await isElementPresent(
-    getTestId(`${genesetName}:geneset-name`)
-  );
-  await expect(result).toBe(false);
-}
-
-export async function assertGenesetExists(genesetName) {
-  const handle = await waitByID(`${genesetName}:geneset-name`);
-
-  const result = await handle.evaluate((node) => {
-    return node.getAttribute("aria-label");
-  });
-
-  return expect(result).toBe(genesetName);
-}
-
-/* 
-
-  GENE
-
-*/
-
-export async function addGeneToSet(genesetName, geneToAddToSet) {
-  const submitButton = `${genesetName}:submit-gene`;
-
-  await clickOn(`${genesetName}:add-new-gene-to-geneset`);
-  await typeInto("add-genes", geneToAddToSet);
-  await clickOn(submitButton);
-}
-
-export async function removeGene(geneSymbol) {
-  const targetId = `delete-from-geneset:${geneSymbol}`;
-
-  await clickOn(targetId);
-
-  await waitByClass("autosave-complete");
-}
-
-export async function assertGeneExistsInGeneset(geneSymbol) {
-  const handle = await waitByID(`${geneSymbol}:gene-label`);
-
-  const result = await handle.evaluate((node) => {
-    return node.getAttribute("aria-label");
-  });
-
-  return expect(result).toBe(geneSymbol);
-}
-
-export async function assertGeneDoesNotExist(geneSymbol) {
-  const result = await isElementPresent(getTestId(`${geneSymbol}:gene-label`));
-
-  await expect(result).toBe(false);
-}
-
-export async function expandGene(geneSymbol) {
-  await clickOn(`maximize-${geneSymbol}`);
-}
-
-/* 
-
-  CATEGORY 
-
-*/
-
-export async function duplicateCategory(categoryName) {
-  await clickOn("open-annotation-dialog");
-
-  await typeInto("new-category-name", categoryName);
-
-  const dropdownOptionClass = "duplicate-category-dropdown-option";
-
-  await clickOnUntil("duplicate-category-dropdown", async () => {
-    await expect(page).toMatchElement(getTestClass(dropdownOptionClass));
-  });
-
-  const option = await expect(page).toMatchElement(
-    getTestClass(dropdownOptionClass)
-  );
-
-  await option.click();
-
-  await clickOnUntil("submit-category", async () => {
-    await expect(page).toMatchElement(
-      getTestId(`${categoryName}:category-expand`)
-    );
-  });
-
-  await waitByClass("autosave-complete");
-}
-
-export async function renameCategory(oldCategoryName, newCategoryName) {
-  await clickOn(`${oldCategoryName}:see-actions`);
-  await clickOn(`${oldCategoryName}:edit-category-mode`);
-  await clearInputAndTypeInto(
-    `${oldCategoryName}:edit-category-name-text`,
-    newCategoryName
-  );
-  await clickOn(`${oldCategoryName}:submit-category-edit`);
-}
-
-export async function deleteCategory(categoryName) {
-  const targetId = `${categoryName}:delete-category`;
-
-  await clickOnUntil(`${categoryName}:see-actions`, async () => {
-    await expect(page).toMatchElement(getTestId(targetId));
-  });
-
-  await clickOn(targetId);
-
-  await assertCategoryDoesNotExist();
-}
-
-export async function createLabel(categoryName, labelName) {
-  /**
-   * (thuang): This explicit wait is needed, since currently showing
-   * the modal again quickly after the previous action dismissing the
-   * modal will persist the input value from the previous action.
-   *
-   * To reproduce:
-   * 1. Click on the plus sign to show the modal to add a new label to the category
-   * 2. Type `123` in the input box
-   * 3. Hover over your mouse over the plus sign and double click to quickly dismiss and
-   * invoke the modal again
-   * 4. You will see `123` is persisted in the input box
-   * 5. Expected behavior is to get an empty input box
-   */
-  await page.waitForTimeout(500);
-
-  await clickOn(`${categoryName}:see-actions`);
-
-  await clickOn(`${categoryName}:add-new-label-to-category`);
-
-  await typeInto(`${categoryName}:new-label-name`, labelName);
-
-  await clickOn(`${categoryName}:submit-label`);
-}
-
-export async function deleteLabel(categoryName, labelName) {
-  await expandCategory(categoryName);
-  await clickOn(`${categoryName}:${labelName}:see-actions`);
-  await clickOn(`${categoryName}:${labelName}:delete-label`);
-}
-
-export async function renameLabel(categoryName, oldLabelName, newLabelName) {
-  await expandCategory(categoryName);
-  await clickOn(`${categoryName}:${oldLabelName}:see-actions`);
-  await clickOn(`${categoryName}:${oldLabelName}:edit-label`);
-  await clearInputAndTypeInto(
-    `${categoryName}:${oldLabelName}:edit-label-name`,
-    newLabelName
-  );
-  await clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
-}
-
-export async function addGeneToSearch(geneName) {
-  await typeInto("gene-search", geneName);
-  await page.keyboard.press("Enter");
-  await page.waitForSelector(`[data-testid='histogram-${geneName}']`);
-}
-
-export async function subset(coordinatesAsPercent) {
-  // In order to deselect the selection after the subset, make sure we have some clear part
-  // of the scatterplot we can click on
-  assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
-  const lassoSelection = await calcDragCoordinates(
-    "layout-graph",
-    coordinatesAsPercent
-  );
-  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
-  await clickOn("subset-button");
-  const clearCoordinate = await calcCoordinate("layout-graph", 0.5, 0.99);
-  await clickOnCoordinate("layout-graph", clearCoordinate);
-}
-
-export async function setSellSet(cellSet, cellSetNum) {
-  const selections = cellSet.filter((sel) => sel.kind === "categorical");
-
-  for (const selection of selections) {
-    await selectCategory(selection.metadata, selection.values, true);
-  }
-
-  await getCellSetCount(cellSetNum);
-}
-
-export async function runDiffExp(cellSet1, cellSet2) {
-  await setSellSet(cellSet1, 1);
-  await setSellSet(cellSet2, 2);
-  await clickOn("diffexp-button");
-}
-
-export async function bulkAddGenes(geneNames) {
-  await clickOn("section-bulk-add");
-  await typeInto("input-bulk-add", geneNames.join(","));
-  await page.keyboard.press("Enter");
-}
-
-export async function assertCategoryDoesNotExist(categoryName) {
-  const result = await isElementPresent(
-    getTestId(`${categoryName}:category-label`)
-  );
-
-  await expect(result).toBe(false);
-}
-
-export async function login() {
-  await goToPage(appUrlBase);
-
-  await clickOn("log-in");
-
-  // (thuang): Auth0 form is unstable and unsafe for input until verified
-  await waitUntilFormFieldStable('[name="email"]');
-
-  await expect(page).toFillForm("form", {
-    email: TEST_EMAIL,
-    password: TEST_PASSWORD,
-  });
-
-  await Promise.all([
-    page.waitForNavigation({ waitUntil: "networkidle0" }),
-    expect(page).toClick('[name="submit"]'),
-  ]);
-
-  expect(page.url()).toContain(appUrlBase);
-}
-
-export async function logout() {
-  await clickOnUntil("user-info", async () => {
-    await waitByID("log-out");
-    await Promise.all([
-      page.waitForNavigation({ waitUntil: "networkidle0" }),
-      clickOn("log-out"),
-    ]);
-  });
-
-  await waitByID("log-in");
-}
-
-async function waitUntilFormFieldStable(selector) {
-  const MAX_RETRY = 10;
-  const WAIT_FOR_MS = 200;
-
-  const EXPECTED_VALUE = "aaa";
-
-  let retry = 0;
-
-  while (retry < MAX_RETRY) {
-    try {
-      await expect(page).toFill(selector, EXPECTED_VALUE);
-
-      const fieldHandle = await expect(page).toMatchElement(selector);
-
-      const fieldValue = await page.evaluate(
-        (input) => input.value,
-        fieldHandle
-      );
-
-      expect(fieldValue).toBe(EXPECTED_VALUE);
-
-      break;
-    } catch (error) {
-      retry += 1;
-
-      await page.waitForTimeout(WAIT_FOR_MS);
-    }
-  }
-
-  if (retry === MAX_RETRY) {
-    throw Error("clickOnUntil() assertion failed!");
-  }
-}
-/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/cellxgeneActions.ts

@@ -0,0 +1,596 @@
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
+import { strict as assert } from "assert";
+import {
+  clearInputAndTypeInto,
+  clickOn,
+  getAllByClass,
+  getOneElementInnerText,
+  typeInto,
+  waitByID,
+  waitByClass,
+  waitForAllByIds,
+  clickOnUntil,
+  getTestClass,
+  getTestId,
+  isElementPresent,
+  goToPage,
+} from "./puppeteerUtils";
+
+import { appUrlBase, TEST_EMAIL, TEST_PASSWORD } from "./config";
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function drag(testId: any, start: any, end: any, lasso = false) {
+  const layout = await waitByID(testId);
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const elBox = await layout.boxModel();
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const x1 = elBox.content[0].x + start.x;
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const x2 = elBox.content[0].x + end.x;
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const y1 = elBox.content[0].y + start.y;
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const y2 = elBox.content[0].y + end.y;
+  await page.mouse.move(x1, y1);
+  await page.mouse.down();
+  if (lasso) {
+    await page.mouse.move(x2, y1);
+    await page.mouse.move(x2, y2);
+    await page.mouse.move(x1, y2);
+    await page.mouse.move(x1, y1);
+  } else {
+    await page.mouse.move(x2, y2);
+  }
+  await page.mouse.up();
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function clickOnCoordinate(testId: any, coord: any) {
+  const layout = await expect(page).toMatchElement(getTestId(testId));
+  const elBox = await layout.boxModel();
+
+  if (!elBox) {
+    throw Error("Layout's boxModel is not available!");
+  }
+
+  const x = elBox.content[0].x + coord.x;
+  const y = elBox.content[0].y + coord.y;
+  await page.mouse.click(x, y);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function getAllHistograms(testclass: any, testIds: any) {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  const histTestIds = testIds.map((tid: any) => `histogram-${tid}`);
+
+  // these load asynchronously, so we need to wait for each histogram individually,
+  // and they may be quite slow in some cases.
+  // @ts-expect-error ts-migrate(2554) FIXME: Expected 1 arguments, but got 2.
+  await waitForAllByIds(histTestIds, { timeout: 4 * 60 * 1000 });
+
+  const allHistograms = await getAllByClass(testclass);
+
+  const testIDs = await Promise.all(
+    allHistograms.map((hist) =>
+      page.evaluate((elem) => elem.dataset.testid, hist)
+    )
+  );
+
+  return testIDs.map((id) => id.replace(/^histogram-/, ""));
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function getAllCategoriesAndCounts(category: any) {
+  // these load asynchronously, so we have to wait for the specific category.
+  await waitByID(`category-${category}`);
+
+  return page.$$eval(
+    `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+    (rows) =>
+      Object.fromEntries(
+        rows.map((row) => {
+          // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+          const cat = row
+            .querySelector("[data-testclass='categorical-value']")
+            .getAttribute("aria-label");
+
+          const count = (row.querySelector(
+            "[data-testclass='categorical-value-count']"
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          ) as any).innerText;
+
+          return [cat, count];
+        })
+      )
+  );
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function getCellSetCount(num: any) {
+  await clickOn(`cellset-button-${num}`);
+  return getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function resetCategory(category: any) {
+  const checkboxId = `${category}:category-select`;
+  await waitByID(checkboxId);
+  const checkedPseudoclass = await page.$eval(
+    `[data-testid='${checkboxId}']`,
+    (el) => el.matches(":checked")
+  );
+  if (!checkedPseudoclass) await clickOn(checkboxId);
+
+  const categoryRow = await waitByID(`${category}:category-expand`);
+
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const isExpanded = await categoryRow.$(
+    "[data-testclass='category-expand-is-expanded']"
+  );
+
+  if (isExpanded) await clickOn(`${category}:category-expand`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS.
+export async function calcCoordinate(
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  testId: any,
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  xAsPercent: any,
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  yAsPercent: any
+) {
+  const el = await waitByID(testId);
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const size = await el.boxModel();
+  return {
+    // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+    x: Math.floor(size.width * xAsPercent),
+    // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+    y: Math.floor(size.height * yAsPercent),
+  };
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS.
+export async function calcDragCoordinates(
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  testId: any,
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  coordinateAsPercent: any
+) {
+  return {
+    start: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x1,
+      coordinateAsPercent.y1
+    ),
+    end: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x2,
+      coordinateAsPercent.y2
+    ),
+  };
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function selectCategory(category: any, values: any, reset = true) {
+  if (reset) await resetCategory(category);
+
+  await clickOn(`${category}:category-expand`);
+  await clickOn(`${category}:category-select`);
+
+  for (const value of values) {
+    await clickOn(`categorical-value-select-${category}-${value}`);
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function expandCategory(category: any) {
+  const expand = await waitByID(`${category}:category-expand`);
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const notExpanded = await expand.$(
+    "[data-testclass='category-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${category}:category-expand`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS.
+export async function clip(min = 0, max = 100) {
+  await clickOn("visualization-settings");
+  await clearInputAndTypeInto("clip-min-input", min);
+  await clearInputAndTypeInto("clip-max-input", max);
+  await clickOn("clip-commit");
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function createCategory(categoryName: any) {
+  await clickOnUntil("open-annotation-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("new-category-name"));
+  });
+
+  await typeInto("new-category-name", categoryName);
+  await clickOn("submit-category");
+}
+
+/**
+ * GENESET
+ */
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function colorByGeneset(genesetName: any) {
+  await clickOn(`${genesetName}:colorby-entire-geneset`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function colorByGene(gene: any) {
+  await clickOn(`colorby-${gene}`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function assertColorLegendLabel(label: any) {
+  const handle = await waitByID("continuous_legend_color_by_label");
+
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(label);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function expandGeneset(genesetName: any) {
+  const expand = await waitByID(`${genesetName}:geneset-expand`);
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const notExpanded = await expand.$(
+    "[data-testclass='geneset-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${genesetName}:geneset-expand`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function createGeneset(genesetName: any) {
+  await clickOnUntil("open-create-geneset-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("create-geneset-input"));
+  });
+
+  await typeInto("create-geneset-input", genesetName);
+  await clickOn("submit-geneset");
+  await waitByClass("autosave-complete");
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function editGenesetName(genesetName: any, editText: any) {
+  const editButton = `${genesetName}:edit-genesetName-mode`;
+  const submitButton = `${genesetName}:submit-geneset`;
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(editButton));
+  });
+  await clickOn(editButton);
+  await typeInto("rename-geneset-modal", editText);
+  await clickOn(submitButton);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function deleteGeneset(genesetName: any) {
+  const targetId = `${genesetName}:delete-geneset`;
+
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertGenesetDoesNotExist(genesetName);
+  await waitByClass("autosave-complete");
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function assertGenesetDoesNotExist(genesetName: any) {
+  // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
+  const result = await isElementPresent(
+    getTestId(`${genesetName}:geneset-name`)
+  );
+  await expect(result).toBe(false);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function assertGenesetExists(genesetName: any) {
+  const handle = await waitByID(`${genesetName}:geneset-name`);
+
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(genesetName);
+}
+
+/**
+ * GENE
+ */
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function addGeneToSet(genesetName: any, geneToAddToSet: any) {
+  const submitButton = `${genesetName}:submit-gene`;
+
+  await clickOn(`${genesetName}:add-new-gene-to-geneset`);
+  await typeInto("add-genes", geneToAddToSet);
+  await clickOn(submitButton);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function removeGene(geneSymbol: any) {
+  const targetId = `delete-from-geneset:${geneSymbol}`;
+
+  await clickOn(targetId);
+
+  await waitByClass("autosave-complete");
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function assertGeneExistsInGeneset(geneSymbol: any) {
+  const handle = await waitByID(`${geneSymbol}:gene-label`);
+
+  // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(geneSymbol);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function assertGeneDoesNotExist(geneSymbol: any) {
+  // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
+  const result = await isElementPresent(getTestId(`${geneSymbol}:gene-label`));
+
+  await expect(result).toBe(false);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function expandGene(geneSymbol: any) {
+  await clickOn(`maximize-${geneSymbol}`);
+}
+
+/**
+ * CATEGORY
+ */
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function duplicateCategory(categoryName: any) {
+  await clickOn("open-annotation-dialog");
+
+  await typeInto("new-category-name", categoryName);
+
+  const dropdownOptionClass = "duplicate-category-dropdown-option";
+
+  await clickOnUntil("duplicate-category-dropdown", async () => {
+    await expect(page).toMatchElement(getTestClass(dropdownOptionClass));
+  });
+
+  const option = await expect(page).toMatchElement(
+    getTestClass(dropdownOptionClass)
+  );
+
+  await option.click();
+
+  await clickOnUntil("submit-category", async () => {
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+  });
+
+  await waitByClass("autosave-complete");
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS.
+export async function renameCategory(
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  oldCategoryName: any,
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  newCategoryName: any
+) {
+  await clickOn(`${oldCategoryName}:see-actions`);
+  await clickOn(`${oldCategoryName}:edit-category-mode`);
+  await clearInputAndTypeInto(
+    `${oldCategoryName}:edit-category-name-text`,
+    newCategoryName
+  );
+  await clickOn(`${oldCategoryName}:submit-category-edit`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function deleteCategory(categoryName: any) {
+  const targetId = `${categoryName}:delete-category`;
+
+  await clickOnUntil(`${categoryName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  // @ts-expect-error ts-migrate(2554) FIXME: Expected 1 arguments, but got 0.
+  await assertCategoryDoesNotExist();
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function createLabel(categoryName: any, labelName: any) {
+  /**
+   * (thuang): This explicit wait is needed, since currently showing
+   * the modal again quickly after the previous action dismissing the
+   * modal will persist the input value from the previous action.
+   *
+   * To reproduce:
+   * 1. Click on the plus sign to show the modal to add a new label to the category
+   * 2. Type `123` in the input box
+   * 3. Hover over your mouse over the plus sign and double click to quickly dismiss and
+   * invoke the modal again
+   * 4. You will see `123` is persisted in the input box
+   * 5. Expected behavior is to get an empty input box
+   */
+  await page.waitForTimeout(500);
+
+  await clickOn(`${categoryName}:see-actions`);
+
+  await clickOn(`${categoryName}:add-new-label-to-category`);
+
+  await typeInto(`${categoryName}:new-label-name`, labelName);
+
+  await clickOn(`${categoryName}:submit-label`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function deleteLabel(categoryName: any, labelName: any) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${labelName}:see-actions`);
+  await clickOn(`${categoryName}:${labelName}:delete-label`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS.
+export async function renameLabel(
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  categoryName: any,
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  oldLabelName: any,
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  newLabelName: any
+) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${oldLabelName}:see-actions`);
+  await clickOn(`${categoryName}:${oldLabelName}:edit-label`);
+  await clearInputAndTypeInto(
+    `${categoryName}:${oldLabelName}:edit-label-name`,
+    newLabelName
+  );
+  await clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function addGeneToSearch(geneName: any) {
+  await typeInto("gene-search", geneName);
+  await page.keyboard.press("Enter");
+  await page.waitForSelector(`[data-testid='histogram-${geneName}']`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function subset(coordinatesAsPercent: any) {
+  // In order to deselect the selection after the subset, make sure we have some clear part
+  // of the scatterplot we can click on
+  assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await clickOn("subset-button");
+  const clearCoordinate = await calcCoordinate("layout-graph", 0.5, 0.99);
+  await clickOnCoordinate("layout-graph", clearCoordinate);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function setSellSet(cellSet: any, cellSetNum: any) {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  const selections = cellSet.filter((sel: any) => sel.kind === "categorical");
+
+  for (const selection of selections) {
+    await selectCategory(selection.metadata, selection.values, true);
+  }
+
+  await getCellSetCount(cellSetNum);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function runDiffExp(cellSet1: any, cellSet2: any) {
+  await setSellSet(cellSet1, 1);
+  await setSellSet(cellSet2, 2);
+  await clickOn("diffexp-button");
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function bulkAddGenes(geneNames: any) {
+  await clickOn("section-bulk-add");
+  await typeInto("input-bulk-add", geneNames.join(","));
+  await page.keyboard.press("Enter");
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function assertCategoryDoesNotExist(categoryName: any) {
+  // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
+  const result = await isElementPresent(
+    getTestId(`${categoryName}:category-label`)
+  );
+
+  await expect(result).toBe(false);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS.
+export async function login() {
+  await goToPage(appUrlBase);
+
+  await clickOn("log-in");
+
+  // (thuang): Auth0 form is unstable and unsafe for input until verified
+  await waitUntilFormFieldStable('[name="email"]');
+
+  await expect(page).toFillForm("form", {
+    email: TEST_EMAIL,
+    password: TEST_PASSWORD,
+  });
+
+  await Promise.all([
+    page.waitForNavigation({ waitUntil: "networkidle0" }),
+    expect(page).toClick('[name="submit"]'),
+  ]);
+
+  expect(page.url()).toContain(appUrlBase);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types --- FIXME: disabled temporarily on migrate to TS.
+export async function logout() {
+  await clickOnUntil("user-info", async () => {
+    await waitByID("log-out");
+    await Promise.all([
+      page.waitForNavigation({ waitUntil: "networkidle0" }),
+      clickOn("log-out"),
+    ]);
+  });
+
+  await waitByID("log-in");
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+async function waitUntilFormFieldStable(selector: any) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  const EXPECTED_VALUE = "aaa";
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await expect(page).toFill(selector, EXPECTED_VALUE);
+
+      const fieldHandle = await expect(page).toMatchElement(selector);
+
+      const fieldValue = await page.evaluate(
+        (input) => input.value,
+        fieldHandle
+      );
+
+      expect(fieldValue).toBe(EXPECTED_VALUE);
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitForTimeout(WAIT_FOR_MS);
+    }
+  }
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2e.test.ts

@@ -58,10 +58,12 @@ describe("metadata loads", () => {
       const categories = await getAllCategoriesAndCounts(label);
 
       expect(Object.keys(categories)).toMatchObject(
+        // @ts-expect-error ts-migrate(7053) FIXME: Element implicitly has an 'any' type because expre... Remove this comment to see the full error message
         Object.keys(data.categorical[label])
       );
 
       expect(Object.values(categories)).toMatchObject(
+        // @ts-expect-error ts-migrate(7053) FIXME: Element implicitly has an 'any' type because expre... Remove this comment to see the full error message
         Object.values(data.categorical[label])
       );
     }
@@ -159,10 +161,12 @@ describe("subset", () => {
       const categories = await getAllCategoriesAndCounts(label);
 
       expect(Object.keys(categories)).toMatchObject(
+        // @ts-expect-error ts-migrate(7053) FIXME: Element implicitly has an 'any' type because expre... Remove this comment to see the full error message
         Object.keys(data.subset.categorical[label])
       );
 
       expect(Object.values(categories)).toMatchObject(
+        // @ts-expect-error ts-migrate(7053) FIXME: Element implicitly has an 'any' type because expre... Remove this comment to see the full error message
         Object.values(data.subset.categorical[label])
       );
     }
@@ -195,6 +199,7 @@ describe("clipping", () => {
   test("clip continuous", async () => {
     await goToPage(appUrlBase);
 
+    // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'string' is not assignable to par... Remove this comment to see the full error message
     await clip(data.clip.min, data.clip.max);
     const histBrushableAreaId = `histogram-${data.clip.metadata}-plot-brushable-area`;
     const coords = await calcDragCoordinates(
@@ -254,6 +259,7 @@ describe("centroid labels", () => {
       const generatedLabels = await getAllByClass("centroid-label");
       // Number of labels generated should be equal to size of the object
       expect(generatedLabels).toHaveLength(
+        // @ts-expect-error ts-migrate(7053) FIXME: Element implicitly has an 'any' type because expre... Remove this comment to see the full error message
         Object.keys(data.categorical[label]).length
       );
     }
@@ -275,6 +281,7 @@ describe("graph overlay", () => {
       data.pan["coordinates-as-percent"]
     );
 
+    // @ts-expect-error ts-migrate(7053) FIXME: Element implicitly has an 'any' type because expre... Remove this comment to see the full error message
     const categoryValue = Object.keys(data.categorical[category])[0];
     const initialCoordinates = await getElementCoordinates(
       `${categoryValue}-centroid-label`

client/__tests__/e2e/e2eAnnotations.test.ts

@@ -12,6 +12,7 @@ import {
   getTestId,
   getTestClass,
   getAllByClass,
+  clickOnUntil,
   getOneElementInnerHTML,
 } from "./puppeteerUtils";
 
@@ -76,7 +77,13 @@ const brushThisGeneGeneset = "brush_this_gene";
 const geneBrushedCellCount = "109";
 const subsetGeneBrushedCellCount = "96";
 
-async function setup(config) {
+const genesetDescriptionID =
+  "geneset-description-tooltip-fourth_gene_set: fourth description";
+const genesetDescriptionString = "fourth_gene_set: fourth description";
+const genesetToCheckForDescription = "fourth_gene_set";
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+async function setup(config: any) {
   await goToPage(appUrlBase);
 
   if (config.categoricalAnno) {
@@ -150,7 +157,8 @@ describe.each([
     await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
 
     await page.waitForFunction(
-      (selector) => !document.querySelector(selector),
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (selector: any) => !document.querySelector(selector),
       {},
       getTestClass("gene-loading-spinner")
     );
@@ -165,7 +173,8 @@ describe.each([
     await expect(page).toClick(getTestClass("pop-2-geneset-expand"));
 
     await page.waitForFunction(
-      (selector) => !document.querySelector(selector),
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (selector: any) => !document.querySelector(selector),
       {},
       getTestClass("gene-loading-spinner")
     );
@@ -174,7 +183,7 @@ describe.each([
 
     expect(genesHTML).toMatchSnapshot();
   });
-  test("create a new geneset", async () => {
+  test("create a new geneset and undo/redo", async () => {
     if (config.withSubset) return;
 
     await setup(config);
@@ -184,19 +193,45 @@ describe.each([
     await createGeneset(genesetName);
     /* note: as of June 2021, the aria label is in the truncate component which clones the element */
     await assertGenesetExists(genesetName);
+    await clickOn("undo");
+    await assertGenesetDoesNotExist(genesetName);
+    await clickOn("redo");
+    await assertGenesetExists(genesetName);
   });
-  test("edit geneset name", async () => {
+  test("edit geneset name and undo/redo", async () => {
     await setup(config);
 
     await editGenesetName(editableGenesetName, editText);
     await assertGenesetExists(newGenesetName);
+    await clickOn("undo");
+    await assertGenesetExists(editableGenesetName);
+    await clickOn("redo");
+    await assertGenesetExists(newGenesetName);
   });
-  test("delete a geneset", async () => {
+  test("delete a geneset and undo/redo", async () => {
     if (config.withSubset) return;
 
     await setup(config);
 
     await deleteGeneset(genesetToDeleteName);
+    await clickOn("undo");
+    await assertGenesetExists(genesetToDeleteName);
+    await clickOn("redo");
+    await assertGenesetDoesNotExist(genesetToDeleteName);
+  });
+  test("geneset description", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await clickOnUntil(
+      `${genesetToCheckForDescription}:geneset-expand`,
+      async () => {
+        expect(page).toMatchElement(getTestId(genesetDescriptionID), {
+          text: genesetDescriptionString,
+        });
+      }
+    );
   });
 });
 
@@ -204,12 +239,16 @@ describe.each([
   { withSubset: true, tag: "subset" },
   { withSubset: false, tag: "whole" },
 ])("GENE crud operations and interactions", (config) => {
-  test("add a gene to geneset", async () => {
+  test("add a gene to geneset and undo/redo", async () => {
     await setup(config);
 
     await addGeneToSet(setToAddGeneTo, geneToAddToSet);
     await expandGeneset(setToAddGeneTo);
     await assertGeneExistsInGeneset(geneToAddToSet);
+    await clickOn("undo");
+    await assertGeneDoesNotExist(geneToAddToSet);
+    await clickOn("redo");
+    await assertGeneExistsInGeneset(geneToAddToSet);
   });
   test("expand gene and brush", async () => {
     await setup(config);
@@ -240,7 +279,7 @@ describe.each([
     await colorByGene(geneToBrushAndColorBy);
     await assertColorLegendLabel(geneToBrushAndColorBy);
   });
-  test("delete gene from geneset", async () => {
+  test("delete gene from geneset and undo/redo", async () => {
     // We've already deleted the gene
     if (config.withSubset) return;
 
@@ -249,6 +288,10 @@ describe.each([
     await expandGeneset(setToRemoveFrom);
     await removeGene(geneToRemove);
     await assertGeneDoesNotExist(geneToRemove);
+    await clickOn("undo");
+    await assertGeneExistsInGeneset(geneToRemove);
+    await clickOn("redo");
+    await assertGeneDoesNotExist(geneToRemove);
   });
 });
 
@@ -362,8 +405,13 @@ describe.each([
     expect(actualLabelName).toBe(expectedLabelName);
     expect(actualLabelCount).toBe(expectedLabelCount);
 
-    async function getInnerText(element, className) {
-      return element.$eval(getTestClass(className), (node) => node?.innerText);
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    async function getInnerText(element: any, className: any) {
+      return element.$eval(
+        getTestClass(className),
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+        (node: any) => node?.innerText
+      );
     }
   });
 
@@ -388,7 +436,9 @@ describe.each([
       `categorical-value-count-${perTestCategoryName}-${perTestLabelName}`
     );
 
+    // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
     expect(await result.evaluate((node) => node.innerText)).toBe(
+      // @ts-expect-error ts-migrate(2538) FIXME: Type 'boolean' cannot be used as an index type.
       data.categoryLabel.newCount.bySubsetConfig[config.withSubset]
     );
   });
@@ -448,6 +498,7 @@ describe.each([
     await createLabel(perTestCategoryName, labelName);
     await assertLabelExists(perTestCategoryName, labelName);
     await clickOn("undo");
+    // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
     await assertLabelDoesNotExist(perTestCategoryName);
     await clickOn("redo");
     await assertLabelExists(perTestCategoryName, labelName);
@@ -457,10 +508,12 @@ describe.each([
     await setup(config);
 
     await deleteLabel(perTestCategoryName, perTestLabelName);
+    // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
     await assertLabelDoesNotExist(perTestCategoryName);
     await clickOn("undo");
     await assertLabelExists(perTestCategoryName, perTestLabelName);
     await clickOn("redo");
+    // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
     await assertLabelDoesNotExist(perTestCategoryName);
   });
 
@@ -491,11 +544,9 @@ describe.each([
     const labels = await getAllByClass("categorical-row");
 
     const result = await Promise.all(
-      labels.map((label) => {
-        return page.evaluate((element) => {
-          return element.outerHTML;
-        }, label);
-      })
+      labels.map((label) =>
+        page.evaluate((element) => element.outerHTML, label)
+      )
     );
 
     expect(result).toMatchSnapshot();
@@ -523,9 +574,11 @@ describe.each([
     expect(result).toMatchSnapshot();
   });
 
-  async function assertCategoryExists(categoryName) {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  async function assertCategoryExists(categoryName: any) {
     const handle = await waitByID(`${categoryName}:category-label`);
 
+    // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
     const result = await handle.evaluate((node) =>
       node.getAttribute("aria-label")
     );
@@ -533,7 +586,8 @@ describe.each([
     return expect(result).toBe(categoryName);
   }
 
-  async function assertLabelExists(categoryName, labelName) {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  async function assertLabelExists(categoryName: any, labelName: any) {
     await expect(page).toMatchElement(
       getTestId(`${categoryName}:category-expand`)
     );
@@ -545,11 +599,13 @@ describe.each([
     );
 
     expect(
+      // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
       await previous.evaluate((node) => node.getAttribute("aria-label"))
     ).toBe(labelName);
   }
 
-  async function assertLabelDoesNotExist(categoryName, labelName) {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  async function assertLabelDoesNotExist(categoryName: any, labelName: any) {
     await expandCategory(categoryName);
     const result = await page.$(
       `[data-testid='categorical-value-${categoryName}-${labelName}']`

client/__tests__/e2e/e2eJestConfig.json

@@ -1,10 +1,10 @@
 {
   "testRunner": "jest-circus/runner",
   "preset": "jest-puppeteer",
-  "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
-  "setupFiles": ["../setupMissingGlobals.js"],
-  "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
-  "globalSetup": "../globalSetup.js",
+  "testMatch": ["**/__tests__/**/?(*.)(spec|test).ts?(x)"],
+  "setupFiles": ["../setupMissingGlobals.ts"],
+  "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.ts"],
+  "globalSetup": "../globalSetup.ts",
   "globalTeardown": "jest-environment-puppeteer/teardown",
   "testEnvironment": "./screenshot_env.js"
 }

client/__tests__/e2e/puppeteer.setup.ts

@@ -23,6 +23,7 @@ beforeEach(async () => {
   const userAgent = await browser.userAgent();
   await page.setUserAgent(`${userAgent}bot`);
 
+  // @ts-expect-error ts-migrate(2341) FIXME: Property '_client' is private and only accessible ... Remove this comment to see the full error message
   await page._client.send("Animation.setPlaybackRate", { playbackRate: 12 });
 
   page.on("pageerror", (err) => {
@@ -49,7 +50,8 @@ beforeEach(async () => {
         }
         const errorMsgText = await Promise.all(
           // TODO can we do this without internal properties?
-          msg.args().map((arg) => arg._remoteObject.description)
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          msg.args().map((arg: any) => arg._remoteObject.description)
         );
         throw new Error(`Console error: ${errorMsgText}`);
       }

client/__tests__/e2e/puppeteerUtils.js

@@ -1,131 +0,0 @@
-/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
-export function getTestId(id) {
-  return `[data-testid='${id}']`;
-}
-
-export function getTestClass(className) {
-  return `[data-testclass='${className}']`;
-}
-
-export async function waitByID(testId, props = {}) {
-  return page.waitForSelector(getTestId(testId), props);
-}
-
-export async function waitByClass(testClass, props = {}) {
-  return page.waitForSelector(`[data-testclass='${testClass}']`, props);
-}
-
-export async function waitForAllByIds(testIds) {
-  await Promise.all(
-    testIds.map((testId) => page.waitForSelector(getTestId(testId)))
-  );
-}
-
-export async function getAllByClass(testClass) {
-  return page.$$(`[data-testclass=${testClass}]`);
-}
-
-export async function typeInto(testId, text) {
-  // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
-  // only works for text without special characters
-  await waitByID(testId);
-  const selector = getTestId(testId);
-  // type ahead can be annoying if you don't pause before you type
-  await page.click(selector);
-  await page.waitForTimeout(200);
-  await page.type(selector, text);
-}
-
-export async function clearInputAndTypeInto(testId, text) {
-  await waitByID(testId);
-  const selector = getTestId(testId);
-  // only works for text without special characters
-  // type ahead can be annoying if you don't pause before you type
-  await page.click(selector);
-  await page.waitForTimeout(200);
-  // select all
-  await page.click(selector, { clickCount: 3 });
-  await page.keyboard.press("Backspace");
-  await page.type(selector, text);
-}
-
-export async function clickOn(testId, options = {}) {
-  await expect(page).toClick(getTestId(testId), options);
-}
-
-/**
- * (thuang): There are times when Puppeteer clicks on a button and the page doesn't respond.
- * So I added clickOnUntil() to retry clicking until a given condition is met.
- */
-export async function clickOnUntil(testId, assert) {
-  const MAX_RETRY = 10;
-  const WAIT_FOR_MS = 200;
-
-  let retry = 0;
-
-  while (retry < MAX_RETRY) {
-    try {
-      await clickOn(testId);
-      await assert();
-
-      break;
-    } catch (error) {
-      retry += 1;
-
-      await page.waitForTimeout(WAIT_FOR_MS);
-    }
-  }
-
-  if (retry === MAX_RETRY) {
-    throw Error("clickOnUntil() assertion failed!");
-  }
-}
-
-export async function getOneElementInnerHTML(selector, options = {}) {
-  await page.waitForSelector(selector, options);
-
-  return page.$eval(selector, (el) => el.innerHTML);
-}
-
-export async function getOneElementInnerText(selector) {
-  expect(page).toMatchElement(selector);
-
-  return page.$eval(selector, (el) => el.innerText);
-}
-
-export async function getElementCoordinates(testId) {
-  return page.$eval(getTestId(testId), (elem) => {
-    const { left, top } = elem.getBoundingClientRect();
-    return [left, top];
-  });
-}
-
-async function clickTermsOfService() {
-  if (!(await isElementPresent(getTestId("tos-cookies-accept")))) return;
-
-  await clickOn("tos-cookies-accept");
-}
-
-async function nameNewAnnotation() {
-  if (await isElementPresent(getTestId("annotation-dialog"))) {
-    await typeInto("new-annotation-name", "ignoreE2E");
-    await clickOn("submit-annotation");
-
-    // wait for the page to load
-    await waitByClass("autosave-complete");
-  }
-}
-
-export async function goToPage(url) {
-  await page.goto(url, {
-    waitUntil: "networkidle0",
-  });
-
-  await nameNewAnnotation();
-  await clickTermsOfService();
-}
-
-export async function isElementPresent(selector, options) {
-  return Boolean(await page.$(selector, options));
-}
-/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/puppeteerUtils.ts

@@ -0,0 +1,151 @@
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export function getTestId(id: any) {
+  return `[data-testid='${id}']`;
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export function getTestClass(className: any) {
+  return `[data-testclass='${className}']`;
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function waitByID(testId: any, props = {}) {
+  return page.waitForSelector(getTestId(testId), props);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function waitByClass(testClass: any, props = {}) {
+  return page.waitForSelector(`[data-testclass='${testClass}']`, props);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function waitForAllByIds(testIds: any) {
+  await Promise.all(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    testIds.map((testId: any) => page.waitForSelector(getTestId(testId)))
+  );
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function getAllByClass(testClass: any) {
+  return page.$$(`[data-testclass=${testClass}]`);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function typeInto(testId: any, text: any) {
+  // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+  // only works for text without special characters
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitForTimeout(200);
+  await page.type(selector, text);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function clearInputAndTypeInto(testId: any, text: any) {
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // only works for text without special characters
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitForTimeout(200);
+  // select all
+  await page.click(selector, { clickCount: 3 });
+  await page.keyboard.press("Backspace");
+  await page.type(selector, text);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function clickOn(testId: any, options = {}) {
+  await expect(page).toClick(getTestId(testId), options);
+}
+
+/**
+ * (thuang): There are times when Puppeteer clicks on a button and the page doesn't respond.
+ * So I added clickOnUntil() to retry clicking until a given condition is met.
+ */
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function clickOnUntil(testId: any, assert: any) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await clickOn(testId);
+      await assert();
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitForTimeout(WAIT_FOR_MS);
+    }
+  }
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function getOneElementInnerHTML(selector: any, options = {}) {
+  await page.waitForSelector(selector, options);
+
+  return page.$eval(selector, (el) => el.innerHTML);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function getOneElementInnerText(selector: any) {
+  expect(page).toMatchElement(selector);
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  return page.$eval(selector, (el) => (el as any).innerText);
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function getElementCoordinates(testId: any) {
+  return page.$eval(getTestId(testId), (elem) => {
+    const { left, top } = elem.getBoundingClientRect();
+    return [left, top];
+  });
+}
+
+async function clickTermsOfService() {
+  // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
+  if (!(await isElementPresent(getTestId("tos-cookies-accept")))) return;
+
+  await clickOn("tos-cookies-accept");
+}
+
+async function nameNewAnnotation() {
+  // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
+  if (await isElementPresent(getTestId("annotation-dialog"))) {
+    await typeInto("new-annotation-name", "ignoreE2E");
+    await clickOn("submit-annotation");
+
+    // wait for the page to load
+    await waitByClass("autosave-complete");
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function goToPage(url: any) {
+  await page.goto(url, {
+    waitUntil: "networkidle0",
+  });
+
+  await nameNewAnnotation();
+  await clickTermsOfService();
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+export async function isElementPresent(selector: any, options: any) {
+  // @ts-expect-error ts-migrate(2554) FIXME: Expected 1 arguments, but got 2.
+  return Boolean(await page.$(selector, options));
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/screenshot_env.js

@@ -1,7 +1,11 @@
+// eslint-disable-next-line @typescript-eslint/no-var-requires --- FIXME: disabled temporarily on migrate to TS.
 const PuppeteerEnvironment = require("jest-environment-puppeteer");
 require("jest-circus");
+// eslint-disable-next-line @typescript-eslint/no-var-requires --- FIXME: disabled temporarily on migrate to TS.
 const ENV_DEFAULT = require("../../../environment.default.json");
 
+// @ts-expect-error ts-migrate(2451) FIXME: Cannot redeclare block-scoped variable 'takeScreen... Remove this comment to see the full error message
+// eslint-disable-next-line @typescript-eslint/no-var-requires --- FIXME: disabled temporarily on migrate to TS.
 const takeScreenshot = require("./takeScreenshot");
 
 class ScreenshotEnvironment extends PuppeteerEnvironment {

client/__tests__/e2e/test_config.yaml

@@ -22,10 +22,6 @@ dataset:
     local_file_csv:
       directory: null
       file: null
-    ontology:
-      enable: false
-      obo_location: null
 
   embeddings:
     names: []
-    enable_reembedding: false

client/__tests__/globalSetup.ts

@@ -1,8 +1,12 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment --- FIXME: disabled temporarily on migrate to TS.
+// @ts-ignore FIXME: 'globalSetup.ts' cannot be compiled under '--isola... Remove this comment to see the full error message
 const {
   SecretsManagerClient,
   GetSecretValueCommand,
+  // eslint-disable-next-line @typescript-eslint/no-var-requires --- FIXME: disabled temporarily on migrate to TS.
 } = require("@aws-sdk/client-secrets-manager");
 
+// eslint-disable-next-line @typescript-eslint/no-var-requires --- FIXME: disabled temporarily on migrate to TS.
 const { setup } = require("jest-environment-puppeteer");
 
 const client = new SecretsManagerClient({ region: "us-west-2" });

client/__tests__/reducers/cascade.test.ts

@@ -20,7 +20,16 @@ describe("cascade", () => {
     const reducer = cascadeReducers([
       [
         "foo",
-        (currentState, action, nextSharedState, prevSharedState) => {
+        (
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          currentState: any,
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          action: any,
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          nextSharedState: any,
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          prevSharedState: any
+        ) => {
           expect(currentState).toBeUndefined();
           expect(action).toEqual(topLevelAction);
           expect(nextSharedState).toStrictEqual({});
@@ -30,7 +39,16 @@ describe("cascade", () => {
       ],
       [
         "bar",
-        (currentState, action, nextSharedState, prevSharedState) => {
+        (
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          currentState: any,
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          action: any,
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          nextSharedState: any,
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          prevSharedState: any
+        ) => {
           expect(currentState).toBeUndefined();
           expect(action).toEqual(topLevelAction);
           expect(nextSharedState).toStrictEqual({ foo: 0 });

client/__tests__/reducers/genesets.test.ts

@@ -501,6 +501,7 @@ describe("geneset: set tid", () => {
   test("not a number error", () => {
     expect(() => {
       genesetsReducer(
+        // @ts-expect-error ts-migrate(2322) FIXME: Type 'number' is not assignable to type 'undefined... Remove this comment to see the full error message
         { lastTid: 1 },
         {
           type: "geneset: set tid",
@@ -513,6 +514,7 @@ describe("geneset: set tid", () => {
   test("decrement error", () => {
     expect(() => {
       genesetsReducer(
+        // @ts-expect-error ts-migrate(2322) FIXME: Type 'number' is not assignable to type 'undefined... Remove this comment to see the full error message
         { lastTid: 1 },
         {
           type: "geneset: set tid",

client/__tests__/reducers/undoable.test.ts

@@ -1,9 +1,12 @@
+import { Reducer } from "redux";
 import undoable from "../../src/reducers/undoable";
 
 describe("create", () => {
   test("no keys", () => {
-    expect(() => undoable(() => {})).toThrow();
-    expect(() => undoable(() => {}, null)).toThrow();
+    expect(() =>
+      undoable(() => {}, undefined as unknown as string[])
+    ).toThrow();
+    expect(() => undoable(() => {}, null as unknown as string[])).toThrow();
     expect(() => undoable(() => {}, [])).toThrow();
     expect(() => undoable(() => {}, [], {})).toThrow();
   });
@@ -23,9 +26,7 @@ describe("create", () => {
 describe("undo", () => {
   test("expected state modifications", () => {
     const initialState = { a: 0, b: 1000 };
-    const reducer = (state) => {
-      return { a: state.a + 1, b: state.b + 1 };
-    };
+    const reducer: Reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
     const undoableReducer = undoable(reducer, ["a"]);
 
     const s1 = undoableReducer(initialState, { type: "test" });
@@ -43,10 +44,8 @@ describe("undo", () => {
 
 describe("redo", () => {
   const initialState = { a: 0, b: 1000 };
-  const reducer = (state) => {
-    return { a: state.a + 1, b: state.b + 1 };
-  };
-  let UR;
+  const reducer: Reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
+  let UR: Reducer;
 
   beforeEach(() => {
     UR = undoable(reducer, ["a"]);

client/__tests__/setupMissingGlobals.ts

@@ -5,5 +5,6 @@ the jest test environment).
 
 import { TextDecoder, TextEncoder } from "util";
 
+// @ts-expect-error ts-migrate(2322) FIXME: Type 'typeof TextDecoder' is not assignable to typ... Remove this comment to see the full error message
 global.TextDecoder = TextDecoder;
 global.TextEncoder = TextEncoder;

client/__tests__/util/annoMatrix/annoMatrix.test.ts

@@ -14,10 +14,13 @@ import { Dataframe } from "../../../src/util/dataframe";
 enableFetchMocks();
 
 describe("AnnoMatrix", () => {
-  let annoMatrix;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  let annoMatrix: any;
 
   beforeEach(async () => {
-    fetch.resetMocks(); // reset all fetch mocking state
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    (fetch as any).resetMocks(); // reset all fetch mocking state
+    // reset all fetch mocking state
     annoMatrix = new AnnoMatrixLoader(
       serverMocks.baseDataURL,
       serverMocks.schema.schema
@@ -36,7 +39,8 @@ describe("AnnoMatrix", () => {
     });
 
     test("simple single column fetch", async () => {
-      fetch.once(serverMocks.annotationsObs(["name_0"]));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(serverMocks.annotationsObs(["name_0"]));
 
       const df = await annoMatrix.fetch("obs", "name_0");
       expect(df).toBeInstanceOf(Dataframe);
@@ -45,7 +49,8 @@ describe("AnnoMatrix", () => {
     });
 
     test("simple multi column fetch", async () => {
-      fetch
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any)
         .once(serverMocks.annotationsObs(["name_0"]))
         .once(serverMocks.annotationsObs(["n_genes"]));
 
@@ -55,9 +60,13 @@ describe("AnnoMatrix", () => {
     });
 
     describe("fetch from field", () => {
-      const getLastTwo = async (field) => {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      const getLastTwo = async (field: any) => {
         const columnNames = annoMatrix.getMatrixColumns(field).slice(-2);
-        fetch.mockResponses(...columnNames.map(() => serverMocks.responder));
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+        (fetch as any).mockResponses(
+          ...columnNames.map(() => serverMocks.responder)
+        );
         await expect(
           annoMatrix.fetch(field, columnNames)
         ).resolves.toBeInstanceOf(Dataframe);
@@ -70,19 +79,22 @@ describe("AnnoMatrix", () => {
 
     test("fetch - test all query forms", async () => {
       // single string is a column name
-      fetch.once(serverMocks.annotationsObs(["n_genes"]));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(serverMocks.annotationsObs(["n_genes"]));
       await expect(annoMatrix.fetch("obs", "n_genes")).resolves.toBeInstanceOf(
         Dataframe
       );
 
       // array of column names, expecting n_genes to be cached.
-      fetch.once(serverMocks.annotationsObs(["percent_mito"]));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(serverMocks.annotationsObs(["percent_mito"]));
       await expect(
         annoMatrix.fetch("obs", ["n_genes", "percent_mito"])
       ).resolves.toBeInstanceOf(Dataframe);
 
       // more complex value filter query, enumerated
-      fetch.once(serverMocks.responder);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(serverMocks.responder);
       await expect(
         annoMatrix.fetch("X", {
           where: {
@@ -95,7 +107,8 @@ describe("AnnoMatrix", () => {
 
       // more complex value filter query, range
       const varIndex = annoMatrix.schema.annotations.var.index;
-      fetch
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any)
         .once(
           serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "SUMO3"]])
         )
@@ -171,7 +184,8 @@ describe("AnnoMatrix", () => {
       expect(am1.nObs).toEqual(am2.nObs);
       expect(am1.nVar).toEqual(am2.nVar);
 
-      fetch
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any)
         .once(serverMocks.annotationsObs(["n_genes"]))
         .once(serverMocks.annotationsObs(["n_genes"]));
       const ng1 = await am1.fetch("obs", "n_genes");
@@ -185,9 +199,11 @@ describe("AnnoMatrix", () => {
   });
 
   describe("add/drop column", () => {
+    // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'base' implicitly has an 'any' type.
     async function addDrop(base) {
       expect(base.getMatrixColumns("obs")).not.toContain("foo");
-      fetch.mockRejectOnce(new Error("unknown column name"));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockRejectOnce(new Error("unknown column name"));
       await expect(base.fetch("obs", "foo")).rejects.toThrow(
         "unknown column name"
       );
@@ -212,7 +228,8 @@ describe("AnnoMatrix", () => {
       const am2 = am1.dropObsColumn("foo");
       expect(base.getMatrixColumns("obs")).not.toContain("foo");
       expect(am2.getMatrixColumns("obs")).not.toContain("foo");
-      fetch.mockRejectOnce(new Error("unknown column name"));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockRejectOnce(new Error("unknown column name"));
       await expect(am2.fetch("obs", "foo")).rejects.toThrow(
         "unknown column name"
       );
@@ -235,14 +252,16 @@ describe("AnnoMatrix", () => {
       const am4 = clip(am3, 0, 1);
       await addDrop(am4);
 
-      fetch.mockResponse(serverMocks.responder);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockResponse(serverMocks.responder);
 
       await am1.fetch("obs", am1.getMatrixColumns("obs"));
       await am2.fetch("obs", am2.getMatrixColumns("obs"));
       await am3.fetch("obs", am3.getMatrixColumns("obs"));
       await am4.fetch("obs", am4.getMatrixColumns("obs"));
 
-      fetch.resetMocks();
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).resetMocks();
 
       await addDrop(am1);
       await addDrop(am2);
@@ -252,6 +271,7 @@ describe("AnnoMatrix", () => {
   });
 
   describe("setObsColumnValues", () => {
+    // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'base' implicitly has an 'any' type.
     async function addSetDrop(base) {
       /* add column */
       let am = base.addObsColumn(
@@ -287,7 +307,8 @@ describe("AnnoMatrix", () => {
       );
 
       /* drop column */
-      fetch.mockRejectOnce(new Error("unknown column name"));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockRejectOnce(new Error("unknown column name"));
       am = am1.dropObsColumn("test");
       await expect(am.fetch("obs", "test")).rejects.toThrow(
         "unknown column name"
@@ -308,7 +329,8 @@ describe("AnnoMatrix", () => {
       const am3 = isubset(annoMatrix, [10, 1, 0, 30, 2]);
       await addSetDrop(am3);
 
-      fetch.mockResponse(serverMocks.responder);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockResponse(serverMocks.responder);
 
       await am1.fetch("obs", am1.getMatrixColumns("obs"));
       await am2.fetch("obs", am2.getMatrixColumns("obs"));

client/__tests__/util/annoMatrix/crossfilter.test.ts

@@ -17,11 +17,15 @@ import { rangeFill } from "../../../src/util/range";
 enableFetchMocks();
 
 describe("AnnoMatrixCrossfilter", () => {
-  let annoMatrix;
-  let crossfilter;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  let annoMatrix: any;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  let crossfilter: any;
 
   beforeEach(async () => {
-    fetch.resetMocks(); // reset all fetch mocking state
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    (fetch as any).resetMocks(); // reset all fetch mocking state
+    // reset all fetch mocking state
     annoMatrix = new AnnoMatrixLoader(
       serverMocks.baseDataURL,
       serverMocks.schema.schema
@@ -67,7 +71,10 @@ describe("AnnoMatrixCrossfilter", () => {
         crossfilter.obsCrossfilter.hasDimension("obs/louvain")
       ).toBeFalsy();
 
-      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(
+        serverMocks.dataframeResponse(["louvain"], [obsLouvain])
+      );
       let newCrossfilter = await crossfilter.select("obs", "louvain", {
         mode: "none",
       });
@@ -76,7 +83,8 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(
         newCrossfilter.obsCrossfilter.hasDimension("obs/louvain")
       ).toBeTruthy();
-      expect(fetch.mock.calls).toHaveLength(1);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      expect((fetch as any).mock.calls).toHaveLength(1);
 
       newCrossfilter = await crossfilter.select("obs", "louvain", {
         mode: "all",
@@ -87,7 +95,10 @@ describe("AnnoMatrixCrossfilter", () => {
     test("simple column select", async () => {
       let xfltr;
 
-      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(
+        serverMocks.dataframeResponse(["louvain"], [obsLouvain])
+      );
       xfltr = await crossfilter.select("obs", "louvain", {
         mode: "exact",
         values: ["NK cells", "B cells"],
@@ -105,6 +116,7 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(xfltr.allSelectedLabels()).toEqual(
         Int32Array.from(
           obsLouvain.reduce((acc, val, idx) => {
+            // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'number' is not assignable to par... Remove this comment to see the full error message
             if (val === "NK cells" || val === "B cells") acc.push(idx);
             return acc;
           }, [])
@@ -124,10 +136,13 @@ describe("AnnoMatrixCrossfilter", () => {
       const values = df.col("louvain").asArray();
       const selected = xfltr.allSelectedMask();
       values.every(
-        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+        (val: any, idx: any) =>
+          !["NK cells", "B cells"].includes(val) !== !selected[idx]
       );
 
-      fetch.once(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(
         serverMocks.dataframeResponse(["n_genes"], [new Int32Array(obsNGenes)])
       );
       xfltr = await xfltr.select("obs", "n_genes", {
@@ -146,6 +161,7 @@ describe("AnnoMatrixCrossfilter", () => {
               val < 500 &&
               (louvain === "NK cells" || louvain === "B cells")
             )
+              // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'number' is not assignable to par... Remove this comment to see the full error message
               acc.push(idx);
             return acc;
           }, [])
@@ -160,7 +176,8 @@ describe("AnnoMatrixCrossfilter", () => {
       const varIndex = annoMatrix.schema.annotations.var.index;
 
       const { nObs } = annoMatrix.schema.dataframe;
-      fetch.once(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(
         serverMocks.dataframeResponse(
           ["TEST"],
           [rangeFill(new Float32Array(nObs), 0, 0.1)]
@@ -196,14 +213,19 @@ describe("AnnoMatrixCrossfilter", () => {
       });
       const values = df.icol(0).asArray();
       const selected = xfltr.allSelectedMask();
-      values.every((val, idx) => !(val >= 0 && val <= 50) !== !selected[idx]);
-      expect(selected.reduce((acc, val) => (val ? acc + 1 : acc), 0)).toEqual(
-        xfltr.countSelected()
+      values.every(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+        (val: any, idx: any) => !(val >= 0 && val <= 50) !== !selected[idx]
       );
+      expect(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+        selected.reduce((acc: any, val: any) => (val ? acc + 1 : acc), 0)
+      ).toEqual(xfltr.countSelected());
     });
 
     test("spatial column select", async () => {
-      fetch.once(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(
         serverMocks.dataframeResponse(
           ["umap_0", "umap_1"],
           [Float32Array.from(embUmap[0]), Float32Array.from(embUmap[1])]
@@ -222,6 +244,7 @@ describe("AnnoMatrixCrossfilter", () => {
     test("select on subset", async () => {
       const mask = new Uint8Array(annoMatrix.nObs).fill(0);
       for (let i = 0; i < mask.length; i += 2) {
+        // @ts-expect-error ts-migrate(2322) FIXME: Type 'boolean' is not assignable to type 'number'.
         mask[i] = true;
       }
       const annoMatrixSubset = isubsetMask(annoMatrix, mask);
@@ -230,7 +253,10 @@ describe("AnnoMatrixCrossfilter", () => {
       let xfltr = new AnnoMatrixObsCrossfilter(annoMatrixSubset);
       expect(xfltr.countSelected()).toEqual(annoMatrixSubset.nObs);
 
-      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(
+        serverMocks.dataframeResponse(["louvain"], [obsLouvain])
+      );
       xfltr = await xfltr.select("obs", "louvain", {
         mode: "exact",
         values: ["NK cells", "B cells"],
@@ -243,7 +269,9 @@ describe("AnnoMatrixCrossfilter", () => {
       const values = df.col("louvain").asArray();
       const selected = xfltr.allSelectedMask();
       values.every(
-        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+        (val: any, idx: any) =>
+          !["NK cells", "B cells"].includes(val) !== !selected[idx]
       );
     });
 
@@ -256,7 +284,8 @@ describe("AnnoMatrixCrossfilter", () => {
         "unable to obsSelect upon the var dimension"
       );
 
-      fetch.mockRejectOnce(new Error("unknown column name"));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockRejectOnce(new Error("unknown column name"));
       await expect(crossfilter.select("obs", "foo")).rejects.toThrow(
         "unknown column name"
       );
@@ -267,24 +296,29 @@ describe("AnnoMatrixCrossfilter", () => {
     /*
     test the matrix mutators via crossfilter proxy
     */
-    async function helperAddTestCol(cf, colName, colSchema = null) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    async function helperAddTestCol(cf: any, colName: any, colSchema = null) {
       expect(
         cf.annoMatrix.getMatrixColumns("obs").includes(colName)
       ).toBeFalsy();
 
       if (colSchema === null) {
+        // @ts-expect-error ts-migrate(2322) FIXME: Type '{ name: any; type: string; categories: strin... Remove this comment to see the full error message
         colSchema = {
           name: colName,
           type: "categorical",
           categories: ["toasty"],
         };
       }
+      // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
       colSchema.name = colName;
+      // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
       const initValue = colSchema.categories[0];
       const xfltr = cf.addObsColumn(colSchema, Array, initValue);
       expect(
         xfltr.annoMatrix.schema.annotations.obs.columns.filter(
-          (v) => v.name === colName
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          (v: any) => v.name === colName
         )
       ).toHaveLength(1);
       const df = await xfltr.annoMatrix.fetch("obs", colName);
@@ -314,7 +348,8 @@ describe("AnnoMatrixCrossfilter", () => {
       });
       expect(
         xfltr.annoMatrix.schema.annotations.obs.columns.filter(
-          (v) => v.name === "foo"
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          (v: any) => v.name === "foo"
         )
       ).toHaveLength(1);
 
@@ -323,8 +358,8 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(
         df
           .col("foo")
-          .asArray()
-          .every((v) => v === "A")
+          .asArray() // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          .every((v: any) => v === "A")
       ).toBeTruthy();
 
       // check that we catch dups
@@ -361,11 +396,13 @@ describe("AnnoMatrixCrossfilter", () => {
       xfltr = xfltr.dropObsColumn("foo");
       expect(
         xfltr.annoMatrix.schema.annotations.obs.columns.filter(
-          (v) => v.name === "foo"
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          (v: any) => v.name === "foo"
         )
       ).toHaveLength(0);
       expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toBeUndefined();
-      fetch.mockRejectOnce(new Error("unknown column name"));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockRejectOnce(new Error("unknown column name"));
       await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
         "unknown column name"
       );
@@ -378,7 +415,8 @@ describe("AnnoMatrixCrossfilter", () => {
       });
       xfltr = xfltr.dropObsColumn("bar");
 
-      fetch.mockRejectOnce(new Error("unknown column name"));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockRejectOnce(new Error("unknown column name"));
       await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
         "unknown column name"
       );
@@ -404,7 +442,8 @@ describe("AnnoMatrixCrossfilter", () => {
         type: "categorical",
       });
 
-      fetch.mockRejectOnce(new Error("unknown column name"));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockRejectOnce(new Error("unknown column name"));
       await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
         "unknown column name"
       );
@@ -419,7 +458,8 @@ describe("AnnoMatrixCrossfilter", () => {
       });
       xfltr = xfltr.renameObsColumn("bar", "xyz");
 
-      fetch.mockRejectOnce(new Error("unknown column name"));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).mockRejectOnce(new Error("unknown column name"));
       await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
         "unknown column name"
       );
@@ -429,7 +469,8 @@ describe("AnnoMatrixCrossfilter", () => {
     });
 
     test("addObsAnnoCategory", async () => {
-      let xfltr;
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      let xfltr: any;
 
       // catch unknown or readonly columns
       expect(() => crossfilter.addObsAnnoCategory("louvain", "mumble")).toThrow(
@@ -440,6 +481,7 @@ describe("AnnoMatrixCrossfilter", () => {
       ).toThrow("Unknown or readonly obs column");
 
       // add a column and then add category to it
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '{ name: string; type: string; ca... Remove this comment to see the full error message
       xfltr = await helperAddTestCol(crossfilter, "foo", {
         name: "foo",
         type: "categorical",
@@ -458,6 +500,7 @@ describe("AnnoMatrixCrossfilter", () => {
       );
 
       // now same, but ensure we have built an index before doing the operation
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '{ name: string; type: string; ca... Remove this comment to see the full error message
       xfltr = await helperAddTestCol(crossfilter, "bar", {
         name: "bar",
         type: "categorical",
@@ -486,6 +529,7 @@ describe("AnnoMatrixCrossfilter", () => {
         crossfilter.removeObsAnnoCategory("undefined-name", "mumble")
       ).rejects.toThrow("Unknown or readonly obs column");
 
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '{ name: string; type: string; ca... Remove this comment to see the full error message
       xfltr = await helperAddTestCol(crossfilter, "foo", {
         name: "foo",
         type: "categorical",
@@ -495,8 +539,8 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(
         (await xfltr.annoMatrix.fetch("obs", "foo"))
           .col("foo")
-          .asArray()
-          .every((v) => v === "unassigned")
+          .asArray() // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          .every((v: any) => v === "unassigned")
       ).toBeTruthy();
       expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
         name: "foo",
@@ -514,8 +558,8 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(
         (await xfltr1.annoMatrix.fetch("obs", "foo"))
           .col("foo")
-          .asArray()
-          .every((v) => v === "unassigned")
+          .asArray() // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          .every((v: any) => v === "unassigned")
       ).toBeTruthy();
       expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
         name: "foo",
@@ -537,8 +581,8 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(
         (await xfltr2.annoMatrix.fetch("obs", "foo"))
           .col("foo")
-          .asArray()
-          .every((v) => v === "red")
+          .asArray() // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          .every((v: any) => v === "red")
       ).toBeTruthy();
       expect(xfltr2.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
         name: "foo",
@@ -556,6 +600,7 @@ describe("AnnoMatrixCrossfilter", () => {
         crossfilter.setObsColumnValues("undefined-name", [0], "mumble")
       ).rejects.toThrow("Unknown or readonly obs column");
 
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '{ name: string; type: string; ca... Remove this comment to see the full error message
       let xfltr = await helperAddTestCol(crossfilter, "foo", {
         name: "foo",
         type: "categorical",
@@ -572,8 +617,8 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(
         (await xfltr.annoMatrix.fetch("obs", "foo"))
           .col("foo")
-          .asArray()
-          .every((v) => v === "unassigned")
+          .asArray() // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          .every((v: any) => v === "unassigned")
       ).toBeTruthy();
       const xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
       expect(
@@ -581,7 +626,8 @@ describe("AnnoMatrixCrossfilter", () => {
           .col("foo")
           .asArray()
           .every(
-            (v, i) =>
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+            (v: any, i: any) =>
               v === "unassigned" || (v === "purple" && (i === 0 || i === 10))
           )
       ).toBeTruthy();
@@ -615,6 +661,7 @@ describe("AnnoMatrixCrossfilter", () => {
         crossfilter.resetObsColumnValues("undefined-name", "red", "blue")
       ).rejects.toThrow("Unknown or readonly obs column");
 
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '{ name: string; type: string; ca... Remove this comment to see the full error message
       let xfltr = await helperAddTestCol(crossfilter, "foo", {
         name: "foo",
         type: "categorical",
@@ -638,22 +685,22 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(
         (await xfltr1.annoMatrix.fetch("obs", "foo"))
           .col("foo")
-          .asArray()
-          .filter((v) => v === "purple")
+          .asArray() // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          .filter((v: any) => v === "purple")
       ).toHaveLength(2);
 
       xfltr1 = await xfltr1.resetObsColumnValues("foo", "purple", "magenta");
       expect(
         (await xfltr1.annoMatrix.fetch("obs", "foo"))
           .col("foo")
-          .asArray()
-          .filter((v) => v === "magenta")
+          .asArray() // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          .filter((v: any) => v === "magenta")
       ).toHaveLength(2);
       expect(
         (await xfltr1.annoMatrix.fetch("obs", "foo"))
           .col("foo")
-          .asArray()
-          .filter((v) => v === "purple")
+          .asArray() // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          .filter((v: any) => v === "purple")
       ).toHaveLength(0);
       expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
         name: "foo",
@@ -673,12 +720,16 @@ describe("AnnoMatrixCrossfilter", () => {
   describe("edge cases", () => {
     test("transition from empty annoMatrix", async () => {
       // select before fetch needs to work
-      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (fetch as any).once(
+        serverMocks.dataframeResponse(["louvain"], [obsLouvain])
+      );
       const xfltr = await crossfilter.select("obs", "louvain", {
         mode: "exact",
         values: "B cells",
       });
-      expect(fetch.mock.calls).toHaveLength(1);
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      expect((fetch as any).mock.calls).toHaveLength(1);
       expect(xfltr.obsCrossfilter.hasDimension("obs/louvain")).toBeTruthy();
       expect(xfltr.obsCrossfilter.all()).toBe(xfltr.annoMatrix._cache.obs);
       expect(xfltr.countSelected()).toEqual(

client/__tests__/util/annoMatrix/serverMocks/index.ts

@@ -1,6 +1,7 @@
 export const baseDataURL = "https://a.fake.url/api/v0.2";
 
-window.CELLXGENE = {
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+(window as any).CELLXGENE = {
   API: {
     prefix: baseDataURL,
     version: "v0.2/",

client/__tests__/util/annoMatrix/serverMocks/routes.js

@@ -1,211 +0,0 @@
-import { schema } from "./schema";
-import { Dataframe, KeyIndex } from "../../../../src/util/dataframe";
-import { encodeMatrixFBS } from "../../../../src/util/stateManager/matrix";
-
-const indexedSchema = {
-  obsByName: Object.fromEntries(
-    schema.schema.annotations.obs.columns.map((v) => [v.name, v]) ?? []
-  ),
-  varByName: Object.fromEntries(
-    schema.schema.annotations.var.columns.map((v) => [v.name, v]) ?? []
-  ),
-  embByName: Object.fromEntries(
-    schema.schema.layout.obs.map((v) => [v.name, v]) ?? []
-  ),
-};
-
-function makeMockColumn(s, length) {
-  const { type } = s;
-  switch (type) {
-    case "int32":
-      return new Int32Array(length).fill(Math.floor(99 * Math.random()));
-
-    case "string":
-      return new Array(length).fill("test");
-
-    case "float32":
-      return new Float32Array(length).fill(99 * Math.random());
-
-    case "boolean":
-      return new Array(length).fill(false);
-
-    case "categorical":
-      return new Array(length).fill(s.categories[0]);
-
-    default:
-      throw new Error("unkonwn type");
-  }
-}
-
-function getEncodedDataframe(colNames, length, colSchemas) {
-  const colIndex = new KeyIndex(colNames);
-  const columns = colSchemas.map((s) => makeMockColumn(s, length));
-  const df = new Dataframe([length, colNames.length], columns, null, colIndex);
-  const body = encodeMatrixFBS(df);
-  return body;
-}
-
-export function dataframeResponse(colNames, columns) {
-  const colIndex = new KeyIndex(colNames);
-  const df = new Dataframe(
-    [columns[0].length, colNames.length],
-    columns,
-    null,
-    colIndex
-  );
-  const body = encodeMatrixFBS(df);
-  const headers = new Headers({
-    "Content-Type": "application/octet-stream",
-  });
-  return () => Promise.resolve({ body, init: { status: 200, headers } });
-}
-
-function annotationObsResponse(request) {
-  const url = new URL(request.url);
-  const params = Array.from(url.searchParams.entries());
-  const names = params
-    .filter(([k]) => k === "annotation-name")
-    .map(([, v]) => v);
-  if (!names.every((n) => indexedSchema.obsByName[n])) {
-    return Promise.reject(new Error("bad obs annotation name in URL"));
-  }
-  const colSchemas = names.map((n) => indexedSchema.obsByName[n]);
-  const body = getEncodedDataframe(
-    names,
-    schema.schema.dataframe.nObs,
-    colSchemas
-  );
-
-  const headers = new Headers({
-    "Content-Type": "application/octet-stream",
-  });
-  return Promise.resolve({
-    body,
-    init: { status: 200, headers },
-  });
-}
-
-function annotationVarResponse(request) {
-  const url = new URL(request.url);
-  const params = Array.from(url.searchParams.entries());
-  const names = params
-    .filter(([k]) => k === "annotation-name")
-    .map(([, v]) => v);
-  if (!names.every((n) => indexedSchema.varByName[n])) {
-    return Promise.reject(new Error("bad var annotation name in URL"));
-  }
-  const colSchemas = names.map((n) => indexedSchema.varByName[n]);
-  const body = getEncodedDataframe(
-    names,
-    schema.schema.dataframe.nVar,
-    colSchemas
-  );
-
-  const headers = new Headers({
-    "Content-Type": "application/octet-stream",
-  });
-  return Promise.resolve({
-    body,
-    init: { status: 200, headers },
-  });
-}
-
-function layoutObsResponse(request) {
-  const url = new URL(request.url);
-  const params = Array.from(url.searchParams.entries());
-  const names = params.filter(([k]) => k === "layout-name").map(([, v]) => v);
-  if (!names.every((n) => indexedSchema.embByName[n])) {
-    return Promise.reject(new Error("bad layout name in URL"));
-  }
-  const dims = names.map((n) => indexedSchema.embByName[n].dims).flat();
-  const colSchemas = names
-    .map((n) => [indexedSchema.embByName[n], indexedSchema.embByName[n]])
-    .flat();
-  const body = getEncodedDataframe(
-    dims,
-    schema.schema.dataframe.nObs,
-    colSchemas
-  );
-
-  const headers = new Headers({
-    "Content-Type": "application/octet-stream",
-  });
-  return Promise.resolve({
-    body,
-    init: { status: 200, headers },
-  });
-}
-
-function dataVarResponse(request) {
-  const url = new URL(request.url);
-  const params = Array.from(url.searchParams.entries());
-
-  const colNames = params.map((v) => `${v[0]}/${v[1]}`);
-  const colSchemas = colNames.map(() => schema.schema.dataframe);
-  const body = getEncodedDataframe(
-    colNames,
-    schema.schema.dataframe.nObs,
-    colSchemas
-  );
-
-  const headers = new Headers({
-    "Content-Type": "application/octet-stream",
-  });
-  return Promise.resolve({
-    body,
-    init: { status: 200, headers },
-  });
-}
-
-export function responder(request) {
-  const url = new URL(request.url);
-  const { pathname } = url;
-  if (pathname.endsWith("/annotations/obs")) {
-    return annotationObsResponse(request);
-  }
-  if (pathname.endsWith("/annotations/var")) {
-    return annotationVarResponse(request);
-  }
-  if (pathname.endsWith("/layout/obs")) {
-    return layoutObsResponse(request);
-  }
-  if (pathname.endsWith("/data/var")) {
-    return dataVarResponse(request);
-  }
-  return Promise.reject(new Error("bad URL"));
-}
-
-export function withExpected(expectedURL, expectedParams) {
-  /*
-  Do some additional error checking
-  */
-  return (request) => {
-    // if URL is bogus, reject the promise
-    const url = new URL(request.url);
-    if (!url.pathname.endsWith(expectedURL)) {
-      return Promise.reject(new Error("Unexpected URL!"));
-    }
-    const params = Array.from(url.searchParams.entries()).sort(
-      (a, b) => a[0] < b[0]
-    );
-    expectedParams = expectedParams.slice().sort((a, b) => a[0] < b[0]);
-
-    if (
-      params.length !== expectedParams.length ||
-      !params.every(
-        (p, i) => p[0] === expectedParams[i][0] && p[1] === expectedParams[i][1]
-      )
-    ) {
-      return Promise.reject(new Error("unexpected name requested in URL"));
-    }
-
-    return responder(request);
-  };
-}
-
-export function annotationsObs(names) {
-  return withExpected(
-    "/annotations/obs",
-    names.map((name) => ["annotation-name", name])
-  );
-}

client/__tests__/util/annoMatrix/serverMocks/routes.ts

@@ -0,0 +1,251 @@
+import { schema } from "./schema";
+import { Dataframe, KeyIndex } from "../../../../src/util/dataframe";
+import { encodeMatrixFBS } from "../../../../src/util/stateManager/matrix";
+
+const indexedSchema = {
+  obsByName: Object.fromEntries(
+    schema.schema.annotations.obs.columns.map((v) => [v.name, v]) ?? []
+  ),
+  varByName: Object.fromEntries(
+    schema.schema.annotations.var.columns.map((v) => [v.name, v]) ?? []
+  ),
+  embByName: Object.fromEntries(
+    schema.schema.layout.obs.map((v) => [v.name, v]) ?? []
+  ),
+};
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+function makeMockColumn(s: any, length: any) {
+  const { type } = s;
+  switch (type) {
+    case "int32":
+      return new Int32Array(length).fill(Math.floor(99 * Math.random()));
+
+    case "string":
+      return new Array(length).fill("test");
+
+    case "float32":
+      return new Float32Array(length).fill(99 * Math.random());
+
+    case "boolean":
+      return new Array(length).fill(false);
+
+    case "categorical":
+      return new Array(length).fill(s.categories[0]);
+
+    default:
+      throw new Error("unknown type");
+  }
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+function getEncodedDataframe(colNames: any, length: any, colSchemas: any) {
+  const colIndex = new KeyIndex(colNames);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  const columns = colSchemas.map((s: any) => makeMockColumn(s, length));
+  // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
+  const df = new Dataframe([length, colNames.length], columns, null, colIndex);
+  const body = encodeMatrixFBS(df);
+  return body;
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
+export function dataframeResponse(colNames: any, columns: any) {
+  const colIndex = new KeyIndex(colNames);
+  const df = new Dataframe(
+    [columns[0].length, colNames.length],
+    columns,
+    null,
+    // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
+    colIndex
+  );
+  const body = encodeMatrixFBS(df);
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return () => Promise.resolve({ body, init: { status: 200, headers } });
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+function annotationObsResponse(request: any) {
+  const url = new URL(request.url);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  const params = Array.from((url.searchParams as any).entries());
+  const names = params
+    // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
+    .filter(([k]) => k === "annotation-name")
+    // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '([, v]: [any, any]) => any' is n... Remove this comment to see the full error message
+    .map(([, v]) => v);
+  // @ts-expect-error ts-migrate(2538) FIXME: Type 'unknown' cannot be used as an index type.
+  if (!names.every((n) => indexedSchema.obsByName[n])) {
+    return Promise.reject(new Error("bad obs annotation name in URL"));
+  }
+  // @ts-expect-error ts-migrate(2538) FIXME: Type 'unknown' cannot be used as an index type.
+  const colSchemas = names.map((n) => indexedSchema.obsByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+function annotationVarResponse(request: any) {
+  const url = new URL(request.url);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  const params = Array.from((url.searchParams as any).entries());
+  const names = params
+    // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
+    .filter(([k]) => k === "annotation-name")
+    // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '([, v]: [any, any]) => any' is n... Remove this comment to see the full error message
+    .map(([, v]) => v);
+  // @ts-expect-error ts-migrate(2538) FIXME: Type 'unknown' cannot be used as an index type.
+  if (!names.every((n) => indexedSchema.varByName[n])) {
+    return Promise.reject(new Error("bad var annotation name in URL"));
+  }
+  // @ts-expect-error ts-migrate(2538) FIXME: Type 'unknown' cannot be used as an index type.
+  const colSchemas = names.map((n) => indexedSchema.varByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nVar,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+function layoutObsResponse(request: any) {
+  const url = new URL(request.url);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  const params = Array.from((url.searchParams as any).entries());
+  // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
+  const names = params.filter(([k]) => k === "layout-name").map(([, v]) => v);
+  // @ts-expect-error ts-migrate(2538) FIXME: Type 'unknown' cannot be used as an index type.
+  if (!names.every((n) => indexedSchema.embByName[n])) {
+    return Promise.reject(new Error("bad layout name in URL"));
+  }
+  // @ts-expect-error ts-migrate(2538) FIXME: Type 'unknown' cannot be used as an index type.
+  const dims = names.map((n) => indexedSchema.embByName[n].dims).flat();
+  const colSchemas = names
+    // @ts-expect-error ts-migrate(2538) FIXME: Type 'unknown' cannot be used as an index type.
+    .map((n) => [indexedSchema.embByName[n], indexedSchema.embByName[n]])
+    .flat();
+  const body = getEncodedDataframe(
+    dims,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+function dataVarResponse(request: any) {
+  const url = new URL(request.url);
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  const params = Array.from((url.searchParams as any).entries());
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  const colNames = params.map((v) => `${(v as any)[0]}/${(v as any)[1]}`);
+  const colSchemas = colNames.map(() => schema.schema.dataframe);
+  const body = getEncodedDataframe(
+    colNames,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
+export function responder(request: any) {
+  const url = new URL(request.url);
+  const { pathname } = url;
+  if (pathname.endsWith("/annotations/obs")) {
+    return annotationObsResponse(request);
+  }
+  if (pathname.endsWith("/annotations/var")) {
+    return annotationVarResponse(request);
+  }
+  if (pathname.endsWith("/layout/obs")) {
+    return layoutObsResponse(request);
+  }
+  if (pathname.endsWith("/data/var")) {
+    return dataVarResponse(request);
+  }
+  return Promise.reject(new Error("bad URL"));
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
+export function withExpected(expectedURL: any, expectedParams: any) {
+  /*
+  Do some additional error checking
+  */
+  // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
+  return (request: any) => {
+    // if URL is bogus, reject the promise
+    const url = new URL(request.url);
+    if (!url.pathname.endsWith(expectedURL)) {
+      return Promise.reject(new Error("Unexpected URL!"));
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    const params = Array.from((url.searchParams as any).entries()).sort(
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type '(a: unknown, b: unknown) => bool... Remove this comment to see the full error message
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (a, b) => (a as any)[0] < (b as any)[0]
+    );
+    expectedParams = expectedParams
+      .slice() // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      .sort((a: any, b: any) => a[0] < b[0]);
+
+    if (
+      params.length !== expectedParams.length ||
+      !params.every(
+        (p, i) =>
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          (p as any)[0] === expectedParams[i][0] && // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+          (p as any)[1] === expectedParams[i][1]
+      )
+    ) {
+      return Promise.reject(new Error("unexpected name requested in URL"));
+    }
+
+    return responder(request);
+  };
+}
+
+// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any -- - FIXME: disabled temporarily on migrate to TS.
+export function annotationsObs(names: any) {
+  return withExpected(
+    "/annotations/obs",
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    names.map((name: any) => ["annotation-name", name])
+  );
+}

client/__tests__/util/annoMatrix/serverMocks/schema.ts

@@ -1,4 +1,6 @@
-export const schema = {
+import { RawSchema } from "../../../../src/common/types/schema";
+
+export const schema: { schema: RawSchema } = {
   schema: {
     annotations: {
       obs: {

client/__tests__/util/annoMatrix/whereCache.test.ts

@@ -218,10 +218,18 @@ describe("whereCache", () => {
         },
       })
     );
-    expect(wc.where.field.queryField.has("queryColumn")).toEqual(true);
-    expect(wc.where.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    expect((wc.where as any).field.queryField.has("queryColumn")).toEqual(true);
     expect(
-      wc.where.field.queryField.get("queryColumn").has("queryValue")
+      // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (wc.where as any).field.queryField.get("queryColumn")
+    ).toBeInstanceOf(Map);
+    expect(
+      // @ts-expect-error ts-migrate(2531) FIXME: Object is possibly 'null'.
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      (wc.where as any).field.queryField.get("queryColumn").has("queryValue")
     ).toEqual(true);
     expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
   });

client/__tests__/util/centroid.test.ts

@@ -5,19 +5,22 @@ import quantile from "../../src/util/quantile";
 import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
 import * as REST from "./stateManager/sampleResponses";
 import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
-import { _normalizeCategoricalSchema } from "../../src/annoMatrix/schema";
+import { normalizeWritableCategoricalSchema } from "../../src/annoMatrix/normalize";
 
 describe("centroid", () => {
-  let schema;
-  let obsAnnotations;
-  let obsLayout;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  let schema: any;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  let obsAnnotations: any;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  let obsLayout: any;
 
   beforeAll(() => {
     schema = indexEntireSchema(cloneDeep(REST.schema.schema));
     obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
     obsLayout = matrixFBSToDataframe(REST.layoutObs);
 
-    _normalizeCategoricalSchema(
+    normalizeWritableCategoricalSchema(
       schema.annotations.obsByName.field3,
       obsAnnotations.col("field3")
     );
@@ -44,7 +47,8 @@ describe("centroid", () => {
       quantile([0.5], obsLayout.col("umap_1").asArray())[0],
     ];
 
-    centroidResult.forEach((coordinate) => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    centroidResult.forEach((coordinate: any) => {
       expect(coordinate).toEqual(expectedResult);
     });
   });
@@ -68,7 +72,8 @@ describe("centroid", () => {
       quantile([0.5], obsLayout.col("umap_1").asArray())[0],
     ];
 
-    centroidResult.forEach((coordinate) => {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    centroidResult.forEach((coordinate: any) => {
       expect(coordinate).toEqual(expectedResult);
     });
   });

client/__tests__/util/dataframe/dataframe.test.ts

@@ -29,6 +29,7 @@ describe("dataframe constructor", () => {
     const df = new Dataframe.Dataframe(
       [3, 2],
       [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'DenseInt32Index' is not assignab... Remove this comment to see the full error message
       new Dataframe.DenseInt32Index([2, 1, 0]),
       new Dataframe.KeyIndex(["A", "B"])
     );
@@ -55,6 +56,7 @@ describe("simple data access", () => {
       new Float64Array([0.0, Number.NaN, Number.POSITIVE_INFINITY, 3.14159]),
       ["red", "blue", "green", "nan"],
     ],
+    // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'DenseInt32Index' is not assignab... Remove this comment to see the full error message
     new Dataframe.DenseInt32Index([3, 2, 1, 0]),
     new Dataframe.KeyIndex(["numbers", "colors"])
   );
@@ -139,10 +141,12 @@ describe("dataframe subsetting", () => {
         ["red", "green", "blue"],
       ],
       null, // identity index
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
     );
 
     test("all rows, one column", () => {
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'string[]' is not assignable to p... Remove this comment to see the full error message
       const dfA = sourceDf.subset(null, ["colors"]);
       expect(dfA).toBeDefined();
       expect(dfA.dims).toEqual([3, 1]);
@@ -158,6 +162,7 @@ describe("dataframe subsetting", () => {
     });
 
     test("all rows, two columns", () => {
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'string[]' is not assignable to p... Remove this comment to see the full error message
       const dfB = sourceDf.subset(null, ["float32", "colors"]);
       expect(dfB).toBeDefined();
       expect(dfB.dims).toEqual([3, 2]);
@@ -227,6 +232,7 @@ describe("dataframe subsetting", () => {
     });
 
     test("two rows, two colums", () => {
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'string[]' is not assignable to p... Remove this comment to see the full error message
       const dfF = sourceDf.subset([0, 2], ["int32", "float32"]);
       expect(dfF).toBeDefined();
       expect(dfF.dims).toEqual([2, 2]);
@@ -236,6 +242,7 @@ describe("dataframe subsetting", () => {
       expect(dfF.colIndex.labels()).toEqual(["int32", "float32"]);
 
       // reverse the row and column order
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'string[]' is not assignable to p... Remove this comment to see the full error message
       const dfFr = sourceDf.subset([2, 0], ["float32", "int32"]);
       expect(dfFr).toBeDefined();
       expect(dfFr.dims).toEqual([2, 2]);
@@ -248,6 +255,7 @@ describe("dataframe subsetting", () => {
     test("withRowIndex", () => {
       const df = sourceDf.subset(
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'string[]' is not assignable to p... Remove this comment to see the full error message
         ["int32", "float32"],
         new Dataframe.DenseInt32Index([3, 2, 1])
       );
@@ -258,12 +266,15 @@ describe("dataframe subsetting", () => {
 
     test("withRowIndex error checks", () => {
       expect(() =>
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'string[]' is not assignable to p... Remove this comment to see the full error message
         sourceDf.subset(null, ["red"], new Dataframe.IdentityInt32Index(1))
       ).toThrow(RangeError);
       expect(() =>
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'string[]' is not assignable to p... Remove this comment to see the full error message
         sourceDf.subset(null, ["red"], new Dataframe.DenseInt32Index([0, 1]))
       ).toThrow(RangeError);
       expect(() =>
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'string[]' is not assignable to p... Remove this comment to see the full error message
         sourceDf.subset(null, ["red"], new Dataframe.KeyIndex([0, 1, 2, 3]))
       ).toThrow(RangeError);
     });
@@ -278,12 +289,14 @@ describe("dataframe subsetting", () => {
         new Float32Array([4.4, 5.5, 6.6]),
         ["red", "green", "blue"],
       ],
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'DenseInt32Index' is not assignab... Remove this comment to see the full error message
       new Dataframe.DenseInt32Index([2, 4, 6]),
       new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
     );
 
     const dfA = sourceDf.isubsetMask(
       new Uint8Array([0, 1, 1]),
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'Uint8Array' is not assignable to... Remove this comment to see the full error message
       new Uint8Array([1, 0, 0, 1])
     );
     expect(dfA.dims).toEqual([2, 2]);
@@ -303,6 +316,7 @@ describe("dataframe subsetting", () => {
         ["red", "green", "blue"],
       ],
       null, // identity index
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
     );
 
@@ -316,6 +330,7 @@ describe("dataframe subsetting", () => {
     });
 
     test("all rows, two cols", () => {
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'number[]' is not assignable to p... Remove this comment to see the full error message
       const dfA = sourceDf.isubset(null, [1, 2]);
       expect(dfA.dims).toEqual([3, 2]);
       expect(dfA.icol(0).asArray()).toEqual(["A", "B", "C"]);
@@ -361,6 +376,7 @@ describe("dataframe factories", () => {
     const dfA = new Dataframe.Dataframe(
       [3, 2],
       [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'DenseInt32Index' is not assignab... Remove this comment to see the full error message
       new Dataframe.DenseInt32Index([2, 1, 0]),
       new Dataframe.KeyIndex(["A", "B"])
     );
@@ -385,6 +401,7 @@ describe("dataframe factories", () => {
           [true, false],
         ],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["colors", "bools"])
       );
       const dfA = df.withCol("numbers", [1, 0]);
@@ -408,6 +425,7 @@ describe("dataframe factories", () => {
           [true, false],
         ],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'DenseInt32Index' is not assignab... Remove this comment to see the full error message
         new Dataframe.DenseInt32Index([74, 75])
       );
       const dfA = df.withCol(72, [1, 0]);
@@ -433,6 +451,7 @@ describe("dataframe factories", () => {
           [true, false],
         ],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'DenseInt32Index' is not assignab... Remove this comment to see the full error message
         new Dataframe.DenseInt32Index([74, 75])
       );
       const dfA = df.withCol(999, [1, 0]);
@@ -541,6 +560,7 @@ describe("dataframe factories", () => {
           [1, 0],
         ],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["colors", "bools", "numbers"])
       );
 
@@ -549,11 +569,14 @@ describe("dataframe factories", () => {
         [3, 1],
         [["red", "blue", "green"]],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["colorsA"])
       );
+      // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
       expect(() => dfA.withColsFrom(dfB)).toThrow(RangeError);
 
       /* duplicate labels should throw an error */
+      // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
       expect(() => dfA.withColsFrom(dfA)).toThrow(Error);
     });
 
@@ -564,15 +587,18 @@ describe("dataframe factories", () => {
         [2, 1],
         [["red", "blue"]],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["colors"])
       );
       const dfB = new Dataframe.Dataframe(
         [2, 1],
         [[true, false]],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["bools"])
       );
 
+      // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
       const dfLikeA = dfEmpty.withColsFrom(dfA);
       expect(dfLikeA).toBeDefined();
       expect(dfLikeA.dims).toEqual(dfA.dims);
@@ -581,6 +607,7 @@ describe("dataframe factories", () => {
       expect(dfLikeA.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
       expect(dfLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
+      // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
       const dfAlsoLikeA = dfA.withColsFrom(dfEmpty);
       expect(dfAlsoLikeA).toBeDefined();
       expect(dfAlsoLikeA.dims).toEqual(dfA.dims);
@@ -589,6 +616,7 @@ describe("dataframe factories", () => {
       expect(dfAlsoLikeA.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
       expect(dfAlsoLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
+      // @ts-expect-error ts-migrate(2554) FIXME: Expected 2 arguments, but got 1.
       const dfC = dfA.withColsFrom(dfB);
       expect(dfC).toBeDefined();
       expect(dfC.dims).toEqual([2, 2]);
@@ -605,6 +633,7 @@ describe("dataframe factories", () => {
         [2, 1],
         [["red", "blue"]],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["colors"])
       );
       const dfB = new Dataframe.Dataframe(
@@ -615,6 +644,7 @@ describe("dataframe factories", () => {
           [1, 0],
         ],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["colors", "bools", "numbers"])
       );
 
@@ -647,6 +677,7 @@ describe("dataframe factories", () => {
         [2, 1],
         [["red", "blue"]],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["colors"])
       );
       const dfB = new Dataframe.Dataframe(
@@ -657,6 +688,7 @@ describe("dataframe factories", () => {
           [1, 0],
         ],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["colors", "bools", "numbers"])
       );
 
@@ -680,6 +712,7 @@ describe("dataframe factories", () => {
           [1, 0],
         ],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["colors", "bools", "numbers"])
       );
       const dfA = df.dropCol("colors");
@@ -751,6 +784,7 @@ describe("dataframe factories", () => {
           [1, 0],
         ],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'DenseInt32Index' is not assignab... Remove this comment to see the full error message
         new Dataframe.DenseInt32Index([102, 101, 100])
       );
       const dfA = df.dropCol(101);
@@ -777,7 +811,8 @@ describe("dataframe factories", () => {
           new Float64Array(3).fill(1.1),
         ]
       );
-      const dfB = dfA.mapColumns((col, idx) => {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+      const dfB = dfA.mapColumns((col: any, idx: any) => {
         expect(dfA.icol(idx).asArray()).toBe(col);
         return col;
       });
@@ -793,9 +828,7 @@ describe("dataframe factories", () => {
         [3, 3],
         [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
       );
-      const dfB = dfA.mapColumns(() => {
-        return new Array(3).fill(1);
-      });
+      const dfB = dfA.mapColumns(() => new Array(3).fill(1));
       expect(dfA).not.toBe(dfB);
       expect(dfB.iat(0, 0)).toEqual(1);
       expect(dfB.iat(0, 1)).toEqual(1);
@@ -822,6 +855,7 @@ describe("dataframe factories", () => {
           [1, 0],
         ],
         null,
+        // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
         new Dataframe.KeyIndex(["A", "B"])
       );
       const dfB = dfA.renameCol("B", "C");
@@ -834,7 +868,8 @@ describe("dataframe factories", () => {
 });
 
 describe("dataframe col", () => {
-  let df = null;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+  let df: any = null;
   beforeEach(() => {
     df = new Dataframe.Dataframe(
       [2, 2],
@@ -843,6 +878,7 @@ describe("dataframe col", () => {
         [1, 0],
       ],
       null,
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex(["A", "B"])
     );
   });
@@ -1195,6 +1231,7 @@ describe("label indexing", () => {
     test("create", () => {
       expect(Dataframe.isLabelIndex(idx)).toBeTruthy();
       expect(() => new Dataframe.KeyIndex(["dup", "dup"])).toThrow(Error);
+      // @ts-expect-error ts-migrate(2554) FIXME: Expected 1 arguments, but got 0.
       expect(new Dataframe.KeyIndex().size()).toEqual(0);
     });
 
@@ -1367,6 +1404,7 @@ describe("corner cases", () => {
         [1, 0],
       ],
       null,
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex(["A", "B"])
     );
 

client/__tests__/util/dataframe/histogram.test.ts

@@ -6,6 +6,7 @@ describe("Dataframe column histogram", () => {
       [3, 3],
       [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
       null,
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex(["name", "cat", "value"])
     );
 
@@ -26,6 +27,7 @@ describe("Dataframe column histogram", () => {
       [3, 3],
       [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
       null,
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex(["name", "cat", "value"])
     );
 
@@ -48,6 +50,7 @@ describe("Dataframe column histogram", () => {
       [3, 3],
       [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
       null,
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex(["name", "cat", "value"])
     );
 
@@ -68,6 +71,7 @@ describe("Dataframe column histogram", () => {
       [3, 3],
       [["n1", "n2", "n3"], ["c1", "c2", "c3"], new Int32Array([0, 1, 2])],
       null,
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex(["name", "cat", "value"])
     );
 

client/__tests__/util/dataframe/summarize.test.ts

@@ -1,6 +1,7 @@
 import * as Dataframe from "../../../src/util/dataframe";
 
-function float32Conversion(f) {
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+function float32Conversion(f: any) {
   return new Float32Array([f])[0];
 }
 
@@ -30,6 +31,7 @@ describe("Dataframe column summary", () => {
         [1],
       ],
       null,
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex([
         "name",
         "nameString",
@@ -106,6 +108,7 @@ describe("Dataframe column summary", () => {
         [1, false, "0"],
       ],
       null,
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex([
         "name",
         "nameString",
@@ -174,6 +177,7 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
+        // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
         categoryCounts: new Map([
           [1, 1],
           [false, 1],
@@ -201,6 +205,7 @@ describe("Dataframe column summary", () => {
         [1, false, "0", "0"],
       ],
       null,
+      // @ts-expect-error ts-migrate(2345) FIXME: Argument of type 'KeyIndex' is not assignable to p... Remove this comment to see the full error message
       new Dataframe.KeyIndex([
         "name",
         "nameString",
@@ -269,6 +274,7 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
+        // @ts-expect-error ts-migrate(2769) FIXME: No overload matches this call.
         categoryCounts: new Map([
           [1, 1],
           [false, 1],

client/__tests__/util/promiseLimit.test.ts

@@ -1,7 +1,8 @@
 import PromiseLimit from "../../src/util/promiseLimit";
 import { range } from "../../src/util/range";
 
-const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
+// eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+const delay = (t: any) => new Promise((resolve) => setTimeout(resolve, t));
 
 describe("PromiseLimit", () => {
   test("simple evaluation, concurrency 1", async () => {
@@ -51,7 +52,9 @@ describe("PromiseLimit", () => {
       running -= 1;
     };
 
-    await Promise.all(range(10).map((i) => plimit.add(() => callback(i))));
+    // @ts-expect-error ts-migrate(2554) FIXME: Expected 3 arguments, but got 1.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any --- FIXME: disabled temporarily on migrate to TS.
+    await Promise.all(range(10).map((i: any) => plimit.add(() => callback(i))));
 
     expect(maxRunning).toEqual(2);
   });