Refactoring cxg utility classes in preparation for CXG conversion tooling (#1739)

server/__init__.py

@@ -1,7 +1,8 @@
-from server.common.utils import import_plugins
 import logging
 import sys
 
+from server.common.utils.utils import import_plugins
+
 __version__ = "0.16.0"
 display_version = "cellxgene v" + __version__
 

server/app/app.py

@@ -1,22 +1,19 @@
 import datetime
 import logging
+from functools import wraps
+from http import HTTPStatus
 
-from flask import Flask, redirect, current_app, make_response, render_template, abort
-from flask import Blueprint, request
+from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
-from http import HTTPStatus
-
 import server.common.rest as common_rest
-from server.common.errors import DatasetAccessError, RequestException
-from server.common.utils import path_join, Float32JSONEncoder
 from server.common.data_locator import DataLocator
+from server.common.errors import DatasetAccessError, RequestException
 from server.common.health import health_check
+from server.common.utils.utils import path_join, Float32JSONEncoder
 from server.data_common.matrix_loader import MatrixDataLoader
 
-from functools import wraps
-
 webbp = Blueprint("webapp", "server.common.web", template_folder="templates")
 
 ONE_WEEK = 7 * 24 * 60 * 60

server/cli/launch.py

@@ -1,19 +1,19 @@
 import errno
 import functools
 import logging
-from os import devnull
 import sys
 import webbrowser
+from os import devnull
 
 import click
 from flask_compress import Compress
 from flask_cors import CORS
 
-from server.common.utils import sort_options
-from server.common.errors import DatasetAccessError, ConfigurationError
+from server.app.app import Server
 from server.common.app_config import AppConfig
 from server.common.default_config import default_config
-from server.app.app import Server
+from server.common.errors import DatasetAccessError, ConfigurationError
+from server.common.utils.utils import sort_options
 
 DEFAULT_CONFIG = AppConfig()
 
@@ -33,7 +33,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<path>",
         help="CSV file to initialize editing of existing annotations; will be altered in-place. "
-        "Incompatible with --annotations-dir.",
+             "Incompatible with --annotations-dir.",
     )
     @click.option(
         "--annotations-dir",
@@ -42,7 +42,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<directory path>",
         help="Directory of where to save output annotations; filename will be specified in the application. "
-        "Incompatible with --annotations-file.",
+             "Incompatible with --annotations-file.",
     )
     @click.option(
         "--experimental-annotations-ontology",
@@ -170,7 +170,7 @@ def server_args(func):
         default=DEFAULT_CONFIG.server_config.app__debug,
         show_default=True,
         help="Run in debug mode. This is helpful for cellxgene developers, "
-        "or when you want more information about an error condition.",
+             "or when you want more information about an error condition.",
     )
     @click.option(
         "--verbose",
@@ -203,7 +203,7 @@ def server_args(func):
         multiple=True,
         metavar="<text>",
         help="Additional script files to include in HTML page. If not specified, "
-        "no additional script files will be included.",
+             "no additional script files will be included.",
         show_default=False,
     )
     @functools.wraps(func)
@@ -223,7 +223,7 @@ def launch_args(func):
         default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
         metavar="<data directory>",
         help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
-        " to folder containing H5AD and/or CXG datasets.",
+             " to folder containing H5AD and/or CXG datasets.",
         hidden=True,
     )  # TODO, unhide when dataroot is supported)
     @click.argument("datapath", required=False, metavar="<path to data file>")
@@ -307,32 +307,32 @@ class CliLaunchServer(Server):
 )
 @launch_args
 def launch(
-    datapath,
-    dataroot,
-    verbose,
-    debug,
-    open_browser,
-    port,
-    host,
-    embedding,
-    obs_names,
-    var_names,
-    max_category_items,
-    disable_custom_colors,
-    diffexp_lfc_cutoff,
-    title,
-    scripts,
-    about,
-    disable_annotations,
-    annotations_file,
-    annotations_dir,
-    backed,
-    disable_diffexp,
-    experimental_annotations_ontology,
-    experimental_annotations_ontology_obo,
-    experimental_enable_reembedding,
-    config_file,
-    dump_default_config,
+        datapath,
+        dataroot,
+        verbose,
+        debug,
+        open_browser,
+        port,
+        host,
+        embedding,
+        obs_names,
+        var_names,
+        max_category_items,
+        disable_custom_colors,
+        diffexp_lfc_cutoff,
+        title,
+        scripts,
+        about,
+        disable_annotations,
+        annotations_file,
+        annotations_dir,
+        backed,
+        disable_diffexp,
+        experimental_annotations_ontology,
+        experimental_annotations_ontology_obo,
+        experimental_enable_reembedding,
+        config_file,
+        dump_default_config,
 ):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.

server/cli/prepare.py

@@ -5,7 +5,7 @@ import pandas as pd
 from numpy import ndarray, unique
 from scipy.sparse.csc import csc_matrix
 
-from server.common.utils import sort_options
+from server.common.utils.utils import sort_options
 
 
 @sort_options
@@ -37,7 +37,7 @@ from server.common.utils import sort_options
     default=False,
     is_flag=True,
     help="Do not run quality control metrics. By default cellxgene runs them "
-    "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
+         "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
 )
 @click.option(
     "--make-obs-names-unique/--no-make-obs-names-unique",
@@ -53,18 +53,18 @@ from server.common.utils import sort_options
 )
 @click.help_option("--help", "-h", help="Show this message and exit.")
 def prepare(
-    data,
-    embedding,
-    recipe,
-    output,
-    plotting,
-    sparse,
-    overwrite,
-    set_obs_names,
-    set_var_names,
-    skip_qc,
-    make_obs_names_unique,
-    make_var_names_unique,
+        data,
+        embedding,
+        recipe,
+        output,
+        plotting,
+        sparse,
+        overwrite,
+        set_obs_names,
+        set_var_names,
+        skip_qc,
+        make_obs_names_unique,
+        make_var_names_unique,
 ):
     """
     Preprocess data for use with cellxgene.

server/common/annotations/annotations.py

@@ -2,9 +2,9 @@ from abc import ABCMeta, abstractmethod
 
 import fastobo
 import fsspec
-from server.common.errors import OntologyLoadFailure
 
-from server.common.utils import series_to_schema
+from server.common.errors import OntologyLoadFailure
+from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
 
 
 class Annotations(metaclass=ABCMeta):
@@ -44,7 +44,7 @@ class Annotations(metaclass=ABCMeta):
         if labels is not None and not labels.empty:
             for col in labels.columns:
                 col_schema = dict(name=col, writable=True)
-                col_schema.update(series_to_schema(labels[col]))
+                col_schema.update(get_schema_type_hint_of_array(labels[col]))
                 schema.append(col_schema)
 
         return schema

server/common/annotations/local_file_csv.py

@@ -14,7 +14,6 @@ from server.common.errors import AnnotationsError
 
 
 class AnnotationsLocalFile(Annotations):
-
     CXG_ANNO_COLLECTION = "cxg_anno_collection"
 
     def __init__(self, output_dir, output_file):

server/common/app_config.py

@@ -1,23 +1,23 @@
-from server import display_version as cellxgene_display_version
-from flatten_dict import flatten, unflatten
-import os
-from os.path import splitext, basename, isdir
-import sys
-from urllib.parse import urlparse, quote_plus
-import yaml
 import copy
-
-from server.common.default_config import get_default_config
-from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
-from server.common.utils import find_available_port, is_port_available
+import os
+import sys
 import warnings
+from os.path import splitext, basename, isdir
+from urllib.parse import urlparse, quote_plus
+
+import yaml
+from flatten_dict import flatten, unflatten
+
+import server.compute.diffexp_cxg as diffexp_tiledb
+from server import display_version as cellxgene_display_version
+from server.auth.auth import AuthTypeFactory
 from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
 from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.utils import custom_format_warning
-import server.compute.diffexp_cxg as diffexp_tiledb
 from server.common.data_locator import discover_s3_region_name
-from server.auth.auth import AuthTypeFactory
+from server.common.default_config import get_default_config
+from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
+from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available
+from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
 from server.db.db_utils import DbUtils
 
 DEFAULT_SERVER_PORT = 5005
@@ -150,7 +150,6 @@ class AppConfig(object):
         parameters is done"""
 
         if messagefn is None:
-
             def noop(message):
                 pass
 
@@ -284,7 +283,7 @@ class AppConfig(object):
             if auth.requires_client_login():
                 config["authentication"].update({
                     "login": auth.get_login_url(data_adaptor),
-                    "logout" : auth.get_logout_url(data_adaptor),
+                    "logout": auth.get_logout_url(data_adaptor),
                 })
 
         return c
@@ -748,7 +747,8 @@ class DatasetConfig(BaseConfig):
             self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
             self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
             self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
-            self.user_annotations__hosted_tiledb_array__hosted_file_directory = dc["user_annotations"]["hosted_tiledb_array"]["hosted_file_directory"]  # noqa E501
+            self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
+            dc["user_annotations"]["hosted_tiledb_array"]["hosted_file_directory"]  # noqa E501
 
             self.embeddings__names = dc["embeddings"]["names"]
             self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
@@ -896,7 +896,7 @@ class DatasetConfig(BaseConfig):
         server_config = self.app_config.server_config
         if server_config.single_dataset__datapath:
             with server_config.matrix_data_cache_manager.data_adaptor(
-                self.tag, server_config.single_dataset__datapath, self.app_config
+                    self.tag, server_config.single_dataset__datapath, self.app_config
             ) as data_adaptor:
                 if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
                     context["messagefn"](

server/common/utils/matrix_utils.py

@@ -0,0 +1,112 @@
+import logging
+
+import numpy as np
+from scipy.stats import mode
+
+
+def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
+    """
+    Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has
+    a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of
+    elements evaluated.  This function may return before evaluating the whole matrix if it can be determined that matrix
+    is not sparse enough.
+    """
+
+    if sparse_threshold == 100.0:
+        return True
+    if sparse_threshold == 0.0:
+        return False
+
+    total_number_of_rows = matrix.shape[0]
+    total_number_of_columns = matrix.shape[1]
+    total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
+
+    # For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the
+    # maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the
+    # quit early for large dense matrices.
+    row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000)
+
+    maximum_number_of_non_zero_elements_in_matrix = int(
+        total_number_of_rows * total_number_of_columns * sparse_threshold / 100
+    )
+    number_of_non_zero_elements = 0
+
+    for start_row_index in range(0, total_number_of_rows, row_stride):
+        end_row_index = min(start_row_index + row_stride, total_number_of_rows)
+
+        matrix_subset = matrix[start_row_index:end_row_index, :]
+        if not isinstance(matrix_subset, np.ndarray):
+            matrix_subset = matrix_subset.toarray()
+
+        number_of_non_zero_elements += np.count_nonzero(matrix_subset)
+        if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
+            if end_row_index != total_number_of_rows:
+                percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / (
+                    end_row_index * total_number_of_columns)
+                logging.info(
+                    f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
+                    f"{percentage_of_non_zero_elements:6.2f}")
+            else:
+                percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
+                logging.info(
+                    f"Matrix is not sparse. Percentage of non-zero elements (exact): "
+                    f"{percentage_of_non_zero_elements:6.2f}")
+            return False
+
+    is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
+    return is_sparse
+
+
+def get_column_shift_encode_for_matrix(matrix, sparse_threshold):
+    """
+    Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column
+    shift encoding works by taking the most common value in each column, then subtracting that value from each  element
+    of the column.  If each column mostly contains its most common value, then the resulting matrix can be very sparse.
+
+    This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a
+    sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns
+    the number of non-zeros encountered and number of elements evaluated.  This function may return before evaluating
+    the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding.
+    """
+
+    total_number_of_rows = matrix.shape[0]
+    total_number_of_columns = matrix.shape[1]
+    total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
+
+    stride = max(1, 128_000_000 // total_number_of_rows)
+    column_shift = np.zeros(total_number_of_columns)
+
+    maximum_number_of_non_zero_elements_in_matrix = int(
+        total_number_of_rows * total_number_of_columns * sparse_threshold / 100
+    )
+    number_of_non_zero_elements = 0
+
+    for start_column_index in range(0, total_number_of_columns, stride):
+        end_column_index = min(start_column_index + stride, total_number_of_columns)
+
+        matrix_subset = matrix[:, start_column_index:end_column_index]
+        if not isinstance(matrix_subset, np.ndarray):
+            matrix_subset = matrix_subset.toarray()
+
+        matrix_subset_mode = mode(matrix_subset)
+
+        column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode
+        number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum(
+            matrix_subset_mode.count
+        )
+
+        if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
+            if end_column_index != total_number_of_columns:
+                logging.info(
+                    "Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f"
+                    % (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows)
+                )
+            else:
+                logging.info(
+                    "Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f"
+                    % (100 * number_of_non_zero_elements / total_number_of_matrix_elements)
+                )
+            return None
+
+    is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
+    return column_shift if is_sparse else None

server/common/utils/sanitization_utils.py

@@ -0,0 +1,40 @@
+import re
+
+
+def sanitize_values_in_list(list_of_keys: list):
+    """
+    Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both
+    safe and unique.
+    """
+
+    if not all([isinstance(key, str) for key in list_of_keys]):
+        raise Exception("List of keys to sanitize must contain all strings.")
+
+    # Mask out [~/.] and anything outside the ASCII range.
+    mask = re.compile(r"[^ -\-0-\[\]-\}]")
+    clean_keys_list = [mask.sub("_", key) for key in list_of_keys]
+
+    # Dedupe the clean keys list
+    deduped_clean_keys_list = []
+    for index, clean_key in enumerate(clean_keys_list):
+        total_occurrences_of_clean_key = clean_keys_list.count(clean_key)
+        total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key)
+        deduped_clean_keys_list.append(
+            clean_key + "_" + str(total_occurrences_up_until_current_index + 1)
+            if total_occurrences_of_clean_key > 1
+            else clean_key
+        )
+
+    return dict(zip(list_of_keys, deduped_clean_keys_list))
+
+
+def sanitize_keys_in_dictionary(dict_to_sanitize: dict):
+    """
+    Clean and dedupe the keys in the given dictionary.
+    """
+
+    clean_keys = sanitize_values_in_list(dict_to_sanitize.keys())
+    for original_key, sanitized_key in clean_keys.items():
+        if original_key != sanitized_key:
+            dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key]
+            del dict_to_sanitize[original_key]

server/common/utils/type_conversion_utils.py

@@ -0,0 +1,93 @@
+import logging
+
+import numpy as np
+import pandas as pd
+
+
+def get_dtype_of_array(array: pd.Series):
+    return get_dtype_and_schema_of_array(array)[0]
+
+
+def get_schema_type_hint_of_array(array: pd.Series):
+    return get_dtype_and_schema_of_array(array)[1]
+
+
+def get_dtype_and_schema_of_array(array: pd.Series):
+    return (get_dtype_from_dtype(array.dtype, array_values=array),
+            get_schema_type_hint_from_dtype(array.dtype, array_values=array))
+
+
+def get_dtype_from_dtype(dtype, array_values=None):
+    """
+    Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
+    for 64 bit values which will get downcast to 32 bit.
+    """
+
+    dtype_name = dtype.name
+    dtype_kind = dtype.kind
+
+    if dtype == np.float32 or dtype == np.int32:
+        return dtype
+    if dtype_name == "bool":
+        return np.uint8
+    if dtype_name == "object" and dtype_kind == "O":
+        return np.unicode
+    if dtype_name == "category":
+        return get_dtype_from_dtype(dtype.categories.dtype, dtype.categories)
+
+    if can_cast_to_float32(dtype):
+        return np.float32
+    if can_cast_to_int32(dtype, array_values):
+        return np.int32
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def get_schema_type_hint_from_dtype(dtype, array_values=None):
+    """
+    Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
+    and will be downcast to 32 bit.
+    """
+
+    dtype_name = dtype.name
+    dtype_kind = dtype.kind
+
+    if dtype == np.float32 or dtype == np.int32:
+        return {"type": dtype_name}
+    if dtype_name == "bool":
+        return {"type": "boolean"}
+    if dtype_name == "object" and dtype_kind == "O":
+        return {"type": "string"}
+    if dtype_name == "category":
+        return {"type": "categorical", "categories": dtype.categories.tolist()}
+
+    if can_cast_to_float32(dtype):
+        return {"type": "float32"}
+    if can_cast_to_int32(dtype, array_values):
+        return {"type": "int32"}
+
+    raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def can_cast_to_float32(dtype):
+    if dtype.kind == "f":
+        if not np.can_cast(dtype, np.float32):
+            logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
+        return True
+    return False
+
+
+def can_cast_to_int32(dtype, array_values=None):
+    """
+    A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
+    the higher precision type has values that are entirely within the range of the downcast type.
+    """
+
+    if dtype.kind in ["i", "u"]:
+        if np.can_cast(dtype, np.int32):
+            return True
+        ii32 = np.iinfo(np.int32)
+        if not array_values.empty and (
+                array_values.min() >= ii32.min and array_values.max() <= ii32.max) or array_values.empty:
+            return True
+    return False

server/common/utils/utils.py

@@ -5,12 +5,11 @@ import logging
 import os
 import pkgutil
 import socket
-import warnings
-
-from flask import json
 from urllib.parse import urlsplit, urljoin
+
 import numpy as np
-import pandas as pd
+from flask import json
+
 from server.common.errors import ConfigurationError
 
 
@@ -94,61 +93,6 @@ def jsonify_numpy(data):
     return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
 
 
-def dtype_to_schema(dtype):
-    schema = {}
-    if dtype == np.float32:
-        schema["type"] = "float32"
-    elif dtype == np.int32:
-        schema["type"] = "int32"
-    elif dtype == np.bool_:
-        schema["type"] = "boolean"
-    elif dtype == np.str:
-        schema["type"] = "string"
-    elif dtype == "category":
-        schema["type"] = "categorical"
-        schema["categories"] = dtype.categories.tolist()
-    else:
-        raise TypeError(f"Annotations of type {dtype} are unsupported.")
-    return schema
-
-
-def can_cast_to_float32(array):
-    if array.dtype.kind == "f":
-        if not np.can_cast(array.dtype, np.float32):
-            warnings.warn(f"Annotation {array.name} will be converted to 32 bit float and may lose precision.")
-        return True
-    return False
-
-
-def can_cast_to_int32(array):
-    if array.dtype.kind in ["i", "u"]:
-        if np.can_cast(array.dtype, np.int32):
-            return True
-        ii32 = np.iinfo(np.int32)
-        if array.min() >= ii32.min and array.max() <= ii32.max:
-            return True
-    return False
-
-
-def series_to_schema(array):
-    assert type(array) == pd.Series
-    try:
-        return dtype_to_schema(array.dtype)
-    except TypeError:
-        dtype = array.dtype
-        data_kind = dtype.kind
-        schema = {}
-        if can_cast_to_float32(array):
-            schema["type"] = "float32"
-        elif can_cast_to_int32(array):
-            schema["type"] = "int32"
-        elif data_kind == "O" and dtype == "object":
-            schema["type"] = "string"
-        else:
-            raise TypeError(f"Annotations of type {dtype} are unsupported.")
-        return schema
-
-
 def import_plugins(plugin_module):
     """
     Load optional plugin modules from server.common.plugins

server/data_anndata/anndata_adaptor.py

@@ -1,22 +1,22 @@
 import warnings
-
-import numpy as np
-from pandas.core.dtypes.dtypes import CategoricalDtype
-import anndata
-from scipy import sparse
-from packaging import version
 from datetime import datetime
+
+import anndata
+import numpy as np
+from packaging import version
+from pandas.core.dtypes.dtypes import CategoricalDtype
+from scipy import sparse
 from server_timing import Timing as ServerTiming
 
-from server.data_common.data_adaptor import DataAdaptor
-from server.data_common.fbs.matrix import encode_matrix_fbs
-from server.common.utils import series_to_schema
+import server.compute.diffexp_generic as diffexp_generic
 from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from server.common.constants import Axis, MAX_LAYOUTS
-from server.common.errors import PrepareError, DatasetAccessError, FilterError
-from server.compute.scanpy import scanpy_umap
-import server.compute.diffexp_generic as diffexp_generic
 from server.common.corpora import corpora_get_props_from_anndata
+from server.common.errors import PrepareError, DatasetAccessError, FilterError
+from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array
+from server.compute.scanpy import scanpy_umap
+from server.data_common.data_adaptor import DataAdaptor
+from server.data_common.fbs.matrix import encode_matrix_fbs
 
 anndata_version = version.parse(str(anndata.__version__)).release
 
@@ -137,7 +137,7 @@ class AnndataAdaptor(DataAdaptor):
             curr_axis = getattr(self.data, str(ax))
             for ann in curr_axis:
                 ann_schema = {"name": ann, "writable": False}
-                ann_schema.update(series_to_schema(curr_axis[ann]))
+                ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann]))
                 self.schema["annotations"][ax]["columns"].append(ann_schema)
 
         for layout in self.get_embedding_names():

server/data_common/data_adaptor.py

@@ -1,14 +1,15 @@
 from abc import ABCMeta, abstractmethod
-from server_timing import Timing as ServerTiming
-import numpy as np
-import pandas as pd
 from os.path import basename, splitext
 
-from server.data_common.fbs.matrix import encode_matrix_fbs
+import numpy as np
+import pandas as pd
+from server_timing import Timing as ServerTiming
+
+from server.common.app_config import AppFeature, AppConfig
 from server.common.constants import Axis
 from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
-from server.common.utils import jsonify_numpy
-from server.common.app_config import AppFeature, AppConfig
+from server.common.utils.utils import jsonify_numpy
+from server.data_common.fbs.matrix import encode_matrix_fbs
 
 
 class DataAdaptor(metaclass=ABCMeta):
@@ -172,7 +173,7 @@ class DataAdaptor(metaclass=ABCMeta):
         mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0] : i[1]] = True
+                mask[i[0]: i[1]] = True
             else:
                 mask[i] = True
         return mask
@@ -313,7 +314,7 @@ class DataAdaptor(metaclass=ABCMeta):
             top_n = self.dataset_config.diffexp__top_n
 
         if self.server_config.exceeds_limit(
-            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+                "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 

server/data_cxg/cxg_adaptor.py

@@ -1,9 +1,9 @@
 import os
 import json
 import logging
-from server.common.utils import dtype_to_schema
+from server.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype
 from server.common.errors import DatasetAccessError, ConfigurationError
-from server.common.utils import path_join
+from server.common.utils.utils import path_join
 from server.common.constants import Axis
 from server.data_common.data_adaptor import DataAdaptor
 from server.data_common.fbs.matrix import encode_matrix_fbs
@@ -389,7 +389,7 @@ class CxgAdaptor(DataAdaptor):
                     if schema["type"] == "categorical" and "categories" in type_hint:
                         schema["categories"] = type_hint["categories"]
                 else:
-                    schema.update(dtype_to_schema(attr.dtype))
+                    schema.update(get_schema_type_hint_from_dtype(attr.dtype))
                 cols.append(schema)
 
             annotations[ax] = dict(columns=cols)

server/test/__init__.py

@@ -1,22 +1,21 @@
+import os
 import random
 import shutil
 import string
 import tempfile
-import requests
 import time
-import os
-from subprocess import Popen
-from os import path, popen
 from contextlib import contextmanager
+from os import path, popen
+from subprocess import Popen
 
 import pandas as pd
+import requests
 
 from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
 from server.common.annotations.local_file_csv import AnnotationsLocalFile
-
-from server.common.data_locator import DataLocator
 from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT
-from server.common.utils import find_available_port
+from server.common.data_locator import DataLocator
+from server.common.utils.utils import find_available_port
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 from server.db.db_utils import DbUtils
@@ -137,7 +136,7 @@ def start_test_server(command_line_args=[], app_config=None):
     yaml config file, which this server will read and parse.
     """
 
-    start = random.randint(DEFAULT_SERVER_PORT, 2**16 - 1)
+    start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
     port = int(os.environ.get("CXG_SERVER_PORT", start))
     port = find_available_port("localhost", port)
     command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose", "--port=%d" % port] + command_line_args

server/test/unit/common/test_app_config.py

@@ -3,10 +3,11 @@ import unittest
 from unittest import mock
 from unittest.mock import patch
 
+import requests
+
 from server.common.app_config import AppConfig
 from server.common.errors import ConfigurationError
 from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
-import requests
 
 
 # NOTE, there are more tests that should be written for AppConfig.
@@ -119,7 +120,6 @@ class AppConfigTest(unittest.TestCase):
         config = AppConfig()
 
         with self.assertLogs(level="INFO") as logger:
-
             from server.common.aws_secret_utils import handle_config_from_secret
             # should not throw error
             # "AttributeError: 'XConfig' object has no attribute 'x'"
@@ -133,4 +133,3 @@ class AppConfigTest(unittest.TestCase):
             self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
             self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
             self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
-

server/test/unit/common/utils/test_matrix_utils.py

@@ -0,0 +1,67 @@
+import unittest
+
+import numpy as np
+
+from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+
+
+class TestMatrixUtils(unittest.TestCase):
+
+    def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
+        matrix = np.array([1, 2, 3])
+
+        self.assertFalse(is_matrix_sparse(matrix, 0))
+        self.assertTrue(is_matrix_sparse(matrix, 100))
+
+    def test__is_matrix_sparse__partially_populated_sparse_matrix_returns_true(self):
+        matrix = np.zeros([3, 4])
+        matrix[2][3] = 1.0
+        matrix[1][1] = 2.2
+
+        self.assertTrue(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__partially_populated_dense_matrix_returns_false(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+        matrix[0][1] = 2.2
+        matrix[1][1] = 3.7
+
+        self.assertFalse(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__giant_matrix_returns_false_early(self):
+        matrix = np.ones([20000, 20])
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
+
+    def test__is_matrix_sparse_with_column_shift_encoding__regular_sparse_returns_true(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+
+        self.assertIsNotNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__column_shift_returns_same_value(self):
+        matrix = np.ones([2, 2])
+        expected_column_shift = [1, 1]
+
+        actual_column_shift = get_column_shift_encode_for_matrix(matrix, 50)
+        self.assertTrue((expected_column_shift == actual_column_shift).all())
+
+    def test__is_matrix_sparse_with_column_shift_encoding__impossible_column_shift_returns_none(self):
+        matrix = np.array([[1, 2], [3, 4]])
+
+        self.assertIsNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__giant_matrix_returns_false_early(self):
+        matrix = np.random.rand(20000, 20)
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])

server/test/unit/common/utils/test_sanitization_utils.py

@@ -0,0 +1,56 @@
+import unittest
+
+from server.common.utils.sanitization_utils import sanitize_values_in_list, sanitize_keys_in_dictionary
+
+
+class TestSanitizationUtils(unittest.TestCase):
+
+    def test__sanitize_values_in_list__not_strings_raises_exception(self):
+        keys_to_sanitize = [1, 2, 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__not_all_strings_raises_exception(self):
+        keys_to_sanitize = ["1", "2", 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__replace_non_ascii_character_with_underscore(self):
+        keys_to_sanitize = ["abc.", "~abc", "a~b/c"]
+        expected_sanitized_keys_dict = dict(zip(keys_to_sanitize, ["abc_", "_abc", "a_b_c"]))
+
+        actual_sanitized_keys_dict = sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertEqual(expected_sanitized_keys_dict, actual_sanitized_keys_dict)
+
+    def test__sanitize_keys_in_dictionary__replace_non_ascii_character_with_underscore(self):
+        dictionary_to_sanitize = {"abc.": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc_": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
+
+    def test__sanitize_keys_in_dictionary__non_string_key_raises_exception(self):
+        dictionary_to_sanitize = {4: 3, "~abc": 4, "a~b/c": 5}
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_keys_in_dictionary(dictionary_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_keys_in_dictionary__replace_only_some_keys(self):
+        dictionary_to_sanitize = {"abc": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)

server/test/unit/common/utils/test_type_conversion_utils.py

@@ -0,0 +1,121 @@
+import unittest
+from unittest.mock import patch
+
+import numpy as np
+from pandas import Series
+
+from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \
+    get_schema_type_hint_of_array
+
+
+class TestTypeConversionUtils(unittest.TestCase):
+
+    def test__can_cast_to_float32__string_is_false(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=str)
+
+        can_cast = can_cast_to_float32(array_to_convert.dtype)
+
+        self.assertFalse(can_cast)
+
+    def test__can_cast_to_float32__int_is_true_warning_outputted(self):
+        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
+
+        with self.assertLogs(level="WARN") as logger:
+            can_cast = can_cast_to_float32(array_to_convert.dtype)
+            self.assertIn("may lose precision", logger.output[0])
+
+        self.assertTrue(can_cast)
+
+    @patch("logging.warning")
+    def test__can_cast_to_float64__int_is_false(self, mock_log_warning):
+        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32))
+
+        can_cast = can_cast_to_float32(array_to_convert.dtype)
+
+        self.assertTrue(can_cast)
+        assert not mock_log_warning.called
+
+    def test__can_cast_to_int32__string_is_false(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=str)
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertFalse(can_cast)
+
+    def test__can_cast_to_int32__int64_is_true(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64))
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertTrue(can_cast)
+
+    def test__can_cast_to_int32__int16_is_true(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16))
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertTrue(can_cast)
+
+    def test__can_cast_to_int32__int64_with_large_value_is_false(self):
+        array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64))
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertFalse(can_cast)
+
+    def test__get_dtype_of_array__supported_dtypes_return_as_expected(self):
+        types = [np.float32, np.int32, np.bool_, str]
+        expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}",
+                              i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
+
+    def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self):
+        types = [np.float32, np.int32, np.bool_, str]
+        expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}",
+                              i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
+
+    def test__get_dtype_of_array__categories_return_as_expected(self):
+        array = Series(data=["a", "b", "c"], dtype="category")
+        expected_dtype = np.unicode
+
+        actual_dtype = get_dtype_of_array(array)
+
+        self.assertEqual(expected_dtype, actual_dtype)
+
+    def test__get_schema_type_hint_of_array__categories_return_as_expected(self):
+        array = Series(data=["a", "b", "b"], dtype="category")
+        expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]}
+
+        actual_schema_hint = get_schema_type_hint_of_array(array)
+
+        self.assertEqual(expected_schema_hint, actual_schema_hint)
+
+    def test__get_dtype_of_array__castable_dtypes_return_as_expected(self):
+        types = [np.float64, np.int64]
+        expected_dtypes = [np.float32, np.int32]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}",
+                              i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
+
+    def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self):
+        types = [np.float64, np.int64]
+        expected_schema_hints = [{"type": "float32"}, {"type": "int32"}]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(
+                    f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
+                    i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])

server/test/unit/common/utils/test_utils.py

@@ -2,7 +2,7 @@ import os
 import shutil
 import unittest
 
-from server.common.utils import import_plugins
+from server.common.utils.utils import import_plugins
 from server.test import PROJECT_ROOT, random_string
 
 

server/test/unit/data_anndata/test_nan_anndata_adaptor.py

@@ -22,8 +22,9 @@ class NaNTest(unittest.TestCase):
             self.data._create_schema()
 
     def test_load(self):
-        with self.assertWarns(UserWarning):
+        with self.assertLogs(level="WARN") as logger:
             self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.assertTrue(logger.output)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 100)