From 508889f74bb754260f8e13e225ac9c26bf5b18e2 Mon Sep 17 00:00:00 2001 From: maniarathi Date: Fri, 14 Aug 2020 16:51:13 -0700 Subject: [PATCH] Refactoring cxg utility classes in preparation for CXG conversion tooling (#1739) --- server/__init__.py | 3 +- server/app/app.py | 13 +- server/cli/launch.py | 70 +++++----- server/cli/prepare.py | 28 ++-- server/common/annotations/annotations.py | 6 +- server/common/annotations/local_file_csv.py | 1 - server/common/app_config.py | 38 +++--- server/common/utils/__init__.py | 0 server/common/utils/matrix_utils.py | 112 ++++++++++++++++ server/common/utils/sanitization_utils.py | 40 ++++++ server/common/utils/type_conversion_utils.py | 93 ++++++++++++++ server/common/{ => utils}/utils.py | 62 +-------- server/data_anndata/anndata_adaptor.py | 26 ++-- server/data_common/data_adaptor.py | 17 +-- server/data_cxg/cxg_adaptor.py | 6 +- server/test/__init__.py | 15 +-- server/test/unit/common/test_app_config.py | 5 +- server/test/unit/common/utils/__init__.py | 0 .../unit/common/utils/test_matrix_utils.py | 67 ++++++++++ .../common/utils/test_sanitization_utils.py | 56 ++++++++ .../utils/test_type_conversion_utils.py | 121 ++++++++++++++++++ .../unit/common/{ => utils}/test_utils.py | 2 +- .../data_anndata/test_nan_anndata_adaptor.py | 3 +- 23 files changed, 607 insertions(+), 177 deletions(-) create mode 100644 server/common/utils/__init__.py create mode 100644 server/common/utils/matrix_utils.py create mode 100644 server/common/utils/sanitization_utils.py create mode 100644 server/common/utils/type_conversion_utils.py rename server/common/{ => utils}/utils.py (69%) create mode 100644 server/test/unit/common/utils/__init__.py create mode 100644 server/test/unit/common/utils/test_matrix_utils.py create mode 100644 server/test/unit/common/utils/test_sanitization_utils.py create mode 100644 server/test/unit/common/utils/test_type_conversion_utils.py rename server/test/unit/common/{ => utils}/test_utils.py (95%) diff --git a/server/__init__.py b/server/__init__.py index 30bcb57c..94238d9a 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -1,7 +1,8 @@ -from server.common.utils import import_plugins import logging import sys +from server.common.utils.utils import import_plugins + __version__ = "0.16.0" display_version = "cellxgene v" + __version__ diff --git a/server/app/app.py b/server/app/app.py index cbbb6e4b..d01b1964 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -1,22 +1,19 @@ import datetime import logging +from functools import wraps +from http import HTTPStatus -from flask import Flask, redirect, current_app, make_response, render_template, abort -from flask import Blueprint, request +from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request from flask_restful import Api, Resource from server_timing import Timing as ServerTiming -from http import HTTPStatus - import server.common.rest as common_rest -from server.common.errors import DatasetAccessError, RequestException -from server.common.utils import path_join, Float32JSONEncoder from server.common.data_locator import DataLocator +from server.common.errors import DatasetAccessError, RequestException from server.common.health import health_check +from server.common.utils.utils import path_join, Float32JSONEncoder from server.data_common.matrix_loader import MatrixDataLoader -from functools import wraps - webbp = Blueprint("webapp", "server.common.web", template_folder="templates") ONE_WEEK = 7 * 24 * 60 * 60 diff --git a/server/cli/launch.py b/server/cli/launch.py index 00900092..d46161d4 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -1,19 +1,19 @@ import errno import functools import logging -from os import devnull import sys import webbrowser +from os import devnull import click from flask_compress import Compress from flask_cors import CORS -from server.common.utils import sort_options -from server.common.errors import DatasetAccessError, ConfigurationError +from server.app.app import Server from server.common.app_config import AppConfig from server.common.default_config import default_config -from server.app.app import Server +from server.common.errors import DatasetAccessError, ConfigurationError +from server.common.utils.utils import sort_options DEFAULT_CONFIG = AppConfig() @@ -33,7 +33,7 @@ def annotation_args(func): multiple=False, metavar="", help="CSV file to initialize editing of existing annotations; will be altered in-place. " - "Incompatible with --annotations-dir.", + "Incompatible with --annotations-dir.", ) @click.option( "--annotations-dir", @@ -42,7 +42,7 @@ def annotation_args(func): multiple=False, metavar="", help="Directory of where to save output annotations; filename will be specified in the application. " - "Incompatible with --annotations-file.", + "Incompatible with --annotations-file.", ) @click.option( "--experimental-annotations-ontology", @@ -170,7 +170,7 @@ def server_args(func): default=DEFAULT_CONFIG.server_config.app__debug, show_default=True, help="Run in debug mode. This is helpful for cellxgene developers, " - "or when you want more information about an error condition.", + "or when you want more information about an error condition.", ) @click.option( "--verbose", @@ -203,7 +203,7 @@ def server_args(func): multiple=True, metavar="", help="Additional script files to include in HTML page. If not specified, " - "no additional script files will be included.", + "no additional script files will be included.", show_default=False, ) @functools.wraps(func) @@ -223,7 +223,7 @@ def launch_args(func): default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot, metavar="", help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)" - " to folder containing H5AD and/or CXG datasets.", + " to folder containing H5AD and/or CXG datasets.", hidden=True, ) # TODO, unhide when dataroot is supported) @click.argument("datapath", required=False, metavar="") @@ -307,32 +307,32 @@ class CliLaunchServer(Server): ) @launch_args def launch( - datapath, - dataroot, - verbose, - debug, - open_browser, - port, - host, - embedding, - obs_names, - var_names, - max_category_items, - disable_custom_colors, - diffexp_lfc_cutoff, - title, - scripts, - about, - disable_annotations, - annotations_file, - annotations_dir, - backed, - disable_diffexp, - experimental_annotations_ontology, - experimental_annotations_ontology_obo, - experimental_enable_reembedding, - config_file, - dump_default_config, + datapath, + dataroot, + verbose, + debug, + open_browser, + port, + host, + embedding, + obs_names, + var_names, + max_category_items, + disable_custom_colors, + diffexp_lfc_cutoff, + title, + scripts, + about, + disable_annotations, + annotations_file, + annotations_dir, + backed, + disable_diffexp, + experimental_annotations_ontology, + experimental_annotations_ontology_obo, + experimental_enable_reembedding, + config_file, + dump_default_config, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. diff --git a/server/cli/prepare.py b/server/cli/prepare.py index a282c14d..df535db0 100644 --- a/server/cli/prepare.py +++ b/server/cli/prepare.py @@ -5,7 +5,7 @@ import pandas as pd from numpy import ndarray, unique from scipy.sparse.csc import csc_matrix -from server.common.utils import sort_options +from server.common.utils.utils import sort_options @sort_options @@ -37,7 +37,7 @@ from server.common.utils import sort_options default=False, is_flag=True, help="Do not run quality control metrics. By default cellxgene runs them " - "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", + "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", ) @click.option( "--make-obs-names-unique/--no-make-obs-names-unique", @@ -53,18 +53,18 @@ from server.common.utils import sort_options ) @click.help_option("--help", "-h", help="Show this message and exit.") def prepare( - data, - embedding, - recipe, - output, - plotting, - sparse, - overwrite, - set_obs_names, - set_var_names, - skip_qc, - make_obs_names_unique, - make_var_names_unique, + data, + embedding, + recipe, + output, + plotting, + sparse, + overwrite, + set_obs_names, + set_var_names, + skip_qc, + make_obs_names_unique, + make_var_names_unique, ): """ Preprocess data for use with cellxgene. diff --git a/server/common/annotations/annotations.py b/server/common/annotations/annotations.py index d15a9fff..855f19ab 100644 --- a/server/common/annotations/annotations.py +++ b/server/common/annotations/annotations.py @@ -2,9 +2,9 @@ from abc import ABCMeta, abstractmethod import fastobo import fsspec -from server.common.errors import OntologyLoadFailure -from server.common.utils import series_to_schema +from server.common.errors import OntologyLoadFailure +from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array class Annotations(metaclass=ABCMeta): @@ -44,7 +44,7 @@ class Annotations(metaclass=ABCMeta): if labels is not None and not labels.empty: for col in labels.columns: col_schema = dict(name=col, writable=True) - col_schema.update(series_to_schema(labels[col])) + col_schema.update(get_schema_type_hint_of_array(labels[col])) schema.append(col_schema) return schema diff --git a/server/common/annotations/local_file_csv.py b/server/common/annotations/local_file_csv.py index 49efcf5c..b463540b 100644 --- a/server/common/annotations/local_file_csv.py +++ b/server/common/annotations/local_file_csv.py @@ -14,7 +14,6 @@ from server.common.errors import AnnotationsError class AnnotationsLocalFile(Annotations): - CXG_ANNO_COLLECTION = "cxg_anno_collection" def __init__(self, output_dir, output_file): diff --git a/server/common/app_config.py b/server/common/app_config.py index f1dc942b..b3715ecc 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -1,23 +1,23 @@ -from server import display_version as cellxgene_display_version -from flatten_dict import flatten, unflatten -import os -from os.path import splitext, basename, isdir -import sys -from urllib.parse import urlparse, quote_plus -import yaml import copy - -from server.common.default_config import get_default_config -from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure -from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType -from server.common.utils import find_available_port, is_port_available +import os +import sys import warnings +from os.path import splitext, basename, isdir +from urllib.parse import urlparse, quote_plus + +import yaml +from flatten_dict import flatten, unflatten + +import server.compute.diffexp_cxg as diffexp_tiledb +from server import display_version as cellxgene_display_version +from server.auth.auth import AuthTypeFactory from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB from server.common.annotations.local_file_csv import AnnotationsLocalFile -from server.common.utils import custom_format_warning -import server.compute.diffexp_cxg as diffexp_tiledb from server.common.data_locator import discover_s3_region_name -from server.auth.auth import AuthTypeFactory +from server.common.default_config import get_default_config +from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure +from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available +from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType from server.db.db_utils import DbUtils DEFAULT_SERVER_PORT = 5005 @@ -150,7 +150,6 @@ class AppConfig(object): parameters is done""" if messagefn is None: - def noop(message): pass @@ -284,7 +283,7 @@ class AppConfig(object): if auth.requires_client_login(): config["authentication"].update({ "login": auth.get_login_url(data_adaptor), - "logout" : auth.get_logout_url(data_adaptor), + "logout": auth.get_logout_url(data_adaptor), }) return c @@ -748,7 +747,8 @@ class DatasetConfig(BaseConfig): self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"] self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"] self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"] - self.user_annotations__hosted_tiledb_array__hosted_file_directory = dc["user_annotations"]["hosted_tiledb_array"]["hosted_file_directory"] # noqa E501 + self.user_annotations__hosted_tiledb_array__hosted_file_directory = \ + dc["user_annotations"]["hosted_tiledb_array"]["hosted_file_directory"] # noqa E501 self.embeddings__names = dc["embeddings"]["names"] self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"] @@ -896,7 +896,7 @@ class DatasetConfig(BaseConfig): server_config = self.app_config.server_config if server_config.single_dataset__datapath: with server_config.matrix_data_cache_manager.data_adaptor( - self.tag, server_config.single_dataset__datapath, self.app_config + self.tag, server_config.single_dataset__datapath, self.app_config ) as data_adaptor: if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): context["messagefn"]( diff --git a/server/common/utils/__init__.py b/server/common/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/common/utils/matrix_utils.py b/server/common/utils/matrix_utils.py new file mode 100644 index 00000000..3eeddc10 --- /dev/null +++ b/server/common/utils/matrix_utils.py @@ -0,0 +1,112 @@ +import logging + +import numpy as np +from scipy.stats import mode + + +def is_matrix_sparse(matrix: np.ndarray, sparse_threshold): + """ + Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has + a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of + elements evaluated. This function may return before evaluating the whole matrix if it can be determined that matrix + is not sparse enough. + """ + + if sparse_threshold == 100.0: + return True + if sparse_threshold == 0.0: + return False + + total_number_of_rows = matrix.shape[0] + total_number_of_columns = matrix.shape[1] + total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns + + # For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the + # maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the + # quit early for large dense matrices. + row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000) + + maximum_number_of_non_zero_elements_in_matrix = int( + total_number_of_rows * total_number_of_columns * sparse_threshold / 100 + ) + number_of_non_zero_elements = 0 + + for start_row_index in range(0, total_number_of_rows, row_stride): + end_row_index = min(start_row_index + row_stride, total_number_of_rows) + + matrix_subset = matrix[start_row_index:end_row_index, :] + if not isinstance(matrix_subset, np.ndarray): + matrix_subset = matrix_subset.toarray() + + number_of_non_zero_elements += np.count_nonzero(matrix_subset) + if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix: + if end_row_index != total_number_of_rows: + percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / ( + end_row_index * total_number_of_columns) + logging.info( + f"Matrix is not sparse. Percentage of non-zero elements (estimate): " + f"{percentage_of_non_zero_elements:6.2f}") + else: + percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements + logging.info( + f"Matrix is not sparse. Percentage of non-zero elements (exact): " + f"{percentage_of_non_zero_elements:6.2f}") + return False + + is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold + return is_sparse + + +def get_column_shift_encode_for_matrix(matrix, sparse_threshold): + """ + Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column + shift encoding works by taking the most common value in each column, then subtracting that value from each element + of the column. If each column mostly contains its most common value, then the resulting matrix can be very sparse. + + This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a + sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns + the number of non-zeros encountered and number of elements evaluated. This function may return before evaluating + the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding. + """ + + total_number_of_rows = matrix.shape[0] + total_number_of_columns = matrix.shape[1] + total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns + + stride = max(1, 128_000_000 // total_number_of_rows) + column_shift = np.zeros(total_number_of_columns) + + maximum_number_of_non_zero_elements_in_matrix = int( + total_number_of_rows * total_number_of_columns * sparse_threshold / 100 + ) + number_of_non_zero_elements = 0 + + for start_column_index in range(0, total_number_of_columns, stride): + end_column_index = min(start_column_index + stride, total_number_of_columns) + + matrix_subset = matrix[:, start_column_index:end_column_index] + if not isinstance(matrix_subset, np.ndarray): + matrix_subset = matrix_subset.toarray() + + matrix_subset_mode = mode(matrix_subset) + + column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode + number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum( + matrix_subset_mode.count + ) + + if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix: + if end_column_index != total_number_of_columns: + logging.info( + "Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f" + % (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows) + ) + else: + logging.info( + "Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f" + % (100 * number_of_non_zero_elements / total_number_of_matrix_elements) + ) + return None + + is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold + return column_shift if is_sparse else None diff --git a/server/common/utils/sanitization_utils.py b/server/common/utils/sanitization_utils.py new file mode 100644 index 00000000..af6f299f --- /dev/null +++ b/server/common/utils/sanitization_utils.py @@ -0,0 +1,40 @@ +import re + + +def sanitize_values_in_list(list_of_keys: list): + """ + Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both + safe and unique. + """ + + if not all([isinstance(key, str) for key in list_of_keys]): + raise Exception("List of keys to sanitize must contain all strings.") + + # Mask out [~/.] and anything outside the ASCII range. + mask = re.compile(r"[^ -\-0-\[\]-\}]") + clean_keys_list = [mask.sub("_", key) for key in list_of_keys] + + # Dedupe the clean keys list + deduped_clean_keys_list = [] + for index, clean_key in enumerate(clean_keys_list): + total_occurrences_of_clean_key = clean_keys_list.count(clean_key) + total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key) + deduped_clean_keys_list.append( + clean_key + "_" + str(total_occurrences_up_until_current_index + 1) + if total_occurrences_of_clean_key > 1 + else clean_key + ) + + return dict(zip(list_of_keys, deduped_clean_keys_list)) + + +def sanitize_keys_in_dictionary(dict_to_sanitize: dict): + """ + Clean and dedupe the keys in the given dictionary. + """ + + clean_keys = sanitize_values_in_list(dict_to_sanitize.keys()) + for original_key, sanitized_key in clean_keys.items(): + if original_key != sanitized_key: + dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key] + del dict_to_sanitize[original_key] diff --git a/server/common/utils/type_conversion_utils.py b/server/common/utils/type_conversion_utils.py new file mode 100644 index 00000000..8b467b47 --- /dev/null +++ b/server/common/utils/type_conversion_utils.py @@ -0,0 +1,93 @@ +import logging + +import numpy as np +import pandas as pd + + +def get_dtype_of_array(array: pd.Series): + return get_dtype_and_schema_of_array(array)[0] + + +def get_schema_type_hint_of_array(array: pd.Series): + return get_dtype_and_schema_of_array(array)[1] + + +def get_dtype_and_schema_of_array(array: pd.Series): + return (get_dtype_from_dtype(array.dtype, array_values=array), + get_schema_type_hint_from_dtype(array.dtype, array_values=array)) + + +def get_dtype_from_dtype(dtype, array_values=None): + """ + Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant + for 64 bit values which will get downcast to 32 bit. + """ + + dtype_name = dtype.name + dtype_kind = dtype.kind + + if dtype == np.float32 or dtype == np.int32: + return dtype + if dtype_name == "bool": + return np.uint8 + if dtype_name == "object" and dtype_kind == "O": + return np.unicode + if dtype_name == "category": + return get_dtype_from_dtype(dtype.categories.dtype, dtype.categories) + + if can_cast_to_float32(dtype): + return np.float32 + if can_cast_to_int32(dtype, array_values): + return np.int32 + + raise TypeError(f"Annotations of type {dtype} are unsupported.") + + +def get_schema_type_hint_from_dtype(dtype, array_values=None): + """ + Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit + and will be downcast to 32 bit. + """ + + dtype_name = dtype.name + dtype_kind = dtype.kind + + if dtype == np.float32 or dtype == np.int32: + return {"type": dtype_name} + if dtype_name == "bool": + return {"type": "boolean"} + if dtype_name == "object" and dtype_kind == "O": + return {"type": "string"} + if dtype_name == "category": + return {"type": "categorical", "categories": dtype.categories.tolist()} + + if can_cast_to_float32(dtype): + return {"type": "float32"} + if can_cast_to_int32(dtype, array_values): + return {"type": "int32"} + + raise TypeError(f"Annotations of type {dtype} are unsupported.") + + +def can_cast_to_float32(dtype): + if dtype.kind == "f": + if not np.can_cast(dtype, np.float32): + logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.") + return True + return False + + +def can_cast_to_int32(dtype, array_values=None): + """ + A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of + the higher precision type has values that are entirely within the range of the downcast type. + """ + + if dtype.kind in ["i", "u"]: + if np.can_cast(dtype, np.int32): + return True + ii32 = np.iinfo(np.int32) + if not array_values.empty and ( + array_values.min() >= ii32.min and array_values.max() <= ii32.max) or array_values.empty: + return True + return False diff --git a/server/common/utils.py b/server/common/utils/utils.py similarity index 69% rename from server/common/utils.py rename to server/common/utils/utils.py index 7e876194..c6bb24f0 100644 --- a/server/common/utils.py +++ b/server/common/utils/utils.py @@ -5,12 +5,11 @@ import logging import os import pkgutil import socket -import warnings - -from flask import json from urllib.parse import urlsplit, urljoin + import numpy as np -import pandas as pd +from flask import json + from server.common.errors import ConfigurationError @@ -94,61 +93,6 @@ def jsonify_numpy(data): return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False) -def dtype_to_schema(dtype): - schema = {} - if dtype == np.float32: - schema["type"] = "float32" - elif dtype == np.int32: - schema["type"] = "int32" - elif dtype == np.bool_: - schema["type"] = "boolean" - elif dtype == np.str: - schema["type"] = "string" - elif dtype == "category": - schema["type"] = "categorical" - schema["categories"] = dtype.categories.tolist() - else: - raise TypeError(f"Annotations of type {dtype} are unsupported.") - return schema - - -def can_cast_to_float32(array): - if array.dtype.kind == "f": - if not np.can_cast(array.dtype, np.float32): - warnings.warn(f"Annotation {array.name} will be converted to 32 bit float and may lose precision.") - return True - return False - - -def can_cast_to_int32(array): - if array.dtype.kind in ["i", "u"]: - if np.can_cast(array.dtype, np.int32): - return True - ii32 = np.iinfo(np.int32) - if array.min() >= ii32.min and array.max() <= ii32.max: - return True - return False - - -def series_to_schema(array): - assert type(array) == pd.Series - try: - return dtype_to_schema(array.dtype) - except TypeError: - dtype = array.dtype - data_kind = dtype.kind - schema = {} - if can_cast_to_float32(array): - schema["type"] = "float32" - elif can_cast_to_int32(array): - schema["type"] = "int32" - elif data_kind == "O" and dtype == "object": - schema["type"] = "string" - else: - raise TypeError(f"Annotations of type {dtype} are unsupported.") - return schema - - def import_plugins(plugin_module): """ Load optional plugin modules from server.common.plugins diff --git a/server/data_anndata/anndata_adaptor.py b/server/data_anndata/anndata_adaptor.py index f9c781ad..3145a833 100644 --- a/server/data_anndata/anndata_adaptor.py +++ b/server/data_anndata/anndata_adaptor.py @@ -1,22 +1,22 @@ import warnings - -import numpy as np -from pandas.core.dtypes.dtypes import CategoricalDtype -import anndata -from scipy import sparse -from packaging import version from datetime import datetime + +import anndata +import numpy as np +from packaging import version +from pandas.core.dtypes.dtypes import CategoricalDtype +from scipy import sparse from server_timing import Timing as ServerTiming -from server.data_common.data_adaptor import DataAdaptor -from server.data_common.fbs.matrix import encode_matrix_fbs -from server.common.utils import series_to_schema +import server.compute.diffexp_generic as diffexp_generic from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors from server.common.constants import Axis, MAX_LAYOUTS -from server.common.errors import PrepareError, DatasetAccessError, FilterError -from server.compute.scanpy import scanpy_umap -import server.compute.diffexp_generic as diffexp_generic from server.common.corpora import corpora_get_props_from_anndata +from server.common.errors import PrepareError, DatasetAccessError, FilterError +from server.common.utils.type_conversion_utils import get_schema_type_hint_of_array +from server.compute.scanpy import scanpy_umap +from server.data_common.data_adaptor import DataAdaptor +from server.data_common.fbs.matrix import encode_matrix_fbs anndata_version = version.parse(str(anndata.__version__)).release @@ -137,7 +137,7 @@ class AnndataAdaptor(DataAdaptor): curr_axis = getattr(self.data, str(ax)) for ann in curr_axis: ann_schema = {"name": ann, "writable": False} - ann_schema.update(series_to_schema(curr_axis[ann])) + ann_schema.update(get_schema_type_hint_of_array(curr_axis[ann])) self.schema["annotations"][ax]["columns"].append(ann_schema) for layout in self.get_embedding_names(): diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index 84ac5e64..20dafdd0 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -1,14 +1,15 @@ from abc import ABCMeta, abstractmethod -from server_timing import Timing as ServerTiming -import numpy as np -import pandas as pd from os.path import basename, splitext -from server.data_common.fbs.matrix import encode_matrix_fbs +import numpy as np +import pandas as pd +from server_timing import Timing as ServerTiming + +from server.common.app_config import AppFeature, AppConfig from server.common.constants import Axis from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError -from server.common.utils import jsonify_numpy -from server.common.app_config import AppFeature, AppConfig +from server.common.utils.utils import jsonify_numpy +from server.data_common.fbs.matrix import encode_matrix_fbs class DataAdaptor(metaclass=ABCMeta): @@ -172,7 +173,7 @@ class DataAdaptor(metaclass=ABCMeta): mask = np.zeros((count,), dtype=np.bool) for i in filter: if type(i) == list: - mask[i[0] : i[1]] = True + mask[i[0]: i[1]] = True else: mask[i] = True return mask @@ -313,7 +314,7 @@ class DataAdaptor(metaclass=ABCMeta): top_n = self.dataset_config.diffexp__top_n if self.server_config.exceeds_limit( - "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) + "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) ): raise ExceedsLimitError("Diffexp request exceeds max cell count limit") diff --git a/server/data_cxg/cxg_adaptor.py b/server/data_cxg/cxg_adaptor.py index 41822c29..db2fdd2a 100644 --- a/server/data_cxg/cxg_adaptor.py +++ b/server/data_cxg/cxg_adaptor.py @@ -1,9 +1,9 @@ import os import json import logging -from server.common.utils import dtype_to_schema +from server.common.utils.type_conversion_utils import get_schema_type_hint_from_dtype from server.common.errors import DatasetAccessError, ConfigurationError -from server.common.utils import path_join +from server.common.utils.utils import path_join from server.common.constants import Axis from server.data_common.data_adaptor import DataAdaptor from server.data_common.fbs.matrix import encode_matrix_fbs @@ -389,7 +389,7 @@ class CxgAdaptor(DataAdaptor): if schema["type"] == "categorical" and "categories" in type_hint: schema["categories"] = type_hint["categories"] else: - schema.update(dtype_to_schema(attr.dtype)) + schema.update(get_schema_type_hint_from_dtype(attr.dtype)) cols.append(schema) annotations[ax] = dict(columns=cols) diff --git a/server/test/__init__.py b/server/test/__init__.py index 589eccdf..49e28ce5 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -1,22 +1,21 @@ +import os import random import shutil import string import tempfile -import requests import time -import os -from subprocess import Popen -from os import path, popen from contextlib import contextmanager +from os import path, popen +from subprocess import Popen import pandas as pd +import requests from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB from server.common.annotations.local_file_csv import AnnotationsLocalFile - -from server.common.data_locator import DataLocator from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT -from server.common.utils import find_available_port +from server.common.data_locator import DataLocator +from server.common.utils.utils import find_available_port from server.data_common.fbs.matrix import encode_matrix_fbs from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType from server.db.db_utils import DbUtils @@ -137,7 +136,7 @@ def start_test_server(command_line_args=[], app_config=None): yaml config file, which this server will read and parse. """ - start = random.randint(DEFAULT_SERVER_PORT, 2**16 - 1) + start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1) port = int(os.environ.get("CXG_SERVER_PORT", start)) port = find_available_port("localhost", port) command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose", "--port=%d" % port] + command_line_args diff --git a/server/test/unit/common/test_app_config.py b/server/test/unit/common/test_app_config.py index e7fc7434..a299bf51 100644 --- a/server/test/unit/common/test_app_config.py +++ b/server/test/unit/common/test_app_config.py @@ -3,10 +3,11 @@ import unittest from unittest import mock from unittest.mock import patch +import requests + from server.common.app_config import AppConfig from server.common.errors import ConfigurationError from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT -import requests # NOTE, there are more tests that should be written for AppConfig. @@ -119,7 +120,6 @@ class AppConfigTest(unittest.TestCase): config = AppConfig() with self.assertLogs(level="INFO") as logger: - from server.common.aws_secret_utils import handle_config_from_secret # should not throw error # "AttributeError: 'XConfig' object has no attribute 'x'" @@ -133,4 +133,3 @@ class AppConfigTest(unittest.TestCase): self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret") self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret") self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri") - diff --git a/server/test/unit/common/utils/__init__.py b/server/test/unit/common/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/server/test/unit/common/utils/test_matrix_utils.py b/server/test/unit/common/utils/test_matrix_utils.py new file mode 100644 index 00000000..ffda1045 --- /dev/null +++ b/server/test/unit/common/utils/test_matrix_utils.py @@ -0,0 +1,67 @@ +import unittest + +import numpy as np + +from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix + + +class TestMatrixUtils(unittest.TestCase): + + def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self): + matrix = np.array([1, 2, 3]) + + self.assertFalse(is_matrix_sparse(matrix, 0)) + self.assertTrue(is_matrix_sparse(matrix, 100)) + + def test__is_matrix_sparse__partially_populated_sparse_matrix_returns_true(self): + matrix = np.zeros([3, 4]) + matrix[2][3] = 1.0 + matrix[1][1] = 2.2 + + self.assertTrue(is_matrix_sparse(matrix, 50)) + + def test__is_matrix_sparse__partially_populated_dense_matrix_returns_false(self): + matrix = np.zeros([2, 2]) + matrix[0][0] = 1.0 + matrix[0][1] = 2.2 + matrix[1][1] = 3.7 + + self.assertFalse(is_matrix_sparse(matrix, 50)) + + def test__is_matrix_sparse__giant_matrix_returns_false_early(self): + matrix = np.ones([20000, 20]) + + with self.assertLogs(level="INFO") as logger: + self.assertFalse(is_matrix_sparse(matrix, 1)) + + # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of + # non-zero elements in the matrix. + self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0]) + + def test__is_matrix_sparse_with_column_shift_encoding__regular_sparse_returns_true(self): + matrix = np.zeros([2, 2]) + matrix[0][0] = 1.0 + + self.assertIsNotNone(get_column_shift_encode_for_matrix(matrix, 50)) + + def test__is_matrix_sparse_with_column_shift_encoding__column_shift_returns_same_value(self): + matrix = np.ones([2, 2]) + expected_column_shift = [1, 1] + + actual_column_shift = get_column_shift_encode_for_matrix(matrix, 50) + self.assertTrue((expected_column_shift == actual_column_shift).all()) + + def test__is_matrix_sparse_with_column_shift_encoding__impossible_column_shift_returns_none(self): + matrix = np.array([[1, 2], [3, 4]]) + + self.assertIsNone(get_column_shift_encode_for_matrix(matrix, 50)) + + def test__is_matrix_sparse_with_column_shift_encoding__giant_matrix_returns_false_early(self): + matrix = np.random.rand(20000, 20) + + with self.assertLogs(level="INFO") as logger: + self.assertFalse(is_matrix_sparse(matrix, 1)) + + # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of + # non-zero elements in the matrix. + self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0]) diff --git a/server/test/unit/common/utils/test_sanitization_utils.py b/server/test/unit/common/utils/test_sanitization_utils.py new file mode 100644 index 00000000..8ef04218 --- /dev/null +++ b/server/test/unit/common/utils/test_sanitization_utils.py @@ -0,0 +1,56 @@ +import unittest + +from server.common.utils.sanitization_utils import sanitize_values_in_list, sanitize_keys_in_dictionary + + +class TestSanitizationUtils(unittest.TestCase): + + def test__sanitize_values_in_list__not_strings_raises_exception(self): + keys_to_sanitize = [1, 2, 3] + + with self.assertRaises(Exception) as exception_context: + sanitize_values_in_list(keys_to_sanitize) + + self.assertIn("must contain all strings", str(exception_context.exception)) + + def test__sanitize_values_in_list__not_all_strings_raises_exception(self): + keys_to_sanitize = ["1", "2", 3] + + with self.assertRaises(Exception) as exception_context: + sanitize_values_in_list(keys_to_sanitize) + + self.assertIn("must contain all strings", str(exception_context.exception)) + + def test__sanitize_values_in_list__replace_non_ascii_character_with_underscore(self): + keys_to_sanitize = ["abc.", "~abc", "a~b/c"] + expected_sanitized_keys_dict = dict(zip(keys_to_sanitize, ["abc_", "_abc", "a_b_c"])) + + actual_sanitized_keys_dict = sanitize_values_in_list(keys_to_sanitize) + + self.assertEqual(expected_sanitized_keys_dict, actual_sanitized_keys_dict) + + def test__sanitize_keys_in_dictionary__replace_non_ascii_character_with_underscore(self): + dictionary_to_sanitize = {"abc.": 3, "~abc": 4, "a~b/c": 5} + expected_sanitized_dict = {"abc_": 3, "_abc": 4, "a_b_c": 5} + + actual_sanitized_dict = dictionary_to_sanitize + sanitize_keys_in_dictionary(actual_sanitized_dict) + + self.assertEqual(expected_sanitized_dict, actual_sanitized_dict) + + def test__sanitize_keys_in_dictionary__non_string_key_raises_exception(self): + dictionary_to_sanitize = {4: 3, "~abc": 4, "a~b/c": 5} + + with self.assertRaises(Exception) as exception_context: + sanitize_keys_in_dictionary(dictionary_to_sanitize) + + self.assertIn("must contain all strings", str(exception_context.exception)) + + def test__sanitize_keys_in_dictionary__replace_only_some_keys(self): + dictionary_to_sanitize = {"abc": 3, "~abc": 4, "a~b/c": 5} + expected_sanitized_dict = {"abc": 3, "_abc": 4, "a_b_c": 5} + + actual_sanitized_dict = dictionary_to_sanitize + sanitize_keys_in_dictionary(actual_sanitized_dict) + + self.assertEqual(expected_sanitized_dict, actual_sanitized_dict) diff --git a/server/test/unit/common/utils/test_type_conversion_utils.py b/server/test/unit/common/utils/test_type_conversion_utils.py new file mode 100644 index 00000000..a5bb0395 --- /dev/null +++ b/server/test/unit/common/utils/test_type_conversion_utils.py @@ -0,0 +1,121 @@ +import unittest +from unittest.mock import patch + +import numpy as np +from pandas import Series + +from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \ + get_schema_type_hint_of_array + + +class TestTypeConversionUtils(unittest.TestCase): + + def test__can_cast_to_float32__string_is_false(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=str) + + can_cast = can_cast_to_float32(array_to_convert.dtype) + + self.assertFalse(can_cast) + + def test__can_cast_to_float32__int_is_true_warning_outputted(self): + array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64)) + + with self.assertLogs(level="WARN") as logger: + can_cast = can_cast_to_float32(array_to_convert.dtype) + self.assertIn("may lose precision", logger.output[0]) + + self.assertTrue(can_cast) + + @patch("logging.warning") + def test__can_cast_to_float64__int_is_false(self, mock_log_warning): + array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32)) + + can_cast = can_cast_to_float32(array_to_convert.dtype) + + self.assertTrue(can_cast) + assert not mock_log_warning.called + + def test__can_cast_to_int32__string_is_false(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=str) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__can_cast_to_int32__int64_is_true(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64)) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_int32__int16_is_true(self): + array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16)) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertTrue(can_cast) + + def test__can_cast_to_int32__int64_with_large_value_is_false(self): + array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64)) + + can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) + + self.assertFalse(can_cast) + + def test__get_dtype_of_array__supported_dtypes_return_as_expected(self): + types = [np.float32, np.int32, np.bool_, str] + expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode] + + for test_type_index in range(len(types)): + with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", + i=test_type_index): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) + + def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self): + types = [np.float32, np.int32, np.bool_, str] + expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}] + + for test_type_index in range(len(types)): + with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", + i=test_type_index): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) + + def test__get_dtype_of_array__categories_return_as_expected(self): + array = Series(data=["a", "b", "c"], dtype="category") + expected_dtype = np.unicode + + actual_dtype = get_dtype_of_array(array) + + self.assertEqual(expected_dtype, actual_dtype) + + def test__get_schema_type_hint_of_array__categories_return_as_expected(self): + array = Series(data=["a", "b", "b"], dtype="category") + expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]} + + actual_schema_hint = get_schema_type_hint_of_array(array) + + self.assertEqual(expected_schema_hint, actual_schema_hint) + + def test__get_dtype_of_array__castable_dtypes_return_as_expected(self): + types = [np.float64, np.int64] + expected_dtypes = [np.float32, np.int32] + + for test_type_index in range(len(types)): + with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", + i=test_type_index): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) + + def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self): + types = [np.float64, np.int64] + expected_schema_hints = [{"type": "float32"}, {"type": "int32"}] + + for test_type_index in range(len(types)): + with self.subTest( + f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", + i=test_type_index): + array = Series(data=[], dtype=types[test_type_index]) + self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) diff --git a/server/test/unit/common/test_utils.py b/server/test/unit/common/utils/test_utils.py similarity index 95% rename from server/test/unit/common/test_utils.py rename to server/test/unit/common/utils/test_utils.py index b5f2a5b5..7d5b6272 100644 --- a/server/test/unit/common/test_utils.py +++ b/server/test/unit/common/utils/test_utils.py @@ -2,7 +2,7 @@ import os import shutil import unittest -from server.common.utils import import_plugins +from server.common.utils.utils import import_plugins from server.test import PROJECT_ROOT, random_string diff --git a/server/test/unit/data_anndata/test_nan_anndata_adaptor.py b/server/test/unit/data_anndata/test_nan_anndata_adaptor.py index 29ca178f..48c4e020 100644 --- a/server/test/unit/data_anndata/test_nan_anndata_adaptor.py +++ b/server/test/unit/data_anndata/test_nan_anndata_adaptor.py @@ -22,8 +22,9 @@ class NaNTest(unittest.TestCase): self.data._create_schema() def test_load(self): - with self.assertWarns(UserWarning): + with self.assertLogs(level="WARN") as logger: self.data = AnndataAdaptor(self.data_locator, self.config) + self.assertTrue(logger.output) def test_init(self): self.assertEqual(self.data.cell_count, 100)