mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-02 06:48:12 +08:00
Refactoring cxg utility classes in preparation for CXG conversion tooling (#1739)
This commit is contained in:
maniarathi
@@ -0,0 +1,112 @@
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
from scipy.stats import mode
|
||||
|
||||
|
||||
def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
|
||||
"""
|
||||
Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has
|
||||
a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of
|
||||
elements evaluated. This function may return before evaluating the whole matrix if it can be determined that matrix
|
||||
is not sparse enough.
|
||||
"""
|
||||
|
||||
if sparse_threshold == 100.0:
|
||||
return True
|
||||
if sparse_threshold == 0.0:
|
||||
return False
|
||||
|
||||
total_number_of_rows = matrix.shape[0]
|
||||
total_number_of_columns = matrix.shape[1]
|
||||
total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
|
||||
|
||||
# For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the
|
||||
# maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the
|
||||
# quit early for large dense matrices.
|
||||
row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000)
|
||||
|
||||
maximum_number_of_non_zero_elements_in_matrix = int(
|
||||
total_number_of_rows * total_number_of_columns * sparse_threshold / 100
|
||||
)
|
||||
number_of_non_zero_elements = 0
|
||||
|
||||
for start_row_index in range(0, total_number_of_rows, row_stride):
|
||||
end_row_index = min(start_row_index + row_stride, total_number_of_rows)
|
||||
|
||||
matrix_subset = matrix[start_row_index:end_row_index, :]
|
||||
if not isinstance(matrix_subset, np.ndarray):
|
||||
matrix_subset = matrix_subset.toarray()
|
||||
|
||||
number_of_non_zero_elements += np.count_nonzero(matrix_subset)
|
||||
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
|
||||
if end_row_index != total_number_of_rows:
|
||||
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / (
|
||||
end_row_index * total_number_of_columns)
|
||||
logging.info(
|
||||
f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
|
||||
f"{percentage_of_non_zero_elements:6.2f}")
|
||||
else:
|
||||
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
|
||||
logging.info(
|
||||
f"Matrix is not sparse. Percentage of non-zero elements (exact): "
|
||||
f"{percentage_of_non_zero_elements:6.2f}")
|
||||
return False
|
||||
|
||||
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
|
||||
return is_sparse
|
||||
|
||||
|
||||
def get_column_shift_encode_for_matrix(matrix, sparse_threshold):
|
||||
"""
|
||||
Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column
|
||||
shift encoding works by taking the most common value in each column, then subtracting that value from each element
|
||||
of the column. If each column mostly contains its most common value, then the resulting matrix can be very sparse.
|
||||
|
||||
This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a
|
||||
sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns
|
||||
the number of non-zeros encountered and number of elements evaluated. This function may return before evaluating
|
||||
the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding.
|
||||
"""
|
||||
|
||||
total_number_of_rows = matrix.shape[0]
|
||||
total_number_of_columns = matrix.shape[1]
|
||||
total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
|
||||
|
||||
stride = max(1, 128_000_000 // total_number_of_rows)
|
||||
column_shift = np.zeros(total_number_of_columns)
|
||||
|
||||
maximum_number_of_non_zero_elements_in_matrix = int(
|
||||
total_number_of_rows * total_number_of_columns * sparse_threshold / 100
|
||||
)
|
||||
number_of_non_zero_elements = 0
|
||||
|
||||
for start_column_index in range(0, total_number_of_columns, stride):
|
||||
end_column_index = min(start_column_index + stride, total_number_of_columns)
|
||||
|
||||
matrix_subset = matrix[:, start_column_index:end_column_index]
|
||||
if not isinstance(matrix_subset, np.ndarray):
|
||||
matrix_subset = matrix_subset.toarray()
|
||||
|
||||
matrix_subset_mode = mode(matrix_subset)
|
||||
|
||||
column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode
|
||||
number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum(
|
||||
matrix_subset_mode.count
|
||||
)
|
||||
|
||||
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
|
||||
if end_column_index != total_number_of_columns:
|
||||
logging.info(
|
||||
"Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f"
|
||||
% (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows)
|
||||
)
|
||||
else:
|
||||
logging.info(
|
||||
"Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f"
|
||||
% (100 * number_of_non_zero_elements / total_number_of_matrix_elements)
|
||||
)
|
||||
return None
|
||||
|
||||
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
|
||||
return column_shift if is_sparse else None
|
||||
@@ -0,0 +1,40 @@
|
||||
import re
|
||||
|
||||
|
||||
def sanitize_values_in_list(list_of_keys: list):
|
||||
"""
|
||||
Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both
|
||||
safe and unique.
|
||||
"""
|
||||
|
||||
if not all([isinstance(key, str) for key in list_of_keys]):
|
||||
raise Exception("List of keys to sanitize must contain all strings.")
|
||||
|
||||
# Mask out [~/.] and anything outside the ASCII range.
|
||||
mask = re.compile(r"[^ -\-0-\[\]-\}]")
|
||||
clean_keys_list = [mask.sub("_", key) for key in list_of_keys]
|
||||
|
||||
# Dedupe the clean keys list
|
||||
deduped_clean_keys_list = []
|
||||
for index, clean_key in enumerate(clean_keys_list):
|
||||
total_occurrences_of_clean_key = clean_keys_list.count(clean_key)
|
||||
total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key)
|
||||
deduped_clean_keys_list.append(
|
||||
clean_key + "_" + str(total_occurrences_up_until_current_index + 1)
|
||||
if total_occurrences_of_clean_key > 1
|
||||
else clean_key
|
||||
)
|
||||
|
||||
return dict(zip(list_of_keys, deduped_clean_keys_list))
|
||||
|
||||
|
||||
def sanitize_keys_in_dictionary(dict_to_sanitize: dict):
|
||||
"""
|
||||
Clean and dedupe the keys in the given dictionary.
|
||||
"""
|
||||
|
||||
clean_keys = sanitize_values_in_list(dict_to_sanitize.keys())
|
||||
for original_key, sanitized_key in clean_keys.items():
|
||||
if original_key != sanitized_key:
|
||||
dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key]
|
||||
del dict_to_sanitize[original_key]
|
||||
@@ -0,0 +1,93 @@
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def get_dtype_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[0]
|
||||
|
||||
|
||||
def get_schema_type_hint_of_array(array: pd.Series):
|
||||
return get_dtype_and_schema_of_array(array)[1]
|
||||
|
||||
|
||||
def get_dtype_and_schema_of_array(array: pd.Series):
|
||||
return (get_dtype_from_dtype(array.dtype, array_values=array),
|
||||
get_schema_type_hint_from_dtype(array.dtype, array_values=array))
|
||||
|
||||
|
||||
def get_dtype_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
|
||||
for 64 bit values which will get downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype == np.float32 or dtype == np.int32:
|
||||
return dtype
|
||||
if dtype_name == "bool":
|
||||
return np.uint8
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return np.unicode
|
||||
if dtype_name == "category":
|
||||
return get_dtype_from_dtype(dtype.categories.dtype, dtype.categories)
|
||||
|
||||
if can_cast_to_float32(dtype):
|
||||
return np.float32
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return np.int32
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def get_schema_type_hint_from_dtype(dtype, array_values=None):
|
||||
"""
|
||||
Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
|
||||
and will be downcast to 32 bit.
|
||||
"""
|
||||
|
||||
dtype_name = dtype.name
|
||||
dtype_kind = dtype.kind
|
||||
|
||||
if dtype == np.float32 or dtype == np.int32:
|
||||
return {"type": dtype_name}
|
||||
if dtype_name == "bool":
|
||||
return {"type": "boolean"}
|
||||
if dtype_name == "object" and dtype_kind == "O":
|
||||
return {"type": "string"}
|
||||
if dtype_name == "category":
|
||||
return {"type": "categorical", "categories": dtype.categories.tolist()}
|
||||
|
||||
if can_cast_to_float32(dtype):
|
||||
return {"type": "float32"}
|
||||
if can_cast_to_int32(dtype, array_values):
|
||||
return {"type": "int32"}
|
||||
|
||||
raise TypeError(f"Annotations of type {dtype} are unsupported.")
|
||||
|
||||
|
||||
def can_cast_to_float32(dtype):
|
||||
if dtype.kind == "f":
|
||||
if not np.can_cast(dtype, np.float32):
|
||||
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def can_cast_to_int32(dtype, array_values=None):
|
||||
"""
|
||||
A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
|
||||
the higher precision type has values that are entirely within the range of the downcast type.
|
||||
"""
|
||||
|
||||
if dtype.kind in ["i", "u"]:
|
||||
if np.can_cast(dtype, np.int32):
|
||||
return True
|
||||
ii32 = np.iinfo(np.int32)
|
||||
if not array_values.empty and (
|
||||
array_values.min() >= ii32.min and array_values.max() <= ii32.max) or array_values.empty:
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1,118 @@
|
||||
import contextlib
|
||||
import errno
|
||||
import importlib.util
|
||||
import logging
|
||||
import os
|
||||
import pkgutil
|
||||
import socket
|
||||
from urllib.parse import urlsplit, urljoin
|
||||
|
||||
import numpy as np
|
||||
from flask import json
|
||||
|
||||
from server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
def find_available_port(host, port=5005):
|
||||
"""
|
||||
Helper method to find open port on host. Tries 5000 ports incremented from the specified port
|
||||
"""
|
||||
# Takes approx 2 seconds to do a scan of 5000 ports on my laptop
|
||||
num_ports_to_try = 5000
|
||||
for port_to_try in range(port, port + num_ports_to_try):
|
||||
if is_port_available(host, port_to_try):
|
||||
return port_to_try
|
||||
raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
|
||||
|
||||
|
||||
def is_port_available(host, port):
|
||||
is_available = False
|
||||
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
|
||||
try:
|
||||
s.bind((host, port))
|
||||
is_available = True
|
||||
except socket.error:
|
||||
pass
|
||||
return is_available
|
||||
|
||||
|
||||
def sort_options(command):
|
||||
"""
|
||||
Helper for the click options - will sort options in a command, and can
|
||||
be used as a decorator.
|
||||
"""
|
||||
command.params.sort(key=lambda p: p.name)
|
||||
return command
|
||||
|
||||
|
||||
def path_join(base, *urls):
|
||||
"""
|
||||
this is like urllib.parse.urljoin, except it works around the scheme-specific
|
||||
cleverness in the aforementioned code, ignores anything in the url except the path,
|
||||
and accepts more than one url.
|
||||
"""
|
||||
if not base.endswith("/"):
|
||||
base += "/"
|
||||
btpl = urlsplit(base)
|
||||
path = btpl.path
|
||||
for url in urls:
|
||||
utpl = urlsplit(url)
|
||||
if btpl.scheme == "":
|
||||
path = os.path.join(path, utpl.path)
|
||||
path = os.path.normpath(path)
|
||||
else:
|
||||
path = urljoin(path, utpl.path)
|
||||
return btpl._replace(path=path).geturl()
|
||||
|
||||
|
||||
class Float32JSONEncoder(json.JSONEncoder):
|
||||
def __init__(self, *args, **kwargs):
|
||||
"""
|
||||
NaN/Infinities are illegal in standard JSON. Python extends JSON with
|
||||
non-standard symbols that most JavaScript JSON parsers do not understand.
|
||||
The `allow_nan` parameter will force Python simplejson to throw an ValueError
|
||||
if it runs into non-finite floating point values which are unsupported by
|
||||
standard JSON.
|
||||
"""
|
||||
kwargs["allow_nan"] = False
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def default(self, obj):
|
||||
if isinstance(obj, np.float32):
|
||||
return float(obj)
|
||||
elif isinstance(obj, np.integer):
|
||||
return int(obj)
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
|
||||
def custom_format_warning(msg, *args, **kwargs):
|
||||
return f"[cellxgene] Warning: {msg} \n"
|
||||
|
||||
|
||||
def jsonify_numpy(data):
|
||||
return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
|
||||
|
||||
|
||||
def import_plugins(plugin_module):
|
||||
"""
|
||||
Load optional plugin modules from server.common.plugins
|
||||
|
||||
If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
|
||||
This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
|
||||
normal.
|
||||
"""
|
||||
loaded_modules = []
|
||||
try:
|
||||
pkg = importlib.import_module(plugin_module)
|
||||
for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
|
||||
full_name = f"{plugin_module}.{name}"
|
||||
try:
|
||||
module = importlib.import_module(full_name)
|
||||
except Exception as e:
|
||||
raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}")
|
||||
loaded_modules.append(module)
|
||||
except ModuleNotFoundError as e:
|
||||
# This exception occurs when the plugin_module does not exist (not an error).
|
||||
logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
|
||||
|
||||
return loaded_modules
|
||||
Reference in New Issue
Block a user