fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-28 09:18:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

Browse Source 
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
2021-03-26 00:27:07 -05:00
committed by GitHub
parent e6e358ddc8
commit 78c9d24ed4
425 changed files with 734 additions and 5317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/common/utils/__init__.py
View File
server/common/utils/corpora_constants.py
-22
View File
@@ -1,22 +0,0 @@
class CorporaConstants(object):
REQUIRED_SIMPLE_METADATA_FIELDS = [
"version",
"title",
"layer_descriptions",
"organism",
"organism_ontology_term_id",
]
# The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
# types.
OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
OPTIONAL_SIMPLE_METADATA_FIELDS = [
"preprint_doi",
"publication_doi",
"default_embedding",
"default_field",
"tags",
"project_name",
"project_description",
]
server/common/utils/cxg_constants.py
-4
View File
@@ -1,4 +0,0 @@
class CxgConstants(object):
# The CXG container version number. Must be a semver string (major.minor.patch)
# DO NOT UPDATE THIS WITHOUT ALSO UPDATING CXG SPECIFICATION.
CXG_VERSION = "0.2.0"
server/common/utils/cxg_generation_utils.py
-178
View File
@@ -1,178 +0,0 @@
import json
import numpy as np
import tiledb
from server.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
"""
Saves the contents of the dictionary to the CXG output directory specified.
This function is primarily used to save metadata about a dataset to the CXG directory. At some point, tiledb will
have support for metadata on groups at which point the utility of this function should be revisited. Until such
feature exists, this function create an empty array and annotate that array.
For more information, visit https://github.com/TileDB-Inc/TileDB-Py/issues/254.
"""
array_name = f"{cxg_container}/{group_metadata_name}"
# Because TileDB does not allow one to attach metadata directly to a CXG group, we need to have a workaround
# where we create an empty array and attached the metadata onto to this empty array. Below we construct this empty
# array.
tiledb.from_numpy(array_name, np.zeros((1,)))
with tiledb.DenseArray(array_name, mode="w") as metadata_array:
for key, value in metadata_dict.items():
metadata_array.meta[key] = value
def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, index_column_name, ctx):
"""
Saves the contents of the dataframe to the CXG output directory specified.
Current access patterns are oriented toward reading very large slices of the dataframe, one attribute at a time.
Attribute data also tends to be (often) repetitive (bools, categories, strings). Given this, we use a large tile
size (1000) and very aggressive compression levels.
"""
def create_dataframe_array(array_name, dataframe):
tiledb_filter = tiledb.FilterList(
[
# Attempt aggressive compression as many of these dataframes are very repetitive strings, bools and
# other non-float data.
tiledb.ZstdFilter(level=22),
]
)
attrs = [
tiledb.Attr(name=column, dtype=get_dtype_of_array(dataframe[column]), filters=tiledb_filter)
for column in dataframe
]
domain = tiledb.Domain(
tiledb.Dim(domain=(0, dataframe.shape[0] - 1), tile=min(dataframe.shape[0], 1000), dtype=np.uint32)
)
schema = tiledb.ArraySchema(
domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major"
)
tiledb.DenseArray.create(array_name, schema)
array_name = f"{cxg_container}/{dataframe_name}"
create_dataframe_array(array_name, dataframe)
with tiledb.DenseArray(array_name, mode="w", ctx=ctx) as array:
value = {}
schema_hints = {}
for column_name, column_values in dataframe.items():
dtype, hints = get_dtype_and_schema_of_array(column_values)
value[column_name] = column_values.to_numpy(dtype=dtype)
if hints:
schema_hints.update({column_name: hints})
schema_hints.update({"index": index_column_name})
array[:] = value
array.meta["cxg_schema"] = json.dumps(schema_hints)
tiledb.consolidate(array_name, ctx=ctx)
def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
"""
Saves contents of ndarray to the CXG output directory specified.
Generally this function is used to convert dataset embeddings. Because embeddings are typically accessed with
very large slices (or all of the embedding), they do not benefit from overly aggressive compression due to their
format. Given this, we use a large tile size (1000) but only default compression level.
"""
def create_ndarray_array(ndarray_name, ndarray):
filters = tiledb.FilterList([tiledb.ZstdFilter()])
attrs = [tiledb.Attr(dtype=ndarray.dtype, filters=filters)]
dimensions = [
tiledb.Dim(
domain=(0, ndarray.shape[dimension] - 1), tile=min(ndarray.shape[dimension], 1000), dtype=np.uint32
)
for dimension in range(ndarray.ndim)
]
domain = tiledb.Domain(*dimensions)
schema = tiledb.ArraySchema(
domain=domain, sparse=False, attrs=attrs, capacity=1_000_000, cell_order="row-major", tile_order="row-major"
)
tiledb.DenseArray.create(ndarray_name, schema)
create_ndarray_array(ndarray_name, ndarray)
with tiledb.DenseArray(ndarray_name, mode="w", ctx=ctx) as array:
array[:] = ndarray
tiledb.consolidate(ndarray_name, ctx=ctx)
def convert_matrix_to_cxg_array(
matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
):
"""
Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
is true or not. Note that when the matrix is encoded as a SparseArray, it only writes the values that are
nonzero. This means that if you count the number of elements in the SparseArray, it will not equal the total
number of elements in the matrix, only the number of nonzero elements.
Furthermore, if the `column_shift_for_sparse_encoding` matrix is not None, this function will subtract the sparse
encoding from the original given matrix and as previously stated, only write the nonzero values to the TileDB
SparseArray.
"""
def create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array):
filters = tiledb.FilterList([tiledb.ZstdFilter()])
attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
if encode_as_sparse_array:
domain = tiledb.Domain(
tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 512), dtype=np.uint32),
tiledb.Dim(
name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 2048), dtype=np.uint32
),
)
else:
domain = tiledb.Domain(
tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 50), dtype=np.uint32),
tiledb.Dim(
name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 100), dtype=np.uint32
),
)
schema = tiledb.ArraySchema(
domain=domain, sparse=encode_as_sparse_array, attrs=attrs, cell_order="row-major", tile_order="col-major"
)
if encode_as_sparse_array:
tiledb.SparseArray.create(matrix_name, schema)
else:
tiledb.DenseArray.create(matrix_name, schema)
number_of_rows = matrix.shape[0]
number_of_columns = matrix.shape[1]
stride = min(int(np.power(10, np.around(np.log10(1e9 / number_of_columns)))), 10_000)
create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array)
if encode_as_sparse_array:
with tiledb.SparseArray(matrix_name, mode="w", ctx=ctx) as array:
for start_row_index in range(0, number_of_rows, stride):
end_row_index = min(start_row_index + stride, number_of_rows)
matrix_subset = matrix[start_row_index:end_row_index, :]
if not isinstance(matrix_subset, np.ndarray):
matrix_subset = matrix_subset.toarray()
if column_shift_for_sparse_encoding is not None:
matrix_subset = matrix_subset - column_shift_for_sparse_encoding
indices = np.nonzero(matrix_subset)
trow = indices[0] + start_row_index
array[trow, indices[1]] = matrix_subset[indices[0], indices[1]]
else:
with tiledb.DenseArray(matrix_name, mode="w", ctx=ctx) as array:
for start_row_index in range(0, number_of_rows, stride):
end_row_index = min(start_row_index + stride, number_of_rows)
matrix_subset = matrix[start_row_index:end_row_index, :]
if not isinstance(matrix_subset, np.ndarray):
matrix_subset = matrix_subset.toarray()
array[start_row_index:end_row_index, :] = matrix_subset
server/common/utils/matrix_utils.py
-115
View File
@@ -1,115 +0,0 @@
import logging
import numpy as np
from scipy.stats import mode
def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
"""
Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has
a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of
elements evaluated. This function may return before evaluating the whole matrix if it can be determined that matrix
is not sparse enough.
"""
if sparse_threshold == 100.0:
return True
if sparse_threshold == 0.0:
return False
total_number_of_rows = matrix.shape[0]
total_number_of_columns = matrix.shape[1]
total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
# For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the
# maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the
# quit early for large dense matrices.
row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000)
maximum_number_of_non_zero_elements_in_matrix = int(
total_number_of_rows * total_number_of_columns * sparse_threshold / 100
)
number_of_non_zero_elements = 0
for start_row_index in range(0, total_number_of_rows, row_stride):
end_row_index = min(start_row_index + row_stride, total_number_of_rows)
matrix_subset = matrix[start_row_index:end_row_index, :]
if not isinstance(matrix_subset, np.ndarray):
matrix_subset = matrix_subset.toarray()
number_of_non_zero_elements += np.count_nonzero(matrix_subset)
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
if end_row_index != total_number_of_rows:
percentage_of_non_zero_elements = (
100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
)
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
f"{percentage_of_non_zero_elements:6.2f}"
)
else:
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (exact): "
f"{percentage_of_non_zero_elements:6.2f}"
)
return False
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
return is_sparse
def get_column_shift_encode_for_matrix(matrix, sparse_threshold):
"""
Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column
shift encoding works by taking the most common value in each column, then subtracting that value from each element
of the column. If each column mostly contains its most common value, then the resulting matrix can be very sparse.
This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a
sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns
the number of non-zeros encountered and number of elements evaluated. This function may return before evaluating
the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding.
"""
total_number_of_rows = matrix.shape[0]
total_number_of_columns = matrix.shape[1]
total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
stride = max(1, 128_000_000 // total_number_of_rows)
column_shift = np.zeros(total_number_of_columns)
maximum_number_of_non_zero_elements_in_matrix = int(
total_number_of_rows * total_number_of_columns * sparse_threshold / 100
)
number_of_non_zero_elements = 0
for start_column_index in range(0, total_number_of_columns, stride):
end_column_index = min(start_column_index + stride, total_number_of_columns)
matrix_subset = matrix[:, start_column_index:end_column_index]
if not isinstance(matrix_subset, np.ndarray):
matrix_subset = matrix_subset.toarray()
matrix_subset_mode = mode(matrix_subset)
column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode
number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum(
matrix_subset_mode.count
)
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
if end_column_index != total_number_of_columns:
logging.info(
"Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f"
% (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows)
)
else:
logging.info(
"Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f"
% (100 * number_of_non_zero_elements / total_number_of_matrix_elements)
)
return None
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
return column_shift if is_sparse else None
server/common/utils/sanitization_utils.py
-40
View File
@@ -1,40 +0,0 @@
import re
def sanitize_values_in_list(list_of_keys: list):
"""
Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both
safe and unique.
"""
if not all([isinstance(key, str) for key in list_of_keys]):
raise Exception("List of keys to sanitize must contain all strings.")
# Mask out [~/.] and anything outside the ASCII range.
mask = re.compile(r"[^ -\-0-\[\]-\}]")
clean_keys_list = [mask.sub("_", key) for key in list_of_keys]
# Dedupe the clean keys list
deduped_clean_keys_list = []
for index, clean_key in enumerate(clean_keys_list):
total_occurrences_of_clean_key = clean_keys_list.count(clean_key)
total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key)
deduped_clean_keys_list.append(
clean_key + "_" + str(total_occurrences_up_until_current_index + 1)
if total_occurrences_of_clean_key > 1
else clean_key
)
return dict(zip(list_of_keys, deduped_clean_keys_list))
def sanitize_keys_in_dictionary(dict_to_sanitize: dict):
"""
Clean and dedupe the keys in the given dictionary.
"""
clean_keys = sanitize_values_in_list(dict_to_sanitize.keys())
for original_key, sanitized_key in clean_keys.items():
if original_key != sanitized_key:
dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key]
del dict_to_sanitize[original_key]
server/common/utils/type_conversion_utils.py
-158
View File
@@ -1,158 +0,0 @@
import logging
import numpy as np
import pandas as pd
def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
dtypes_by_column_name = {}
schema_type_hints_by_column_name = {}
for column_name, column_values in dataframe.items():
(
dtypes_by_column_name[column_name],
schema_type_hints_by_column_name[column_name],
) = get_dtype_and_schema_of_array(column_values)
return dtypes_by_column_name, schema_type_hints_by_column_name
def get_dtype_of_array(array: pd.Series):
return get_dtype_and_schema_of_array(array)[0]
def get_schema_type_hint_of_array(array: pd.Series):
return get_dtype_and_schema_of_array(array)[1]
def get_dtype_and_schema_of_array(array: pd.Series):
return (
get_dtype_from_dtype(array.dtype, array_values=array),
get_schema_type_hint_from_dtype(array.dtype, array_values=array),
)
def get_dtype_from_dtype(dtype, array_values=None):
"""
Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
for 64 bit values which will get downcast to 32 bit.
"""
dtype_name = dtype.name
dtype_kind = dtype.kind
if dtype_name == "bool":
return np.uint8
if dtype_name == "object" and dtype_kind == "O":
return str
if dtype_name == "category":
return get_dtype_from_dtype(dtype.categories.dtype, array_values)
if can_cast_to_int32(dtype, array_values):
return np.int32
if can_cast_to_float32(dtype, array_values):
return np.float32
if not can_cast_to_float32(dtype, array_values):
return np.float64
raise TypeError(f"Annotations of type {dtype} are unsupported.")
def get_schema_type_hint_from_dtype(dtype, array_values=None):
"""
Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
and will be downcast to 32 bit.
"""
dtype_name = dtype.name
dtype_kind = dtype.kind
if dtype == np.float32 or dtype == np.int32:
return {"type": dtype_name}
if dtype_name == "bool":
return {"type": "boolean"}
if dtype_name == "object" and dtype_kind == "O":
return {"type": "string"}
if dtype_name == "category":
return {"type": "categorical", "categories": dtype.categories.tolist()}
if can_cast_to_int32(dtype, array_values):
return {"type": "int32"}
if can_cast_to_float32(dtype, array_values):
return {"type": "float32"}
if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
return {"type": "float64"}
raise TypeError(f"Annotations of type {dtype} are unsupported.")
def can_cast_to_float32(dtype, array_values):
"""
Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
a float.
We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
"""
if dtype.kind == "f":
if not np.can_cast(dtype, np.float32):
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
return True
if dtype.kind == "O" and array_values.hasnans:
return True
return False
def can_cast_to_int32(dtype, array_values=None):
"""
A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
the higher precision type has values that are entirely within the range of the downcast type.
"""
# Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
# return False.
if array_values.hasnans:
return False
# If the array is categorical, then we need to order the array values so that functions min and max that occur
# later, can function. They do not function on unordered categories.
ordered_array_values = array_values
if array_values.dtype.name == "category" and not array_values.cat.ordered:
ordered_array_values = array_values.cat.as_ordered()
if dtype.kind in ["i", "u"]:
if np.can_cast(dtype, np.int32):
return True
ii32 = np.iinfo(np.int32)
if (
not ordered_array_values.empty
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
or ordered_array_values.empty
):
return True
return False
def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
if series_to_convert.hasnans and dtype == np.int32:
logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
return series_to_convert.to_numpy(dtype)
def convert_string_to_value(value: str):
"""convert a string to value with the most appropriate type"""
if value.lower() == "true":
return True
if value.lower() == "false":
return False
if value == "null":
return None
try:
return eval(value)
except: # noqa E722
return value
server/common/utils/utils.py
-118
View File
@@ -1,118 +0,0 @@
import contextlib
import errno
import importlib.util
import logging
import os
import pkgutil
import socket
from urllib.parse import urlsplit, urljoin
import numpy as np
from flask import json
from server.common.errors import ConfigurationError
def find_available_port(host, port=5005):
"""
Helper method to find open port on host. Tries 5000 ports incremented from the specified port
"""
# Takes approx 2 seconds to do a scan of 5000 ports on my laptop
num_ports_to_try = 5000
for port_to_try in range(port, port + num_ports_to_try):
if is_port_available(host, port_to_try):
return port_to_try
raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
def is_port_available(host, port):
is_available = False
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
try:
s.bind((host, port))
is_available = True
except socket.error:
pass
return is_available
def sort_options(command):
"""
Helper for the click options - will sort options in a command, and can
be used as a decorator.
"""
command.params.sort(key=lambda p: p.name)
return command
def path_join(base, *urls):
"""
this is like urllib.parse.urljoin, except it works around the scheme-specific
cleverness in the aforementioned code, ignores anything in the url except the path,
and accepts more than one url.
"""
if not base.endswith("/"):
base += "/"
btpl = urlsplit(base)
path = btpl.path
for url in urls:
utpl = urlsplit(url)
if btpl.scheme == "":
path = os.path.join(path, utpl.path)
path = os.path.normpath(path)
else:
path = urljoin(path, utpl.path)
return btpl._replace(path=path).geturl()
class Float32JSONEncoder(json.JSONEncoder):
def __init__(self, *args, **kwargs):
"""
NaN/Infinities are illegal in standard JSON. Python extends JSON with
non-standard symbols that most JavaScript JSON parsers do not understand.
The `allow_nan` parameter will force Python simplejson to throw an ValueError
if it runs into non-finite floating point values which are unsupported by
standard JSON.
"""
kwargs["allow_nan"] = False
super().__init__(*args, **kwargs)
def default(self, obj):
if isinstance(obj, np.float32):
return float(obj)
elif isinstance(obj, np.integer):
return int(obj)
return json.JSONEncoder.default(self, obj)
def custom_format_warning(msg, *args, **kwargs):
return f"[cellxgene] Warning: {msg} \n"
def jsonify_numpy(data):
return json.dumps(data, cls=Float32JSONEncoder, allow_nan=False)
def import_plugins(plugin_module):
"""
Load optional plugin modules from server.common.plugins
If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
normal.
"""
loaded_modules = []
try:
pkg = importlib.import_module(plugin_module)
for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
full_name = f"{plugin_module}.{name}"
try:
module = importlib.import_module(full_name)
except Exception as e:
raise ConfigurationError(f"Unexpected error while importing plugin: {plugin_module}.{name}: {str(e)}")
loaded_modules.append(module)
except ModuleNotFoundError as e:
# This exception occurs when the plugin_module does not exist (not an error).
logging.debug(f"No plugins found in module: {plugin_module}: {str(e)}")
return loaded_modules