fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-03 21:18:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

Browse Source 
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
2021-03-26 00:27:07 -05:00
committed by GitHub
parent e6e358ddc8
commit 78c9d24ed4
425 changed files with 734 additions and 5317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/czi_hosted/common/utils/__init__.py
View File
backend/czi_hosted/common/utils/corpora_constants.py
+22
View File
@@ -0,0 +1,22 @@
class CorporaConstants(object):
REQUIRED_SIMPLE_METADATA_FIELDS = [
"version",
"title",
"layer_descriptions",
"organism",
"organism_ontology_term_id",
]
# The Corpora specification requires some values encoded as JSON due to the inability of AnnData to store complex
# types.
OPTIONAL_JSON_ENCODED_METADATA_FIELD = ["contributors", "project_links"]
OPTIONAL_SIMPLE_METADATA_FIELDS = [
"preprint_doi",
"publication_doi",
"default_embedding",
"default_field",
"tags",
"project_name",
"project_description",
]
backend/czi_hosted/common/utils/cxg_constants.py
+4
View File
@@ -0,0 +1,4 @@
class CxgConstants(object):
# The CXG container version number. Must be a semver string (major.minor.patch)
# DO NOT UPDATE THIS WITHOUT ALSO UPDATING CXG SPECIFICATION.
CXG_VERSION = "0.2.0"
backend/czi_hosted/common/utils/cxg_generation_utils.py
+178
View File
@@ -0,0 +1,178 @@
import json
import numpy as np
import tiledb
from backend.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
"""
Saves the contents of the dictionary to the CXG output directory specified.
This function is primarily used to save metadata about a dataset to the CXG directory. At some point, tiledb will
have support for metadata on groups at which point the utility of this function should be revisited. Until such
feature exists, this function create an empty array and annotate that array.
For more information, visit https://github.com/TileDB-Inc/TileDB-Py/issues/254.
"""
array_name = f"{cxg_container}/{group_metadata_name}"
# Because TileDB does not allow one to attach metadata directly to a CXG group, we need to have a workaround
# where we create an empty array and attached the metadata onto to this empty array. Below we construct this empty
# array.
tiledb.from_numpy(array_name, np.zeros((1,)))
with tiledb.DenseArray(array_name, mode="w") as metadata_array:
for key, value in metadata_dict.items():
metadata_array.meta[key] = value
def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, index_column_name, ctx):
"""
Saves the contents of the dataframe to the CXG output directory specified.
Current access patterns are oriented toward reading very large slices of the dataframe, one attribute at a time.
Attribute data also tends to be (often) repetitive (bools, categories, strings). Given this, we use a large tile
size (1000) and very aggressive compression levels.
"""
def create_dataframe_array(array_name, dataframe):
tiledb_filter = tiledb.FilterList(
[
# Attempt aggressive compression as many of these dataframes are very repetitive strings, bools and
# other non-float data.
tiledb.ZstdFilter(level=22),
]
)
attrs = [
tiledb.Attr(name=column, dtype=get_dtype_of_array(dataframe[column]), filters=tiledb_filter)
for column in dataframe
]
domain = tiledb.Domain(
tiledb.Dim(domain=(0, dataframe.shape[0] - 1), tile=min(dataframe.shape[0], 1000), dtype=np.uint32)
)
schema = tiledb.ArraySchema(
domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="row-major"
)
tiledb.DenseArray.create(array_name, schema)
array_name = f"{cxg_container}/{dataframe_name}"
create_dataframe_array(array_name, dataframe)
with tiledb.DenseArray(array_name, mode="w", ctx=ctx) as array:
value = {}
schema_hints = {}
for column_name, column_values in dataframe.items():
dtype, hints = get_dtype_and_schema_of_array(column_values)
value[column_name] = column_values.to_numpy(dtype=dtype)
if hints:
schema_hints.update({column_name: hints})
schema_hints.update({"index": index_column_name})
array[:] = value
array.meta["cxg_schema"] = json.dumps(schema_hints)
tiledb.consolidate(array_name, ctx=ctx)
def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
"""
Saves contents of ndarray to the CXG output directory specified.
Generally this function is used to convert dataset embeddings. Because embeddings are typically accessed with
very large slices (or all of the embedding), they do not benefit from overly aggressive compression due to their
format. Given this, we use a large tile size (1000) but only default compression level.
"""
def create_ndarray_array(ndarray_name, ndarray):
filters = tiledb.FilterList([tiledb.ZstdFilter()])
attrs = [tiledb.Attr(dtype=ndarray.dtype, filters=filters)]
dimensions = [
tiledb.Dim(
domain=(0, ndarray.shape[dimension] - 1), tile=min(ndarray.shape[dimension], 1000), dtype=np.uint32
)
for dimension in range(ndarray.ndim)
]
domain = tiledb.Domain(*dimensions)
schema = tiledb.ArraySchema(
domain=domain, sparse=False, attrs=attrs, capacity=1_000_000, cell_order="row-major", tile_order="row-major"
)
tiledb.DenseArray.create(ndarray_name, schema)
create_ndarray_array(ndarray_name, ndarray)
with tiledb.DenseArray(ndarray_name, mode="w", ctx=ctx) as array:
array[:] = ndarray
tiledb.consolidate(ndarray_name, ctx=ctx)
def convert_matrix_to_cxg_array(
matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
):
"""
Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
is true or not. Note that when the matrix is encoded as a SparseArray, it only writes the values that are
nonzero. This means that if you count the number of elements in the SparseArray, it will not equal the total
number of elements in the matrix, only the number of nonzero elements.
Furthermore, if the `column_shift_for_sparse_encoding` matrix is not None, this function will subtract the sparse
encoding from the original given matrix and as previously stated, only write the nonzero values to the TileDB
SparseArray.
"""
def create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array):
filters = tiledb.FilterList([tiledb.ZstdFilter()])
attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
if encode_as_sparse_array:
domain = tiledb.Domain(
tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 512), dtype=np.uint32),
tiledb.Dim(
name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 2048), dtype=np.uint32
),
)
else:
domain = tiledb.Domain(
tiledb.Dim(name="obs", domain=(0, number_of_rows - 1), tile=min(number_of_rows, 50), dtype=np.uint32),
tiledb.Dim(
name="var", domain=(0, number_of_columns - 1), tile=min(number_of_columns, 100), dtype=np.uint32
),
)
schema = tiledb.ArraySchema(
domain=domain, sparse=encode_as_sparse_array, attrs=attrs, cell_order="row-major", tile_order="col-major"
)
if encode_as_sparse_array:
tiledb.SparseArray.create(matrix_name, schema)
else:
tiledb.DenseArray.create(matrix_name, schema)
number_of_rows = matrix.shape[0]
number_of_columns = matrix.shape[1]
stride = min(int(np.power(10, np.around(np.log10(1e9 / number_of_columns)))), 10_000)
create_matrix_array(matrix_name, number_of_rows, number_of_columns, encode_as_sparse_array)
if encode_as_sparse_array:
with tiledb.SparseArray(matrix_name, mode="w", ctx=ctx) as array:
for start_row_index in range(0, number_of_rows, stride):
end_row_index = min(start_row_index + stride, number_of_rows)
matrix_subset = matrix[start_row_index:end_row_index, :]
if not isinstance(matrix_subset, np.ndarray):
matrix_subset = matrix_subset.toarray()
if column_shift_for_sparse_encoding is not None:
matrix_subset = matrix_subset - column_shift_for_sparse_encoding
indices = np.nonzero(matrix_subset)
trow = indices[0] + start_row_index
array[trow, indices[1]] = matrix_subset[indices[0], indices[1]]
else:
with tiledb.DenseArray(matrix_name, mode="w", ctx=ctx) as array:
for start_row_index in range(0, number_of_rows, stride):
end_row_index = min(start_row_index + stride, number_of_rows)
matrix_subset = matrix[start_row_index:end_row_index, :]
if not isinstance(matrix_subset, np.ndarray):
matrix_subset = matrix_subset.toarray()
array[start_row_index:end_row_index, :] = matrix_subset
backend/czi_hosted/common/utils/matrix_utils.py
+115
View File
@@ -0,0 +1,115 @@
import logging
import numpy as np
from scipy.stats import mode
def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
"""
Returns whether `matrix` is sparse or not (i.e. dense). This is determined by figuring out whether the matrix has
a sparsity percentage below the sparse_threshold, returning the number of non-zeros encountered and number of
elements evaluated. This function may return before evaluating the whole matrix if it can be determined that matrix
is not sparse enough.
"""
if sparse_threshold == 100.0:
return True
if sparse_threshold == 0.0:
return False
total_number_of_rows = matrix.shape[0]
total_number_of_columns = matrix.shape[1]
total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
# For efficiency, we count the number of non-zero elements in chunks of the matrix at a time until we hit the
# maximum number of non zero values allowed before the matrix is deemed "dense." This allows the function the
# quit early for large dense matrices.
row_stride = min(int(np.power(10, np.around(np.log10(1e9 / total_number_of_columns)))), 10_000)
maximum_number_of_non_zero_elements_in_matrix = int(
total_number_of_rows * total_number_of_columns * sparse_threshold / 100
)
number_of_non_zero_elements = 0
for start_row_index in range(0, total_number_of_rows, row_stride):
end_row_index = min(start_row_index + row_stride, total_number_of_rows)
matrix_subset = matrix[start_row_index:end_row_index, :]
if not isinstance(matrix_subset, np.ndarray):
matrix_subset = matrix_subset.toarray()
number_of_non_zero_elements += np.count_nonzero(matrix_subset)
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
if end_row_index != total_number_of_rows:
percentage_of_non_zero_elements = (
100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
)
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
f"{percentage_of_non_zero_elements:6.2f}"
)
else:
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (exact): "
f"{percentage_of_non_zero_elements:6.2f}"
)
return False
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
return is_sparse
def get_column_shift_encode_for_matrix(matrix, sparse_threshold):
"""
Returns a column shift if there is a column shift that allows the given matrix to be considered as sparse. Column
shift encoding works by taking the most common value in each column, then subtracting that value from each element
of the column. If each column mostly contains its most common value, then the resulting matrix can be very sparse.
This function determines if column shift encoding can be used to transform the matrix into a sparse matrix with a
sparsity below the sparse_threshold. If so, returns the array that stores this encoding. This function also returns
the number of non-zeros encountered and number of elements evaluated. This function may return before evaluating
the whole matrix if it can be determined that the matrix cannot benefit from column shift encoding.
"""
total_number_of_rows = matrix.shape[0]
total_number_of_columns = matrix.shape[1]
total_number_of_matrix_elements = total_number_of_rows * total_number_of_columns
stride = max(1, 128_000_000 // total_number_of_rows)
column_shift = np.zeros(total_number_of_columns)
maximum_number_of_non_zero_elements_in_matrix = int(
total_number_of_rows * total_number_of_columns * sparse_threshold / 100
)
number_of_non_zero_elements = 0
for start_column_index in range(0, total_number_of_columns, stride):
end_column_index = min(start_column_index + stride, total_number_of_columns)
matrix_subset = matrix[:, start_column_index:end_column_index]
if not isinstance(matrix_subset, np.ndarray):
matrix_subset = matrix_subset.toarray()
matrix_subset_mode = mode(matrix_subset)
column_shift[start_column_index:end_column_index] = matrix_subset_mode.mode
number_of_non_zero_elements += total_number_of_rows * (end_column_index - start_column_index) - np.sum(
matrix_subset_mode.count
)
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
if end_column_index != total_number_of_columns:
logging.info(
"Matrix is not sparse even with column shift. Percentage of non-zero elements (estimate): %6.2f"
% (100 * number_of_non_zero_elements / end_column_index * total_number_of_rows)
)
else:
logging.info(
"Matrix is not sparse even with column shift. Percentage of non-zero elements (exact): %6.2f"
% (100 * number_of_non_zero_elements / total_number_of_matrix_elements)
)
return None
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
return column_shift if is_sparse else None
backend/czi_hosted/common/utils/sanitization_utils.py
+40
View File
@@ -0,0 +1,40 @@
import re
def sanitize_values_in_list(list_of_keys: list):
"""
Returns a dictionary mapping of the old keys in the list of `list_of_keys` to its new, clean name that is both
safe and unique.
"""
if not all([isinstance(key, str) for key in list_of_keys]):
raise Exception("List of keys to sanitize must contain all strings.")
# Mask out [~/.] and anything outside the ASCII range.
mask = re.compile(r"[^ -\-0-\[\]-\}]")
clean_keys_list = [mask.sub("_", key) for key in list_of_keys]
# Dedupe the clean keys list
deduped_clean_keys_list = []
for index, clean_key in enumerate(clean_keys_list):
total_occurrences_of_clean_key = clean_keys_list.count(clean_key)
total_occurrences_up_until_current_index = clean_keys_list[:index].count(clean_key)
deduped_clean_keys_list.append(
clean_key + "_" + str(total_occurrences_up_until_current_index + 1)
if total_occurrences_of_clean_key > 1
else clean_key
)
return dict(zip(list_of_keys, deduped_clean_keys_list))
def sanitize_keys_in_dictionary(dict_to_sanitize: dict):
"""
Clean and dedupe the keys in the given dictionary.
"""
clean_keys = sanitize_values_in_list(dict_to_sanitize.keys())
for original_key, sanitized_key in clean_keys.items():
if original_key != sanitized_key:
dict_to_sanitize[sanitized_key] = dict_to_sanitize[original_key]
del dict_to_sanitize[original_key]