mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-08 06:58:11 +08:00
Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
@@ -0,0 +1,250 @@
|
||||
import json
|
||||
import logging
|
||||
from os import path
|
||||
|
||||
import anndata
|
||||
import numpy as np
|
||||
import tiledb
|
||||
|
||||
from backend.common.colors import convert_anndata_category_colors_to_cxg_category_colors
|
||||
from backend.czi_hosted.common.corpora import corpora_get_props_from_anndata
|
||||
from backend.common.errors import ColorFormatException
|
||||
from backend.czi_hosted.common.utils.cxg_constants import CxgConstants
|
||||
from backend.czi_hosted.common.utils.cxg_generation_utils import (
|
||||
convert_dictionary_to_cxg_group,
|
||||
convert_dataframe_to_cxg_array,
|
||||
convert_ndarray_to_cxg_dense_array,
|
||||
convert_matrix_to_cxg_array,
|
||||
)
|
||||
from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
|
||||
|
||||
|
||||
class H5ADDataFile:
|
||||
""" Class encapsulating required information about an H5AD datafile that ultimately will be transformed into
|
||||
another format (currently just CXG is supported). """
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_filename,
|
||||
backed=False,
|
||||
dataset_title=None,
|
||||
dataset_about=None,
|
||||
obs_index_column_name=None,
|
||||
vars_index_column_name=None,
|
||||
use_corpora_schema=True,
|
||||
):
|
||||
self.input_filename = input_filename
|
||||
self.backed = backed
|
||||
self.dataset_title = dataset_title
|
||||
self.dataset_about = dataset_about
|
||||
self.obs_index_column_name = obs_index_column_name
|
||||
self.vars_index_column_name = vars_index_column_name
|
||||
|
||||
self.use_corpora_schema = use_corpora_schema
|
||||
|
||||
self.validate_input_file_type()
|
||||
|
||||
self.extract_anndata_elements_from_file()
|
||||
self.extract_metadata_about_dataset()
|
||||
|
||||
self.validate_anndata()
|
||||
|
||||
def to_cxg(self, output_cxg_directory, sparse_threshold, convert_anndata_colors_to_cxg_colors=True):
|
||||
"""
|
||||
Writes the following attributes of the anndata to CXG: 1) the metadata as metadata attached to an empty
|
||||
DenseArray, 2) the obs DataFrame as a DenseArray, 3) the var DataFrame as a DenseArray, 4) all valid
|
||||
embeddings stored in obsm, each one as a DenseArray, 5) the main X matrix of the anndata as either a
|
||||
SparseArray or DenseArray based on the `sparse_threshold`, and optionally 6) the column shift of the main X
|
||||
matrix that might turn an otherwise Dense matrix into a Sparse matrix.
|
||||
"""
|
||||
|
||||
logging.info("Beginning writing to CXG.")
|
||||
ctx = tiledb.Ctx(
|
||||
{
|
||||
"sm.num_reader_threads": 32,
|
||||
"sm.num_writer_threads": 32,
|
||||
"sm.consolidation.buffer_size": 1 * 1024 * 1024 * 1024,
|
||||
}
|
||||
)
|
||||
|
||||
tiledb.group_create(output_cxg_directory, ctx=ctx)
|
||||
logging.info(f"\t...group created, with name {output_cxg_directory}")
|
||||
|
||||
convert_dictionary_to_cxg_group(
|
||||
output_cxg_directory, self.generate_cxg_metadata(convert_anndata_colors_to_cxg_colors)
|
||||
)
|
||||
logging.info("\t...dataset metadata saved")
|
||||
|
||||
convert_dataframe_to_cxg_array(output_cxg_directory, "obs", self.obs, self.obs_index_column_name, ctx)
|
||||
logging.info("\t...dataset obs dataframe saved")
|
||||
|
||||
convert_dataframe_to_cxg_array(output_cxg_directory, "var", self.var, self.var_index_column_name, ctx)
|
||||
logging.info("\t...dataset var dataframe saved")
|
||||
|
||||
self.write_anndata_embeddings_to_cxg(output_cxg_directory, ctx)
|
||||
logging.info("\t...dataset embeddings saved")
|
||||
|
||||
self.write_anndata_x_matrix_to_cxg(output_cxg_directory, ctx, sparse_threshold)
|
||||
logging.info("\t...dataset X matrix saved")
|
||||
|
||||
logging.info("Completed writing to CXG.")
|
||||
|
||||
def write_anndata_x_matrix_to_cxg(self, output_cxg_directory, ctx, sparse_threshold):
|
||||
matrix_container = f"{output_cxg_directory}/X"
|
||||
|
||||
x_matrix_data = self.anndata.X
|
||||
is_sparse = is_matrix_sparse(x_matrix_data, sparse_threshold)
|
||||
if not is_sparse:
|
||||
col_shift = get_column_shift_encode_for_matrix(x_matrix_data, sparse_threshold)
|
||||
is_sparse = col_shift is not None
|
||||
else:
|
||||
col_shift = None
|
||||
|
||||
if col_shift is not None:
|
||||
logging.info("Converting matrix X as sparse matrix with column shift encoding")
|
||||
x_col_shift_name = f"{output_cxg_directory}/X_col_shift"
|
||||
convert_ndarray_to_cxg_dense_array(x_col_shift_name, col_shift, ctx)
|
||||
|
||||
convert_matrix_to_cxg_array(matrix_container, x_matrix_data, is_sparse, ctx, col_shift)
|
||||
|
||||
tiledb.consolidate(matrix_container, ctx=ctx)
|
||||
if hasattr(tiledb, "vacuum"):
|
||||
tiledb.vacuum(matrix_container)
|
||||
|
||||
def write_anndata_embeddings_to_cxg(self, output_cxg_directory, ctx):
|
||||
def is_valid_embedding(adata, embedding_name, embedding_array):
|
||||
"""
|
||||
Returns true if this layout data is a valid array for front-end presentation with the following criteria:
|
||||
* ndarray, with shape (n_obs, >= 2), dtype float/int/uint
|
||||
* follows ScanPy embedding naming conventions
|
||||
* with all values finite or NaN (no +Inf or -Inf)
|
||||
"""
|
||||
|
||||
is_valid = isinstance(embedding_name, str) and embedding_name.startswith("X_") and len(embedding_name) > 2
|
||||
is_valid = is_valid and isinstance(embedding_array, np.ndarray) and embedding_array.dtype.kind in "fiu"
|
||||
is_valid = is_valid and embedding_array.shape[0] == adata.n_obs and embedding_array.shape[1] >= 2
|
||||
is_valid = is_valid and not np.any(np.isinf(embedding_array)) and not np.all(np.isnan(embedding_array))
|
||||
return is_valid
|
||||
|
||||
embedding_container = f"{output_cxg_directory}/emb"
|
||||
tiledb.group_create(embedding_container, ctx=ctx)
|
||||
|
||||
for embedding_name, embedding_values in self.anndata.obsm.items():
|
||||
if is_valid_embedding(self.anndata, embedding_name, embedding_values):
|
||||
embedding_name = f"{embedding_container}/{embedding_name[2:]}"
|
||||
convert_ndarray_to_cxg_dense_array(embedding_name, embedding_values, ctx)
|
||||
logging.info(f"\t\t...{embedding_name} embedding created")
|
||||
|
||||
def generate_cxg_metadata(self, convert_anndata_colors_to_cxg_colors):
|
||||
"""
|
||||
Return a dictionary containing metadata about CXG dataset. This include data about the version as well as
|
||||
Corpora schema properties if they exist, among other pieces of metadata.
|
||||
"""
|
||||
|
||||
cxg_group_metadata = {
|
||||
"cxg_version": CxgConstants.CXG_VERSION,
|
||||
"cxg_properties": json.dumps({"title": self.dataset_title, "about": self.dataset_about}),
|
||||
}
|
||||
if self.corpora_properties is not None:
|
||||
cxg_group_metadata["corpora"] = json.dumps(self.corpora_properties)
|
||||
|
||||
if convert_anndata_colors_to_cxg_colors:
|
||||
try:
|
||||
cxg_group_metadata["cxg_category_colors"] = json.dumps(
|
||||
convert_anndata_category_colors_to_cxg_category_colors(self.anndata)
|
||||
)
|
||||
except ColorFormatException:
|
||||
logging.warning(
|
||||
"Failed to extract colors from H5AD file! Fix the H5AD file or rerun with "
|
||||
"--disable-custom-colors. See help for more details."
|
||||
)
|
||||
|
||||
return cxg_group_metadata
|
||||
|
||||
def validate_input_file_type(self):
|
||||
"""
|
||||
Validate that the input file is of a type that we can handle. Currently the only valid file type is `.h5ad`.
|
||||
"""
|
||||
|
||||
if not self.input_filename.endswith(".h5ad"):
|
||||
raise Exception(f"Cannot process input file {self.input_filename}. File must be an H5AD.")
|
||||
|
||||
if self.dataset_title or self.dataset_about:
|
||||
logging.warning(
|
||||
"If you convert this dataset into CXG and you explicit specify values for the dataset title metadata "
|
||||
"or the dataset about metadata, it will override any metadata that is extracted as part of the "
|
||||
"Corpora schema fields."
|
||||
)
|
||||
|
||||
def validate_anndata(self):
|
||||
if not self.var.index.is_unique:
|
||||
raise ValueError("Variable index in AnnData object is not unique.")
|
||||
if not self.obs.index.is_unique:
|
||||
raise ValueError("Observation index in AnnData object is not unique.")
|
||||
|
||||
def extract_anndata_elements_from_file(self):
|
||||
logging.info(f"Reading in AnnData dataset: {path.basename(self.input_filename)}")
|
||||
self.anndata = anndata.read_h5ad(self.input_filename, backed="r" if self.backed else None)
|
||||
logging.info("Completed reading in AnnData dataset!")
|
||||
|
||||
self.obs = self.transform_dataframe_index_into_column(self.anndata.obs, "obs", self.obs_index_column_name)
|
||||
self.var = self.transform_dataframe_index_into_column(self.anndata.var, "var", self.vars_index_column_name)
|
||||
|
||||
def extract_metadata_about_dataset(self):
|
||||
"""
|
||||
Extract metadata information about the dataset that upon conversion will be saved as group metadata with the
|
||||
CXG that is generated. This metadata information includes Corpora schema properties, the dataset title and
|
||||
a link that details more information about the dataset.
|
||||
"""
|
||||
|
||||
self.corpora_properties = corpora_get_props_from_anndata(self.anndata) if self.use_corpora_schema else None
|
||||
if self.corpora_properties is None and self.use_corpora_schema:
|
||||
# If the return value is None, this means that we were not able to figure out what version of the Corpora
|
||||
# schema the object is using and therefore cannot extract any properties.
|
||||
raise ValueError("Unknown source file schema version is unsupported.")
|
||||
|
||||
# The title and about properties of the dataset are set by the following order: if they are explicitly defined
|
||||
# then use the explicit value. If the dataset is a Corpora-schema based schema, then extract the title and about
|
||||
# from the corpora_properties. Otherwise, use the input filename (only for title, about will be blank).
|
||||
if self.corpora_properties:
|
||||
corpora_project_links = self.corpora_properties.get("project_links", [])
|
||||
corpora_about_link = next(
|
||||
(link for link in corpora_project_links if (link.get("link_type", None) == "SUMMARY")), {}
|
||||
)
|
||||
else:
|
||||
corpora_about_link = {}
|
||||
|
||||
filename = path.splitext(path.basename(self.input_filename))[0]
|
||||
|
||||
self.dataset_title = self.dataset_title if self.dataset_title else corpora_about_link.get("link_name", filename)
|
||||
self.dataset_about = self.dataset_about if self.dataset_about else corpora_about_link.get("link_url")
|
||||
|
||||
def transform_dataframe_index_into_column(self, dataframe, dataframe_name, index_column_name):
|
||||
"""
|
||||
Convert the dataframe's index into another column in the dataframe. If an index_column_name is specified,
|
||||
use that column as the index instead.
|
||||
"""
|
||||
|
||||
if index_column_name is None:
|
||||
# Create a unique column name for the index.
|
||||
suffix = 0
|
||||
while f"name_{suffix}" in dataframe.columns:
|
||||
suffix += 1
|
||||
index_column_name = f"name_{suffix}"
|
||||
|
||||
# Turn the index into a normal column
|
||||
dataframe.rename_axis(index_column_name, inplace=True)
|
||||
dataframe.reset_index(inplace=True)
|
||||
|
||||
elif index_column_name in dataframe.columns:
|
||||
# User has specified alternative column for unique names, and it exists
|
||||
if not dataframe[index_column_name].is_unique:
|
||||
raise KeyError(
|
||||
f"Values in {dataframe_name}.{index_column_name} must be unique. Please prepare data to contain "
|
||||
f"unique values."
|
||||
)
|
||||
else:
|
||||
raise KeyError(f"Column {index_column_name} does not exist.")
|
||||
|
||||
setattr(self, f"{dataframe_name}_index_column_name", index_column_name)
|
||||
return dataframe
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Helpers for converting and checking HGNC gene symbols."""
|
||||
|
||||
import argparse
|
||||
import enum
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def get_upgraded_var_index(var, hgnc_path=None):
|
||||
"""Given an anndata var dataframe, return a new index for the dataframe
|
||||
where human gene symbols have been upgraded to the current HGNC set.
|
||||
"""
|
||||
|
||||
if not hgnc_path:
|
||||
hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
|
||||
|
||||
hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
|
||||
|
||||
return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
|
||||
|
||||
|
||||
class SymbolStatus(enum.Enum):
|
||||
"""The status of a symbol in the HGNC database.
|
||||
|
||||
APPROVED: Currently a valid symbol
|
||||
WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
|
||||
not to exist _unless_ that symbol is also approved
|
||||
AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
|
||||
multiple approved symbols
|
||||
UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
|
||||
symbol
|
||||
UNKNOWN: A symbol that does not appear in HGNC
|
||||
"""
|
||||
|
||||
APPROVED = 1
|
||||
WITHDRAWN = 2
|
||||
AMBIGUOUS = 3
|
||||
UPGRADABLE = 4
|
||||
UNKNOWN = 5
|
||||
|
||||
|
||||
class HGNCSymbolChecker:
|
||||
"""Handle checking and correcting HGNC symbols."""
|
||||
|
||||
def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
|
||||
self.approved_symbols = approved_symbols
|
||||
self.withdrawn_symbols = withdrawn_symbols
|
||||
self.ambiguous_symbols = ambiguous_symbols
|
||||
self.symbol_map = symbol_map
|
||||
|
||||
def print_symbol_map(self):
|
||||
"""Print out a map from old symbol to new symbol."""
|
||||
|
||||
for symbol_pair in self.symbol_map.items():
|
||||
print("\t".join(symbol_pair))
|
||||
|
||||
def check_symbol(self, symbol):
|
||||
"""See if a symbol if approved or something else."""
|
||||
if symbol in self.approved_symbols:
|
||||
return SymbolStatus.APPROVED
|
||||
|
||||
if symbol in self.withdrawn_symbols:
|
||||
return SymbolStatus.WITHDRAWN
|
||||
|
||||
if symbol in self.ambiguous_symbols:
|
||||
return SymbolStatus.AMBIGUOUS
|
||||
|
||||
if symbol in self.symbol_map:
|
||||
return SymbolStatus.UPGRADABLE
|
||||
|
||||
return SymbolStatus.UNKNOWN
|
||||
|
||||
def upgrade_symbol(self, symbol):
|
||||
"""Return the approved symbol for the given symbol.
|
||||
|
||||
If the symbol cannot be upgraded, just return the original symbol.
|
||||
"""
|
||||
|
||||
fixed_symbol, stripped_symbol = format_symbol(symbol)
|
||||
|
||||
if fixed_symbol in self.approved_symbols:
|
||||
return fixed_symbol
|
||||
elif fixed_symbol in self.symbol_map:
|
||||
return self.symbol_map[fixed_symbol]
|
||||
elif stripped_symbol in self.approved_symbols:
|
||||
return stripped_symbol
|
||||
elif stripped_symbol in self.symbol_map:
|
||||
return self.symbol_map[stripped_symbol]
|
||||
|
||||
return symbol
|
||||
|
||||
@classmethod
|
||||
def from_hgnc_records(cls, hgnc_dataset_path):
|
||||
"""Parse a hgnc database download into a HGNCSymbolChecker object."""
|
||||
|
||||
def all_symbols(record):
|
||||
"""Get all the symbols associated with an HGNC record including previous, alias,
|
||||
and approved."""
|
||||
yield format_symbol(record["symbol"])[0]
|
||||
for symbol in alias_and_previous_symbols(record):
|
||||
yield symbol
|
||||
|
||||
def alias_and_previous_symbols(record):
|
||||
"""Get alias and previous symbols from an HGNC record."""
|
||||
for field in ("alias_symbol", "prev_symbol"):
|
||||
if record[field] is not np.nan:
|
||||
for symbol in record[field].split("|"):
|
||||
yield format_symbol(symbol)[0]
|
||||
# Sometimes something like HGNC:1234 appears in datasets, which we
|
||||
# want to fix as well.
|
||||
yield record["hgnc_id"]
|
||||
|
||||
hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
|
||||
|
||||
# Get all symbols that are currently approved.
|
||||
approved_symbols = set()
|
||||
for record in hgnc_records:
|
||||
if record["status"] == "Approved":
|
||||
approved_symbols.add(format_symbol(record["symbol"])[0])
|
||||
|
||||
# Get all symbols that have been withdrawn
|
||||
withdrawn_symbols = set()
|
||||
for record in hgnc_records:
|
||||
if record["status"] == "Entry Withdrawn":
|
||||
for symbol in all_symbols(record):
|
||||
withdrawn_symbols.add(symbol)
|
||||
|
||||
# If a symbol is both approved and withdrawn, be optimistic and call it approved
|
||||
logging.warning(
|
||||
f"Some symbols are simulaneously withdrawn and approved\n"
|
||||
f"We will treat them at approved:\n"
|
||||
f"{withdrawn_symbols.intersection(approved_symbols)}"
|
||||
)
|
||||
withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
|
||||
|
||||
# Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
|
||||
alias_previous_to_approved = {}
|
||||
ambiguous_symbols = set()
|
||||
|
||||
for record in hgnc_records:
|
||||
if record["status"] == "Approved":
|
||||
|
||||
# The approved symbol is what we'll map to
|
||||
approved_symbol = format_symbol(record["symbol"])[0]
|
||||
|
||||
for symbol in alias_and_previous_symbols(record):
|
||||
|
||||
# If the alias or previous symbol is also an approved symbol,
|
||||
# we'll just leave it alone
|
||||
if symbol in approved_symbols:
|
||||
continue
|
||||
|
||||
# If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
|
||||
if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
|
||||
ambiguous_symbols.add(symbol)
|
||||
else:
|
||||
alias_previous_to_approved[symbol] = approved_symbol
|
||||
|
||||
# Remove all the ambiguous symbols from the map
|
||||
for ambiguous_symbol in ambiguous_symbols:
|
||||
alias_previous_to_approved.pop(ambiguous_symbol)
|
||||
|
||||
return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
|
||||
|
||||
|
||||
def format_symbol(symbol):
|
||||
"""HGNC rules say symbols should all be upper case except for C#orf#. However, case is
|
||||
variable in both alias and previous symbols as well as in the symbols we get in
|
||||
submissions. So, upper case everything except for the one situation where mixed-case
|
||||
is allowed, which are the genes like C2orf157.
|
||||
|
||||
Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
|
||||
names persist throughout the life of the object. They won't match against the HGNC database
|
||||
and we want to merge them, so we need to strip off the suffix and try matching again.
|
||||
|
||||
This function takes a symbol and returns the symbol with the fixed case and also with the
|
||||
seurat/scanpy suffix stripped off.
|
||||
"""
|
||||
|
||||
match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
|
||||
|
||||
if match:
|
||||
fixed_case = f"C{match.group(2)}orf{match.group(4)}"
|
||||
else:
|
||||
fixed_case = symbol.upper()
|
||||
|
||||
suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
|
||||
|
||||
return fixed_case, suffix_stripped
|
||||
|
||||
|
||||
def main():
|
||||
"""When called as main, parse a given hgnc download and print out a map from old to new
|
||||
symbol.
|
||||
"""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
|
||||
|
||||
hgnc_symbol_checker.print_symbol_map()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
@@ -0,0 +1,86 @@
|
||||
"""Methods for working with ontologies and the OLS."""
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import requests
|
||||
|
||||
OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api"
|
||||
|
||||
# Curie means something like CL:0000001
|
||||
|
||||
|
||||
def _ontology_name(curie):
|
||||
"""Get the name of the ontology from the curie, CL or UBERON for example."""
|
||||
return curie.split(":")[0]
|
||||
|
||||
|
||||
def _ontology_value(curie):
|
||||
"""Get the id component of the curie, 0000001 from CL:0000001 for example."""
|
||||
return curie.split(":")[1]
|
||||
|
||||
|
||||
def _double_encode(url):
|
||||
"""Double url encode a url. This is required by the OLS API."""
|
||||
return quote_plus(quote_plus(url))
|
||||
|
||||
|
||||
def _iri(curie):
|
||||
"""Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org"""
|
||||
if _ontology_name(curie) == "EFO":
|
||||
return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}"
|
||||
return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}"
|
||||
|
||||
|
||||
class OntologyLookupError(Exception):
|
||||
"""Exception for some problem with looking up ontology information."""
|
||||
|
||||
|
||||
def _ontology_info_url(curie):
|
||||
"""Get the to make a GET to to get information about an ontology term."""
|
||||
|
||||
# If the curie is empty, just return an empty string. This happens when there is no
|
||||
# valid ontology value.
|
||||
if not curie:
|
||||
return ""
|
||||
else:
|
||||
return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}"
|
||||
|
||||
|
||||
def get_ontology_label(curie):
|
||||
"""For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'"""
|
||||
|
||||
url = _ontology_info_url(curie)
|
||||
|
||||
if not url:
|
||||
return ""
|
||||
|
||||
response = requests.get(url)
|
||||
|
||||
if not response.ok:
|
||||
raise OntologyLookupError(
|
||||
f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}"
|
||||
)
|
||||
return response.json()["label"]
|
||||
|
||||
|
||||
def lookup_candidate_term(label, ontology="cl", method="select"):
|
||||
"""Lookup candidate terms for a label. This is useful when there is an existing label in a
|
||||
submitted dataset, and you want to find an appropriate ontology term.
|
||||
|
||||
Args:
|
||||
label: the label to find ontology terms for
|
||||
ontology: the ontology to search in, cl or uberon or efo for example
|
||||
method: select or search. search provides much broader results
|
||||
|
||||
Returns:
|
||||
list of (curie, label) tuples returned by OLS
|
||||
"""
|
||||
# using OLS REST API [https://www.ebi.ac.uk/ols/docs/api]
|
||||
url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}"
|
||||
response = requests.get(url)
|
||||
|
||||
if not response.ok:
|
||||
raise OntologyLookupError(
|
||||
f"Label {label} lookup failed, got status code {response.status_code}: {response.text}"
|
||||
)
|
||||
|
||||
return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]]
|
||||
@@ -0,0 +1,264 @@
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import string
|
||||
|
||||
import anndata
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import yaml
|
||||
|
||||
from . import gene_symbol
|
||||
from . import ontology
|
||||
from . import validate
|
||||
|
||||
REPLACE_SUFFIX = "_original"
|
||||
ONTOLOGY_SUFFIX = "_ontology_term_id"
|
||||
|
||||
|
||||
def is_curie(value):
|
||||
"""Return True iff the value is an OBO-id CURIE like EFO:000001"""
|
||||
return (value.count(":")
|
||||
and all(len(part) > 0 for part in value.split(":"))
|
||||
and all(c in string.digits for c in value.split(":")[1]))
|
||||
|
||||
|
||||
def is_ontology_field(field_name):
|
||||
"""Return True iff the field_name is an ontology field like tissue_ontology_term_id"""
|
||||
return field_name.endswith(ONTOLOGY_SUFFIX)
|
||||
|
||||
|
||||
def get_label_field_name(field_name):
|
||||
"""Get the associated label field from an ontology field, assay_ontology_term_id --> assay"""
|
||||
return field_name[: -len(ONTOLOGY_SUFFIX)]
|
||||
|
||||
|
||||
def split_suffix(maybe_curie):
|
||||
"""Split off the (cell culture) or (organoid) suffix."""
|
||||
|
||||
suffixes = [" (cell culture)", " (organoid)"]
|
||||
for suffix in suffixes:
|
||||
if maybe_curie.endswith(suffix):
|
||||
return maybe_curie[:-len(suffix)], suffix
|
||||
return maybe_curie, ""
|
||||
|
||||
|
||||
def get_curie_and_label(maybe_curie):
|
||||
"""Given a string that might be a curie, return a (curie, label) pair"""
|
||||
|
||||
maybe_curie, suffix = split_suffix(maybe_curie)
|
||||
if not is_curie(maybe_curie):
|
||||
return ("", maybe_curie + suffix)
|
||||
return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix)
|
||||
|
||||
|
||||
def safe_add_field(adata_attr, field_name, field_value):
|
||||
"""Add a field and value to an AnnData, but don't clobber an exising value."""
|
||||
|
||||
if (
|
||||
isinstance(field_value, list)
|
||||
and field_value
|
||||
and isinstance(field_value[0], dict)
|
||||
):
|
||||
field_value = json.dumps(field_value)
|
||||
if field_name in adata_attr:
|
||||
adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name]
|
||||
adata_attr[field_name] = field_value
|
||||
|
||||
|
||||
def remix_uns(adata, uns_config):
|
||||
"""Add fields from the config to adata.uns"""
|
||||
for field_name, field_value in uns_config.items():
|
||||
|
||||
if is_ontology_field(field_name):
|
||||
# If it's an ontology field, look it up
|
||||
label_field_name = get_label_field_name(field_name)
|
||||
ontology_term, ontology_label = get_curie_and_label(field_value)
|
||||
safe_add_field(adata.uns, field_name, ontology_term)
|
||||
safe_add_field(adata.uns, label_field_name, ontology_label)
|
||||
else:
|
||||
safe_add_field(adata.uns, field_name, field_value)
|
||||
|
||||
|
||||
def remix_obs(adata, obs_config):
|
||||
"""Add fields from the config to adata.obs"""
|
||||
|
||||
for field_name, field_value in obs_config.items():
|
||||
|
||||
if isinstance(field_value, dict):
|
||||
# If the value is a dict, that means we are supposed to map from an
|
||||
# existing column to the new one
|
||||
source_column, column_map = next(iter(field_value.items()))
|
||||
nan_value = None
|
||||
for key in column_map:
|
||||
if isinstance(key, float) and math.isnan(key):
|
||||
nan_value = column_map[key]
|
||||
if nan_value is not None:
|
||||
column_map["nan"] = nan_value
|
||||
|
||||
for key in column_map:
|
||||
if key not in adata.obs[source_column].unique():
|
||||
logging.warning(f'Key {key} not in adata.obs["{source_column}"]')
|
||||
|
||||
for value in adata.obs[source_column].unique():
|
||||
if value not in column_map:
|
||||
logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict')
|
||||
|
||||
if is_ontology_field(field_name):
|
||||
ontology_term_map, ontology_label_map = {}, {}
|
||||
logging.info(f"Looking up labels for {field_name}")
|
||||
for original_value, maybe_curie in column_map.items():
|
||||
curie, label = get_curie_and_label(maybe_curie)
|
||||
ontology_term_map[original_value] = curie
|
||||
ontology_label_map[original_value] = label
|
||||
logging.info(f"Mapping {original_value} -> {curie} -> {label}")
|
||||
|
||||
ontology_column = adata.obs[source_column].replace(
|
||||
ontology_term_map, inplace=False
|
||||
)
|
||||
label_column = adata.obs[source_column].replace(
|
||||
ontology_label_map, inplace=False
|
||||
)
|
||||
|
||||
safe_add_field(adata.obs, field_name, ontology_column)
|
||||
safe_add_field(
|
||||
adata.obs, get_label_field_name(field_name), label_column
|
||||
)
|
||||
else:
|
||||
label_column = adata.obs[source_column].replace(
|
||||
column_map, inplace=False
|
||||
)
|
||||
safe_add_field(adata.obs, field_name, label_column)
|
||||
|
||||
else:
|
||||
if is_ontology_field(field_name):
|
||||
# If it's an ontology field, look it up
|
||||
label_field_name = get_label_field_name(field_name)
|
||||
ontology_term, ontology_label = get_curie_and_label(field_value)
|
||||
safe_add_field(adata.obs, field_name, ontology_term)
|
||||
safe_add_field(adata.obs, label_field_name, ontology_label)
|
||||
else:
|
||||
safe_add_field(adata.obs, field_name, field_value)
|
||||
|
||||
|
||||
def merge_df(df, domain, index, columns):
|
||||
"""
|
||||
Given a dataframe with duplicate column labels, merge and return a dataframe where
|
||||
the duplicates have been merged together, resulting in a dataframe with unique column
|
||||
labels.
|
||||
|
||||
"merge" depends on the value of domain. If the domain is "raw", then duplicate columns
|
||||
can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then
|
||||
summed, and then logged or sqrt'd again.
|
||||
"""
|
||||
|
||||
if not isinstance(df, np.ndarray):
|
||||
to_merge = df.toarray()
|
||||
else:
|
||||
to_merge = df
|
||||
if domain == "raw":
|
||||
merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum(
|
||||
axis=1, level=0, skipna=False
|
||||
)
|
||||
elif domain == "log1p":
|
||||
merged_df = (
|
||||
pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns)
|
||||
.sum(axis=1, level=0, skipna=False)
|
||||
)
|
||||
merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
|
||||
elif domain == "sqrt":
|
||||
merged_df = (
|
||||
pd.DataFrame(np.square(to_merge), index=index, columns=columns)
|
||||
.sum(axis=1, level=0, skipna=False)
|
||||
)
|
||||
merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
|
||||
|
||||
return merged_df
|
||||
|
||||
|
||||
def fixup_gene_symbols(adata, fixup_config):
|
||||
"""Update the var index to hold a consistent set of HGNC gene symbols."""
|
||||
|
||||
upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var)
|
||||
|
||||
merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index)
|
||||
fixup_adata = anndata.AnnData(
|
||||
X=merged_X,
|
||||
obs=adata.obs,
|
||||
var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
|
||||
uns=adata.uns,
|
||||
obsm=adata.obsm,
|
||||
)
|
||||
|
||||
for layer, domain in fixup_config.items():
|
||||
if layer == "X":
|
||||
continue
|
||||
if layer == "raw.X":
|
||||
df = adata.raw.X
|
||||
else:
|
||||
df = adata.layers[layer]
|
||||
|
||||
merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index)
|
||||
assert merged_df.index.equals(merged_X.index)
|
||||
assert merged_df.columns.equals(merged_X.columns)
|
||||
|
||||
if domain == "raw":
|
||||
fixup_raw = anndata.AnnData(
|
||||
X=merged_df,
|
||||
obs=adata.obs,
|
||||
var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
|
||||
)
|
||||
fixup_adata.raw = fixup_raw
|
||||
else:
|
||||
fixup_adata.layers[layer] = merged_df
|
||||
|
||||
return fixup_adata
|
||||
|
||||
def _strip_version(adata):
|
||||
"""Remove version information from the AnnData object."""
|
||||
|
||||
if "version" in adata.uns_keys():
|
||||
del adata.uns["version"]
|
||||
|
||||
def apply_schema(source_h5ad, remix_config, output_filename):
|
||||
|
||||
try:
|
||||
import scanpy
|
||||
except ImportError:
|
||||
raise ImportError("scanpy must be installed for cellxgene schema")
|
||||
adata = scanpy.read_h5ad(source_h5ad)
|
||||
config = yaml.load(open(remix_config), Loader=yaml.FullLoader)
|
||||
remix_uns(adata, config["uns"])
|
||||
remix_obs(adata, config["obs"])
|
||||
|
||||
if config.get("fixup_gene_symbols"):
|
||||
adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"])
|
||||
|
||||
if ("version" in adata.uns_keys()
|
||||
and isinstance(adata.uns["version"], collections.Mapping)
|
||||
and "corpora_schema_version" in adata.uns["version"]):
|
||||
schema_version = adata.uns["version"]["corpora_schema_version"]
|
||||
try:
|
||||
validate.get_schema_definition(schema_version)
|
||||
except ValueError:
|
||||
logging.warning(f"Stripping version information out of AnnData because schema "
|
||||
f"version {schema_version} is unknown.")
|
||||
_strip_version(adata)
|
||||
|
||||
if not validate.validate_adata(adata, shallow=False):
|
||||
logging.warning(f"Stripping version information out of AnnData because it does not "
|
||||
f"follow schema version {schema_version} .")
|
||||
_strip_version(adata)
|
||||
|
||||
adata.write_h5ad(output_filename, compression="gzip")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-h5ad", required=True)
|
||||
parser.add_argument("--remix-config", required=True)
|
||||
parser.add_argument("--output-filename", required=True)
|
||||
args = parser.parse_args()
|
||||
apply_schema(args.source_h5ad, args.remix_config, args.output_filename)
|
||||
@@ -0,0 +1,95 @@
|
||||
title: Corpora schema version 1.0.0
|
||||
type: anndata
|
||||
components:
|
||||
uns:
|
||||
type: dict
|
||||
keys:
|
||||
version:
|
||||
type: dict
|
||||
keys:
|
||||
corpora_schema_version: null
|
||||
corpora_encoding_version: null
|
||||
title:
|
||||
type: string
|
||||
contributors:
|
||||
type: stringified list of dicts
|
||||
layer_descriptions:
|
||||
type: dict
|
||||
keys:
|
||||
X: null
|
||||
organism:
|
||||
type: string
|
||||
nullable: false
|
||||
organism_ontology_term_id:
|
||||
type: curie
|
||||
prefixes:
|
||||
- NCBITaxon
|
||||
var:
|
||||
type: dataframe
|
||||
index:
|
||||
type: human-readable string
|
||||
unique: true
|
||||
obs:
|
||||
type: dataframe
|
||||
index:
|
||||
unique: true
|
||||
columns:
|
||||
tissue:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
tissue_ontology_term_id:
|
||||
type: suffixed curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- UBERON
|
||||
assay:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
assay_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- EFO
|
||||
disease:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
disease_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- MONDO
|
||||
- PATO
|
||||
cell_type:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
cell_type_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- CL
|
||||
- UBERON
|
||||
sex:
|
||||
type: string
|
||||
enum:
|
||||
- male
|
||||
- female
|
||||
- mixed
|
||||
- unknown
|
||||
- other
|
||||
ethnicity:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
ethnicity_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- HANCESTRO
|
||||
development_stage:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
development_stage_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- HsapDv
|
||||
- EFO
|
||||
@@ -0,0 +1,93 @@
|
||||
title: Corpora schema version 1.1.0
|
||||
type: anndata
|
||||
components:
|
||||
uns:
|
||||
type: dict
|
||||
keys:
|
||||
version:
|
||||
type: dict
|
||||
keys:
|
||||
corpora_schema_version: null
|
||||
corpora_encoding_version: null
|
||||
title:
|
||||
type: string
|
||||
layer_descriptions:
|
||||
type: dict
|
||||
keys:
|
||||
X: null
|
||||
organism:
|
||||
type: string
|
||||
nullable: false
|
||||
organism_ontology_term_id:
|
||||
type: curie
|
||||
prefixes:
|
||||
- NCBITaxon
|
||||
var:
|
||||
type: dataframe
|
||||
index:
|
||||
type: human-readable string
|
||||
unique: true
|
||||
obs:
|
||||
type: dataframe
|
||||
index:
|
||||
unique: true
|
||||
columns:
|
||||
tissue:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
tissue_ontology_term_id:
|
||||
type: suffixed curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- UBERON
|
||||
assay:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
assay_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- EFO
|
||||
disease:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
disease_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- MONDO
|
||||
- PATO
|
||||
cell_type:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
cell_type_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- CL
|
||||
- UBERON
|
||||
sex:
|
||||
type: string
|
||||
enum:
|
||||
- male
|
||||
- female
|
||||
- mixed
|
||||
- unknown
|
||||
- other
|
||||
ethnicity:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
ethnicity_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- HANCESTRO
|
||||
development_stage:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
development_stage_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- HsapDv
|
||||
- EFO
|
||||
@@ -0,0 +1,236 @@
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
import yaml
|
||||
|
||||
|
||||
def _is_null(v):
|
||||
"""Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad."""
|
||||
return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0)
|
||||
|
||||
|
||||
def _validate_stringified_list_of_dicts(s):
|
||||
"""Verify that a string can be parsed into a list.
|
||||
|
||||
We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to
|
||||
json.dumps them. This verifies that we can load them back.
|
||||
"""
|
||||
|
||||
try:
|
||||
list_ = json.loads(s)
|
||||
if not isinstance(list_, list):
|
||||
return False
|
||||
for el in list_:
|
||||
if not isinstance(el, dict):
|
||||
return False
|
||||
return True
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _validate_human_readable_string(s):
|
||||
"""Verify that a string is human-readable.
|
||||
|
||||
There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind
|
||||
of vague and subjective. I feel like I can read many strings. So here we just check for the main ways
|
||||
that fails: someone puts in an ontology term id or and ensembl gene/transcript id.
|
||||
|
||||
Returns False if s is not a string or is one of those bad string types.
|
||||
"""
|
||||
|
||||
return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s))
|
||||
|
||||
|
||||
def _validate_curie(c, prefixes):
|
||||
"""Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the
|
||||
prefix of the curies is in prefixes.
|
||||
"""
|
||||
|
||||
if not c:
|
||||
return True
|
||||
|
||||
match = re.match(r"([A-Z]\w+):\d+$", c)
|
||||
|
||||
if prefixes:
|
||||
return match and match.group(1) in prefixes
|
||||
else:
|
||||
return match
|
||||
|
||||
|
||||
def _validate_suffixed_curie(c, prefixes):
|
||||
"""Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'"""
|
||||
|
||||
# Pull off the suffix
|
||||
suffix = re.findall(r"\ \(.*\)$", c)
|
||||
if suffix:
|
||||
c = c[: -len(suffix[0])]
|
||||
return _validate_curie(c, prefixes)
|
||||
|
||||
|
||||
def _validate_column(column, column_name, df_name, schema_def):
|
||||
"""Given a schema definition and the column of a dataframe, verify that the column satifies
|
||||
the schema.
|
||||
"""
|
||||
|
||||
errors = []
|
||||
|
||||
if schema_def.get("unique"):
|
||||
if column.nunique() != len(column):
|
||||
errors.append(f"Column {column_name} in dataframe {df_name} is not unique.")
|
||||
|
||||
if "nullable" in schema_def and not schema_def["nullable"]:
|
||||
if any(_is_null(v) for v in column):
|
||||
errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.")
|
||||
|
||||
if schema_def.get("type") == "human-readable string":
|
||||
non_readables = [v for v in column if not _validate_human_readable_string(v)]
|
||||
if non_readables:
|
||||
errors.append(
|
||||
f"Column {column_name} in dataframe {df_name} contains non-human-readable "
|
||||
f"values like {non_readables[0]}"
|
||||
)
|
||||
|
||||
if schema_def.get("type") in ("curie", "suffixed curie"):
|
||||
validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie
|
||||
non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))]
|
||||
if non_valid_curies:
|
||||
errors.append(
|
||||
f"Column {column_name} in dataframe {df_name} contains invalid ontology values like "
|
||||
f"{non_valid_curies[0]}."
|
||||
)
|
||||
if "prefixes" in schema_def:
|
||||
errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}."
|
||||
|
||||
if "enum" in schema_def:
|
||||
bad_enums = [v for v in column if v not in schema_def["enum"]]
|
||||
if bad_enums:
|
||||
errors.append(
|
||||
f"Column {column_name} in dataframe {df_name} contains unpermitted values like "
|
||||
f"{bad_enums[0]}. Values must be one of {schema_def['enum']}."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_dict(dict_, dict_name, schema_def):
|
||||
"""Given a schema definition and dict, verify that the dict satifies the schema."""
|
||||
|
||||
errors = []
|
||||
|
||||
for key in schema_def.get("keys", []):
|
||||
if key not in dict_:
|
||||
errors.append(f"{dict_name} is missing key {key}.")
|
||||
elif schema_def["keys"][key]:
|
||||
if schema_def["keys"][key]["type"] == "stringified list of dicts":
|
||||
if not _validate_stringified_list_of_dicts(dict_[key]):
|
||||
errors.append(
|
||||
f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}"
|
||||
)
|
||||
elif schema_def["keys"][key]["type"] == "dict":
|
||||
errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key]))
|
||||
elif schema_def["keys"][key]["type"] == "curie":
|
||||
if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]):
|
||||
errors.append(f"Key {key} in {dict_name} contains invalid ontology value.")
|
||||
if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]:
|
||||
if _is_null(dict_[key]):
|
||||
errors.append(f"Key {key} in dict {dict_name} is an empty value.")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_dataframe(df, df_name, schema_def):
|
||||
"""Given a dataframe and schema definition, verify that the dataframe follows the schema."""
|
||||
|
||||
errors = []
|
||||
|
||||
if "index" in schema_def:
|
||||
errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"]))
|
||||
|
||||
for column in schema_def.get("columns", []):
|
||||
if column not in df.columns:
|
||||
errors.append(f"Dataframe {df_name} is missing column {column}.")
|
||||
else:
|
||||
errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column]))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def get_schema_definition(version):
|
||||
"""Look up and read a schema definition based on a version number like "1.0.0"."""
|
||||
|
||||
path = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml"
|
||||
)
|
||||
|
||||
if not os.path.isfile(path):
|
||||
raise ValueError(f"No definition for version {version} found.")
|
||||
|
||||
return yaml.load(open(path), Loader=yaml.FullLoader)
|
||||
|
||||
|
||||
def deep_check(adata, schema_def):
|
||||
"""Perform a "deep" check of the AnnData object using the schema definition.
|
||||
|
||||
This checks all the columns and unstructured metadata rather than just the version.
|
||||
|
||||
Returns a list of error messages. If that list is empty, the object passed validation.
|
||||
"""
|
||||
|
||||
errors = []
|
||||
|
||||
for component, component_def in schema_def["components"].items():
|
||||
if component_def["type"] == "dataframe":
|
||||
errors.extend(_validate_dataframe(getattr(adata, component), component, component_def))
|
||||
elif component_def["type"] == "dict":
|
||||
errors.extend(_validate_dict(getattr(adata, component), component, component_def))
|
||||
else:
|
||||
raise ValueError(f"Unexpected component type {component['type']}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate_adata(adata, shallow):
|
||||
"""Validate an AnnData object. If shallow, just check that the required version information is
|
||||
present.
|
||||
"""
|
||||
|
||||
# Does it have the version information written into uns?
|
||||
if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]:
|
||||
print("AnnData file is missing corpora version information")
|
||||
return False
|
||||
|
||||
# We can stop here if it's a "shallow" check, that is, if we're just
|
||||
# checking that version is present.
|
||||
if shallow:
|
||||
return True
|
||||
|
||||
schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"])
|
||||
|
||||
errors = deep_check(adata, schema_def)
|
||||
|
||||
for error in errors:
|
||||
print(error)
|
||||
|
||||
return not errors
|
||||
|
||||
|
||||
def validate(h5ad_path, shallow=False):
|
||||
"""Entry point for validation."""
|
||||
|
||||
try:
|
||||
import scanpy
|
||||
except ImportError:
|
||||
raise ImportError("scanpy must be installed for cellxgene schema")
|
||||
|
||||
try:
|
||||
adata = scanpy.read_h5ad(h5ad_path, backed="r")
|
||||
except (OSError, TypeError):
|
||||
print(f"Unable to open {h5ad_path} with scanpy.")
|
||||
sys.exit(1)
|
||||
|
||||
if not validate_adata(adata, shallow):
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,81 @@
|
||||
"""
|
||||
Script to create a sparse dataset in CXG format based on an input dataset in CXG format.
|
||||
The input dataset is not modified.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
import tiledb
|
||||
|
||||
from backend.czi_hosted.common.utils.cxg_generation_utils import convert_ndarray_to_cxg_dense_array, \
|
||||
convert_matrix_to_cxg_array
|
||||
from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("input", help="input cxg directory")
|
||||
parser.add_argument("output", help="output cxg directory")
|
||||
parser.add_argument("--overwrite", action="store_true", help="replace output cxg directory")
|
||||
parser.add_argument("--verbose", "-v", action="count", default=0, help="verbose output")
|
||||
parser.add_argument(
|
||||
"--sparse-threshold",
|
||||
"-s",
|
||||
type=float,
|
||||
default=5.0, # default is 5% non-zero values
|
||||
help="The X array will be sparse if the percent of non-zeros falls below this value",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.path.exists(args.output):
|
||||
print("output dir exists:", args.output)
|
||||
if args.overwrite:
|
||||
print("output dir removed:", args.output)
|
||||
shutil.rmtree(args.output)
|
||||
else:
|
||||
print("use the overwrite option to remove the output directory")
|
||||
sys.exit(1)
|
||||
|
||||
if not os.path.isdir(args.input):
|
||||
print("input is not a directory", args.input)
|
||||
sys.exit(1)
|
||||
|
||||
shutil.copytree(args.input, args.output, ignore=shutil.ignore_patterns("X", "X_col_shift"))
|
||||
|
||||
ctx = tiledb.Ctx(
|
||||
{
|
||||
"sm.num_reader_threads": 32,
|
||||
"sm.num_writer_threads": 32,
|
||||
"sm.consolidation.buffer_size": 1 * 1024 * 1024 * 1024,
|
||||
}
|
||||
)
|
||||
|
||||
with tiledb.DenseArray(os.path.join(args.input, "X"), mode="r", ctx=ctx) as X_in:
|
||||
x_matrix_data = X_in[:, :]
|
||||
matrix_container = args.output
|
||||
|
||||
is_sparse = is_matrix_sparse(x_matrix_data, args.sparse_threshold)
|
||||
if not is_sparse:
|
||||
col_shift = get_column_shift_encode_for_matrix(x_matrix_data, args.sparse_threshold)
|
||||
is_sparse = col_shift is not None
|
||||
else:
|
||||
col_shift = None
|
||||
|
||||
if col_shift is not None:
|
||||
x_col_shift_name = f"{args.output}/X_col_shift"
|
||||
convert_ndarray_to_cxg_dense_array(x_col_shift_name, col_shift, ctx)
|
||||
tiledb.consolidate(matrix_container, ctx=ctx)
|
||||
if is_sparse:
|
||||
convert_matrix_to_cxg_array(matrix_container, x_matrix_data, is_sparse, ctx, col_shift)
|
||||
tiledb.consolidate(matrix_container, ctx=ctx)
|
||||
|
||||
if not is_sparse:
|
||||
print("The array is not sparse, cleaning up, abort.")
|
||||
shutil.rmtree(args.output)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user