mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-15 12:47:56 +08:00
Clean up dead/hosted code [zh2310] (#2430)
* Clean up dead/hosted code * Remove schema conversion tool and related * Remove cxg references * Remove locust * missed a spot * Remove aws secret manager * Merge branch 'main' into brodgers/2310/code-cleanup-v1 * cleanup merge
This commit is contained in:
Ben MR
2
.gitignore
vendored
2
.gitignore
vendored
@@ -15,7 +15,7 @@ dist/
|
||||
*.egg-info
|
||||
|
||||
# Environments
|
||||
venv/
|
||||
venv*/
|
||||
cellxgene/
|
||||
|
||||
# client build
|
||||
|
||||
@@ -50,6 +50,5 @@ define_request_exception(
|
||||
|
||||
define_exception("ConfigurationError", "Raised when checking configuration errors")
|
||||
define_exception("PrepareError", "Raised when data is misprepared")
|
||||
define_exception("SecretKeyRetrievalError", "Raised when get_secret_key from AWS fails")
|
||||
define_exception("ObsoleteRequest", "Raised when the request is no longer valid.")
|
||||
define_exception("UnsupportedSummaryMethod", "Raised when a gene set summary method is unknown or unsupported.")
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
import logging
|
||||
|
||||
import boto3
|
||||
from flask import json
|
||||
|
||||
from backend.common.errors import SecretKeyRetrievalError
|
||||
|
||||
|
||||
def get_secret_key(region_name, secret_name):
|
||||
session = boto3.session.Session()
|
||||
client = session.client(service_name="secretsmanager", region_name=region_name)
|
||||
|
||||
try:
|
||||
get_secret_value_response = client.get_secret_value(SecretId=secret_name)
|
||||
if "SecretString" in get_secret_value_response:
|
||||
var = get_secret_value_response["SecretString"]
|
||||
secret = json.loads(var)
|
||||
return secret
|
||||
except Exception as e:
|
||||
logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
|
||||
raise SecretKeyRetrievalError(str(e))
|
||||
|
||||
return None
|
||||
@@ -6,7 +6,7 @@ import pandas as pd
|
||||
|
||||
"""
|
||||
These routines drive all type inference for the schema generation and the
|
||||
FBS (REST OTA) encoding. They are also used for CXG generation.
|
||||
FBS (REST OTA) encoding.
|
||||
|
||||
|
||||
H5AD Type REST REST
|
||||
|
||||
@@ -3,7 +3,6 @@ import click
|
||||
from .launch import launch
|
||||
from .prepare import prepare
|
||||
from .upgrade import log_upgrade_check
|
||||
from .schema import schema_cli
|
||||
from .. import __version__
|
||||
|
||||
|
||||
@@ -30,4 +29,3 @@ def cli(upgrade_check):
|
||||
|
||||
cli.add_command(launch)
|
||||
cli.add_command(prepare)
|
||||
cli.add_command(schema_cli)
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
import click
|
||||
|
||||
from backend.server.converters.schema import remix, validate
|
||||
|
||||
|
||||
@click.group(
|
||||
name="schema",
|
||||
subcommand_metavar="COMMAND <args>",
|
||||
short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
|
||||
context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
|
||||
)
|
||||
def schema_cli():
|
||||
try:
|
||||
import scanpy # noqa: F401
|
||||
except ImportError:
|
||||
raise click.ClickException(
|
||||
"[cellxgene] cellxgene schema requires scanpy"
|
||||
)
|
||||
|
||||
|
||||
@click.command(
|
||||
name="apply",
|
||||
short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
|
||||
help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
|
||||
"h5ad file, apply the schema changes and create a new, conforming h5ad.",
|
||||
)
|
||||
@click.option(
|
||||
"--source-h5ad",
|
||||
help="Input h5ad file.",
|
||||
nargs=1,
|
||||
required=True,
|
||||
type=click.Path(exists=True, dir_okay=False),
|
||||
)
|
||||
@click.option(
|
||||
"--remix-config",
|
||||
help="Config yaml with information on how to apply the schema.",
|
||||
nargs=1,
|
||||
required=True,
|
||||
type=click.Path(exists=True, dir_okay=False),
|
||||
)
|
||||
@click.option(
|
||||
"--output-filename",
|
||||
help="Filename for the new, schema-conforming h5ad file.",
|
||||
required=True,
|
||||
nargs=1
|
||||
)
|
||||
def schema_apply(source_h5ad, remix_config, output_filename):
|
||||
remix.apply_schema(source_h5ad, remix_config, output_filename)
|
||||
|
||||
|
||||
@click.command(
|
||||
name="validate",
|
||||
short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
|
||||
)
|
||||
@click.argument(
|
||||
"h5ad",
|
||||
nargs=1,
|
||||
type=click.Path(exists=True, dir_okay=False),
|
||||
)
|
||||
@click.option(
|
||||
"--shallow",
|
||||
help="When true, just check that the correct version information is present.",
|
||||
default=False,
|
||||
show_default=True,
|
||||
is_flag=True,
|
||||
)
|
||||
def schema_validate(h5ad, shallow):
|
||||
validate.validate(h5ad, shallow)
|
||||
|
||||
|
||||
schema_cli.add_command(schema_apply)
|
||||
schema_cli.add_command(schema_validate)
|
||||
@@ -1,4 +1,2 @@
|
||||
from backend.common.utils.aws_secret_utils import get_secret_key # noqa F504
|
||||
|
||||
DEFAULT_SERVER_PORT = 5005
|
||||
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
|
||||
|
||||
@@ -2,28 +2,23 @@ import os
|
||||
|
||||
from backend.server.common.config.base_config import BaseConfig
|
||||
from backend.common.errors import ConfigurationError
|
||||
from backend.server.common.config import get_secret_key
|
||||
from backend.common.errors import SecretKeyRetrievalError
|
||||
from backend.common.utils.type_conversion_utils import convert_string_to_value
|
||||
|
||||
|
||||
class ExternalConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with external configuration sources, such as
|
||||
environment variables or the AWS Secrets Manager."""
|
||||
environment variables."""
|
||||
|
||||
def __init__(self, app_config, default_config):
|
||||
super().__init__(app_config, default_config)
|
||||
try:
|
||||
self.environment = default_config["environment"]
|
||||
self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
|
||||
self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_environment(context)
|
||||
self.handle_aws_secrets_manager(context)
|
||||
|
||||
def handle_environment(self, context):
|
||||
"""For each environment variable defined, get the value (if it is set),
|
||||
@@ -47,50 +42,3 @@ class ExternalConfig(BaseConfig):
|
||||
else:
|
||||
value = convert_string_to_value(value)
|
||||
self.app_config.update_single_config_from_path_and_value(path, value)
|
||||
|
||||
def handle_aws_secrets_manager(self, context):
|
||||
"""For each aws secret defined, get the key/values, and set the specified config parameter"""
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
|
||||
|
||||
if not self.aws_secrets_manager__secrets:
|
||||
return
|
||||
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
|
||||
|
||||
for secret in self.aws_secrets_manager__secrets:
|
||||
secret_name = secret.get("name")
|
||||
if secret_name is None:
|
||||
raise ConfigurationError("aws_secrets_manager: 'name' is missing")
|
||||
if not isinstance(secret_name, str):
|
||||
raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
|
||||
|
||||
try:
|
||||
secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
|
||||
except SecretKeyRetrievalError as e:
|
||||
raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
|
||||
|
||||
values = secret.get("values")
|
||||
if values is None:
|
||||
raise ConfigurationError("aws_secrets_manager: 'values' is missing")
|
||||
if not isinstance(values, list):
|
||||
raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
|
||||
|
||||
for value in values:
|
||||
key = value.get("key")
|
||||
if key is None:
|
||||
raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
|
||||
path = value.get("path")
|
||||
if path is None:
|
||||
raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
|
||||
required = value.get("required", False)
|
||||
if type(required) != bool:
|
||||
raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
|
||||
|
||||
secret_value = secret_dict.get(key)
|
||||
if secret_value is None:
|
||||
if required:
|
||||
raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
|
||||
else:
|
||||
secret_value = convert_string_to_value(secret_value)
|
||||
self.app_config.update_single_config_from_path_and_value(path, secret_value)
|
||||
|
||||
@@ -1,211 +0,0 @@
|
||||
"""Helpers for converting and checking HGNC gene symbols."""
|
||||
|
||||
import argparse
|
||||
import enum
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def get_upgraded_var_index(var, hgnc_path=None):
|
||||
"""Given an anndata var dataframe, return a new index for the dataframe
|
||||
where human gene symbols have been upgraded to the current HGNC set.
|
||||
"""
|
||||
|
||||
if not hgnc_path:
|
||||
hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
|
||||
|
||||
hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
|
||||
|
||||
return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
|
||||
|
||||
|
||||
class SymbolStatus(enum.Enum):
|
||||
"""The status of a symbol in the HGNC database.
|
||||
|
||||
APPROVED: Currently a valid symbol
|
||||
WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
|
||||
not to exist _unless_ that symbol is also approved
|
||||
AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
|
||||
multiple approved symbols
|
||||
UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
|
||||
symbol
|
||||
UNKNOWN: A symbol that does not appear in HGNC
|
||||
"""
|
||||
|
||||
APPROVED = 1
|
||||
WITHDRAWN = 2
|
||||
AMBIGUOUS = 3
|
||||
UPGRADABLE = 4
|
||||
UNKNOWN = 5
|
||||
|
||||
|
||||
class HGNCSymbolChecker:
|
||||
"""Handle checking and correcting HGNC symbols."""
|
||||
|
||||
def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
|
||||
self.approved_symbols = approved_symbols
|
||||
self.withdrawn_symbols = withdrawn_symbols
|
||||
self.ambiguous_symbols = ambiguous_symbols
|
||||
self.symbol_map = symbol_map
|
||||
|
||||
def print_symbol_map(self):
|
||||
"""Print out a map from old symbol to new symbol."""
|
||||
|
||||
for symbol_pair in self.symbol_map.items():
|
||||
print("\t".join(symbol_pair))
|
||||
|
||||
def check_symbol(self, symbol):
|
||||
"""See if a symbol if approved or something else."""
|
||||
if symbol in self.approved_symbols:
|
||||
return SymbolStatus.APPROVED
|
||||
|
||||
if symbol in self.withdrawn_symbols:
|
||||
return SymbolStatus.WITHDRAWN
|
||||
|
||||
if symbol in self.ambiguous_symbols:
|
||||
return SymbolStatus.AMBIGUOUS
|
||||
|
||||
if symbol in self.symbol_map:
|
||||
return SymbolStatus.UPGRADABLE
|
||||
|
||||
return SymbolStatus.UNKNOWN
|
||||
|
||||
def upgrade_symbol(self, symbol):
|
||||
"""Return the approved symbol for the given symbol.
|
||||
|
||||
If the symbol cannot be upgraded, just return the original symbol.
|
||||
"""
|
||||
|
||||
fixed_symbol, stripped_symbol = format_symbol(symbol)
|
||||
|
||||
if fixed_symbol in self.approved_symbols:
|
||||
return fixed_symbol
|
||||
elif fixed_symbol in self.symbol_map:
|
||||
return self.symbol_map[fixed_symbol]
|
||||
elif stripped_symbol in self.approved_symbols:
|
||||
return stripped_symbol
|
||||
elif stripped_symbol in self.symbol_map:
|
||||
return self.symbol_map[stripped_symbol]
|
||||
|
||||
return symbol
|
||||
|
||||
@classmethod
|
||||
def from_hgnc_records(cls, hgnc_dataset_path):
|
||||
"""Parse a hgnc database download into a HGNCSymbolChecker object."""
|
||||
|
||||
def all_symbols(record):
|
||||
"""Get all the symbols associated with an HGNC record including previous, alias,
|
||||
and approved."""
|
||||
yield format_symbol(record["symbol"])[0]
|
||||
for symbol in alias_and_previous_symbols(record):
|
||||
yield symbol
|
||||
|
||||
def alias_and_previous_symbols(record):
|
||||
"""Get alias and previous symbols from an HGNC record."""
|
||||
for field in ("alias_symbol", "prev_symbol"):
|
||||
if record[field] is not np.nan:
|
||||
for symbol in record[field].split("|"):
|
||||
yield format_symbol(symbol)[0]
|
||||
# Sometimes something like HGNC:1234 appears in datasets, which we
|
||||
# want to fix as well.
|
||||
yield record["hgnc_id"]
|
||||
|
||||
hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
|
||||
|
||||
# Get all symbols that are currently approved.
|
||||
approved_symbols = set()
|
||||
for record in hgnc_records:
|
||||
if record["status"] == "Approved":
|
||||
approved_symbols.add(format_symbol(record["symbol"])[0])
|
||||
|
||||
# Get all symbols that have been withdrawn
|
||||
withdrawn_symbols = set()
|
||||
for record in hgnc_records:
|
||||
if record["status"] == "Entry Withdrawn":
|
||||
for symbol in all_symbols(record):
|
||||
withdrawn_symbols.add(symbol)
|
||||
|
||||
# If a symbol is both approved and withdrawn, be optimistic and call it approved
|
||||
logging.warning(
|
||||
f"Some symbols are simulaneously withdrawn and approved\n"
|
||||
f"We will treat them at approved:\n"
|
||||
f"{withdrawn_symbols.intersection(approved_symbols)}"
|
||||
)
|
||||
withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
|
||||
|
||||
# Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
|
||||
alias_previous_to_approved = {}
|
||||
ambiguous_symbols = set()
|
||||
|
||||
for record in hgnc_records:
|
||||
if record["status"] == "Approved":
|
||||
|
||||
# The approved symbol is what we'll map to
|
||||
approved_symbol = format_symbol(record["symbol"])[0]
|
||||
|
||||
for symbol in alias_and_previous_symbols(record):
|
||||
|
||||
# If the alias or previous symbol is also an approved symbol,
|
||||
# we'll just leave it alone
|
||||
if symbol in approved_symbols:
|
||||
continue
|
||||
|
||||
# If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
|
||||
if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
|
||||
ambiguous_symbols.add(symbol)
|
||||
else:
|
||||
alias_previous_to_approved[symbol] = approved_symbol
|
||||
|
||||
# Remove all the ambiguous symbols from the map
|
||||
for ambiguous_symbol in ambiguous_symbols:
|
||||
alias_previous_to_approved.pop(ambiguous_symbol)
|
||||
|
||||
return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
|
||||
|
||||
|
||||
def format_symbol(symbol):
|
||||
"""HGNC rules say symbols should all be upper case except for C#orf#. However, case is
|
||||
variable in both alias and previous symbols as well as in the symbols we get in
|
||||
submissions. So, upper case everything except for the one situation where mixed-case
|
||||
is allowed, which are the genes like C2orf157.
|
||||
|
||||
Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
|
||||
names persist throughout the life of the object. They won't match against the HGNC database
|
||||
and we want to merge them, so we need to strip off the suffix and try matching again.
|
||||
|
||||
This function takes a symbol and returns the symbol with the fixed case and also with the
|
||||
seurat/scanpy suffix stripped off.
|
||||
"""
|
||||
|
||||
match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
|
||||
|
||||
if match:
|
||||
fixed_case = f"C{match.group(2)}orf{match.group(4)}"
|
||||
else:
|
||||
fixed_case = symbol.upper()
|
||||
|
||||
suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
|
||||
|
||||
return fixed_case, suffix_stripped
|
||||
|
||||
|
||||
def main():
|
||||
"""When called as main, parse a given hgnc download and print out a map from old to new
|
||||
symbol.
|
||||
"""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
|
||||
|
||||
hgnc_symbol_checker.print_symbol_map()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
@@ -1,86 +0,0 @@
|
||||
"""Methods for working with ontologies and the OLS."""
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
import requests
|
||||
|
||||
OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api"
|
||||
|
||||
# Curie means something like CL:0000001
|
||||
|
||||
|
||||
def _ontology_name(curie):
|
||||
"""Get the name of the ontology from the curie, CL or UBERON for example."""
|
||||
return curie.split(":")[0]
|
||||
|
||||
|
||||
def _ontology_value(curie):
|
||||
"""Get the id component of the curie, 0000001 from CL:0000001 for example."""
|
||||
return curie.split(":")[1]
|
||||
|
||||
|
||||
def _double_encode(url):
|
||||
"""Double url encode a url. This is required by the OLS API."""
|
||||
return quote_plus(quote_plus(url))
|
||||
|
||||
|
||||
def _iri(curie):
|
||||
"""Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org"""
|
||||
if _ontology_name(curie) == "EFO":
|
||||
return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}"
|
||||
return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}"
|
||||
|
||||
|
||||
class OntologyLookupError(Exception):
|
||||
"""Exception for some problem with looking up ontology information."""
|
||||
|
||||
|
||||
def _ontology_info_url(curie):
|
||||
"""Get the to make a GET to to get information about an ontology term."""
|
||||
|
||||
# If the curie is empty, just return an empty string. This happens when there is no
|
||||
# valid ontology value.
|
||||
if not curie:
|
||||
return ""
|
||||
else:
|
||||
return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}"
|
||||
|
||||
|
||||
def get_ontology_label(curie):
|
||||
"""For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'"""
|
||||
|
||||
url = _ontology_info_url(curie)
|
||||
|
||||
if not url:
|
||||
return ""
|
||||
|
||||
response = requests.get(url)
|
||||
|
||||
if not response.ok:
|
||||
raise OntologyLookupError(
|
||||
f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}"
|
||||
)
|
||||
return response.json()["label"]
|
||||
|
||||
|
||||
def lookup_candidate_term(label, ontology="cl", method="select"):
|
||||
"""Lookup candidate terms for a label. This is useful when there is an existing label in a
|
||||
submitted dataset, and you want to find an appropriate ontology term.
|
||||
|
||||
Args:
|
||||
label: the label to find ontology terms for
|
||||
ontology: the ontology to search in, cl or uberon or efo for example
|
||||
method: select or search. search provides much broader results
|
||||
|
||||
Returns:
|
||||
list of (curie, label) tuples returned by OLS
|
||||
"""
|
||||
# using OLS REST API [https://www.ebi.ac.uk/ols/docs/api]
|
||||
url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}"
|
||||
response = requests.get(url)
|
||||
|
||||
if not response.ok:
|
||||
raise OntologyLookupError(
|
||||
f"Label {label} lookup failed, got status code {response.status_code}: {response.text}"
|
||||
)
|
||||
|
||||
return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]]
|
||||
@@ -1,266 +0,0 @@
|
||||
import argparse
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import string
|
||||
|
||||
import anndata
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import yaml
|
||||
|
||||
from . import gene_symbol
|
||||
from . import ontology
|
||||
from . import validate
|
||||
|
||||
REPLACE_SUFFIX = "_original"
|
||||
ONTOLOGY_SUFFIX = "_ontology_term_id"
|
||||
|
||||
|
||||
def is_curie(value):
|
||||
"""Return True iff the value is an OBO-id CURIE like EFO:000001"""
|
||||
return (value.count(":")
|
||||
and all(len(part) > 0 for part in value.split(":"))
|
||||
and all(c in string.digits for c in value.split(":")[1]))
|
||||
|
||||
|
||||
def is_ontology_field(field_name):
|
||||
"""Return True iff the field_name is an ontology field like tissue_ontology_term_id"""
|
||||
return field_name.endswith(ONTOLOGY_SUFFIX)
|
||||
|
||||
|
||||
def get_label_field_name(field_name):
|
||||
"""Get the associated label field from an ontology field, assay_ontology_term_id --> assay"""
|
||||
return field_name[: -len(ONTOLOGY_SUFFIX)]
|
||||
|
||||
|
||||
def split_suffix(maybe_curie):
|
||||
"""Split off the (cell culture) or (organoid) suffix."""
|
||||
|
||||
suffixes = [" (cell culture)", " (organoid)"]
|
||||
for suffix in suffixes:
|
||||
if maybe_curie.endswith(suffix):
|
||||
return maybe_curie[:-len(suffix)], suffix
|
||||
return maybe_curie, ""
|
||||
|
||||
|
||||
def get_curie_and_label(maybe_curie):
|
||||
"""Given a string that might be a curie, return a (curie, label) pair"""
|
||||
|
||||
maybe_curie, suffix = split_suffix(maybe_curie)
|
||||
if not is_curie(maybe_curie):
|
||||
return ("", maybe_curie + suffix)
|
||||
return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix)
|
||||
|
||||
|
||||
def safe_add_field(adata_attr, field_name, field_value):
|
||||
"""Add a field and value to an AnnData, but don't clobber an exising value."""
|
||||
|
||||
if (
|
||||
isinstance(field_value, list)
|
||||
and field_value
|
||||
and isinstance(field_value[0], dict)
|
||||
):
|
||||
field_value = json.dumps(field_value)
|
||||
if field_name in adata_attr:
|
||||
adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name]
|
||||
adata_attr[field_name] = field_value
|
||||
|
||||
|
||||
def remix_uns(adata, uns_config):
|
||||
"""Add fields from the config to adata.uns"""
|
||||
for field_name, field_value in uns_config.items():
|
||||
|
||||
if is_ontology_field(field_name):
|
||||
# If it's an ontology field, look it up
|
||||
label_field_name = get_label_field_name(field_name)
|
||||
ontology_term, ontology_label = get_curie_and_label(field_value)
|
||||
safe_add_field(adata.uns, field_name, ontology_term)
|
||||
safe_add_field(adata.uns, label_field_name, ontology_label)
|
||||
else:
|
||||
safe_add_field(adata.uns, field_name, field_value)
|
||||
|
||||
|
||||
def remix_obs(adata, obs_config):
|
||||
"""Add fields from the config to adata.obs"""
|
||||
|
||||
for field_name, field_value in obs_config.items():
|
||||
|
||||
if isinstance(field_value, dict):
|
||||
# If the value is a dict, that means we are supposed to map from an
|
||||
# existing column to the new one
|
||||
source_column, column_map = next(iter(field_value.items()))
|
||||
nan_value = None
|
||||
for key in column_map:
|
||||
if isinstance(key, float) and math.isnan(key):
|
||||
nan_value = column_map[key]
|
||||
if nan_value is not None:
|
||||
column_map["nan"] = nan_value
|
||||
|
||||
for key in column_map:
|
||||
if key not in adata.obs[source_column].unique():
|
||||
logging.warning(f'Key {key} not in adata.obs["{source_column}"]')
|
||||
|
||||
for value in adata.obs[source_column].unique():
|
||||
if value not in column_map:
|
||||
logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict')
|
||||
|
||||
if is_ontology_field(field_name):
|
||||
ontology_term_map, ontology_label_map = {}, {}
|
||||
logging.info(f"Looking up labels for {field_name}")
|
||||
for original_value, maybe_curie in column_map.items():
|
||||
curie, label = get_curie_and_label(maybe_curie)
|
||||
ontology_term_map[original_value] = curie
|
||||
ontology_label_map[original_value] = label
|
||||
logging.info(f"Mapping {original_value} -> {curie} -> {label}")
|
||||
|
||||
ontology_column = adata.obs[source_column].replace(
|
||||
ontology_term_map, inplace=False
|
||||
)
|
||||
label_column = adata.obs[source_column].replace(
|
||||
ontology_label_map, inplace=False
|
||||
)
|
||||
|
||||
safe_add_field(adata.obs, field_name, ontology_column)
|
||||
safe_add_field(
|
||||
adata.obs, get_label_field_name(field_name), label_column
|
||||
)
|
||||
else:
|
||||
label_column = adata.obs[source_column].replace(
|
||||
column_map, inplace=False
|
||||
)
|
||||
safe_add_field(adata.obs, field_name, label_column)
|
||||
|
||||
else:
|
||||
if is_ontology_field(field_name):
|
||||
# If it's an ontology field, look it up
|
||||
label_field_name = get_label_field_name(field_name)
|
||||
ontology_term, ontology_label = get_curie_and_label(field_value)
|
||||
safe_add_field(adata.obs, field_name, ontology_term)
|
||||
safe_add_field(adata.obs, label_field_name, ontology_label)
|
||||
else:
|
||||
safe_add_field(adata.obs, field_name, field_value)
|
||||
|
||||
|
||||
def merge_df(df, domain, index, columns):
|
||||
"""
|
||||
Given a dataframe with duplicate column labels, merge and return a dataframe where
|
||||
the duplicates have been merged together, resulting in a dataframe with unique column
|
||||
labels.
|
||||
|
||||
"merge" depends on the value of domain. If the domain is "raw", then duplicate columns
|
||||
can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then
|
||||
summed, and then logged or sqrt'd again.
|
||||
"""
|
||||
|
||||
if not isinstance(df, np.ndarray):
|
||||
to_merge = df.toarray()
|
||||
else:
|
||||
to_merge = df
|
||||
if domain == "raw":
|
||||
merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum(
|
||||
axis=1, level=0, skipna=False
|
||||
)
|
||||
elif domain == "log1p":
|
||||
merged_df = (
|
||||
pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns)
|
||||
.sum(axis=1, level=0, skipna=False)
|
||||
)
|
||||
merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
|
||||
elif domain == "sqrt":
|
||||
merged_df = (
|
||||
pd.DataFrame(np.square(to_merge), index=index, columns=columns)
|
||||
.sum(axis=1, level=0, skipna=False)
|
||||
)
|
||||
merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
|
||||
|
||||
return merged_df
|
||||
|
||||
|
||||
def fixup_gene_symbols(adata, fixup_config):
|
||||
"""Update the var index to hold a consistent set of HGNC gene symbols."""
|
||||
|
||||
upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var)
|
||||
|
||||
merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index)
|
||||
fixup_adata = anndata.AnnData(
|
||||
X=merged_X,
|
||||
obs=adata.obs,
|
||||
var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
|
||||
uns=adata.uns,
|
||||
obsm=adata.obsm,
|
||||
)
|
||||
|
||||
for layer, domain in fixup_config.items():
|
||||
if layer == "X":
|
||||
continue
|
||||
if layer == "raw.X":
|
||||
df = adata.raw.X
|
||||
else:
|
||||
df = adata.layers[layer]
|
||||
|
||||
merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index)
|
||||
assert merged_df.index.equals(merged_X.index)
|
||||
assert merged_df.columns.equals(merged_X.columns)
|
||||
|
||||
if domain == "raw":
|
||||
fixup_raw = anndata.AnnData(
|
||||
X=merged_df,
|
||||
obs=adata.obs,
|
||||
var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
|
||||
)
|
||||
fixup_adata.raw = fixup_raw
|
||||
else:
|
||||
fixup_adata.layers[layer] = merged_df
|
||||
|
||||
return fixup_adata
|
||||
|
||||
|
||||
def _strip_version(adata):
|
||||
"""Remove version information from the AnnData object."""
|
||||
|
||||
if "version" in adata.uns_keys():
|
||||
del adata.uns["version"]
|
||||
|
||||
|
||||
def apply_schema(source_h5ad, remix_config, output_filename):
|
||||
|
||||
try:
|
||||
import scanpy
|
||||
except ImportError:
|
||||
raise ImportError("scanpy must be installed for cellxgene schema")
|
||||
adata = scanpy.read_h5ad(source_h5ad)
|
||||
config = yaml.load(open(remix_config), Loader=yaml.FullLoader)
|
||||
remix_uns(adata, config["uns"])
|
||||
remix_obs(adata, config["obs"])
|
||||
|
||||
if config.get("fixup_gene_symbols"):
|
||||
adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"])
|
||||
|
||||
if ("version" in adata.uns_keys()
|
||||
and isinstance(adata.uns["version"], collections.Mapping)
|
||||
and "corpora_schema_version" in adata.uns["version"]):
|
||||
schema_version = adata.uns["version"]["corpora_schema_version"]
|
||||
try:
|
||||
validate.get_schema_definition(schema_version)
|
||||
except ValueError:
|
||||
logging.warning(f"Stripping version information out of AnnData because schema "
|
||||
f"version {schema_version} is unknown.")
|
||||
_strip_version(adata)
|
||||
|
||||
if not validate.validate_adata(adata, shallow=False):
|
||||
logging.warning(f"Stripping version information out of AnnData because it does not "
|
||||
f"follow schema version {schema_version} .")
|
||||
_strip_version(adata)
|
||||
|
||||
adata.write_h5ad(output_filename, compression="gzip")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--source-h5ad", required=True)
|
||||
parser.add_argument("--remix-config", required=True)
|
||||
parser.add_argument("--output-filename", required=True)
|
||||
args = parser.parse_args()
|
||||
apply_schema(args.source_h5ad, args.remix_config, args.output_filename)
|
||||
@@ -1,95 +0,0 @@
|
||||
title: Corpora schema version 1.0.0
|
||||
type: anndata
|
||||
components:
|
||||
uns:
|
||||
type: dict
|
||||
keys:
|
||||
version:
|
||||
type: dict
|
||||
keys:
|
||||
corpora_schema_version: null
|
||||
corpora_encoding_version: null
|
||||
title:
|
||||
type: string
|
||||
contributors:
|
||||
type: stringified list of dicts
|
||||
layer_descriptions:
|
||||
type: dict
|
||||
keys:
|
||||
X: null
|
||||
organism:
|
||||
type: string
|
||||
nullable: false
|
||||
organism_ontology_term_id:
|
||||
type: curie
|
||||
prefixes:
|
||||
- NCBITaxon
|
||||
var:
|
||||
type: dataframe
|
||||
index:
|
||||
type: human-readable string
|
||||
unique: true
|
||||
obs:
|
||||
type: dataframe
|
||||
index:
|
||||
unique: true
|
||||
columns:
|
||||
tissue:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
tissue_ontology_term_id:
|
||||
type: suffixed curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- UBERON
|
||||
assay:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
assay_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- EFO
|
||||
disease:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
disease_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- MONDO
|
||||
- PATO
|
||||
cell_type:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
cell_type_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- CL
|
||||
- UBERON
|
||||
sex:
|
||||
type: string
|
||||
enum:
|
||||
- male
|
||||
- female
|
||||
- mixed
|
||||
- unknown
|
||||
- other
|
||||
ethnicity:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
ethnicity_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- HANCESTRO
|
||||
development_stage:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
development_stage_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- HsapDv
|
||||
- EFO
|
||||
@@ -1,93 +0,0 @@
|
||||
title: Corpora schema version 1.1.0
|
||||
type: anndata
|
||||
components:
|
||||
uns:
|
||||
type: dict
|
||||
keys:
|
||||
version:
|
||||
type: dict
|
||||
keys:
|
||||
corpora_schema_version: null
|
||||
corpora_encoding_version: null
|
||||
title:
|
||||
type: string
|
||||
layer_descriptions:
|
||||
type: dict
|
||||
keys:
|
||||
X: null
|
||||
organism:
|
||||
type: string
|
||||
nullable: false
|
||||
organism_ontology_term_id:
|
||||
type: curie
|
||||
prefixes:
|
||||
- NCBITaxon
|
||||
var:
|
||||
type: dataframe
|
||||
index:
|
||||
type: human-readable string
|
||||
unique: true
|
||||
obs:
|
||||
type: dataframe
|
||||
index:
|
||||
unique: true
|
||||
columns:
|
||||
tissue:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
tissue_ontology_term_id:
|
||||
type: suffixed curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- UBERON
|
||||
assay:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
assay_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- EFO
|
||||
disease:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
disease_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- MONDO
|
||||
- PATO
|
||||
cell_type:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
cell_type_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- CL
|
||||
- UBERON
|
||||
sex:
|
||||
type: string
|
||||
enum:
|
||||
- male
|
||||
- female
|
||||
- mixed
|
||||
- unknown
|
||||
- other
|
||||
ethnicity:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
ethnicity_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- HANCESTRO
|
||||
development_stage:
|
||||
type: human-readable string
|
||||
nullable: false
|
||||
development_stage_ontology_term_id:
|
||||
type: curie
|
||||
nullable: true
|
||||
prefixes:
|
||||
- HsapDv
|
||||
- EFO
|
||||
@@ -1,236 +0,0 @@
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
import yaml
|
||||
|
||||
|
||||
def _is_null(v):
|
||||
"""Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad."""
|
||||
return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0)
|
||||
|
||||
|
||||
def _validate_stringified_list_of_dicts(s):
|
||||
"""Verify that a string can be parsed into a list.
|
||||
|
||||
We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to
|
||||
json.dumps them. This verifies that we can load them back.
|
||||
"""
|
||||
|
||||
try:
|
||||
list_ = json.loads(s)
|
||||
if not isinstance(list_, list):
|
||||
return False
|
||||
for el in list_:
|
||||
if not isinstance(el, dict):
|
||||
return False
|
||||
return True
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _validate_human_readable_string(s):
|
||||
"""Verify that a string is human-readable.
|
||||
|
||||
There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind
|
||||
of vague and subjective. I feel like I can read many strings. So here we just check for the main ways
|
||||
that fails: someone puts in an ontology term id or and ensembl gene/transcript id.
|
||||
|
||||
Returns False if s is not a string or is one of those bad string types.
|
||||
"""
|
||||
|
||||
return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s))
|
||||
|
||||
|
||||
def _validate_curie(c, prefixes):
|
||||
"""Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the
|
||||
prefix of the curies is in prefixes.
|
||||
"""
|
||||
|
||||
if not c:
|
||||
return True
|
||||
|
||||
match = re.match(r"([A-Z]\w+):\d+$", c)
|
||||
|
||||
if prefixes:
|
||||
return match and match.group(1) in prefixes
|
||||
else:
|
||||
return match
|
||||
|
||||
|
||||
def _validate_suffixed_curie(c, prefixes):
|
||||
"""Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'"""
|
||||
|
||||
# Pull off the suffix
|
||||
suffix = re.findall(r"\ \(.*\)$", c)
|
||||
if suffix:
|
||||
c = c[: -len(suffix[0])]
|
||||
return _validate_curie(c, prefixes)
|
||||
|
||||
|
||||
def _validate_column(column, column_name, df_name, schema_def):
|
||||
"""Given a schema definition and the column of a dataframe, verify that the column satifies
|
||||
the schema.
|
||||
"""
|
||||
|
||||
errors = []
|
||||
|
||||
if schema_def.get("unique"):
|
||||
if column.nunique() != len(column):
|
||||
errors.append(f"Column {column_name} in dataframe {df_name} is not unique.")
|
||||
|
||||
if "nullable" in schema_def and not schema_def["nullable"]:
|
||||
if any(_is_null(v) for v in column):
|
||||
errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.")
|
||||
|
||||
if schema_def.get("type") == "human-readable string":
|
||||
non_readables = [v for v in column if not _validate_human_readable_string(v)]
|
||||
if non_readables:
|
||||
errors.append(
|
||||
f"Column {column_name} in dataframe {df_name} contains non-human-readable "
|
||||
f"values like {non_readables[0]}"
|
||||
)
|
||||
|
||||
if schema_def.get("type") in ("curie", "suffixed curie"):
|
||||
validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie
|
||||
non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))]
|
||||
if non_valid_curies:
|
||||
errors.append(
|
||||
f"Column {column_name} in dataframe {df_name} contains invalid ontology values like "
|
||||
f"{non_valid_curies[0]}."
|
||||
)
|
||||
if "prefixes" in schema_def:
|
||||
errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}."
|
||||
|
||||
if "enum" in schema_def:
|
||||
bad_enums = [v for v in column if v not in schema_def["enum"]]
|
||||
if bad_enums:
|
||||
errors.append(
|
||||
f"Column {column_name} in dataframe {df_name} contains unpermitted values like "
|
||||
f"{bad_enums[0]}. Values must be one of {schema_def['enum']}."
|
||||
)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_dict(dict_, dict_name, schema_def):
|
||||
"""Given a schema definition and dict, verify that the dict satifies the schema."""
|
||||
|
||||
errors = []
|
||||
|
||||
for key in schema_def.get("keys", []):
|
||||
if key not in dict_:
|
||||
errors.append(f"{dict_name} is missing key {key}.")
|
||||
elif schema_def["keys"][key]:
|
||||
if schema_def["keys"][key]["type"] == "stringified list of dicts":
|
||||
if not _validate_stringified_list_of_dicts(dict_[key]):
|
||||
errors.append(
|
||||
f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}"
|
||||
)
|
||||
elif schema_def["keys"][key]["type"] == "dict":
|
||||
errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key]))
|
||||
elif schema_def["keys"][key]["type"] == "curie":
|
||||
if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]):
|
||||
errors.append(f"Key {key} in {dict_name} contains invalid ontology value.")
|
||||
if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]:
|
||||
if _is_null(dict_[key]):
|
||||
errors.append(f"Key {key} in dict {dict_name} is an empty value.")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def _validate_dataframe(df, df_name, schema_def):
|
||||
"""Given a dataframe and schema definition, verify that the dataframe follows the schema."""
|
||||
|
||||
errors = []
|
||||
|
||||
if "index" in schema_def:
|
||||
errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"]))
|
||||
|
||||
for column in schema_def.get("columns", []):
|
||||
if column not in df.columns:
|
||||
errors.append(f"Dataframe {df_name} is missing column {column}.")
|
||||
else:
|
||||
errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column]))
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def get_schema_definition(version):
|
||||
"""Look up and read a schema definition based on a version number like "1.0.0"."""
|
||||
|
||||
path = os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml"
|
||||
)
|
||||
|
||||
if not os.path.isfile(path):
|
||||
raise ValueError(f"No definition for version {version} found.")
|
||||
|
||||
return yaml.load(open(path), Loader=yaml.FullLoader)
|
||||
|
||||
|
||||
def deep_check(adata, schema_def):
|
||||
"""Perform a "deep" check of the AnnData object using the schema definition.
|
||||
|
||||
This checks all the columns and unstructured metadata rather than just the version.
|
||||
|
||||
Returns a list of error messages. If that list is empty, the object passed validation.
|
||||
"""
|
||||
|
||||
errors = []
|
||||
|
||||
for component, component_def in schema_def["components"].items():
|
||||
if component_def["type"] == "dataframe":
|
||||
errors.extend(_validate_dataframe(getattr(adata, component), component, component_def))
|
||||
elif component_def["type"] == "dict":
|
||||
errors.extend(_validate_dict(getattr(adata, component), component, component_def))
|
||||
else:
|
||||
raise ValueError(f"Unexpected component type {component['type']}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def validate_adata(adata, shallow):
|
||||
"""Validate an AnnData object. If shallow, just check that the required version information is
|
||||
present.
|
||||
"""
|
||||
|
||||
# Does it have the version information written into uns?
|
||||
if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]:
|
||||
print("AnnData file is missing corpora version information")
|
||||
return False
|
||||
|
||||
# We can stop here if it's a "shallow" check, that is, if we're just
|
||||
# checking that version is present.
|
||||
if shallow:
|
||||
return True
|
||||
|
||||
schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"])
|
||||
|
||||
errors = deep_check(adata, schema_def)
|
||||
|
||||
for error in errors:
|
||||
print(error)
|
||||
|
||||
return not errors
|
||||
|
||||
|
||||
def validate(h5ad_path, shallow=False):
|
||||
"""Entry point for validation."""
|
||||
|
||||
try:
|
||||
import scanpy
|
||||
except ImportError:
|
||||
raise ImportError("scanpy must be installed for cellxgene schema")
|
||||
|
||||
try:
|
||||
adata = scanpy.read_h5ad(h5ad_path, backed="r")
|
||||
except (OSError, TypeError):
|
||||
print(f"Unable to open {h5ad_path} with scanpy.")
|
||||
sys.exit(1)
|
||||
|
||||
if not validate_adata(adata, shallow):
|
||||
sys.exit(1)
|
||||
@@ -71,7 +71,7 @@ dataset:
|
||||
|
||||
external:
|
||||
# You can retrieve configuration parameters from this config file, the environment,
|
||||
# the AWS secrets manager, or from the "cellxgene launch" command line arguments.
|
||||
# or from the "cellxgene launch" command line arguments.
|
||||
# They are applied in that order, meaning that if a parameter is defined in more
|
||||
# than one location, the last one applied takes effect.
|
||||
|
||||
@@ -87,31 +87,6 @@ external:
|
||||
- name: CXG_SECRET_KEY
|
||||
path: [server, app, flask_secret_key]
|
||||
required: false
|
||||
|
||||
# AWS Secrets Manager
|
||||
# This section describes how to map aws secrets to configuration parameters.
|
||||
# The format is the region for the secrets manager, then a list of secrets.
|
||||
# each secret has a name, and a list of values.
|
||||
# Each entry in the list of values is a dictionary with three entries:
|
||||
# key: the key of the aws secret.
|
||||
# path: the path within the cellxgene configuration to update.
|
||||
# required: (default=False) a boolean. If true, then it is an error if the key does not exist in the secret.
|
||||
#
|
||||
# example:
|
||||
# aws_secrets_manager:
|
||||
# region: us-west-2
|
||||
# - name: my_first_secret
|
||||
# values:
|
||||
# - key: flask_secret_key
|
||||
# path: [server, app, flask_secret_key]
|
||||
# required: true
|
||||
# - key: db_uri
|
||||
# path: [dataset, user_annotations, db_uri]
|
||||
# required: true
|
||||
|
||||
aws_secrets_manager:
|
||||
region: null
|
||||
secrets: []
|
||||
"""
|
||||
|
||||
|
||||
|
||||
@@ -1,139 +0,0 @@
|
||||
#!/bin/bash
|
||||
wget "https://s3-us-west-2.amazonaws.com/10x.files/samples/cell/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz"
|
||||
tar xf "pbmc3k_filtered_gene_bc_matrices.tar.gz"
|
||||
|
||||
python3 - <<MERGE_GENES
|
||||
import os
|
||||
from scipy.io import mmread, mmwrite
|
||||
import scipy.sparse
|
||||
import pandas as pd
|
||||
from server.converters.schema import gene_symbol
|
||||
|
||||
mat = mmread("filtered_gene_bc_matrices/hg19/matrix.mtx").todense()
|
||||
genes = pd.read_csv("filtered_gene_bc_matrices/hg19/genes.tsv", sep='\t', names=["gene_id", "gene_symbol"])
|
||||
|
||||
upgraded_genes = gene_symbol.get_upgraded_var_index(pd.DataFrame(index=genes["gene_symbol"]))
|
||||
df = pd.DataFrame(data=mat, index=upgraded_genes).T
|
||||
merged = df.sum(axis=1, level=0, skipna=False)
|
||||
|
||||
os.makedirs("merged")
|
||||
merged.columns.to_frame().to_csv("merged/genes.tsv", index=False, header=False)
|
||||
mmwrite("merged/matrix.mtx", scipy.sparse.coo_matrix(merged).T)
|
||||
MERGE_GENES
|
||||
|
||||
cp "filtered_gene_bc_matrices/hg19/barcodes.tsv" "merged/barcodes.tsv"
|
||||
awk '{print $1"\t"$1}' merged/genes.tsv > genes_tmp.tsv; mv genes_tmp.tsv merged/genes.tsv
|
||||
|
||||
echo -e "\n\n\nRunning tutorial on original\n\n\n"
|
||||
Rscript - <<TUTORIAL
|
||||
library(Seurat)
|
||||
|
||||
pbmc.data <- Read10X(data.dir = "filtered_gene_bc_matrices/hg19/")
|
||||
pbmc <- CreateSeuratObject(counts = pbmc.data, project = "pbmc3k", min.features = 200)
|
||||
pbmc <- NormalizeData(pbmc, normalization.method = "LogNormalize", scale.factor = 10000)
|
||||
pbmc <- FindVariableFeatures(pbmc, selection.method = "vst", nfeatures = 2000)
|
||||
pbmc[["percent.mt"]] <- PercentageFeatureSet(pbmc, pattern = "^MT-")
|
||||
all.genes <- rownames(pbmc)
|
||||
pbmc <- ScaleData(pbmc, features = all.genes)
|
||||
|
||||
pbmc <- RunPCA(pbmc, features = VariableFeatures(object = pbmc))
|
||||
pbmc <- FindNeighbors(pbmc, dims = 1:10)
|
||||
pbmc <- FindClusters(pbmc, resolution = 0.5)
|
||||
pbmc <- RunUMAP(pbmc, dims = 1:10)
|
||||
saveRDS(pbmc, file = "./seurat_tutorial.rds")
|
||||
TUTORIAL
|
||||
|
||||
echo -e "\n\n\nRunning tutorial on merged\n\n\n"
|
||||
Rscript - <<TUTORIAL_MERGED
|
||||
library(Seurat)
|
||||
|
||||
pbmc.data <- Read10X(data.dir = "merged/")
|
||||
pbmc <- CreateSeuratObject(counts = pbmc.data, project = "pbmc3k", min.features = 200)
|
||||
pbmc <- NormalizeData(pbmc, normalization.method = "LogNormalize", scale.factor = 10000)
|
||||
pbmc <- FindVariableFeatures(pbmc, selection.method = "vst", nfeatures = 2000)
|
||||
pbmc[["percent.mt"]] <- PercentageFeatureSet(pbmc, pattern = "^MT-")
|
||||
all.genes <- rownames(pbmc)
|
||||
pbmc <- ScaleData(pbmc, features = all.genes)
|
||||
|
||||
pbmc <- RunPCA(pbmc, features = VariableFeatures(object = pbmc))
|
||||
pbmc <- FindNeighbors(pbmc, dims = 1:10)
|
||||
pbmc <- FindClusters(pbmc, resolution = 0.5)
|
||||
pbmc <- RunUMAP(pbmc, dims = 1:10)
|
||||
saveRDS(pbmc, file = "./seurat_tutorial_merged.rds")
|
||||
TUTORIAL_MERGED
|
||||
|
||||
echo -e "\n\n\nRunning SCTransform on original\n\n\n"
|
||||
Rscript - <<SCTRANSFORM
|
||||
library(Seurat)
|
||||
library(sctransform)
|
||||
|
||||
pbmc.data <- Read10X(data.dir = "filtered_gene_bc_matrices/hg19/")
|
||||
pbmc <- CreateSeuratObject(counts = pbmc.data)
|
||||
pbmc <- PercentageFeatureSet(pbmc, pattern = "^MT-", col.name = "percent.mt")
|
||||
pbmc <- SCTransform(pbmc, vars.to.regress = "percent.mt", verbose = FALSE)
|
||||
pbmc <- RunPCA(pbmc, verbose = FALSE)
|
||||
pbmc <- RunUMAP(pbmc, dims = 1:30, verbose = FALSE)
|
||||
pbmc <- FindNeighbors(pbmc, dims = 1:30, verbose = FALSE)
|
||||
pbmc <- FindClusters(pbmc, verbose = FALSE)
|
||||
saveRDS(pbmc, file = "./sctransform.rds")
|
||||
SCTRANSFORM
|
||||
|
||||
echo -e "\n\n\nRunning SCTransform on merged\n\n\n"
|
||||
Rscript - <<SCTRANSFORM_MERGED
|
||||
library(Seurat)
|
||||
library(sctransform)
|
||||
|
||||
pbmc.data <- Read10X(data.dir = "merged/")
|
||||
pbmc <- CreateSeuratObject(counts = pbmc.data)
|
||||
pbmc <- PercentageFeatureSet(pbmc, pattern = "^MT-", col.name = "percent.mt")
|
||||
pbmc <- SCTransform(pbmc, vars.to.regress = "percent.mt", verbose = FALSE)
|
||||
pbmc <- RunPCA(pbmc, verbose = FALSE)
|
||||
pbmc <- RunUMAP(pbmc, dims = 1:30, verbose = FALSE)
|
||||
pbmc <- FindNeighbors(pbmc, dims = 1:30, verbose = FALSE)
|
||||
pbmc <- FindClusters(pbmc, verbose = FALSE)
|
||||
saveRDS(pbmc, file = "./sctransform_merged.rds")
|
||||
SCTRANSFORM_MERGED
|
||||
|
||||
echo -e "\n\n\nConverting\n\n\n"
|
||||
Rscript - <<SCEASY
|
||||
library(sceasy)
|
||||
srt <- readRDS("seurat_tutorial.rds")
|
||||
sceasy::convertFormat(srt,
|
||||
outFile = "seurat_tutorial.h5ad",
|
||||
from = "seurat",
|
||||
to = "anndata",
|
||||
assay = "RNA",
|
||||
main_layer = "data",
|
||||
transfer_layers = c("data", "counts", "scale.data"),
|
||||
drop_single_values = FALSE)
|
||||
|
||||
srt <- readRDS("seurat_tutorial_merged.rds")
|
||||
sceasy::convertFormat(srt,
|
||||
outFile = "seurat_tutorial_merged.h5ad",
|
||||
from = "seurat",
|
||||
to = "anndata",
|
||||
assay = "RNA",
|
||||
main_layer = "data",
|
||||
transfer_layers = c("data", "counts", "scale.data"),
|
||||
drop_single_values = FALSE)
|
||||
|
||||
srt <- readRDS("sctransform.rds")
|
||||
sceasy::convertFormat(srt,
|
||||
outFile = "sctransform.h5ad",
|
||||
from = "seurat",
|
||||
to = "anndata",
|
||||
assay = "SCT",
|
||||
main_layer = "data",
|
||||
transfer_layers = c("data", "counts", "scale.data"),
|
||||
drop_single_values = FALSE)
|
||||
|
||||
srt <- readRDS("sctransform_merged.rds")
|
||||
sceasy::convertFormat(srt,
|
||||
outFile = "sctransform_merged.h5ad",
|
||||
from = "seurat",
|
||||
to = "anndata",
|
||||
assay = "SCT",
|
||||
main_layer = "data",
|
||||
transfer_layers = c("data", "counts", "scale.data"),
|
||||
drop_single_values = FALSE)
|
||||
SCEASY
|
||||
@@ -1,39 +0,0 @@
|
||||
# Locust Load Test
|
||||
|
||||
This directory contains scripts to load test cellxgene's backend. It
|
||||
primary simulates initial data loading and expression data fetch, which
|
||||
are the most common data routes. It currently does not include tests
|
||||
for differential expression or re-clustering routes.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
You need:
|
||||
|
||||
- Python 3.6+, and pip
|
||||
- cellxgene installed
|
||||
- install the locust dependencies in `requirements-locust.txt`
|
||||
|
||||
## To test
|
||||
|
||||
1. Choose to run cellxgene in either single dataset or data root mode.
|
||||
2. Edit config.py to indicate which datasets to load:
|
||||
- in single dataset mode, just set `DataSets=[""]`
|
||||
- in dataroot (multi-dataset) mode, add the route names, eg, `DataSets=['foo.cxg', 'bar.cxg']`
|
||||
3. Launch cellxgene in the appropriate mode
|
||||
4. launch locust, specifying the correct --host argument
|
||||
5. point your web browser to the locust http server, usually `http://localhost:8089/`
|
||||
|
||||
### Single dataset mode
|
||||
|
||||
- Edit config.py and set `DataSets=[""]`
|
||||
- in a shell, run `cellxgene launch somefile.h5ad`
|
||||
- launch locust in another shell, `locust --host http://localhost:5005/` (or wherever you are running cellxgene)
|
||||
- point a browser to the locust port, usually http://localhost:8089/
|
||||
- run test
|
||||
|
||||
### Multi-dataset mode
|
||||
|
||||
- Edit config.py and set `DataSets=["datapath1", ...]`
|
||||
- in a shell, run `cellxgene launch --dataroot path`
|
||||
|
||||
The remainder of the steps are same as single dataset.
|
||||
@@ -1,15 +0,0 @@
|
||||
"""
|
||||
Locust test config
|
||||
"""
|
||||
|
||||
|
||||
""" Data routes that will be tested """
|
||||
|
||||
# single dataset, for non-dataroot tests
|
||||
# DataSets = [""]
|
||||
|
||||
# multi-dataset, for dataroot tests. these are varied in size/shape
|
||||
DataSets = [
|
||||
"GSE60361.cxg",
|
||||
"WongAdultRetina.cxg",
|
||||
]
|
||||
@@ -1,165 +0,0 @@
|
||||
import json
|
||||
import random
|
||||
|
||||
import requests
|
||||
from config import DataSets
|
||||
from locust import HttpUser, SequentialTaskSet, task, between, TaskSet
|
||||
from locust.clients import HttpSession
|
||||
from requests.packages.urllib3.exceptions import InsecureRequestWarning
|
||||
|
||||
import backend.test.decode_fbs as decode_fbs
|
||||
|
||||
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
||||
|
||||
"""
|
||||
Simple locust stress test definition for cellxgene
|
||||
"""
|
||||
|
||||
API_SUFFIX = "api/v0.2"
|
||||
|
||||
|
||||
class CellXGeneTasks(TaskSet):
|
||||
"""
|
||||
Simulate use against a single dataset
|
||||
"""
|
||||
|
||||
def on_start(self):
|
||||
|
||||
self.client.verify = False
|
||||
self.dataset = random.choice(DataSets)
|
||||
|
||||
with self.client.get(
|
||||
f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True
|
||||
) as schema_response:
|
||||
if schema_response.status_code == 200:
|
||||
self.schema = schema_response.json()["schema"]
|
||||
else:
|
||||
self.schema = None
|
||||
|
||||
with self.client.get(
|
||||
f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True
|
||||
) as config_response:
|
||||
if config_response.status_code == 200:
|
||||
self.config = config_response.json()["config"]
|
||||
else:
|
||||
self.config = None
|
||||
|
||||
with self.client.get(
|
||||
f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.var_index_name()}",
|
||||
headers={"Accept": "application/octet-stream"},
|
||||
catch_response=True,
|
||||
) as var_index_response:
|
||||
if var_index_response.status_code == 200:
|
||||
df = decode_fbs.decode_matrix_FBS(var_index_response.content)
|
||||
gene_names_idx = df["col_idx"].index(self.var_index_name())
|
||||
self.gene_names = df["columns"][gene_names_idx]
|
||||
else:
|
||||
self.gene_names = []
|
||||
|
||||
def var_index_name(self):
|
||||
if self.schema is not None:
|
||||
return self.schema["annotations"]["var"]["index"]
|
||||
return None
|
||||
|
||||
def obs_annotation_names(self):
|
||||
if self.schema is not None:
|
||||
return [col["name"] for col in self.schema["annotations"]["obs"]["columns"]]
|
||||
return []
|
||||
|
||||
def layout_names(self):
|
||||
if self.schema is not None:
|
||||
return [layout["name"] for layout in self.schema["layout"]["obs"]]
|
||||
else:
|
||||
return []
|
||||
|
||||
@task(2)
|
||||
class InitializeClient(SequentialTaskSet):
|
||||
"""
|
||||
Initial loading of cellxgene - when the user hits the main route.
|
||||
|
||||
Currently this sequence skips some of the static assets, which are quite small and should be served by the
|
||||
HTTP server directly.
|
||||
|
||||
1. Load index.html, etc.
|
||||
2. Concurrently load /config, /schema
|
||||
3. Concurrently load /layout/obs, /annotations/var?annotation-name=<the index>
|
||||
-- Does initial render --
|
||||
4. Concurrently load all /annotations/obs and all /layouts/obs
|
||||
-- Fully initialized --
|
||||
"""
|
||||
|
||||
# Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency.
|
||||
wait_time = between(0.01, 0.1)
|
||||
|
||||
def on_start(self):
|
||||
self.dataset = self.parent.dataset
|
||||
self.client.verify = False
|
||||
self.api_less_client = HttpSession(
|
||||
base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""),
|
||||
request_success=self.client.request_success,
|
||||
request_failure=self.client.request_failure,
|
||||
)
|
||||
|
||||
@task
|
||||
def index(self):
|
||||
self.api_less_client.get(f"{self.dataset}", stream=True)
|
||||
|
||||
@task
|
||||
def loadConfigAndSchema(self):
|
||||
self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True)
|
||||
self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True)
|
||||
|
||||
@task
|
||||
def loadBootstrapData(self):
|
||||
self.client.get(
|
||||
f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True
|
||||
)
|
||||
self.client.get(
|
||||
f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}",
|
||||
headers={"Accept": "application/octet-stream"},
|
||||
catch_response=True,
|
||||
)
|
||||
|
||||
@task
|
||||
def loadObsAnnotationsAndLayouts(self):
|
||||
obs_names = self.parent.obs_annotation_names()
|
||||
for name in obs_names:
|
||||
self.client.get(
|
||||
f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}",
|
||||
headers={"Accept": "application/octet-stream"},
|
||||
stream=True,
|
||||
)
|
||||
|
||||
layouts = self.parent.layout_names()
|
||||
for name in layouts:
|
||||
self.client.get(
|
||||
f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}",
|
||||
headers={"Accept": "application/octet-stream"},
|
||||
stream=True,
|
||||
)
|
||||
|
||||
@task
|
||||
def done(self):
|
||||
self.interrupt()
|
||||
|
||||
@task(1)
|
||||
def load_expression(self):
|
||||
"""
|
||||
Simulate user occasionally loading some expression data for a gene
|
||||
"""
|
||||
|
||||
gene_name = random.choice(self.gene_names)
|
||||
filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}}
|
||||
self.client.put(
|
||||
f"{self.dataset}/{API_SUFFIX}/data/var",
|
||||
data=json.dumps(filter),
|
||||
headers={"Content-Type": "application/json", "Accept": "application/octet-stream"},
|
||||
stream=True,
|
||||
).close()
|
||||
|
||||
|
||||
class CellxgeneUser(HttpUser):
|
||||
tasks = [CellXGeneTasks]
|
||||
|
||||
# Most ops do not require back-end interaction, so slow cadence for users
|
||||
wait_time = between(10, 60)
|
||||
@@ -1,2 +0,0 @@
|
||||
locust
|
||||
-r ../../requirements.txt
|
||||
@@ -98,8 +98,6 @@ class ConfigTests(unittest.TestCase):
|
||||
lfc_cutoff=0.01,
|
||||
top_n=10,
|
||||
environment=None,
|
||||
aws_secrets_manager_region=None,
|
||||
aws_secrets_manager_secrets=[],
|
||||
X_approximate_distribution="auto",
|
||||
config_file_name="app_config.yml",
|
||||
):
|
||||
@@ -151,8 +149,6 @@ class ConfigTests(unittest.TestCase):
|
||||
)
|
||||
external_config = self.custom_external_config(
|
||||
environment=environment,
|
||||
aws_secrets_manager_region=aws_secrets_manager_region,
|
||||
aws_secrets_manager_secrets=aws_secrets_manager_secrets,
|
||||
config_file_name=f"temp_external_config_{random_num}.yml",
|
||||
)
|
||||
|
||||
@@ -197,8 +193,6 @@ class ConfigTests(unittest.TestCase):
|
||||
def custom_external_config(
|
||||
self,
|
||||
environment=None,
|
||||
aws_secrets_manager_region=None,
|
||||
aws_secrets_manager_secrets=[],
|
||||
config_file_name="external_config.yaml",
|
||||
):
|
||||
# set to the default if environment is None
|
||||
@@ -209,7 +203,6 @@ class ConfigTests(unittest.TestCase):
|
||||
external_config = {
|
||||
"external": {
|
||||
"environment": environment,
|
||||
"aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import requests
|
||||
|
||||
@@ -13,7 +12,7 @@ from backend.test.test_server.unit.common.config import ConfigTests
|
||||
|
||||
class TestExternalConfig(ConfigTests):
|
||||
def test_type_convert(self):
|
||||
# The values from environment variables and aws secrets are returned as strings.
|
||||
# The values from environment variables are returned as strings.
|
||||
# These values need to be converted to the proper types.
|
||||
|
||||
self.assertEqual(convert_string_to_value("1"), int(1))
|
||||
@@ -88,129 +87,3 @@ class TestExternalConfig(ConfigTests):
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
|
||||
|
||||
@patch("backend.server.common.config.external_config.get_secret_key")
|
||||
def test_aws_secrets_manager(self, mock_get_secret_key):
|
||||
mock_get_secret_key.return_value = {
|
||||
"flask_secret_key": "mock_flask_secret_key",
|
||||
}
|
||||
configfile = self.custom_external_config(
|
||||
aws_secrets_manager_region="us-west-2",
|
||||
aws_secrets_manager_secrets=[
|
||||
dict(
|
||||
name="my_secret",
|
||||
values=[
|
||||
dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=True),
|
||||
],
|
||||
)
|
||||
],
|
||||
config_file_name="secret_external_config.yaml",
|
||||
)
|
||||
|
||||
app_config = AppConfig()
|
||||
app_config.update_from_config_file(configfile)
|
||||
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
|
||||
|
||||
app_config.complete_config()
|
||||
|
||||
self.assertEqual(app_config.server_config.app__flask_secret_key, "mock_flask_secret_key")
|
||||
|
||||
@patch("backend.server.common.config.external_config.get_secret_key")
|
||||
def test_aws_secrets_manager_error(self, mock_get_secret_key):
|
||||
mock_get_secret_key.return_value = {
|
||||
"db_uri": "mock_db_uri",
|
||||
}
|
||||
|
||||
# no region
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = None
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(
|
||||
config_error.exception.message,
|
||||
"Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
|
||||
)
|
||||
|
||||
# missing secret name
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
|
||||
|
||||
# secret name wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
|
||||
|
||||
# missing values name
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
|
||||
|
||||
# values wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
|
||||
|
||||
# entry missing key
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
|
||||
|
||||
# entry required is wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
|
||||
|
||||
# entry missing path
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=[dict(key="db_uri", required=True)])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
|
||||
|
||||
# secret missing required key
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(
|
||||
name="mysecret",
|
||||
values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
|
||||
)
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
import os
|
||||
import unittest
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from backend.test import FIXTURES_ROOT
|
||||
from backend.server.converters.schema import gene_symbol
|
||||
|
||||
|
||||
class TestHGNCSymbolChecker(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz")
|
||||
self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path)
|
||||
|
||||
def test_symbol_upgrade(self):
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2")
|
||||
|
||||
# Strip off seurat endings when appropriate
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2")
|
||||
|
||||
# DIFF6 is ambiguous so don't upgrade it
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6")
|
||||
|
||||
# ARG1 is approved
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1")
|
||||
|
||||
# HAP1 is both approved and withdrawn
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1")
|
||||
|
||||
# Leave unknown symbols alone
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol")
|
||||
|
||||
# Upgrade HGNC ids unless you can't find it
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1")
|
||||
self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456")
|
||||
|
||||
def test_check_symbol(self):
|
||||
self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE)
|
||||
self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS)
|
||||
self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN)
|
||||
|
||||
# HAP1 is one of the approved and withdrawn symbols
|
||||
self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED)
|
||||
|
||||
def test_upgrade_index(self):
|
||||
index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"])
|
||||
var_df = pd.DataFrame([[0] * len(index)], index=index)
|
||||
upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path)
|
||||
self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"])
|
||||
@@ -1,128 +0,0 @@
|
||||
import json
|
||||
|
||||
import unittest.mock
|
||||
|
||||
from backend.server.converters.schema import ontology
|
||||
|
||||
|
||||
class TestOntologyParsing(unittest.TestCase):
|
||||
def setUp(self):
|
||||
|
||||
self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"]
|
||||
|
||||
self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"]
|
||||
|
||||
self.values = ["0002048", "0000174", "9606", "0008995"]
|
||||
|
||||
self.iris = [
|
||||
"http://purl.obolibrary.org/obo/UBERON_0002048",
|
||||
"http://purl.obolibrary.org/obo/HsapDv_0000174",
|
||||
"http://purl.obolibrary.org/obo/NCBITaxon_9606",
|
||||
"http://www.ebi.ac.uk/efo/EFO_0008995",
|
||||
]
|
||||
|
||||
URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/"
|
||||
self.urls = [
|
||||
URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048",
|
||||
URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174",
|
||||
URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606",
|
||||
URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995",
|
||||
]
|
||||
|
||||
self.responses = {
|
||||
"UBERON:0002048": {
|
||||
"iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
|
||||
"description": ["Respiration organ that develops as an outpocketing of the esophagus."],
|
||||
"label": "lung",
|
||||
},
|
||||
"HsapDv:0000174": {
|
||||
"iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
|
||||
"description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
|
||||
"label": "1-month-old human stage",
|
||||
},
|
||||
"NCBITaxon:9606": {
|
||||
"iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
|
||||
"description": None,
|
||||
"label": "Homo sapiens",
|
||||
},
|
||||
"EFO:0008995": {
|
||||
"iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
|
||||
"description": [
|
||||
(
|
||||
'10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
|
||||
"gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent "
|
||||
"of 1 million pipetting steps. Successive versions of the 10x chemistry use different "
|
||||
"barcode locations to improve the sequencing yield and quality of 10x experiments."
|
||||
)
|
||||
],
|
||||
"label": "10X sequencing",
|
||||
},
|
||||
}
|
||||
|
||||
def test_ontololgy_name(self):
|
||||
for curie, expected_name in zip(self.curies, self.names):
|
||||
self.assertEqual(ontology._ontology_name(curie), expected_name)
|
||||
|
||||
def test_ontololgy_value(self):
|
||||
for curie, expected_value in zip(self.curies, self.values):
|
||||
self.assertEqual(ontology._ontology_value(curie), expected_value)
|
||||
|
||||
def test_iri(self):
|
||||
for curie, expected_iri in zip(self.curies, self.iris):
|
||||
self.assertEqual(ontology._iri(curie), expected_iri)
|
||||
|
||||
def test_ontology_info_url(self):
|
||||
for curie, expected_url in zip(self.curies, self.urls):
|
||||
self.assertEqual(ontology._ontology_info_url(curie), expected_url)
|
||||
|
||||
def test_empty_ontology_info_url(self):
|
||||
self.assertEqual(ontology._ontology_info_url(""), "")
|
||||
|
||||
|
||||
class TestOntologyLookup(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.responses = {
|
||||
"UBERON:0002048": {
|
||||
"iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
|
||||
"description": ["Respiration organ that develops as an outpocketing of the esophagus."],
|
||||
"label": "lung",
|
||||
},
|
||||
"HsapDv:0000174": {
|
||||
"iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
|
||||
"description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
|
||||
"label": "1-month-old human stage",
|
||||
},
|
||||
"NCBITaxon:9606": {
|
||||
"iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
|
||||
"description": None,
|
||||
"label": "Homo sapiens",
|
||||
},
|
||||
"EFO:0008995": {
|
||||
"iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
|
||||
"description": [
|
||||
('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
|
||||
'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent '
|
||||
'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode '
|
||||
'locations to improve the sequencing yield and quality of 10x experiments.')
|
||||
],
|
||||
"label": "10X sequencing",
|
||||
},
|
||||
}
|
||||
|
||||
self.labels = {
|
||||
"UBERON:0002048": "lung",
|
||||
"HsapDv:0000174": "1-month-old human stage",
|
||||
"NCBITaxon:9606": "Homo sapiens",
|
||||
"EFO:0008995": "10X sequencing",
|
||||
}
|
||||
|
||||
@unittest.mock.patch("requests.get")
|
||||
def test_lookup_label(self, mock_get):
|
||||
|
||||
for curie, response in self.responses.items():
|
||||
mock_get.return_value.content = json.dumps(response)
|
||||
mock_get.return_value.json.return_value = response
|
||||
mock_get.return_value.status_code = 200
|
||||
|
||||
label = ontology.get_ontology_label(curie)
|
||||
self.assertEqual(label, self.labels[curie])
|
||||
@@ -1,256 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import unittest
|
||||
import unittest.mock
|
||||
|
||||
import anndata
|
||||
import numpy
|
||||
import pandas as pd
|
||||
import scanpy as sc
|
||||
|
||||
from backend.server.converters.schema import remix
|
||||
from backend.test import PROJECT_ROOT
|
||||
|
||||
|
||||
class TestApplySchema(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.source_h5ad_path = f"{PROJECT_ROOT}/backend/test/fixtures/pbmc3k-CSC-gz.h5ad"
|
||||
self.output_h5ad_path = f"{PROJECT_ROOT}/backend/test/fixtures/test_remix.h5ad"
|
||||
self.config_path = f"{PROJECT_ROOT}/backend/test/fixtures/test_config.yaml"
|
||||
self.bad_config_path = f"{PROJECT_ROOT}/backend/test/fixtures/test_bad_config.yaml"
|
||||
|
||||
def tearDown(self):
|
||||
try:
|
||||
os.remove(self.output_h5ad_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
|
||||
def test_apply_schema(self, mock_get_ontology_label):
|
||||
mock_get_ontology_label.return_value = "test label"
|
||||
remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path)
|
||||
new_adata = sc.read_h5ad(self.output_h5ad_path)
|
||||
|
||||
self.assertIn("cell_type", new_adata.obs.columns)
|
||||
self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist())
|
||||
self.assertListEqual(
|
||||
["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"],
|
||||
sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist())
|
||||
)
|
||||
|
||||
self.assertIn("version", new_adata.uns_keys())
|
||||
|
||||
@unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
|
||||
def test_apply_bad_schema(self, mock_get_ontology_label):
|
||||
mock_get_ontology_label.return_value = "test label"
|
||||
remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path)
|
||||
new_adata = sc.read_h5ad(self.output_h5ad_path)
|
||||
|
||||
# Should refuse to write the version
|
||||
self.assertNotIn("version", new_adata.uns_keys())
|
||||
|
||||
class TestFieldParsing(unittest.TestCase):
|
||||
|
||||
def test_is_curie(self):
|
||||
self.assertTrue(remix.is_curie("EFO:00001"))
|
||||
self.assertTrue(remix.is_curie("UBERON:123456"))
|
||||
self.assertTrue(remix.is_curie("HsapDv:0001"))
|
||||
self.assertFalse(remix.is_curie("UBERON"))
|
||||
self.assertFalse(remix.is_curie("UBERON:"))
|
||||
self.assertFalse(remix.is_curie("123456"))
|
||||
|
||||
def test_is_ontology_field(self):
|
||||
self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id"))
|
||||
self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id"))
|
||||
self.assertFalse(remix.is_ontology_field("cell_ontology"))
|
||||
self.assertFalse(remix.is_ontology_field("method"))
|
||||
|
||||
def test_get_label_field_name(self):
|
||||
self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id"))
|
||||
self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id"))
|
||||
|
||||
def test_split_suffix(self):
|
||||
self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)"))
|
||||
self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)"))
|
||||
self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234"))
|
||||
self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)"))
|
||||
|
||||
@unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
|
||||
def test_get_curie_and_label(self, mock_get_ontology_label):
|
||||
mock_get_ontology_label.return_value = "test label"
|
||||
self.assertEqual(
|
||||
remix.get_curie_and_label("UBERON:1234"),
|
||||
("UBERON:1234", "test label")
|
||||
)
|
||||
self.assertEqual(
|
||||
remix.get_curie_and_label("UBERON:1234 (cell culture)"),
|
||||
("UBERON:1234 (cell culture)", "test label (cell culture)")
|
||||
)
|
||||
self.assertEqual(
|
||||
remix.get_curie_and_label("whatever"),
|
||||
("", "whatever")
|
||||
)
|
||||
|
||||
|
||||
class TestManipulateAnndata(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
||||
self.cell_count = 20
|
||||
self.gene_count = 200
|
||||
X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count))
|
||||
uns = {"organism": "monkey", "experiment": "monkey experiment"}
|
||||
obs = pd.DataFrame(
|
||||
index=[f"Cell{d}" for d in range(self.cell_count)],
|
||||
columns=["tissue", "CellType"],
|
||||
data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2)
|
||||
)
|
||||
var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)])
|
||||
|
||||
self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns)
|
||||
|
||||
def test_safe_add_field(self):
|
||||
|
||||
remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count)
|
||||
self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count)
|
||||
self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count)
|
||||
|
||||
remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}])
|
||||
self.assertEqual(
|
||||
self.adata.uns["contributors"],
|
||||
json.dumps([{"name": "contributor1"}, {"name": "contributor2"}])
|
||||
)
|
||||
|
||||
@unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
|
||||
def test_remix_uns(self, mock_get_ontology_label):
|
||||
mock_get_ontology_label.return_value = "Pan troglodytes"
|
||||
uns_config = {
|
||||
"version": {
|
||||
"corpora_schema_version": "1.0.0",
|
||||
"corpora_encoding_version": "0.1.0"
|
||||
},
|
||||
"organism_ontology_term_id": "NCBITaxon:9598",
|
||||
"contributors": [
|
||||
{
|
||||
"name": "scientist",
|
||||
"email": "scientist@science.com"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
remix.remix_uns(self.adata, uns_config)
|
||||
|
||||
self.assertEqual(
|
||||
sorted(self.adata.uns_keys()),
|
||||
sorted(["organism_original", "organism", "organism_ontology_term_id",
|
||||
"contributors", "version", "experiment"])
|
||||
)
|
||||
|
||||
self.assertEqual(self.adata.uns['organism'], "Pan troglodytes")
|
||||
self.assertEqual(self.adata.uns['organism_original'], "monkey")
|
||||
self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598")
|
||||
self.assertEqual(self.adata.uns['contributors'],
|
||||
json.dumps([{"name": "scientist", "email": "scientist@science.com"}]))
|
||||
|
||||
@unittest.mock.patch("backend.server.converters.schema.ontology.get_ontology_label")
|
||||
def test_remix_obs(self, mock_get_ontology_label):
|
||||
mock_get_ontology_label.return_value = "lung (in a monkey)"
|
||||
obs_config = {
|
||||
"tissue_ontology_term_id": {
|
||||
"tissue": {
|
||||
"lung": "UBERON:00000"
|
||||
}
|
||||
},
|
||||
"cell_color": {
|
||||
"CellType": {
|
||||
"epithelial": "fuschia",
|
||||
"endothelial": "khaki"
|
||||
}
|
||||
},
|
||||
"sex": "male"
|
||||
}
|
||||
|
||||
remix.remix_obs(self.adata, obs_config)
|
||||
self.assertEqual(
|
||||
sorted(self.adata.obs_keys()),
|
||||
sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"])
|
||||
)
|
||||
|
||||
self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original))
|
||||
self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id))
|
||||
self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue))
|
||||
self.assertTrue(all(v == "male" for v in self.adata.obs.sex))
|
||||
self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki"))
|
||||
for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color)))
|
||||
|
||||
|
||||
class TestFixupGeneSymbols(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.seurat_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/seurat_tutorial.h5ad"
|
||||
self.seurat_merged_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad"
|
||||
self.sctransform_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/sctransform.h5ad"
|
||||
self.sctransform_merged_path = f"{PROJECT_ROOT}/server/test/fixtures/schema_test_data/sctransform_merged.h5ad"
|
||||
|
||||
# There's lots of MALAT1, but it doesn't collide with any other names,
|
||||
# so it shouldn't change during merging.
|
||||
self.stable_gene = "MALAT1"
|
||||
|
||||
def test_fixup_gene_symbols_seurat(self):
|
||||
|
||||
if not os.path.isfile(self.seurat_path):
|
||||
return unittest.skip(
|
||||
"Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
|
||||
"run server/test/fixtures/schema_test_data/generate_test_data.sh"
|
||||
)
|
||||
|
||||
original_adata = sc.read_h5ad(self.seurat_path)
|
||||
merged_adata = sc.read_h5ad(self.seurat_merged_path)
|
||||
|
||||
fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"}
|
||||
|
||||
fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
|
||||
|
||||
self.assertEqual(
|
||||
merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(),
|
||||
fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
|
||||
)
|
||||
self.assertAlmostEqual(
|
||||
merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
|
||||
fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum()
|
||||
)
|
||||
|
||||
self.assertAlmostEqual(
|
||||
merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(),
|
||||
fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum()
|
||||
)
|
||||
|
||||
def test_fixup_gene_symbols_sctransform(self):
|
||||
|
||||
if not os.path.isfile(self.sctransform_path):
|
||||
return unittest.skip(
|
||||
"Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
|
||||
"run server/test/fixtures/schema_test_data/generate_test_data.sh"
|
||||
)
|
||||
|
||||
original_adata = sc.read_h5ad(self.sctransform_path)
|
||||
merged_adata = sc.read_h5ad(self.sctransform_merged_path)
|
||||
|
||||
fixup_config = {"X": "log1p", "counts": "raw"}
|
||||
|
||||
fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
|
||||
|
||||
# sctransform does a bunch of stuff, including slightly modifying the
|
||||
# raw counts. So we can't assert for exact equality the way we do with
|
||||
# the vanilla seurat tutorial. But, the results should still be very
|
||||
# close.
|
||||
merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum()
|
||||
fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
|
||||
self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable)
|
||||
|
||||
self.assertAlmostEqual(
|
||||
merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
|
||||
fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(),
|
||||
0
|
||||
)
|
||||
@@ -1,434 +0,0 @@
|
||||
import json
|
||||
import unittest
|
||||
|
||||
import pandas as pd
|
||||
import scanpy as sc
|
||||
|
||||
from backend.server.converters.schema import validate
|
||||
|
||||
from backend.test import PROJECT_ROOT
|
||||
|
||||
|
||||
class TestFieldValidation(unittest.TestCase):
|
||||
|
||||
def test_validate_stringified_list_of_dicts(self):
|
||||
|
||||
good = json.dumps([{"a": 1}, {2: "x", "z": "y"}])
|
||||
not_stringified = [{"a": 1}, {2: "x", "z": "y"}]
|
||||
not_a_list = json.dumps({"bad": "dict"})
|
||||
not_json = "oh hey!"
|
||||
|
||||
self.assertTrue(validate._validate_stringified_list_of_dicts(good))
|
||||
|
||||
self.assertFalse(validate._validate_stringified_list_of_dicts(not_stringified))
|
||||
self.assertFalse(validate._validate_stringified_list_of_dicts(not_a_list))
|
||||
self.assertFalse(validate._validate_stringified_list_of_dicts(not_json))
|
||||
|
||||
def test_validate_human_readable_string(self):
|
||||
|
||||
good = "oh hey!"
|
||||
curie = "EFO:0001"
|
||||
ensg = "ENSG000001234"
|
||||
enst = "ENST000005678"
|
||||
|
||||
self.assertTrue(validate._validate_human_readable_string(good))
|
||||
|
||||
self.assertFalse(validate._validate_human_readable_string(curie))
|
||||
self.assertFalse(validate._validate_human_readable_string(ensg))
|
||||
self.assertFalse(validate._validate_human_readable_string(enst))
|
||||
|
||||
def test_validate_curie(self):
|
||||
|
||||
self.assertTrue(validate._validate_curie("UBERON:00001", ["UBERON", "EFO"]))
|
||||
self.assertTrue(validate._validate_curie("HsapDv:00002", ["HsapDv"]))
|
||||
|
||||
self.assertFalse(validate._validate_curie("HsapDv:00002", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_curie("EFO:00002 (organoid)", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_curie("EFO:00002 extra", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_curie("UBERON:ABCD", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_curie("Uberon:00002", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_curie("UBERON:", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_curie("UBERON", ["UBERON", "EFO"]))
|
||||
|
||||
def test_validate_suffixed_curie(self):
|
||||
|
||||
self.assertTrue(validate._validate_suffixed_curie("EFO:00001", ["UBERON", "EFO"]))
|
||||
self.assertTrue(validate._validate_suffixed_curie("UBERON:00001 (cell culture)", ["UBERON", "EFO"]))
|
||||
|
||||
self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002 (organoid)", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002(organoid)", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_suffixed_curie("EFO:00002 extra", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_suffixed_curie("UBERON:ABCD", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_suffixed_curie("Uberon:00002", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_suffixed_curie("UBERON:", ["UBERON", "EFO"]))
|
||||
self.assertFalse(validate._validate_suffixed_curie("UBERON", ["UBERON", "EFO"]))
|
||||
|
||||
|
||||
class TestColumnValidation(unittest.TestCase):
|
||||
|
||||
def test_validate_unique(self):
|
||||
unique = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
|
||||
index=["X", "Y", "Z"], columns=["col1", "col2"])
|
||||
duped = pd.DataFrame([["abc", "def"], ["ghi", "qrs"], ["abc", "qrs"]],
|
||||
index=["X", "Y", "X"], columns=["col1", "col2"])
|
||||
|
||||
schema_def = {"unique": True}
|
||||
|
||||
errors = validate._validate_column(unique.index, "index", "unique_df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
errors = validate._validate_column(duped.index, "index", "duped_df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("is not unique", errors[0])
|
||||
|
||||
errors = validate._validate_column(unique["col1"], "col1", "unique_df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("is not unique", errors[0])
|
||||
|
||||
schema_def = {"unique": False}
|
||||
errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
def test_validate_nullable(self):
|
||||
non_null = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
|
||||
index=["X", "Y", "Z"], columns=["col1", "col2"])
|
||||
has_null = pd.DataFrame([["abc", "", None], ["ghi", "jkl", 1], ["mnop", "qrs", 2]],
|
||||
index=["X", "Y", "Z"], columns=["col1", "col2", "col3"])
|
||||
|
||||
schema_def = {"nullable": False}
|
||||
errors = validate._validate_column(non_null["col1"], "col1", "nonnull_df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
errors = validate._validate_column(has_null["col1"], "col1", "hasnull_df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("contains empty values", errors[0])
|
||||
errors = validate._validate_column(has_null["col3"], "col3", "hasnull_df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("contains empty values", errors[0])
|
||||
|
||||
schema_def = {"nullable": True}
|
||||
errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
def test_human_readable(self):
|
||||
hr_df = pd.DataFrame(
|
||||
[["for you, a human", "UBERON:12345", "UBERON:1234 (thundercat)"],
|
||||
["hope you're well", "bit of lungs", "brain"]],
|
||||
index=["ENSG00001", "ENSG00002"],
|
||||
columns=["good", "curie", "suffixed_curie"])
|
||||
|
||||
schema_def = {"type": "human-readable string"}
|
||||
errors = validate._validate_column(hr_df["good"], "good", "hr", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
errors = validate._validate_column(hr_df["curie"], "curie", "hr", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("non-human-readable", errors[0])
|
||||
|
||||
errors = validate._validate_column(hr_df["suffixed_curie"], "suffixed_curie", "hr", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("non-human-readable", errors[0])
|
||||
|
||||
errors = validate._validate_column(hr_df.index, "ensg", "hr", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("non-human-readable", errors[0])
|
||||
|
||||
def test_curie(self):
|
||||
|
||||
curie_df = pd.DataFrame(
|
||||
[["EFO:00001", "HsapDv:00001 (cell culture)", "EFO:", "MONDO:0001 cell culture"],
|
||||
["UBERON:00002", "HsapDv:00002 (organoid)", "EFO:12345", "MONDO:0002 (baba yaga)"],
|
||||
["EFO:0000000005", "HsapDv:000004 (humanzee)", "EFO:000002", "MONDO:0004 (TMNT)"]],
|
||||
index=["X", "Y", "Z"],
|
||||
columns=["good", "good_suffix", "bad", "bad_suffix"])
|
||||
|
||||
# Good
|
||||
schema_def = {"type": "curie", "prefixes": ["EFO", "UBERON"]}
|
||||
errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
# Good suffix
|
||||
schema_def = {"type": "suffixed curie", "prefixes": ["HsapDv", "WHATEVER"]}
|
||||
errors = validate._validate_column(curie_df["good_suffix"], "good_suffix", "curie_df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
# Bad prefix
|
||||
schema_def = {"type": "curie", "prefixes": ["EFO"]}
|
||||
errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("invalid ontology", errors[0])
|
||||
self.assertIn("must be curies from one of these", errors[0])
|
||||
|
||||
# Bad curies
|
||||
schema_def = {"type": "curie", "prefixes": ["EFO"]}
|
||||
errors = validate._validate_column(curie_df["bad"], "bad", "curie_df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("invalid ontology", errors[0])
|
||||
|
||||
# Bad suffixes
|
||||
schema_def = {"type": "suffixed curie", "prefixes": ["EFO"]}
|
||||
errors = validate._validate_column(curie_df["bad_suffix"], "bad_suffix", "curie_df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("invalid ontology", errors[0])
|
||||
|
||||
def test_enum(self):
|
||||
enum_df = pd.DataFrame(
|
||||
[["abc", "ghi"],
|
||||
["def", "jkl"]],
|
||||
index=["X", "Y"],
|
||||
columns=["col1", "col2"])
|
||||
|
||||
# All match
|
||||
schema_def = {"type": "string", "enum": ["abc", "def", "xyz"]}
|
||||
errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
# Missing value
|
||||
schema_def = {"type": "string", "enum": ["abc", "xyz"]}
|
||||
errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("unpermitted values", errors[0])
|
||||
|
||||
|
||||
class TestDictValidations(unittest.TestCase):
|
||||
|
||||
|
||||
def test_key_presence(self):
|
||||
|
||||
schema_def = {"keys": {"abc": None, "def": None}}
|
||||
|
||||
dict_ = {"abc": "123", "def": "456"}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
# Missing keys are bad
|
||||
dict_ = {"abc": "123"}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("missing key", errors[0])
|
||||
|
||||
# Extra keys are okay
|
||||
dict_ = {"abc": "123", "def": "456", "xyz": "789"}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
# Better not be empty come on
|
||||
dict_ = {}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertEqual(len(errors), 2)
|
||||
|
||||
def test_nullable(self):
|
||||
|
||||
schema_def = {"keys": {"abc": {"type": "string", "nullable": False},
|
||||
"def": {"type": "string", "nullable": True}}}
|
||||
|
||||
dict_ = {"abc": "xyz", "def": ""}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
dict_ = {"abc": "", "def": ""}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("empty value", errors[0])
|
||||
|
||||
def test_recurse(self):
|
||||
|
||||
schema_def = {
|
||||
"keys": {
|
||||
"subdict": {
|
||||
"type": "dict",
|
||||
"keys": {
|
||||
"subdict_key1": None,
|
||||
"subdict_key2": None
|
||||
}
|
||||
},
|
||||
"ontology": {
|
||||
"type": "curie",
|
||||
"prefixes": ["ONTOLOGY"]
|
||||
},
|
||||
"blob": {
|
||||
"type": "stringified list of dicts"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dict_ = {
|
||||
"subdict": {"subdict_key1": "any", "subdict_key2": "any"},
|
||||
"ontology": "ONTOLOGY:123456",
|
||||
"blob": json.dumps([{"abc": 123}, {"def": 456}])
|
||||
}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
dict_ = {
|
||||
"subdict": {"subdict_key1": "any"},
|
||||
"ontology": "ONTOLOGY:123456",
|
||||
"blob": json.dumps([{"abc": 123}, {"def": 456}])
|
||||
}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("missing key", errors[0])
|
||||
|
||||
dict_ = {
|
||||
"subdict": {"subdict_key1": "any", "subdict_key2": "any"},
|
||||
"ontology": "oh no not an ontology term",
|
||||
"blob": json.dumps([{"abc": 123}, {"def": 456}])
|
||||
}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("invalid ontology", errors[0])
|
||||
|
||||
dict_ = {
|
||||
"subdict": {"subdict_key1": "any", "subdict_key2": "any"},
|
||||
"ontology": "ONTOLOGY:123456",
|
||||
"blob": [{"abc": 123}, {"def": 456}]
|
||||
}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("JSON-encoded list of dicts", errors[0])
|
||||
|
||||
# Multiple errors
|
||||
dict_ = {
|
||||
"subdict": {"subdict_key1": "any"},
|
||||
"ontology": "oh no not an ontology term",
|
||||
"blob": json.dumps([{"abc": 123}, {"def": 456}])
|
||||
}
|
||||
errors = validate._validate_dict(dict_, "d", schema_def)
|
||||
self.assertEqual(len(errors), 2)
|
||||
|
||||
|
||||
class TestDataframeValidation(unittest.TestCase):
|
||||
|
||||
def test_column_presence(self):
|
||||
df = pd.DataFrame(
|
||||
[["abc", "EFO:123"],
|
||||
["def", "UBERON:456"]],
|
||||
columns=["hr_string", "ontology"],
|
||||
index=["X", "Y"]
|
||||
)
|
||||
|
||||
schema_def = {
|
||||
"columns": {
|
||||
"hr_string": {"type": "human-readable string"},
|
||||
"ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
|
||||
}
|
||||
}
|
||||
errors = validate._validate_dataframe(df, "df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
schema_def = {
|
||||
"columns": {
|
||||
"hr_string": {"type": "human-readable string"},
|
||||
"another_hr_string": {"type": "human-readable string"},
|
||||
"ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
|
||||
}
|
||||
}
|
||||
errors = validate._validate_dataframe(df, "df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("missing column", errors[0])
|
||||
|
||||
# Extra is okay
|
||||
df = pd.DataFrame(
|
||||
[["abc", "EFO:123", "extra"],
|
||||
["def", "UBERON:456", "extra"]],
|
||||
columns=["hr_string", "ontology", "extra"],
|
||||
index=["X", "Y"]
|
||||
)
|
||||
schema_def = {
|
||||
"columns": {
|
||||
"hr_string": {"type": "human-readable string"},
|
||||
"ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
|
||||
}
|
||||
}
|
||||
errors = validate._validate_dataframe(df, "df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
|
||||
def test_index(self):
|
||||
df = pd.DataFrame(
|
||||
[["abc", "123"],
|
||||
["def", "456"]],
|
||||
columns=["col1", "col2"],
|
||||
index=["ENSG0001", "ENSG0002"]
|
||||
)
|
||||
|
||||
schema_def = {"index": {"unique": True}}
|
||||
errors = validate._validate_dataframe(df, "df", schema_def)
|
||||
self.assertFalse(errors)
|
||||
|
||||
schema_def = {"index": {"type": "human-readable string"}}
|
||||
errors = validate._validate_dataframe(df, "df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("non-human-readable", errors[0])
|
||||
|
||||
df = pd.DataFrame(
|
||||
[["abc", "123"],
|
||||
["def", "456"]],
|
||||
columns=["col1", "col2"],
|
||||
index=["ENSG0001", "ENSG0001"]
|
||||
)
|
||||
schema_def = {"index": {"unique": True}}
|
||||
errors = validate._validate_dataframe(df, "df", schema_def)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("is not unique", errors[0])
|
||||
|
||||
def test_recurse(self):
|
||||
|
||||
df = pd.DataFrame(
|
||||
[["abc", "HsapDv:0001"],
|
||||
["EFO:123", "UBERON:456"]],
|
||||
columns=["hr_string", "ontology"],
|
||||
index=["X", "Y"]
|
||||
)
|
||||
schema_def = {
|
||||
"columns": {
|
||||
"hr_string": {"type": "human-readable string"},
|
||||
"ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
|
||||
}
|
||||
}
|
||||
errors = validate._validate_dataframe(df, "df", schema_def)
|
||||
self.assertEqual(len(errors), 2)
|
||||
self.assertEqual(len([e for e in errors if "non-human-readable" in e]), 1)
|
||||
self.assertEqual(len([e for e in errors if "invalid ontology" in e]), 1)
|
||||
|
||||
|
||||
class TestGetSchema(unittest.TestCase):
|
||||
|
||||
def test_get_schema(self):
|
||||
self.assertIsInstance(validate.get_schema_definition("1.0.0"), dict)
|
||||
|
||||
with self.assertRaises(ValueError):
|
||||
validate.get_schema_definition("10.1.5")
|
||||
|
||||
|
||||
class TestValidate(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.source_h5ad_path = f"{PROJECT_ROOT}/backend/test/fixtures/pbmc3k-CSC-gz.h5ad"
|
||||
|
||||
def test_shallow(self):
|
||||
|
||||
adata = sc.read_h5ad(self.source_h5ad_path)
|
||||
self.assertFalse(validate.validate_adata(adata, True))
|
||||
|
||||
adata.uns["version"] = {
|
||||
"corpora_schema_version": "1.0.0",
|
||||
"corpora_encoding_version": "0.1.0"
|
||||
}
|
||||
self.assertTrue(validate.validate_adata(adata, True))
|
||||
|
||||
def test_deep(self):
|
||||
adata = sc.read_h5ad(self.source_h5ad_path)
|
||||
self.assertFalse(validate.validate_adata(adata, False))
|
||||
|
||||
adata.uns["version"] = {
|
||||
"corpora_schema_version": "1.0.0",
|
||||
"corpora_encoding_version": "0.1.0"
|
||||
}
|
||||
self.assertFalse(validate.validate_adata(adata, False))
|
||||
@@ -10,7 +10,6 @@
|
||||
"dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
|
||||
"e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
|
||||
"e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
|
||||
"e2e-prod": "CXG_URL_BASE='https://cellxgene.cziscience.com/d/pbmc3k.cxg/' jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
|
||||
"fmt": "eslint --fix src __tests__",
|
||||
"lint": "eslint --fix src __tests__",
|
||||
"prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
|
||||
|
||||
164
dev_docs/cxg.md
164
dev_docs/cxg.md
@@ -1,164 +0,0 @@
|
||||
## UPDATE (9/30/2020): Starting today, the name Corpora will only be used as the internal project name, with cellxgene Data Portal being the official product name
|
||||
|
||||
# CXG Data Format Specification
|
||||
|
||||
Document Status: _draft_
|
||||
|
||||
Version: 0.2.0 (_DRAFT, not yet approved_)
|
||||
|
||||
Date Last Modified: 2020-07-23
|
||||
|
||||
## Introduction
|
||||
|
||||
CXG is a cellxgene-private data format, used for at-rest storage of annotated matrix data. It is similar to [AnnData](https://anndata.readthedocs.io/en/stable/), but with performance and access characteristics amenable to a multi-dataset, multi-user serving environment.
|
||||
|
||||
CXG is built upon the [TileDB](https://tiledb.com/) embedded database. Each CXG is a TileDB [group](https://docs.tiledb.com/main/api-usage/object-management), which in turn includes one or more TileDB multi-dimensional arrays.
|
||||
|
||||
This document presumes familiarity with [TileDB terminology and concepts](https://docs.tiledb.com/main/), the [Corpora schema](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md) and its [H5AD encoding](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md), and the AnnData/H5AD data model.
|
||||
|
||||
This document also leverages the current cellxgene schema, which is documented in the [REST API spec](./REST_API.md).
|
||||
|
||||
### Terminology
|
||||
|
||||
Unless explicitly noted, the AnnData conventions and terminology are adopted when referring to general annotated matrix characteristics (eg, `n_obs` is the number of observations/rows/cells in the annotated matrix). Where implied by context, eg, "TileDB array attribute", domain-specific terms are used.
|
||||
|
||||
Where capitalized, [IETF RFC 2119](https://www.ietf.org/rfc/rfc2119.txt) conventions are followed (ie, conventions MUST be followed).
|
||||
|
||||
_Author's note:_ if you see any ambiguous terms, please call them out for clarification.
|
||||
|
||||
### Reserved
|
||||
|
||||
The `cxg` prefix is used for CXG-specific names.
|
||||
|
||||
### Encoding Data With TileDB Arrays
|
||||
|
||||
The TileDB array schema authoritatively defines the characteristics of each array (eg, the type of `X` is defined by [`X.schema`](https://tiledb-inc-tiledb-py.readthedocs-hosted.com/en/stable/python-api.html#tiledb.libtiledb.Array.schema)). In some cases, additional metadata is required for the CXG, and is attched to the array using the TileDB [array metadata](https://docs.tiledb.com/main/basic-concepts/array-metadata) capability.
|
||||
|
||||
All TileDB arrays MUST have a uint32 domain, zero based. All X counts and embedding coordinates SHOULD be coerced to float32, which is ample precision for visualization purposes, and MUST be a numeric type. Dataframe (metadata) types are generally preserved, or where that is not possible, converted to something with equal representative value in the cellxgene application (eg, categorical types are converted to string, bools to uint8, etc).
|
||||
|
||||
CXG consumers (readers) MUST be prepared to handle any legal TileDB compression, global layout and tile size. CXG writers SHOULD attempt to encode data using best-effort heuristics for time and space considerations (eg, dense/sparse encoding tradeoffs).
|
||||
|
||||
## Entities
|
||||
|
||||
### CXG
|
||||
|
||||
The CXG is a TileDB group containing all data and metadata for a single annotated matrix. The following objects MUST be present in a CXG, except where noted as optional:
|
||||
* __obs__: a TileDB array, of shape (n_obs,), containing obs annotations, each annotation stored in a separate TileDB array attribute.
|
||||
* __var__: a TileDB array, of shape (n_var,), containing var annotations, each annotation stored in a separate TileDB array attribute.
|
||||
* __X__: a TileDB array, of shape (n_obs, n_var), with a single TileDB attribute of numeric type.
|
||||
* __X_col_shift__: (optional) TilebDB Array used in column shift encoding, shape (n_var,), dtype = X.dtype. Single unnamed numeric attribute.
|
||||
* __emb__: a TileDB group, which in turn contains all (zero or more) embeddings.
|
||||
* __emb__/\<embedding_name\>__: a TileDB array, with a single anonymous attribute, of numeric type, and shape (n_obs, N>=2).
|
||||
* __cxg_group_metadata__: an empty TileDB array, used to store CXG-wide metadata
|
||||
|
||||
### obs and var
|
||||
|
||||
All per-observation (obs) and per-feature (var) data is encoded in a TileDB array named `obs` and `var` respectively, with shape (n_obs,) and (n_var,). Each TileDB array has an array attribute for each obs/var column. All TileDB array attributes will have the same type and value as the original data, eg, float32, with the following exceptions:
|
||||
* bool is encoded as uint8 (1/0)
|
||||
* categorical is encoded as string
|
||||
* Numeric types are cast to 32-bit equivalents
|
||||
|
||||
In addition to the obs/var data, both TileDB arrays contain an optional 'cxg_schema' metadata field that is a JSON string containing per-column (attribute) schema hinting. This is used where the TileDB native typing information is insufficient to reconstruct useful information such as categorical typing from Pandas DataFrames, and to communicate which column is the preferred human-readable index for obs & var.
|
||||
|
||||
The `cxg_schema` JSON string is attached to the TileDB array metadata, and is a dictionary containing the following top-level names:
|
||||
* "index": string, containing the name of the index column
|
||||
* \<column-name\>: optional, a JSON dict, contain a schema definition using the same format as the cellxgene REST API /schema route
|
||||
|
||||
For example:
|
||||
```
|
||||
{
|
||||
"index": "obs_index",
|
||||
"louvain": { "type": "categorical", "categories": [ "0", "1", "2", "3", "4" ]}
|
||||
"is_useful": { "type": "boolean" }
|
||||
}
|
||||
```
|
||||
|
||||
### X
|
||||
|
||||
TileDB array, with a single anonymous attribute, shape (n_obs, n_var), containing the count matrix (equivalent to the AnnData `X` array). MUST have numeric type, and SHOULD be float32. The TileDB schema defines type and sparsity, and both dense and sparse encoding are supported.
|
||||
|
||||
### X_col_shift
|
||||
|
||||
Optional TileDB array, used to encode-per column offsets for column-shift sparse encoding. The TileDB array will have a single anonymous attribute, of the same type as the X array, and shape (n_var,).
|
||||
|
||||
If the X array is sparse, and X_col_shift exists, then all values in the i'th column were subtracted by X_col_shift[i].
|
||||
|
||||
### emb and embedding arrays
|
||||
|
||||
A CXG must have a group named `emb`, which will contain all embeddings. Embeddings are encoded as TileDB arrays, of numeric type and shape (n_obs, >=2). The arrays SHOULD be coerced to float32, and MUST be a numeric type. The TileDB array name will be assumed to be the embedding name (conventionally, embedding names in CXG are _not_ prefixed with an `X_` as they are in AnnData).
|
||||
|
||||
CXG supports zero or more embeddings. Note that cellxgene currently _requires_ at least one embedding.
|
||||
|
||||
### cxg_group_metadata
|
||||
|
||||
Required, but empty TileDB array, used to store CXG-wide metadata. The following fields are defined:
|
||||
* __cxg_version__: (required) a semver string identifying the specification version used to encode the CXG.
|
||||
* __cxg_properties__: (optional) a dictionary containing dataset wide properties, defined below.
|
||||
* __cxg_category_colors__: (optional) a categorical color table, defined below.
|
||||
|
||||
#### cxg_properties
|
||||
|
||||
The properties metadata dictionary contains dataset-wide properties, encoded as a JSON dictionary. Currently, the following fields are defined:
|
||||
* title: string, dataset human name (eg, "Lung Tissue")
|
||||
* about: string, fully-qualified http/https URL, linking to more information on the dataset.
|
||||
|
||||
All implementions MUST ignore unrecognized fields.
|
||||
|
||||
#### cxg_category_colors
|
||||
|
||||
This optional field contains a copy of the category color table, which MAY be used to display category-specific color labels. This is a JSON dictionary, containing a per-category color-table. Each color table is named `{category_name}_colors`, and is itself a dictionary mapping label name to RGB color. For example:
|
||||
|
||||
```
|
||||
{
|
||||
"louvain_colors": {
|
||||
"0": "#FFFFFF",
|
||||
"1": "#000000"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Corpora Schema Encoding
|
||||
|
||||
The [Corpora schema](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md) and [Corpora AnnData encoding](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md) define a set of metadata and encoding conventions for annotated matrices. When a Corpora dataset is encoded as a CXG, the following shall apply.
|
||||
|
||||
### Corpora metadata property
|
||||
|
||||
A CXG containing a Corpora dataset will contain a property in the __cxg_group_metadata__ field named `corpora`. The value will be a JSON encoded string, which in turn contains all properties defined in the [Corpora AnnData uns](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md#uns) container. For example:
|
||||
|
||||
```
|
||||
{
|
||||
"corpora": {
|
||||
"version": {
|
||||
"corpora_schema_version": "1.0.0",
|
||||
"corpora_encoding_version": "0.1.0",
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The `corpora` metadata, if present, MUST contain the version information. Optionality of other values in this object will follow the specifications set forth in the relevant Corpora schema specification (ie, optional fields are optional, required are present, etc), with the following changes:
|
||||
* the contents of `corpora_encoding_version` MUST be identical to the `cxg_version`, as this field is defined as the current object encoding version, *NOT* the source data encoding version.
|
||||
* the entire encoding will be JSON, rather than a hybrid Python/JSON encoding, but will otherwise follow the data structure defined by the AnnData Corpora encoding.
|
||||
* the `<obs_column>_colors` will be omitted in favor of `cxg_category_colors`
|
||||
|
||||
### Other Corpora fields
|
||||
|
||||
All other Corpora schema fields will be encoded into a CXG using the conventions defined in the [Corpora Schema AnnData Implementation](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema_h5ad_implementation.md). For example, fields in `AnnData.obs` will be encoded in the CXG `obs`array as defined [above](#obs-and-var).
|
||||
|
||||
### Compatibility with CXG 0.1.0
|
||||
|
||||
For backwards compatibility and continuity with CXG version 0.1.0, the following MUST be implemented.
|
||||
|
||||
#### Presentation Hints
|
||||
* The [Corpora `title`](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md#presentation-metadata) value MUST be saved in the `cxg_properties.title` field.
|
||||
* The [Corpora `color_map`](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md#presentation-hints), when present in the dataset, MUST be saved in the `cxg_category_colors` field and NOT in the `corpora` field.
|
||||
* The [Corpora SUMMARY `project_link`](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md#presentation-hints), if present, MUST be saved in the `cxg_properties.about` field.
|
||||
|
||||
Where these values differ in the final CXG, the `cxg_properties` values WILL take precedence.
|
||||
|
||||
## CXG Version History
|
||||
|
||||
There were several ad hoc version of CXG created prior to this spec. This describes the _proposed_ next version of CXG, which incoporates support for Corpora schema semantics. Prior verisons:
|
||||
* _unnamed_ - an unnamed development version. Did not include explicit versioning support in the data model, but can be detected by the absence of __cxg_group_metadata__ and any version property. Created in early 2020, and not actively used in production
|
||||
* 0.1 - the first and current version, defined to support the capabilities of the mid-2020 cellxgene. Created in early 2020, and in active use. Includes everything in this spec, excluding Corpora schema support. __NOTE:__ this version is encoded with a short-hand (malformed) semver version number.
|
||||
* 0.2.0 - this specification.
|
||||
@@ -1,175 +0,0 @@
|
||||
# Cellxgene Schema Guide
|
||||
|
||||
Datasets included in the [data portal](https://cellxgene.cziscience.com/) and hosted cellxgene need to follow the schema
|
||||
described [here](https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md). That
|
||||
schema defines some required fields, requirements about feature labels, and some optional fields that mostly help with
|
||||
presentation.
|
||||
|
||||
The number of fields is rather low, and we expect that information needed to populate those fields should either already
|
||||
be present in datasets prepared by a submitter or be easy to obtain. However, this still leaves the task of actually
|
||||
manipulating the dataset so that it follows the schema: adjusting field names, ensuring proper ontologies are used,
|
||||
converting gene symbols to a common set, etc. This can be tedious and error-prone, and at the beginning of the hosted
|
||||
cellxgene project, this was always done with engineering support. As we increase the rate at which we add data, we want
|
||||
to eliminate the need for engineering support so that ultimately submitters themselves can create files that follow the
|
||||
schema.
|
||||
|
||||
## `cellxgene schema apply`
|
||||
|
||||
To enable this, we have a new cellxgene subcommand, `cellxgene schema`, that handles applying and verifying the schema.
|
||||
Its first subcommand, `cellxgene schema apply`, takes three inputs:
|
||||
|
||||
1. A source h5ad file. The input needs to be an AnnData file, so if a submitter has, say, a serialized Seurat or
|
||||
SingleCellExperiment object, it needs to be converted to AnnData first. This can be done with
|
||||
[sceasy](https://github.com/cellgeni/sceasy) or via
|
||||
[Seurat](https://satijalab.org/seurat/v3.1/conversion_vignette.html).
|
||||
2. A configuration yaml file that describes the fields to add and conversions to apply (see below).
|
||||
3. A name for the new h5ad file that should follow the schema.
|
||||
|
||||
### Configuration yaml
|
||||
|
||||
The configuration yaml file describes how to apply the schema. This is an example of a "skeleton" yaml that has all the
|
||||
fields required for the 1.0.0 schema but is not yet filled in with any logic:
|
||||
|
||||
```
|
||||
uns:
|
||||
version:
|
||||
corpora_schema_version: 1.0.0
|
||||
corpora_encoding_version: 0.1.0
|
||||
contributors:
|
||||
title:
|
||||
layer_descriptions:
|
||||
preprint_doi:
|
||||
publication_doi:
|
||||
organism_ontology_term_id:
|
||||
obs:
|
||||
tissue_ontology_term_id:
|
||||
assay_ontology_term_id:
|
||||
disease_ontology_term_id:
|
||||
cell_type_ontology_term_id:
|
||||
sex:
|
||||
ethnicity_ontology_term_id:
|
||||
development_stage_ontology_term_id:
|
||||
fixup_gene_symbols:
|
||||
```
|
||||
|
||||
#### Unstructured metadata
|
||||
The first section is `uns`, which includes metadata fields that describe the whole dataset (see
|
||||
[here](https://anndata.readthedocs.io/en/latest/) for further description of `uns` and `obs`.).
|
||||
|
||||
The first line is `version`, which is required for most of our tooling to work. The schema version is set at
|
||||
1.0.0 in the example above, but of course for future versions that should be changed.
|
||||
|
||||
Next is `contributors` which describes who is adding the dataset to the portal. If you consult the schema, you see that
|
||||
contributors is a list where each element can have `name`, `email`, and `institution`. So when filled out, the
|
||||
`contributors` field should look like this:
|
||||
|
||||
```
|
||||
contributors:
|
||||
- name: Mary B. Scientist
|
||||
email: mbs@singlecell.edu
|
||||
institution: Single-Cell University
|
||||
- name: Robert J. Scientist
|
||||
email: rjs@usingle.edu
|
||||
institution: University of Single Cell
|
||||
```
|
||||
|
||||
`title` is the name of the dataset, and is just a string that gets displayed in the portal and cellxgene to identify the
|
||||
dataset.
|
||||
|
||||
`layer_descriptions` is free text descriptions of the different
|
||||
[layers](https://anndata.readthedocs.io/en/latest/anndata.AnnData.layers.html) of the AnnData file. It should look like
|
||||
this when complete, depending on what layers are present:
|
||||
```
|
||||
layer_descriptions:
|
||||
X: CPM and logged
|
||||
raw.X: raw
|
||||
```
|
||||
Note that one of the layers needs to be "raw", that is, the AnnData file must contain raw counts.
|
||||
|
||||
The two DOI fields are optional but can be included if the dataset is associated with a publication or preprint. Note
|
||||
that the DOI should be a full url:
|
||||
```
|
||||
publication_doi: https://doi.org/10.1073%2Fpnas.83.15.5372
|
||||
```
|
||||
|
||||
Finally, the `organism_ontology_term_id` field is the species of the donor organism from the NCBITaxon ontology. The
|
||||
value for _Homo sapiens_ is `NCBITaxon:9606`:
|
||||
```
|
||||
organism_ontology_term_id: NCBITaxon:9606
|
||||
```
|
||||
Note that the schema also requires a human-readable `organism` field, but this doesn't need to be included in the yaml.
|
||||
When the `cellxgene schema apply` script encounters an ontology field, it looks up the label for the term(s) and inserts it
|
||||
into the appropriate field.
|
||||
|
||||
|
||||
#### Observation metadata
|
||||
The next section is `obs`, which is metadata than can vary for each observation (and "observation" usually means cell).
|
||||
These fields are all ontology fields except for `sex`, which has its own enumerated set of permitted values.
|
||||
|
||||
There are two ways to fill in the `obs` fields. The first is useful when there is only one value for all the
|
||||
observations in the dataset. This is not uncommon, for example all cells often come from the same assay. In that case
|
||||
just insert the ontology term:
|
||||
```
|
||||
assay_ontology_term_id: EFO:0009922
|
||||
```
|
||||
|
||||
The second is for when there is an existing field in the dataset that needs to be mapped to the schema field. For
|
||||
example, the submitter may have included cell type annotations in a field called `CellType`, and those annotations may
|
||||
just be free text. This doesn't follow the schema because it needs to be in `cell_type_ontology_term_id` and
|
||||
`cell_type`, and it needs ontology terms and labels, not just any text. In that case the field can be a dictionary:
|
||||
|
||||
```
|
||||
cell_type_ontology_term_id:
|
||||
CellType:
|
||||
t-cell: CL:0000084
|
||||
b-cell: CL:0000236
|
||||
```
|
||||
|
||||
This will look at the `obs.CellType` field in the dataset, and where it has the value "t-cell", it will insert
|
||||
`CL:0000084` into `cell_type_ontology_term_id` and its label `T cell` into `cell_type`.
|
||||
|
||||
Now there are often situations where there is no valid ontology term for some field. For example, the dataset may have
|
||||
been produced via an assay not present in `EFO`. Or, a particular cell type may have no entry in `CL`. In that case, a
|
||||
free text description can be used in the `ontology_term_id` field:
|
||||
|
||||
```
|
||||
assay_ontology_term_id: Sci-Plex
|
||||
cell_type_ontology_term_id:
|
||||
CellType:
|
||||
t-cell: CL:0000084
|
||||
b-cell: CL:0000236
|
||||
new cell type: new cell type
|
||||
```
|
||||
|
||||
In these cases, the `cellxgene schema apply` script will leave the ontology field blank and move the free text
|
||||
description into the label field. So the `assay_ontology_term_id` in the new dataset would be `""` but `assay` would be
|
||||
`Sci-Plex`.
|
||||
|
||||
|
||||
#### Gene symbol harmonization
|
||||
|
||||
The last section describes how gene symbol conversion should be applied to each of the layers. This is similar to the
|
||||
`layer_descriptions` field above, but there are only three permitted values: `raw`, `log1p`, and `sqrt`:
|
||||
|
||||
```
|
||||
fixup_gene_symbols:
|
||||
X: log1p
|
||||
raw.X: raw
|
||||
```
|
||||
|
||||
This tells the script how each each layer was transformed from raw values that can be directly summed. `raw` means that
|
||||
the layer contains raw counts or some linear tranformation of raw counts. `log1p` means that the layer has `log(X + 1)`
|
||||
for each the raw `X` values. `sqrt` means `sqrt(X)` (this is not common). For layers produced by Seurat's normalization
|
||||
or SCTransform functions, the correct choice is usually `log1p`.
|
||||
|
||||
|
||||
### `cellxgene schema validate`
|
||||
|
||||
The next `cellxgene schema` subcommand is `cellxgene schema validate`, and it validates that a given h5ad follows a
|
||||
version of the schema. It accepts two parameters:
|
||||
|
||||
1. The h5ad file to check
|
||||
2. The version of the schema to check against.
|
||||
|
||||
If the validation succeeds, the command will have a zero exit code. If it does not, it will have a non-zero exit code
|
||||
and will print validation failure messages.
|
||||
Reference in New Issue
Block a user