fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-29 03:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Clean up dead/hosted code [zh2310] (#2430)

Browse Source 
* Clean up dead/hosted code

* Remove schema conversion tool and related
* Remove cxg references
* Remove locust

* missed a spot

* Remove aws secret manager

* Merge branch 'main' into brodgers/2310/code-cleanup-v1

* cleanup merge
This commit is contained in:
Ben MR
2021-09-17 20:41:12 +00:00
committed by GitHub
parent 69e159916e
commit ef2ab07ca0
35 changed files with 5 additions and 2882 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/server/cli/cli.py
-2
View File
@@ -3,7 +3,6 @@ import click
from .launch import launch
from .prepare import prepare
from .upgrade import log_upgrade_check
from .schema import schema_cli
from .. import __version__
@@ -30,4 +29,3 @@ def cli(upgrade_check):
cli.add_command(launch)
cli.add_command(prepare)
cli.add_command(schema_cli)
backend/server/cli/schema.py
-72
View File
@@ -1,72 +0,0 @@
import click
from backend.server.converters.schema import remix, validate
@click.group(
name="schema",
subcommand_metavar="COMMAND <args>",
short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
)
def schema_cli():
try:
import scanpy # noqa: F401
except ImportError:
raise click.ClickException(
"[cellxgene] cellxgene schema requires scanpy"
)
@click.command(
name="apply",
short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
"h5ad file, apply the schema changes and create a new, conforming h5ad.",
)
@click.option(
"--source-h5ad",
help="Input h5ad file.",
nargs=1,
required=True,
type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"--remix-config",
help="Config yaml with information on how to apply the schema.",
nargs=1,
required=True,
type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"--output-filename",
help="Filename for the new, schema-conforming h5ad file.",
required=True,
nargs=1
)
def schema_apply(source_h5ad, remix_config, output_filename):
remix.apply_schema(source_h5ad, remix_config, output_filename)
@click.command(
name="validate",
short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
)
@click.argument(
"h5ad",
nargs=1,
type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"--shallow",
help="When true, just check that the correct version information is present.",
default=False,
show_default=True,
is_flag=True,
)
def schema_validate(h5ad, shallow):
validate.validate(h5ad, shallow)
schema_cli.add_command(schema_apply)
schema_cli.add_command(schema_validate)
backend/server/common/config/__init__.py
-2
View File
@@ -1,4 +1,2 @@
from backend.common.utils.aws_secret_utils import get_secret_key # noqa F504
DEFAULT_SERVER_PORT = 5005
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
backend/server/common/config/external_config.py
+1 -53
View File
@@ -2,28 +2,23 @@ import os
from backend.server.common.config.base_config import BaseConfig
from backend.common.errors import ConfigurationError
from backend.server.common.config import get_secret_key
from backend.common.errors import SecretKeyRetrievalError
from backend.common.utils.type_conversion_utils import convert_string_to_value
class ExternalConfig(BaseConfig):
"""Manages the config attribute associated with external configuration sources, such as
environment variables or the AWS Secrets Manager."""
environment variables."""
def __init__(self, app_config, default_config):
super().__init__(app_config, default_config)
try:
self.environment = default_config["environment"]
self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
def complete_config(self, context):
self.handle_environment(context)
self.handle_aws_secrets_manager(context)
def handle_environment(self, context):
"""For each environment variable defined, get the value (if it is set),
@@ -47,50 +42,3 @@ class ExternalConfig(BaseConfig):
else:
value = convert_string_to_value(value)
self.app_config.update_single_config_from_path_and_value(path, value)
def handle_aws_secrets_manager(self, context):
"""For each aws secret defined, get the key/values, and set the specified config parameter"""
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
if not self.aws_secrets_manager__secrets:
return
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
for secret in self.aws_secrets_manager__secrets:
secret_name = secret.get("name")
if secret_name is None:
raise ConfigurationError("aws_secrets_manager: 'name' is missing")
if not isinstance(secret_name, str):
raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
try:
secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
except SecretKeyRetrievalError as e:
raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
values = secret.get("values")
if values is None:
raise ConfigurationError("aws_secrets_manager: 'values' is missing")
if not isinstance(values, list):
raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
for value in values:
key = value.get("key")
if key is None:
raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
path = value.get("path")
if path is None:
raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
required = value.get("required", False)
if type(required) != bool:
raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
secret_value = secret_dict.get(key)
if secret_value is None:
if required:
raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
else:
secret_value = convert_string_to_value(secret_value)
self.app_config.update_single_config_from_path_and_value(path, secret_value)
backend/server/converters/__init__.py
View File
backend/server/converters/schema/__init__.py
View File
backend/server/converters/schema/gene_symbol.py
-211
View File
@@ -1,211 +0,0 @@
"""Helpers for converting and checking HGNC gene symbols."""
import argparse
import enum
import logging
import os
import re
import numpy as np
import pandas as pd
def get_upgraded_var_index(var, hgnc_path=None):
"""Given an anndata var dataframe, return a new index for the dataframe
where human gene symbols have been upgraded to the current HGNC set.
"""
if not hgnc_path:
hgnc_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "hgnc_complete_set.txt.gz")
hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(hgnc_path)
return pd.Index([hgnc_symbol_checker.upgrade_symbol(s) for s in var.index])
class SymbolStatus(enum.Enum):
"""The status of a symbol in the HGNC database.
APPROVED: Currently a valid symbol
WITHDRAWN: A previously approved HGNC symbol for a gene that has since been shown
not to exist _unless_ that symbol is also approved
AMBIGUOUS: A symbol that is not approved but is an alias or previous symbol for
multiple approved symbols
UPGRADABLE: A symbol that is not approved but unambiguously maps to an approved
symbol
UNKNOWN: A symbol that does not appear in HGNC
"""
APPROVED = 1
WITHDRAWN = 2
AMBIGUOUS = 3
UPGRADABLE = 4
UNKNOWN = 5
class HGNCSymbolChecker:
"""Handle checking and correcting HGNC symbols."""
def __init__(self, approved_symbols, withdrawn_symbols, ambiguous_symbols, symbol_map):
self.approved_symbols = approved_symbols
self.withdrawn_symbols = withdrawn_symbols
self.ambiguous_symbols = ambiguous_symbols
self.symbol_map = symbol_map
def print_symbol_map(self):
"""Print out a map from old symbol to new symbol."""
for symbol_pair in self.symbol_map.items():
print("\t".join(symbol_pair))
def check_symbol(self, symbol):
"""See if a symbol if approved or something else."""
if symbol in self.approved_symbols:
return SymbolStatus.APPROVED
if symbol in self.withdrawn_symbols:
return SymbolStatus.WITHDRAWN
if symbol in self.ambiguous_symbols:
return SymbolStatus.AMBIGUOUS
if symbol in self.symbol_map:
return SymbolStatus.UPGRADABLE
return SymbolStatus.UNKNOWN
def upgrade_symbol(self, symbol):
"""Return the approved symbol for the given symbol.
If the symbol cannot be upgraded, just return the original symbol.
"""
fixed_symbol, stripped_symbol = format_symbol(symbol)
if fixed_symbol in self.approved_symbols:
return fixed_symbol
elif fixed_symbol in self.symbol_map:
return self.symbol_map[fixed_symbol]
elif stripped_symbol in self.approved_symbols:
return stripped_symbol
elif stripped_symbol in self.symbol_map:
return self.symbol_map[stripped_symbol]
return symbol
@classmethod
def from_hgnc_records(cls, hgnc_dataset_path):
"""Parse a hgnc database download into a HGNCSymbolChecker object."""
def all_symbols(record):
"""Get all the symbols associated with an HGNC record including previous, alias,
and approved."""
yield format_symbol(record["symbol"])[0]
for symbol in alias_and_previous_symbols(record):
yield symbol
def alias_and_previous_symbols(record):
"""Get alias and previous symbols from an HGNC record."""
for field in ("alias_symbol", "prev_symbol"):
if record[field] is not np.nan:
for symbol in record[field].split("|"):
yield format_symbol(symbol)[0]
# Sometimes something like HGNC:1234 appears in datasets, which we
# want to fix as well.
yield record["hgnc_id"]
hgnc_records = pd.read_csv(hgnc_dataset_path, sep="\t", header=0, low_memory=False).to_dict("records")
# Get all symbols that are currently approved.
approved_symbols = set()
for record in hgnc_records:
if record["status"] == "Approved":
approved_symbols.add(format_symbol(record["symbol"])[0])
# Get all symbols that have been withdrawn
withdrawn_symbols = set()
for record in hgnc_records:
if record["status"] == "Entry Withdrawn":
for symbol in all_symbols(record):
withdrawn_symbols.add(symbol)
# If a symbol is both approved and withdrawn, be optimistic and call it approved
logging.warning(
f"Some symbols are simulaneously withdrawn and approved\n"
f"We will treat them at approved:\n"
f"{withdrawn_symbols.intersection(approved_symbols)}"
)
withdrawn_symbols = withdrawn_symbols.difference(approved_symbols)
# Now try to map from symbols that are not approved but are an alias or previous symbol for an approved symbol
alias_previous_to_approved = {}
ambiguous_symbols = set()
for record in hgnc_records:
if record["status"] == "Approved":
# The approved symbol is what we'll map to
approved_symbol = format_symbol(record["symbol"])[0]
for symbol in alias_and_previous_symbols(record):
# If the alias or previous symbol is also an approved symbol,
# we'll just leave it alone
if symbol in approved_symbols:
continue
# If the alias or previous symbol maps to a different approved symbol, mark it as ambiguous
if symbol in alias_previous_to_approved and alias_previous_to_approved[symbol] != approved_symbol:
ambiguous_symbols.add(symbol)
else:
alias_previous_to_approved[symbol] = approved_symbol
# Remove all the ambiguous symbols from the map
for ambiguous_symbol in ambiguous_symbols:
alias_previous_to_approved.pop(ambiguous_symbol)
return HGNCSymbolChecker(approved_symbols, withdrawn_symbols, ambiguous_symbols, alias_previous_to_approved)
def format_symbol(symbol):
"""HGNC rules say symbols should all be upper case except for C#orf#. However, case is
variable in both alias and previous symbols as well as in the symbols we get in
submissions. So, upper case everything except for the one situation where mixed-case
is allowed, which are the genes like C2orf157.
Also, seurat and scanpy append ".1" or "-1" to duplicated gene names, and these altered
names persist throughout the life of the object. They won't match against the HGNC database
and we want to merge them, so we need to strip off the suffix and try matching again.
This function takes a symbol and returns the symbol with the fixed case and also with the
seurat/scanpy suffix stripped off.
"""
match = re.match(r"^(C)(\d+)(orf)(\d+)$", symbol, re.IGNORECASE)
if match:
fixed_case = f"C{match.group(2)}orf{match.group(4)}"
else:
fixed_case = symbol.upper()
suffix_stripped = re.sub(r"[\.\-]\d+$", "", fixed_case)
return fixed_case, suffix_stripped
def main():
"""When called as main, parse a given hgnc download and print out a map from old to new
symbol.
"""
parser = argparse.ArgumentParser()
parser.add_argument(
"hgnc_dataset", help="HGNC dataset tsv, available from www.genenames.org/download/statistics-and-files/"
)
args = parser.parse_args()
hgnc_symbol_checker = HGNCSymbolChecker.from_hgnc_records(args.hgnc_dataset)
hgnc_symbol_checker.print_symbol_map()
if __name__ == "__main__":
main()
backend/server/converters/schema/hgnc_complete_set.txt.gz
BIN
View File
Binary file not shown.
backend/server/converters/schema/ontology.py
-86
View File
@@ -1,86 +0,0 @@
"""Methods for working with ontologies and the OLS."""
from urllib.parse import quote_plus
import requests
OLS_API_ROOT = "http://www.ebi.ac.uk/ols/api"
# Curie means something like CL:0000001
def _ontology_name(curie):
"""Get the name of the ontology from the curie, CL or UBERON for example."""
return curie.split(":")[0]
def _ontology_value(curie):
"""Get the id component of the curie, 0000001 from CL:0000001 for example."""
return curie.split(":")[1]
def _double_encode(url):
"""Double url encode a url. This is required by the OLS API."""
return quote_plus(quote_plus(url))
def _iri(curie):
"""Get the iri from a curie. This is a bit hopeful that they all map to purl.obolibrary.org"""
if _ontology_name(curie) == "EFO":
return f"http://www.ebi.ac.uk/efo/EFO_{_ontology_value(curie)}"
return f"http://purl.obolibrary.org/obo/{_ontology_name(curie)}_{_ontology_value(curie)}"
class OntologyLookupError(Exception):
"""Exception for some problem with looking up ontology information."""
def _ontology_info_url(curie):
"""Get the to make a GET to to get information about an ontology term."""
# If the curie is empty, just return an empty string. This happens when there is no
# valid ontology value.
if not curie:
return ""
else:
return f"{OLS_API_ROOT}/ontologies/{_ontology_name(curie)}/terms/{_double_encode(_iri(curie))}"
def get_ontology_label(curie):
"""For a given curie like 'CL:1000413', get the label like 'endothelial cell of artery'"""
url = _ontology_info_url(curie)
if not url:
return ""
response = requests.get(url)
if not response.ok:
raise OntologyLookupError(
f"Curie {curie} lookup failed, got status code {response.status_code}: {response.text}"
)
return response.json()["label"]
def lookup_candidate_term(label, ontology="cl", method="select"):
"""Lookup candidate terms for a label. This is useful when there is an existing label in a
submitted dataset, and you want to find an appropriate ontology term.
Args:
label: the label to find ontology terms for
ontology: the ontology to search in, cl or uberon or efo for example
method: select or search. search provides much broader results
Returns:
list of (curie, label) tuples returned by OLS
"""
# using OLS REST API [https://www.ebi.ac.uk/ols/docs/api]
url = f"{OLS_API_ROOT}/{method}?q={quote_plus(label)}&ontology={ontology.lower()}"
response = requests.get(url)
if not response.ok:
raise OntologyLookupError(
f"Label {label} lookup failed, got status code {response.status_code}: {response.text}"
)
return [(r["obo_id"], r["label"]) for r in response.json()["response"]["docs"]]
backend/server/converters/schema/remix.py
-266
View File
@@ -1,266 +0,0 @@
import argparse
import collections
import json
import logging
import math
import string
import anndata
import numpy as np
import pandas as pd
import yaml
from . import gene_symbol
from . import ontology
from . import validate
REPLACE_SUFFIX = "_original"
ONTOLOGY_SUFFIX = "_ontology_term_id"
def is_curie(value):
"""Return True iff the value is an OBO-id CURIE like EFO:000001"""
return (value.count(":")
and all(len(part) > 0 for part in value.split(":"))
and all(c in string.digits for c in value.split(":")[1]))
def is_ontology_field(field_name):
"""Return True iff the field_name is an ontology field like tissue_ontology_term_id"""
return field_name.endswith(ONTOLOGY_SUFFIX)
def get_label_field_name(field_name):
"""Get the associated label field from an ontology field, assay_ontology_term_id --> assay"""
return field_name[: -len(ONTOLOGY_SUFFIX)]
def split_suffix(maybe_curie):
"""Split off the (cell culture) or (organoid) suffix."""
suffixes = [" (cell culture)", " (organoid)"]
for suffix in suffixes:
if maybe_curie.endswith(suffix):
return maybe_curie[:-len(suffix)], suffix
return maybe_curie, ""
def get_curie_and_label(maybe_curie):
"""Given a string that might be a curie, return a (curie, label) pair"""
maybe_curie, suffix = split_suffix(maybe_curie)
if not is_curie(maybe_curie):
return ("", maybe_curie + suffix)
return (maybe_curie + suffix, ontology.get_ontology_label(maybe_curie) + suffix)
def safe_add_field(adata_attr, field_name, field_value):
"""Add a field and value to an AnnData, but don't clobber an exising value."""
if (
isinstance(field_value, list)
and field_value
and isinstance(field_value[0], dict)
):
field_value = json.dumps(field_value)
if field_name in adata_attr:
adata_attr[field_name + REPLACE_SUFFIX] = adata_attr[field_name]
adata_attr[field_name] = field_value
def remix_uns(adata, uns_config):
"""Add fields from the config to adata.uns"""
for field_name, field_value in uns_config.items():
if is_ontology_field(field_name):
# If it's an ontology field, look it up
label_field_name = get_label_field_name(field_name)
ontology_term, ontology_label = get_curie_and_label(field_value)
safe_add_field(adata.uns, field_name, ontology_term)
safe_add_field(adata.uns, label_field_name, ontology_label)
else:
safe_add_field(adata.uns, field_name, field_value)
def remix_obs(adata, obs_config):
"""Add fields from the config to adata.obs"""
for field_name, field_value in obs_config.items():
if isinstance(field_value, dict):
# If the value is a dict, that means we are supposed to map from an
# existing column to the new one
source_column, column_map = next(iter(field_value.items()))
nan_value = None
for key in column_map:
if isinstance(key, float) and math.isnan(key):
nan_value = column_map[key]
if nan_value is not None:
column_map["nan"] = nan_value
for key in column_map:
if key not in adata.obs[source_column].unique():
logging.warning(f'Key {key} not in adata.obs["{source_column}"]')
for value in adata.obs[source_column].unique():
if value not in column_map:
logging.warning(f'Value {value} in adata.obs["{source_column}"] not in translation dict')
if is_ontology_field(field_name):
ontology_term_map, ontology_label_map = {}, {}
logging.info(f"Looking up labels for {field_name}")
for original_value, maybe_curie in column_map.items():
curie, label = get_curie_and_label(maybe_curie)
ontology_term_map[original_value] = curie
ontology_label_map[original_value] = label
logging.info(f"Mapping {original_value} -> {curie} -> {label}")
ontology_column = adata.obs[source_column].replace(
ontology_term_map, inplace=False
)
label_column = adata.obs[source_column].replace(
ontology_label_map, inplace=False
)
safe_add_field(adata.obs, field_name, ontology_column)
safe_add_field(
adata.obs, get_label_field_name(field_name), label_column
)
else:
label_column = adata.obs[source_column].replace(
column_map, inplace=False
)
safe_add_field(adata.obs, field_name, label_column)
else:
if is_ontology_field(field_name):
# If it's an ontology field, look it up
label_field_name = get_label_field_name(field_name)
ontology_term, ontology_label = get_curie_and_label(field_value)
safe_add_field(adata.obs, field_name, ontology_term)
safe_add_field(adata.obs, label_field_name, ontology_label)
else:
safe_add_field(adata.obs, field_name, field_value)
def merge_df(df, domain, index, columns):
"""
Given a dataframe with duplicate column labels, merge and return a dataframe where
the duplicates have been merged together, resulting in a dataframe with unique column
labels.
"merge" depends on the value of domain. If the domain is "raw", then duplicate columns
can just be summed. If it's "log1p" or "sqrt", it needs to be exp1m'd or squared, then
summed, and then logged or sqrt'd again.
"""
if not isinstance(df, np.ndarray):
to_merge = df.toarray()
else:
to_merge = df
if domain == "raw":
merged_df = pd.DataFrame(to_merge, index=index, columns=columns).sum(
axis=1, level=0, skipna=False
)
elif domain == "log1p":
merged_df = (
pd.DataFrame(np.expm1(to_merge, dtype=np.float128), index=index, columns=columns)
.sum(axis=1, level=0, skipna=False)
)
merged_df = pd.DataFrame(np.log1p(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
elif domain == "sqrt":
merged_df = (
pd.DataFrame(np.square(to_merge), index=index, columns=columns)
.sum(axis=1, level=0, skipna=False)
)
merged_df = pd.DataFrame(np.sqrt(merged_df.to_numpy()), index=merged_df.index, columns=merged_df.columns)
return merged_df
def fixup_gene_symbols(adata, fixup_config):
"""Update the var index to hold a consistent set of HGNC gene symbols."""
upgraded_var_index = gene_symbol.get_upgraded_var_index(adata.var)
merged_X = merge_df(adata.X, fixup_config["X"], adata.obs.index, upgraded_var_index)
fixup_adata = anndata.AnnData(
X=merged_X,
obs=adata.obs,
var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
uns=adata.uns,
obsm=adata.obsm,
)
for layer, domain in fixup_config.items():
if layer == "X":
continue
if layer == "raw.X":
df = adata.raw.X
else:
df = adata.layers[layer]
merged_df = merge_df(df, domain, adata.obs.index, upgraded_var_index)
assert merged_df.index.equals(merged_X.index)
assert merged_df.columns.equals(merged_X.columns)
if domain == "raw":
fixup_raw = anndata.AnnData(
X=merged_df,
obs=adata.obs,
var=merged_X.columns.to_frame(name="hgnc_gene_symbol"),
)
fixup_adata.raw = fixup_raw
else:
fixup_adata.layers[layer] = merged_df
return fixup_adata
def _strip_version(adata):
"""Remove version information from the AnnData object."""
if "version" in adata.uns_keys():
del adata.uns["version"]
def apply_schema(source_h5ad, remix_config, output_filename):
try:
import scanpy
except ImportError:
raise ImportError("scanpy must be installed for cellxgene schema")
adata = scanpy.read_h5ad(source_h5ad)
config = yaml.load(open(remix_config), Loader=yaml.FullLoader)
remix_uns(adata, config["uns"])
remix_obs(adata, config["obs"])
if config.get("fixup_gene_symbols"):
adata = fixup_gene_symbols(adata, config["fixup_gene_symbols"])
if ("version" in adata.uns_keys()
and isinstance(adata.uns["version"], collections.Mapping)
and "corpora_schema_version" in adata.uns["version"]):
schema_version = adata.uns["version"]["corpora_schema_version"]
try:
validate.get_schema_definition(schema_version)
except ValueError:
logging.warning(f"Stripping version information out of AnnData because schema "
f"version {schema_version} is unknown.")
_strip_version(adata)
if not validate.validate_adata(adata, shallow=False):
logging.warning(f"Stripping version information out of AnnData because it does not "
f"follow schema version {schema_version} .")
_strip_version(adata)
adata.write_h5ad(output_filename, compression="gzip")
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--source-h5ad", required=True)
parser.add_argument("--remix-config", required=True)
parser.add_argument("--output-filename", required=True)
args = parser.parse_args()
apply_schema(args.source_h5ad, args.remix_config, args.output_filename)
backend/server/converters/schema/schema_definitions/1_0_0.yaml
-95
View File
@@ -1,95 +0,0 @@
title: Corpora schema version 1.0.0
type: anndata
components:
uns:
type: dict
keys:
version:
type: dict
keys:
corpora_schema_version: null
corpora_encoding_version: null
title:
type: string
contributors:
type: stringified list of dicts
layer_descriptions:
type: dict
keys:
X: null
organism:
type: string
nullable: false
organism_ontology_term_id:
type: curie
prefixes:
- NCBITaxon
var:
type: dataframe
index:
type: human-readable string
unique: true
obs:
type: dataframe
index:
unique: true
columns:
tissue:
type: human-readable string
nullable: false
tissue_ontology_term_id:
type: suffixed curie
nullable: true
prefixes:
- UBERON
assay:
type: human-readable string
nullable: false
assay_ontology_term_id:
type: curie
nullable: true
prefixes:
- EFO
disease:
type: human-readable string
nullable: false
disease_ontology_term_id:
type: curie
nullable: true
prefixes:
- MONDO
- PATO
cell_type:
type: human-readable string
nullable: false
cell_type_ontology_term_id:
type: curie
nullable: true
prefixes:
- CL
- UBERON
sex:
type: string
enum:
- male
- female
- mixed
- unknown
- other
ethnicity:
type: human-readable string
nullable: false
ethnicity_ontology_term_id:
type: curie
nullable: true
prefixes:
- HANCESTRO
development_stage:
type: human-readable string
nullable: false
development_stage_ontology_term_id:
type: curie
nullable: true
prefixes:
- HsapDv
- EFO
backend/server/converters/schema/schema_definitions/1_1_0.yaml
-93
View File
@@ -1,93 +0,0 @@
title: Corpora schema version 1.1.0
type: anndata
components:
uns:
type: dict
keys:
version:
type: dict
keys:
corpora_schema_version: null
corpora_encoding_version: null
title:
type: string
layer_descriptions:
type: dict
keys:
X: null
organism:
type: string
nullable: false
organism_ontology_term_id:
type: curie
prefixes:
- NCBITaxon
var:
type: dataframe
index:
type: human-readable string
unique: true
obs:
type: dataframe
index:
unique: true
columns:
tissue:
type: human-readable string
nullable: false
tissue_ontology_term_id:
type: suffixed curie
nullable: true
prefixes:
- UBERON
assay:
type: human-readable string
nullable: false
assay_ontology_term_id:
type: curie
nullable: true
prefixes:
- EFO
disease:
type: human-readable string
nullable: false
disease_ontology_term_id:
type: curie
nullable: true
prefixes:
- MONDO
- PATO
cell_type:
type: human-readable string
nullable: false
cell_type_ontology_term_id:
type: curie
nullable: true
prefixes:
- CL
- UBERON
sex:
type: string
enum:
- male
- female
- mixed
- unknown
- other
ethnicity:
type: human-readable string
nullable: false
ethnicity_ontology_term_id:
type: curie
nullable: true
prefixes:
- HANCESTRO
development_stage:
type: human-readable string
nullable: false
development_stage_ontology_term_id:
type: curie
nullable: true
prefixes:
- HsapDv
- EFO
backend/server/converters/schema/validate.py
-236
View File
@@ -1,236 +0,0 @@
import json
import re
import os
import sys
import pandas as pd
import yaml
def _is_null(v):
"""Return True if v is null, for one of the multiple ways a "null" value shows up in an h5ad."""
return pd.isnull(v) or (hasattr(v, "__len__") and len(v) == 0)
def _validate_stringified_list_of_dicts(s):
"""Verify that a string can be parsed into a list.
We have some types that are lists of dicts. Those cannot be stored directly in an h5ad, so we have to
json.dumps them. This verifies that we can load them back.
"""
try:
list_ = json.loads(s)
if not isinstance(list_, list):
return False
for el in list_:
if not isinstance(el, dict):
return False
return True
except (json.JSONDecodeError, TypeError):
pass
return False
def _validate_human_readable_string(s):
"""Verify that a string is human-readable.
There are parts of the schema where a "human-readable" string is required. "Human-readable" is kind
of vague and subjective. I feel like I can read many strings. So here we just check for the main ways
that fails: someone puts in an ontology term id or and ensembl gene/transcript id.
Returns False if s is not a string or is one of those bad string types.
"""
return isinstance(s, str) and (not re.match(r"[A-Z]\w+:\d+", s)) and (not re.match(r"ENS[GT]\d+$", s))
def _validate_curie(c, prefixes):
"""Verify that a string is a valid compact URI, like EFO:000001. If prefixes is not empty, make sure the
prefix of the curies is in prefixes.
"""
if not c:
return True
match = re.match(r"([A-Z]\w+):\d+$", c)
if prefixes:
return match and match.group(1) in prefixes
else:
return match
def _validate_suffixed_curie(c, prefixes):
"""Verify that a string is a compact URI with an optional suffix like 'EFO:00001 (cell culture)'"""
# Pull off the suffix
suffix = re.findall(r"\ \(.*\)$", c)
if suffix:
c = c[: -len(suffix[0])]
return _validate_curie(c, prefixes)
def _validate_column(column, column_name, df_name, schema_def):
"""Given a schema definition and the column of a dataframe, verify that the column satifies
the schema.
"""
errors = []
if schema_def.get("unique"):
if column.nunique() != len(column):
errors.append(f"Column {column_name} in dataframe {df_name} is not unique.")
if "nullable" in schema_def and not schema_def["nullable"]:
if any(_is_null(v) for v in column):
errors.append(f"Column {column_name} in dataframe {df_name} contains empty values.")
if schema_def.get("type") == "human-readable string":
non_readables = [v for v in column if not _validate_human_readable_string(v)]
if non_readables:
errors.append(
f"Column {column_name} in dataframe {df_name} contains non-human-readable "
f"values like {non_readables[0]}"
)
if schema_def.get("type") in ("curie", "suffixed curie"):
validation_func = _validate_curie if schema_def.get("type") == "curie" else _validate_suffixed_curie
non_valid_curies = [v for v in column if not validation_func(v, schema_def.get("prefixes"))]
if non_valid_curies:
errors.append(
f"Column {column_name} in dataframe {df_name} contains invalid ontology values like "
f"{non_valid_curies[0]}."
)
if "prefixes" in schema_def:
errors[-1] += f" Values must be curies from one of these ontologies {schema_def['prefixes']}."
if "enum" in schema_def:
bad_enums = [v for v in column if v not in schema_def["enum"]]
if bad_enums:
errors.append(
f"Column {column_name} in dataframe {df_name} contains unpermitted values like "
f"{bad_enums[0]}. Values must be one of {schema_def['enum']}."
)
return errors
def _validate_dict(dict_, dict_name, schema_def):
"""Given a schema definition and dict, verify that the dict satifies the schema."""
errors = []
for key in schema_def.get("keys", []):
if key not in dict_:
errors.append(f"{dict_name} is missing key {key}.")
elif schema_def["keys"][key]:
if schema_def["keys"][key]["type"] == "stringified list of dicts":
if not _validate_stringified_list_of_dicts(dict_[key]):
errors.append(
f"Key {key} in {dict_name} should be a JSON-encoded list of dicts, but it is {dict_[key]}"
)
elif schema_def["keys"][key]["type"] == "dict":
errors.extend(_validate_dict(dict_[key], key, schema_def["keys"][key]))
elif schema_def["keys"][key]["type"] == "curie":
if not _validate_curie(dict_[key], schema_def["keys"][key]["prefixes"]):
errors.append(f"Key {key} in {dict_name} contains invalid ontology value.")
if "nullable" in schema_def["keys"][key] and not schema_def["keys"][key]["nullable"]:
if _is_null(dict_[key]):
errors.append(f"Key {key} in dict {dict_name} is an empty value.")
return errors
def _validate_dataframe(df, df_name, schema_def):
"""Given a dataframe and schema definition, verify that the dataframe follows the schema."""
errors = []
if "index" in schema_def:
errors.extend(_validate_column(df.index, "index", df_name, schema_def["index"]))
for column in schema_def.get("columns", []):
if column not in df.columns:
errors.append(f"Dataframe {df_name} is missing column {column}.")
else:
errors.extend(_validate_column(df[column], column, df_name, schema_def["columns"][column]))
return errors
def get_schema_definition(version):
"""Look up and read a schema definition based on a version number like "1.0.0"."""
path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "schema_definitions", version.replace(".", "_") + ".yaml"
)
if not os.path.isfile(path):
raise ValueError(f"No definition for version {version} found.")
return yaml.load(open(path), Loader=yaml.FullLoader)
def deep_check(adata, schema_def):
"""Perform a "deep" check of the AnnData object using the schema definition.
This checks all the columns and unstructured metadata rather than just the version.
Returns a list of error messages. If that list is empty, the object passed validation.
"""
errors = []
for component, component_def in schema_def["components"].items():
if component_def["type"] == "dataframe":
errors.extend(_validate_dataframe(getattr(adata, component), component, component_def))
elif component_def["type"] == "dict":
errors.extend(_validate_dict(getattr(adata, component), component, component_def))
else:
raise ValueError(f"Unexpected component type {component['type']}")
return errors
def validate_adata(adata, shallow):
"""Validate an AnnData object. If shallow, just check that the required version information is
present.
"""
# Does it have the version information written into uns?
if "version" not in adata.uns_keys() or "corpora_schema_version" not in adata.uns["version"]:
print("AnnData file is missing corpora version information")
return False
# We can stop here if it's a "shallow" check, that is, if we're just
# checking that version is present.
if shallow:
return True
schema_def = get_schema_definition(adata.uns["version"]["corpora_schema_version"])
errors = deep_check(adata, schema_def)
for error in errors:
print(error)
return not errors
def validate(h5ad_path, shallow=False):
"""Entry point for validation."""
try:
import scanpy
except ImportError:
raise ImportError("scanpy must be installed for cellxgene schema")
try:
adata = scanpy.read_h5ad(h5ad_path, backed="r")
except (OSError, TypeError):
print(f"Unable to open {h5ad_path} with scanpy.")
sys.exit(1)
if not validate_adata(adata, shallow):
sys.exit(1)
backend/server/default_config.py
+1 -26
View File
@@ -71,7 +71,7 @@ dataset:
external:
# You can retrieve configuration parameters from this config file, the environment,
# the AWS secrets manager, or from the "cellxgene launch" command line arguments.
# or from the "cellxgene launch" command line arguments.
# They are applied in that order, meaning that if a parameter is defined in more
# than one location, the last one applied takes effect.
@@ -87,31 +87,6 @@ external:
- name: CXG_SECRET_KEY
path: [server, app, flask_secret_key]
required: false
# AWS Secrets Manager
# This section describes how to map aws secrets to configuration parameters.
# The format is the region for the secrets manager, then a list of secrets.
# each secret has a name, and a list of values.
# Each entry in the list of values is a dictionary with three entries:
# key: the key of the aws secret.
# path: the path within the cellxgene configuration to update.
# required: (default=False) a boolean. If true, then it is an error if the key does not exist in the secret.
#
# example:
# aws_secrets_manager:
# region: us-west-2
# - name: my_first_secret
# values:
# - key: flask_secret_key
# path: [server, app, flask_secret_key]
# required: true
# - key: db_uri
# path: [dataset, user_annotations, db_uri]
# required: true
aws_secrets_manager:
region: null
secrets: []
"""