fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-25 13:58:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

Browse Source 
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
2021-03-26 00:27:07 -05:00
committed by GitHub
parent e6e358ddc8
commit 78c9d24ed4
425 changed files with 734 additions and 5317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cli/__init__.py
View File
server/cli/cli.py
-35
View File
@@ -1,35 +0,0 @@
import click
from .convert_to_cxg import convert_to_cxg
from .launch import launch
from .prepare import prepare
from .upgrade import log_upgrade_check
from .schema import schema_cli
from .. import __version__
@click.group(
name="cellxgene",
subcommand_metavar="COMMAND <args>",
options_metavar="<options>",
context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
)
@click.help_option("--help", "-h", help="Show this message and exit.")
@click.version_option(
version=__version__,
prog_name="cellxgene",
message="[%(prog)s] Version %(version)s",
help="Show the software version and exit.",
)
@click.option(
"--upgrade-check/--no-upgrade-check", default=True, show_default=True, help="Check for release upgrades on start.",
)
def cli(upgrade_check):
if upgrade_check:
log_upgrade_check()
cli.add_command(launch)
cli.add_command(prepare)
cli.add_command(convert_to_cxg)
cli.add_command(schema_cli)
server/cli/convert_to_cxg.py
-133
View File
@@ -1,133 +0,0 @@
from os import path
import click
from server.converters.h5ad_data_file import H5ADDataFile
@click.command(
name="convert",
short_help="Converts an H5AD dataset to the CXG format.",
help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
"that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
"environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
"usually with the generated CXG file.",
)
@click.argument(
"input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"-o",
"--output-directory",
help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
"CXG extension.",
)
@click.option(
"-b",
"--backed",
help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
"but will use less memory.",
default=False,
show_default=True,
is_flag=True,
)
@click.option(
"-t",
"--title",
help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
"the dataset title will be the filename.",
)
@click.option(
"-a",
"--about",
help="A fully qualified URL that provides more information about the dataset and will be included as "
"metadata about the CXG file.",
)
@click.option(
"-s",
"--sparse-threshold",
help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
"array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
"convert to dense array.",
default=0.0,
show_default=True,
)
@click.option(
"--obs-names",
help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.",
)
@click.option(
"--var-names",
help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
"the one designated by the dataframe generated-index.",
)
@click.option(
"--disable-custom-colors",
help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
default=False,
show_default=True,
is_flag=True,
)
@click.option(
"--disable-corpora-schema",
help="When set, conversion process will neither extract nor store Corpora schema information. See "
"https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
"information.",
default=False,
show_default=True,
is_flag=True,
)
@click.option(
"--overwrite",
help="When set to true, will overwrite the output file if the output file already exists.",
default=False,
show_default=True,
is_flag=True,
)
@click.help_option("--help", "-h", help="Show this message and exit.")
def convert_to_cxg(
input_file,
output_directory,
backed,
title,
about,
sparse_threshold,
obs_names,
var_names,
disable_custom_colors,
disable_corpora_schema,
overwrite,
):
"""
Convert a dataset file into CXG.
"""
h5ad_data_file = H5ADDataFile(
input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
)
# Get the directory that will hold all the CXG files
cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
h5ad_data_file.to_cxg(
cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
)
def get_output_directory(input_filename, output_directory, should_overwrite):
"""
Get the name of the CXG output directory to be created/populated during the dataset conversion.
"""
if output_directory and (not path.isdir(output_directory) or (path.isdir(output_directory) and should_overwrite)):
if output_directory.endswith(".cxg"):
return output_directory
return output_directory + ".cxg"
if output_directory and path.isdir(output_directory) and not should_overwrite:
raise click.BadParameter(
f"Output directory {output_directory} already exists. If you'd like to overwrite, then run the command "
f"with the --overwrite flag."
)
return path.splitext(input_filename)[0] + ".cxg"
server/cli/launch.py
-460
View File
@@ -1,460 +0,0 @@
import errno
import functools
import logging
import sys
import webbrowser
import os
import click
from flask_compress import Compress
from flask_cors import CORS
from server.default_config import default_config
from server.app.app import Server
from server.common.config.app_config import AppConfig
from server.common.errors import DatasetAccessError, ConfigurationError
from server.common.utils.utils import sort_options
DEFAULT_CONFIG = AppConfig()
def annotation_args(func):
@click.option(
"--disable-annotations",
is_flag=True,
default=not DEFAULT_CONFIG.default_dataset_config.user_annotations__enable,
show_default=True,
help="Disable user annotation of data.",
)
@click.option(
"--annotations-file",
default=DEFAULT_CONFIG.default_dataset_config.user_annotations__local_file_csv__file,
show_default=True,
multiple=False,
metavar="<path>",
help="CSV file to initialize editing of existing annotations; will be altered in-place. "
"Incompatible with --annotations-dir.",
)
@click.option(
"--annotations-dir",
default=DEFAULT_CONFIG.default_dataset_config.user_annotations__local_file_csv__directory,
show_default=False,
multiple=False,
metavar="<directory path>",
help="Directory of where to save output annotations; filename will be specified in the application. "
"Incompatible with --annotations-file.",
)
@click.option(
"--experimental-annotations-ontology",
is_flag=True,
default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__enable,
show_default=True,
help="When creating annotations, optionally autocomplete names from ontology terms.",
)
@click.option(
"--experimental-annotations-ontology-obo",
default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__obo_location,
show_default=True,
metavar="<path or url>",
help="Location of OBO file defining cell annotation autosuggest terms.",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
def config_args(func):
@click.option(
"--max-category-items",
default=DEFAULT_CONFIG.default_dataset_config.presentation__max_categories,
metavar="<integer>",
show_default=True,
help="Will not display categories with more distinct values than specified.",
)
@click.option(
"--disable-custom-colors",
is_flag=True,
default=False,
show_default=False,
help="Disable user-defined category-label colors drawn from source data file.",
)
@click.option(
"--diffexp-lfc-cutoff",
"-de",
default=DEFAULT_CONFIG.default_dataset_config.diffexp__lfc_cutoff,
show_default=True,
metavar="<float>",
help="Minimum log fold change threshold for differential expression.",
)
@click.option(
"--disable-diffexp",
is_flag=True,
default=not DEFAULT_CONFIG.default_dataset_config.diffexp__enable,
show_default=False,
help="Disable on-demand differential expression.",
)
@click.option(
"--embedding",
"-e",
default=DEFAULT_CONFIG.default_dataset_config.embeddings__names,
multiple=True,
show_default=False,
metavar="<text>",
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
)
@click.option(
"--experimental-enable-reembedding",
is_flag=True,
default=DEFAULT_CONFIG.default_dataset_config.embeddings__enable_reembedding,
show_default=False,
hidden=True,
help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
def dataset_args(func):
@click.option(
"--obs-names",
"-obs",
default=DEFAULT_CONFIG.server_config.single_dataset__obs_names,
metavar="<text>",
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
)
@click.option(
"--var-names",
"-var",
default=DEFAULT_CONFIG.server_config.single_dataset__var_names,
metavar="<text>",
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
)
@click.option(
"--backed",
"-b",
is_flag=True,
default=DEFAULT_CONFIG.server_config.adaptor__anndata_adaptor__backed,
show_default=False,
help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.",
)
@click.option(
"--title",
"-t",
default=DEFAULT_CONFIG.server_config.single_dataset__title,
metavar="<text>",
help="Title to display. If omitted will use file name.",
)
@click.option(
"--about",
default=DEFAULT_CONFIG.server_config.single_dataset__about,
metavar="<URL>",
help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
def server_args(func):
@click.option(
"--debug",
"-d",
is_flag=True,
default=DEFAULT_CONFIG.server_config.app__debug,
show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",
)
@click.option(
"--verbose",
"-v",
is_flag=True,
default=DEFAULT_CONFIG.server_config.app__verbose,
show_default=True,
help="Provide verbose output, including warnings and all server requests.",
)
@click.option(
"--port",
"-p",
metavar="<port>",
default=DEFAULT_CONFIG.server_config.app__port,
type=int,
show_default=True,
help="Port to run server on. If not specified cellxgene will find an available port.",
)
@click.option(
"--host",
metavar="<IP address>",
default=DEFAULT_CONFIG.server_config.app__host,
show_default=False,
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
)
@click.option(
"--scripts",
"-s",
default=DEFAULT_CONFIG.default_dataset_config.app__scripts,
multiple=True,
metavar="<text>",
help="Additional script files to include in HTML page. If not specified, "
"no additional script files will be included.",
show_default=False,
)
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
def launch_args(func):
@annotation_args
@config_args
@dataset_args
@server_args
@click.option(
"--dataroot",
default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
metavar="<data directory>",
help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
" to folder containing H5AD and/or CXG datasets.",
hidden=True,
) # TODO, unhide when dataroot is supported)
@click.argument("datapath", required=False, metavar="<path to data file>")
@click.option(
"--open",
"-o",
"open_browser",
is_flag=True,
default=DEFAULT_CONFIG.server_config.app__open_browser,
show_default=True,
help="Open web browser after launch.",
)
@click.option(
"--config-file",
"-c",
"config_file",
default=None,
show_default=True,
help="Location to yaml file with configuration settings",
)
@click.option(
"--dump-default-config",
"dump_default_config",
is_flag=True,
default=False,
show_default=True,
help="Print default configuration settings and exit",
)
@click.help_option("--help", "-h", help="Show this message and exit.")
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
return wrapper
def handle_scripts(scripts):
if scripts:
click.echo(
r"""
/ / /\ \ \__ _ _ __ _ __ (_)_ __ __ _
\ \/ \/ / _` | '__| '_ \| | '_ \ / _` |
\ /\ / (_| | | | | | | | | | | (_| |
\/ \/ \__,_|_| |_| |_|_|_| |_|\__, |
|___/
The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a
security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
"""
)
scripts_pretty = ", ".join(scripts)
click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
class CliLaunchServer(Server):
"""
the CLI runs a local web server, and needs to enable a few more features.
"""
def __init__(self, app_config):
super().__init__(app_config)
@staticmethod
def _before_adding_routes(app, app_config):
app.config["COMPRESS_MIMETYPES"] = [
"text/html",
"text/css",
"text/xml",
"application/json",
"application/javascript",
"application/octet-stream",
]
Compress(app)
if app_config.server_config.app__debug:
CORS(app, supports_credentials=True)
@sort_options
@click.command(
short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
options_metavar="<options>",
)
@launch_args
def launch(
datapath,
dataroot,
verbose,
debug,
open_browser,
port,
host,
embedding,
obs_names,
var_names,
max_category_items,
disable_custom_colors,
diffexp_lfc_cutoff,
title,
scripts,
about,
disable_annotations,
annotations_file,
annotations_dir,
backed,
disable_diffexp,
experimental_annotations_ontology,
experimental_annotations_ontology_obo,
experimental_enable_reembedding,
config_file,
dump_default_config,
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
Data must be in a format that cellxgene expects.
Read the "getting started" guide to learn more:
https://chanzuckerberg.github.io/cellxgene/getting-started.html
Examples:
> cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k
> cellxgene launch <your data file> --title <your title>
> cellxgene launch <url>"""
# TODO Examples to provide when "--dataroot" is unhidden
# > cellxgene launch --dataroot example-dataset/
#
# > cellxgene launch --dataroot <url>
if dump_default_config:
print(default_config)
sys.exit(0)
# Startup message
click.echo("[cellxgene] Starting the CLI...")
# app config
app_config = AppConfig()
server_config = app_config.server_config
try:
if config_file:
app_config.update_from_config_file(config_file)
# Determine which config options were give on the command line.
# Those will override the ones provided in the config file (if provided).
cli_config = AppConfig()
cli_config.update_server_config(
app__verbose=verbose,
app__debug=debug,
app__host=host,
app__port=port,
app__open_browser=open_browser,
single_dataset__datapath=datapath,
single_dataset__title=title,
single_dataset__about=about,
single_dataset__obs_names=obs_names,
single_dataset__var_names=var_names,
multi_dataset__dataroot=dataroot,
adaptor__anndata_adaptor__backed=backed,
)
cli_config.update_default_dataset_config(
app__scripts=scripts,
user_annotations__enable=not disable_annotations,
user_annotations__local_file_csv__file=annotations_file,
user_annotations__local_file_csv__directory=annotations_dir,
user_annotations__ontology__enable=experimental_annotations_ontology,
user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
presentation__max_categories=max_category_items,
presentation__custom_colors=not disable_custom_colors,
embeddings__names=embedding,
embeddings__enable_reembedding=experimental_enable_reembedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
)
diff = cli_config.server_config.changes_from_default()
changes = {key: val for key, val, _ in diff}
app_config.update_server_config(**changes)
diff = cli_config.default_dataset_config.changes_from_default()
changes = {key: val for key, val, _ in diff}
app_config.update_default_dataset_config(**changes)
# process the configuration
# any errors will be thrown as an exception.
# any info messages will be passed to the messagefn function.
def messagefn(message):
click.echo("[cellxgene] " + message)
# Use a default secret if one is not provided
if not server_config.app__flask_secret_key:
app_config.update_server_config(app__flask_secret_key="SparkleAndShine")
app_config.complete_config(messagefn)
except (ConfigurationError, DatasetAccessError) as e:
raise click.ClickException(e)
handle_scripts(scripts)
# create the server
server = CliLaunchServer(app_config)
if not server_config.app__verbose:
log = logging.getLogger("werkzeug")
log.setLevel(logging.ERROR)
cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
if server_config.app__open_browser:
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
webbrowser.open(cellxgene_url)
else:
click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.")
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not server_config.app__verbose:
f = open(os.devnull, "w")
sys.stdout = f
try:
server.app.run(
host=server_config.app__host,
debug=server_config.app__debug,
port=server_config.app__port,
threaded=not server_config.app__debug,
use_debugger=False,
use_reloader=False,
)
except OSError as e:
if e.errno == errno.EADDRINUSE:
raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
raise
server/cli/prepare.py
-274
View File
@@ -1,274 +0,0 @@
from os.path import expanduser, isdir, isfile, sep, splitext
import click
import pandas as pd
from numpy import ndarray, unique
from scipy.sparse.csc import csc_matrix
from server.common.utils.utils import sort_options
@sort_options
@click.command(
short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.",
options_metavar="<options>",
)
@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
@click.option(
"--embedding",
"-e",
default=["umap", "tsne"],
multiple=True,
type=click.Choice(["umap", "tsne"]),
help="Embedding algorithm(s). Repeat option for multiple embeddings.",
show_default=True,
)
@click.option(
"--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]), show_default=True,
)
@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True)
@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True)
@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
@click.option(
"--skip-qc",
default=False,
is_flag=True,
help="Do not run quality control metrics. By default cellxgene runs them "
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
)
@click.option(
"--make-obs-names-unique/--no-make-obs-names-unique",
default=True,
help="Ensure obs index is unique.",
show_default=True,
)
@click.option(
"--make-var-names-unique/--no-make-var-names-unique",
default=True,
help="Ensure var index is unique.",
show_default=True,
)
@click.help_option("--help", "-h", help="Show this message and exit.")
def prepare(
data,
embedding,
recipe,
output,
plotting,
sparse,
overwrite,
set_obs_names,
set_var_names,
skip_qc,
make_obs_names_unique,
make_var_names_unique,
):
"""
Preprocess data for use with cellxgene.
This tool runs a series of scanpy routines for preparing a dataset for use
with cellxgene. It loads data from different formats
(h5ad, loom, or a 10x directory), runs dimensionality reduction,
computes nearest neighbors, computes an embedding, performs clustering,
and saves the results. Includes additional options for naming annotations,
ensuring sparsity, and plotting results.
"""
# collect slow imports here to make CLI startup more responsive
click.echo("[cellxgene] Starting CLI...")
try:
import matplotlib
matplotlib.use("Agg")
import scanpy as sc
except ImportError:
raise click.ClickException(
"[cellxgene] cellxgene prepare has not been installed. Please run `pip install 'cellxgene[prepare]'` "
"to install the necessary requirements."
)
# scanpy settings
sc.settings.verbosity = 0
sc.settings.autosave = True
# check args
if sparse and not recipe == "none":
raise click.UsageError("Cannot use a recipe when forcing sparsity")
output = expanduser(output)
if not output:
click.echo(
"Warning: No file will be saved, to save the results of cellxgene prepare include "
"--output <filename> to save output to a new file"
)
if isfile(output) and not overwrite:
raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
def load_data(data):
if isfile(data):
name, extension = splitext(data)
if extension == ".h5ad":
adata = sc.read_h5ad(data)
elif extension == ".loom":
adata = sc.read_loom(data)
else:
raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]")
elif isdir(data):
if not data.endswith(sep):
data += sep
adata = sc.read_10x_mtx(data)
else:
raise click.FileError(data, hint="not a valid file or path")
if not set_obs_names == "":
if set_obs_names not in adata.obs_keys():
raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
adata.obs_names = adata.obs[set_obs_names]
if not set_var_names == "":
if set_var_names not in adata.var_keys():
raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
adata.var_names = adata.var[set_var_names]
if make_obs_names_unique:
adata.obs.index = make_index_unique(adata.obs.index)
if make_var_names_unique:
adata.var.index = make_index_unique(adata.var.index)
if not adata._obs.index.is_unique:
click.echo("Warning: obs index is not unique")
if not adata._var.index.is_unique:
click.echo("Warning: var index is not unique")
return adata
def calculate_qc_metrics(adata):
if not skip_qc:
sc.pp.calculate_qc_metrics(adata, inplace=True)
return adata
def make_sparse(adata):
if (type(adata.X) is ndarray) and sparse:
adata.X = csc_matrix(adata.X)
def run_recipe(adata):
if recipe == "seurat":
sc.pp.recipe_seurat(adata)
elif recipe == "zheng17":
sc.pp.recipe_zheng17(adata)
else:
sc.pp.filter_cells(adata, min_genes=5)
sc.pp.filter_genes(adata, min_cells=25)
if sparse:
sc.pp.scale(adata, zero_center=False)
else:
sc.pp.scale(adata)
def run_pca(adata):
if sparse:
sc.pp.pca(adata, svd_solver="arpack", zero_center=False)
else:
sc.pp.pca(adata, svd_solver="arpack")
def run_neighbors(adata):
sc.pp.neighbors(adata)
def run_louvain(adata):
sc.tl.louvain(adata)
def run_embedding(adata):
if len(unique(adata.obs["louvain"].values)) < 10:
palette = "tab10"
else:
palette = "tab20"
if "umap" in embedding:
sc.tl.umap(adata)
if plotting:
sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain")
if "tsne" in embedding:
sc.tl.tsne(adata)
if plotting:
sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain")
def show_step(item):
if not skip_qc:
qc_name = "Calculating QC metrics"
else:
qc_name = "Skipping QC"
names = {
"calculate_qc_metrics": qc_name,
"make_sparse": "Ensuring sparsity",
"run_recipe": f'Running preprocessing recipe "{recipe}"',
"run_pca": "Running PCA",
"run_neighbors": "Calculating neighbors",
"run_louvain": "Calculating clusters",
"run_embedding": "Computing embedding",
}
if item is not None:
return names[item.__name__]
steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding]
click.echo(f"[cellxgene] Loading data from {data}, please wait...")
adata = load_data(data)
click.echo("[cellxgene] Beginning preprocessing...")
with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar:
for step in bar:
step(adata)
# saving
if not output == "":
click.echo(f"[cellxgene] Saving results to {output}...")
adata.write(output)
click.echo("[cellxgene] Success!")
# TODO (mweiden): remove this once this issue is resolved https://github.com/theislab/anndata/issues/344
# Note: tentative solution here https://github.com/theislab/anndata/pull/345
def make_index_unique(index: pd.Index, join: str = "-"):
"""
Makes the index unique by appending a number string to each duplicate index element: '1', '2', etc.
If a tentative name created by the algorithm already exists in the index, it tries the next integer in the sequence.
The first occurrence of a non-unique value is ignored.
Parameters
----------
join
The connecting string between name and integer.
Examples
--------
>>> from anndata import AnnData
>>> adata1 = AnnData(np.ones((3, 2)), dict(obs_names=['a', 'b', 'c']))
>>> adata2 = AnnData(np.zeros((3, 2)), dict(obs_names=['d', 'b', 'b']))
>>> adata = adata1.concatenate(adata2)
>>> adata.obs_names
Index(['a', 'b', 'c', 'd', 'b', 'b'], dtype='object')
>>> adata.obs_names_make_unique()
>>> adata.obs_names
Index(['a', 'b', 'c', 'd', 'b-1', 'b-2'], dtype='object')
"""
if index.is_unique:
return index
from collections import defaultdict
values = index.values
values_set = set(values)
indices_dup = index.duplicated(keep="first")
values_dup = values[indices_dup]
counter = defaultdict(lambda: 0)
for i, v in enumerate(values_dup):
while True:
counter[v] += 1
tentative_new_name = v + join + str(counter[v])
if tentative_new_name not in values_set:
values_set.add(tentative_new_name)
values_dup[i] = tentative_new_name
break
values[indices_dup] = values_dup
index = pd.Index(values)
return index
server/cli/schema.py
-72
View File
@@ -1,72 +0,0 @@
import click
from server.converters.schema import remix, validate
@click.group(
name="schema",
subcommand_metavar="COMMAND <args>",
short_help="Apply and validate the cellxgene data integration schema to an h5ad file.",
context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
)
def schema_cli():
try:
import scanpy # noqa: F401
except ImportError:
raise click.ClickException(
"[cellxgene] cellxgene schema requires scanpy"
)
@click.command(
name="apply",
short_help="(experimental) Apply the cellxgene data integration schema to an h5ad.",
help="(experimental) Using a yaml file that describes schema values to insert or convert and in input "
"h5ad file, apply the schema changes and create a new, conforming h5ad.",
)
@click.option(
"--source-h5ad",
help="Input h5ad file.",
nargs=1,
required=True,
type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"--remix-config",
help="Config yaml with information on how to apply the schema.",
nargs=1,
required=True,
type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"--output-filename",
help="Filename for the new, schema-conforming h5ad file.",
required=True,
nargs=1
)
def schema_apply(source_h5ad, remix_config, output_filename):
remix.apply_schema(source_h5ad, remix_config, output_filename)
@click.command(
name="validate",
short_help="(experimental) Check that an h5ad follows the cellxgene data integration schema.",
)
@click.argument(
"h5ad",
nargs=1,
type=click.Path(exists=True, dir_okay=False),
)
@click.option(
"--shallow",
help="When true, just check that the correct version information is present.",
default=False,
show_default=True,
is_flag=True,
)
def schema_validate(h5ad, shallow):
validate.validate(h5ad, shallow)
schema_cli.add_command(schema_apply)
schema_cli.add_command(schema_validate)
server/cli/upgrade.py
-85
View File
@@ -1,85 +0,0 @@
import re
import click
import requests
from requests.exceptions import ConnectionError
from .. import __version__
# Official SemVer regex: https://semver.org/
SEMVER_FORMAT = re.compile(
r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
r"?:\.[0-9a-zA-Z-]+)*))?$"
)
def log_upgrade_check():
# Sanity-check that the CLI version is a properly-formatted SemVer string
assert validate_version_str(__version__, release_only=False)
# Get the current latest release
try:
release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases())
latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name))
if version_gt(latest_release, __version__):
click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True)
click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True)
except (ConnectionError, RateLimitException):
click.echo("Upgrade check failed.\n")
class RateLimitException(Exception):
"""
Github API Rate Limit Exception
"""
def _request_cellxgene_releases():
def raise_on_rate_limit(response):
if response.status_code == 403 and res.headers.get("X-RateLimit-Remaining") == "0":
raise RateLimitException
url = "https://api.github.com/repos/chanzuckerberg/cellxgene/releases"
res = requests.get(url)
raise_on_rate_limit(res)
for release in res.json():
yield release
while "next" in res.links.keys():
res = requests.get(res.links["next"]["url"])
raise_on_rate_limit(res)
for release in res.json():
yield release
def validate_version_str(version_str, release_only=True):
"""
Test if a string conforms to SemVer format (https://semver.org/)
:param version_str: a string to be validated
:param release_only: only declare releases (not prereleases) valid
:return: True if the version string is of a valid SemVer format else False
"""
match = SEMVER_FORMAT.match(version_str)
has_match = match is not None
if has_match and release_only:
return not match.group("prerelease")
return has_match
def split_version(version_string):
"""
Split a SemVer-formatted string into its component integers
:param version_string: a SemVer string to be split
:return: an array of three integers
"""
match = SEMVER_FORMAT.match(version_string)
return [int(match.group(group)) for group in ["major", "minor", "patch"]]
def version_gt(left_version, right_version):
for left, right in zip(split_version(left_version), split_version(right_version)):
if left > right:
return True
elif right > left:
return False
return False