mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-25 02:28:12 +08:00
move common code into server, update tests and makefile (#2425)
* move common code into server, update tests and makefile remove backend directory, refactor update smoke tests
This commit is contained in:
@@ -0,0 +1,34 @@
|
||||
import click
|
||||
|
||||
from .launch import launch
|
||||
from .prepare import prepare
|
||||
from .upgrade import log_upgrade_check
|
||||
from .. import __version__
|
||||
|
||||
|
||||
@click.group(
|
||||
name="cellxgene",
|
||||
subcommand_metavar="COMMAND <args>",
|
||||
options_metavar="<options>",
|
||||
context_settings=dict(max_content_width=85, help_option_names=["-h", "--help"]),
|
||||
)
|
||||
@click.help_option("--help", "-h", help="Show this message and exit.")
|
||||
@click.version_option(
|
||||
version=__version__,
|
||||
prog_name="cellxgene",
|
||||
message="[%(prog)s] Version %(version)s",
|
||||
help="Show the software version and exit.",
|
||||
)
|
||||
@click.option(
|
||||
"--upgrade-check/--no-upgrade-check",
|
||||
default=True,
|
||||
show_default=True,
|
||||
help="Check for release upgrades on start.",
|
||||
)
|
||||
def cli(upgrade_check):
|
||||
if upgrade_check:
|
||||
log_upgrade_check()
|
||||
|
||||
|
||||
cli.add_command(launch)
|
||||
cli.add_command(prepare)
|
||||
@@ -0,0 +1,449 @@
|
||||
import errno
|
||||
import functools
|
||||
import logging
|
||||
import sys
|
||||
import webbrowser
|
||||
import os
|
||||
import click
|
||||
from flask_compress import Compress
|
||||
from flask_cors import CORS
|
||||
|
||||
from server.default_config import default_config
|
||||
from server.app.app import Server
|
||||
from server.common.config.app_config import AppConfig
|
||||
from server.common.errors import DatasetAccessError, ConfigurationError
|
||||
from server.common.utils.utils import sort_options
|
||||
|
||||
DEFAULT_CONFIG = AppConfig()
|
||||
|
||||
|
||||
def annotation_args(func):
|
||||
@click.option(
|
||||
"--disable-annotations",
|
||||
is_flag=True,
|
||||
default=not DEFAULT_CONFIG.dataset_config.user_annotations__enable,
|
||||
show_default=True,
|
||||
help="Disable user annotation of data.",
|
||||
)
|
||||
@click.option(
|
||||
"--annotations-file",
|
||||
default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__file,
|
||||
show_default=True,
|
||||
multiple=False,
|
||||
metavar="<path>",
|
||||
help="CSV file to initialize editing of existing annotations; will be altered in-place. "
|
||||
"Incompatible with --user-generated-data-dir.",
|
||||
)
|
||||
@click.option(
|
||||
"--user-generated-data-dir",
|
||||
"--annotations-dir",
|
||||
default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__directory,
|
||||
show_default=False,
|
||||
multiple=False,
|
||||
metavar="<directory path>",
|
||||
help="Directory of where to save output annotations; filename will be specified in the application. "
|
||||
"Incompatible with --annotations-file and --gene-sets-file.",
|
||||
)
|
||||
@click.option(
|
||||
"--disable-gene-sets-save",
|
||||
is_flag=True,
|
||||
default=DEFAULT_CONFIG.dataset_config.user_annotations__gene_sets__readonly,
|
||||
show_default=False,
|
||||
help="Disable saving gene sets. If disabled, users will be able to make changes to gene sets but all "
|
||||
"changes will be lost on browser refresh.",
|
||||
)
|
||||
@click.option(
|
||||
"--gene-sets-file",
|
||||
default=DEFAULT_CONFIG.dataset_config.user_annotations__local_file_csv__gene_sets_file,
|
||||
show_default=True,
|
||||
multiple=False,
|
||||
metavar="<path>",
|
||||
help="CSV file to initialize editing of gene sets; will be altered in-place. Incompatible with "
|
||||
"--user-generated-data-dir.",
|
||||
)
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def config_args(func):
|
||||
@click.option(
|
||||
"--max-category-items",
|
||||
default=DEFAULT_CONFIG.dataset_config.presentation__max_categories,
|
||||
metavar="<integer>",
|
||||
show_default=True,
|
||||
help="Will not display categories with more distinct values than specified.",
|
||||
)
|
||||
@click.option(
|
||||
"--disable-custom-colors",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
show_default=False,
|
||||
help="Disable user-defined category-label colors drawn from source data file.",
|
||||
)
|
||||
@click.option(
|
||||
"--diffexp-lfc-cutoff",
|
||||
"-de",
|
||||
default=DEFAULT_CONFIG.dataset_config.diffexp__lfc_cutoff,
|
||||
show_default=True,
|
||||
metavar="<float>",
|
||||
help="Minimum log fold change threshold for differential expression.",
|
||||
)
|
||||
@click.option(
|
||||
"--disable-diffexp",
|
||||
is_flag=True,
|
||||
default=not DEFAULT_CONFIG.dataset_config.diffexp__enable,
|
||||
show_default=False,
|
||||
help="Disable on-demand differential expression.",
|
||||
)
|
||||
@click.option(
|
||||
"--embedding",
|
||||
"-e",
|
||||
default=DEFAULT_CONFIG.dataset_config.embeddings__names,
|
||||
multiple=True,
|
||||
show_default=False,
|
||||
metavar="<text>",
|
||||
help="Embedding name, eg, 'umap'. Repeat option for multiple embeddings. Defaults to all.",
|
||||
)
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def dataset_args(func):
|
||||
@click.option(
|
||||
"--obs-names",
|
||||
"-obs",
|
||||
default=DEFAULT_CONFIG.server_config.single_dataset__obs_names,
|
||||
metavar="<text>",
|
||||
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
|
||||
)
|
||||
@click.option(
|
||||
"--var-names",
|
||||
"-var",
|
||||
default=DEFAULT_CONFIG.server_config.single_dataset__var_names,
|
||||
metavar="<text>",
|
||||
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
|
||||
)
|
||||
@click.option(
|
||||
"--backed",
|
||||
"-b",
|
||||
is_flag=True,
|
||||
default=DEFAULT_CONFIG.server_config.adaptor__anndata_adaptor__backed,
|
||||
show_default=False,
|
||||
help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.",
|
||||
)
|
||||
@click.option(
|
||||
"--title",
|
||||
"-t",
|
||||
default=DEFAULT_CONFIG.server_config.single_dataset__title,
|
||||
metavar="<text>",
|
||||
help="Title to display. If omitted will use file name.",
|
||||
)
|
||||
@click.option(
|
||||
"--about",
|
||||
default=DEFAULT_CONFIG.server_config.single_dataset__about,
|
||||
metavar="<URL>",
|
||||
help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
|
||||
)
|
||||
@click.option(
|
||||
"--X-approximate-distribution",
|
||||
default=DEFAULT_CONFIG.dataset_config.X_approximate_distribution,
|
||||
show_default=True,
|
||||
type=click.Choice(["auto", "normal", "count"], case_sensitive=False),
|
||||
help="Specify the approximate distribution of X matrix values. 'auto' will use a heuristic "
|
||||
"to determine the approximate distribution. Mode 'auto' is incompatible with --backed.",
|
||||
)
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def server_args(func):
|
||||
@click.option(
|
||||
"--debug",
|
||||
"-d",
|
||||
is_flag=True,
|
||||
default=DEFAULT_CONFIG.server_config.app__debug,
|
||||
show_default=True,
|
||||
help="Run in debug mode. This is helpful for cellxgene developers, "
|
||||
"or when you want more information about an error condition.",
|
||||
)
|
||||
@click.option(
|
||||
"--verbose",
|
||||
"-v",
|
||||
is_flag=True,
|
||||
default=DEFAULT_CONFIG.server_config.app__verbose,
|
||||
show_default=True,
|
||||
help="Provide verbose output, including warnings and all server requests.",
|
||||
)
|
||||
@click.option(
|
||||
"--port",
|
||||
"-p",
|
||||
metavar="<port>",
|
||||
default=DEFAULT_CONFIG.server_config.app__port,
|
||||
type=int,
|
||||
show_default=True,
|
||||
help="Port to run server on. If not specified cellxgene will find an available port.",
|
||||
)
|
||||
@click.option(
|
||||
"--host",
|
||||
metavar="<IP address>",
|
||||
default=DEFAULT_CONFIG.server_config.app__host,
|
||||
show_default=False,
|
||||
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
|
||||
)
|
||||
@click.option(
|
||||
"--scripts",
|
||||
"-s",
|
||||
default=DEFAULT_CONFIG.dataset_config.app__scripts,
|
||||
multiple=True,
|
||||
metavar="<text>",
|
||||
help="Additional script files to include in HTML page. If not specified, "
|
||||
"no additional script files will be included.",
|
||||
show_default=False,
|
||||
)
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def launch_args(func):
|
||||
@annotation_args
|
||||
@config_args
|
||||
@dataset_args
|
||||
@server_args
|
||||
@click.argument("datapath", required=False, metavar="<path to data file>")
|
||||
@click.option(
|
||||
"--open",
|
||||
"-o",
|
||||
"open_browser",
|
||||
is_flag=True,
|
||||
default=DEFAULT_CONFIG.server_config.app__open_browser,
|
||||
show_default=True,
|
||||
help="Open web browser after launch.",
|
||||
)
|
||||
@click.option(
|
||||
"--config-file",
|
||||
"-c",
|
||||
"config_file",
|
||||
default=None,
|
||||
show_default=True,
|
||||
help="Location to yaml file with configuration settings",
|
||||
)
|
||||
@click.option(
|
||||
"--dump-default-config",
|
||||
"dump_default_config",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
show_default=True,
|
||||
help="Print default configuration settings and exit",
|
||||
)
|
||||
@click.help_option("--help", "-h", help="Show this message and exit.")
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def handle_scripts(scripts):
|
||||
if scripts:
|
||||
click.echo(
|
||||
r"""
|
||||
/ / /\ \ \__ _ _ __ _ __ (_)_ __ __ _
|
||||
\ \/ \/ / _` | '__| '_ \| | '_ \ / _` |
|
||||
\ /\ / (_| | | | | | | | | | | (_| |
|
||||
\/ \/ \__,_|_| |_| |_|_|_| |_|\__, |
|
||||
|___/
|
||||
The --scripts flag is intended for developers to include google analytics etc. You could be opening yourself to a
|
||||
security risk by including the --scripts flag. Make sure you trust the scripts that you are including.
|
||||
"""
|
||||
)
|
||||
scripts_pretty = ", ".join(scripts)
|
||||
click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
|
||||
|
||||
|
||||
class CliLaunchServer(Server):
|
||||
"""
|
||||
the CLI runs a local web server, and needs to enable a few more features.
|
||||
"""
|
||||
|
||||
def __init__(self, app_config):
|
||||
super().__init__(app_config)
|
||||
|
||||
@staticmethod
|
||||
def _before_adding_routes(app, app_config):
|
||||
app.config["COMPRESS_MIMETYPES"] = [
|
||||
"text/html",
|
||||
"text/css",
|
||||
"text/xml",
|
||||
"application/json",
|
||||
"application/javascript",
|
||||
"application/octet-stream",
|
||||
]
|
||||
Compress(app)
|
||||
if app_config.server_config.app__debug:
|
||||
CORS(app, supports_credentials=True)
|
||||
|
||||
|
||||
@sort_options
|
||||
@click.command(
|
||||
short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
|
||||
options_metavar="<options>",
|
||||
)
|
||||
@launch_args
|
||||
def launch(
|
||||
datapath,
|
||||
verbose,
|
||||
debug,
|
||||
open_browser,
|
||||
port,
|
||||
host,
|
||||
embedding,
|
||||
obs_names,
|
||||
var_names,
|
||||
max_category_items,
|
||||
disable_custom_colors,
|
||||
diffexp_lfc_cutoff,
|
||||
title,
|
||||
scripts,
|
||||
about,
|
||||
disable_annotations,
|
||||
annotations_file,
|
||||
user_generated_data_dir,
|
||||
gene_sets_file,
|
||||
disable_gene_sets_save,
|
||||
backed,
|
||||
disable_diffexp,
|
||||
config_file,
|
||||
dump_default_config,
|
||||
x_approximate_distribution,
|
||||
):
|
||||
"""Launch the cellxgene data viewer.
|
||||
This web app lets you explore single-cell expression data.
|
||||
Data must be in a format that cellxgene expects.
|
||||
Read the "getting started" guide to learn more:
|
||||
https://chanzuckerberg.github.io/cellxgene/getting-started.html
|
||||
|
||||
Examples:
|
||||
|
||||
> cellxgene launch example-dataset/pbmc3k.h5ad --title pbmc3k
|
||||
|
||||
> cellxgene launch <your data file> --title <your title>
|
||||
|
||||
> cellxgene launch <url>"""
|
||||
|
||||
if dump_default_config:
|
||||
print(default_config)
|
||||
sys.exit(0)
|
||||
|
||||
# Startup message
|
||||
click.echo("[cellxgene] Starting the CLI...")
|
||||
|
||||
# app config
|
||||
app_config = AppConfig()
|
||||
server_config = app_config.server_config
|
||||
|
||||
try:
|
||||
if config_file:
|
||||
app_config.update_from_config_file(config_file)
|
||||
|
||||
# Determine which config options were give on the command line.
|
||||
# Those will override the ones provided in the config file (if provided).
|
||||
cli_config = AppConfig()
|
||||
cli_config.update_server_config(
|
||||
app__verbose=verbose,
|
||||
app__debug=debug,
|
||||
app__host=host,
|
||||
app__port=port,
|
||||
app__open_browser=open_browser,
|
||||
single_dataset__datapath=datapath,
|
||||
single_dataset__title=title,
|
||||
single_dataset__about=about,
|
||||
single_dataset__obs_names=obs_names,
|
||||
single_dataset__var_names=var_names,
|
||||
adaptor__anndata_adaptor__backed=backed,
|
||||
)
|
||||
cli_config.update_dataset_config(
|
||||
app__scripts=scripts,
|
||||
user_annotations__enable=not disable_annotations,
|
||||
user_annotations__local_file_csv__file=annotations_file,
|
||||
user_annotations__local_file_csv__directory=user_generated_data_dir,
|
||||
user_annotations__local_file_csv__gene_sets_file=gene_sets_file,
|
||||
user_annotations__gene_sets__readonly=disable_gene_sets_save,
|
||||
presentation__max_categories=max_category_items,
|
||||
presentation__custom_colors=not disable_custom_colors,
|
||||
embeddings__names=embedding,
|
||||
diffexp__enable=not disable_diffexp,
|
||||
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
|
||||
X_approximate_distribution=x_approximate_distribution,
|
||||
)
|
||||
|
||||
diff = cli_config.server_config.changes_from_default()
|
||||
changes = {key: val for key, val, _ in diff}
|
||||
app_config.update_server_config(**changes)
|
||||
|
||||
diff = cli_config.dataset_config.changes_from_default()
|
||||
changes = {key: val for key, val, _ in diff}
|
||||
app_config.update_dataset_config(**changes)
|
||||
|
||||
# process the configuration
|
||||
# any errors will be thrown as an exception.
|
||||
# any info messages will be passed to the messagefn function.
|
||||
|
||||
def messagefn(message):
|
||||
click.echo("[cellxgene] " + message)
|
||||
|
||||
# Use a default secret if one is not provided
|
||||
if not server_config.app__flask_secret_key:
|
||||
app_config.update_server_config(app__flask_secret_key="SparkleAndShine")
|
||||
|
||||
app_config.complete_config(messagefn)
|
||||
|
||||
except (ConfigurationError, DatasetAccessError) as e:
|
||||
raise click.ClickException(e)
|
||||
|
||||
handle_scripts(scripts)
|
||||
|
||||
# create the server
|
||||
server = CliLaunchServer(app_config)
|
||||
|
||||
if not server_config.app__verbose:
|
||||
log = logging.getLogger("werkzeug")
|
||||
log.setLevel(logging.ERROR)
|
||||
|
||||
cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
|
||||
if server_config.app__open_browser:
|
||||
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
|
||||
webbrowser.open(cellxgene_url)
|
||||
else:
|
||||
click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.")
|
||||
|
||||
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
|
||||
|
||||
if not server_config.app__verbose:
|
||||
f = open(os.devnull, "w")
|
||||
sys.stdout = f
|
||||
|
||||
try:
|
||||
server.app.run(
|
||||
host=server_config.app__host,
|
||||
debug=server_config.app__debug,
|
||||
port=server_config.app__port,
|
||||
threaded=not server_config.app__debug,
|
||||
use_debugger=False,
|
||||
use_reloader=False,
|
||||
)
|
||||
except OSError as e:
|
||||
if e.errno == errno.EADDRINUSE:
|
||||
raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
|
||||
raise
|
||||
@@ -0,0 +1,278 @@
|
||||
from os.path import expanduser, isdir, isfile, sep, splitext
|
||||
|
||||
import click
|
||||
import pandas as pd
|
||||
from numpy import ndarray, unique
|
||||
from scipy.sparse.csc import csc_matrix
|
||||
|
||||
from server.common.utils.utils import sort_options
|
||||
|
||||
|
||||
@sort_options
|
||||
@click.command(
|
||||
short_help="Preprocess data for use with cellxgene. " "Run `cellxgene prepare --help` for more information.",
|
||||
options_metavar="<options>",
|
||||
)
|
||||
@click.argument("data", nargs=1, metavar="<path to data file>", required=True)
|
||||
@click.option(
|
||||
"--embedding",
|
||||
"-e",
|
||||
default=["umap", "tsne"],
|
||||
multiple=True,
|
||||
type=click.Choice(["umap", "tsne"]),
|
||||
help="Embedding algorithm(s). Repeat option for multiple embeddings.",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--recipe",
|
||||
"-r",
|
||||
default="none",
|
||||
type=click.Choice(["none", "seurat", "zheng17"]),
|
||||
show_default=True,
|
||||
)
|
||||
@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
|
||||
@click.option("--plotting", "-p", default=False, is_flag=True, help="Generate plots.", show_default=True)
|
||||
@click.option("--sparse", default=False, is_flag=True, help="Force sparsity.", show_default=True)
|
||||
@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
|
||||
@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
|
||||
@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
|
||||
@click.option(
|
||||
"--skip-qc",
|
||||
default=False,
|
||||
is_flag=True,
|
||||
help="Do not run quality control metrics. By default cellxgene runs them "
|
||||
"(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
|
||||
)
|
||||
@click.option(
|
||||
"--make-obs-names-unique/--no-make-obs-names-unique",
|
||||
default=True,
|
||||
help="Ensure obs index is unique.",
|
||||
show_default=True,
|
||||
)
|
||||
@click.option(
|
||||
"--make-var-names-unique/--no-make-var-names-unique",
|
||||
default=True,
|
||||
help="Ensure var index is unique.",
|
||||
show_default=True,
|
||||
)
|
||||
@click.help_option("--help", "-h", help="Show this message and exit.")
|
||||
def prepare(
|
||||
data,
|
||||
embedding,
|
||||
recipe,
|
||||
output,
|
||||
plotting,
|
||||
sparse,
|
||||
overwrite,
|
||||
set_obs_names,
|
||||
set_var_names,
|
||||
skip_qc,
|
||||
make_obs_names_unique,
|
||||
make_var_names_unique,
|
||||
):
|
||||
"""
|
||||
Preprocess data for use with cellxgene.
|
||||
This tool runs a series of scanpy routines for preparing a dataset for use
|
||||
with cellxgene. It loads data from different formats
|
||||
(h5ad, loom, or a 10x directory), runs dimensionality reduction,
|
||||
computes nearest neighbors, computes an embedding, performs clustering,
|
||||
and saves the results. Includes additional options for naming annotations,
|
||||
ensuring sparsity, and plotting results.
|
||||
"""
|
||||
|
||||
# collect slow imports here to make CLI startup more responsive
|
||||
click.echo("[cellxgene] Starting CLI...")
|
||||
try:
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import scanpy as sc
|
||||
except ImportError:
|
||||
raise click.ClickException(
|
||||
"[cellxgene] cellxgene prepare has not been installed. Please run `pip install 'cellxgene[prepare]'` "
|
||||
"to install the necessary requirements."
|
||||
)
|
||||
|
||||
# scanpy settings
|
||||
sc.settings.verbosity = 0
|
||||
sc.settings.autosave = True
|
||||
|
||||
# check args
|
||||
if sparse and not recipe == "none":
|
||||
raise click.UsageError("Cannot use a recipe when forcing sparsity")
|
||||
|
||||
output = expanduser(output)
|
||||
|
||||
if not output:
|
||||
click.echo(
|
||||
"Warning: No file will be saved, to save the results of cellxgene prepare include "
|
||||
"--output <filename> to save output to a new file"
|
||||
)
|
||||
if isfile(output) and not overwrite:
|
||||
raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
|
||||
|
||||
def load_data(data):
|
||||
if isfile(data):
|
||||
name, extension = splitext(data)
|
||||
if extension == ".h5ad":
|
||||
adata = sc.read_h5ad(data)
|
||||
elif extension == ".loom":
|
||||
adata = sc.read_loom(data)
|
||||
else:
|
||||
raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]")
|
||||
elif isdir(data):
|
||||
if not data.endswith(sep):
|
||||
data += sep
|
||||
adata = sc.read_10x_mtx(data)
|
||||
else:
|
||||
raise click.FileError(data, hint="not a valid file or path")
|
||||
|
||||
if not set_obs_names == "":
|
||||
if set_obs_names not in adata.obs_keys():
|
||||
raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
|
||||
adata.obs_names = adata.obs[set_obs_names]
|
||||
if not set_var_names == "":
|
||||
if set_var_names not in adata.var_keys():
|
||||
raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
|
||||
adata.var_names = adata.var[set_var_names]
|
||||
if make_obs_names_unique:
|
||||
adata.obs.index = make_index_unique(adata.obs.index)
|
||||
if make_var_names_unique:
|
||||
adata.var.index = make_index_unique(adata.var.index)
|
||||
if not adata._obs.index.is_unique:
|
||||
click.echo("Warning: obs index is not unique")
|
||||
if not adata._var.index.is_unique:
|
||||
click.echo("Warning: var index is not unique")
|
||||
return adata
|
||||
|
||||
def calculate_qc_metrics(adata):
|
||||
if not skip_qc:
|
||||
sc.pp.calculate_qc_metrics(adata, inplace=True)
|
||||
return adata
|
||||
|
||||
def make_sparse(adata):
|
||||
if (type(adata.X) is ndarray) and sparse:
|
||||
adata.X = csc_matrix(adata.X)
|
||||
|
||||
def run_recipe(adata):
|
||||
if recipe == "seurat":
|
||||
sc.pp.recipe_seurat(adata)
|
||||
elif recipe == "zheng17":
|
||||
sc.pp.recipe_zheng17(adata)
|
||||
else:
|
||||
sc.pp.filter_cells(adata, min_genes=5)
|
||||
sc.pp.filter_genes(adata, min_cells=25)
|
||||
if sparse:
|
||||
sc.pp.scale(adata, zero_center=False)
|
||||
else:
|
||||
sc.pp.scale(adata)
|
||||
|
||||
def run_pca(adata):
|
||||
if sparse:
|
||||
sc.pp.pca(adata, svd_solver="arpack", zero_center=False)
|
||||
else:
|
||||
sc.pp.pca(adata, svd_solver="arpack")
|
||||
|
||||
def run_neighbors(adata):
|
||||
sc.pp.neighbors(adata)
|
||||
|
||||
def run_louvain(adata):
|
||||
sc.tl.louvain(adata)
|
||||
|
||||
def run_embedding(adata):
|
||||
if len(unique(adata.obs["louvain"].values)) < 10:
|
||||
palette = "tab10"
|
||||
else:
|
||||
palette = "tab20"
|
||||
|
||||
if "umap" in embedding:
|
||||
sc.tl.umap(adata)
|
||||
if plotting:
|
||||
sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain")
|
||||
|
||||
if "tsne" in embedding:
|
||||
sc.tl.tsne(adata)
|
||||
if plotting:
|
||||
sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain")
|
||||
|
||||
def show_step(item):
|
||||
if not skip_qc:
|
||||
qc_name = "Calculating QC metrics"
|
||||
else:
|
||||
qc_name = "Skipping QC"
|
||||
names = {
|
||||
"calculate_qc_metrics": qc_name,
|
||||
"make_sparse": "Ensuring sparsity",
|
||||
"run_recipe": f'Running preprocessing recipe "{recipe}"',
|
||||
"run_pca": "Running PCA",
|
||||
"run_neighbors": "Calculating neighbors",
|
||||
"run_louvain": "Calculating clusters",
|
||||
"run_embedding": "Computing embedding",
|
||||
}
|
||||
if item is not None:
|
||||
return names[item.__name__]
|
||||
|
||||
steps = [calculate_qc_metrics, make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_embedding]
|
||||
|
||||
click.echo(f"[cellxgene] Loading data from {data}, please wait...")
|
||||
adata = load_data(data)
|
||||
|
||||
click.echo("[cellxgene] Beginning preprocessing...")
|
||||
with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar:
|
||||
for step in bar:
|
||||
step(adata)
|
||||
|
||||
# saving
|
||||
if not output == "":
|
||||
click.echo(f"[cellxgene] Saving results to {output}...")
|
||||
adata.write(output)
|
||||
|
||||
click.echo("[cellxgene] Success!")
|
||||
|
||||
|
||||
# TODO (mweiden): remove this once this issue is resolved https://github.com/theislab/anndata/issues/344
|
||||
# Note: tentative solution here https://github.com/theislab/anndata/pull/345
|
||||
def make_index_unique(index: pd.Index, join: str = "-"):
|
||||
"""
|
||||
Makes the index unique by appending a number string to each duplicate index element: '1', '2', etc.
|
||||
|
||||
If a tentative name created by the algorithm already exists in the index, it tries the next integer in the sequence.
|
||||
|
||||
The first occurrence of a non-unique value is ignored.
|
||||
Parameters
|
||||
----------
|
||||
join
|
||||
The connecting string between name and integer.
|
||||
Examples
|
||||
--------
|
||||
>>> from anndata import AnnData
|
||||
>>> adata1 = AnnData(np.ones((3, 2)), dict(obs_names=['a', 'b', 'c']))
|
||||
>>> adata2 = AnnData(np.zeros((3, 2)), dict(obs_names=['d', 'b', 'b']))
|
||||
>>> adata = adata1.concatenate(adata2)
|
||||
>>> adata.obs_names
|
||||
Index(['a', 'b', 'c', 'd', 'b', 'b'], dtype='object')
|
||||
>>> adata.obs_names_make_unique()
|
||||
>>> adata.obs_names
|
||||
Index(['a', 'b', 'c', 'd', 'b-1', 'b-2'], dtype='object')
|
||||
"""
|
||||
if index.is_unique:
|
||||
return index
|
||||
from collections import defaultdict
|
||||
|
||||
values = index.values
|
||||
values_set = set(values)
|
||||
indices_dup = index.duplicated(keep="first")
|
||||
values_dup = values[indices_dup]
|
||||
counter = defaultdict(lambda: 0)
|
||||
for i, v in enumerate(values_dup):
|
||||
while True:
|
||||
counter[v] += 1
|
||||
tentative_new_name = v + join + str(counter[v])
|
||||
if tentative_new_name not in values_set:
|
||||
values_set.add(tentative_new_name)
|
||||
values_dup[i] = tentative_new_name
|
||||
break
|
||||
|
||||
values[indices_dup] = values_dup
|
||||
index = pd.Index(values)
|
||||
return index
|
||||
@@ -0,0 +1,85 @@
|
||||
import re
|
||||
|
||||
import click
|
||||
import requests
|
||||
from requests.exceptions import ConnectionError
|
||||
|
||||
from .. import __version__
|
||||
|
||||
# Official SemVer regex: https://semver.org/
|
||||
SEMVER_FORMAT = re.compile(
|
||||
r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
|
||||
r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
|
||||
r"?:\.[0-9a-zA-Z-]+)*))?$"
|
||||
)
|
||||
|
||||
|
||||
def log_upgrade_check():
|
||||
# Sanity-check that the CLI version is a properly-formatted SemVer string
|
||||
assert validate_version_str(__version__, release_only=False)
|
||||
|
||||
# Get the current latest release
|
||||
try:
|
||||
release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases())
|
||||
latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name))
|
||||
if version_gt(latest_release, __version__):
|
||||
click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True)
|
||||
click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True)
|
||||
except (ConnectionError, RateLimitException):
|
||||
click.echo("Upgrade check failed.\n")
|
||||
|
||||
|
||||
class RateLimitException(Exception):
|
||||
"""
|
||||
Github API Rate Limit Exception
|
||||
"""
|
||||
|
||||
|
||||
def _request_cellxgene_releases():
|
||||
def raise_on_rate_limit(response):
|
||||
if response.status_code == 403 and res.headers.get("X-RateLimit-Remaining") == "0":
|
||||
raise RateLimitException
|
||||
|
||||
url = "https://api.github.com/repos/chanzuckerberg/cellxgene/releases"
|
||||
res = requests.get(url)
|
||||
raise_on_rate_limit(res)
|
||||
for release in res.json():
|
||||
yield release
|
||||
while "next" in res.links.keys():
|
||||
res = requests.get(res.links["next"]["url"])
|
||||
raise_on_rate_limit(res)
|
||||
for release in res.json():
|
||||
yield release
|
||||
|
||||
|
||||
def validate_version_str(version_str, release_only=True):
|
||||
"""
|
||||
Test if a string conforms to SemVer format (https://semver.org/)
|
||||
:param version_str: a string to be validated
|
||||
:param release_only: only declare releases (not prereleases) valid
|
||||
:return: True if the version string is of a valid SemVer format else False
|
||||
"""
|
||||
match = SEMVER_FORMAT.match(version_str)
|
||||
has_match = match is not None
|
||||
if has_match and release_only:
|
||||
return not match.group("prerelease")
|
||||
return has_match
|
||||
|
||||
|
||||
def split_version(version_string):
|
||||
"""
|
||||
Split a SemVer-formatted string into its component integers
|
||||
:param version_string: a SemVer string to be split
|
||||
:return: an array of three integers
|
||||
"""
|
||||
match = SEMVER_FORMAT.match(version_string)
|
||||
return [int(match.group(group)) for group in ["major", "minor", "patch"]]
|
||||
|
||||
|
||||
def version_gt(left_version, right_version):
|
||||
for left, right in zip(split_version(left_version), split_version(right_version)):
|
||||
if left > right:
|
||||
return True
|
||||
elif right > left:
|
||||
return False
|
||||
return False
|
||||
Reference in New Issue
Block a user