fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-30 08:08:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Introduce a config file to cellxgene (#1264)

Browse Source 
* Introduce a config file to cellxgene

The config file format is in yaml.  The default config is located
in server/common/default_config.py.  A user may create a yaml file
that contains a subset of these fields.  It can be used during cellxgene
launch, or for hosted cellxgene.

The code has also been refactored.  Much of the logic to check arguments
has moved from launch to app config.

It is now possible to set the tiledb context parameters using the config
file.  Other feature will soon be handled in a similar way.
This commit is contained in:
bmccandless
2020-03-22 09:34:11 -07:00
committed by GitHub
parent 1351c8f724
commit 8180be83b8
19 changed files with 691 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cli/launch.py
+117 -181
View File
@@ -1,40 +1,32 @@
import errno
import functools
import logging
from os import devnull, mkdir, environ
from os.path import splitext, basename, isdir
from os import devnull
import sys
import warnings
import webbrowser
from urllib.parse import urlparse
import click
from server.common.utils import custom_format_warning
from server.common.utils import find_available_port, is_port_available, sort_options
from server.common.errors import DatasetAccessError
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
from server.common.annotations import AnnotationsLocalFile
from server.common.utils import sort_options
from server.common.errors import DatasetAccessError, ConfigurationError
from server.data_common.matrix_loader import MatrixDataCacheManager
from server.common.app_config import AppConfig
from server.common.default_config import default_config
from server.common.errors import OntologyLoadFailure
# anything bigger than this will generate a special message
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
DEFAULT_SERVER_PORT = int(environ.get("CXG_SERVER_PORT", "5005"))
DEFAULT_CONFIG = AppConfig()
def annotation_args(func):
@click.option(
"--disable-annotations",
is_flag=True,
default=False,
default=not DEFAULT_CONFIG.user_annotations__enable,
show_default=True,
help="Disable user annotation of data.",
)
@click.option(
"--annotations-file",
default=None,
default=DEFAULT_CONFIG.user_annotations__local_file_csv__file,
show_default=True,
multiple=False,
metavar="<path>",
@@ -43,7 +35,7 @@ def annotation_args(func):
)
@click.option(
"--annotations-dir",
default=None,
default=DEFAULT_CONFIG.user_annotations__local_file_csv__directory,
show_default=False,
multiple=False,
metavar="<directory path>",
@@ -53,13 +45,13 @@ def annotation_args(func):
@click.option(
"--experimental-annotations-ontology",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.user_annotations__ontology__enable,
show_default=True,
help="When creating annotations, optionally autocomplete names from ontology terms.",
)
@click.option(
"--experimental-annotations-ontology-obo",
default=None,
default=DEFAULT_CONFIG.user_annotations__ontology__obo_location,
show_default=True,
metavar="<path or url>",
help="Location of OBO file defining cell annotation autosuggest terms.",
@@ -74,7 +66,7 @@ def annotation_args(func):
def config_args(func):
@click.option(
"--max-category-items",
default=1000,
default=DEFAULT_CONFIG.presentation__max_categories,
metavar="<integer>",
show_default=True,
help="Will not display categories with more distinct values than specified.",
@@ -82,7 +74,7 @@ def config_args(func):
@click.option(
"--diffexp-lfc-cutoff",
"-de",
default=0.01,
default=DEFAULT_CONFIG.diffexp__lfc_cutoff,
show_default=True,
metavar="<float>",
help="Minimum log fold change threshold for differential expression.",
@@ -90,14 +82,14 @@ def config_args(func):
@click.option(
"--disable-diffexp",
is_flag=True,
default=False,
default=not DEFAULT_CONFIG.diffexp__enable,
show_default=False,
help="Disable on-demand differential expression.",
)
@click.option(
"--embedding",
"-e",
default=[],
default=DEFAULT_CONFIG.embeddings__names,
multiple=True,
show_default=False,
metavar="<text>",
@@ -106,7 +98,7 @@ def config_args(func):
@click.option(
"--experimental-enable-reembedding",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.embeddings__enable_reembedding,
show_default=False,
hidden=True,
help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
@@ -122,14 +114,14 @@ def dataset_args(func):
@click.option(
"--obs-names",
"-obs",
default=None,
default=DEFAULT_CONFIG.single_dataset__obs_names,
metavar="<text>",
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
)
@click.option(
"--var-names",
"-var",
default=None,
default=DEFAULT_CONFIG.single_dataset__var_names,
metavar="<text>",
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
)
@@ -137,13 +129,20 @@ def dataset_args(func):
"--backed",
"-b",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.adaptor__anndata_adaptor__backed,
show_default=False,
help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.",
)
@click.option("--title", "-t", metavar="<text>", help="Title to display. If omitted will use file name.")
@click.option(
"--title",
"-t",
default=DEFAULT_CONFIG.single_dataset__title,
metavar="<text>",
help="Title to display. If omitted will use file name."
)
@click.option(
"--about",
default=DEFAULT_CONFIG.single_dataset__about,
metavar="<URL>",
help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).",
)
@@ -159,7 +158,7 @@ def server_args(func):
"--debug",
"-d",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.server__debug,
show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",
@@ -168,7 +167,7 @@ def server_args(func):
"--verbose",
"-v",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.server__verbose,
show_default=True,
help="Provide verbose output, including warnings and all server requests.",
)
@@ -176,21 +175,22 @@ def server_args(func):
"--port",
"-p",
metavar="<port>",
default=None,
default=DEFAULT_CONFIG.server__port,
type=int,
show_default=True,
help="Port to run server on. If not specified cellxgene will find an available port.",
)
@click.option(
"--host",
metavar="<IP address>",
default="127.0.0.1",
default=DEFAULT_CONFIG.server__host,
show_default=False,
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
)
@click.option(
"--scripts",
"-s",
default=[],
default=DEFAULT_CONFIG.server__scripts,
multiple=True,
metavar="<text>",
help="Additional script files to include in HTML page. If not specified, "
@@ -211,7 +211,7 @@ def launch_args(func):
@server_args
@click.option(
"--dataroot",
default=None,
default=DEFAULT_CONFIG.multi_dataset__dataroot,
metavar="<data directory>",
help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
" to folder containing H5AD and/or CXG datasets.",
@@ -223,10 +223,27 @@ def launch_args(func):
"-o",
"open_browser",
is_flag=True,
default=False,
default=DEFAULT_CONFIG.server__open_browser,
show_default=True,
help="Open web browser after launch.",
)
@click.option(
"--config-file",
"-c",
"config_file",
is_flag=True,
default=None,
show_default=True,
help="Location to yaml file with configuration settings",
)
@click.option(
"--dump-default-config",
"dump_default_config",
is_flag=True,
default=False,
show_default=True,
help="Print default configuration settings and exit",
)
@click.help_option("--help", "-h", help="Show this message and exit.")
@functools.wraps(func)
def wrapper(*args, **kwargs):
@@ -252,11 +269,6 @@ def handle_scripts(scripts):
click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True)
def handle_verbose(verbose):
if not verbose:
sys.tracebacklimit = 0
@sort_options
@click.command(
short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.",
@@ -287,6 +299,8 @@ def launch(
experimental_annotations_ontology,
experimental_annotations_ontology_obo,
experimental_enable_reembedding,
config_file,
dump_default_config
):
"""Launch the cellxgene data viewer.
This web app lets you explore single-cell expression data.
@@ -307,162 +321,79 @@ def launch(
#
# > cellxgene launch --dataroot <url>
if dump_default_config:
print(default_config)
sys.exit(0)
# Startup message
click.echo("[cellxgene] Starting the CLI...")
if datapath is None and dataroot is None:
# TODO: change the error message once dataroot is fully supported
raise click.ClickException('Missing argument "<path to data file>."')
# raise click.ClickException("must supply either <path to data file> or --dataroot")
if datapath is not None and dataroot is not None:
raise click.ClickException("must supply only one of <path to data file> or --dataroot")
if datapath:
# preload this data set
matrix_data_loader = MatrixDataLoader(datapath)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise click.ClickException(str(e))
if experimental_enable_reembedding:
if matrix_data_loader.matrix_data_type() != MatrixDataType.H5AD:
raise click.ClickException("--experimental-enable-reembedding is only supported with H5AD files.")
if backed:
raise click.ClickException(
"--experimental-enable-reembedding is not supported when run in --backed mode."
)
file_size = matrix_data_loader.file_size()
if file_size > BIG_FILE_SIZE_THRESHOLD:
click.echo(f"[cellxgene] Loading data from {basename(datapath)}, this may take a while...")
else:
click.echo(f"[cellxgene] Loading data from {basename(datapath)}.")
if debug:
verbose = True
open_browser = False
else:
warnings.formatwarning = custom_format_warning
handle_verbose(verbose)
handle_scripts(scripts)
if port:
if debug:
raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).")
if not is_port_available(host, int(port)):
raise click.ClickException(
f"The port selected {port} is in use, please specify an open port using the --port flag."
)
else:
port = find_available_port(host, DEFAULT_SERVER_PORT)
if disable_annotations:
if annotations_file is not None:
click.echo("Warning: --annotations-file ignored as annotations are disabled.")
if annotations_dir is not None:
click.echo("Warning: --annotations-dir ignored as annotations are disabled.")
if experimental_annotations_ontology:
click.echo("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
if experimental_annotations_ontology_obo is not None:
click.echo("Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled.")
else:
if annotations_file is not None and annotations_dir is not None:
raise click.ClickException(
"--annotations-file and --annotations-dir " "may not be used together."
)
if annotations_file is not None:
lf_name, lf_ext = splitext(annotations_file)
if lf_ext and lf_ext != ".csv":
raise click.FileError(basename(annotations_file), hint="annotation file type must be .csv")
if annotations_dir is not None and not isdir(annotations_dir):
try:
mkdir(annotations_dir)
except OSError:
raise click.ClickException(
"Unable to create directory specified by " "--annotations-dir"
)
if about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(about):
raise click.ClickException("Must provide an absolute URL for --about. (Example format: http://example.com)")
# Setup app
cellxgene_url = f"http://{host}:{port}"
# app config
app_config = AppConfig(
datapath=datapath,
dataroot=dataroot,
title=title,
about=about,
scripts=scripts,
layout=embedding,
max_category_items=max_category_items,
diffexp_lfc_cutoff=diffexp_lfc_cutoff,
obs_names=obs_names,
var_names=var_names,
anndata_backed=backed,
disable_diffexp=disable_diffexp,
enable_reembedding=experimental_enable_reembedding,
)
app_config = AppConfig()
matrix_data_cache_manager = MatrixDataCacheManager()
data_adaptor = None
if datapath:
try:
with matrix_data_cache_manager.data_adaptor(datapath, app_config) as data_adaptor:
if not disable_diffexp and data_adaptor.parameters.get("diffexp_may_be_slow", False):
click.echo(
f"[cellxgene] CAUTION: due to the size of your dataset, "
f"running differential expression may take longer or fail."
)
except Exception as e:
raise click.ClickException(str(e))
try:
if config_file:
app_config.update_from_config_file(config_file)
# create an annotations object. Only AnnotationsLocalFile is used (for now)
annotations = None
app_config.update(
server__verbose=verbose,
server__debug=debug,
server__host=host,
server__port=port,
server__scripts=scripts,
server__open_browser=open_browser,
if not disable_annotations:
annotations = AnnotationsLocalFile(annotations_dir, annotations_file)
single_dataset__datapath=datapath,
single_dataset__title=title,
single_dataset__about=about,
single_dataset__obs_names=obs_names,
single_dataset__var_names=var_names,
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
multi_dataset__dataroot=dataroot,
if annotations_file and data_adaptor:
data_adaptor.check_new_labels(annotations.read_labels(data_adaptor))
user_annotations__enable=not disable_annotations,
user_annotations__local_file_csv__file=annotations_file,
user_annotations__local_file_csv__directory=annotations_dir,
user_annotations__ontology__enable=experimental_annotations_ontology,
user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
if experimental_annotations_ontology or bool(experimental_annotations_ontology_obo):
try:
annotations.load_ontology(experimental_annotations_ontology_obo)
except OntologyLoadFailure as e:
raise click.ClickException("Unable to load ontology terms\n" + str(e))
presentation__max_categories=max_category_items,
embeddings__names=embedding,
embeddings__enable_reembedding=experimental_enable_reembedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
adaptor__anndata_adaptor__backed=backed,
)
# process the configuration
# any errors will be thrown as an exception.
# any info messages will be passed to the messagefn function.
matrix_data_cache_manager = MatrixDataCacheManager()
def messagefn(message):
click.echo("[cellxgene] " + message)
app_config.complete_config(matrix_data_cache_manager, messagefn)
except (ConfigurationError, DatasetAccessError) as e:
raise click.ClickException(e)
handle_scripts(scripts)
user_annotations = app_config.user_annotations
# create the server
from server.app.app import Server
server = Server(matrix_data_cache_manager, user_annotations, app_config)
server = Server(matrix_data_cache_manager, annotations, app_config)
if not verbose:
if not app_config.server__verbose:
log = logging.getLogger("werkzeug")
log.setLevel(logging.ERROR)
if open_browser:
cellxgene_url = f"http://{app_config.server__host}:{app_config.server__port}"
if app_config.server__open_browser:
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
webbrowser.open(cellxgene_url)
else:
@@ -470,12 +401,17 @@ def launch(
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not verbose:
if not app_config.server__verbose:
f = open(devnull, "w")
sys.stdout = f
try:
server.app.run(host=host, debug=debug, port=port, threaded=False if debug else True, use_debugger=False)
server.app.run(
host=app_config.server__host,
debug=app_config.server__debug,
port=app_config.server__port,
threaded=not app_config.server__debug,
use_debugger=False)
except OSError as e:
if e.errno == errno.EADDRINUSE:
raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e