fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-30 18:18:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor config to support different config options for datasets in different dataroots. (#1596)

Browse Source 
This will give us the ability to specify different config options for
different dataroots.

the key of the dataroot dictionary is no longer the same as the dataroot_url.
Previously key==dataroot_url, and now those are separated.

Added an "is_multi_dataset" function to simplify logic where it branched on single vs multi.

Simplified the rest.py interface by no longer passing in the user annotations object, since
that can be retrieved from the dataset.
This commit is contained in:
bmccandless
2020-07-10 16:21:40 -07:00
committed by GitHub
parent 13246cb6d1
commit f69d141336
21 changed files with 1021 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cli/launch.py
+52 -46
View File
@@ -22,13 +22,13 @@ def annotation_args(func):
@click.option(
"--disable-annotations",
is_flag=True,
default=not DEFAULT_CONFIG.user_annotations__enable,
default=not DEFAULT_CONFIG.default_dataset_config.user_annotations__enable,
show_default=True,
help="Disable user annotation of data.",
)
@click.option(
"--annotations-file",
default=DEFAULT_CONFIG.user_annotations__local_file_csv__file,
default=DEFAULT_CONFIG.default_dataset_config.user_annotations__local_file_csv__file,
show_default=True,
multiple=False,
metavar="<path>",
@@ -37,7 +37,7 @@ def annotation_args(func):
)
@click.option(
"--annotations-dir",
default=DEFAULT_CONFIG.user_annotations__local_file_csv__directory,
default=DEFAULT_CONFIG.default_dataset_config.user_annotations__local_file_csv__directory,
show_default=False,
multiple=False,
metavar="<directory path>",
@@ -47,13 +47,13 @@ def annotation_args(func):
@click.option(
"--experimental-annotations-ontology",
is_flag=True,
default=DEFAULT_CONFIG.user_annotations__ontology__enable,
default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__enable,
show_default=True,
help="When creating annotations, optionally autocomplete names from ontology terms.",
)
@click.option(
"--experimental-annotations-ontology-obo",
default=DEFAULT_CONFIG.user_annotations__ontology__obo_location,
default=DEFAULT_CONFIG.default_dataset_config.user_annotations__ontology__obo_location,
show_default=True,
metavar="<path or url>",
help="Location of OBO file defining cell annotation autosuggest terms.",
@@ -68,7 +68,7 @@ def annotation_args(func):
def config_args(func):
@click.option(
"--max-category-items",
default=DEFAULT_CONFIG.presentation__max_categories,
default=DEFAULT_CONFIG.default_dataset_config.presentation__max_categories,
metavar="<integer>",
show_default=True,
help="Will not display categories with more distinct values than specified.",
@@ -83,7 +83,7 @@ def config_args(func):
@click.option(
"--diffexp-lfc-cutoff",
"-de",
default=DEFAULT_CONFIG.diffexp__lfc_cutoff,
default=DEFAULT_CONFIG.default_dataset_config.diffexp__lfc_cutoff,
show_default=True,
metavar="<float>",
help="Minimum log fold change threshold for differential expression.",
@@ -91,14 +91,14 @@ def config_args(func):
@click.option(
"--disable-diffexp",
is_flag=True,
default=not DEFAULT_CONFIG.diffexp__enable,
default=not DEFAULT_CONFIG.default_dataset_config.diffexp__enable,
show_default=False,
help="Disable on-demand differential expression.",
)
@click.option(
"--embedding",
"-e",
default=DEFAULT_CONFIG.embeddings__names,
default=DEFAULT_CONFIG.default_dataset_config.embeddings__names,
multiple=True,
show_default=False,
metavar="<text>",
@@ -107,7 +107,7 @@ def config_args(func):
@click.option(
"--experimental-enable-reembedding",
is_flag=True,
default=DEFAULT_CONFIG.embeddings__enable_reembedding,
default=DEFAULT_CONFIG.default_dataset_config.embeddings__enable_reembedding,
show_default=False,
hidden=True,
help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.",
@@ -123,14 +123,14 @@ def dataset_args(func):
@click.option(
"--obs-names",
"-obs",
default=DEFAULT_CONFIG.single_dataset__obs_names,
default=DEFAULT_CONFIG.server_config.single_dataset__obs_names,
metavar="<text>",
help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.",
)
@click.option(
"--var-names",
"-var",
default=DEFAULT_CONFIG.single_dataset__var_names,
default=DEFAULT_CONFIG.server_config.single_dataset__var_names,
metavar="<text>",
help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.",
)
@@ -138,20 +138,20 @@ def dataset_args(func):
"--backed",
"-b",
is_flag=True,
default=DEFAULT_CONFIG.adaptor__anndata_adaptor__backed,
default=DEFAULT_CONFIG.server_config.adaptor__anndata_adaptor__backed,
show_default=False,
help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.",
)
@click.option(
"--title",
"-t",
default=DEFAULT_CONFIG.single_dataset__title,
default=DEFAULT_CONFIG.server_config.single_dataset__title,
metavar="<text>",
help="Title to display. If omitted will use file name.",
)
@click.option(
"--about",
default=DEFAULT_CONFIG.single_dataset__about,
default=DEFAULT_CONFIG.server_config.single_dataset__about,
metavar="<URL>",
help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
)
@@ -167,7 +167,7 @@ def server_args(func):
"--debug",
"-d",
is_flag=True,
default=DEFAULT_CONFIG.server__debug,
default=DEFAULT_CONFIG.server_config.app__debug,
show_default=True,
help="Run in debug mode. This is helpful for cellxgene developers, "
"or when you want more information about an error condition.",
@@ -176,7 +176,7 @@ def server_args(func):
"--verbose",
"-v",
is_flag=True,
default=DEFAULT_CONFIG.server__verbose,
default=DEFAULT_CONFIG.server_config.app__verbose,
show_default=True,
help="Provide verbose output, including warnings and all server requests.",
)
@@ -184,7 +184,7 @@ def server_args(func):
"--port",
"-p",
metavar="<port>",
default=DEFAULT_CONFIG.server__port,
default=DEFAULT_CONFIG.server_config.app__port,
type=int,
show_default=True,
help="Port to run server on. If not specified cellxgene will find an available port.",
@@ -192,14 +192,14 @@ def server_args(func):
@click.option(
"--host",
metavar="<IP address>",
default=DEFAULT_CONFIG.server__host,
default=DEFAULT_CONFIG.server_config.app__host,
show_default=False,
help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).",
)
@click.option(
"--scripts",
"-s",
default=DEFAULT_CONFIG.server__scripts,
default=DEFAULT_CONFIG.default_dataset_config.app__scripts,
multiple=True,
metavar="<text>",
help="Additional script files to include in HTML page. If not specified, "
@@ -220,7 +220,7 @@ def launch_args(func):
@server_args
@click.option(
"--dataroot",
default=DEFAULT_CONFIG.multi_dataset__dataroot,
default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
metavar="<data directory>",
help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
" to folder containing H5AD and/or CXG datasets.",
@@ -232,7 +232,7 @@ def launch_args(func):
"-o",
"open_browser",
is_flag=True,
default=DEFAULT_CONFIG.server__open_browser,
default=DEFAULT_CONFIG.server_config.app__open_browser,
show_default=True,
help="Open web browser after launch.",
)
@@ -296,7 +296,7 @@ class CliLaunchServer(Server):
"application/octet-stream",
]
Compress(app)
if app_config.server__debug:
if app_config.server_config.app__debug:
CORS(app, supports_credentials=True)
@@ -362,6 +362,7 @@ def launch(
# app config
app_config = AppConfig()
server_config = app_config.server_config
try:
if config_file:
@@ -370,19 +371,22 @@ def launch(
# Determine which config options were give on the command line.
# Those will override the ones provided in the config file (if provided).
cli_config = AppConfig()
cli_config.update(
server__verbose=verbose,
server__debug=debug,
server__host=host,
server__port=port,
server__scripts=scripts,
server__open_browser=open_browser,
cli_config.update_server_config(
app__verbose=verbose,
app__debug=debug,
app__host=host,
app__port=port,
app__open_browser=open_browser,
single_dataset__datapath=datapath,
single_dataset__title=title,
single_dataset__about=about,
single_dataset__obs_names=obs_names,
single_dataset__var_names=var_names,
multi_dataset__dataroot=dataroot,
adaptor__anndata_adaptor__backed=backed,
)
cli_config.update_default_dataset_config(
app__scripts=scripts,
user_annotations__enable=not disable_annotations,
user_annotations__local_file_csv__file=annotations_file,
user_annotations__local_file_csv__directory=annotations_dir,
@@ -394,13 +398,15 @@ def launch(
embeddings__enable_reembedding=experimental_enable_reembedding,
diffexp__enable=not disable_diffexp,
diffexp__lfc_cutoff=diffexp_lfc_cutoff,
adaptor__anndata_adaptor__backed=backed,
)
diff = cli_config.changes_from_default()
changes = {}
for key, val, defval in diff:
changes[key] = val
app_config.update(**changes)
diff = cli_config.server_config.changes_from_default()
changes = {key: val for key, val, _ in diff}
app_config.update_server_config(**changes)
diff = cli_config.default_dataset_config.changes_from_default()
changes = {key: val for key, val, _ in diff}
app_config.update_default_dataset_config(**changes)
# process the configuration
# any errors will be thrown as an exception.
@@ -410,8 +416,8 @@ def launch(
click.echo("[cellxgene] " + message)
# Use a default secret if one is not provided
if not app_config.server__flask_secret_key:
app_config.update(server__flask_secret_key="SparkleAndShine")
if not server_config.app__flask_secret_key:
app_config.update_server_config(app__flask_secret_key="SparkleAndShine")
app_config.complete_config(messagefn)
@@ -423,12 +429,12 @@ def launch(
# create the server
server = CliLaunchServer(app_config)
if not app_config.server__verbose:
if not server_config.app__verbose:
log = logging.getLogger("werkzeug")
log.setLevel(logging.ERROR)
cellxgene_url = f"http://{app_config.server__host}:{app_config.server__port}"
if app_config.server__open_browser:
cellxgene_url = f"http://{app_config.server_config.app__host}:{app_config.server_config.app__port}"
if server_config.app__open_browser:
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
webbrowser.open(cellxgene_url)
else:
@@ -436,16 +442,16 @@ def launch(
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
if not app_config.server__verbose:
if not server_config.app__verbose:
f = open(devnull, "w")
sys.stdout = f
try:
server.app.run(
host=app_config.server__host,
debug=app_config.server__debug,
port=app_config.server__port,
threaded=not app_config.server__debug,
host=server_config.app__host,
debug=server_config.app__debug,
port=server_config.app__port,
threaded=not server_config.app__debug,
use_debugger=False,
use_reloader=False,
)