fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-27 05:58:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Launch validation (#414)

Browse Source 
* WIP

* Cleanup

* Validation

* typo

* Max category limit 100 -> 500
This commit is contained in:
Charlotte Weaver
2018-11-07 13:01:13 -08:00
committed by GitHub
parent a9189017c7
commit 7d5c054b90
10 changed files with 204 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cli/cli.py
+2 -2
View File
@@ -4,8 +4,8 @@ from .launch import launch
from .prepare import prepare
@click.group(name='cellxgene', context_settings=dict(max_content_width=85))
@click.version_option(version='0.0.1', prog_name='cellxgene', message='[%(prog)s] Version %(version)s')
@click.group(name="cellxgene", context_settings=dict(max_content_width=85))
@click.version_option(version="0.0.1", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
def cli():
pass
server/cli/launch.py
+42 -33
View File
@@ -1,31 +1,32 @@
import sys
import click
import logging
from os.path import splitext, basename
import webbrowser
from os.path import splitext, basename
from server.app.util.errors import ScanpyFileError
@click.command()
@click.argument('data', metavar='<data file>', type=click.Path(exists=True, file_okay=True, dir_okay=False))
@click.option('--layout', '-l', type=click.Choice(['umap', 'tsne']), default='umap', show_default=True,
help='Method for layout.')
@click.option('--diffexp', '-d', type=click.Choice(['ttest']), default='ttest', show_default=True,
help='Method for differential expression.')
@click.option('--title', '-t', help='Title to display (if omitted will use file name).', metavar='')
@click.option('--verbose', '-v', is_flag=True, default=False, show_default=True,
help='Provide verbose output, including warnings and all server requests.')
@click.option('--debug', '-d', is_flag=True, default=False, show_default=True,
help='Run in debug mode.')
@click.option('--open', '-o', 'open_browser', is_flag=True, default=False, show_default=True,
help='Open the web browser after launch.')
@click.option('--port', '-p', help="Port to run server on.", metavar='', default=5005, show_default=True)
@click.option('--obs-names', default=None, metavar='', help='Name of annotation field to use for observations.')
@click.option('--var-names', default=None, metavar='', help='Name of annotation to use for variables.')
@click.option('--listen-all', is_flag=True, default=False, show_default=True,
help='Bind to all interfaces (this makes the server accessible beyond this computer).')
@click.option('--max-category-items', default=100, metavar='', show_default=True,
help='Limits the number of categorical annotation items displayed.')
@click.argument("data", metavar="<data file>", type=click.Path(exists=True, file_okay=True, dir_okay=False))
@click.option("--layout", "-l", type=click.Choice(["umap", "tsne"]), default="umap", show_default=True,
help="Method for layout.")
@click.option("--diffexp", "-d", type=click.Choice(["ttest"]), default="ttest", show_default=True,
help="Method for differential expression.")
@click.option("--title", "-t", help="Title to display (if omitted will use file name).", metavar="")
@click.option("--verbose", "-v", is_flag=True, default=False, show_default=True,
help="Provide verbose output, including warnings and all server requests.")
@click.option("--debug", "-d", is_flag=True, default=False, show_default=True,
help="Run in debug mode.")
@click.option("--open", "-o", "open_browser", is_flag=True, default=False, show_default=True,
help="Open the web browser after launch.")
@click.option("--port", "-p", help="Port to run server on.", metavar="", default=5005, show_default=True)
@click.option("--obs-names", default=None, metavar="", help="Name of annotation field to use for observations.")
@click.option("--var-names", default=None, metavar="", help="Name of annotation to use for variables.")
@click.option("--listen-all", is_flag=True, default=False, show_default=True,
help="Bind to all interfaces (this makes the server accessible beyond this computer).")
@click.option("--max-category-items", default=100, metavar="", show_default=True,
help="Limits the number of categorical annotation items displayed.")
def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
open_browser, port, listen_all, max_category_items):
"""Launch the cellxgene data viewer.
@@ -40,15 +41,15 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
> cellxgene launch <your data file> --title <your title>"""
# Startup message
click.echo('[cellxgene] Starting the CLI...')
click.echo("[cellxgene] Starting the CLI...")
# Import Flask app
from server.app.app import app
# Argument checking
name, extension = splitext(data)
if extension != '.h5ad':
raise click.FileError(basename(data), hint='file type must be .h5ad')
if extension != ".h5ad":
raise click.FileError(basename(data), hint="file type must be .h5ad")
if debug:
verbose = True
@@ -62,9 +63,9 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
title = file_parts[0]
if listen_all:
host = '0.0.0.0'
host = "0.0.0.0"
else:
host = '127.0.0.1'
host = "127.0.0.1"
# Setup app
cellxgene_url = f"http://{host}:{port}"
@@ -76,24 +77,32 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
)
if not verbose:
log = logging.getLogger('werkzeug')
log = logging.getLogger("werkzeug")
log.setLevel(logging.ERROR)
click.echo(f'[cellxgene] Loading data from {basename(data)}, this may take awhile...')
click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...")
from server.app.scanpy_engine.scanpy_engine import ScanpyEngine
args = {'layout': layout, 'diffexp': diffexp, 'max_category_items': max_category_items,
'obs_names': obs_names, 'var_names': var_names}
args = {
"layout": layout,
"diffexp": diffexp,
"max_category_items": max_category_items,
"obs_names": obs_names,
"var_names": var_names
}
app.data = ScanpyEngine(data, args)
try:
app.data = ScanpyEngine(data, args)
except ScanpyFileError as e:
raise click.ClickException(f"{e}")
if open_browser:
click.echo(f'[cellxgene] Launching! Opening your browser to {cellxgene_url} now.')
click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.")
webbrowser.open(cellxgene_url)
else:
click.echo(f'[cellxgene] Launching! Please go to {cellxgene_url} in your browser.')
click.echo(f"[cellxgene] Launching! Please go to {cellxgene_url} in your browser.")
click.echo('[cellxgene] Type CTRL-C at any time to exit.')
click.echo("[cellxgene] Type CTRL-C at any time to exit.")
app.run(host=host, debug=debug, port=port, threaded=True)
server/cli/prepare.py
+56 -56
View File
@@ -1,26 +1,26 @@
import click
from os.path import expanduser, isdir, isfile, sep, splitext
from numpy import unique, ndarray
import click
from numpy import ndarray, unique
from scipy.sparse.csc import csc_matrix
from os.path import isfile, isdir, splitext, expanduser, sep
@click.command()
@click.argument('data', nargs=1, metavar='<dataset: file or path to data>', required=True)
@click.option('--layout', '-l', default=['umap', 'tsne'], multiple=True, type=click.Choice(['umap', 'tsne']),
help='Layout algorithm', show_default=True)
@click.option('--recipe', '-r', default='none', type=click.Choice(['none', 'seurat', 'zheng17']),
help='Preprocessing to run.', show_default=True)
@click.option('--output', '-o', default='', help='Save a new file to filename.', metavar='<filename>')
@click.option('--plotting', '-p', default=False, is_flag=True, help='Whether to generate plots.', show_default=True)
@click.option('--sparse', default=False, is_flag=True, help='Whether to force sparsity.', show_default=True)
@click.option('--overwrite', default=False, is_flag=True, help='Allow file overwriting.', show_default=True)
@click.option('--set-obs-names', default='', help='Named field to set as index for obs.', metavar='<name>')
@click.option('--set-var-names', default='', help='Named field to set as index for var.', metavar='<name>')
@click.option('--make-obs-names-unique', default=True, is_flag=True,
help='Ensure obs index is unique.', show_default=True)
@click.option('--make-var-names-unique', default=True, is_flag=True,
help='Ensure var index is unique.', show_default=True)
@click.argument("data", nargs=1, metavar="<dataset: file or path to data>", required=True)
@click.option("--layout", "-l", default=["umap", "tsne"], multiple=True, type=click.Choice(["umap", "tsne"]),
help="Layout algorithm", show_default=True)
@click.option("--recipe", "-r", default="none", type=click.Choice(["none", "seurat", "zheng17"]),
help="Preprocessing to run.", show_default=True)
@click.option("--output", "-o", default="", help="Save a new file to filename.", metavar="<filename>")
@click.option("--plotting", "-p", default=False, is_flag=True, help="Whether to generate plots.", show_default=True)
@click.option("--sparse", default=False, is_flag=True, help="Whether to force sparsity.", show_default=True)
@click.option("--overwrite", default=False, is_flag=True, help="Allow file overwriting.", show_default=True)
@click.option("--set-obs-names", default="", help="Named field to set as index for obs.", metavar="<name>")
@click.option("--set-var-names", default="", help="Named field to set as index for var.", metavar="<name>")
@click.option("--make-obs-names-unique", default=True, is_flag=True,
help="Ensure obs index is unique.", show_default=True)
@click.option("--make-var-names-unique", default=True, is_flag=True,
help="Ensure var index is unique.", show_default=True)
def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
set_obs_names, set_var_names, make_obs_names_unique, make_var_names_unique):
"""Preprocesses data for use with cellxgene.
@@ -33,9 +33,9 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
annotations, ensuring sparsity, and plotting results."""
# collect slow imports here to make CLI startup more responsive
click.echo('[cellxgene] Starting CLI...')
click.echo("[cellxgene] Starting CLI...")
import matplotlib
matplotlib.use('Agg')
matplotlib.use("Agg")
import scanpy.api as sc
# scanpy settings
@@ -43,45 +43,45 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
sc.settings.autosave = True
# check args
if sparse and not recipe == 'none':
raise click.UsageError('Cannot use a recipe when forcing sparsity')
if sparse and not recipe == "none":
raise click.UsageError("Cannot use a recipe when forcing sparsity")
output = expanduser(output)
if isfile(output) and not overwrite:
raise click.UsageError(f'Cannot overwrite existing file {output}, try using the flag --overwrite')
raise click.UsageError(f"Cannot overwrite existing file {output}, try using the flag --overwrite")
def load_data(data):
if isfile(data):
name, extension = splitext(data)
if extension == '.h5ad':
if extension == ".h5ad":
adata = sc.read_h5ad(data)
elif extension == '.loom':
elif extension == ".loom":
adata = sc.read_loom(data)
else:
raise click.FileError(data, hint='does not have a valid extension [.h5ad | .loom]')
raise click.FileError(data, hint="does not have a valid extension [.h5ad | .loom]")
elif isdir(data):
if not data.endswith(sep):
data += sep
adata = sc.read_10x_mtx(data)
else:
raise click.FileError(data, hint='not a valid file or path')
raise click.FileError(data, hint="not a valid file or path")
if not set_obs_names == '':
if not set_obs_names == "":
if set_obs_names not in adata.obs_keys():
raise click.UsageError(f'obs {set_obs_names} not found, options are: {adata.obs_keys()}')
raise click.UsageError(f"obs {set_obs_names} not found, options are: {adata.obs_keys()}")
adata.obs_names = adata.obs[set_obs_names]
if not set_var_names == '':
if not set_var_names == "":
if set_var_names not in adata.var_keys():
raise click.UsageError(f'var {set_var_names} not found, options are: {adata.var_keys()}')
raise click.UsageError(f"var {set_var_names} not found, options are: {adata.var_keys()}")
adata.var_names = adata.var[set_var_names]
if make_obs_names_unique:
adata.obs_names_make_unique()
if make_var_names_unique:
adata.var_names_make_unique()
if not adata._obs.index.is_unique:
click.echo('Warning: obs index is not unique')
click.echo("Warning: obs index is not unique")
if not adata._var.index.is_unique:
click.echo('Warning: var index is not unique')
click.echo("Warning: var index is not unique")
return adata
@@ -90,9 +90,9 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
adata.X = csc_matrix(adata.X)
def run_recipe(adata):
if recipe == 'seurat':
if recipe == "seurat":
sc.pp.recipe_seurat(adata)
elif recipe == 'zheng17':
elif recipe == "zheng17":
sc.pp.recipe_zheng17(adata)
else:
sc.pp.filter_cells(adata, min_genes=5)
@@ -104,9 +104,9 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
def run_pca(adata):
if sparse:
sc.pp.pca(adata, svd_solver='arpack', zero_center=False)
sc.pp.pca(adata, svd_solver="arpack", zero_center=False)
else:
sc.pp.pca(adata, svd_solver='arpack')
sc.pp.pca(adata, svd_solver="arpack")
def run_neighbors(adata):
sc.pp.neighbors(adata)
@@ -115,46 +115,46 @@ def prepare(data, layout, recipe, output, plotting, sparse, overwrite,
sc.tl.louvain(adata)
def run_layout(adata):
if len(unique(adata.obs['louvain'].values)) < 10:
palette = 'tab10'
if len(unique(adata.obs["louvain"].values)) < 10:
palette = "tab10"
else:
palette = 'tab20'
palette = "tab20"
if 'umap' in layout:
if "umap" in layout:
sc.tl.umap(adata)
if plotting:
sc.pl.umap(adata, color='louvain', palette=palette, save='_louvain')
sc.pl.umap(adata, color="louvain", palette=palette, save="_louvain")
if 'tsne' in layout:
if "tsne" in layout:
sc.tl.tsne(adata)
if plotting:
sc.pl.tsne(adata, color='louvain', palette=palette, save='_louvain')
sc.pl.tsne(adata, color="louvain", palette=palette, save="_louvain")
def show_step(item):
names = {
'make_sparse': 'Ensuring sparsity',
'run_recipe': 'Running preprocessing recipe "%s"' % recipe,
'run_pca': 'Running PCA',
'run_neighbors': 'Calculating neighbors',
'run_louvain': 'Calculating clusters',
'run_layout': 'Computing layout'
"make_sparse": "Ensuring sparsity",
"run_recipe": f"Running preprocessing recipe \"{recipe}\"",
"run_pca": "Running PCA",
"run_neighbors": "Calculating neighbors",
"run_louvain": "Calculating clusters",
"run_layout": "Computing layout"
}
if item is not None:
return names[item.__name__]
steps = [make_sparse, run_recipe, run_pca, run_neighbors, run_louvain, run_layout]
click.echo(f'[cellxgene] Loading data from {data}, please wait...')
click.echo(f"[cellxgene] Loading data from {data}, please wait...")
adata = load_data(data)
click.echo('[cellxgene] Beginning preprocessing...')
with click.progressbar(steps, label='[cellxgene] Progress', show_eta=False, item_show_func=show_step) as bar:
click.echo("[cellxgene] Beginning preprocessing...")
with click.progressbar(steps, label="[cellxgene] Progress", show_eta=False, item_show_func=show_step) as bar:
for step in bar:
step(adata)
# saving
if not output == '':
click.echo(f'[cellxgene] Saving results to {output}...')
if not output == "":
click.echo(f"[cellxgene] Saving results to {output}...")
adata.write(output)
click.echo('[cellxgene] Success!')
click.echo("[cellxgene] Success!")