From 94f95d656586756c785b2ceaccb2b893e739aa47 Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Wed, 24 Oct 2018 19:28:59 -0700 Subject: [PATCH] CLI Launch (#366) * Scanpy engine now required Without the --engine param we need to error if scanpy engine cannot be imported rather than waiting for all engines * CLI options and help matches proposal (but not all options hooked up yet) * Flesh out top level args * Move computation args to engine * CLI input file (#374) * Fix test command (tests still won't work) * Input is file instead of directory - also renamed example file * Csweaver/debug (#376) * Respect debug flag for logging flask calls * Add loading messages * max categories (#377) * Add max categories * Rename max_categories to category_selection_limit * ensure whole numbers --- README.md | 2 +- example-dataset/{data.h5ad => pbmc3k.h5ad} | Bin server/app/app.py | 132 +++++++++++++++------ server/app/driver/driver.py | 3 +- server/app/rest_api/rest.py | 3 + server/app/scanpy_engine/scanpy_engine.py | 25 ++-- server/app/util/utils.py | 11 ++ server/test/test_api.py | 4 +- server/test/test_scanpy_engine.py | 2 +- 9 files changed, 130 insertions(+), 52 deletions(-) rename example-dataset/{data.h5ad => pbmc3k.h5ad} (100%) diff --git a/README.md b/README.md index 49d97740..cc299543 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Started in the context of the Human Cell Atlas Consortium, cellxgene hopes to bo **Run (with demo data)** - cellxgene --title PBMC3K scanpy example-dataset/ + cellxgene launch --title PBMC3K example-dataset/pbmck3.h5ad **Help** diff --git a/example-dataset/data.h5ad b/example-dataset/pbmc3k.h5ad similarity index 100% rename from example-dataset/data.h5ad rename to example-dataset/pbmc3k.h5ad diff --git a/server/app/app.py b/server/app/app.py index 56d190be..50daf506 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -1,6 +1,6 @@ import argparse +import logging import os -import warnings import webbrowser from flask import Flask @@ -10,7 +10,7 @@ from flask_cors import CORS from flask_restful_swagger_2 import get_swagger_blueprint from .rest_api.rest import get_api_resources -from .util.utils import Float32JSONEncoder +from .util.utils import Float32JSONEncoder, whole_number from .web import webapp REACTIVE_LIMIT = 1_000_000 @@ -46,48 +46,112 @@ app.register_blueprint( app.add_url_rule("/", endpoint="index") +def create_cli(): + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.description = """ +synopsis: +cellxgene [options] + +description: + +cellxgene is a local web application for exploring single cell expression. + """ + parser.add_argument("-V", "--version", help="show version and exit") + subparsers = parser.add_subparsers(dest="program") + subparsers.required = True + launch_group = subparsers.add_parser("launch", help="launch web application", + formatter_class=argparse.RawTextHelpFormatter) + launch_group.description = """ + cellxgene launches a local web application for exploring single cell expression data. + + Data must be in a format that cellxgene expects [[ how to format ]] + """ + launch_group.epilog = """ + annotation names: + The data viewer requires a unique, human readable name for each observation and variable. These are used for + various application features, such as the ability to view expression by gene. When launching cellxgene, appropriate + observation and variable annotations must be identified. + + If --obs-name or --var-name parameters are specified, values in the named annotations will be used. If not + specified, the observation and variable index values will name each respectively. An error will generated if the + values for each are not unique. + + examples: + To run with the example dataset: + + cellxgene example_dataset/pbmc3k.h5ad --title PBMC3K + + To run with your own data with tsne layout: + + cellxgene --title -l tsne + + To indicate that the human-readable variable annotation is named 'gene_names', and the human-readable observation + is 'cell_names': + + cellxgene mydata.h5ad -var-name gene_names -obs-name cell_names + + """ + launch_group.add_argument("data", metavar="data", help="file containing the data to display") + launch_group.add_argument("--title", "-t", help="title to display -- if this is omitted the title will be the name " + "of the data file.") + launch_group.add_argument( + "--listen-all", + help="bind to all interfaces (this makes the server accessible beyond this computer)", + action="store_true") + launch_group.add_argument("--port", help="port to run server on", type=int, default=5005) + launch_group.add_argument("--debug", action="store_true", + help="more verbose output, including outputting warnings and every REST request") + launch_group.add_argument("--flask-debug", action="store_true", help=argparse.SUPPRESS) + launch_group.add_argument("--no-open", help="do not launch the webbrowser", action="store_false", + dest="open_browser") + launch_group.add_argument( + "--category-selection-limit", + type=whole_number, + help="maximum number of categories to display on the front-end. " + "Annotations with more than this number are not displayed", + default=100) + try: + from .scanpy_engine.scanpy_engine import ScanpyEngine + except ImportError as e: + # We will handle more engines when they come + raise ImportError('Scanpy is required for cellxgene, please install scanpy and try again', e) from e + else: + ScanpyEngine.add_to_parser(launch_group) + return parser + + def run_scanpy(args): title = args.title if not title: - title = os.path.basename(os.path.normpath(args.data_directory)) - api_base = f"http://127.0.0.1:{args.port}/api/" - app.config.update( - DATASET_TITLE=title, - CXG_API_BASE=api_base - ) - - from .scanpy_engine.scanpy_engine import ScanpyEngine - app.data = ScanpyEngine(args.data_directory, layout_method=args.layout, diffexp_method=args.diffexp) + file_parts = os.path.splitext(os.path.basename(args.data)) + title = file_parts[0] if args.listen_all: host = "0.0.0.0" else: host = "127.0.0.1" + cellxgene_url = f"http://{host}:{args.port}" + api_base = f"{cellxgene_url}/api/" + app.config.update( + DATASET_TITLE=title, + CXG_API_BASE=api_base + ) + if not args.debug: + log = logging.getLogger('werkzeug') + log.setLevel(logging.ERROR) + from .scanpy_engine.scanpy_engine import ScanpyEngine + print(f"Loading data from {args.data}") + app.data = ScanpyEngine(args.data, layout_method=args.layout, diffexp_method=args.diffexp, + category_selection_limit=args.category_selection_limit) + print(f"Launching cellxgene") if args.open_browser: - webbrowser.open(f"http://{host}:{args.port}") + webbrowser.open(cellxgene_url) + print(f"Please go to {cellxgene_url}") app.run(host=host, debug=args.flask_debug, port=args.port) def main(): - parser = argparse.ArgumentParser(description="Cellxgene is a tool for exploring single cell expression.") - parser.add_argument("--title", "-t", help="Title to display -- if this is omitted the title will be the name " - "of the directory from the data_directory arg") - parser.add_argument("--port", help="Port to run server on.", type=int, default=5005) - parser.add_argument("--flask-debug", action="store_true", help=argparse.SUPPRESS) - parser.add_argument("--no-open", help="Do not launch the webbrowser.", action="store_false", dest="open_browser") - parser.add_argument( - "--listen-all", - help="Bind to all interfaces (this makes the server accessible beyond this computer)", - action="store_true") - subparsers = parser.add_subparsers(dest="engine") - subparsers.required = True - try: - from .scanpy_engine.scanpy_engine import ScanpyEngine - except ImportError: - warnings.simplefilter('default', ImportWarning) # Enable ImportWarning - warnings.warn("Scanpy engine not available", ImportWarning) - else: - ScanpyEngine.add_to_parser(subparsers, run_scanpy) - if len(subparsers.choices) == 0: - raise ImportError('Could not import any engines, see warnings above') + parser = create_cli() args = parser.parse_args() - args.func(args) + # TODO pick engine based on input file + print("cellxgene starting...\n") + run_scanpy(args) diff --git a/server/app/driver/driver.py b/server/app/driver/driver.py index 8166850e..81713c3c 100644 --- a/server/app/driver/driver.py +++ b/server/app/driver/driver.py @@ -12,10 +12,11 @@ Sort order for methods class CXGDriver(metaclass=ABCMeta): - def __init__(self, data, layout_method=None, diffexp_method=None): + def __init__(self, data, layout_method=None, diffexp_method=None, category_selection_limit=100): self.data = self._load_data(data) self.layout_method = layout_method self.diffexp_method = diffexp_method + self.category_selection_limit = category_selection_limit self.cluster = None @property diff --git a/server/app/rest_api/rest.py b/server/app/rest_api/rest.py index ee910330..f4bd9b16 100644 --- a/server/app/rest_api/rest.py +++ b/server/app/rest_api/rest.py @@ -110,6 +110,9 @@ class ConfigAPI(Resource): "displayNames": { "engine": f"cellxgene Scanpy engine version {pkg_resources.get_distribution('cellxgene').version}", "dataset": current_app.config["DATASET_TITLE"] + }, + "parameters": { + "category_selection_limit": current_app.data.category_selection_limit } } } diff --git a/server/app/scanpy_engine/scanpy_engine.py b/server/app/scanpy_engine/scanpy_engine.py index 1c808564..f795cd24 100644 --- a/server/app/scanpy_engine/scanpy_engine.py +++ b/server/app/scanpy_engine/scanpy_engine.py @@ -1,4 +1,3 @@ -import os import warnings import numpy as np @@ -23,12 +22,15 @@ Sort order for methods class ScanpyEngine(CXGDriver): - def __init__(self, data, layout_method=None, diffexp_method=None): - super().__init__(data, layout_method=layout_method, diffexp_method=diffexp_method) + def __init__(self, data, layout_method=None, diffexp_method=None, category_selection_limit=100): + super().__init__(data, layout_method=layout_method, diffexp_method=diffexp_method, + category_selection_limit=category_selection_limit) self._validatate_data_types() self._add_mandatory_annotations() self.cell_count = self.data.shape[0] self.gene_count = self.data.shape[1] + self.layout_options = ["umap", "tsne"] + self.diffexp_options = ["ttest"] self._create_schema() def _create_schema(self): @@ -64,16 +66,13 @@ class ScanpyEngine(CXGDriver): self.schema["annotations"][ax].append(ann_schema) @classmethod - def add_to_parser(cls, subparsers, invocation_function): - scanpy_group = subparsers.add_parser("scanpy", help="run cellxgene using the scanpy engine") + def add_to_parser(cls, subparser): + computation_group = subparser.add_argument_group('computational arguments') # TODO these choices should be generated from the actual available methods see GH issue #94 - scanpy_group.add_argument("-l", "--layout", choices=["umap", "tsne"], default="umap", - help="Algorithm to use for graph layout") - scanpy_group.add_argument("-d", "--diffexp", choices=["ttest"], default="ttest", - help="Algorithm to used to calculate differential expression") - scanpy_group.add_argument("data_directory", metavar="dir", help="Directory containing data and schema file") - scanpy_group.set_defaults(func=invocation_function) - return scanpy_group + computation_group.add_argument("-l", "--layout", choices=["umap", "tsne"], default="umap", + help="Algorithm to use for graph layout") + computation_group.add_argument("-d", "--diffexp", choices=["ttest"], default="ttest", + help="Algorithm to used to calculate differential expression") @staticmethod def _load_data(data): @@ -81,7 +80,7 @@ class ScanpyEngine(CXGDriver): # Based upon this advice, setting cache=True parameter # Note: as of current scanpy/anndata release, setting backed='r' will # result in an error. - return sc.read(os.path.join(data, "data.h5ad"), cache=True) + return sc.read(data, cache=True) @staticmethod def _top_sort(values, sort_order, top_n=None): diff --git a/server/app/util/utils.py b/server/app/util/utils.py index 1579226e..57452f44 100644 --- a/server/app/util/utils.py +++ b/server/app/util/utils.py @@ -1,4 +1,5 @@ import json +from argparse import ArgumentTypeError from numpy import float32, integer @@ -49,3 +50,13 @@ def get_mime_type(default="application/json", acceptable_types=["application/jso if not mime_type: raise MimeTypeError(f"Unsupported mime type(s) {header} in HTTP Accept header") return mime_type + + +def whole_number(value): + try: + value = int(value) + except ValueError as e: + raise ArgumentTypeError(f"{value} is not type int") from e + if value < 0: + raise ArgumentTypeError(f"{value} is not >= 0") + return value diff --git a/server/test/test_api.py b/server/test/test_api.py index a984f6de..f0677698 100644 --- a/server/test/test_api.py +++ b/server/test/test_api.py @@ -25,7 +25,7 @@ class EndPoints(unittest.TestCase): @classmethod def setUpClass(cls): - cls.ps = Popen(["cellxgene", "--no-open", "scanpy", "example-dataset/"]) + cls.ps = Popen(["cellxgene", "launch", "--no-open", "example-dataset/pbmc3k.h5ad"]) session = requests.Session() for i in range(90): try: @@ -58,7 +58,7 @@ class EndPoints(unittest.TestCase): result = self.session.get(url) self.assertEqual(result.status_code, HTTPStatus.OK) result_data = result.json() - self.assertEqual(result_data["config"]["displayNames"]["dataset"], "example-dataset") + self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") self.assertEqual(len(result_data["config"]["features"]), 4) def test_get_layout(self): diff --git a/server/test/test_scanpy_engine.py b/server/test/test_scanpy_engine.py index 5e172d9c..2223747c 100644 --- a/server/test/test_scanpy_engine.py +++ b/server/test/test_scanpy_engine.py @@ -12,7 +12,7 @@ from server.app.scanpy_engine.scanpy_engine import ScanpyEngine class UtilTest(unittest.TestCase): def setUp(self): - self.data = ScanpyEngine("example-dataset/", layout_method="umap", diffexp_method="ttest") + self.data = ScanpyEngine("example-dataset/pbmc3k.h5ad", layout_method="umap", diffexp_method="ttest") self.data._create_schema() def test_init(self):