diff --git a/server/app/app.py b/server/app/app.py index 03e2bb95..4dfd2a67 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -26,14 +26,14 @@ webbp = Blueprint("webapp", "server.common.web", template_folder="templates") def dataset_index(dataset=None): config = current_app.app_config if dataset is None: - if config.datapath: - location = config.datapath + if config.single_dataset__datapath: + location = config.single_dataset__datapath else: return dataroot_index() else: - location = path_join(config.dataroot, dataset) + location = path_join(config.multi_dataset__dataroot, dataset) - scripts = config.scripts + scripts = config.server__scripts try: cache_manager = current_app.matrix_data_cache_manager @@ -59,13 +59,13 @@ def get_data_adaptor(dataset=None): config = current_app.app_config if dataset is None: - datapath = config.datapath + datapath = config.single_dataset__datapath else: - datapath = path_join(config.dataroot, dataset) + datapath = path_join(config.multi_dataset__dataroot, dataset) # path_join returns a normalized path. Therefore it is # sufficient to check that the datapath starts with the # dataroot to determine that the datapath is under the dataroot. - if not datapath.startswith(config.dataroot): + if not datapath.startswith(config.multi_dataset__dataroot): raise DatasetAccessError("Invalid dataset {dataset}") if datapath is None: @@ -100,10 +100,10 @@ def dataroot_test_index(): try: config = current_app.app_config - locator = DataLocator(config.dataroot) + locator = DataLocator(config.multi_dataset__dataroot) datasets = [] for fname in locator.ls(): - location = path_join(config.dataroot, fname) + location = path_join(config.multi_dataset__dataroot, fname) try: MatrixDataLoader(location, app_config=config) datasets.append(fname) @@ -118,7 +118,7 @@ def dataroot_test_index(): data += f"
  • {dataset}
    • " data += "" except Exception as e: - data += f"
    Unable to locate datasets from {config.dataroot}: {str(e)}" + data += f'
    Unable to locate datasets from {config.multi_dataset__dataroot}: {str(e)}' data += "" return make_response(data) @@ -127,12 +127,12 @@ def dataroot_test_index(): def dataroot_index(): # Handle the base url for the cellxgene server when running in multi dataset mode config = current_app.app_config - if not config.multi_dataset_index: + if not config.multi_dataset__index: abort(404) - elif config.multi_dataset_index is True: + elif config.multi_dataset__index is True: return dataroot_test_index() else: - return redirect(config.dataroot_index) + return redirect(config.multi_dataset__index) class SchemaAPI(Resource): @@ -222,7 +222,7 @@ class Server: self.app.register_blueprint(webbp) api_version = "/api/v0.2" - if app_config.datapath: + if app_config.single_dataset__datapath: bp_api = Blueprint("api", __name__, url_prefix=api_version) resources = get_api_resources(bp_api) self.app.register_blueprint(resources.blueprint) diff --git a/server/cli/launch.py b/server/cli/launch.py index bc5661f8..bd5dbd21 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -1,40 +1,32 @@ import errno import functools import logging -from os import devnull, mkdir, environ -from os.path import splitext, basename, isdir +from os import devnull import sys -import warnings import webbrowser -from urllib.parse import urlparse import click -from server.common.utils import custom_format_warning -from server.common.utils import find_available_port, is_port_available, sort_options -from server.common.errors import DatasetAccessError -from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType -from server.common.annotations import AnnotationsLocalFile +from server.common.utils import sort_options +from server.common.errors import DatasetAccessError, ConfigurationError +from server.data_common.matrix_loader import MatrixDataCacheManager from server.common.app_config import AppConfig +from server.common.default_config import default_config -from server.common.errors import OntologyLoadFailure - -# anything bigger than this will generate a special message -BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB -DEFAULT_SERVER_PORT = int(environ.get("CXG_SERVER_PORT", "5005")) +DEFAULT_CONFIG = AppConfig() def annotation_args(func): @click.option( "--disable-annotations", is_flag=True, - default=False, + default=not DEFAULT_CONFIG.user_annotations__enable, show_default=True, help="Disable user annotation of data.", ) @click.option( "--annotations-file", - default=None, + default=DEFAULT_CONFIG.user_annotations__local_file_csv__file, show_default=True, multiple=False, metavar="", @@ -43,7 +35,7 @@ def annotation_args(func): ) @click.option( "--annotations-dir", - default=None, + default=DEFAULT_CONFIG.user_annotations__local_file_csv__directory, show_default=False, multiple=False, metavar="", @@ -53,13 +45,13 @@ def annotation_args(func): @click.option( "--experimental-annotations-ontology", is_flag=True, - default=False, + default=DEFAULT_CONFIG.user_annotations__ontology__enable, show_default=True, help="When creating annotations, optionally autocomplete names from ontology terms.", ) @click.option( "--experimental-annotations-ontology-obo", - default=None, + default=DEFAULT_CONFIG.user_annotations__ontology__obo_location, show_default=True, metavar="", help="Location of OBO file defining cell annotation autosuggest terms.", @@ -74,7 +66,7 @@ def annotation_args(func): def config_args(func): @click.option( "--max-category-items", - default=1000, + default=DEFAULT_CONFIG.presentation__max_categories, metavar="", show_default=True, help="Will not display categories with more distinct values than specified.", @@ -82,7 +74,7 @@ def config_args(func): @click.option( "--diffexp-lfc-cutoff", "-de", - default=0.01, + default=DEFAULT_CONFIG.diffexp__lfc_cutoff, show_default=True, metavar="", help="Minimum log fold change threshold for differential expression.", @@ -90,14 +82,14 @@ def config_args(func): @click.option( "--disable-diffexp", is_flag=True, - default=False, + default=not DEFAULT_CONFIG.diffexp__enable, show_default=False, help="Disable on-demand differential expression.", ) @click.option( "--embedding", "-e", - default=[], + default=DEFAULT_CONFIG.embeddings__names, multiple=True, show_default=False, metavar="", @@ -106,7 +98,7 @@ def config_args(func): @click.option( "--experimental-enable-reembedding", is_flag=True, - default=False, + default=DEFAULT_CONFIG.embeddings__enable_reembedding, show_default=False, hidden=True, help="Enable experimental on-demand re-embedding using UMAP. WARNING: may be very slow.", @@ -122,14 +114,14 @@ def dataset_args(func): @click.option( "--obs-names", "-obs", - default=None, + default=DEFAULT_CONFIG.single_dataset__obs_names, metavar="", help="Name of annotation field to use for observations. If not specified cellxgene will use the the obs index.", ) @click.option( "--var-names", "-var", - default=None, + default=DEFAULT_CONFIG.single_dataset__var_names, metavar="", help="Name of annotation to use for variables. If not specified cellxgene will use the the var index.", ) @@ -137,13 +129,20 @@ def dataset_args(func): "--backed", "-b", is_flag=True, - default=False, + default=DEFAULT_CONFIG.adaptor__anndata_adaptor__backed, show_default=False, help="Load anndata in file-backed mode. " "This may save memory, but may result in slower overall performance.", ) - @click.option("--title", "-t", metavar="", help="Title to display. If omitted will use file name.") + @click.option( + "--title", + "-t", + default=DEFAULT_CONFIG.single_dataset__title, + metavar="", + help="Title to display. If omitted will use file name." + ) @click.option( "--about", + default=DEFAULT_CONFIG.single_dataset__about, metavar="", help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).", ) @@ -159,7 +158,7 @@ def server_args(func): "--debug", "-d", is_flag=True, - default=False, + default=DEFAULT_CONFIG.server__debug, show_default=True, help="Run in debug mode. This is helpful for cellxgene developers, " "or when you want more information about an error condition.", @@ -168,7 +167,7 @@ def server_args(func): "--verbose", "-v", is_flag=True, - default=False, + default=DEFAULT_CONFIG.server__verbose, show_default=True, help="Provide verbose output, including warnings and all server requests.", ) @@ -176,21 +175,22 @@ def server_args(func): "--port", "-p", metavar="", - default=None, + default=DEFAULT_CONFIG.server__port, + type=int, show_default=True, help="Port to run server on. If not specified cellxgene will find an available port.", ) @click.option( "--host", metavar="", - default="127.0.0.1", + default=DEFAULT_CONFIG.server__host, show_default=False, help="Host IP address. By default cellxgene will use localhost (e.g. 127.0.0.1).", ) @click.option( "--scripts", "-s", - default=[], + default=DEFAULT_CONFIG.server__scripts, multiple=True, metavar="", help="Additional script files to include in HTML page. If not specified, " @@ -211,7 +211,7 @@ def launch_args(func): @server_args @click.option( "--dataroot", - default=None, + default=DEFAULT_CONFIG.multi_dataset__dataroot, metavar="", help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)" " to folder containing H5AD and/or CXG datasets.", @@ -223,10 +223,27 @@ def launch_args(func): "-o", "open_browser", is_flag=True, - default=False, + default=DEFAULT_CONFIG.server__open_browser, show_default=True, help="Open web browser after launch.", ) + @click.option( + "--config-file", + "-c", + "config_file", + is_flag=True, + default=None, + show_default=True, + help="Location to yaml file with configuration settings", + ) + @click.option( + "--dump-default-config", + "dump_default_config", + is_flag=True, + default=False, + show_default=True, + help="Print default configuration settings and exit", + ) @click.help_option("--help", "-h", help="Show this message and exit.") @functools.wraps(func) def wrapper(*args, **kwargs): @@ -252,11 +269,6 @@ def handle_scripts(scripts): click.confirm(f"Are you sure you want to inject these scripts: {scripts_pretty}?", abort=True) -def handle_verbose(verbose): - if not verbose: - sys.tracebacklimit = 0 - - @sort_options @click.command( short_help="Launch the cellxgene data viewer. " "Run `cellxgene launch --help` for more information.", @@ -287,6 +299,8 @@ def launch( experimental_annotations_ontology, experimental_annotations_ontology_obo, experimental_enable_reembedding, + config_file, + dump_default_config ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. @@ -307,162 +321,79 @@ def launch( # # > cellxgene launch --dataroot + if dump_default_config: + print(default_config) + sys.exit(0) + # Startup message click.echo("[cellxgene] Starting the CLI...") - if datapath is None and dataroot is None: - # TODO: change the error message once dataroot is fully supported - raise click.ClickException('Missing argument "."') - # raise click.ClickException("must supply either or --dataroot") - if datapath is not None and dataroot is not None: - raise click.ClickException("must supply only one of or --dataroot") - - if datapath: - # preload this data set - matrix_data_loader = MatrixDataLoader(datapath) - - try: - matrix_data_loader.pre_load_validation() - except DatasetAccessError as e: - raise click.ClickException(str(e)) - - if experimental_enable_reembedding: - if matrix_data_loader.matrix_data_type() != MatrixDataType.H5AD: - raise click.ClickException("--experimental-enable-reembedding is only supported with H5AD files.") - if backed: - raise click.ClickException( - "--experimental-enable-reembedding is not supported when run in --backed mode." - ) - - file_size = matrix_data_loader.file_size() - if file_size > BIG_FILE_SIZE_THRESHOLD: - click.echo(f"[cellxgene] Loading data from {basename(datapath)}, this may take a while...") - else: - click.echo(f"[cellxgene] Loading data from {basename(datapath)}.") - - if debug: - verbose = True - open_browser = False - else: - warnings.formatwarning = custom_format_warning - - handle_verbose(verbose) - handle_scripts(scripts) - - if port: - if debug: - raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).") - if not is_port_available(host, int(port)): - raise click.ClickException( - f"The port selected {port} is in use, please specify an open port using the --port flag." - ) - else: - port = find_available_port(host, DEFAULT_SERVER_PORT) - - if disable_annotations: - if annotations_file is not None: - click.echo("Warning: --annotations-file ignored as annotations are disabled.") - if annotations_dir is not None: - click.echo("Warning: --annotations-dir ignored as annotations are disabled.") - if experimental_annotations_ontology: - click.echo("Warning: --experimental-annotations-ontology ignored as annotations are disabled.") - if experimental_annotations_ontology_obo is not None: - click.echo("Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled.") - else: - if annotations_file is not None and annotations_dir is not None: - raise click.ClickException( - "--annotations-file and --annotations-dir " "may not be used together." - ) - - if annotations_file is not None: - lf_name, lf_ext = splitext(annotations_file) - if lf_ext and lf_ext != ".csv": - raise click.FileError(basename(annotations_file), hint="annotation file type must be .csv") - - if annotations_dir is not None and not isdir(annotations_dir): - try: - mkdir(annotations_dir) - except OSError: - raise click.ClickException( - "Unable to create directory specified by " "--annotations-dir" - ) - - if about: - - def url_check(url): - try: - result = urlparse(url) - if all([result.scheme, result.netloc]): - return True - else: - return False - except ValueError: - return False - - if not url_check(about): - raise click.ClickException("Must provide an absolute URL for --about. (Example format: http://example.com)") - - # Setup app - cellxgene_url = f"http://{host}:{port}" - # app config - app_config = AppConfig( - datapath=datapath, - dataroot=dataroot, - title=title, - about=about, - scripts=scripts, - layout=embedding, - max_category_items=max_category_items, - diffexp_lfc_cutoff=diffexp_lfc_cutoff, - obs_names=obs_names, - var_names=var_names, - anndata_backed=backed, - disable_diffexp=disable_diffexp, - enable_reembedding=experimental_enable_reembedding, - ) + app_config = AppConfig() - matrix_data_cache_manager = MatrixDataCacheManager() - data_adaptor = None - if datapath: - try: - with matrix_data_cache_manager.data_adaptor(datapath, app_config) as data_adaptor: - if not disable_diffexp and data_adaptor.parameters.get("diffexp_may_be_slow", False): - click.echo( - f"[cellxgene] CAUTION: due to the size of your dataset, " - f"running differential expression may take longer or fail." - ) - except Exception as e: - raise click.ClickException(str(e)) + try: + if config_file: + app_config.update_from_config_file(config_file) - # create an annotations object. Only AnnotationsLocalFile is used (for now) - annotations = None + app_config.update( + server__verbose=verbose, + server__debug=debug, + server__host=host, + server__port=port, + server__scripts=scripts, + server__open_browser=open_browser, - if not disable_annotations: - annotations = AnnotationsLocalFile(annotations_dir, annotations_file) + single_dataset__datapath=datapath, + single_dataset__title=title, + single_dataset__about=about, + single_dataset__obs_names=obs_names, + single_dataset__var_names=var_names, - # if the user has specified a fixed label file, go ahead and validate it - # so that we can remove errors early in the process. + multi_dataset__dataroot=dataroot, - if annotations_file and data_adaptor: - data_adaptor.check_new_labels(annotations.read_labels(data_adaptor)) + user_annotations__enable=not disable_annotations, + user_annotations__local_file_csv__file=annotations_file, + user_annotations__local_file_csv__directory=annotations_dir, + user_annotations__ontology__enable=experimental_annotations_ontology, + user_annotations__ontology__obo_location=experimental_annotations_ontology_obo, - if experimental_annotations_ontology or bool(experimental_annotations_ontology_obo): - try: - annotations.load_ontology(experimental_annotations_ontology_obo) - except OntologyLoadFailure as e: - raise click.ClickException("Unable to load ontology terms\n" + str(e)) + presentation__max_categories=max_category_items, + + embeddings__names=embedding, + embeddings__enable_reembedding=experimental_enable_reembedding, + + diffexp__enable=not disable_diffexp, + diffexp__lfc_cutoff=diffexp_lfc_cutoff, + + adaptor__anndata_adaptor__backed=backed, + ) + + # process the configuration + # any errors will be thrown as an exception. + # any info messages will be passed to the messagefn function. + matrix_data_cache_manager = MatrixDataCacheManager() + + def messagefn(message): + click.echo("[cellxgene] " + message) + + app_config.complete_config(matrix_data_cache_manager, messagefn) + + except (ConfigurationError, DatasetAccessError) as e: + raise click.ClickException(e) + + handle_scripts(scripts) + user_annotations = app_config.user_annotations # create the server from server.app.app import Server + server = Server(matrix_data_cache_manager, user_annotations, app_config) - server = Server(matrix_data_cache_manager, annotations, app_config) - - if not verbose: + if not app_config.server__verbose: log = logging.getLogger("werkzeug") log.setLevel(logging.ERROR) - if open_browser: + cellxgene_url = f"http://{app_config.server__host}:{app_config.server__port}" + if app_config.server__open_browser: click.echo(f"[cellxgene] Launching! Opening your browser to {cellxgene_url} now.") webbrowser.open(cellxgene_url) else: @@ -470,12 +401,17 @@ def launch( click.echo("[cellxgene] Type CTRL-C at any time to exit.") - if not verbose: + if not app_config.server__verbose: f = open(devnull, "w") sys.stdout = f try: - server.app.run(host=host, debug=debug, port=port, threaded=False if debug else True, use_debugger=False) + server.app.run( + host=app_config.server__host, + debug=app_config.server__debug, + port=app_config.server__port, + threaded=not app_config.server__debug, + use_debugger=False) except OSError as e: if e.errno == errno.EADDRINUSE: raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e diff --git a/server/common/app_config.py b/server/common/app_config.py index e8798e1e..9ef88242 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -1,6 +1,25 @@ # -*- coding: utf-8 -*- + from server import __version__ as cellxgene_version +from flatten_dict import flatten +from os import mkdir, environ +from os.path import splitext, basename, isdir +import sys +from urllib.parse import urlparse +import yaml + +from server.common.default_config import get_default_config +from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure +from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType +from server.common.utils import find_available_port, is_port_available +import warnings +from server.common.annotations import AnnotationsLocalFile +from server.common.utils import custom_format_warning + +DEFAULT_SERVER_PORT = int(environ.get("CXG_SERVER_PORT", "5005")) +# anything bigger than this will generate a special message +BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB class AppFeature(object): @@ -19,78 +38,334 @@ class AppFeature(object): class AppConfig(object): - def __init__(self, **kw): - super().__init__() + def __init__(self): - # app inputs - self.datapath = None - self.dataroot = None - self.title = "" - self.about = None - self.scripts = [] - self.layout = [] - self.max_category_items = 100 - self.diffexp_lfc_cutoff = 0.01 - self.disable_diffexp = False - self.enable_reembedding = False - self.anndata_backed = False + self.default_config = get_default_config() - # The index page when in multi-dataset mode: - # False or None: this returns a 404 code - # True: loads a test index page, which links to the datasets that are available in the dataroot - # string/URL: redirect to this URL: flask.redirect(config.multi_dataset_index) - self.multi_dataset_index = False + dc = self.default_config + try: + self.server__verbose = dc["server"]["verbose"] + self.server__debug = dc["server"]["debug"] + self.server__host = dc["server"]["host"] + self.server__port = dc["server"]["port"] + self.server__scripts = dc["server"]["scripts"] + self.server__open_browser = dc["server"]["open_browser"] + self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"] + self.multi_dataset__index = dc["multi_dataset"]["index"] + self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"] + self.single_dataset__datapath = dc["single_dataset"]["datapath"] + self.single_dataset__obs_names = dc["single_dataset"]["obs_names"] + self.single_dataset__var_names = dc["single_dataset"]["var_names"] + self.single_dataset__about = dc["single_dataset"]["about"] + self.single_dataset__title = dc["single_dataset"]["title"] + self.user_annotations__enable = dc["user_annotations"]["enable"] + self.user_annotations__type = dc["user_annotations"]["type"] + self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"] + self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"] + self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"] + self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"] + self.presentation__max_categories = dc["presentation"]["max_categories"] + self.embeddings__names = dc["embeddings"]["names"] + self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"] + self.diffexp__enable = dc["diffexp"]["enable"] + self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"] + self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"] + self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"] + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") - # A list of allowed matrix types. If an empty list, then all matrix types are allowed - self.multi_dataset_allowed_matrix_type = [] + # The annotation object is created during complete_config and stored here. + self.user_annotations = None - # TODO these options may not apply to all datasets in the multi dataset. - # may need to invent a way to associate these config parameters with - # specific datasets. - self.obs_names = None - self.var_names = None + # Set to true when config_completed is called + self.is_completed = False - # parameters - self.diffexp_may_be_slow = False + def update_from_config_file(self, config_file): + with open(config_file) as fyaml: + config = yaml.load(fyaml, Loader=yaml.FullLoader) - inputs = [ - "datapath", - "dataroot", - "title", - "about", - "scripts", - "layout", - "max_category_items", - "diffexp_lfc_cutoff", - "obs_names", - "var_names", - "anndata_backed", - "disable_diffexp", - "enable_reembedding", - "multi_dataset_index", - "multi_dataset_allowed_matrix_type", - ] + # special case for tiledb_ctx whose value is a dict, and cannot + # be handled by the flattening below + if config.get("adaptor", {}).get("cxg_adaptor", {}).get("tiledb_ctx"): + value = config["adaptor"]["cxg_adaptor"]["tiledb_ctx"] + self.adaptor__cxg_adaptor__tiledb_ctx = value + del config["adaptor"]["cxg_adaptor"]["tiledb_ctx"] - self.update(inputs, kw) + flat_config = flatten(config) + for key, value in flat_config.items(): + # name of the attribute + attr = "__".join(key) + if not hasattr(self, attr): + raise ConfigurationError(f"Unknown key from config file: {key}") + try: + setattr(self, attr, value) + except KeyError: + raise ConfigurationError(f"Unable to set config attribute: {key}") - def update(self, inputs, kw): - for k, v in kw.items(): - if k in inputs: - setattr(self, k, v) - else: - raise RuntimeError(f"unknown config parameter {k}.") + self.is_completed = False + + def update(self, **kw): + for key, value in kw.items(): + if not hasattr(self, key): + raise ConfigurationError(f"unknown config parameter {key}.") + try: + setattr(self, key, value) + except KeyError: + raise ConfigurationError(f"Unable to set config parameter {key}.") + + self.is_completed = False + + def complete_config(self, matrix_data_cache_manager=None, messagefn=None): + """The configure options are checked, and any additional setup based on the config + parameters is done""" + + if matrix_data_cache_manager is None: + matrix_data_cache_manager = MatrixDataCacheManager() + if messagefn is None: + + def noop(message): + pass + + messagefn = noop + + # TODO: to give better error messages we can add a mapping between where each config + # attribute originated (e.g. command line argument or config file), then in the error + # messages we can give correct context for attributes with bad value. + context = dict(matrix_cache=matrix_data_cache_manager, messagefn=messagefn) + + self.handle_server(context) + self.handle_single_dataset(context) + self.handle_multi_dataset(context) + self.handle_user_annotations(context) + self.handle_embeddings(context) + self.handle_diffexp(context) + self.handle_adaptor(context) + + self.is_completed = True + + def __check_attr(self, attrname, vtype): + val = getattr(self, attrname) + if type(vtype) in (list, tuple): + if type(val) not in vtype: + tnames = ",".join([x.__name__ for x in vtype]) + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}" + ) + else: + if type(val) != vtype: + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, " + f"expected type {vtype.__name__}, got {type(val).__name__}" + ) + + def handle_server(self, context): + self.__check_attr("server__verbose", bool) + self.__check_attr("server__debug", bool) + self.__check_attr("server__host", str) + self.__check_attr("server__port", (type(None), int)) + self.__check_attr("server__scripts", (list, tuple)) + self.__check_attr("server__open_browser", bool) + + if self.server__port: + if self.server__debug: + raise ConfigurationError( + "'port' and 'debug' may not be used together (try 'verbose' for error logging)." + ) + if not is_port_available(self.server__host, self.server__port): + raise ConfigurationError( + f"The port selected {self.server__port} is in use, please configure an open port." + ) + else: + self.server__port = find_available_port(self.server__host, DEFAULT_SERVER_PORT) + + if self.server__debug: + context["messagefn"]("in debug mode, setting verbose=True and open_browser=False") + self.server__verbose = True + self.server__open_browser = False + else: + warnings.formatwarning = custom_format_warning + + if not self.server__verbose: + sys.tracebacklimit = 0 + + def handle_presentation(self, context): + self.__check_attr("presentation__max_categories", int) + + def handle_single_dataset(self, context): + self.__check_attr("single_dataset__datapath", (str, type(None))) + self.__check_attr("single_dataset__title", (str, type(None))) + self.__check_attr("single_dataset__about", (str, type(None))) + self.__check_attr("single_dataset__obs_names", (str, type(None))) + self.__check_attr("single_dataset__var_names", (str, type(None))) + + if self.single_dataset__datapath is None: + if self.multi_dataset__dataroot is None: + # TODO: change the error message once dataroot is fully supported + raise ConfigurationError("missing datapath") + return + else: + if self.multi_dataset__dataroot is not None: + raise ConfigurationError("must supply only one of datapath or dataroot") + + # preload this data set + matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath) + try: + matrix_data_loader.pre_load_validation() + except DatasetAccessError as e: + raise ConfigurationError(str(e)) + + file_size = matrix_data_loader.file_size() + file_basename = basename(self.single_dataset__datapath) + if file_size > BIG_FILE_SIZE_THRESHOLD: + context["messagefn"](f"Loading data from {file_basename}, this may take a while...") + else: + context["messagefn"](f"Loading data from {file_basename}.") + + if self.single_dataset__about: + + def url_check(url): + try: + result = urlparse(url) + if all([result.scheme, result.netloc]): + return True + else: + return False + except ValueError: + return False + + if not url_check(self.single_dataset__about): + raise ConfigurationError( + "Must provide an absolute URL for --about. (Example format: http://example.com)" + ) + + def handle_multi_dataset(self, context): + self.__check_attr("multi_dataset__dataroot", (type(None), str)) + self.__check_attr("multi_dataset__index", (type(None), bool, str)) + self.__check_attr("multi_dataset__allowed_matrix_types", (list)) + + if self.multi_dataset__dataroot is None: + return + + # error checking + for mtype in self.multi_dataset__allowed_matrix_types: + try: + MatrixDataType(mtype) + except ValueError: + raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}') + + def handle_user_annotations(self, context): + self.__check_attr("user_annotations__enable", bool) + self.__check_attr("user_annotations__type", str) + self.__check_attr("user_annotations__local_file_csv__directory", (type(None), str)) + self.__check_attr("user_annotations__local_file_csv__file", (type(None), str)) + self.__check_attr("user_annotations__ontology__enable", bool) + self.__check_attr("user_annotations__ontology__obo_location", (type(None), str)) + + if self.user_annotations__enable: + # TODO, replace this with a factory pattern once we have more than one way + # to do annotations. currently only local_file_csv + if self.user_annotations__type != "local_file_csv": + raise ConfigurationError('The only annotation type support is "local_file_csv"') + + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + + if filename is not None and dirname is not None: + raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.") + + if filename is not None: + lf_name, lf_ext = splitext(filename) + if lf_ext and lf_ext != ".csv": + raise ConfigurationError(f"annotation file type must be .csv: {filename}") + + if dirname is not None and not isdir(dirname): + try: + mkdir(dirname) + except OSError: + raise ConfigurationError("Unable to create directory specified by --annotations-dir") + + self.user_annotations = AnnotationsLocalFile(dirname, filename) + + # if the user has specified a fixed label file, go ahead and validate it + # so that we can remove errors early in the process. + if self.single_dataset__datapath and self.user_annotations__local_file_csv__file: + with context["matrix_cache"].data_adaptor(self.single_dataset__datapath, self) as data_adaptor: + data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) + + if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: + try: + self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) + except OntologyLoadFailure as e: + raise ConfigurationError("Unable to load ontology terms\n" + str(e)) + + else: + if self.user_annotations__type == "local_file_csv": + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + if filename is not None: + context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.") + if dirname is not None: + context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.") + + if self.user_annotations__ontology__enable: + context["messagefn"]( + "Warning: --experimental-annotations-ontology" " ignored as annotations are disabled." + ) + if self.user_annotations__ontology__obo_location is not None: + context["messagefn"]( + "Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled." + ) + + def handle_embeddings(self, context): + self.__check_attr("embeddings__names", (list, tuple)) + self.__check_attr("embeddings__enable_reembedding", bool) + + if self.single_dataset__datapath: + if self.embeddings__enable_reembedding: + matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath) + if matrix_data_loader.matrix_data_type() != MatrixDataType.H5AD: + raise ConfigurationError("'enable-reembedding is only supported with H5AD files.") + if self.adaptor__anndata_adaptor__backed: + raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.") + + def handle_diffexp(self, context): + self.__check_attr("diffexp__enable", bool) + self.__check_attr("diffexp__lfc_cutoff", float) + + if self.single_dataset__datapath: + with context["matrix_cache"].data_adaptor(self.single_dataset__datapath, self) as data_adaptor: + if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): + context["messagefn"]( + f"CAUTION: due to the size of your dataset, " + f"running differential expression may take longer or fail." + ) + + def handle_adaptor(self, context): + # cxg + self.__check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict) + from server.data_cxg.cxg_adaptor import CxgAdaptor + CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx) + + # anndata + self.__check_attr("adaptor__anndata_adaptor__backed", bool) def get_title(self, data_adaptor): - return self.title if self.title else data_adaptor.get_title() + return self.single_dataset__title if self.single_dataset__title else data_adaptor.get_title() def get_about(self, data_adaptor): - return self.about if self.about else data_adaptor.get_about() + return self.single_dataset__about if self.single_dataset__about else data_adaptor.get_about() - def get_config(self, data_adaptor, annotation=None): + def get_client_config(self, data_adaptor, annotation=None): + """ + Return the configuration as required by the /config REST route + """ # FIXME The current set of config is not consistently presented: # we have camalCase, hyphen-text, and underscore_text + if not self.is_completed: + raise ConfigurationError("The configuration has not been completed") + # features features = [f.todict() for f in data_adaptor.get_features(annotation)] @@ -110,14 +385,14 @@ class AppConfig(object): # parameters parameters = { - "layout": self.layout, - "max-category-items": self.max_category_items, - "obs_names": self.obs_names, - "var_names": self.var_names, - "diffexp_lfc_cutoff": self.diffexp_lfc_cutoff, - "backed": self.anndata_backed, - "disable-diffexp": self.disable_diffexp, - "enable-reembedding": self.enable_reembedding, + "layout": self.embeddings__names, + "max-category-items": self.presentation__max_categories, + "obs_names": self.single_dataset__obs_names, + "var_names": self.single_dataset__var_names, + "diffexp_lfc_cutoff": self.diffexp__lfc_cutoff, + "backed": self.adaptor__anndata_adaptor__backed, + "disable-diffexp": not self.diffexp__enable, + "enable-reembedding": self.embeddings__enable_reembedding, "annotations": False, "annotations_file": None, "annotations_dir": None, diff --git a/server/common/default_config.py b/server/common/default_config.py new file mode 100644 index 00000000..ded28029 --- /dev/null +++ b/server/common/default_config.py @@ -0,0 +1,67 @@ +import yaml + +default_config = """ +# cellxgene configuration + +server: + verbose: false + debug: false + host: "127.0.0.1" + port : null + scripts : [] + open_browser: false + +presentation: + max_categories: 1000 + +multi_dataset: + dataroot: null + + # The index page when in multi-dataset mode: + # false or null: this returns a 404 code + # true: loads a test index page, which links to the datasets that are available in the dataroot + # string/URL: redirect to this URL: flask.redirect(config.multi_dataset__index) + index: false + + # A list of allowed matrix types. If an empty list, then all matrix types are allowed + allowed_matrix_types: [] + +single_dataset: + datapath: null + obs_names: null + var_names: null + about: null + title: null + +user_annotations: + enable: true + type: local_file_csv + local_file_csv: + directory: null + file: null + ontology: + enable: false + obo_location: null + +embeddings: + names : [] + enable_reembedding: false + +diffexp: + enable: true + lfc_cutoff: 0.01 + +adaptor: + cxg_adaptor: + tiledb_ctx: + sm.tile_cache_size: 8589934592 + sm.num_reader_threads: 32 + vfs.s3.region: us-east-1 + + anndata_adaptor: + backed: false +""" + + +def get_default_config(): + return yaml.load(default_config, Loader=yaml.Loader) diff --git a/server/common/errors.py b/server/common/errors.py index 3d96a728..6ef18c57 100644 --- a/server/common/errors.py +++ b/server/common/errors.py @@ -60,3 +60,11 @@ class OntologyLoadFailure(Exception): """ pass + + +class ConfigurationError(Exception): + """ + Raised when checking configuration errors + """ + + pass diff --git a/server/common/rest.py b/server/common/rest.py index a446ae02..5f920bb4 100644 --- a/server/common/rest.py +++ b/server/common/rest.py @@ -33,7 +33,7 @@ def schema_get(data_adaptor, annotations): def config_get(app_config, data_adaptor, annotations): - config = app_config.get_config(data_adaptor, annotations) + config = app_config.get_client_config(data_adaptor, annotations) return make_response(jsonify(config), HTTPStatus.OK) @@ -131,7 +131,7 @@ def data_var_put(request, data_adaptor): def diffexp_obs_post(request, data_adaptor): - if data_adaptor.config.disable_diffexp: + if not data_adaptor.config.diffexp__enable: return make_response(f"diffexp not supported.", HTTPStatus.BAD_REQUEST) args = request.get_json() @@ -198,7 +198,7 @@ def layout_obs_put(request, data_adaptor): preferred_mimetype = request.accept_mimetypes.best_match(["application/octet-stream"]) if preferred_mimetype != "application/octet-stream": return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE) - if not data_adaptor.config.enable_reembedding: + if not data_adaptor.config.embedding__enable_reembedding: return make_response(f"Computed embedding not supported.", HTTPStatus.BAD_REQUEST) args = request.get_json() diff --git a/server/data_anndata/anndata_adaptor.py b/server/data_anndata/anndata_adaptor.py index 7f39c81a..d8441e4f 100644 --- a/server/data_anndata/anndata_adaptor.py +++ b/server/data_anndata/anndata_adaptor.py @@ -95,7 +95,9 @@ class AnndataAdaptor(DataAdaptor): """ self.original_obs_index = self.data.obs.index - for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")): + for (ax_name, var_name) in ((Axis.OBS, "obs"), (Axis.VAR, "var")): + config_name = f"single_dataset__{var_name}_names" + parameter_name = f"{var_name}_names" name = getattr(self.config, config_name) df_axis = getattr(self.data, str(ax_name)) if name is None: @@ -107,7 +109,7 @@ class AnndataAdaptor(DataAdaptor): "alternative with --{ax_name}-name." ) name = self._create_unique_column_name(df_axis.columns, "name_") - self.parameters[config_name] = name + self.parameters[parameter_name] = name # reset index to simple range; alias name to point at the # previously specified index. df_axis.rename_axis(name, inplace=True) @@ -119,7 +121,7 @@ class AnndataAdaptor(DataAdaptor): f"Values in {ax_name}.{name} must be unique. " "Please prepare data to contain unique values." ) df_axis.reset_index(drop=True, inplace=True) - self.parameters[config_name] = name + self.parameters[parameter_name] = name else: # user specified a non-existent column name raise KeyError(f"Annotation name {name}, specified in --{ax_name}-name does not exist.") @@ -157,7 +159,7 @@ class AnndataAdaptor(DataAdaptor): with data_locator.local_handle() as lh: # as of AnnData 0.6.19, backed mode performs initial load fast, but at the # cost of significantly slower access to X data. - backed = "r" if self.config.anndata_backed else None + backed = "r" if self.config.adaptor__anndata_adaptor__backed else None self.data = anndata.read_h5ad(lh, backed=backed) except ValueError: @@ -177,7 +179,7 @@ class AnndataAdaptor(DataAdaptor): ) def _validate_and_initialize(self): - if anndata_version_is_pre_070() and self.config.anndata_backed: + if anndata_version_is_pre_070() and self.config.adaptor__anndata_adaptor__backed: warnings.warn( f"Use of --backed mode with anndata versions older than 0.7 will have serious " "performance issues. Please update to at least anndata 0.7 or later." @@ -195,7 +197,7 @@ class AnndataAdaptor(DataAdaptor): # heuristic n_values = self.data.shape[0] * self.data.shape[1] - if (n_values > 1e8 and self.config.anndata_backed is True) or (n_values > 5e8): + if (n_values > 1e8 and self.config.adaptor__anndata_adaptor__backed is True) or (n_values > 5e8): self.parameters.update({"diffexp_may_be_slow": True}) def _is_valid_layout(self, arr): @@ -242,7 +244,7 @@ class AnndataAdaptor(DataAdaptor): ) if isinstance(datatype, CategoricalDtype): category_num = len(curr_axis[ann].dtype.categories) - if category_num > 500 and category_num > self.config.max_category_items: + if category_num > 500 and category_num > self.config.presentation__max_categories: warnings.warn( f"{str(ax).title()} annotation '{ann}' has {category_num} categories, this may be " f"cumbersome or slow to display. We recommend setting the " @@ -273,7 +275,7 @@ class AnndataAdaptor(DataAdaptor): c) cap total list of layouts at global const MAX_LAYOUTS """ # load default layouts from the data. - layouts = self.config.layout + layouts = self.config.embeddings__names if layouts is None or len(layouts) == 0: layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")] @@ -342,7 +344,7 @@ class AnndataAdaptor(DataAdaptor): return getattr(self.data.obs, term_name) def get_obs_index(self): - name = getattr(self.config, "obs_names") + name = self.config.single_dataset__obs_names if name is None: return self.original_obs_index else: diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index e3607f81..4fd9d7ae 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -16,12 +16,8 @@ class DataAdaptor(metaclass=ABCMeta): """Base class for loading and accessing matrix data""" def __init__(self, config): - # config will normally be a type that inherits from AppConfig. - # the following is for backwards compatability with tests - if config is None: - config = AppConfig() - elif type(config) == dict: - config = AppConfig(**config) + if type(config) != AppConfig: + raise TypeError("config expected to be of type AppConfig") # config is the application configuration self.config = config @@ -140,8 +136,8 @@ class DataAdaptor(metaclass=ABCMeta): features = [ AppFeature("/cluster/", method="POST", available=False), AppFeature("/layout/obs", method="GET", available=self.get_embedding_names() is not None), - AppFeature("/layout/obs", method="PUT", available=self.config.enable_reembedding), - AppFeature("/diffexp/", method="POST", available=not self.config.disable_diffexp), + AppFeature("/layout/obs", method="PUT", available=self.config.embeddings__enable_reembedding), + AppFeature("/diffexp/", method="POST", available=self.config.diffexp__enable), AppFeature("/annotations/obs", method="PUT", available=annotations is not None), ] return features @@ -290,7 +286,7 @@ class DataAdaptor(metaclass=ABCMeta): if top_n is None: top_n = DEFAULT_TOP_N - result = diffexp_ttest(self, obs_mask_A, obs_mask_B, top_n, self.config.diffexp_lfc_cutoff) + result = diffexp_ttest(self, obs_mask_A, obs_mask_B, top_n, self.config.diffexp__lfc_cutoff) try: return jsonify_numpy(result) diff --git a/server/data_common/matrix_loader.py b/server/data_common/matrix_loader.py index e684d730..a775868a 100644 --- a/server/data_common/matrix_loader.py +++ b/server/data_common/matrix_loader.py @@ -174,12 +174,12 @@ class MatrixDataLoader(object): if not app_config: return True - if not app_config.dataroot: + if not app_config.multi_dataset__dataroot: return True - if len(app_config.multi_dataset_allowed_matrix_type) == 0: + if len(app_config.multi_dataset__allowed_matrix_types) == 0: return True - for val in app_config.multi_dataset_allowed_matrix_type: + for val in app_config.multi_dataset__allowed_matrix_types: try: if self.matrix_data_type == MatrixDataType(val): return True diff --git a/server/data_cxg/cxg_adaptor.py b/server/data_cxg/cxg_adaptor.py index 71775edd..3439ea2a 100644 --- a/server/data_cxg/cxg_adaptor.py +++ b/server/data_cxg/cxg_adaptor.py @@ -1,7 +1,7 @@ import os import json from server.common.utils import dtype_to_schema -from server.common.errors import DatasetAccessError +from server.common.errors import DatasetAccessError, ConfigurationError from server.common.utils import path_join from server.common.constants import Axis from server.data_common.data_adaptor import DataAdaptor @@ -16,7 +16,11 @@ import threading class CxgAdaptor(DataAdaptor): # TODO: The tiledb context parameters should be a configuration option - tiledb_ctx = tiledb.Ctx({"sm.tile_cache_size": 8 * 1024 * 1024 * 1024, "sm.num_reader_threads": 32}) + tiledb_ctx = tiledb.Ctx({ + "sm.tile_cache_size": 8 * 1024 * 1024 * 1024, + "sm.num_reader_threads": 32, + "vfs.s3.region": "us-east-1" + }) def __init__(self, data_locator, config=None): super().__init__(config) @@ -36,6 +40,14 @@ class CxgAdaptor(DataAdaptor): array.close() self.arrays.clear() + @staticmethod + def set_tiledb_context(context_params): + """Set the tiledb context. This should be set before any instances of CxgAdaptor are created""" + try: + CxgAdaptor.tiledb_ctx = tiledb.Ctx(context_params) + except tiledb.libtiledb.TileDBError as e: + raise ConfigurationError(f"Invalid tiledb context: {str(e)}") + @staticmethod def pre_load_validation(data_locator): location = data_locator.uri_or_path @@ -100,15 +112,15 @@ class CxgAdaptor(DataAdaptor): Return True if this looks like a valid CXG, False if not. Just a quick/cheap test, not to be fully trusted. """ - if not tiledb.object_type(url) == "group": + if not tiledb.object_type(url, ctx=CxgAdaptor.tiledb_ctx) == "group": return False - if not tiledb.object_type(path_join(url, "obs")) == "array": + if not tiledb.object_type(path_join(url, "obs"), ctx=CxgAdaptor.tiledb_ctx) == "array": return False - if not tiledb.object_type(path_join(url, "var")) == "array": + if not tiledb.object_type(path_join(url, "var"), ctx=CxgAdaptor.tiledb_ctx) == "array": return False - if not tiledb.object_type(path_join(url, "X")) == "array": + if not tiledb.object_type(path_join(url, "X"), ctx=CxgAdaptor.tiledb_ctx) == "array": return False - if not tiledb.object_type(path_join(url, "emb")) == "group": + if not tiledb.object_type(path_join(url, "emb"), ctx=CxgAdaptor.tiledb_ctx) == "group": return False return True @@ -126,7 +138,7 @@ class CxgAdaptor(DataAdaptor): * version 0.1 -- metadata attache to cxg_group_metadata array. Same as 0, except it adds group metadata. """ - a_type = tiledb.object_type(path_join(self.url, "cxg_group_metadata")) + a_type = tiledb.object_type(path_join(self.url, "cxg_group_metadata"), ctx=self.tiledb_ctx) if a_type is None: # version 0 cxg_version = "0.0" diff --git a/server/eb/Makefile b/server/eb/Makefile index edef6797..66ab68a5 100644 --- a/server/eb/Makefile +++ b/server/eb/Makefile @@ -20,6 +20,9 @@ build: clean cp app.py artifact.dir/application.py; \ cp ../requirements.txt artifact.dir; \ cp -r .ebextensions artifact.dir; \ + if [ -f config.yaml ] ; then \ + cp config.yaml artifact.dir; \ + fi; \ (cd artifact.dir; \ cp -r server/common/web/static static; \ zip -r ../artifact.zip . --exclude server/test/\* server/eb/\* ; ); \ diff --git a/server/eb/README.md b/server/eb/README.md index bf751280..9d89b46a 100644 --- a/server/eb/README.md +++ b/server/eb/README.md @@ -40,6 +40,10 @@ There are many more options to these commands that may be important or necessary 3. Create the artifact.zip file for the application + If you have additional configuration for the application, + place that application in server/eb/config.yaml. This will + be included in the eb deployment. + ``` make build ``` diff --git a/server/eb/app.py b/server/eb/app.py index 5ff0600e..ae322bf0 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -24,49 +24,53 @@ try: from server.app.app import Server from server.data_common.matrix_loader import MatrixDataCacheManager except Exception: - logging.exception("Exception importing server modules") + logging.critical("Exception importing server modules", exc_info=True) sys.exit(1) try: dataroot = os.getenv("CXG_DATAROOT") - if dataroot is None: - logging.error("CXG_DATAROOT environment variable must be set") - sys.exit(1) + app_config = AppConfig() - app_config = AppConfig( - datapath=None, - dataroot=dataroot, - title="", - about=None, - scripts=[], - layout=[], - max_category_items=100, - diffexp_lfc_cutoff=0.01, - obs_names=None, - var_names=None, - anndata_backed=False, - disable_diffexp=True, - multi_dataset_index=None, - multi_dataset_allowed_matrix_type=["cxg"], + config_file = "config.yaml" + if os.path.exists(config_file): + logging.info(f"Configuration from {config_file}") + app_config.update_from_config_file(config_file) + + if dataroot: + logging.info(f"Configuration from CXG_DATAROOT") + app_config.update( + multi_dataset__dataroot=dataroot, + ) + + # features are unsupported in the current hosted server + app_config.update( + diffexp__enable=False, + user_annotations__enable=False, + embeddings__enable_reembedding=False, + multi_dataset__allowed_matrix_types=["cxg"], ) matrix_data_cache_manager = MatrixDataCacheManager() - annotations = None + app_config.complete_config(matrix_data_cache_manager, logging.info) + user_annotations = app_config.user_annotations - server = Server(matrix_data_cache_manager, annotations, app_config) + server = Server(matrix_data_cache_manager, user_annotations, app_config) debug = False application = server.app except Exception: - logging.exception("Caught exception during initialization") + logging.critical("Caught exception during initialization", exc_info=True) sys.exit(1) -logging.info(f"starting server with CXG_DATAROOT={dataroot}") +if app_config.multi_dataset__dataroot: + logging.info(f"starting server with multi_dataset__dataroot={app_config.multi_dataset__dataroot}") +elif app_config.single_dataset__datapath: + logging.info(f"starting server with single_dataset__datapath={app_config.single_dataset__datapath}") if __name__ == "__main__": try: application.run(debug=debug, threaded=not debug, use_debugger=False) except Exception: - logging.exception("Caught exception during server run") + logging.critical("Caught exception during initialization", exc_info=True) sys.exit(1) diff --git a/server/requirements.txt b/server/requirements.txt index dbb6a2ee..69ea7553 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -8,10 +8,12 @@ Flask-Cors>=3.0.6 Flask-RESTful>=0.3.6 flask-server-timing>=0.1.2 flatbuffers>=1.10.0 +flatten-dict>=0.2.0 fsspec>=0.4.4 numpy>=1.16.0 packaging>=20.0 pandas>=0.24.2 +PyYAML>=5.3 scipy>=1.3.0 requests>=2.22.0 tiledb>=0.5.3 diff --git a/server/test/__init__.py b/server/test/__init__.py index 5689d1e1..d096f4c4 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -6,6 +6,7 @@ import pandas as pd from server.common.annotations import AnnotationsLocalFile from server.common.data_locator import DataLocator +from server.common.app_config import AppConfig from server.data_common.fbs.matrix import encode_matrix_fbs from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType @@ -16,18 +17,22 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False): if annotations_fixture: shutil.copyfile(f"test/test_datasets/pbmc3k-annotations.csv", annotations_file) args = { - "layout": ["umap"], - "max_category_items": 100, - "obs_names": None, - "var_names": None, - "diffexp_lfc_cutoff": 0.01, + "embeddings__names": ["umap"], + "presentation__max_categories": 100, + "single_dataset__obs_names": None, + "single_dataset__var_names": None, + "diffexp__lfc_cutoff": 0.01, } fname = { MatrixDataType.H5AD: "../example-dataset/pbmc3k.h5ad", MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg", }[ext] data_locator = DataLocator(fname) - data = MatrixDataLoader(data_locator.abspath()).open(args) + config = AppConfig() + config.update(**args) + config.update(single_dataset__datapath=data_locator.path) + config.complete_config() + data = MatrixDataLoader(data_locator.abspath()).open(config) annotations = AnnotationsLocalFile(None, annotations_file) return data, tmp_dir, annotations diff --git a/server/test/test_anndata_adaptor.py b/server/test/test_anndata_adaptor.py index d7514ae9..dc7f2f37 100644 --- a/server/test/test_anndata_adaptor.py +++ b/server/test/test_anndata_adaptor.py @@ -13,6 +13,7 @@ import pandas as pd from server.data_anndata.anndata_adaptor import AnndataAdaptor from server.common.errors import FilterError from server.common.data_locator import DataLocator +from server.common.app_config import AppConfig """ Test the anndata adaptor using the pbmc3k data set. @@ -33,14 +34,18 @@ Test the anndata adaptor using the pbmc3k data set. class AdaptorTest(unittest.TestCase): def setUp(self): args = { - "layout": ["umap"], - "max_category_items": 100, - "obs_names": None, - "var_names": None, - "diffexp_lfc_cutoff": 0.01, - "anndata_backed": self.backed, + "embeddings__names": ["umap"], + "presentation__max_categories": 100, + "single_dataset__obs_names": None, + "single_dataset__var_names": None, + "diffexp__lfc_cutoff": 0.01, + "adaptor__anndata_adaptor__backed": self.backed, + "single_dataset__datapath" : self.data_locator } - self.data = AnndataAdaptor(DataLocator(self.data_locator), args) + config = AppConfig() + config.update(**args) + config.complete_config() + self.data = AnndataAdaptor(DataLocator(self.data_locator), config) def test_init(self): self.assertEqual(self.data.cell_count, 2638) @@ -88,7 +93,7 @@ class AdaptorTest(unittest.TestCase): def test_get_schema(self): with open(path.join(path.dirname(__file__), "schema.json")) as fh: schema = json.load(fh) - self.assertEqual(self.data.get_schema(), schema) + self.assertDictEqual(self.data.get_schema(), schema) def test_schema_produces_error(self): self.data.data.obs["time"] = pd.Series( @@ -109,9 +114,9 @@ class AdaptorTest(unittest.TestCase): self.assertEqual(len(feature), 1) check_feature("POST", "/cluster/", False) - check_feature("POST", "/diffexp/", not self.data.config.disable_diffexp) + check_feature("POST", "/diffexp/", self.data.config.diffexp__enable) check_feature("GET", "/layout/obs", True) - check_feature("PUT", "/layout/obs", self.data.config.enable_reembedding) + check_feature("PUT", "/layout/obs", self.data.config.embeddings__enable_reembedding) check_feature("PUT", "/annotations/obs", False) def test_layout(self): diff --git a/server/test/test_anndata_adaptor_data_load.py b/server/test/test_anndata_adaptor_data_load.py index 2a7b1fb8..c16bd7f5 100644 --- a/server/test/test_anndata_adaptor_data_load.py +++ b/server/test/test_anndata_adaptor_data_load.py @@ -3,6 +3,7 @@ import json from server.data_anndata.anndata_adaptor import AnndataAdaptor from server.common.data_locator import DataLocator +from server.common.app_config import AppConfig class DataLoadAdaptorTest(unittest.TestCase): @@ -12,7 +13,10 @@ class DataLoadAdaptorTest(unittest.TestCase): def setUp(self): self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad") - self.data = AnndataAdaptor(self.data_file) + config = AppConfig() + config.update(single_dataset__datapath=self.data_file.path) + config.complete_config() + self.data = AnndataAdaptor(self.data_file, config) def test_delayed_load_data(self): self.data._create_schema() @@ -37,11 +41,11 @@ class DataLocatorAdaptorTest(unittest.TestCase): def setUp(self): self.args = { - "layout": ["umap"], - "max_category_items": 100, - "obs_names": None, - "var_names": None, - "diffexp_lfc_cutoff": 0.01, + "embeddings__names": ["umap"], + "presentation__max_categories": 100, + "single_dataset__obs_names": None, + "single_dataset__var_names": None, + "diffexp__lfc_cutoff": 0.01, } def stdAsserts(self, data): @@ -52,17 +56,25 @@ class DataLocatorAdaptorTest(unittest.TestCase): def test_posix_file(self): locator = DataLocator("../example-dataset/pbmc3k.h5ad") - data = AnndataAdaptor(locator, self.args) + config = AppConfig() + config.update(**self.args) + config.update(single_dataset__datapath=locator.path) + config.complete_config() + data = AnndataAdaptor(locator, config) self.stdAsserts(data) def test_url_https(self): url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad" locator = DataLocator(url) - data = AnndataAdaptor(locator, self.args) + config = AppConfig() + config.update(**self.args) + data = AnndataAdaptor(locator, config) self.stdAsserts(data) def test_url_http(self): url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/example-dataset/pbmc3k.h5ad" locator = DataLocator(url) - data = AnndataAdaptor(locator, self.args) + config = AppConfig() + config.update(**self.args) + data = AnndataAdaptor(locator, config) self.stdAsserts(data) diff --git a/server/test/test_nan_anndata_adaptor.py b/server/test/test_nan_anndata_adaptor.py index c56ccc20..6ce560ab 100644 --- a/server/test/test_nan_anndata_adaptor.py +++ b/server/test/test_nan_anndata_adaptor.py @@ -8,25 +8,37 @@ import server.test.decode_fbs as decode_fbs from server.data_anndata.anndata_adaptor import AnndataAdaptor from server.common.errors import FilterError from server.common.data_locator import DataLocator +from server.common.app_config import AppConfig class NaNTest(unittest.TestCase): def setUp(self): self.args = { - "layout": ["umap"], - "max_category_items": 100, - "obs_names": None, - "var_names": None, - "diffexp_lfc_cutoff": 0.01, + "embeddings__names": ["umap"], + "presentation__max_categories": 100, + "single_dataset__obs_names": None, + "single_dataset__var_names": None, + "diffexp__lfc_cutoff": 0.01, } + config = AppConfig() + config.update(**self.args) + locator = DataLocator("test/test_datasets/nan.h5ad") + config.update(single_dataset__datapath=locator.path) + config.complete_config() + with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) - self.data = AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args) + self.data = AnndataAdaptor(locator, config) self.data._create_schema() def test_load(self): with self.assertWarns(UserWarning): - AnndataAdaptor(DataLocator("test/test_datasets/nan.h5ad"), self.args) + config = AppConfig() + config.update(**self.args) + locator = DataLocator("test/test_datasets/nan.h5ad") + config.update(single_dataset__datapath=locator.path) + config.complete_config() + self.data = AnndataAdaptor(locator, config) def test_init(self): self.assertEqual(self.data.cell_count, 100) diff --git a/server/test/test_writable_annotation.py b/server/test/test_writable_annotation.py index 57ac9784..693bc373 100644 --- a/server/test/test_writable_annotation.py +++ b/server/test/test_writable_annotation.py @@ -149,7 +149,7 @@ class WritableAnnotationTest(unittest.TestCase): self.assertEqual(len(feature), 1) check_feature("POST", "/cluster/", False) - check_feature("POST", "/diffexp/", not self.data.config.disable_diffexp) + check_feature("POST", "/diffexp/", self.data.config.diffexp__enable) check_feature("GET", "/layout/obs", True) - check_feature("PUT", "/layout/obs", self.data.config.enable_reembedding) + check_feature("PUT", "/layout/obs", self.data.config.embeddings__enable_reembedding) check_feature("PUT", "/annotations/obs", True)