config refactor (#1854)

* split out config

* add tests for base and app config, refactor client config out of app config

* refactor default config retrieval

* create config test class and helper functions

* move default_config into server to fix import issue

server/cli/convert_to_cxg.py

@@ -9,26 +9,24 @@ from server.converters.h5ad_data_file import H5ADDataFile
     name="convert",
     short_help="Converts an H5AD dataset to the CXG format.",
     help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
-         "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
-         "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
-         "usually with the generated CXG file.",
+    "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
+    "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
+    "usually with the generated CXG file.",
 )
 @click.argument(
-    "input-file",
-    nargs=1,
-    type=click.Path(exists=True, dir_okay=False),
+    "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
 )
 @click.option(
     "-o",
     "--output-directory",
     help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
-         "CXG extension.",
+    "CXG extension.",
 )
 @click.option(
     "-b",
     "--backed",
     help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
-         "but will use less memory.",
+    "but will use less memory.",
     default=False,
     show_default=True,
     is_flag=True,
@@ -37,29 +35,33 @@ from server.converters.h5ad_data_file import H5ADDataFile
     "-t",
     "--title",
     help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
-         "the dataset title will be the filename.",
+    "the dataset title will be the filename.",
 )
 @click.option(
     "-a",
     "--about",
     help="A fully qualified URL that provides more information about the dataset and will be included as "
-         "metadata about the CXG file.",
+    "metadata about the CXG file.",
 )
 @click.option(
     "-s",
     "--sparse-threshold",
     help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
-         "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
-         "convert to dense array.",
+    "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
+    "convert to dense array.",
     default=0.0,
     show_default=True,
 )
-@click.option("--obs-names",
-              help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
-                   "the one designated by the dataframe generated-index.")
-@click.option("--var-names",
-              help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
-                   "the one designated by the dataframe generated-index.")
+@click.option(
+    "--obs-names",
+    help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
+    "the one designated by the dataframe generated-index.",
+)
+@click.option(
+    "--var-names",
+    help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
+    "the one designated by the dataframe generated-index.",
+)
 @click.option(
     "--disable-custom-colors",
     help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
@@ -70,8 +72,8 @@ from server.converters.h5ad_data_file import H5ADDataFile
 @click.option(
     "--disable-corpora-schema",
     help="When set, conversion process will neither extract nor store Corpora schema information. See "
-         "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
-         "information.",
+    "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
+    "information.",
     default=False,
     show_default=True,
     is_flag=True,
@@ -85,30 +87,32 @@ from server.converters.h5ad_data_file import H5ADDataFile
 )
 @click.help_option("--help", "-h", help="Show this message and exit.")
 def convert_to_cxg(
-        input_file,
-        output_directory,
-        backed,
-        title,
-        about,
-        sparse_threshold,
-        obs_names,
-        var_names,
-        disable_custom_colors,
-        disable_corpora_schema,
-        overwrite,
+    input_file,
+    output_directory,
+    backed,
+    title,
+    about,
+    sparse_threshold,
+    obs_names,
+    var_names,
+    disable_custom_colors,
+    disable_corpora_schema,
+    overwrite,
 ):
     """
     Convert a dataset file into CXG.
     """
 
-    h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names,
-                                  use_corpora_schema=not disable_corpora_schema)
+    h5ad_data_file = H5ADDataFile(
+        input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
+    )
 
     # Get the directory that will hold all the CXG files
     cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
 
-    h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold,
-                          convert_anndata_colors_to_cxg_colors=not disable_custom_colors)
+    h5ad_data_file.to_cxg(
+        cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
+    )
 
 
 def get_output_directory(input_filename, output_directory, should_overwrite):

server/cli/launch.py

@@ -3,15 +3,14 @@ import functools
 import logging
 import sys
 import webbrowser
-from os import devnull
-
+import os
 import click
 from flask_compress import Compress
 from flask_cors import CORS
 
+from server.default_config import default_config
 from server.app.app import Server
-from server.common.app_config import AppConfig
-from server.common.default_config import default_config
+from server.common.config.app_config import AppConfig
 from server.common.errors import DatasetAccessError, ConfigurationError
 from server.common.utils.utils import sort_options
 
@@ -33,7 +32,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<path>",
         help="CSV file to initialize editing of existing annotations; will be altered in-place. "
-             "Incompatible with --annotations-dir.",
+        "Incompatible with --annotations-dir.",
     )
     @click.option(
         "--annotations-dir",
@@ -42,7 +41,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<directory path>",
         help="Directory of where to save output annotations; filename will be specified in the application. "
-             "Incompatible with --annotations-file.",
+        "Incompatible with --annotations-file.",
     )
     @click.option(
         "--experimental-annotations-ontology",
@@ -170,7 +169,7 @@ def server_args(func):
         default=DEFAULT_CONFIG.server_config.app__debug,
         show_default=True,
         help="Run in debug mode. This is helpful for cellxgene developers, "
-             "or when you want more information about an error condition.",
+        "or when you want more information about an error condition.",
     )
     @click.option(
         "--verbose",
@@ -203,7 +202,7 @@ def server_args(func):
         multiple=True,
         metavar="<text>",
         help="Additional script files to include in HTML page. If not specified, "
-             "no additional script files will be included.",
+        "no additional script files will be included.",
         show_default=False,
     )
     @functools.wraps(func)
@@ -223,7 +222,7 @@ def launch_args(func):
         default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
         metavar="<data directory>",
         help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
-             " to folder containing H5AD and/or CXG datasets.",
+        " to folder containing H5AD and/or CXG datasets.",
         hidden=True,
     )  # TODO, unhide when dataroot is supported)
     @click.argument("datapath", required=False, metavar="<path to data file>")
@@ -307,32 +306,32 @@ class CliLaunchServer(Server):
 )
 @launch_args
 def launch(
-        datapath,
-        dataroot,
-        verbose,
-        debug,
-        open_browser,
-        port,
-        host,
-        embedding,
-        obs_names,
-        var_names,
-        max_category_items,
-        disable_custom_colors,
-        diffexp_lfc_cutoff,
-        title,
-        scripts,
-        about,
-        disable_annotations,
-        annotations_file,
-        annotations_dir,
-        backed,
-        disable_diffexp,
-        experimental_annotations_ontology,
-        experimental_annotations_ontology_obo,
-        experimental_enable_reembedding,
-        config_file,
-        dump_default_config,
+    datapath,
+    dataroot,
+    verbose,
+    debug,
+    open_browser,
+    port,
+    host,
+    embedding,
+    obs_names,
+    var_names,
+    max_category_items,
+    disable_custom_colors,
+    diffexp_lfc_cutoff,
+    title,
+    scripts,
+    about,
+    disable_annotations,
+    annotations_file,
+    annotations_dir,
+    backed,
+    disable_diffexp,
+    experimental_annotations_ontology,
+    experimental_annotations_ontology_obo,
+    experimental_enable_reembedding,
+    config_file,
+    dump_default_config,
 ):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.
@@ -443,7 +442,7 @@ def launch(
     click.echo("[cellxgene] Type CTRL-C at any time to exit.")
 
     if not server_config.app__verbose:
-        f = open(devnull, "w")
+        f = open(os.devnull, "w")
         sys.stdout = f
 
     try:

server/cli/prepare.py

@@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options
     default=False,
     is_flag=True,
     help="Do not run quality control metrics. By default cellxgene runs them "
-         "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
+    "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
 )
 @click.option(
     "--make-obs-names-unique/--no-make-obs-names-unique",
@@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options
 )
 @click.help_option("--help", "-h", help="Show this message and exit.")
 def prepare(
-        data,
-        embedding,
-        recipe,
-        output,
-        plotting,
-        sparse,
-        overwrite,
-        set_obs_names,
-        set_var_names,
-        skip_qc,
-        make_obs_names_unique,
-        make_var_names_unique,
+    data,
+    embedding,
+    recipe,
+    output,
+    plotting,
+    sparse,
+    overwrite,
+    set_obs_names,
+    set_var_names,
+    skip_qc,
+    make_obs_names_unique,
+    make_var_names_unique,
 ):
     """
     Preprocess data for use with cellxgene.

server/cli/upgrade.py

@@ -10,7 +10,8 @@ from .. import __version__
 SEMVER_FORMAT = re.compile(
     r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
     r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
-    r"?:\.[0-9a-zA-Z-]+)*))?$")
+    r"?:\.[0-9a-zA-Z-]+)*))?$"
+)
 
 
 def log_upgrade_check():