config refactor (#1854)

* split out config

* add tests for base and app config, refactor client config out of app config

* refactor default config retrieval

* create config test class and helper functions

* move default_config into server to fix import issue

Makefile

@@ -83,7 +83,8 @@ lint: lint-server lint-client
 
 .PHONY: lint-server
 lint-server:
-	flake8 server
+	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821'
+
 
 .PHONY: lint-client
 lint-client:

server/__init__.py

@@ -1,6 +1,5 @@
 import logging
 import sys
-
 from server.common.utils.utils import import_plugins
 
 __version__ = "0.16.0"

server/app/app.py

@@ -6,8 +6,17 @@ from urllib.parse import urlparse
 import hashlib
 import os
 
-from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \
+from flask import (
-    send_from_directory
+    Flask,
+    redirect,
+    current_app,
+    make_response,
+    render_template,
+    abort,
+    Blueprint,
+    request,
+    send_from_directory,
+)
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
@@ -87,10 +96,7 @@ def dataset_index(url_dataroot=None, dataset=None):
         cache_manager = current_app.matrix_data_cache_manager
         with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
             data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
-            args = {
+            args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
-                "SCRIPTS" : scripts,
-                "INLINE_SCRIPTS" : inline_scripts
-            }
             return render_template("index.html", **args)
 
     except DatasetAccessError as e:
@@ -417,8 +423,9 @@ class Server:
             for dataroot_dict in server_config.multi_dataset__dataroot.values():
                 url_dataroot = dataroot_dict["base_url"]
                 bp_dataroot = Blueprint(
-                    f"api_dataset_{url_dataroot}", __name__,
+                    f"api_dataset_{url_dataroot}",
-                    url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version
+                    __name__,
+                    url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version,
                 )
                 dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
                 self.app.register_blueprint(dataroot_resources.blueprint)
@@ -433,7 +440,7 @@ class Server:
                     f"/{url_dataroot}/<dataset>/static/<path:filename>",
                     f"static_assets_{url_dataroot}",
                     view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename),
-                    methods=["GET"]
+                    methods=["GET"],
                 )
 
         else:
@@ -444,7 +451,7 @@ class Server:
                 "/static/<path:filename>",
                 "static_assets",
                 view_func=lambda filename: send_from_directory("../common/web/static", filename),
-                methods=["GET"]
+                methods=["GET"],
             )
 
         self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager

server/auth/__init__.py

@@ -1,4 +1,3 @@
-
 # import the built in auth types so they can be registered
 
 import server.auth.auth_none  # noqa: F401

server/auth/auth.py

@@ -76,7 +76,7 @@ class AuthTypeFactory:
 
     @staticmethod
     def register(name, auth_type):
-        assert(issubclass(auth_type, AuthTypeBase))
+        assert issubclass(auth_type, AuthTypeBase)
         AuthTypeFactory.auth_types[name] = auth_type
 
     @staticmethod

server/auth/auth_none.py

@@ -2,7 +2,6 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory
 
 
 class AuthTypeNone(AuthTypeBase):
-
     def __init__(self, app_config):
         super().__init__()
 

server/auth/auth_oauth.py

@@ -97,8 +97,17 @@ class AuthTypeOAuth(AuthTypeClientBase):
             return
 
         valid_keys = {
-            "verify_signature", "verify_aud", "verify_iat", "verify_exp", "verify_nbf", "verify_iss",
+            "verify_signature",
-            "verify_sub", "verify_jti", "verify_at_hash", "leeway"}
+            "verify_aud",
+            "verify_iat",
+            "verify_exp",
+            "verify_nbf",
+            "verify_iss",
+            "verify_sub",
+            "verify_jti",
+            "verify_at_hash",
+            "leeway",
+        }
         keys = set(self.jwt_decode_options.keys())
         unknown = keys - valid_keys
         if unknown:

server/cli/convert_to_cxg.py

@@ -9,26 +9,24 @@ from server.converters.h5ad_data_file import H5ADDataFile
     name="convert",
     short_help="Converts an H5AD dataset to the CXG format.",
     help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
-         "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
+    "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
-         "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
+    "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
-         "usually with the generated CXG file.",
+    "usually with the generated CXG file.",
 )
 @click.argument(
-    "input-file",
+    "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
-    nargs=1,
-    type=click.Path(exists=True, dir_okay=False),
 )
 @click.option(
     "-o",
     "--output-directory",
     help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
-         "CXG extension.",
+    "CXG extension.",
 )
 @click.option(
     "-b",
     "--backed",
     help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
-         "but will use less memory.",
+    "but will use less memory.",
     default=False,
     show_default=True,
     is_flag=True,
@@ -37,29 +35,33 @@ from server.converters.h5ad_data_file import H5ADDataFile
     "-t",
     "--title",
     help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
-         "the dataset title will be the filename.",
+    "the dataset title will be the filename.",
 )
 @click.option(
     "-a",
     "--about",
     help="A fully qualified URL that provides more information about the dataset and will be included as "
-         "metadata about the CXG file.",
+    "metadata about the CXG file.",
 )
 @click.option(
     "-s",
     "--sparse-threshold",
     help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
-         "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
+    "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
-         "convert to dense array.",
+    "convert to dense array.",
     default=0.0,
     show_default=True,
 )
-@click.option("--obs-names",
+@click.option(
-              help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
+    "--obs-names",
-                   "the one designated by the dataframe generated-index.")
+    help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
-@click.option("--var-names",
+    "the one designated by the dataframe generated-index.",
-              help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
+)
-                   "the one designated by the dataframe generated-index.")
+@click.option(
+    "--var-names",
+    help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
+    "the one designated by the dataframe generated-index.",
+)
 @click.option(
     "--disable-custom-colors",
     help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
@@ -70,8 +72,8 @@ from server.converters.h5ad_data_file import H5ADDataFile
 @click.option(
     "--disable-corpora-schema",
     help="When set, conversion process will neither extract nor store Corpora schema information. See "
-         "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
+    "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
-         "information.",
+    "information.",
     default=False,
     show_default=True,
     is_flag=True,
@@ -85,30 +87,32 @@ from server.converters.h5ad_data_file import H5ADDataFile
 )
 @click.help_option("--help", "-h", help="Show this message and exit.")
 def convert_to_cxg(
-        input_file,
+    input_file,
-        output_directory,
+    output_directory,
-        backed,
+    backed,
-        title,
+    title,
-        about,
+    about,
-        sparse_threshold,
+    sparse_threshold,
-        obs_names,
+    obs_names,
-        var_names,
+    var_names,
-        disable_custom_colors,
+    disable_custom_colors,
-        disable_corpora_schema,
+    disable_corpora_schema,
-        overwrite,
+    overwrite,
 ):
     """
     Convert a dataset file into CXG.
     """
 
-    h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names,
+    h5ad_data_file = H5ADDataFile(
-                                  use_corpora_schema=not disable_corpora_schema)
+        input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
+    )
 
     # Get the directory that will hold all the CXG files
     cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
 
-    h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold,
+    h5ad_data_file.to_cxg(
-                          convert_anndata_colors_to_cxg_colors=not disable_custom_colors)
+        cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
+    )
 
 
 def get_output_directory(input_filename, output_directory, should_overwrite):

server/cli/launch.py

@@ -3,15 +3,14 @@ import functools
 import logging
 import sys
 import webbrowser
-from os import devnull
+import os
-
 import click
 from flask_compress import Compress
 from flask_cors import CORS
 
+from server.default_config import default_config
 from server.app.app import Server
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
-from server.common.default_config import default_config
 from server.common.errors import DatasetAccessError, ConfigurationError
 from server.common.utils.utils import sort_options
 
@@ -33,7 +32,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<path>",
         help="CSV file to initialize editing of existing annotations; will be altered in-place. "
-             "Incompatible with --annotations-dir.",
+        "Incompatible with --annotations-dir.",
     )
     @click.option(
         "--annotations-dir",
@@ -42,7 +41,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<directory path>",
         help="Directory of where to save output annotations; filename will be specified in the application. "
-             "Incompatible with --annotations-file.",
+        "Incompatible with --annotations-file.",
     )
     @click.option(
         "--experimental-annotations-ontology",
@@ -170,7 +169,7 @@ def server_args(func):
         default=DEFAULT_CONFIG.server_config.app__debug,
         show_default=True,
         help="Run in debug mode. This is helpful for cellxgene developers, "
-             "or when you want more information about an error condition.",
+        "or when you want more information about an error condition.",
     )
     @click.option(
         "--verbose",
@@ -203,7 +202,7 @@ def server_args(func):
         multiple=True,
         metavar="<text>",
         help="Additional script files to include in HTML page. If not specified, "
-             "no additional script files will be included.",
+        "no additional script files will be included.",
         show_default=False,
     )
     @functools.wraps(func)
@@ -223,7 +222,7 @@ def launch_args(func):
         default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
         metavar="<data directory>",
         help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
-             " to folder containing H5AD and/or CXG datasets.",
+        " to folder containing H5AD and/or CXG datasets.",
         hidden=True,
     )  # TODO, unhide when dataroot is supported)
     @click.argument("datapath", required=False, metavar="<path to data file>")
@@ -307,32 +306,32 @@ class CliLaunchServer(Server):
 )
 @launch_args
 def launch(
-        datapath,
+    datapath,
-        dataroot,
+    dataroot,
-        verbose,
+    verbose,
-        debug,
+    debug,
-        open_browser,
+    open_browser,
-        port,
+    port,
-        host,
+    host,
-        embedding,
+    embedding,
-        obs_names,
+    obs_names,
-        var_names,
+    var_names,
-        max_category_items,
+    max_category_items,
-        disable_custom_colors,
+    disable_custom_colors,
-        diffexp_lfc_cutoff,
+    diffexp_lfc_cutoff,
-        title,
+    title,
-        scripts,
+    scripts,
-        about,
+    about,
-        disable_annotations,
+    disable_annotations,
-        annotations_file,
+    annotations_file,
-        annotations_dir,
+    annotations_dir,
-        backed,
+    backed,
-        disable_diffexp,
+    disable_diffexp,
-        experimental_annotations_ontology,
+    experimental_annotations_ontology,
-        experimental_annotations_ontology_obo,
+    experimental_annotations_ontology_obo,
-        experimental_enable_reembedding,
+    experimental_enable_reembedding,
-        config_file,
+    config_file,
-        dump_default_config,
+    dump_default_config,
 ):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.
@@ -443,7 +442,7 @@ def launch(
     click.echo("[cellxgene] Type CTRL-C at any time to exit.")
 
     if not server_config.app__verbose:
-        f = open(devnull, "w")
+        f = open(os.devnull, "w")
         sys.stdout = f
 
     try:

server/cli/prepare.py

@@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options
     default=False,
     is_flag=True,
     help="Do not run quality control metrics. By default cellxgene runs them "
-         "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
+    "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
 )
 @click.option(
     "--make-obs-names-unique/--no-make-obs-names-unique",
@@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options
 )
 @click.help_option("--help", "-h", help="Show this message and exit.")
 def prepare(
-        data,
+    data,
-        embedding,
+    embedding,
-        recipe,
+    recipe,
-        output,
+    output,
-        plotting,
+    plotting,
-        sparse,
+    sparse,
-        overwrite,
+    overwrite,
-        set_obs_names,
+    set_obs_names,
-        set_var_names,
+    set_var_names,
-        skip_qc,
+    skip_qc,
-        make_obs_names_unique,
+    make_obs_names_unique,
-        make_var_names_unique,
+    make_var_names_unique,
 ):
     """
     Preprocess data for use with cellxgene.

server/cli/upgrade.py

@@ -10,7 +10,8 @@ from .. import __version__
 SEMVER_FORMAT = re.compile(
     r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
     r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
-    r"?:\.[0-9a-zA-Z-]+)*))?$")
+    r"?:\.[0-9a-zA-Z-]+)*))?$"
+)
 
 
 def log_upgrade_check():

server/common/annotations/hosted_tiledb.py

@@ -31,7 +31,8 @@ class AnnotationsHostedTileDB(Annotations):
         unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
         if unsanitary_original_category_names:
             raise AnnotationCategoryNameError(
-                f"{unsanitary_original_category_names} are not valid category names, please resubmit")
+                f"{unsanitary_original_category_names} are not valid category names, please resubmit"
+            )
 
     def is_safe_collection_name(self, name):
         """
@@ -68,11 +69,11 @@ class AnnotationsHostedTileDB(Annotations):
         index_dims = None
         schema_hints = json.loads(schema_hints)
 
-        if '__pandas_attribute_repr' in tileDBArray.meta:
+        if "__pandas_attribute_repr" in tileDBArray.meta:
             # backwards compatibility... unsure if necessary at this point
-            repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr'])
+            repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
-        if '__pandas_index_dims' in tileDBArray.meta:
+        if "__pandas_index_dims" in tileDBArray.meta:
-            index_dims = json.loads(tileDBArray.meta['__pandas_index_dims'])
+            index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
 
         data = tileDBArray[:]
         indexes = list()
@@ -80,12 +81,12 @@ class AnnotationsHostedTileDB(Annotations):
         for col_name, col_val in data.items():
             # If the column values are byte literals, decode them
             if isinstance(col_val[0], bytes):
-                col_val = [value.decode('utf-8') for value in col_val]
+                col_val = [value.decode("utf-8") for value in col_val]
 
             if schema_hints and col_name in schema_hints:
                 type = schema_hints.get(col_name).get("type")
                 if type and type == "categorical":
-                    new_col = pd.Series(col_val, dtype='category')
+                    new_col = pd.Series(col_val, dtype="category")
                     data[col_name] = new_col
             elif repr_meta and col_name in repr_meta:
                 new_col = pd.Series(col_val, dtype=repr_meta[col_name])
@@ -127,7 +128,7 @@ class AnnotationsHostedTileDB(Annotations):
             tiledb_uri=uri,
             user_id=user_id,
             dataset_id=str(dataset_id),
-            schema_hints=json.dumps(dataframe_schema_type_hints)
+            schema_hints=json.dumps(dataframe_schema_type_hints),
         )
         if not df.empty:
             self.check_category_names(df)

server/common/app_config.py

@@ -1,963 +0,0 @@
-import copy
-import os
-import sys
-import warnings
-from os.path import splitext, basename, isdir
-from urllib.parse import urlparse, quote_plus
-
-import yaml
-from flatten_dict import flatten, unflatten
-
-import server.compute.diffexp_cxg as diffexp_tiledb
-import server.compute.scanpy
-from server import display_version as cellxgene_display_version
-from server.auth.auth import AuthTypeFactory
-from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.data_locator import discover_s3_region_name
-from server.common.default_config import get_default_config
-from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
-from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
-from server.db.db_utils import DbUtils
-
-DEFAULT_SERVER_PORT = 5005
-# anything bigger than this will generate a special message
-BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB
-
-
-class AppFeature(object):
-    def __init__(self, path, available=False, method="POST", extra={}):
-        self.path = path
-        self.available = available
-        self.method = method
-        self.extra = extra
-        for k, v in extra.items():
-            setattr(self, k, v)
-
-    def todict(self):
-        d = dict(available=self.available, method=self.method, path=self.path)
-        d.update(self.extra)
-        return d
-
-
-class AppConfig(object):
-    """AppConfig stores all the configuration for cellxgene.  The configuration is divided into two main parts:
-    server attributes, and dataset attributes. The server_config contains attributes that refer to the server process
-    as a whole.   The default_dataset_config referes to attributes that are associated with the features and
-    presentations of a dataset.  The dataset config attributes can be overridden depending on the url by which the
-    dataset was accessed.  These are stored in dataroot_config.
-    AppConfig has methods to initialize, modify, and access the configuration.
-    """
-
-    def __init__(self):
-
-        # the default configuration (see default_config.py)
-        self.default_config = get_default_config()
-        # the server configuration
-        self.server_config = ServerConfig(self, self.default_config["server"])
-        # the dataset config, unless overridden by an entry in dataroot_config
-        self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
-        # a dictionary of keys to DatasetConfig objects.  Each key must exist in the multi_dataset__dataroot
-        # attribute of the server_config.
-        self.dataroot_config = {}
-
-        # Set to true when config_completed is called
-        self.is_completed = False
-
-    def get_dataset_config(self, dataroot_key):
-        if self.server_config.single_dataset__datapath:
-            return self.default_dataset_config
-        else:
-            return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
-
-    def check_config(self):
-        """Verify all the attributes have been checked"""
-        if not self.is_completed:
-            raise ConfigurationError("The configuration has not been completed")
-        self.server_config.check_config()
-        self.default_dataset_config.check_config()
-        for dataset_config in self.dataroot_config.values():
-            dataset_config.check_config()
-
-    def update_server_config(self, **kw):
-        self.server_config.update(**kw)
-        self.is_complete = False
-
-    def update_default_dataset_config(self, **kw):
-        self.default_dataset_config.update(**kw)
-        # update all the other dataset configs, if any
-        for value in self.dataroot_config.values():
-            value.update(**kw)
-        self.is_complete = False
-
-    def update_from_config_file(self, config_file):
-        with open(config_file) as fyaml:
-            config = yaml.load(fyaml, Loader=yaml.FullLoader)
-
-        if config.get("server"):
-            self.server_config.update_from_config(config["server"], "server")
-        if config.get("dataset"):
-            self.default_dataset_config.update_from_config(config["dataset"], "dataset")
-
-        per_dataset_config = config.get("per_dataset_config", {})
-        for key, dataroot_config in per_dataset_config.items():
-            # first create and initialize the dataroot with the default config
-            self.add_dataroot_config(key, **config["dataset"])
-            # then apply the per dataset configuration
-            self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
-
-        self.is_complete = False
-
-    def write_config(self, config_file):
-        """output the config to a yaml file"""
-        server = self.server_config.create_mapping(self.server_config.default_config)
-        dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
-        config = dict(server={}, dataset={})
-        for attrname in server.keys():
-            config["server__" + attrname] = getattr(self.server_config, attrname)
-        for attrname in dataset.keys():
-            config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
-        if self.dataroot_config:
-            config["per_dataset_config"] = {}
-        for dataroot_tag, dataroot_config in self.dataroot_config.items():
-            dataset = dataroot_config.create_mapping(dataroot_config.default_config)
-            for attrname in dataset.keys():
-                config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
-
-        config = unflatten(config, splitter=lambda key: key.split("__"))
-        yaml.dump(config, open(config_file, "w"))
-
-    def changes_from_default(self):
-        """Return all the attribute that are different from the default"""
-        diff_server = self.server_config.changes_from_default()
-        diff_dataset = self.default_dataset_config.changes_from_default()
-        diff = dict(server=diff_server, dataset=diff_dataset)
-        return diff
-
-    def add_dataroot_config(self, dataroot_tag, **kw):
-        """Create a new dataset config object based on the default dataset config, and kw parameters"""
-        if dataroot_tag in self.dataroot_config:
-            raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
-        if type(self.server_config.multi_dataset__dataroot) != dict:
-            raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
-        if dataroot_tag not in self.server_config.multi_dataset__dataroot:
-            raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
-
-        self.is_completed = False
-        self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
-        flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
-        config = {key: value[1] for key, value in flat_config.items()}
-        self.dataroot_config[dataroot_tag].update(**config)
-        self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
-
-    def complete_config(self, messagefn=None):
-        """The configure options are checked, and any additional setup based on the config
-        parameters is done"""
-
-        if messagefn is None:
-            def noop(message):
-                pass
-
-            messagefn = noop
-
-        # TODO: to give better error messages we can add a mapping between where each config
-        # attribute originated (e.g. command line argument or config file), then in the error
-        # messages we can give correct context for attributes with bad value.
-        context = dict(messagefn=messagefn)
-
-        self.server_config.complete_config(context)
-        self.default_dataset_config.complete_config(context)
-        for dataroot_config in self.dataroot_config.values():
-            dataroot_config.complete_config(context)
-
-        self.is_completed = True
-        self.check_config()
-
-    def get_matrix_data_cache_manager(self):
-        return self.server_config.matrix_data_cache_manager
-
-    def is_multi_dataset(self):
-        return self.server_config.multi_dataset__dataroot is not None
-
-    def get_title(self, data_adaptor):
-        return (
-            self.server_config.single_dataset__title
-            if self.server_config.single_dataset__title
-            else data_adaptor.get_title()
-        )
-
-    def get_about(self, data_adaptor):
-        return (
-            self.server_config.single_dataset__about
-            if self.server_config.single_dataset__about
-            else data_adaptor.get_about()
-        )
-
-    def get_client_config(self, data_adaptor):
-        """
-        Return the configuration as required by the /config REST route
-        """
-
-        server_config = self.server_config
-        dataset_config = data_adaptor.dataset_config
-        annotation = dataset_config.user_annotations
-        auth = server_config.auth
-
-        # FIXME The current set of config is not consistently presented:
-        # we have camalCase, hyphen-text, and underscore_text
-
-        # make sure the configuration has been checked.
-        self.check_config()
-
-        # features
-        features = [f.todict() for f in data_adaptor.get_features(annotation)]
-
-        # display_names
-        title = self.get_title(data_adaptor)
-        about = self.get_about(data_adaptor)
-
-        display_names = dict(engine=data_adaptor.get_name(), dataset=title)
-
-        # library_versions
-        library_versions = {}
-        library_versions.update(data_adaptor.get_library_versions())
-        library_versions["cellxgene"] = cellxgene_display_version
-
-        # links
-        links = {"about-dataset": about}
-
-        # parameters
-        parameters = {
-            "layout": dataset_config.embeddings__names,
-            "max-category-items": dataset_config.presentation__max_categories,
-            "obs_names": server_config.single_dataset__obs_names,
-            "var_names": server_config.single_dataset__var_names,
-            "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
-            "backed": server_config.adaptor__anndata_adaptor__backed,
-            "disable-diffexp": not dataset_config.diffexp__enable,
-            "enable-reembedding": dataset_config.embeddings__enable_reembedding,
-            "annotations": False,
-            "annotations_file": None,
-            "annotations_dir": None,
-            "annotations_cell_ontology_enabled": False,
-            "annotations_cell_ontology_obopath": None,
-            "annotations_cell_ontology_terms": None,
-            "custom_colors": dataset_config.presentation__custom_colors,
-            "diffexp-may-be-slow": False,
-            "about_legal_tos": dataset_config.app__about_legal_tos,
-            "about_legal_privacy": dataset_config.app__about_legal_privacy,
-        }
-
-        # corpora dataset_props
-        # TODO/Note: putting info from the dataset into the /config is not ideal.
-        # However, it is definitely not part of /schema, and we do not have a top-level
-        # route for data properties.  Consider creating one at some point.
-        corpora_props = data_adaptor.get_corpora_props()
-        if corpora_props and "default_embedding" in corpora_props:
-            default_embedding = corpora_props["default_embedding"]
-            if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
-                default_embedding = default_embedding[2:]  # drop X_ prefix
-            if default_embedding in data_adaptor.get_embedding_names():
-                parameters["default_embedding"] = default_embedding
-
-        data_adaptor.update_parameters(parameters)
-        if annotation:
-            annotation.update_parameters(parameters, data_adaptor)
-
-        # gather it all together
-        c = {}
-        config = c["config"] = {}
-        config["features"] = features
-        config["displayNames"] = display_names
-        config["library_versions"] = library_versions
-        config["links"] = links
-        config["parameters"] = parameters
-        config["corpora_props"] = corpora_props
-        config["limits"] = {
-            "column_request_max": server_config.limits__column_request_max,
-            "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
-        }
-
-        if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
-            config["authentication"] = {
-                "requires_client_login": auth.requires_client_login(),
-            }
-            if auth.requires_client_login():
-                config["authentication"].update({
-                    "login": auth.get_login_url(data_adaptor),
-                    "logout": auth.get_logout_url(data_adaptor),
-                })
-
-        return c
-
-    def get_client_userinfo(self, data_adaptor):
-        """
-        Return the userinfo as required by the /userinfo REST route
-        """
-
-        server_config = self.server_config
-        dataset_config = data_adaptor.dataset_config
-        auth = server_config.auth
-
-        # make sure the configuration has been checked.
-        self.check_config()
-
-        if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
-            userinfo = {}
-            userinfo["userinfo"] = {
-                "is_authenticated": auth.is_user_authenticated(),
-                "username": auth.get_user_name(),
-                "user_id": auth.get_user_id(),
-                "email": auth.get_user_email()
-            }
-            return userinfo
-        else:
-            return None
-
-
-class BaseConfig(object):
-    """This class handles the mechanics of updating and checking attributes.
-    Derived classes are expected to store the actual attributes"""
-
-    def __init__(self, app_config, default_config, dictval_cases={}):
-        # reference back to the app_config
-        self.app_config = app_config
-        # the complete set of attribute and their default values (unflattened)
-        self.default_config = default_config
-        # attributes where the value may be a dict (and therefore are not flattened)
-        self.dictval_cases = dictval_cases
-        # used to make sure every attribute value is checked
-        self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()}
-
-    def create_mapping(self, config):
-        """Create a mapping from attribute names to (location in the config tree, value)"""
-        dc = copy.deepcopy(config)
-        mapping = {}
-
-        # special cases where the value could be a dict.
-        # If its value is not None, the entry is added to the mapping, and not included
-        # in the flattening below.
-        for dictval_case in self.dictval_cases:
-            cur = dc
-            for part in dictval_case[:-1]:
-                cur = cur.get(part, {})
-            val = cur.get(dictval_case[-1])
-            if val is not None:
-                key = "__".join(dictval_case)
-                mapping[key] = (dictval_case, val)
-                del cur[dictval_case[-1]]
-
-        flat_config = flatten(dc)
-        for key, value in flat_config.items():
-            # name of the attribute
-            attr = "__".join(key)
-            mapping[attr] = (key, value)
-
-        return mapping
-
-    def check_attr(self, attrname, vtype):
-        val = getattr(self, attrname)
-        if type(vtype) in (list, tuple):
-            if type(val) not in vtype:
-                tnames = ",".join([x.__name__ for x in vtype])
-                raise ConfigurationError(
-                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
-                )
-        else:
-            if type(val) != vtype:
-                raise ConfigurationError(
-                    f"Invalid type for attribute: {attrname}, "
-                    f"expected type {vtype.__name__}, got {type(val).__name__}"
-                )
-
-        self.attr_checked[attrname] = True
-
-    def check_config(self):
-        mapping = self.create_mapping(self.default_config)
-        for key in mapping.keys():
-            if not self.attr_checked[key]:
-                raise ConfigurationError(f"The attr '{key}' has not been checked")
-
-    def update(self, **kw):
-        for key, value in kw.items():
-            if not hasattr(self, key):
-                raise ConfigurationError(f"unknown config parameter {key}.")
-            try:
-                if type(value) == tuple:
-                    # convert tuple values to list values
-                    value = list(value)
-                setattr(self, key, value)
-            except KeyError:
-                raise ConfigurationError(f"Unable to set config parameter {key}.")
-
-            self.attr_checked[key] = False
-
-    def update_from_config(self, config, prefix):
-        mapping = self.create_mapping(config)
-        for attr, (key, value) in mapping.items():
-            if not hasattr(self, attr):
-                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
-            try:
-                setattr(self, attr, value)
-            except KeyError:
-                raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}")
-
-            self.attr_checked[attr] = False
-
-    def changes_from_default(self):
-        """Return all the attribute that are different from the default"""
-        mapping = self.create_mapping(self.default_config)
-        diff = []
-        for attrname, (key, defval) in mapping.items():
-            curval = getattr(self, attrname)
-            if curval != defval:
-                diff.append((attrname, curval, defval))
-        return diff
-
-
-class ServerConfig(BaseConfig):
-    """Manages the config attribute associated with the server."""
-
-    def __init__(self, app_config, default_config):
-        dictval_cases = [
-            ("app", "csp_directives"),
-            ("authentication", "params_oauth", "cookie"),
-            ("authentication", "params_oauth", "jwt_decode_options"),
-            ("adaptor", "cxg_adaptor", "tiledb_ctx"),
-            ("multi_dataset", "dataroot"),
-        ]
-        super().__init__(app_config, default_config, dictval_cases)
-
-        dc = default_config
-        try:
-            self.app__verbose = dc["app"]["verbose"]
-            self.app__debug = dc["app"]["debug"]
-            self.app__host = dc["app"]["host"]
-            self.app__port = dc["app"]["port"]
-            self.app__open_browser = dc["app"]["open_browser"]
-            self.app__force_https = dc["app"]["force_https"]
-            self.app__flask_secret_key = dc["app"]["flask_secret_key"]
-            self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"]
-            self.app__server_timing_headers = dc["app"]["server_timing_headers"]
-            self.app__csp_directives = dc["app"]["csp_directives"]
-            self.app__api_base_url = dc["app"]["api_base_url"]
-            self.app__web_base_url = dc["app"]["web_base_url"]
-
-            self.authentication__type = dc["authentication"]["type"]
-            self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][
-                "oauth_api_base_url"
-            ]
-            self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"]
-            self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"]
-            self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][
-                "jwt_decode_options"]
-            self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"]
-            self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"]
-
-            self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
-            self.multi_dataset__index = dc["multi_dataset"]["index"]
-            self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
-            self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"]
-            self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"]
-
-            self.single_dataset__datapath = dc["single_dataset"]["datapath"]
-            self.single_dataset__obs_names = dc["single_dataset"]["obs_names"]
-            self.single_dataset__var_names = dc["single_dataset"]["var_names"]
-            self.single_dataset__about = dc["single_dataset"]["about"]
-            self.single_dataset__title = dc["single_dataset"]["title"]
-
-            self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"]
-            self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"]
-            self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"]
-
-            self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"]
-
-            self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
-            self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"]
-
-            self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"]
-            self.limits__column_request_max = dc["limits"]["column_request_max"]
-
-        except KeyError as e:
-            raise ConfigurationError(f"Unexpected config: {str(e)}")
-
-        # The matrix data cache manager is created during the complete_config and stored here.
-        self.matrix_data_cache_manager = None
-
-        # The authentication object
-        self.auth = None
-
-    def complete_config(self, context):
-        self.handle_app(context)
-        self.handle_data_source(context)
-        self.handle_authentication(context)
-        self.handle_data_locator(context)
-        self.handle_adaptor(context)  # may depend on data_locator
-        self.handle_single_dataset(context)  # may depend on adaptor
-        self.handle_multi_dataset(context)  # may depend on adaptor
-        self.handle_diffexp(context)
-        self.handle_limits(context)
-
-        self.check_config()
-
-    def handle_app(self, context):
-        self.check_attr("app__verbose", bool)
-        self.check_attr("app__debug", bool)
-        self.check_attr("app__host", str)
-        self.check_attr("app__port", (type(None), int))
-        self.check_attr("app__open_browser", bool)
-        self.check_attr("app__force_https", bool)
-        self.check_attr("app__flask_secret_key", (type(None), str))
-        self.check_attr("app__generate_cache_control_headers", bool)
-        self.check_attr("app__server_timing_headers", bool)
-        self.check_attr("app__csp_directives", (type(None), dict))
-        self.check_attr("app__api_base_url", (type(None), str))
-        self.check_attr("app__web_base_url", (type(None), str))
-
-        if self.app__port:
-            try:
-                if not is_port_available(self.app__host, self.app__port):
-                    raise ConfigurationError(
-                        f"The port selected {self.app__port} is in use, please configure an open port."
-                    )
-            except OverflowError:
-                raise ConfigurationError(f"Invalid port: {self.app__port}")
-        else:
-            try:
-                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
-            except ValueError:
-                raise ConfigurationError(
-                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
-                )
-            try:
-                self.app__port = find_available_port(self.app__host, default_server_port)
-            except OverflowError:
-                raise ConfigurationError(f"Invalid port: {default_server_port}")
-
-        if self.app__debug:
-            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
-            self.app__verbose = True
-            self.app__open_browser = False
-        else:
-            warnings.formatwarning = custom_format_warning
-
-        if not self.app__verbose:
-            sys.tracebacklimit = 0
-
-        # secret key:
-        #   first, from CXG_SECRET_KEY environment variable
-        #   second, from config file
-        self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
-
-        # CSP Directives are a dict of string: list(string) or string: string
-        if self.app__csp_directives is not None:
-            for k, v in self.app__csp_directives.items():
-                if not isinstance(k, str):
-                    raise ConfigurationError("CSP directive names must be a string.")
-                if isinstance(v, list):
-                    for policy in v:
-                        if not isinstance(policy, str):
-                            raise ConfigurationError("CSP directive value must be a string or list of strings.")
-                elif not isinstance(v, str):
-                    raise ConfigurationError("CSP directive value must be a string or list of strings.")
-
-        if self.app__web_base_url is None:
-            self.app__web_base_url = self.app__api_base_url
-
-    def handle_authentication(self, context):
-        self.check_attr("authentication__type", (type(None), str))
-
-        # oauth
-        ptypes = str if self.authentication__type == "oauth" else (type(None), str)
-        self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes)
-        self.check_attr("authentication__params_oauth__client_id", ptypes)
-        self.check_attr("authentication__params_oauth__client_secret", ptypes)
-        self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict))
-        self.check_attr("authentication__params_oauth__session_cookie", bool)
-
-        if self.authentication__params_oauth__session_cookie:
-            self.check_attr("authentication__params_oauth__cookie", (type(None), dict))
-        else:
-            self.check_attr("authentication__params_oauth__cookie", dict)
-        #   secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
-        #   second, from config file
-        self.authentication__params__oauth__client_secret = os.environ.get(
-            "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret)
-
-        self.auth = AuthTypeFactory.create(self.authentication__type, self)
-        if self.auth is None:
-            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
-
-    def handle_data_locator(self, context):
-        self.check_attr("data_locator__s3__region_name", (type(None), bool, str))
-        if self.data_locator__s3__region_name is True:
-            path = self.single_dataset__datapath or self.multi_dataset__dataroot
-            if type(path) == dict:
-                # if multi_dataset__dataroot is a dict, then use the first key
-                # that is in s3.   NOTE:  it is not supported to have dataroots
-                # in different regions.
-                paths = [val.get("dataroot") for val in path.values()]
-                for path in paths:
-                    if path.startswith("s3://"):
-                        break
-            if path.startswith("s3://"):
-                region_name = discover_s3_region_name(path)
-                if region_name is None:
-                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
-            else:
-                region_name = None
-            self.data_locator__s3__region_name = region_name
-
-    def handle_data_source(self, context):
-        self.check_attr("single_dataset__datapath", (str, type(None)))
-        self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
-
-        if self.single_dataset__datapath is None:
-            if self.multi_dataset__dataroot is None:
-                # TODO:  change the error message once dataroot is fully supported
-                raise ConfigurationError("missing datapath")
-            return
-        else:
-            if self.multi_dataset__dataroot is not None:
-                raise ConfigurationError("must supply only one of datapath or dataroot")
-
-    def handle_single_dataset(self, context):
-        self.check_attr("single_dataset__datapath", (str, type(None)))
-        self.check_attr("single_dataset__title", (str, type(None)))
-        self.check_attr("single_dataset__about", (str, type(None)))
-        self.check_attr("single_dataset__obs_names", (str, type(None)))
-        self.check_attr("single_dataset__var_names", (str, type(None)))
-
-        if self.single_dataset__datapath is None:
-            return
-
-        # create the matrix data cache manager:
-        if self.matrix_data_cache_manager is None:
-            self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
-
-        # preload this data set
-        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
-        try:
-            matrix_data_loader.pre_load_validation()
-        except DatasetAccessError as e:
-            raise ConfigurationError(str(e))
-
-        file_size = matrix_data_loader.file_size()
-        file_basename = basename(self.single_dataset__datapath)
-        if file_size > BIG_FILE_SIZE_THRESHOLD:
-            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
-        else:
-            context["messagefn"](f"Loading data from {file_basename}.")
-
-        if self.single_dataset__about:
-
-            def url_check(url):
-                try:
-                    result = urlparse(url)
-                    if all([result.scheme, result.netloc]):
-                        return True
-                    else:
-                        return False
-                except ValueError:
-                    return False
-
-            if not url_check(self.single_dataset__about):
-                raise ConfigurationError(
-                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
-                )
-
-    def handle_multi_dataset(self, context):
-        self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
-        self.check_attr("multi_dataset__index", (type(None), bool, str))
-        self.check_attr("multi_dataset__allowed_matrix_types", list)
-        self.check_attr("multi_dataset__matrix_cache__max_datasets", int)
-        self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float))
-
-        if self.multi_dataset__dataroot is None:
-            return
-
-        if type(self.multi_dataset__dataroot) == str:
-            default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
-            self.multi_dataset__dataroot = dict(d=default_dict)
-
-        for tag, dataroot_dict in self.multi_dataset__dataroot.items():
-            if "base_url" not in dataroot_dict:
-                raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
-            if "dataroot" not in dataroot_dict:
-                raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
-
-            base_url = dataroot_dict["base_url"]
-
-            # sanity check for well formed base urls
-            bad = False
-            if type(base_url) != str:
-                bad = True
-            elif os.path.normpath(base_url) != base_url:
-                bad = True
-            else:
-                base_url_parts = base_url.split("/")
-                if [quote_plus(part) for part in base_url_parts] != base_url_parts:
-                    bad = True
-                if ".." in base_url_parts:
-                    bad = True
-            if bad:
-                raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
-
-        # verify all the base_urls are unique
-        base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
-        if len(base_urls) > len(set(base_urls)):
-            raise ConfigurationError("error in multi_dataset__dataroot:  base_urls must be unique")
-
-        # error checking
-        for mtype in self.multi_dataset__allowed_matrix_types:
-            try:
-                MatrixDataType(mtype)
-            except ValueError:
-                raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
-
-        # create the matrix data cache manager:
-        if self.matrix_data_cache_manager is None:
-            self.matrix_data_cache_manager = MatrixDataCacheManager(
-                max_cached=self.multi_dataset__matrix_cache__max_datasets,
-                timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
-            )
-
-    def handle_diffexp(self, context):
-        self.check_attr("diffexp__alg_cxg__max_workers", (str, int))
-        self.check_attr("diffexp__alg_cxg__cpu_multiplier", int)
-        self.check_attr("diffexp__alg_cxg__target_workunit", int)
-
-        max_workers = self.diffexp__alg_cxg__max_workers
-        cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
-        cpu_count = os.cpu_count()
-        max_workers = min(max_workers, cpu_multiplier * cpu_count)
-        diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
-
-    def handle_adaptor(self, context):
-        # cxg
-        self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict)
-        regionkey = "vfs.s3.region"
-        if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
-            if type(self.data_locator__s3__region_name) == str:
-                self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
-
-        from server.data_cxg.cxg_adaptor import CxgAdaptor
-
-        CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
-
-        # anndata
-        self.check_attr("adaptor__anndata_adaptor__backed", bool)
-
-    def handle_limits(self, context):
-        self.check_attr("limits__diffexp_cellcount_max", (type(None), int))
-        self.check_attr("limits__column_request_max", (type(None), int))
-
-    def exceeds_limit(self, limit_name, value):
-        limit_value = getattr(self, "limits__" + limit_name, None)
-        if limit_value is None:  # disabled
-            return False
-        return value > limit_value
-
-    def get_api_base_url(self):
-        if self.app__api_base_url == "local":
-            return f"http://{self.app__host}:{self.app__port}"
-        if self.app__api_base_url and self.app__api_base_url.endswith("/"):
-            return self.app__api_base_url[:-1]
-        return self.app__api_base_url
-
-    def get_web_base_url(self):
-        if self.app__web_base_url == "local":
-            return f"http://{self.app__host}:{self.app__port}"
-        if self.app__web_base_url is None:
-            return self.get_api_base_url()
-        if self.app__web_base_url.endswith("/"):
-            return self.app__web_base_url[:-1]
-        return self.api__web_base_url
-
-
-class DatasetConfig(BaseConfig):
-    """Manages the config attribute associated with a dataset."""
-
-    def __init__(self, tag, app_config, default_config):
-        super().__init__(app_config, default_config)
-        self.tag = tag
-        dc = default_config
-        try:
-            self.app__scripts = dc["app"]["scripts"]
-            self.app__inline_scripts = dc["app"]["inline_scripts"]
-            self.app__about_legal_tos = dc["app"]["about_legal_tos"]
-            self.app__about_legal_privacy = dc["app"]["about_legal_privacy"]
-            self.app__authentication_enable = dc["app"]["authentication_enable"]
-
-            self.presentation__max_categories = dc["presentation"]["max_categories"]
-            self.presentation__custom_colors = dc["presentation"]["custom_colors"]
-
-            self.user_annotations__enable = dc["user_annotations"]["enable"]
-            self.user_annotations__type = dc["user_annotations"]["type"]
-            self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"]
-            self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"]
-            self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
-            self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
-            self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
-            self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
-                dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ]  # noqa E501
-
-            self.embeddings__names = dc["embeddings"]["names"]
-            self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
-
-            self.diffexp__enable = dc["diffexp"]["enable"]
-            self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"]
-            self.diffexp__top_n = dc["diffexp"]["top_n"]
-
-        except KeyError as e:
-            raise ConfigurationError(f"Unexpected config: {str(e)}")
-
-        # The annotation object is created during complete_config and stored here.
-        self.user_annotations = None
-
-    def complete_config(self, context):
-        self.handle_app(context)
-        self.handle_presentation(context)
-        self.handle_user_annotations(context)
-        self.handle_embeddings(context)
-        self.handle_diffexp(context)
-
-    def handle_app(self, context):
-        self.check_attr("app__scripts", list)
-        self.check_attr("app__inline_scripts", list)
-        self.check_attr("app__about_legal_tos", (type(None), str))
-        self.check_attr("app__about_legal_privacy", (type(None), str))
-        self.check_attr("app__authentication_enable", bool)
-
-        # scripts can be string (filename) or dict (attributes).   Convert string to dict.
-        scripts = []
-        for s in self.app__scripts:
-            if isinstance(s, str):
-                scripts.append({"src": s})
-            elif isinstance(s, dict) and isinstance(s["src"], str):
-                scripts.append(s)
-            else:
-                raise ConfigurationError("Scripts must be string or dict")
-        self.app__scripts = scripts
-
-    def handle_presentation(self, context):
-        self.check_attr("presentation__max_categories", int)
-        self.check_attr("presentation__custom_colors", bool)
-
-    def handle_user_annotations(self, context):
-        self.check_attr("user_annotations__enable", bool)
-        self.check_attr("user_annotations__type", str)
-        self.check_attr("user_annotations__local_file_csv__directory", (type(None), str))
-        self.check_attr("user_annotations__local_file_csv__file", (type(None), str))
-        self.check_attr("user_annotations__ontology__enable", bool)
-        self.check_attr("user_annotations__ontology__obo_location", (type(None), str))
-        self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str))
-        self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str))
-
-        if self.user_annotations__enable:
-            server_config = self.app_config.server_config
-            if not self.app__authentication_enable:
-                raise ConfigurationError("user annotations requires authentication to be enabled")
-            if not server_config.auth.is_valid_authentication_type():
-                auth_type = server_config.authentication__type
-                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
-
-            # TODO, replace this with a factory pattern once we have more than one way
-            # to do annotations.  currently only local_file_csv
-            if self.user_annotations__type == "local_file_csv":
-                dirname = self.user_annotations__local_file_csv__directory
-                filename = self.user_annotations__local_file_csv__file
-
-                if filename is not None and dirname is not None:
-                    raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
-
-                if filename is not None:
-                    lf_name, lf_ext = splitext(filename)
-                    if lf_ext and lf_ext != ".csv":
-                        raise ConfigurationError(f"annotation file type must be .csv: {filename}")
-
-                if dirname is not None and not isdir(dirname):
-                    try:
-                        os.mkdir(dirname)
-                    except OSError:
-                        raise ConfigurationError("Unable to create directory specified by --annotations-dir")
-
-                self.user_annotations = AnnotationsLocalFile(dirname, filename)
-
-                # if the user has specified a fixed label file, go ahead and validate it
-                # so that we can remove errors early in the process.
-                server_config = self.app_config.server_config
-                if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
-                    with server_config.matrix_data_cache_manager.data_adaptor(
-                            self.tag, server_config.single_dataset__datapath, self.app_config
-                    ) as data_adaptor:
-                        data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
-
-                if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
-                    try:
-                        self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
-                    except OntologyLoadFailure as e:
-                        raise ConfigurationError("Unable to load ontology terms\n" + str(e))
-            elif self.user_annotations__type == "hosted_tiledb_array":
-                self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str)
-                self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str)
-                self.user_annotations = AnnotationsHostedTileDB(
-                    directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
-                    db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
-                )
-            else:
-                raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
-        else:
-            if self.user_annotations__type == "local_file_csv":
-                dirname = self.user_annotations__local_file_csv__directory
-                filename = self.user_annotations__local_file_csv__file
-                if filename is not None:
-                    context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
-                if dirname is not None:
-                    context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
-
-            if self.user_annotations__ontology__enable:
-                context["messagefn"](
-                    "Warning: --experimental-annotations-ontology" " ignored as annotations are disabled."
-                )
-            if self.user_annotations__ontology__obo_location is not None:
-                context["messagefn"](
-                    "Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled."
-                )
-
-    def handle_embeddings(self, context):
-        self.check_attr("embeddings__names", list)
-        self.check_attr("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                matrix_data_loader = MatrixDataLoader(
-                    server_config.single_dataset__datapath, app_config=self.app_config
-                )
-                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
-                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                server.compute.scanpy.get_scanpy_module()
-            except NotImplementedError:
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
-
-    def handle_diffexp(self, context):
-        self.check_attr("diffexp__enable", bool)
-        self.check_attr("diffexp__lfc_cutoff", float)
-        self.check_attr("diffexp__top_n", int)
-
-        server_config = self.app_config.server_config
-        if server_config.single_dataset__datapath:
-            with server_config.matrix_data_cache_manager.data_adaptor(
-                    self.tag, server_config.single_dataset__datapath, self.app_config
-            ) as data_adaptor:
-                if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
-                    context["messagefn"](
-                        "CAUTION: due to the size of your dataset, "
-                        "running differential expression may take longer or fail."
-                    )

server/common/aws_secret_utils.py

@@ -1,72 +1,11 @@
 import logging
-import os
-import sys
 
 import boto3
 from flask import json
 
-from server.common.data_locator import discover_s3_region_name
 from server.common.errors import SecretKeyRetrievalError
 
 
-def handle_config_from_secret(app_config):
-    """Update configuration from the secret manager"""
-    secret_name = os.getenv("CXG_AWS_SECRET_NAME")
-    if not secret_name:
-        return
-
-    # need to find the secret manager region.
-    #  1. from CXG_AWS_SECRET_REGION_NAME
-    #  2. discover from dataroot location (if on s3)
-    #  3. discover from config file location (if on s3)
-    secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
-    if secret_region_name is None:
-        secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
-        if not secret_region_name:
-            from server.eb.app import config_file
-            secret_region_name = discover_s3_region_name(config_file)
-    if not secret_region_name:
-        logging.error("Could not determine the AWS Secret Manager region")
-        sys.exit(1)
-
-    secrets = get_secret_key(secret_region_name, secret_name)
-
-    if not secrets:
-        return
-
-    server_attrs = (
-        ("flask_secret_key", "app__flask_secret_key"),
-        ("oauth_client_secret", "authentication__params_oauth__client_secret"),
-    )
-    default_dataset_attrs = (
-        ("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),
-    )
-
-    # update server configuration attributes
-    for key, attr in server_attrs:
-        cur_val = getattr(app_config.server_config, attr)
-        if cur_val:
-            continue
-
-        # replace the attr with the secret if it is not set
-        val = secrets.get(key)
-        if val:
-            logging.info(f"set {attr} from secret")
-            app_config.update_server_config(**{attr : val})
-
-    # update default dataset configuration attributes
-    for key, attr in default_dataset_attrs:
-        cur_val = getattr(app_config.default_dataset_config, attr)
-        if cur_val:
-            continue
-
-        # replace the attr with the secret if it is not set
-        val = secrets.get(key)
-        if val:
-            logging.info(f"set {attr} from secret")
-            app_config.update_default_dataset_config(**{attr : val})
-
-
 def get_secret_key(region_name, secret_name):
     session = boto3.session.Session()
     client = session.client(service_name="secretsmanager", region_name=region_name)

server/common/config/__init__.py

@@ -0,0 +1,66 @@
+import logging
+import os
+import sys
+
+from server.common.aws_secret_utils import get_secret_key
+from server.common.data_locator import discover_s3_region_name
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB
+
+
+def handle_config_from_secret(app_config):
+    """Update configuration from the secret manager"""
+    secret_name = os.getenv("CXG_AWS_SECRET_NAME")
+    if not secret_name:
+        return
+
+    # need to find the secret manager region.
+    #  1. from CXG_AWS_SECRET_REGION_NAME
+    #  2. discover from dataroot location (if on s3)
+    #  3. discover from config file location (if on s3)
+    secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
+    if secret_region_name is None:
+        secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
+        if not secret_region_name:
+            from server.eb.app import config_file
+
+            secret_region_name = discover_s3_region_name(config_file)
+    if not secret_region_name:
+        logging.error("Could not determine the AWS Secret Manager region")
+        sys.exit(1)
+
+    secrets = get_secret_key(secret_region_name, secret_name)
+
+    if not secrets:
+        return
+
+    server_attrs = (
+        ("flask_secret_key", "app__flask_secret_key"),
+        ("oauth_client_secret", "authentication__params_oauth__client_secret"),
+    )
+    default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),)
+
+    # update server configuration attributes
+    for key, attr in server_attrs:
+        cur_val = getattr(app_config.server_config, attr)
+        if cur_val:
+            continue
+
+        # replace the attr with the secret if it is not set
+        val = secrets.get(key)
+        if val:
+            logging.info(f"set {attr} from secret")
+            app_config.update_server_config(**{attr: val})
+
+    # update default dataset configuration attributes
+    for key, attr in default_dataset_attrs:
+        cur_val = getattr(app_config.default_dataset_config, attr)
+        if cur_val:
+            continue
+
+        # replace the attr with the secret if it is not set
+        val = secrets.get(key)
+        if val:
+            logging.info(f"set {attr} from secret")
+            app_config.update_default_dataset_config(**{attr: val})

server/common/config/app_config.py

@@ -0,0 +1,183 @@
+import yaml
+from flatten_dict import unflatten
+
+from server.default_config import get_default_config
+from server.common.config.dataset_config import DatasetConfig
+from server.common.config.server_config import ServerConfig
+from server.common.errors import ConfigurationError
+
+
+class AppConfig(object):
+    """
+    AppConfig stores all the configuration for cellxgene.
+    AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
+    The server_config contains attributes that refer to the server process as a whole.
+    The default_dataset_config refers to attributes that are associated with the features and
+    presentations of a dataset.
+    The dataset config attributes can be overridden depending on the url by which the
+    dataset was accessed.  These are stored in dataroot_config.
+    AppConfig has methods to initialize, modify, and access the configuration.
+    """
+
+    def __init__(self):
+
+        # the default configuration (see default_config.py)
+        # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
+        #  defaults within the config class?
+        self.default_config = get_default_config()
+        # the server configuration
+        self.server_config = ServerConfig(self, self.default_config["server"])
+        # the dataset config, unless overridden by an entry in dataroot_config
+        self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
+        # a dictionary of keys to DatasetConfig objects.  Each key must exist in the multi_dataset__dataroot
+        # attribute of the server_config. The default dataset config will apply to all datasets unless a different set
+        # of config vars was passed for a specific dataset under the multidataset config. For example:
+        """
+        per_dataset_config:
+            d1:
+               user_annotations:
+                   enable:  false
+            d2:
+               user_annotations:
+                   enable:  true
+         """
+        #   dataroot config
+        self.dataroot_config = {}
+
+        # Set to true when config_completed is called
+        self.is_completed = False
+
+    def get_dataset_config(self, dataroot_key):
+        if self.server_config.single_dataset__datapath:
+            return self.default_dataset_config
+        else:
+            return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
+
+    def check_config(self):
+        """Verify all the attributes in the config have been type checked"""
+        if not self.is_completed:
+            raise ConfigurationError("The configuration has not been completed")
+        self.server_config.check_config()
+        self.default_dataset_config.check_config()
+        for dataset_config in self.dataroot_config.values():
+            dataset_config.check_config()
+
+    def update_server_config(self, **kw):
+        self.server_config.update(**kw)
+        self.is_complete = False
+
+    def update_default_dataset_config(self, **kw):
+        self.default_dataset_config.update(**kw)
+        # update all the other dataset configs, if any
+        for value in self.dataroot_config.values():
+            value.update(**kw)
+        self.is_complete = False
+
+    def update_from_config_file(self, config_file):
+        try:
+            with open(config_file) as yml_file:
+                config = yaml.safe_load(yml_file)
+        except yaml.YAMLError as e:
+            raise ConfigurationError(f"The specified config file contained an error: {e}")
+        except OSError as e:
+            raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
+
+        if config.get("server"):
+            self.server_config.update_from_config(config["server"], "server")
+        if config.get("dataset"):
+            self.default_dataset_config.update_from_config(config["dataset"], "dataset")
+
+        per_dataset_config = config.get("per_dataset_config", {})
+        for key, dataroot_config in per_dataset_config.items():
+            # first create and initialize the dataroot with the default config
+            self.add_dataroot_config(key, **config["dataset"])
+            # then apply the per dataset configuration
+            self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
+
+        self.is_complete = False
+
+    def write_config(self, config_file):
+        """output the config to a yaml file"""
+        server = self.server_config.create_mapping(self.server_config.default_config)
+        dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        config = dict(server={}, dataset={})
+        for attrname in server.keys():
+            config["server__" + attrname] = getattr(self.server_config, attrname)
+        for attrname in dataset.keys():
+            config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
+        if self.dataroot_config:
+            config["per_dataset_config"] = {}
+        for dataroot_tag, dataroot_config in self.dataroot_config.items():
+            dataset = dataroot_config.create_mapping(dataroot_config.default_config)
+            for attrname in dataset.keys():
+                config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
+
+        config = unflatten(config, splitter=lambda key: key.split("__"))
+        yaml.dump(config, open(config_file, "w"))
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        diff_server = self.server_config.changes_from_default()
+        diff_dataset = self.default_dataset_config.changes_from_default()
+        diff = dict(server=diff_server, dataset=diff_dataset)
+        return diff
+
+    def add_dataroot_config(self, dataroot_tag, **kw):
+        """Create a new dataset config object based on the default dataset config, and kw parameters"""
+        if dataroot_tag in self.dataroot_config:
+            raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
+        if type(self.server_config.multi_dataset__dataroot) != dict:
+            raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
+        if dataroot_tag not in self.server_config.multi_dataset__dataroot:
+            raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
+
+        self.is_completed = False
+        self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
+        flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        config = {key: value[1] for key, value in flat_config.items()}
+        self.dataroot_config[dataroot_tag].update(**config)
+        self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
+
+    def complete_config(self, messagefn=None):
+        """The configure options are checked, and any additional setup based on the config
+        parameters is done"""
+
+        if messagefn is None:
+
+            def noop(message):
+                pass
+
+            messagefn = noop
+
+        # TODO: to give better error messages we can add a mapping between where each config
+        # attribute originated (e.g. command line argument or config file), then in the error
+        # messages we can give correct context for attributes with bad value.
+        context = dict(messagefn=messagefn)
+
+        self.server_config.complete_config(context)
+        self.default_dataset_config.complete_config(context)
+        for dataroot_config in self.dataroot_config.values():
+            dataroot_config.complete_config(context)
+
+        self.is_completed = True
+        self.check_config()
+
+    def get_matrix_data_cache_manager(self):
+        return self.server_config.matrix_data_cache_manager
+
+    def is_multi_dataset(self):
+        return self.server_config.multi_dataset__dataroot is not None
+
+    def get_title(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__title
+            if self.server_config.single_dataset__title
+            else data_adaptor.get_title()
+        )
+
+    def get_about(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__about
+            if self.server_config.single_dataset__about
+            else data_adaptor.get_about()
+        )

server/common/config/base_config.py

@@ -0,0 +1,113 @@
+import copy
+
+from flatten_dict import flatten
+from server.common.errors import ConfigurationError
+
+
+class BaseConfig(object):
+    """
+    This class handles the mechanics of updating and checking attributes.
+    Derived classes are expected to store the actual attributes
+    Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
+    """
+
+    def __init__(self, app_config, default_config, dictval_cases={}):
+        # reference back to the app_config
+        self.app_config = app_config
+        # the complete set of attributes and their default values (unflattened)
+        self.default_config = default_config
+        # attributes where the value may be a dict (and therefore are not flattened)
+        self.dictval_cases = dictval_cases
+        # used to make sure every attribute value is checked
+        self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
+
+    def create_mapping(self, config):
+        """
+        Create a dictionary where the keys are the name of attributes (using double underscore convention)
+        For example: authentication__type
+
+        The values are a tuple,
+        - the first item of the tuple is a tuple of path elements (location in config 'tree')
+        - the second item is the value of the config parameter
+
+        For example: (('authentication', 'type'), 'session'))
+        """
+        config_copy = copy.deepcopy(config)
+        mapping = {}
+
+        # special cases where the value could be a dict.
+        # If its value is not None, the entry is added to the mapping, and not included
+        # in the flattening below.
+        for dictval_case in self.dictval_cases:
+            cur = config_copy
+            for part in dictval_case[:-1]:
+                cur = cur.get(part, {})
+            val = cur.get(dictval_case[-1])
+            if val is not None:
+                key = "__".join(dictval_case)
+                mapping[key] = (dictval_case, val)
+                del cur[dictval_case[-1]]
+
+        flat_config = flatten(config_copy)
+        for key, value in flat_config.items():
+            # name of the attribute
+            attr = "__".join(key)
+            mapping[attr] = (key, value)
+
+        return mapping
+
+    def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
+        val = getattr(self, attrname)
+        if type(vtype) in (list, tuple):
+            if type(val) not in vtype:
+                tnames = ",".join([x.__name__ for x in vtype])
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
+                )
+        else:
+            if type(val) != vtype:
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, "
+                    f"expected type {vtype.__name__}, got {type(val).__name__}"
+                )
+
+        self.attr_checked[attrname] = True
+
+    def check_config(self):
+        mapping = self.create_mapping(self.default_config)
+        for key in mapping.keys():
+            if not self.attr_checked[key]:
+                raise ConfigurationError(f"The attr '{key}' has not been checked")
+
+    def update(self, **kw):
+        for key, value in kw.items():
+            if not hasattr(self, key):
+                raise ConfigurationError(f"unknown config parameter {key}.")
+            try:
+                if type(value) == tuple:
+                    # convert tuple values to list values
+                    value = list(value)
+                setattr(self, key, value)
+            except KeyError:
+                raise ConfigurationError(f"Unable to set config parameter {key}.")
+
+            self.attr_checked[key] = False
+
+    def update_from_config(self, config, prefix):
+        mapping = self.create_mapping(config)
+        for attr, (key, value) in mapping.items():
+            if not hasattr(self, attr):
+                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
+            setattr(self, attr, value)
+
+            self.attr_checked[attr] = False
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        mapping = self.create_mapping(self.default_config)
+        diff = []
+        for attrname, (key, defval) in mapping.items():
+            curval = getattr(self, attrname)
+            if curval != defval:
+                diff.append((attrname, curval, defval))
+        return diff

server/common/config/client_config.py

@@ -0,0 +1,125 @@
+from server import display_version as cellxgene_display_version
+
+
+def get_client_config(app_config, data_adaptor):
+    """
+    Return the configuration as required by the /config REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    annotation = dataset_config.user_annotations
+    auth = server_config.auth
+
+    # FIXME The current set of config is not consistently presented:
+    # we have camalCase, hyphen-text, and underscore_text
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    # features
+    features = [f.todict() for f in data_adaptor.get_features(annotation)]
+
+    # display_names
+    title = app_config.get_title(data_adaptor)
+    about = app_config.get_about(data_adaptor)
+
+    display_names = dict(engine=data_adaptor.get_name(), dataset=title)
+
+    # library_versions
+    library_versions = {}
+    library_versions.update(data_adaptor.get_library_versions())
+    library_versions["cellxgene"] = cellxgene_display_version
+
+    # links
+    links = {"about-dataset": about}
+
+    # parameters
+    parameters = {
+        "layout": dataset_config.embeddings__names,
+        "max-category-items": dataset_config.presentation__max_categories,
+        "obs_names": server_config.single_dataset__obs_names,
+        "var_names": server_config.single_dataset__var_names,
+        "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
+        "backed": server_config.adaptor__anndata_adaptor__backed,
+        "disable-diffexp": not dataset_config.diffexp__enable,
+        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
+        "annotations": False,
+        "annotations_file": None,
+        "annotations_dir": None,
+        "annotations_cell_ontology_enabled": False,
+        "annotations_cell_ontology_obopath": None,
+        "annotations_cell_ontology_terms": None,
+        "custom_colors": dataset_config.presentation__custom_colors,
+        "diffexp-may-be-slow": False,
+        "about_legal_tos": dataset_config.app__about_legal_tos,
+        "about_legal_privacy": dataset_config.app__about_legal_privacy,
+    }
+
+    # corpora dataset_props
+    # TODO/Note: putting info from the dataset into the /config is not ideal.
+    # However, it is definitely not part of /schema, and we do not have a top-level
+    # route for data properties.  Consider creating one at some point.
+    corpora_props = data_adaptor.get_corpora_props()
+    if corpora_props and "default_embedding" in corpora_props:
+        default_embedding = corpora_props["default_embedding"]
+        if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
+            default_embedding = default_embedding[2:]  # drop X_ prefix
+        if default_embedding in data_adaptor.get_embedding_names():
+            parameters["default_embedding"] = default_embedding
+
+    data_adaptor.update_parameters(parameters)
+    if annotation:
+        annotation.update_parameters(parameters, data_adaptor)
+
+    # gather it all together
+    client_config = {}
+    config = client_config["config"] = {}
+    config["features"] = features
+    config["displayNames"] = display_names
+    config["library_versions"] = library_versions
+    config["links"] = links
+    config["parameters"] = parameters
+    config["corpora_props"] = corpora_props
+    config["limits"] = {
+        "column_request_max": server_config.limits__column_request_max,
+        "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
+    }
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        config["authentication"] = {
+            "requires_client_login": auth.requires_client_login(),
+        }
+        if auth.requires_client_login():
+            config["authentication"].update(
+                {
+                    # Todo why are these stored on the data_adaptor?
+                    "login": auth.get_login_url(data_adaptor),
+                    "logout": auth.get_logout_url(data_adaptor),
+                }
+            )
+
+    return client_config
+
+
+def get_client_userinfo(app_config, data_adaptor):
+    """
+    Return the userinfo as required by the /userinfo REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    auth = server_config.auth
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        userinfo = {}
+        userinfo["userinfo"] = {
+            "is_authenticated": auth.is_user_authenticated(),
+            "username": auth.get_user_name(),
+            "user_id": auth.get_user_id(),
+            "email": auth.get_user_email(),
+        }
+        return userinfo

server/common/config/dataset_config.py

@@ -0,0 +1,234 @@
+import os
+from os.path import splitext, isdir
+
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.base_config import BaseConfig
+from server.common.errors import ConfigurationError, OntologyLoadFailure
+from server.compute.scanpy import get_scanpy_module
+from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from server.db.db_utils import DbUtils
+
+
+class DatasetConfig(BaseConfig):
+    """Manages the config attribute associated with a dataset."""
+
+    def __init__(self, tag, app_config, default_config):
+        super().__init__(app_config, default_config)
+        self.tag = tag
+        try:
+            self.app__scripts = default_config["app"]["scripts"]
+            self.app__inline_scripts = default_config["app"]["inline_scripts"]
+            self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
+            self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
+            self.app__authentication_enable = default_config["app"]["authentication_enable"]
+
+            self.presentation__max_categories = default_config["presentation"]["max_categories"]
+            self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
+
+            self.user_annotations__enable = default_config["user_annotations"]["enable"]
+            self.user_annotations__type = default_config["user_annotations"]["type"]
+            self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
+                "directory"
+            ]  # noqa E501
+            self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
+            self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
+            self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
+                "obo_location"
+            ]  # noqa E501
+            self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ][
+                "db_uri"
+            ]  # noqa E501
+            self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ][
+                "hosted_file_directory"
+            ]  # noqa E501
+
+            self.embeddings__names = default_config["embeddings"]["names"]
+            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
+
+            self.diffexp__enable = default_config["diffexp"]["enable"]
+            self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
+            self.diffexp__top_n = default_config["diffexp"]["top_n"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The annotation object is created during complete_config and stored here.
+        self.user_annotations = None
+
+    def complete_config(self, context):
+        self.handle_app()
+        self.handle_presentation()
+        self.handle_user_annotations(context)
+        self.handle_embeddings()
+        self.handle_diffexp(context)
+
+    def handle_app(self):
+        self.validate_correct_type_of_configuration_attribute("app__scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
+
+        # scripts can be string (filename) or dict (attributes). Convert string to dict.
+        scripts = []
+        for script in self.app__scripts:
+            try:
+                if isinstance(script, str):
+                    scripts.append({"src": script})
+                elif isinstance(script, dict) and isinstance(script["src"], str):
+                    scripts.append(script)
+                else:
+                    raise Exception
+            except Exception as e:
+                raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
+
+        self.app__scripts = scripts
+
+    def handle_presentation(self):
+        self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
+        self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
+
+    def handle_user_annotations(self, context):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__directory", (type(None), str)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__file", (type(None), str)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__ontology__obo_location", (type(None), str)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
+        )  # noqa E501
+        if self.user_annotations__enable:
+            server_config = self.app_config.server_config
+            if not self.app__authentication_enable:
+                raise ConfigurationError("user annotations requires authentication to be enabled")
+            if not server_config.auth.is_valid_authentication_type():
+                auth_type = server_config.authentication__type
+                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
+
+            if self.user_annotations__type == "local_file_csv":
+                self.handle_local_file_csv_annotations()
+            elif self.user_annotations__type == "hosted_tiledb_array":
+                self.handle_hosted_tiledb_annotations()
+            else:
+                raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
+            if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
+                try:
+                    self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
+                except OntologyLoadFailure as e:
+                    raise ConfigurationError("Unable to load ontology terms\n" + str(e))
+        else:
+            self.check_annotation_config_vars_not_set(context)
+
+    def handle_local_file_csv_annotations(self):
+        dirname = self.user_annotations__local_file_csv__directory
+        filename = self.user_annotations__local_file_csv__file
+        if filename is not None and dirname is not None:
+            raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
+
+        if filename is not None:
+            lf_name, lf_ext = splitext(filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"annotation file type must be .csv: {filename}")
+
+        if dirname is not None and not isdir(dirname):
+            try:
+                os.mkdir(dirname)
+            except OSError:
+                raise ConfigurationError("Unable to create directory specified by --annotations-dir")
+
+        self.user_annotations = AnnotationsLocalFile(dirname, filename)
+
+        # if the user has specified a fixed label file, go ahead and validate it
+        # so that we can remove errors early in the process.
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+
+    def handle_hosted_tiledb_annotations(self):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", str
+        )  # noqa E501
+        self.user_annotations = AnnotationsHostedTileDB(
+            directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
+            db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
+        )
+
+    def check_annotation_config_vars_not_set(self, context):
+        if self.user_annotations__type is not None:
+            dirname = self.user_annotations__local_file_csv__directory
+            filename = self.user_annotations__local_file_csv__file
+            db_uri = self.user_annotations__hosted_tiledb_array__db_uri
+            hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
+            if filename is not None:
+                context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
+            if dirname is not None:
+                context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
+            if db_uri is not None:
+                context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
+            if hosted_file_dirname is not None:
+                context["messagefn"](
+                    "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
+                )
+
+        if self.user_annotations__ontology__enable:
+            context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
+        if self.user_annotations__ontology__obo_location is not None:
+            context["messagefn"](
+                "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
+            )
+
+    def handle_embeddings(self):
+        self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
+        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
+
+        server_config = self.app_config.server_config
+        if self.embeddings__enable_reembedding:
+            if server_config.single_dataset__datapath:
+                matrix_data_loader = MatrixDataLoader(
+                    server_config.single_dataset__datapath, app_config=self.app_config
+                )
+                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
+                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
+                if server_config.adaptor__anndata_adaptor__backed:
+                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
+
+            try:
+                get_scanpy_module()
+            except NotImplementedError:
+                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
+                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
+
+    def handle_diffexp(self, context):
+        self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
+        self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
+
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
+                    context["messagefn"](
+                        "CAUTION: due to the size of your dataset, "
+                        "running differential expression may take longer or fail."
+                    )

server/common/config/server_config.py

@@ -0,0 +1,390 @@
+import os
+import sys
+import warnings
+from os.path import basename
+from urllib.parse import urlparse, quote_plus
+
+from server.auth.auth import AuthTypeFactory
+from server.common.config.base_config import BaseConfig
+from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from server.common.errors import ConfigurationError, DatasetAccessError
+from server.common.data_locator import discover_s3_region_name
+from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from server.compute import diffexp_cxg as diffexp_tiledb
+from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+
+
+class ServerConfig(BaseConfig):
+    """Manages the config attribute associated with the server."""
+
+    def __init__(self, app_config, default_config):
+        dictval_cases = [
+            ("app", "csp_directives"),
+            ("authentication", "params_oauth", "cookie"),
+            ("authentication", "params_oauth", "jwt_decode_options"),
+            ("adaptor", "cxg_adaptor", "tiledb_ctx"),
+            ("multi_dataset", "dataroot"),
+        ]
+        super().__init__(app_config, default_config, dictval_cases)
+
+        try:
+            self.app__verbose = default_config["app"]["verbose"]
+            self.app__debug = default_config["app"]["debug"]
+            self.app__host = default_config["app"]["host"]
+            self.app__port = default_config["app"]["port"]
+            self.app__open_browser = default_config["app"]["open_browser"]
+            self.app__force_https = default_config["app"]["force_https"]
+            self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
+            self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
+            self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
+            self.app__csp_directives = default_config["app"]["csp_directives"]
+            self.app__api_base_url = default_config["app"]["api_base_url"]
+            self.app__web_base_url = default_config["app"]["web_base_url"]
+
+            self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
+                "oauth_api_base_url"
+            ]  # noqa E501
+            self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
+            self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
+                "client_secret"
+            ]  # noqa E501
+            self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
+                "jwt_decode_options"
+            ]  # noqa E501
+            self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
+                "session_cookie"
+            ]  # noqa E501
+            self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
+
+            self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
+            self.multi_dataset__index = default_config["multi_dataset"]["index"]
+            self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
+            self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
+                "max_datasets"
+            ]  # noqa E501
+            self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
+                "timelimit_s"
+            ]  # noqa E501
+
+            self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
+            self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
+            self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
+            self.single_dataset__about = default_config["single_dataset"]["about"]
+            self.single_dataset__title = default_config["single_dataset"]["title"]
+
+            self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
+            self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
+            self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
+
+            self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
+
+            self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
+            self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
+
+            self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
+            self.limits__column_request_max = default_config["limits"]["column_request_max"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The matrix data cache manager is created during the complete_config and stored here.
+        self.matrix_data_cache_manager = None
+
+        # The authentication object
+        self.auth = None
+
+    def complete_config(self, context):
+        self.handle_app(context)
+        self.handle_data_source()
+        self.handle_authentication()
+        self.handle_data_locator()
+        self.handle_adaptor()  # may depend on data_locator
+        self.handle_single_dataset(context)  # may depend on adaptor
+        self.handle_multi_dataset()  # may depend on adaptor
+        self.handle_diffexp()
+        self.handle_limits()
+
+        self.check_config()
+
+    def handle_app(self, context):
+        self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
+        self.validate_correct_type_of_configuration_attribute("app__debug", bool)
+        self.validate_correct_type_of_configuration_attribute("app__host", str)
+        self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
+        self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
+        self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
+        self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
+
+        if self.app__port:
+            try:
+                if not is_port_available(self.app__host, self.app__port):
+                    raise ConfigurationError(
+                        f"The port selected {self.app__port} is in use, please configure an open port."
+                    )
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {self.app__port}")
+        else:
+            try:
+                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
+            except ValueError:
+                raise ConfigurationError(
+                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
+                )
+            try:
+                self.app__port = find_available_port(self.app__host, default_server_port)
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {default_server_port}")
+
+        if self.app__debug:
+            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
+            self.app__verbose = True
+            self.app__open_browser = False
+        else:
+            warnings.formatwarning = custom_format_warning
+
+        if not self.app__verbose:
+            sys.tracebacklimit = 0
+
+        # secret key:
+        #   first, from CXG_SECRET_KEY environment variable
+        #   second, from config file
+        self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
+
+        # CSP Directives are a dict of string: list(string) or string: string
+        if self.app__csp_directives is not None:
+            for k, v in self.app__csp_directives.items():
+                if not isinstance(k, str):
+                    raise ConfigurationError("CSP directive names must be a string.")
+                if isinstance(v, list):
+                    for policy in v:
+                        if not isinstance(policy, str):
+                            raise ConfigurationError("CSP directive value must be a string or list of strings.")
+                elif not isinstance(v, str):
+                    raise ConfigurationError("CSP directive value must be a string or list of strings.")
+
+        if self.app__web_base_url is None:
+            self.app__web_base_url = self.app__api_base_url
+
+    def handle_authentication(self):
+        self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+
+        # oauth
+        ptypes = str if self.authentication__type == "oauth" else (type(None), str)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__oauth_api_base_url", ptypes
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__jwt_decode_options", (type(None), dict)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
+
+        if self.authentication__params_oauth__session_cookie:
+            self.validate_correct_type_of_configuration_attribute(
+                "authentication__params_oauth__cookie", (type(None), dict)
+            )  # noqa E501
+        else:
+            self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
+        #   secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
+        #   second, from config file
+        self.authentication__params_oauth__client_secret = os.environ.get(
+            "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret
+        )
+
+        self.auth = AuthTypeFactory.create(self.authentication__type, self)
+        if self.auth is None:
+            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
+
+    def handle_data_locator(self):
+        self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
+        if self.data_locator__s3__region_name is True:
+            path = self.single_dataset__datapath or self.multi_dataset__dataroot
+
+            if type(path) == dict:
+                # if multi_dataset__dataroot is a dict, then use the first key
+                # that is in s3.   NOTE:  it is not supported to have dataroots
+                # in different regions.
+                paths = [val.get("dataroot") for val in path.values()]
+                for path in paths:
+                    if path.startswith("s3://"):
+                        break
+            if path.startswith("s3://"):
+                region_name = discover_s3_region_name(path)
+                if region_name is None:
+                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
+            else:
+                region_name = None
+            self.data_locator__s3__region_name = region_name
+
+    def handle_data_source(self):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+
+        if self.single_dataset__datapath and self.multi_dataset__dataroot:
+            raise ConfigurationError(
+                "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
+            )
+        if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
+            raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
+
+    def handle_single_dataset(self, context):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
+
+        if self.single_dataset__datapath is None:
+            return
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
+
+        # preload this data set
+        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
+        try:
+            matrix_data_loader.pre_load_validation()
+        except DatasetAccessError as e:
+            raise ConfigurationError(str(e))
+
+        file_size = matrix_data_loader.file_size()
+        file_basename = basename(self.single_dataset__datapath)
+        if file_size > BIG_FILE_SIZE_THRESHOLD:
+            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
+        else:
+            context["messagefn"](f"Loading data from {file_basename}.")
+
+        if self.single_dataset__about:
+
+            def url_check(url):
+                try:
+                    result = urlparse(url)
+                    if all([result.scheme, result.netloc]):
+                        return True
+                    else:
+                        return False
+                except ValueError:
+                    return False
+
+            if not url_check(self.single_dataset__about):
+                raise ConfigurationError(
+                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
+                )
+
+    def handle_multi_dataset(self):
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
+        self.validate_correct_type_of_configuration_attribute(
+            "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
+        )  # noqa E501
+
+        if self.multi_dataset__dataroot is None:
+            return
+
+        if type(self.multi_dataset__dataroot) == str:
+            default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
+            self.multi_dataset__dataroot = dict(d=default_dict)
+
+        for tag, dataroot_dict in self.multi_dataset__dataroot.items():
+            if "base_url" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
+            if "dataroot" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
+
+            base_url = dataroot_dict["base_url"]
+
+            # sanity check for well formed base urls
+            bad = False
+            if type(base_url) != str:
+                bad = True
+            elif os.path.normpath(base_url) != base_url:
+                bad = True
+            else:
+                base_url_parts = base_url.split("/")
+                if [quote_plus(part) for part in base_url_parts] != base_url_parts:
+                    bad = True
+                if ".." in base_url_parts:
+                    bad = True
+            if bad:
+                raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
+
+        # verify all the base_urls are unique
+        base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
+        if len(base_urls) > len(set(base_urls)):
+            raise ConfigurationError("error in multi_dataset__dataroot:  base_urls must be unique")
+
+        # error checking
+        for mtype in self.multi_dataset__allowed_matrix_types:
+            try:
+                MatrixDataType(mtype)
+            except ValueError:
+                raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(
+                max_cached=self.multi_dataset__matrix_cache__max_datasets,
+                timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
+            )
+
+    def handle_diffexp(self):
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
+
+        max_workers = self.diffexp__alg_cxg__max_workers
+        cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
+        cpu_count = os.cpu_count()
+        max_workers = min(max_workers, cpu_multiplier * cpu_count)
+        diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
+
+    def handle_adaptor(self):
+        # cxg
+        self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
+        regionkey = "vfs.s3.region"
+        if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
+            if type(self.data_locator__s3__region_name) == str:
+                self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
+
+        from server.data_cxg.cxg_adaptor import CxgAdaptor
+
+        CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
+
+        # anndata
+        self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
+
+    def handle_limits(self):
+        self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
+
+    def exceeds_limit(self, limit_name, value):
+        limit_value = getattr(self, "limits__" + limit_name, None)
+        if limit_value is None:  # disabled
+            return False
+        return value > limit_value
+
+    def get_api_base_url(self):
+        if self.app__api_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__api_base_url and self.app__api_base_url.endswith("/"):
+            return self.app__api_base_url[:-1]
+        return self.app__api_base_url
+
+    def get_web_base_url(self):
+        if self.app__web_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__web_base_url is None:
+            return self.get_api_base_url()
+        if self.app__web_base_url.endswith("/"):
+            return self.app__web_base_url[:-1]
+        return self.app__web_base_url

server/common/errors.py

@@ -42,14 +42,14 @@ define_request_exception(
 define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
 define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
 define_request_exception(
-    "AuthenticationError",
+    "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
-    "Raised when there is an authentication error",
+)
-    default_status_code=HTTPStatus.UNAUTHORIZED)
 
 define_request_exception(
     "AnnotationCategoryNameError",
     "Raised when an annotation category name cant be saved",
-    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+)
 
 define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
 define_exception("ConfigurationError", "Raised when checking configuration errors")

server/common/rest.py

@@ -6,6 +6,7 @@ from http import HTTPStatus
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
 
+from server.common.config.client_config import get_client_config, get_client_userinfo
 from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
 from server.common.errors import (
     FilterError,
@@ -117,12 +118,12 @@ def schema_get(data_adaptor):
 
 
 def config_get(app_config, data_adaptor):
-    config = app_config.get_client_config(data_adaptor)
+    config = get_client_config(app_config, data_adaptor)
     return make_response(jsonify(config), HTTPStatus.OK)
 
 
 def userinfo_get(app_config, data_adaptor):
-    config = app_config.get_client_userinfo(data_adaptor)
+    config = get_client_userinfo(app_config, data_adaptor)
     return make_response(jsonify(config), HTTPStatus.OK)
 
 

server/common/utils/cxg_generation_utils.py

@@ -111,7 +111,7 @@ def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
 
 
 def convert_matrix_to_cxg_array(
-        matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
+    matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
 ):
     """
     Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`

server/common/utils/matrix_utils.py

@@ -41,16 +41,19 @@ def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
         number_of_non_zero_elements += np.count_nonzero(matrix_subset)
         if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
             if end_row_index != total_number_of_rows:
-                percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / (
+                percentage_of_non_zero_elements = (
-                    end_row_index * total_number_of_columns)
+                    100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
+                )
                 logging.info(
                     f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
-                    f"{percentage_of_non_zero_elements:6.2f}")
+                    f"{percentage_of_non_zero_elements:6.2f}"
+                )
             else:
                 percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
                 logging.info(
                     f"Matrix is not sparse. Percentage of non-zero elements (exact): "
-                    f"{percentage_of_non_zero_elements:6.2f}")
+                    f"{percentage_of_non_zero_elements:6.2f}"
+                )
             return False
 
     is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold

server/common/utils/type_conversion_utils.py

@@ -9,8 +9,10 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
     schema_type_hints_by_column_name = {}
 
     for column_name, column_values in dataframe.items():
-        dtypes_by_column_name[column_name], schema_type_hints_by_column_name[column_name] = \
+        (
-            get_dtype_and_schema_of_array(column_values)
+            dtypes_by_column_name[column_name],
+            schema_type_hints_by_column_name[column_name],
+        ) = get_dtype_and_schema_of_array(column_values)
 
     return dtypes_by_column_name, schema_type_hints_by_column_name
 
@@ -24,8 +26,10 @@ def get_schema_type_hint_of_array(array: pd.Series):
 
 
 def get_dtype_and_schema_of_array(array: pd.Series):
-    return (get_dtype_from_dtype(array.dtype, array_values=array),
+    return (
-            get_schema_type_hint_from_dtype(array.dtype, array_values=array))
+        get_dtype_from_dtype(array.dtype, array_values=array),
+        get_schema_type_hint_from_dtype(array.dtype, array_values=array),
+    )
 
 
 def get_dtype_from_dtype(dtype, array_values=None):
@@ -133,9 +137,11 @@ def can_cast_to_int32(dtype, array_values=None):
         if np.can_cast(dtype, np.int32):
             return True
         ii32 = np.iinfo(np.int32)
-        if not ordered_array_values.empty and (
+        if (
-                ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \
+            not ordered_array_values.empty
-                ordered_array_values.empty:
+            and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
+            or ordered_array_values.empty
+        ):
             return True
     return False
 

server/converters/h5ad_data_file.py

@@ -24,14 +24,14 @@ class H5ADDataFile:
     another format (currently just CXG is supported). """
 
     def __init__(
-            self,
+        self,
-            input_filename,
+        input_filename,
-            backed=False,
+        backed=False,
-            dataset_title=None,
+        dataset_title=None,
-            dataset_about=None,
+        dataset_about=None,
-            obs_index_column_name=None,
+        obs_index_column_name=None,
-            vars_index_column_name=None,
+        vars_index_column_name=None,
-            use_corpora_schema=True,
+        use_corpora_schema=True,
     ):
         self.input_filename = input_filename
         self.backed = backed

server/data_common/data_adaptor.py

@@ -5,7 +5,7 @@ import numpy as np
 import pandas as pd
 from server_timing import Timing as ServerTiming
 
-from server.common.app_config import AppFeature, AppConfig
+from server.common.config.app_config import AppConfig
 from server.common.constants import Axis
 from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
 from server.common.utils.utils import jsonify_numpy
@@ -173,7 +173,7 @@ class DataAdaptor(metaclass=ABCMeta):
         mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0]: i[1]] = True
+                mask[i[0] : i[1]] = True
             else:
                 mask[i] = True
         return mask
@@ -314,7 +314,7 @@ class DataAdaptor(metaclass=ABCMeta):
             top_n = self.dataset_config.diffexp__top_n
 
         if self.server_config.exceeds_limit(
-                "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 
@@ -388,3 +388,17 @@ class DataAdaptor(metaclass=ABCMeta):
         except RuntimeError:
             lastmod = None
         return lastmod
+
+
+class AppFeature(object):
+    def __init__(self, path, available=False, method="POST", extra={}):
+        self.path = path
+        self.available = available
+        self.method = method
+        self.extra = extra
+        [setattr(self, key, value) for key, value in extra.items()]
+
+    def todict(self):
+        d = dict(available=self.available, method=self.method, path=self.path)
+        d.update(self.extra)
+        return d

server/db/cellxgene_orm.py

@@ -1,11 +1,6 @@
 import uuid
 
-from sqlalchemy import (
+from sqlalchemy import Column, DateTime, ForeignKey, String, func, JSON
-    Column,
-    DateTime,
-    ForeignKey,
-    String,
-    func, JSON)
 from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship

server/db/db_utils.py

@@ -42,9 +42,9 @@ class DbUtils:
 
     def get_or_create_dataset(self, dataset_name):
         try:
-            dataset_id = self.query(
+            dataset_id = self.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])[
-                table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]
+                0
-            )[0].id
+            ].id
         except IndexError:
             dataset_id = uuid.uuid4()
             dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
@@ -54,9 +54,7 @@ class DbUtils:
 
     def get_or_create_user(self, user_id):
         try:
-            user_id = self.query(
+            user_id = self.query(table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id])[0].id
-                table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id]
-            )[0].id
         except IndexError:
             user = CellxGeneUser(id=user_id)
             self.session.add(user)

server/default_config.py

@@ -204,7 +204,6 @@ dataset:
     enable: true
     lfc_cutoff: 0.01
     top_n: 10
-
 """
 
 

server/eb/app.py

@@ -9,7 +9,7 @@ from flask import json
 import logging
 from flask_talisman import Talisman
 from flask_cors import CORS
-from server.common.aws_secret_utils import handle_config_from_secret
+from server.common.config import handle_config_from_secret
 from server.common.errors import SecretKeyRetrievalError
 
 
@@ -26,7 +26,7 @@ SERVERDIR = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(SERVERDIR)
 
 try:
-    from server.common.app_config import AppConfig
+    from server.common.config.app_config import AppConfig
     from server.app.app import Server
     from server.common.data_locator import DataLocator, discover_s3_region_name
 except Exception:
@@ -61,8 +61,7 @@ class WSGIServer(Server):
         csp = {
             "default-src": ["'self'"],
             "connect-src": ["'self'"] + extra_connect_src,
-            "script-src": ["'self'", "'unsafe-eval'"]
+            "script-src": ["'self'", "'unsafe-eval'"] + obsolete_browser_script_hash + script_hashes,
-            + obsolete_browser_script_hash + script_hashes,
             "style-src": ["'self'", "'unsafe-inline'"],
             "img-src": ["'self'", "https://cellxgene.cziscience.com", "data:"],
             "object-src": ["'none'"],
@@ -104,7 +103,7 @@ class WSGIServer(Server):
         if len(script_hashes) == 0:
             logging.error("Content security policy hashes are missing, falling back to unsafe-inline policy")
 
-        return (script_hashes)
+        return script_hashes
 
     @staticmethod
     def compute_inline_csp_hashes(app, app_config):
@@ -173,9 +172,7 @@ try:
         sys.exit(1)
 
     # features are unsupported in the current hosted server
-    app_config.update_default_dataset_config(
+    app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
-        embeddings__enable_reembedding=False,
-    )
     app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
     app_config.complete_config(logging.info)
 

server/test/__init__.py

@@ -13,7 +13,8 @@ import requests
 
 from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
 from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT
+from server.common.config.app_config import AppConfig
+from server.common.config import DEFAULT_SERVER_PORT
 from server.common.data_locator import DataLocator
 from server.common.utils.utils import find_available_port
 from server.data_common.fbs.matrix import encode_matrix_fbs
@@ -33,8 +34,7 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
     data_locator = DataLocator(fname)
     config = AppConfig()
     config.update_server_config(
-        multi_dataset__dataroot=data_locator.path,
+        multi_dataset__dataroot=data_locator.path, authentication__type="test",
-        authentication__type="test",
     )
     config.update_default_dataset_config(
         embeddings__names=["umap"],
@@ -42,16 +42,13 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
         diffexp__lfc_cutoff=0.01,
         user_annotations__type="hosted_tiledb_array",
         user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432",
-        user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir
+        user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
     )
 
     config.complete_config()
 
     data = MatrixDataLoader(data_locator.abspath()).open(config)
-    annotations = AnnotationsHostedTileDB(
+    annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),)
-        tmp_dir,
-        DbUtils("postgresql://postgres:test_pw@localhost:5432"),
-    )
     return data, tmp_dir, annotations
 
 

server/test/fixtures/database/__init__.py

@@ -29,34 +29,26 @@ class TestDatabase:
 
     def _create_test_user(self):
         user = CellxGeneUser(id="test_user_id")
-        user2 = CellxGeneUser(id='1234')
+        user2 = CellxGeneUser(id="1234")
         self.db.session.add(user)
         self.db.session.add(user2)
         self.db.session.commit()
 
     def _create_test_dataset(self):
-        dataset = CellxGeneDataset(
+        dataset = CellxGeneDataset(name="test_dataset",)
-            name="test_dataset",
-        )
         self.db.session.add(dataset)
         self.db.session.commit()
 
     def _create_test_annotation(self):
-        dataset = self.db.query([CellxGeneDataset],
+        dataset = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == "test_dataset"],)[0]
-                                [CellxGeneDataset.name == "test_dataset"],
+        annotation = Annotation(tiledb_uri="tiledb_uri", user_id="test_user_id", dataset_id=str(dataset.id))
-                                )[0]
-        annotation = Annotation(
-            tiledb_uri="tiledb_uri",
-            user_id="test_user_id",
-            dataset_id=str(dataset.id)
-        )
         self.db.session.add(annotation)
         self.db.session.commit()
 
     @staticmethod
     def get_random_string():
         letters = string.ascii_lowercase
-        return ''.join(random.choice(letters) for i in range(12))
+        return "".join(random.choice(letters) for i in range(12))
 
     def _create_test_users(self, user_count: int = 10):
         users = []
@@ -80,10 +72,8 @@ class TestDatabase:
         for i in range(annotation_count):
             dataset = self.order_by_random(CellxGeneDataset)
             user = self.order_by_random(CellxGeneUser)
-            annotations.append(Annotation(
+            annotations.append(
-                tiledb_uri=self.get_random_string(),
+                Annotation(tiledb_uri=self.get_random_string(), user_id=user.id, dataset_id=str(dataset.id))
-                user_id=user.id,
+            )
-                dataset_id=str(dataset.id)
-            ))
         self.db.session.add_all(annotations)
         self.db.session.commit()

server/test/fixtures/dataset_config_outline.py

@@ -0,0 +1,37 @@
+f"""
+dataset:
+  app:
+    scripts: {scripts} #list of strs (filenames) or dicts containing keys
+    inline_scripts: {inline_scripts} #list of strs (filenames)
+
+    about_legal_tos: {about_legal_tos}
+    about_legal_privacy: {about_legal_privacy}
+
+    authentication_enable: {authentication_enable}
+
+  presentation:
+    max_categories: {max_categories}
+    custom_colors: {custom_colors}
+
+  user_annotations:
+    enable: {enable_users_annotations}
+    type: {annotation_type}
+    hosted_tiledb_array:
+      db_uri: {db_uri}
+      hosted_file_directory: {hosted_file_directory}
+    local_file_csv:
+      directory: {local_file_csv_directory}
+      file: {local_file_csv_file}
+    ontology:
+      enable: {ontology_enabled}
+      obo_location: {obo_location}
+
+  embeddings:
+    names: {embedding_names}
+    enable_reembedding: {enable_reembedding}
+
+  diffexp:
+    enable: {enable_difexp}
+    lfc_cutoff: {lfc_cutoff}
+    top_n: {top_n}
+"""

server/test/fixtures/server_config_outline.py

@@ -0,0 +1,62 @@
+f"""server:
+  app:
+    verbose: {verbose}
+    debug: {debug}
+    host: {host}
+    port: {port}
+    open_browser: {open_browser}
+    force_https: {force_https}
+    flask_secret_key: {flask_secret_key}
+    generate_cache_control_headers: {generate_cache_control_headers}
+    server_timing_headers: {server_timing_headers}
+    csp_directives: {csp_directives}
+    api_base_url: {api_base_url}
+    web_base_url: {web_base_url}
+  authentication:
+    type: {auth_type}
+    params_oauth:
+      oauth_api_base_url: {oauth_api_base_url}
+      client_id: {client_id}
+      client_secret: {client_secret}
+      jwt_decode_options: {jwt_decode_options}
+      session_cookie:  {session_cookie}
+      cookie:  {cookie}
+
+  multi_dataset:
+    dataroot: {dataroot}
+    index: {index}
+    allowed_matrix_types: {allowed_matrix_types}
+    matrix_cache:
+      max_datasets: {max_cached_datasets}
+      timelimit_s: {timelimit_s}
+
+  single_dataset:
+    datapath: {dataset_datapath}
+    obs_names: {obs_names}
+    var_names: {var_names}
+    about: {about}
+    title: {title}
+
+  diffexp:
+    alg_cxg: # number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count)
+      max_workers: {diffexp_max_workers}
+      cpu_multiplier: {cpu_multiplier}
+      target_workunit: {target_workunit}  # The target number of matrix elements that are evaluated in one thread.
+
+  data_locator:
+    s3:
+      region_name: {data_locater_region_name}
+
+  adaptor:
+    cxg_adaptor:
+      tiledb_ctx:
+        sm.tile_cache_size:  {cxg_tile_cache_size}
+        sm.num_reader_threads:  {cxg_num_reader_threads}
+
+    anndata_adaptor:
+      backed: {anndata_backed}
+
+  limits:
+    column_request_max: {column_request_max}
+    diffexp_cellcount_max: {diffexp_cellcount_max}
+"""

server/test/performance/run_diffexp.py

@@ -7,7 +7,7 @@ import numpy as np
 import server.compute.diffexp_cxg as diffexp_cxg
 import server.compute.diffexp_generic as diffexp_generic
 
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
 from server.data_common.matrix_loader import MatrixDataLoader
 from server.data_cxg.cxg_adaptor import CxgAdaptor
 

server/test/test_database/test_database.py

@@ -16,45 +16,48 @@ class DatabaseTest(unittest.TestCase):
         del cls.db
 
     def test_user_creation(self):
-        one_user = self.db.get(table=CellxGeneUser, entity_id='test_user_id')
+        one_user = self.db.get(table=CellxGeneUser, entity_id="test_user_id")
-        self.assertEqual(one_user.id, 'test_user_id')
+        self.assertEqual(one_user.id, "test_user_id")
         user_count = self.db.session.query(CellxGeneUser).count()
         self.assertGreater(user_count, 10)
 
     def test_dataset_creation(self):
-        one_dataset = self.db.query(table_args=[CellxGeneDataset],
+        one_dataset = self.db.query(
-                                    filter_args=[CellxGeneDataset.name == 'test_dataset'])
+            table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"]
-        self.assertEqual(one_dataset[0].name, 'test_dataset')
+        )
+        self.assertEqual(one_dataset[0].name, "test_dataset")
         dataset_count = self.db.session.query(CellxGeneDataset).count()
         self.assertGreater(dataset_count, 10)
 
     def test_annotation_creation(self):
-        one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0]
+        one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == "tiledb_uri"])[0]
-        self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri')
+        self.assertEqual(one_annotation.tiledb_uri, "tiledb_uri")
         annotation_count = self.db.session.query(Annotation).count()
         self.assertGreater(annotation_count, 10)
 
     def test_get_most_recent_annotation_for_user_dataset(self):
-        dataset_id = str(self.db.query(table_args=[CellxGeneDataset],
+        dataset_id = str(
-                                       filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id)
+            self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"])[0].id
+        )
 
         # have to commit separately because created_at time written on the db server
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_0"))
         self.db.session.commit()
 
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_1"))
         self.db.session.commit()
 
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_2"))
         self.db.session.commit()
 
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_3"))
         self.db.session.commit()
 
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_4"))
         self.db.session.commit()
 
-        most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id,
+        most_recent_annotation = self.db.query_for_most_recent(
-                                                                            Annotation.user_id == 'test_user_id'])
+            Annotation, [Annotation.dataset_id == dataset_id, Annotation.user_id == "test_user_id"]
+        )
 
-        self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4')
+        self.assertEqual(most_recent_annotation.tiledb_uri, "tiledb_uri_4")

server/test/unit/auth/test_auth.py

@@ -2,7 +2,7 @@ import unittest
 
 import requests
 
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
 from server.test import FIXTURES_ROOT, test_server
 
 
@@ -12,9 +12,7 @@ class AuthTest(unittest.TestCase):
 
     def test_auth_none(self):
         c = AppConfig()
-        c.update_server_config(
+        c.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
-            authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot
-        )
         c.update_default_dataset_config(user_annotations__enable=False)
 
         c.complete_config()
@@ -28,9 +26,7 @@ class AuthTest(unittest.TestCase):
 
     def test_auth_session(self):
         c = AppConfig()
-        c.update_server_config(
+        c.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
-            authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot
-        )
         c.update_default_dataset_config(user_annotations__enable=True)
         c.complete_config()
 
@@ -107,8 +103,8 @@ class AuthTest(unittest.TestCase):
     def test_auth_test_single(self):
         c = AppConfig()
         c.update_server_config(
-            authentication__type="test",
+            authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
-            single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg")
+        )
 
         c.complete_config()
 

server/test/unit/auth/test_oauth.py

@@ -9,7 +9,7 @@ from flask import Flask, jsonify, make_response, request, redirect
 from multiprocessing import Process
 
 import jose
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
 from server.test import FIXTURES_ROOT, test_server
 
 # This tests the oauth authentication type.
@@ -46,7 +46,7 @@ def token():
         "scope": "openid profile email",
         "expires_in": TOKEN_EXPIRES,
         "token_type": "Bearer",
-        "expires_at": expires_at
+        "expires_at": expires_at,
     }
     return make_response(jsonify(r))
 
@@ -89,9 +89,8 @@ class AuthTest(unittest.TestCase):
             authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}",
             authentication__params_oauth__client_id="mock_client_id",
             authentication__params_oauth__client_secret="mock_client_secret",
-            authentication__params_oauth__jwt_decode_options={
+            authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False},
-                "verify_signature": False, "verify_iss": False
+        )
-            })
 
         app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
         app_config.complete_config()
@@ -161,9 +160,7 @@ class AuthTest(unittest.TestCase):
     def test_auth_oauth_session(self):
         # test with session cookies
         app_config = AppConfig()
-        app_config.update_server_config(
+        app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
-            authentication__params_oauth__session_cookie=True,
-        )
         self.auth_flow(app_config)
 
     def test_auth_oauth_cookie(self):

server/test/unit/cli/test_launch.py

@@ -0,0 +1,28 @@
+import filecmp
+import os
+import shutil
+import unittest
+
+import yaml
+
+from server.default_config import default_config
+from server.test import FIXTURES_ROOT
+
+
+class CLIPLaunchTests(unittest.TestCase):
+    tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.mkdir(cls.tmp_dir)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_dir)
+
+
+def test_dump_default_config(self):
+    os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
+    with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
+        expected_config.write(yaml.dump(default_config))
+    filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")

server/test/unit/common/config/__init__.py

@@ -0,0 +1,246 @@
+import os
+import shutil
+import unittest
+import random
+from unittest import mock
+
+from server.test import FIXTURES_ROOT
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class ConfigTests(unittest.TestCase):
+    tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_fixtures_directory)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.makedirs(cls.tmp_fixtures_directory)
+
+    def custom_server_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="null",
+        generate_cache_control_headers="false",
+        server_timing_headers="false",
+        csp_directives="null",
+        api_base_url="null",
+        web_base_url="null",
+        auth_type="session",
+        oauth_api_base_url="null",
+        client_id="null",
+        client_secret="null",
+        jwt_decode_options="null",
+        session_cookie="true",
+        cookie="null",
+        dataroot="null",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        diffexp_max_workers=64,
+        cpu_multiplier=4,
+        target_workunit="16_000_000",
+        data_locater_region_name="us-east-1",
+        cxg_tile_cache_size=8589934592,
+        cxg_num_reader_threads=32,
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        config_file_name="server_config.yaml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py")
+        with open(server_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            server_config = eval(config)
+        with open(configfile, "w") as server_config_file:
+            server_config_file.write(server_config)
+        return configfile
+
+    def custom_app_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="null",
+        generate_cache_control_headers="false",
+        server_timing_headers="false",
+        csp_directives="null",
+        api_base_url="null",
+        web_base_url="null",
+        auth_type="session",
+        oauth_api_base_url="null",
+        client_id="null",
+        client_secret="null",
+        jwt_decode_options="null",
+        session_cookie="true",
+        cookie="null",
+        dataroot="null",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        diffexp_max_workers=64,
+        cpu_multiplier=4,
+        target_workunit="16_000_000",
+        data_locater_region_name="us-east-1",
+        cxg_tile_cache_size=8589934592,
+        cxg_num_reader_threads=32,
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        scripts=[],
+        inline_scripts=[],
+        about_legal_tos="null",
+        about_legal_privacy="null",
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        config_file_name="app_config.yml",
+    ):
+        random_num = random.randrange(999999)
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config = self.custom_server_config(
+            verbose=verbose,
+            debug=debug,
+            host=host,
+            port=port,
+            open_browser=open_browser,
+            force_https=force_https,
+            flask_secret_key=flask_secret_key,
+            generate_cache_control_headers=generate_cache_control_headers,
+            server_timing_headers=server_timing_headers,
+            csp_directives=csp_directives,
+            api_base_url=api_base_url,
+            web_base_url=web_base_url,
+            auth_type=auth_type,
+            oauth_api_base_url=oauth_api_base_url,
+            client_id=client_id,
+            client_secret=client_secret,
+            jwt_decode_options=jwt_decode_options,
+            session_cookie=session_cookie,
+            cookie=cookie,
+            dataroot=dataroot,
+            index=index,
+            allowed_matrix_types=allowed_matrix_types,
+            max_cached_datasets=max_cached_datasets,
+            timelimit_s=timelimit_s,
+            dataset_datapath=dataset_datapath,
+            obs_names=obs_names,
+            var_names=var_names,
+            about=about,
+            title=title,
+            diffexp_max_workers=diffexp_max_workers,
+            cpu_multiplier=cpu_multiplier,
+            target_workunit=target_workunit,
+            data_locater_region_name=data_locater_region_name,
+            cxg_tile_cache_size=cxg_tile_cache_size,
+            cxg_num_reader_threads=cxg_num_reader_threads,
+            anndata_backed=anndata_backed,
+            column_request_max=column_request_max,
+            diffexp_cellcount_max=diffexp_cellcount_max,
+            config_file_name=f"temp_server_config_{random_num}.yml",
+        )
+        dataset_config = self.custom_dataset_config(
+            scripts=scripts,
+            inline_scripts=inline_scripts,
+            about_legal_tos=about_legal_tos,
+            about_legal_privacy=about_legal_privacy,
+            authentication_enable=authentication_enable,
+            max_categories=max_categories,
+            custom_colors=custom_colors,
+            enable_users_annotations=enable_users_annotations,
+            annotation_type=annotation_type,
+            db_uri=db_uri,
+            hosted_file_directory=hosted_file_directory,
+            local_file_csv_directory=local_file_csv_directory,
+            local_file_csv_file=local_file_csv_file,
+            ontology_enabled=ontology_enabled,
+            obo_location=obo_location,
+            embedding_names=embedding_names,
+            enable_reembedding=enable_reembedding,
+            enable_difexp=enable_difexp,
+            lfc_cutoff=lfc_cutoff,
+            top_n=top_n,
+            config_file_name=f"temp_dataset_config_{random_num}.yml",
+        )
+        with open(server_config) as server_config:
+            with open(dataset_config) as dataset_config:
+                with open(configfile, "w") as app_config_file:
+                    for line in server_config:
+                        app_config_file.write(line)
+                    for line in dataset_config:
+                        app_config_file.write(line)
+
+        return configfile
+
+    def custom_dataset_config(
+        self,
+        scripts=[],
+        inline_scripts=[],
+        about_legal_tos="null",
+        about_legal_privacy="null",
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        config_file_name="dataset_config.yml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py")
+        with open(dataset_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            dataset_config = eval(config)
+        with open(configfile, "w") as dataset_config_file:
+            dataset_config_file.write(dataset_config)
+
+        return configfile

server/test/unit/common/config/test_app_config.py

@@ -0,0 +1,140 @@
+import os
+import tempfile
+import unittest
+
+import yaml
+
+from server.default_config import default_config
+from server.common.config.app_config import AppConfig
+from server.test.unit.common.config import ConfigTests
+from server.common.errors import ConfigurationError
+from server.test import FIXTURES_ROOT
+
+
+class AppConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_get_default_config_correctly_reads_default_config_file(self):
+        app_default_config = AppConfig().default_config
+
+        expected_config = yaml.load(default_config, Loader=yaml.Loader)
+
+        server_config = app_default_config['server']
+        dataset_config = app_default_config['dataset']
+
+        expected_server_config = expected_config['server']
+        expected_dataset_config = expected_config['dataset']
+
+        self.assertDictEqual(app_default_config, expected_config)
+        self.assertDictEqual(server_config, expected_server_config)
+        self.assertDictEqual(dataset_config, expected_dataset_config)
+
+    def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self):
+        datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
+        file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+
+        self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
+
+    def test_update_server_config_updates_server_config_and_config_status(self):
+        config = self.get_config()
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.check_config()
+
+    def test_write_config_outputs_yaml_with_all_config_vars(self):
+        config = self.get_config()
+        config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
+        with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
+            default_config_yml = yaml.safe_load(default_config)
+
+        with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
+            output_config_yml = yaml.safe_load(output_config)
+        self.maxDiff = None
+        self.assertEqual(default_config_yml, output_config_yml)
+
+    def test_update_app_config(self):
+        config = AppConfig()
+        config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
+        vars = config.default_dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
+        vars = config.default_dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
+
+    def test_configfile_no_dataset_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    multi_dataset:
+                        dataroot: test_dataroot
+
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.default_dataset_config.changes_from_default()
+            self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
+            self.assertEqual(dataset_changes, [])
+
+    def test_configfile_no_server_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                dataset:
+                    user_annotations:
+                        enable: false
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.default_dataset_config.changes_from_default()
+            self.assertEqual(server_changes, [])
+            self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])

server/test/unit/common/config/test_base_config.py

@@ -0,0 +1,63 @@
+import unittest
+
+from server.common.config.app_config import AppConfig
+from server.test import FIXTURES_ROOT
+from server.test.unit.common.config import ConfigTests
+from server.common.errors import ConfigurationError
+
+
+class BaseConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
+        config = AppConfig()
+        mapping = config.default_dataset_config.create_mapping(config.default_config)
+        self.assertIsNotNone(mapping["server__app__verbose"])
+        self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
+        self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
+        self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
+
+    def test_changes_from_default_returns_list_of_nondefault_config_values(self):
+        config = self.get_config(verbose="true", lfc_cutoff=0.05)
+        server_changes = config.server_config.changes_from_default()
+        dataset_changes = config.default_dataset_config.changes_from_default()
+
+        self.assertEqual(
+            server_changes,
+            [
+                ("app__verbose", True, False),
+                ("multi_dataset__dataroot", FIXTURES_ROOT, None),
+                ("multi_dataset__matrix_cache__timelimit_s", 5, 30),
+                ("data_locator__s3__region_name", "us-east-1", True),
+            ],
+        )
+        self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
+
+    def test_check_config_throws_error_if_attr_has_not_been_checked(self):
+        config = self.get_config(verbose="true")
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(app__verbose=False)
+        with self.assertRaises(ConfigurationError):
+            config.check_config()

server/test/unit/common/config/test_dataset_config.py

@@ -0,0 +1,260 @@
+import os
+import tempfile
+
+import requests
+import unittest
+from unittest.mock import patch
+
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.app_config import AppConfig
+from server.common.config.base_config import BaseConfig
+from server.test import test_server, PROJECT_ROOT, FIXTURES_ROOT
+
+from server.common.errors import ConfigurationError
+from server.test.unit.common.config import ConfigTests
+
+
+class TestDatasetConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.dataset_config = self.config.default_dataset_config
+        self.config.complete_config()
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_datatset_config_sets_vars_from_default_config(self):
+        config = AppConfig()
+        self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
+        self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
+        self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
+        self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location)
+
+    @patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.dataset_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 21)
+
+    def test_app_sets_script_vars(self):
+        config = self.get_config(scripts=["path/to/script"])
+        config.default_dataset_config.handle_app()
+
+        self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}])
+
+        config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
+        config.default_dataset_config.handle_app()
+        self.assertEqual(
+            config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=["path/to/script", "different/script/path"])
+        config.default_dataset_config.handle_app()
+        # TODO @madison -- is this the desired functionality?
+        self.assertEqual(
+            config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=[{"more": "different/script/path"}])
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_app()
+
+    def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
+        config = self.get_config(enable_users_annotations="true", authentication_enable="false")
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_user_annotations(self.context)
+
+        config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
+        with self.assertRaises(ConfigurationError):
+            config.server_config.complete_config(self.context)
+
+    def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self):
+        config = self.get_config(
+            enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set"
+        )
+        config.default_dataset_config.handle_user_annotations(self.context)
+
+        self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."])
+
+    @patch("server.common.config.dataset_config.DbUtils")
+    def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils):
+        mock_db_utils.return_value = "123"
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+
+        config = self.get_config(
+            enable_users_annotations="true",
+            authentication_enable="true",
+            annotation_type="hosted_tiledb_array",
+            db_uri="gotta/set/this",
+            hosted_file_directory="and/this",
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB)
+
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
+        )
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_user_annotations(self.context)
+
+    def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_local_file_csv_annotations()
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+        cwd = os.getcwd()
+        self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
+
+    def test_handle_embeddings__checks_data_file_types(self):
+        file_name = self.custom_app_config(
+            embedding_names=["name1", "name2"],
+            enable_reembedding="true",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+            anndata_backed="true",
+            config_file_name=self.config_file_name,
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_embeddings()
+
+    def test_handle_diffexp__raises_warning_for_large_datasets(self):
+        config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_diffexp(self.context)
+        self.assertEqual(len(self.context["messages"]), 0)
+
+    def test_multi_dataset(self):
+        config = AppConfig()
+        # test for illegal url_dataroots
+        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+            config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            with self.assertRaises(ConfigurationError):
+                config.complete_config()
+
+        # test for legal url_dataroots
+        for legal in ("d", "this.is-okay_", "a/b"):
+            config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            config.complete_config()
+
+        # test that multi dataroots work end to end
+        config.update_server_config(
+            multi_dataset__dataroot=dict(
+                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+            )
+        )
+
+        # Change this default to test if the dataroot overrides below work.
+        config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+        # specialize the configs for set1
+        config.add_dataroot_config(
+            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+        )
+
+        # specialize the configs for set2
+        config.add_dataroot_config(
+            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+        )
+
+        # no specializations for set3 (they get the default dataset config)
+        config.complete_config()
+
+        with test_server(app_config=config) as server:
+            session = requests.Session()
+
+            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is False
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    def test_configfile_with_specialization(self):
+        # test that per_dataset_config config load the default config, then the specialized config
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    multi_dataset:
+                        dataroot:
+                            test:
+                                base_url: test
+                                dataroot: fake_dataroot
+
+                dataset:
+                    user_annotations:
+                        enable: false
+                        type: hosted_tiledb_array
+                        hosted_tiledb_array:
+                            db_uri: fake_db_uri
+                            hosted_file_directory: fake_dir
+
+                per_dataset_config:
+                    test:
+                        user_annotations:
+                            enable: true
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+
+            test_config = app_config.dataroot_config["test"]
+
+            # test config from default
+            self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
+            self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
+
+            # test config from specialization
+            self.assertTrue(test_config.user_annotations__enable)

server/test/unit/common/config/test_server_config.py

@@ -0,0 +1,335 @@
+import os
+import unittest
+from unittest import mock
+from unittest.mock import patch
+
+from server.common.config.base_config import BaseConfig
+from server.common.utils.utils import find_available_port
+from server.test import PROJECT_ROOT, FIXTURES_ROOT
+
+import requests
+
+from server.common.config.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from server.test import test_server
+from server.test.unit.common.config import ConfigTests
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class TestServerConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_raises_error_if_default_config_is_invalid(self):
+        invalid_config = self.get_config(port="not_valid")
+        with self.assertRaises(ConfigurationError):
+            invalid_config.complete_config()
+
+    @patch("server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.server_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 40)
+
+    def test_handle_app__throws_error_if_port_doesnt_exist(self):
+        config = self.get_config(port=99999999)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_app(self.context)
+
+    @patch("server.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = None
+        # Default config
+        self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
+        # hard coded
+        config = self.get_config()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
+        # incorrectly formatted
+        dataroot = {
+            "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+            "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"},
+        }
+        file_name = self.custom_app_config(
+            dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_locator()
+
+    @patch("server.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = "us-west-2"
+        dataroot = {
+            "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+            "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
+        }
+        file_name = self.custom_app_config(
+            dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_data_locator()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
+        mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
+
+    def test_handle_app___can_use_envar_port(self):
+        config = self.get_config(port=24)
+        self.assertEqual(config.server_config.app__port, 24)
+
+        # Note if the port is set in the config file it will NOT be overwritten by a different envvar
+        os.environ["CXG_SERVER_PORT"] = "4008"
+        self.config = AppConfig()
+        self.config.server_config.handle_app(self.context)
+        self.assertEqual(self.config.server_config.app__port, 4008)
+
+    def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
+        config = self.get_config(flask_secret_key="KEY_FROM_FILE")
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
+
+        os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
+        config.server_config.handle_app(self.context)
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
+
+    def test_handle_app__sets_web_base_url(self):
+        config = self.get_config(web_base_url="anything.com")
+        self.assertEqual(config.server_config.app__web_base_url, "anything.com")
+
+    def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self):
+        config = self.get_config(client_secret="KEY_FROM_FILE")
+        config.server_config.handle_authentication()
+        self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
+
+        os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
+        config.server_config.handle_authentication()
+
+        self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
+
+    def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}",
+            config_file_name="two_data_roots.yml",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_source()
+
+        file_name = self.custom_app_config(config_file_name="zero_roots.yml")
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_source()
+
+    def test_get_api_base_url_works(self):
+
+        # test the api_base_url feature, and that it can contain a path
+        config = AppConfig()
+        backend_port = find_available_port("localhost", 10000)
+        config.update_server_config(
+            app__api_base_url=f"http://localhost:{backend_port}/additional/path",
+            multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
+        )
+
+        config.complete_config()
+
+        with test_server(["-p", str(backend_port)], app_config=config) as server:
+            session = requests.Session()
+            self.assertEqual(server, f"http://localhost:{backend_port}")
+            response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
+            self.assertEqual(response.status_code, 200)
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
+
+            # test the health check at the correct url
+            response = session.get(f"{server}/additional/path/health")
+            assert response.json()["status"] == "pass"
+
+            # also check that the old URL still works.
+            # NOTE:  this old URL location will soon be deprecated, and when that happens
+            # this check can be removed.
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    def test_get_web_base_url_works(self):
+        config = self.get_config(web_base_url="www.thisisawebsite.com")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+        config = self.get_config(web_base_url="local", port=12)
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "http://localhost:12")
+
+        config = self.get_config(web_base_url="www.thisisawebsite.com/")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+        config = self.get_config(api_base_url="www.api_base.com/")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.api_base.com")
+
+    def test_config_for_single_dataset(self):
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_single_dataset(self.context)
+        self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
+
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset_with_about.yml",
+            about="www.cziscience.com",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_single_dataset(self.context)
+
+    def test_multi_dataset_raises_error_for_illegal_routes(self):
+        # test for illegal url_dataroots
+        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+            self.config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            with self.assertRaises(ConfigurationError):
+                self.config.complete_config()
+
+    def test_multidataset_works_for_legal_routes(self):
+        # test for legal url_dataroots
+        for legal in ("d", "this.is-okay_", "a/b"):
+            self.config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            self.config.complete_config()
+
+    def test_mulitdatasets_work_e2e(self):
+        # test that multi dataroots work end to end
+        self.config.update_server_config(
+            multi_dataset__dataroot=dict(
+                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+            )
+        )
+
+        # Change this default to test if the dataroot overrides below work.
+        self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+        # specialize the configs for set1
+        self.config.add_dataroot_config(
+            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+        )
+
+        # specialize the configs for set2
+        self.config.add_dataroot_config(
+            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+        )
+
+        # no specializations for set3 (they get the default dataset config)
+        self.config.complete_config()
+
+        with test_server(app_config=self.config) as server:
+            session = requests.Session()
+
+            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is False
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    @patch("server.common.config.server_config.diffexp_tiledb.set_config")
+    def test_handle_diffexp(self, mock_tiledb_config):
+        custom_config_file = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}",
+            cpu_multiplier=3,
+            diffexp_max_workers=1,
+            target_workunit=4,
+            config_file_name=self.config_file_name,
+        )
+        config = AppConfig()
+        config.update_from_config_file(custom_config_file)
+        config.server_config.handle_diffexp()
+        # called with the min of diffexp_max_workers and cpus*cpu_multiplier
+        mock_tiledb_config.assert_called_once_with(1, 4)
+
+    @patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
+    def test_handle_adaptor(self, mock_tiledb_context):
+        custom_config = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
+        )
+        config = AppConfig()
+        config.update_from_config_file(custom_config)
+        config.server_config.handle_adaptor()
+        mock_tiledb_context.assert_called_once_with(
+            {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
+        )
+
+    @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
+    @patch("server.common.config.get_secret_key")
+    def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
+        mock_get_secret_key.return_value = {
+            "flask_secret_key": "mock_flask_secret",
+            "oauth_client_secret": "mock_oauth_secret",
+            "db_uri": "mock_db_uri",
+        }
+
+        config = AppConfig()
+
+        with self.assertLogs(level="INFO") as logger:
+            from server.common.config import handle_config_from_secret
+
+            # should not throw error
+            # "AttributeError: 'XConfig' object has no attribute 'x'"
+            handle_config_from_secret(config)
+
+            # should log 3 lines (one for each var set from a secret)
+            self.assertEqual(len(logger.output), 3)
+            self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0])
+            self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1])
+            self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2])
+            self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
+            self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
+            self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")

server/test/unit/common/test_api.py

@@ -8,8 +8,14 @@ import requests
 
 import server.test.unit.decode_fbs as decode_fbs
 from server.data_common.matrix_loader import MatrixDataType
-from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server,
+from server.test import (
-                         stop_test_server)
+    data_with_tmp_annotations,
+    make_fbs,
+    PROJECT_ROOT,
+    FIXTURES_ROOT,
+    start_test_server,
+    stop_test_server,
+)
 from server.test.fixtures.fixtures import pbmc3k_colors
 
 BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
@@ -381,11 +387,14 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
 
     @classmethod
     def setUpClass(cls):
-        cls._setupClass(cls, [
+        cls._setupClass(
-            f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+            cls,
-            "--disable-annotations",
+            [
-            "--experimental-enable-reembedding",
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
-        ])
+                "--disable-annotations",
+                "--experimental-enable-reembedding",
+            ],
+        )
 
     @classmethod
     def tearDownClass(cls):
@@ -403,10 +412,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
 
     @classmethod
     def setUpClass(cls):
-        cls._setupClass(cls, [
+        cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"])
-            f"{FIXTURES_ROOT}/pbmc3k.cxg",
-            "--disable-annotations",
-        ])
 
     @classmethod
     def tearDownClass(cls):
@@ -423,7 +429,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
             MatrixDataType.H5AD, annotations_fixture=True
         )
-        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ])
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
 
     @classmethod
     def tearDownClass(cls):
@@ -439,11 +445,7 @@ class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
     @classmethod
     def setUpClass(cls):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
-        cls._setupClass(cls, [
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
-            "--annotations-file",
-            cls.annotations.output_file,
-            cls.data.get_location(),
-        ])
 
     @classmethod
     def tearDownClass(cls):

server/test/unit/common/test_app_config.py

@@ -1,249 +0,0 @@
-import os
-import unittest
-from unittest import mock
-from unittest.mock import patch
-import tempfile
-
-import requests
-
-from server.common.app_config import AppConfig
-from server.common.errors import ConfigurationError
-from server.common.utils.utils import find_available_port
-from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
-
-
-# NOTE, there are more tests that should be written for AppConfig.
-# this is just a start.
-
-def mockenv(**envvars):
-    return mock.patch.dict(os.environ, envvars)
-
-
-class AppConfigTest(unittest.TestCase):
-    def test_update(self):
-        config = AppConfig()
-        config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
-        vars = config.server_config.changes_from_default()
-        self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
-
-        config = AppConfig()
-        config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
-        vars = config.server_config.changes_from_default()
-        self.assertCountEqual(vars, [])
-
-        config = AppConfig()
-        config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
-        vars = config.default_dataset_config.changes_from_default()
-        self.assertCountEqual(vars, [])
-
-        config = AppConfig()
-        config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
-        vars = config.default_dataset_config.changes_from_default()
-        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
-
-    def test_multi_dataset(self):
-
-        config = AppConfig()
-        # test for illegal url_dataroots
-        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
-            config.update_server_config(
-                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
-            )
-            with self.assertRaises(ConfigurationError):
-                config.complete_config()
-
-        # test for legal url_dataroots
-        for legal in ("d", "this.is-okay_", "a/b"):
-            config.update_server_config(
-                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
-            )
-            config.complete_config()
-
-        # test that multi dataroots work end to end
-        config.update_server_config(
-            multi_dataset__dataroot=dict(
-                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
-                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
-                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
-            )
-        )
-
-        # Change this default to test if the dataroot overrides below work.
-        config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
-
-        # specialize the configs for set1
-        config.add_dataroot_config(
-            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
-        )
-
-        # specialize the configs for set2
-        config.add_dataroot_config(
-            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
-        )
-
-        # no specializations for set3 (they get the default dataset config)
-        config.complete_config()
-
-        with test_server(app_config=config) as server:
-            session = requests.Session()
-
-            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
-            data_config = response.json()
-            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-            assert data_config["config"]["parameters"]["annotations"] is False
-            assert data_config["config"]["parameters"]["disable-diffexp"] is False
-            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
-
-            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
-            data_config = response.json()
-            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-            assert data_config["config"]["parameters"]["annotations"] is True
-            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
-
-            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
-            data_config = response.json()
-            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-            assert data_config["config"]["parameters"]["annotations"] is True
-            assert data_config["config"]["parameters"]["disable-diffexp"] is False
-            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
-
-            response = session.get(f"{server}/health")
-            assert response.json()["status"] == "pass"
-
-    @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
-    @patch('server.common.aws_secret_utils.get_secret_key')
-    def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
-        mock_get_secret_key.return_value = {
-            "flask_secret_key": "mock_flask_secret",
-            "oauth_client_secret": "mock_oauth_secret",
-            "db_uri": "mock_db_uri"
-        }
-
-        config = AppConfig()
-
-        with self.assertLogs(level="INFO") as logger:
-            from server.common.aws_secret_utils import handle_config_from_secret
-            # should not throw error
-            # "AttributeError: 'XConfig' object has no attribute 'x'"
-            handle_config_from_secret(config)
-
-            # should log 3 lines (one for each var set from a secret)
-            self.assertEqual(len(logger.output), 3)
-            self.assertIn('INFO:root:set app__flask_secret_key from secret', logger.output[0])
-            self.assertIn('INFO:root:set authentication__params_oauth__client_secret from secret', logger.output[1])
-            self.assertIn('INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret', logger.output[2])
-            self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
-            self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
-            self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
-
-    def test_api_base_url(self):
-
-        # test the api_base_url feature, and that it can contain a path
-        config = AppConfig()
-        backend_port = find_available_port("localhost", 10000)
-        config.update_server_config(
-            app__api_base_url=f"http://localhost:{backend_port}/additional/path",
-            multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset"
-        )
-
-        config.complete_config()
-
-        with test_server(["-p", str(backend_port)], app_config=config) as server:
-            session = requests.Session()
-            self.assertEqual(server, f"http://localhost:{backend_port}")
-            response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
-            self.assertEqual(response.status_code, 200)
-            data_config = response.json()
-            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
-
-            # test the health check at the correct url
-            response = session.get(f"{server}/additional/path/health")
-            assert response.json()["status"] == "pass"
-
-            # also check that the old URL still works.
-            # NOTE:  this old URL location will soon be deprecated, and when that happens
-            # this check can be removed.
-            response = session.get(f"{server}/health")
-            assert response.json()["status"] == "pass"
-
-    def test_configfile_with_specialization(self):
-        # test that per_dataset_config config load the default config, then the specialized config
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            configfile = os.path.join(tempdir, "config.yaml")
-            with open(configfile, "w") as fconfig:
-                config = """
-                server:
-                    multi_dataset:
-                        dataroot:
-                            test:
-                                base_url: test
-                                dataroot: fake_dataroot
-
-                dataset:
-                    user_annotations:
-                        enable: false
-                        type: hosted_tiledb_array
-                        hosted_tiledb_array:
-                            db_uri: fake_db_uri
-                            hosted_file_directory: fake_dir
-
-                per_dataset_config:
-                    test:
-                        user_annotations:
-                            enable: true
-                """
-                fconfig.write(config)
-
-            app_config = AppConfig()
-            app_config.update_from_config_file(configfile)
-
-            test_config = app_config.dataroot_config["test"]
-
-            # test config from default
-            self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
-            self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
-
-            # test config from specialization
-            self.assertTrue(test_config.user_annotations__enable)
-
-    def test_configfile_no_dataset_section(self):
-        # test a config file without a dataset section
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            configfile = os.path.join(tempdir, "config.yaml")
-            with open(configfile, "w") as fconfig:
-                config = """
-                server:
-                    multi_dataset:
-                        dataroot: test_dataroot
-
-                """
-                fconfig.write(config)
-
-            app_config = AppConfig()
-            app_config.update_from_config_file(configfile)
-            server_changes = app_config.server_config.changes_from_default()
-            dataset_changes = app_config.default_dataset_config.changes_from_default()
-            self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
-            self.assertEqual(dataset_changes, [])
-
-    def test_configfile_no_server_section(self):
-        # test a config file without a dataset section
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            configfile = os.path.join(tempdir, "config.yaml")
-            with open(configfile, "w") as fconfig:
-                config = """
-                dataset:
-                    user_annotations:
-                        enable: false
-                """
-                fconfig.write(config)
-
-            app_config = AppConfig()
-            app_config.update_from_config_file(configfile)
-            server_changes = app_config.server_config.changes_from_default()
-            dataset_changes = app_config.default_dataset_config.changes_from_default()
-            self.assertEqual(server_changes, [])
-            self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])

server/test/unit/common/test_corpora.py

@@ -87,24 +87,17 @@ class CorporaRESTAPITest(unittest.TestCase):
     def setCorporaFields(cls, path):
         adata = anndata.read_h5ad(path)
         corpora_props = {
-            "version": {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
-                "corpora_schema_version": "1.0.0",
-                "corpora_encoding_version": "0.1.0"
-            },
             "title": "PBMC3K",
-            "contributors": json.dumps([
+            "contributors": json.dumps([{"name": "name"}]),
-                {"name": "name"}
+            "layer_descriptions": {"X": "raw counts"},
-            ]),
-            "layer_descriptions": {
-                "X": "raw counts"
-            },
             "organism": "human",
             "organism_ontology_term_id": "unknown",
             "project_name": "test project",
             "project_description": "test description",
-            "project_links": json.dumps([
+            "project_links": json.dumps(
-                {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}
+                [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
-            ]),
+            ),
             "default_embedding": "X_tsne",
         }
         adata.uns.update(corpora_props)

server/test/unit/common/test_writable_annotation.py

@@ -27,7 +27,7 @@ class auth(object):
 
 class WritableTileDBStoredAnnotationTest(unittest.TestCase):
     def setUp(self):
-        self.user_id = '1234'
+        self.user_id = "1234"
         self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
         self.data.dataset_config.user_annotations = self.annotations
         self.db = self.annotations.db
@@ -38,7 +38,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
         }
         self.fbs = make_fbs(self.test_dict)
         self.df = pd.DataFrame(self.test_dict)
-        self.app = Flask('fake_app')
+        self.app = Flask("fake_app")
         self.app.__setattr__("auth", auth)
 
     def tearDown(self):
@@ -65,8 +65,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
             self.annotations.write_labels(self.df, self.data)
             dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
             annotation = self.db.query_for_most_recent(
-                Annotation,
+                Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
-                [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
             )
             # retrieve tiledb array
             df = tiledb.open(annotation.tiledb_uri)
@@ -78,7 +77,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
 
     def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
         with self.app.test_request_context():
-            new_name = 'new_dataset/location'
+            new_name = "new_dataset/location"
             self.data.get_location = MagicMock(return_value=new_name)
             num_datasets = len(self.db.query([CellxGeneDataset]))
             self.annotation_put_fbs(self.fbs)
@@ -130,15 +129,14 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
             with self.assertRaises(KeyError):
                 self.annotation_put_fbs(fbs_bad)
 
-    @patch('server.common.annotations.hosted_tiledb.current_app')
+    @patch("server.common.annotations.hosted_tiledb.current_app")
     def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id):
-        mock_user_id.auth.get_user_id.return_value = '1234'
+        mock_user_id.auth.get_user_id.return_value = "1234"
         self.annotations.write_labels(self.df, self.data)
         # get uri
         dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
         annotation = self.db.query_for_most_recent(
-            Annotation,
+            Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
-            [Annotation.user_id == '1234', Annotation.dataset_id == str(dataset_id)]
         )
 
         df = tiledb.open(annotation.tiledb_uri)

server/test/unit/common/utils/test_cxg_generation_utils.py

@@ -8,8 +8,12 @@ import numpy as np
 import tiledb
 from pandas import Series, DataFrame
 
-from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array,
+from server.common.utils.cxg_generation_utils import (
-                                                      convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array)
+    convert_dictionary_to_cxg_group,
+    convert_dataframe_to_cxg_array,
+    convert_ndarray_to_cxg_dense_array,
+    convert_matrix_to_cxg_array,
+)
 
 PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
 
@@ -28,8 +32,9 @@ class TestCxgGenerationUtils(unittest.TestCase):
         dictionary_name = "favorite_desserts"
         expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
 
-        convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary,
+        convert_dictionary_to_cxg_group(
-                                        group_metadata_name=dictionary_name)
+            self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
+        )
 
         array = tiledb.open(expected_array_directory)
         actual_stored_metadata = dict(array.meta.items())
@@ -44,13 +49,16 @@ class TestCxgGenerationUtils(unittest.TestCase):
         random_dataframe_name = f"random_dataframe_{uuid4()}"
         random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
 
-        convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe,
+        convert_dataframe_to_cxg_array(
-                                       "int_category", tiledb.Ctx())
+            self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
+        )
 
         expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
         expected_array_metadata = {
-            "cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"},
+            "cxg_schema": json.dumps(
-                                      "index": "int_category"})}
+                {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
+            )
+        }
 
         actual_stored_dataframe_array = tiledb.open(expected_array_directory)
         actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
@@ -95,7 +103,7 @@ class TestCxgGenerationUtils(unittest.TestCase):
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[:, :][''].size == 0)
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
 
     def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
         matrix = np.zeros([3, 3])
@@ -110,10 +118,10 @@ class TestCxgGenerationUtils(unittest.TestCase):
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[0, 0][''] == 1)
+        self.assertTrue(actual_stored_array[0, 0][""] == 1)
-        self.assertTrue(actual_stored_array[1, 1][''] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
-        self.assertTrue(actual_stored_array[2, 2][''] == 2)
+        self.assertTrue(actual_stored_array[2, 2][""] == 2)
-        self.assertTrue(actual_stored_array[:, :][''].size == 3)
+        self.assertTrue(actual_stored_array[:, :][""].size == 3)
 
     def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
         matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -122,14 +130,15 @@ class TestCxgGenerationUtils(unittest.TestCase):
         # a matrix of zeros which is sparse.
         column_shift = np.ones((3, 2))
 
-        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
+        convert_matrix_to_cxg_array(
-                                    column_shift_for_sparse_encoding=column_shift)
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
 
         actual_stored_array = tiledb.open(matrix_name)
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[:, :][''].size == 0)
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
 
     def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
         matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -137,13 +146,14 @@ class TestCxgGenerationUtils(unittest.TestCase):
         # Only column shift the first column of ones.
         column_shift = np.array([[1, 0], [1, 0]])
 
-        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
+        convert_matrix_to_cxg_array(
-                                    column_shift_for_sparse_encoding=column_shift)
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
 
         actual_stored_array = tiledb.open(matrix_name)
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[0, 1][''] == 1)
+        self.assertTrue(actual_stored_array[0, 1][""] == 1)
-        self.assertTrue(actual_stored_array[1, 1][''] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
-        self.assertTrue(actual_stored_array[:, :][''].size == 2)
+        self.assertTrue(actual_stored_array[:, :][""].size == 2)

server/test/unit/common/utils/test_matrix_utils.py

@@ -6,7 +6,6 @@ from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_
 
 
 class TestMatrixUtils(unittest.TestCase):
-
     def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
         matrix = np.array([1, 2, 3])
 

server/test/unit/common/utils/test_sanitization_utils.py

@@ -4,7 +4,6 @@ from server.common.utils.sanitization_utils import sanitize_values_in_list, sani
 
 
 class TestSanitizationUtils(unittest.TestCase):
-
     def test__sanitize_values_in_list__not_strings_raises_exception(self):
         keys_to_sanitize = [1, 2, 3]
 

server/test/unit/common/utils/test_type_conversion_utils.py

@@ -5,12 +5,17 @@ from unittest.mock import patch
 import numpy as np
 from pandas import Series, DataFrame
 
-from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \
+from server.common.utils.type_conversion_utils import (
-    get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe, convert_pandas_series_to_numpy
+    can_cast_to_float32,
+    can_cast_to_int32,
+    get_dtype_of_array,
+    get_schema_type_hint_of_array,
+    get_dtypes_and_schemas_of_dataframe,
+    convert_pandas_series_to_numpy,
+)
 
 
 class TestTypeConversionUtils(unittest.TestCase):
-
     def test__can_cast_to_float32__string_is_false(self):
         array_to_convert = Series(data=["1", "2", "3"], dtype=str)
 
@@ -97,8 +102,9 @@ class TestTypeConversionUtils(unittest.TestCase):
         expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode]
 
         for test_type_index in range(len(types)):
-            with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}",
+            with self.subTest(
-                              i=test_type_index):
+                f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index
+            ):
                 array = Series(data=[], dtype=types[test_type_index])
                 self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
 
@@ -123,8 +129,9 @@ class TestTypeConversionUtils(unittest.TestCase):
         expected_dtypes = [np.float32, np.int32]
 
         for test_type_index in range(len(types)):
-            with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}",
+            with self.subTest(
-                              i=test_type_index):
+                f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index
+            ):
                 array = Series(data=[], dtype=types[test_type_index])
                 self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
 
@@ -141,8 +148,9 @@ class TestTypeConversionUtils(unittest.TestCase):
         expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}]
 
         for test_type_index in range(len(types)):
-            with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}",
+            with self.subTest(
-                              i=test_type_index):
+                f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index
+            ):
                 array = Series(data=[], dtype=types[test_type_index])
                 self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
 
@@ -160,8 +168,9 @@ class TestTypeConversionUtils(unittest.TestCase):
 
         for test_type_index in range(len(types)):
             with self.subTest(
-                    f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
+                f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
-                    i=test_type_index):
+                i=test_type_index,
+            ):
                 array = Series(data=[], dtype=types[test_type_index])
                 self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
 
@@ -171,8 +180,10 @@ class TestTypeConversionUtils(unittest.TestCase):
         dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
 
         expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode}
-        expected_schema_type_hints_dict = {"float_array": {"type": "float32"},
+        expected_schema_type_hints_dict = {
-                                           "category_array": {"type": "categorical", "categories": ["a", "b"]}}
+            "float_array": {"type": "float32"},
+            "category_array": {"type": "categorical", "categories": ["a", "b"]},
+        }
 
         actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe)
 
@@ -201,5 +212,6 @@ class TestTypeConversionUtils(unittest.TestCase):
         with self.assertLogs(level="ERROR") as logger:
             convert_pandas_series_to_numpy(int_series, np.int32)
 
-            self.assertIn("Cannot convert a pandas Series object to an integer dtype if it contains NaNs",
+            self.assertIn(
-                          logger.output[0])
+                "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0]
+            )

server/test/unit/converters/test_h5ad_data_file.py

@@ -16,7 +16,6 @@ PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
 
 
 class TestH5ADDataFile(unittest.TestCase):
-
     def setUp(self):
         self.sample_anndata = self._create_sample_anndata_dataset()
         self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata)
@@ -40,8 +39,12 @@ class TestH5ADDataFile(unittest.TestCase):
 
     def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self):
         with self.assertLogs(level="WARN") as logger:
-            H5ADDataFile(self.sample_h5ad_filename, dataset_title="My Awesome Dataset",
+            H5ADDataFile(
-                         dataset_about="http://www.awesomedataset.com", use_corpora_schema=False)
+                self.sample_h5ad_filename,
+                dataset_title="My Awesome Dataset",
+                dataset_about="http://www.awesomedataset.com",
+                use_corpora_schema=False,
+            )
 
         self.assertIn("will override any metadata that is extracted", logger.output[0])
 
@@ -49,10 +52,12 @@ class TestH5ADDataFile(unittest.TestCase):
         h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
 
         self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all())
-        self.assertEqual(h5ad_file.anndata.obs.sort_index(inplace=True),
+        self.assertEqual(
-                         self.sample_anndata.obs.sort_index(inplace=True))
+            h5ad_file.anndata.obs.sort_index(inplace=True), self.sample_anndata.obs.sort_index(inplace=True)
-        self.assertEqual(h5ad_file.anndata.var.sort_index(inplace=True),
+        )
-                         self.sample_anndata.var.sort_index(inplace=True))
+        self.assertEqual(
+            h5ad_file.anndata.var.sort_index(inplace=True), self.sample_anndata.var.sort_index(inplace=True)
+        )
 
         for key in h5ad_file.anndata.obsm.keys():
             self.assertIn(key, self.sample_anndata.obsm.keys())
@@ -73,8 +78,12 @@ class TestH5ADDataFile(unittest.TestCase):
         self.assertIn("name_0", h5ad_file.var.columns)
 
     def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self):
-        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
+        h5ad_file = H5ADDataFile(
-                                 obs_index_column_name="float_category", vars_index_column_name="int_category")
+            self.sample_h5ad_filename,
+            use_corpora_schema=False,
+            obs_index_column_name="float_category",
+            vars_index_column_name="int_category",
+        )
 
         self.assertNotIn("name_0", h5ad_file.obs.columns)
         self.assertNotIn("name_0", h5ad_file.var.columns)
@@ -82,15 +91,23 @@ class TestH5ADDataFile(unittest.TestCase):
     def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self):
 
         with self.assertRaises(Exception) as exception_context:
-            H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
+            H5ADDataFile(
-                         obs_index_column_name="float_category", vars_index_column_name="bool_category")
+                self.sample_h5ad_filename,
+                use_corpora_schema=False,
+                obs_index_column_name="float_category",
+                vars_index_column_name="bool_category",
+            )
 
         self.assertIn("Please prepare data to contain unique values", str(exception_context.exception))
 
     def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self):
         with self.assertRaises(Exception) as exception_context:
-            H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
+            H5ADDataFile(
-                         obs_index_column_name="unknown_category", vars_index_column_name="i_dont_exist")
+                self.sample_h5ad_filename,
+                use_corpora_schema=False,
+                obs_index_column_name="unknown_category",
+                vars_index_column_name="i_dont_exist",
+            )
 
         self.assertIn("does not exist", str(exception_context.exception))
 
@@ -101,8 +118,9 @@ class TestH5ADDataFile(unittest.TestCase):
         self.assertEqual(h5ad_file.dataset_about, "www.link.com")
 
     def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self):
-        h5ad_file = H5ADDataFile(self.sample_h5ad_filename, dataset_about="override_about",
+        h5ad_file = H5ADDataFile(
-                                 dataset_title="override_title")
+            self.sample_h5ad_filename, dataset_about="override_about", dataset_title="override_title"
+        )
 
         self.assertEqual(h5ad_file.dataset_title, "override_title")
         self.assertEqual(h5ad_file.dataset_about, "override_about")
@@ -145,8 +163,11 @@ class TestH5ADDataFile(unittest.TestCase):
         remove(sparse_with_column_shift_filename)
 
     def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False):
-        expected_directories, expected_obs_files, expected_var_files = \
+        (
-            self._get_expected_generated_list_of_tiledb_files()
+            expected_directories,
+            expected_obs_files,
+            expected_var_files,
+        ) = self._get_expected_generated_list_of_tiledb_files()
 
         for directory in expected_directories:
             self.assertTrue(path.isdir(directory))
@@ -187,8 +208,18 @@ class TestH5ADDataFile(unittest.TestCase):
         var_files.append("bool_category.tdb")
         var_files.append("int_category.tdb")
 
-        return [metadata_directory, main_x_directory, overall_embedding_directory, specific_embedding_directory,
+        return (
-                obs_directory, var_directory], obs_files, var_files
+            [
+                metadata_directory,
+                main_x_directory,
+                overall_embedding_directory,
+                specific_embedding_directory,
+                obs_directory,
+                var_directory,
+            ],
+            obs_files,
+            var_files,
+        )
 
     def _write_anndata_to_file(self, anndata):
         temporary_filename = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}.h5ad"
@@ -204,7 +235,8 @@ class TestH5ADDataFile(unittest.TestCase):
         random_string_category = Series(data=["a", "b", "b"], dtype="category")
         random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32)
         obs_dataframe = DataFrame(
-            data={"string_category": random_string_category, "float_category": random_float_category})
+            data={"string_category": random_string_category, "float_category": random_float_category}
+        )
         obs = obs_dataframe
 
         # Create vars
@@ -230,6 +262,7 @@ class TestH5ADDataFile(unittest.TestCase):
 
         # Set project links to be a dictionary
         uns["project_links"] = json.dumps(
-            [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}])
+            [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}]
+        )
 
         return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns)

server/test/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -3,7 +3,7 @@ import json
 
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
 from server.test import PROJECT_ROOT
 
 

server/test/unit/data_common/test_matrix_loader.py

@@ -4,7 +4,7 @@ import tempfile
 import time
 import unittest
 
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
 from server.common.errors import DatasetAccessError
 from server.data_common.matrix_loader import MatrixDataCacheManager
 from server.test import FIXTURES_ROOT
@@ -38,7 +38,7 @@ class MatrixCacheTest(unittest.TestCase):
         result = {}
         for k, v in datasets.items():
             # filter out the dirname and the .cxg from the name
-            newk = int(k[1][len(dirname) + 1: -4])
+            newk = int(k[1][len(dirname) + 1 : -4])
             result[newk] = v
 
         return result

server/test/unit/eb/test_eb.py

@@ -3,7 +3,7 @@ import tempfile
 import requests
 import subprocess
 from server.test import PROJECT_ROOT, FIXTURES_ROOT
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
 from contextlib import contextmanager
 import time
 
@@ -36,9 +36,7 @@ class Elastic_Beanstalk_Test(unittest.TestCase):
 
         c = AppConfig()
         # test that eb works
-        c.update_server_config(
+        c.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
-            multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame"
-        )
 
         c.complete_config()
         c.write_config(f"{tempdirname}/config.yaml")