From af3c6e1d8e853dc474b3ddf690b939d04ac7da5e Mon Sep 17 00:00:00 2001 From: Madison Dunitz Date: Tue, 29 Sep 2020 16:42:46 -0500 Subject: [PATCH] config refactor (#1854) * split out config * add tests for base and app config, refactor client config out of app config * refactor default config retrieval * create config test class and helper functions * move default_config into server to fix import issue --- Makefile | 3 +- server/__init__.py | 1 - server/app/app.py | 27 +- server/auth/__init__.py | 1 - server/auth/auth.py | 2 +- server/auth/auth_none.py | 1 - server/auth/auth_oauth.py | 13 +- server/cli/convert_to_cxg.py | 74 +- server/cli/launch.py | 71 +- server/cli/prepare.py | 26 +- server/cli/upgrade.py | 3 +- server/common/annotations/hosted_tiledb.py | 17 +- server/common/app_config.py | 963 ------------------ server/common/aws_secret_utils.py | 61 -- server/common/config/__init__.py | 66 ++ server/common/config/app_config.py | 183 ++++ server/common/config/base_config.py | 113 ++ server/common/config/client_config.py | 125 +++ server/common/config/dataset_config.py | 234 +++++ server/common/config/server_config.py | 390 +++++++ server/common/errors.py | 8 +- server/common/rest.py | 5 +- server/common/utils/cxg_generation_utils.py | 2 +- server/common/utils/matrix_utils.py | 11 +- server/common/utils/type_conversion_utils.py | 20 +- server/converters/h5ad_data_file.py | 16 +- server/data_common/data_adaptor.py | 20 +- server/db/cellxgene_orm.py | 7 +- server/db/db_utils.py | 10 +- server/{common => }/default_config.py | 1 - server/eb/app.py | 13 +- server/test/__init__.py | 13 +- server/test/fixtures/database/__init__.py | 26 +- .../test/fixtures/dataset_config_outline.py | 37 + server/test/fixtures/server_config_outline.py | 62 ++ server/test/performance/run_diffexp.py | 2 +- server/test/test_database/test_database.py | 37 +- server/test/unit/auth/test_auth.py | 14 +- server/test/unit/auth/test_oauth.py | 13 +- server/test/unit/cli/test_launch.py | 28 + server/test/unit/common/config/__init__.py | 246 +++++ .../unit/common/config/test_app_config.py | 140 +++ .../unit/common/config/test_base_config.py | 63 ++ .../unit/common/config/test_dataset_config.py | 260 +++++ .../unit/common/config/test_server_config.py | 335 ++++++ server/test/unit/common/test_api.py | 36 +- server/test/unit/common/test_app_config.py | 249 ----- server/test/unit/common/test_corpora.py | 19 +- .../unit/common/test_writable_annotation.py | 16 +- .../common/utils/test_cxg_generation_utils.py | 52 +- .../unit/common/utils/test_matrix_utils.py | 1 - .../common/utils/test_sanitization_utils.py | 1 - .../utils/test_type_conversion_utils.py | 42 +- .../unit/converters/test_h5ad_data_file.py | 75 +- .../test_anndata_adaptor_data_load.py | 2 +- .../unit/data_common/test_matrix_loader.py | 4 +- server/test/unit/eb/test_eb.py | 6 +- 57 files changed, 2667 insertions(+), 1599 deletions(-) delete mode 100644 server/common/app_config.py create mode 100644 server/common/config/__init__.py create mode 100644 server/common/config/app_config.py create mode 100644 server/common/config/base_config.py create mode 100644 server/common/config/client_config.py create mode 100644 server/common/config/dataset_config.py create mode 100644 server/common/config/server_config.py rename server/{common => }/default_config.py (99%) create mode 100644 server/test/fixtures/dataset_config_outline.py create mode 100644 server/test/fixtures/server_config_outline.py create mode 100644 server/test/unit/cli/test_launch.py create mode 100644 server/test/unit/common/config/__init__.py create mode 100644 server/test/unit/common/config/test_app_config.py create mode 100644 server/test/unit/common/config/test_base_config.py create mode 100644 server/test/unit/common/config/test_dataset_config.py create mode 100644 server/test/unit/common/config/test_server_config.py delete mode 100644 server/test/unit/common/test_app_config.py diff --git a/Makefile b/Makefile index 5b588b77..56dccf59 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,8 @@ lint: lint-server lint-client .PHONY: lint-server lint-server: - flake8 server + flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821' + .PHONY: lint-client lint-client: diff --git a/server/__init__.py b/server/__init__.py index 94238d9a..100b1874 100644 --- a/server/__init__.py +++ b/server/__init__.py @@ -1,6 +1,5 @@ import logging import sys - from server.common.utils.utils import import_plugins __version__ = "0.16.0" diff --git a/server/app/app.py b/server/app/app.py index 7e02b4e1..da167453 100644 --- a/server/app/app.py +++ b/server/app/app.py @@ -6,8 +6,17 @@ from urllib.parse import urlparse import hashlib import os -from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \ - send_from_directory +from flask import ( + Flask, + redirect, + current_app, + make_response, + render_template, + abort, + Blueprint, + request, + send_from_directory, +) from flask_restful import Api, Resource from server_timing import Timing as ServerTiming @@ -87,10 +96,7 @@ def dataset_index(url_dataroot=None, dataset=None): cache_manager = current_app.matrix_data_cache_manager with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor: data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}") - args = { - "SCRIPTS" : scripts, - "INLINE_SCRIPTS" : inline_scripts - } + args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts} return render_template("index.html", **args) except DatasetAccessError as e: @@ -417,8 +423,9 @@ class Server: for dataroot_dict in server_config.multi_dataset__dataroot.values(): url_dataroot = dataroot_dict["base_url"] bp_dataroot = Blueprint( - f"api_dataset_{url_dataroot}", __name__, - url_prefix=f"{api_path}/{url_dataroot}/" + api_version + f"api_dataset_{url_dataroot}", + __name__, + url_prefix=f"{api_path}/{url_dataroot}/" + api_version, ) dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot) self.app.register_blueprint(dataroot_resources.blueprint) @@ -433,7 +440,7 @@ class Server: f"/{url_dataroot}//static/", f"static_assets_{url_dataroot}", view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename), - methods=["GET"] + methods=["GET"], ) else: @@ -444,7 +451,7 @@ class Server: "/static/", "static_assets", view_func=lambda filename: send_from_directory("../common/web/static", filename), - methods=["GET"] + methods=["GET"], ) self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager diff --git a/server/auth/__init__.py b/server/auth/__init__.py index 1b33bebc..dfadadde 100644 --- a/server/auth/__init__.py +++ b/server/auth/__init__.py @@ -1,4 +1,3 @@ - # import the built in auth types so they can be registered import server.auth.auth_none # noqa: F401 diff --git a/server/auth/auth.py b/server/auth/auth.py index bb86ea64..145616cc 100644 --- a/server/auth/auth.py +++ b/server/auth/auth.py @@ -76,7 +76,7 @@ class AuthTypeFactory: @staticmethod def register(name, auth_type): - assert(issubclass(auth_type, AuthTypeBase)) + assert issubclass(auth_type, AuthTypeBase) AuthTypeFactory.auth_types[name] = auth_type @staticmethod diff --git a/server/auth/auth_none.py b/server/auth/auth_none.py index c482e3c4..61f82400 100644 --- a/server/auth/auth_none.py +++ b/server/auth/auth_none.py @@ -2,7 +2,6 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory class AuthTypeNone(AuthTypeBase): - def __init__(self, app_config): super().__init__() diff --git a/server/auth/auth_oauth.py b/server/auth/auth_oauth.py index a2b6077f..d7162318 100644 --- a/server/auth/auth_oauth.py +++ b/server/auth/auth_oauth.py @@ -97,8 +97,17 @@ class AuthTypeOAuth(AuthTypeClientBase): return valid_keys = { - "verify_signature", "verify_aud", "verify_iat", "verify_exp", "verify_nbf", "verify_iss", - "verify_sub", "verify_jti", "verify_at_hash", "leeway"} + "verify_signature", + "verify_aud", + "verify_iat", + "verify_exp", + "verify_nbf", + "verify_iss", + "verify_sub", + "verify_jti", + "verify_at_hash", + "leeway", + } keys = set(self.jwt_decode_options.keys()) unknown = keys - valid_keys if unknown: diff --git a/server/cli/convert_to_cxg.py b/server/cli/convert_to_cxg.py index 4e456c08..0cd67e3c 100644 --- a/server/cli/convert_to_cxg.py +++ b/server/cli/convert_to_cxg.py @@ -9,26 +9,24 @@ from server.converters.h5ad_data_file import H5ADDataFile name="convert", short_help="Converts an H5AD dataset to the CXG format.", help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format " - "that has performance and access characteristics amenable to a multi-dataset, multi-user serving " - "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as " - "usually with the generated CXG file.", + "that has performance and access characteristics amenable to a multi-dataset, multi-user serving " + "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as " + "usually with the generated CXG file.", ) @click.argument( - "input-file", - nargs=1, - type=click.Path(exists=True, dir_okay=False), + "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False), ) @click.option( "-o", "--output-directory", help="Name of the output CXG directory. If not provided, will default to be the input filename with a " - "CXG extension.", + "CXG extension.", ) @click.option( "-b", "--backed", help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, " - "but will use less memory.", + "but will use less memory.", default=False, show_default=True, is_flag=True, @@ -37,29 +35,33 @@ from server.converters.h5ad_data_file import H5ADDataFile "-t", "--title", help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, " - "the dataset title will be the filename.", + "the dataset title will be the filename.", ) @click.option( "-a", "--about", help="A fully qualified URL that provides more information about the dataset and will be included as " - "metadata about the CXG file.", + "metadata about the CXG file.", ) @click.option( "-s", "--sparse-threshold", help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X " - "array of the dataset will be sparse. Since the default value is 0.0, the default will be to " - "convert to dense array.", + "array of the dataset will be sparse. Since the default value is 0.0, the default will be to " + "convert to dense array.", default=0.0, show_default=True, ) -@click.option("--obs-names", - help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of " - "the one designated by the dataframe generated-index.") -@click.option("--var-names", - help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of " - "the one designated by the dataframe generated-index.") +@click.option( + "--obs-names", + help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of " + "the one designated by the dataframe generated-index.", +) +@click.option( + "--var-names", + help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of " + "the one designated by the dataframe generated-index.", +) @click.option( "--disable-custom-colors", help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.", @@ -70,8 +72,8 @@ from server.converters.h5ad_data_file import H5ADDataFile @click.option( "--disable-corpora-schema", help="When set, conversion process will neither extract nor store Corpora schema information. See " - "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more " - "information.", + "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more " + "information.", default=False, show_default=True, is_flag=True, @@ -85,30 +87,32 @@ from server.converters.h5ad_data_file import H5ADDataFile ) @click.help_option("--help", "-h", help="Show this message and exit.") def convert_to_cxg( - input_file, - output_directory, - backed, - title, - about, - sparse_threshold, - obs_names, - var_names, - disable_custom_colors, - disable_corpora_schema, - overwrite, + input_file, + output_directory, + backed, + title, + about, + sparse_threshold, + obs_names, + var_names, + disable_custom_colors, + disable_corpora_schema, + overwrite, ): """ Convert a dataset file into CXG. """ - h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names, - use_corpora_schema=not disable_corpora_schema) + h5ad_data_file = H5ADDataFile( + input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema + ) # Get the directory that will hold all the CXG files cxg_output_container = get_output_directory(input_file, output_directory, overwrite) - h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold, - convert_anndata_colors_to_cxg_colors=not disable_custom_colors) + h5ad_data_file.to_cxg( + cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors + ) def get_output_directory(input_filename, output_directory, should_overwrite): diff --git a/server/cli/launch.py b/server/cli/launch.py index d46161d4..a05fd6a1 100644 --- a/server/cli/launch.py +++ b/server/cli/launch.py @@ -3,15 +3,14 @@ import functools import logging import sys import webbrowser -from os import devnull - +import os import click from flask_compress import Compress from flask_cors import CORS +from server.default_config import default_config from server.app.app import Server -from server.common.app_config import AppConfig -from server.common.default_config import default_config +from server.common.config.app_config import AppConfig from server.common.errors import DatasetAccessError, ConfigurationError from server.common.utils.utils import sort_options @@ -33,7 +32,7 @@ def annotation_args(func): multiple=False, metavar="", help="CSV file to initialize editing of existing annotations; will be altered in-place. " - "Incompatible with --annotations-dir.", + "Incompatible with --annotations-dir.", ) @click.option( "--annotations-dir", @@ -42,7 +41,7 @@ def annotation_args(func): multiple=False, metavar="", help="Directory of where to save output annotations; filename will be specified in the application. " - "Incompatible with --annotations-file.", + "Incompatible with --annotations-file.", ) @click.option( "--experimental-annotations-ontology", @@ -170,7 +169,7 @@ def server_args(func): default=DEFAULT_CONFIG.server_config.app__debug, show_default=True, help="Run in debug mode. This is helpful for cellxgene developers, " - "or when you want more information about an error condition.", + "or when you want more information about an error condition.", ) @click.option( "--verbose", @@ -203,7 +202,7 @@ def server_args(func): multiple=True, metavar="", help="Additional script files to include in HTML page. If not specified, " - "no additional script files will be included.", + "no additional script files will be included.", show_default=False, ) @functools.wraps(func) @@ -223,7 +222,7 @@ def launch_args(func): default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot, metavar="", help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)" - " to folder containing H5AD and/or CXG datasets.", + " to folder containing H5AD and/or CXG datasets.", hidden=True, ) # TODO, unhide when dataroot is supported) @click.argument("datapath", required=False, metavar="") @@ -307,32 +306,32 @@ class CliLaunchServer(Server): ) @launch_args def launch( - datapath, - dataroot, - verbose, - debug, - open_browser, - port, - host, - embedding, - obs_names, - var_names, - max_category_items, - disable_custom_colors, - diffexp_lfc_cutoff, - title, - scripts, - about, - disable_annotations, - annotations_file, - annotations_dir, - backed, - disable_diffexp, - experimental_annotations_ontology, - experimental_annotations_ontology_obo, - experimental_enable_reembedding, - config_file, - dump_default_config, + datapath, + dataroot, + verbose, + debug, + open_browser, + port, + host, + embedding, + obs_names, + var_names, + max_category_items, + disable_custom_colors, + diffexp_lfc_cutoff, + title, + scripts, + about, + disable_annotations, + annotations_file, + annotations_dir, + backed, + disable_diffexp, + experimental_annotations_ontology, + experimental_annotations_ontology_obo, + experimental_enable_reembedding, + config_file, + dump_default_config, ): """Launch the cellxgene data viewer. This web app lets you explore single-cell expression data. @@ -443,7 +442,7 @@ def launch( click.echo("[cellxgene] Type CTRL-C at any time to exit.") if not server_config.app__verbose: - f = open(devnull, "w") + f = open(os.devnull, "w") sys.stdout = f try: diff --git a/server/cli/prepare.py b/server/cli/prepare.py index df535db0..67735172 100644 --- a/server/cli/prepare.py +++ b/server/cli/prepare.py @@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options default=False, is_flag=True, help="Do not run quality control metrics. By default cellxgene runs them " - "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", + "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).", ) @click.option( "--make-obs-names-unique/--no-make-obs-names-unique", @@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options ) @click.help_option("--help", "-h", help="Show this message and exit.") def prepare( - data, - embedding, - recipe, - output, - plotting, - sparse, - overwrite, - set_obs_names, - set_var_names, - skip_qc, - make_obs_names_unique, - make_var_names_unique, + data, + embedding, + recipe, + output, + plotting, + sparse, + overwrite, + set_obs_names, + set_var_names, + skip_qc, + make_obs_names_unique, + make_var_names_unique, ): """ Preprocess data for use with cellxgene. diff --git a/server/cli/upgrade.py b/server/cli/upgrade.py index 953f92ba..222d7e81 100644 --- a/server/cli/upgrade.py +++ b/server/cli/upgrade.py @@ -10,7 +10,8 @@ from .. import __version__ SEMVER_FORMAT = re.compile( r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[" r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(" - r"?:\.[0-9a-zA-Z-]+)*))?$") + r"?:\.[0-9a-zA-Z-]+)*))?$" +) def log_upgrade_check(): diff --git a/server/common/annotations/hosted_tiledb.py b/server/common/annotations/hosted_tiledb.py index f3df75ed..54f7dac2 100644 --- a/server/common/annotations/hosted_tiledb.py +++ b/server/common/annotations/hosted_tiledb.py @@ -31,7 +31,8 @@ class AnnotationsHostedTileDB(Annotations): unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names) if unsanitary_original_category_names: raise AnnotationCategoryNameError( - f"{unsanitary_original_category_names} are not valid category names, please resubmit") + f"{unsanitary_original_category_names} are not valid category names, please resubmit" + ) def is_safe_collection_name(self, name): """ @@ -68,11 +69,11 @@ class AnnotationsHostedTileDB(Annotations): index_dims = None schema_hints = json.loads(schema_hints) - if '__pandas_attribute_repr' in tileDBArray.meta: + if "__pandas_attribute_repr" in tileDBArray.meta: # backwards compatibility... unsure if necessary at this point - repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr']) - if '__pandas_index_dims' in tileDBArray.meta: - index_dims = json.loads(tileDBArray.meta['__pandas_index_dims']) + repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"]) + if "__pandas_index_dims" in tileDBArray.meta: + index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"]) data = tileDBArray[:] indexes = list() @@ -80,12 +81,12 @@ class AnnotationsHostedTileDB(Annotations): for col_name, col_val in data.items(): # If the column values are byte literals, decode them if isinstance(col_val[0], bytes): - col_val = [value.decode('utf-8') for value in col_val] + col_val = [value.decode("utf-8") for value in col_val] if schema_hints and col_name in schema_hints: type = schema_hints.get(col_name).get("type") if type and type == "categorical": - new_col = pd.Series(col_val, dtype='category') + new_col = pd.Series(col_val, dtype="category") data[col_name] = new_col elif repr_meta and col_name in repr_meta: new_col = pd.Series(col_val, dtype=repr_meta[col_name]) @@ -127,7 +128,7 @@ class AnnotationsHostedTileDB(Annotations): tiledb_uri=uri, user_id=user_id, dataset_id=str(dataset_id), - schema_hints=json.dumps(dataframe_schema_type_hints) + schema_hints=json.dumps(dataframe_schema_type_hints), ) if not df.empty: self.check_category_names(df) diff --git a/server/common/app_config.py b/server/common/app_config.py deleted file mode 100644 index 080e1892..00000000 --- a/server/common/app_config.py +++ /dev/null @@ -1,963 +0,0 @@ -import copy -import os -import sys -import warnings -from os.path import splitext, basename, isdir -from urllib.parse import urlparse, quote_plus - -import yaml -from flatten_dict import flatten, unflatten - -import server.compute.diffexp_cxg as diffexp_tiledb -import server.compute.scanpy -from server import display_version as cellxgene_display_version -from server.auth.auth import AuthTypeFactory -from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB -from server.common.annotations.local_file_csv import AnnotationsLocalFile -from server.common.data_locator import discover_s3_region_name -from server.common.default_config import get_default_config -from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure -from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available -from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType -from server.db.db_utils import DbUtils - -DEFAULT_SERVER_PORT = 5005 -# anything bigger than this will generate a special message -BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB - - -class AppFeature(object): - def __init__(self, path, available=False, method="POST", extra={}): - self.path = path - self.available = available - self.method = method - self.extra = extra - for k, v in extra.items(): - setattr(self, k, v) - - def todict(self): - d = dict(available=self.available, method=self.method, path=self.path) - d.update(self.extra) - return d - - -class AppConfig(object): - """AppConfig stores all the configuration for cellxgene. The configuration is divided into two main parts: - server attributes, and dataset attributes. The server_config contains attributes that refer to the server process - as a whole. The default_dataset_config referes to attributes that are associated with the features and - presentations of a dataset. The dataset config attributes can be overridden depending on the url by which the - dataset was accessed. These are stored in dataroot_config. - AppConfig has methods to initialize, modify, and access the configuration. - """ - - def __init__(self): - - # the default configuration (see default_config.py) - self.default_config = get_default_config() - # the server configuration - self.server_config = ServerConfig(self, self.default_config["server"]) - # the dataset config, unless overridden by an entry in dataroot_config - self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"]) - # a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot - # attribute of the server_config. - self.dataroot_config = {} - - # Set to true when config_completed is called - self.is_completed = False - - def get_dataset_config(self, dataroot_key): - if self.server_config.single_dataset__datapath: - return self.default_dataset_config - else: - return self.dataroot_config.get(dataroot_key, self.default_dataset_config) - - def check_config(self): - """Verify all the attributes have been checked""" - if not self.is_completed: - raise ConfigurationError("The configuration has not been completed") - self.server_config.check_config() - self.default_dataset_config.check_config() - for dataset_config in self.dataroot_config.values(): - dataset_config.check_config() - - def update_server_config(self, **kw): - self.server_config.update(**kw) - self.is_complete = False - - def update_default_dataset_config(self, **kw): - self.default_dataset_config.update(**kw) - # update all the other dataset configs, if any - for value in self.dataroot_config.values(): - value.update(**kw) - self.is_complete = False - - def update_from_config_file(self, config_file): - with open(config_file) as fyaml: - config = yaml.load(fyaml, Loader=yaml.FullLoader) - - if config.get("server"): - self.server_config.update_from_config(config["server"], "server") - if config.get("dataset"): - self.default_dataset_config.update_from_config(config["dataset"], "dataset") - - per_dataset_config = config.get("per_dataset_config", {}) - for key, dataroot_config in per_dataset_config.items(): - # first create and initialize the dataroot with the default config - self.add_dataroot_config(key, **config["dataset"]) - # then apply the per dataset configuration - self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}") - - self.is_complete = False - - def write_config(self, config_file): - """output the config to a yaml file""" - server = self.server_config.create_mapping(self.server_config.default_config) - dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) - config = dict(server={}, dataset={}) - for attrname in server.keys(): - config["server__" + attrname] = getattr(self.server_config, attrname) - for attrname in dataset.keys(): - config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname) - if self.dataroot_config: - config["per_dataset_config"] = {} - for dataroot_tag, dataroot_config in self.dataroot_config.items(): - dataset = dataroot_config.create_mapping(dataroot_config.default_config) - for attrname in dataset.keys(): - config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname) - - config = unflatten(config, splitter=lambda key: key.split("__")) - yaml.dump(config, open(config_file, "w")) - - def changes_from_default(self): - """Return all the attribute that are different from the default""" - diff_server = self.server_config.changes_from_default() - diff_dataset = self.default_dataset_config.changes_from_default() - diff = dict(server=diff_server, dataset=diff_dataset) - return diff - - def add_dataroot_config(self, dataroot_tag, **kw): - """Create a new dataset config object based on the default dataset config, and kw parameters""" - if dataroot_tag in self.dataroot_config: - raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}") - if type(self.server_config.multi_dataset__dataroot) != dict: - raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary") - if dataroot_tag not in self.server_config.multi_dataset__dataroot: - raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot") - - self.is_completed = False - self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"]) - flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) - config = {key: value[1] for key, value in flat_config.items()} - self.dataroot_config[dataroot_tag].update(**config) - self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag) - - def complete_config(self, messagefn=None): - """The configure options are checked, and any additional setup based on the config - parameters is done""" - - if messagefn is None: - def noop(message): - pass - - messagefn = noop - - # TODO: to give better error messages we can add a mapping between where each config - # attribute originated (e.g. command line argument or config file), then in the error - # messages we can give correct context for attributes with bad value. - context = dict(messagefn=messagefn) - - self.server_config.complete_config(context) - self.default_dataset_config.complete_config(context) - for dataroot_config in self.dataroot_config.values(): - dataroot_config.complete_config(context) - - self.is_completed = True - self.check_config() - - def get_matrix_data_cache_manager(self): - return self.server_config.matrix_data_cache_manager - - def is_multi_dataset(self): - return self.server_config.multi_dataset__dataroot is not None - - def get_title(self, data_adaptor): - return ( - self.server_config.single_dataset__title - if self.server_config.single_dataset__title - else data_adaptor.get_title() - ) - - def get_about(self, data_adaptor): - return ( - self.server_config.single_dataset__about - if self.server_config.single_dataset__about - else data_adaptor.get_about() - ) - - def get_client_config(self, data_adaptor): - """ - Return the configuration as required by the /config REST route - """ - - server_config = self.server_config - dataset_config = data_adaptor.dataset_config - annotation = dataset_config.user_annotations - auth = server_config.auth - - # FIXME The current set of config is not consistently presented: - # we have camalCase, hyphen-text, and underscore_text - - # make sure the configuration has been checked. - self.check_config() - - # features - features = [f.todict() for f in data_adaptor.get_features(annotation)] - - # display_names - title = self.get_title(data_adaptor) - about = self.get_about(data_adaptor) - - display_names = dict(engine=data_adaptor.get_name(), dataset=title) - - # library_versions - library_versions = {} - library_versions.update(data_adaptor.get_library_versions()) - library_versions["cellxgene"] = cellxgene_display_version - - # links - links = {"about-dataset": about} - - # parameters - parameters = { - "layout": dataset_config.embeddings__names, - "max-category-items": dataset_config.presentation__max_categories, - "obs_names": server_config.single_dataset__obs_names, - "var_names": server_config.single_dataset__var_names, - "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff, - "backed": server_config.adaptor__anndata_adaptor__backed, - "disable-diffexp": not dataset_config.diffexp__enable, - "enable-reembedding": dataset_config.embeddings__enable_reembedding, - "annotations": False, - "annotations_file": None, - "annotations_dir": None, - "annotations_cell_ontology_enabled": False, - "annotations_cell_ontology_obopath": None, - "annotations_cell_ontology_terms": None, - "custom_colors": dataset_config.presentation__custom_colors, - "diffexp-may-be-slow": False, - "about_legal_tos": dataset_config.app__about_legal_tos, - "about_legal_privacy": dataset_config.app__about_legal_privacy, - } - - # corpora dataset_props - # TODO/Note: putting info from the dataset into the /config is not ideal. - # However, it is definitely not part of /schema, and we do not have a top-level - # route for data properties. Consider creating one at some point. - corpora_props = data_adaptor.get_corpora_props() - if corpora_props and "default_embedding" in corpora_props: - default_embedding = corpora_props["default_embedding"] - if isinstance(default_embedding, str) and default_embedding.startswith("X_"): - default_embedding = default_embedding[2:] # drop X_ prefix - if default_embedding in data_adaptor.get_embedding_names(): - parameters["default_embedding"] = default_embedding - - data_adaptor.update_parameters(parameters) - if annotation: - annotation.update_parameters(parameters, data_adaptor) - - # gather it all together - c = {} - config = c["config"] = {} - config["features"] = features - config["displayNames"] = display_names - config["library_versions"] = library_versions - config["links"] = links - config["parameters"] = parameters - config["corpora_props"] = corpora_props - config["limits"] = { - "column_request_max": server_config.limits__column_request_max, - "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, - } - - if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): - config["authentication"] = { - "requires_client_login": auth.requires_client_login(), - } - if auth.requires_client_login(): - config["authentication"].update({ - "login": auth.get_login_url(data_adaptor), - "logout": auth.get_logout_url(data_adaptor), - }) - - return c - - def get_client_userinfo(self, data_adaptor): - """ - Return the userinfo as required by the /userinfo REST route - """ - - server_config = self.server_config - dataset_config = data_adaptor.dataset_config - auth = server_config.auth - - # make sure the configuration has been checked. - self.check_config() - - if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): - userinfo = {} - userinfo["userinfo"] = { - "is_authenticated": auth.is_user_authenticated(), - "username": auth.get_user_name(), - "user_id": auth.get_user_id(), - "email": auth.get_user_email() - } - return userinfo - else: - return None - - -class BaseConfig(object): - """This class handles the mechanics of updating and checking attributes. - Derived classes are expected to store the actual attributes""" - - def __init__(self, app_config, default_config, dictval_cases={}): - # reference back to the app_config - self.app_config = app_config - # the complete set of attribute and their default values (unflattened) - self.default_config = default_config - # attributes where the value may be a dict (and therefore are not flattened) - self.dictval_cases = dictval_cases - # used to make sure every attribute value is checked - self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()} - - def create_mapping(self, config): - """Create a mapping from attribute names to (location in the config tree, value)""" - dc = copy.deepcopy(config) - mapping = {} - - # special cases where the value could be a dict. - # If its value is not None, the entry is added to the mapping, and not included - # in the flattening below. - for dictval_case in self.dictval_cases: - cur = dc - for part in dictval_case[:-1]: - cur = cur.get(part, {}) - val = cur.get(dictval_case[-1]) - if val is not None: - key = "__".join(dictval_case) - mapping[key] = (dictval_case, val) - del cur[dictval_case[-1]] - - flat_config = flatten(dc) - for key, value in flat_config.items(): - # name of the attribute - attr = "__".join(key) - mapping[attr] = (key, value) - - return mapping - - def check_attr(self, attrname, vtype): - val = getattr(self, attrname) - if type(vtype) in (list, tuple): - if type(val) not in vtype: - tnames = ",".join([x.__name__ for x in vtype]) - raise ConfigurationError( - f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}" - ) - else: - if type(val) != vtype: - raise ConfigurationError( - f"Invalid type for attribute: {attrname}, " - f"expected type {vtype.__name__}, got {type(val).__name__}" - ) - - self.attr_checked[attrname] = True - - def check_config(self): - mapping = self.create_mapping(self.default_config) - for key in mapping.keys(): - if not self.attr_checked[key]: - raise ConfigurationError(f"The attr '{key}' has not been checked") - - def update(self, **kw): - for key, value in kw.items(): - if not hasattr(self, key): - raise ConfigurationError(f"unknown config parameter {key}.") - try: - if type(value) == tuple: - # convert tuple values to list values - value = list(value) - setattr(self, key, value) - except KeyError: - raise ConfigurationError(f"Unable to set config parameter {key}.") - - self.attr_checked[key] = False - - def update_from_config(self, config, prefix): - mapping = self.create_mapping(config) - for attr, (key, value) in mapping.items(): - if not hasattr(self, attr): - raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}") - try: - setattr(self, attr, value) - except KeyError: - raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}") - - self.attr_checked[attr] = False - - def changes_from_default(self): - """Return all the attribute that are different from the default""" - mapping = self.create_mapping(self.default_config) - diff = [] - for attrname, (key, defval) in mapping.items(): - curval = getattr(self, attrname) - if curval != defval: - diff.append((attrname, curval, defval)) - return diff - - -class ServerConfig(BaseConfig): - """Manages the config attribute associated with the server.""" - - def __init__(self, app_config, default_config): - dictval_cases = [ - ("app", "csp_directives"), - ("authentication", "params_oauth", "cookie"), - ("authentication", "params_oauth", "jwt_decode_options"), - ("adaptor", "cxg_adaptor", "tiledb_ctx"), - ("multi_dataset", "dataroot"), - ] - super().__init__(app_config, default_config, dictval_cases) - - dc = default_config - try: - self.app__verbose = dc["app"]["verbose"] - self.app__debug = dc["app"]["debug"] - self.app__host = dc["app"]["host"] - self.app__port = dc["app"]["port"] - self.app__open_browser = dc["app"]["open_browser"] - self.app__force_https = dc["app"]["force_https"] - self.app__flask_secret_key = dc["app"]["flask_secret_key"] - self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"] - self.app__server_timing_headers = dc["app"]["server_timing_headers"] - self.app__csp_directives = dc["app"]["csp_directives"] - self.app__api_base_url = dc["app"]["api_base_url"] - self.app__web_base_url = dc["app"]["web_base_url"] - - self.authentication__type = dc["authentication"]["type"] - self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][ - "oauth_api_base_url" - ] - self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"] - self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"] - self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][ - "jwt_decode_options"] - self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"] - self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"] - - self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"] - self.multi_dataset__index = dc["multi_dataset"]["index"] - self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"] - self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"] - self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"] - - self.single_dataset__datapath = dc["single_dataset"]["datapath"] - self.single_dataset__obs_names = dc["single_dataset"]["obs_names"] - self.single_dataset__var_names = dc["single_dataset"]["var_names"] - self.single_dataset__about = dc["single_dataset"]["about"] - self.single_dataset__title = dc["single_dataset"]["title"] - - self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"] - self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"] - self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"] - - self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"] - - self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"] - self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"] - - self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"] - self.limits__column_request_max = dc["limits"]["column_request_max"] - - except KeyError as e: - raise ConfigurationError(f"Unexpected config: {str(e)}") - - # The matrix data cache manager is created during the complete_config and stored here. - self.matrix_data_cache_manager = None - - # The authentication object - self.auth = None - - def complete_config(self, context): - self.handle_app(context) - self.handle_data_source(context) - self.handle_authentication(context) - self.handle_data_locator(context) - self.handle_adaptor(context) # may depend on data_locator - self.handle_single_dataset(context) # may depend on adaptor - self.handle_multi_dataset(context) # may depend on adaptor - self.handle_diffexp(context) - self.handle_limits(context) - - self.check_config() - - def handle_app(self, context): - self.check_attr("app__verbose", bool) - self.check_attr("app__debug", bool) - self.check_attr("app__host", str) - self.check_attr("app__port", (type(None), int)) - self.check_attr("app__open_browser", bool) - self.check_attr("app__force_https", bool) - self.check_attr("app__flask_secret_key", (type(None), str)) - self.check_attr("app__generate_cache_control_headers", bool) - self.check_attr("app__server_timing_headers", bool) - self.check_attr("app__csp_directives", (type(None), dict)) - self.check_attr("app__api_base_url", (type(None), str)) - self.check_attr("app__web_base_url", (type(None), str)) - - if self.app__port: - try: - if not is_port_available(self.app__host, self.app__port): - raise ConfigurationError( - f"The port selected {self.app__port} is in use, please configure an open port." - ) - except OverflowError: - raise ConfigurationError(f"Invalid port: {self.app__port}") - else: - try: - default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT)) - except ValueError: - raise ConfigurationError( - "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT") - ) - try: - self.app__port = find_available_port(self.app__host, default_server_port) - except OverflowError: - raise ConfigurationError(f"Invalid port: {default_server_port}") - - if self.app__debug: - context["messagefn"]("in debug mode, setting verbose=True and open_browser=False") - self.app__verbose = True - self.app__open_browser = False - else: - warnings.formatwarning = custom_format_warning - - if not self.app__verbose: - sys.tracebacklimit = 0 - - # secret key: - # first, from CXG_SECRET_KEY environment variable - # second, from config file - self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key) - - # CSP Directives are a dict of string: list(string) or string: string - if self.app__csp_directives is not None: - for k, v in self.app__csp_directives.items(): - if not isinstance(k, str): - raise ConfigurationError("CSP directive names must be a string.") - if isinstance(v, list): - for policy in v: - if not isinstance(policy, str): - raise ConfigurationError("CSP directive value must be a string or list of strings.") - elif not isinstance(v, str): - raise ConfigurationError("CSP directive value must be a string or list of strings.") - - if self.app__web_base_url is None: - self.app__web_base_url = self.app__api_base_url - - def handle_authentication(self, context): - self.check_attr("authentication__type", (type(None), str)) - - # oauth - ptypes = str if self.authentication__type == "oauth" else (type(None), str) - self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes) - self.check_attr("authentication__params_oauth__client_id", ptypes) - self.check_attr("authentication__params_oauth__client_secret", ptypes) - self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict)) - self.check_attr("authentication__params_oauth__session_cookie", bool) - - if self.authentication__params_oauth__session_cookie: - self.check_attr("authentication__params_oauth__cookie", (type(None), dict)) - else: - self.check_attr("authentication__params_oauth__cookie", dict) - # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable - # second, from config file - self.authentication__params__oauth__client_secret = os.environ.get( - "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret) - - self.auth = AuthTypeFactory.create(self.authentication__type, self) - if self.auth is None: - raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") - - def handle_data_locator(self, context): - self.check_attr("data_locator__s3__region_name", (type(None), bool, str)) - if self.data_locator__s3__region_name is True: - path = self.single_dataset__datapath or self.multi_dataset__dataroot - if type(path) == dict: - # if multi_dataset__dataroot is a dict, then use the first key - # that is in s3. NOTE: it is not supported to have dataroots - # in different regions. - paths = [val.get("dataroot") for val in path.values()] - for path in paths: - if path.startswith("s3://"): - break - if path.startswith("s3://"): - region_name = discover_s3_region_name(path) - if region_name is None: - raise ConfigurationError(f"Unable to discover s3 region name from {path}") - else: - region_name = None - self.data_locator__s3__region_name = region_name - - def handle_data_source(self, context): - self.check_attr("single_dataset__datapath", (str, type(None))) - self.check_attr("multi_dataset__dataroot", (type(None), dict, str)) - - if self.single_dataset__datapath is None: - if self.multi_dataset__dataroot is None: - # TODO: change the error message once dataroot is fully supported - raise ConfigurationError("missing datapath") - return - else: - if self.multi_dataset__dataroot is not None: - raise ConfigurationError("must supply only one of datapath or dataroot") - - def handle_single_dataset(self, context): - self.check_attr("single_dataset__datapath", (str, type(None))) - self.check_attr("single_dataset__title", (str, type(None))) - self.check_attr("single_dataset__about", (str, type(None))) - self.check_attr("single_dataset__obs_names", (str, type(None))) - self.check_attr("single_dataset__var_names", (str, type(None))) - - if self.single_dataset__datapath is None: - return - - # create the matrix data cache manager: - if self.matrix_data_cache_manager is None: - self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None) - - # preload this data set - matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config) - try: - matrix_data_loader.pre_load_validation() - except DatasetAccessError as e: - raise ConfigurationError(str(e)) - - file_size = matrix_data_loader.file_size() - file_basename = basename(self.single_dataset__datapath) - if file_size > BIG_FILE_SIZE_THRESHOLD: - context["messagefn"](f"Loading data from {file_basename}, this may take a while...") - else: - context["messagefn"](f"Loading data from {file_basename}.") - - if self.single_dataset__about: - - def url_check(url): - try: - result = urlparse(url) - if all([result.scheme, result.netloc]): - return True - else: - return False - except ValueError: - return False - - if not url_check(self.single_dataset__about): - raise ConfigurationError( - "Must provide an absolute URL for --about. (Example format: http://example.com)" - ) - - def handle_multi_dataset(self, context): - self.check_attr("multi_dataset__dataroot", (type(None), dict, str)) - self.check_attr("multi_dataset__index", (type(None), bool, str)) - self.check_attr("multi_dataset__allowed_matrix_types", list) - self.check_attr("multi_dataset__matrix_cache__max_datasets", int) - self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)) - - if self.multi_dataset__dataroot is None: - return - - if type(self.multi_dataset__dataroot) == str: - default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot) - self.multi_dataset__dataroot = dict(d=default_dict) - - for tag, dataroot_dict in self.multi_dataset__dataroot.items(): - if "base_url" not in dataroot_dict: - raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}") - if "dataroot" not in dataroot_dict: - raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}") - - base_url = dataroot_dict["base_url"] - - # sanity check for well formed base urls - bad = False - if type(base_url) != str: - bad = True - elif os.path.normpath(base_url) != base_url: - bad = True - else: - base_url_parts = base_url.split("/") - if [quote_plus(part) for part in base_url_parts] != base_url_parts: - bad = True - if ".." in base_url_parts: - bad = True - if bad: - raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}") - - # verify all the base_urls are unique - base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()] - if len(base_urls) > len(set(base_urls)): - raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique") - - # error checking - for mtype in self.multi_dataset__allowed_matrix_types: - try: - MatrixDataType(mtype) - except ValueError: - raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}') - - # create the matrix data cache manager: - if self.matrix_data_cache_manager is None: - self.matrix_data_cache_manager = MatrixDataCacheManager( - max_cached=self.multi_dataset__matrix_cache__max_datasets, - timelimit_s=self.multi_dataset__matrix_cache__timelimit_s, - ) - - def handle_diffexp(self, context): - self.check_attr("diffexp__alg_cxg__max_workers", (str, int)) - self.check_attr("diffexp__alg_cxg__cpu_multiplier", int) - self.check_attr("diffexp__alg_cxg__target_workunit", int) - - max_workers = self.diffexp__alg_cxg__max_workers - cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier - cpu_count = os.cpu_count() - max_workers = min(max_workers, cpu_multiplier * cpu_count) - diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit) - - def handle_adaptor(self, context): - # cxg - self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict) - regionkey = "vfs.s3.region" - if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx: - if type(self.data_locator__s3__region_name) == str: - self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name - - from server.data_cxg.cxg_adaptor import CxgAdaptor - - CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx) - - # anndata - self.check_attr("adaptor__anndata_adaptor__backed", bool) - - def handle_limits(self, context): - self.check_attr("limits__diffexp_cellcount_max", (type(None), int)) - self.check_attr("limits__column_request_max", (type(None), int)) - - def exceeds_limit(self, limit_name, value): - limit_value = getattr(self, "limits__" + limit_name, None) - if limit_value is None: # disabled - return False - return value > limit_value - - def get_api_base_url(self): - if self.app__api_base_url == "local": - return f"http://{self.app__host}:{self.app__port}" - if self.app__api_base_url and self.app__api_base_url.endswith("/"): - return self.app__api_base_url[:-1] - return self.app__api_base_url - - def get_web_base_url(self): - if self.app__web_base_url == "local": - return f"http://{self.app__host}:{self.app__port}" - if self.app__web_base_url is None: - return self.get_api_base_url() - if self.app__web_base_url.endswith("/"): - return self.app__web_base_url[:-1] - return self.api__web_base_url - - -class DatasetConfig(BaseConfig): - """Manages the config attribute associated with a dataset.""" - - def __init__(self, tag, app_config, default_config): - super().__init__(app_config, default_config) - self.tag = tag - dc = default_config - try: - self.app__scripts = dc["app"]["scripts"] - self.app__inline_scripts = dc["app"]["inline_scripts"] - self.app__about_legal_tos = dc["app"]["about_legal_tos"] - self.app__about_legal_privacy = dc["app"]["about_legal_privacy"] - self.app__authentication_enable = dc["app"]["authentication_enable"] - - self.presentation__max_categories = dc["presentation"]["max_categories"] - self.presentation__custom_colors = dc["presentation"]["custom_colors"] - - self.user_annotations__enable = dc["user_annotations"]["enable"] - self.user_annotations__type = dc["user_annotations"]["type"] - self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"] - self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"] - self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"] - self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"] - self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"] - self.user_annotations__hosted_tiledb_array__hosted_file_directory = \ - dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ] # noqa E501 - - self.embeddings__names = dc["embeddings"]["names"] - self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"] - - self.diffexp__enable = dc["diffexp"]["enable"] - self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"] - self.diffexp__top_n = dc["diffexp"]["top_n"] - - except KeyError as e: - raise ConfigurationError(f"Unexpected config: {str(e)}") - - # The annotation object is created during complete_config and stored here. - self.user_annotations = None - - def complete_config(self, context): - self.handle_app(context) - self.handle_presentation(context) - self.handle_user_annotations(context) - self.handle_embeddings(context) - self.handle_diffexp(context) - - def handle_app(self, context): - self.check_attr("app__scripts", list) - self.check_attr("app__inline_scripts", list) - self.check_attr("app__about_legal_tos", (type(None), str)) - self.check_attr("app__about_legal_privacy", (type(None), str)) - self.check_attr("app__authentication_enable", bool) - - # scripts can be string (filename) or dict (attributes). Convert string to dict. - scripts = [] - for s in self.app__scripts: - if isinstance(s, str): - scripts.append({"src": s}) - elif isinstance(s, dict) and isinstance(s["src"], str): - scripts.append(s) - else: - raise ConfigurationError("Scripts must be string or dict") - self.app__scripts = scripts - - def handle_presentation(self, context): - self.check_attr("presentation__max_categories", int) - self.check_attr("presentation__custom_colors", bool) - - def handle_user_annotations(self, context): - self.check_attr("user_annotations__enable", bool) - self.check_attr("user_annotations__type", str) - self.check_attr("user_annotations__local_file_csv__directory", (type(None), str)) - self.check_attr("user_annotations__local_file_csv__file", (type(None), str)) - self.check_attr("user_annotations__ontology__enable", bool) - self.check_attr("user_annotations__ontology__obo_location", (type(None), str)) - self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str)) - self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)) - - if self.user_annotations__enable: - server_config = self.app_config.server_config - if not self.app__authentication_enable: - raise ConfigurationError("user annotations requires authentication to be enabled") - if not server_config.auth.is_valid_authentication_type(): - auth_type = server_config.authentication__type - raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") - - # TODO, replace this with a factory pattern once we have more than one way - # to do annotations. currently only local_file_csv - if self.user_annotations__type == "local_file_csv": - dirname = self.user_annotations__local_file_csv__directory - filename = self.user_annotations__local_file_csv__file - - if filename is not None and dirname is not None: - raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.") - - if filename is not None: - lf_name, lf_ext = splitext(filename) - if lf_ext and lf_ext != ".csv": - raise ConfigurationError(f"annotation file type must be .csv: {filename}") - - if dirname is not None and not isdir(dirname): - try: - os.mkdir(dirname) - except OSError: - raise ConfigurationError("Unable to create directory specified by --annotations-dir") - - self.user_annotations = AnnotationsLocalFile(dirname, filename) - - # if the user has specified a fixed label file, go ahead and validate it - # so that we can remove errors early in the process. - server_config = self.app_config.server_config - if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file: - with server_config.matrix_data_cache_manager.data_adaptor( - self.tag, server_config.single_dataset__datapath, self.app_config - ) as data_adaptor: - data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) - - if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: - try: - self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) - except OntologyLoadFailure as e: - raise ConfigurationError("Unable to load ontology terms\n" + str(e)) - elif self.user_annotations__type == "hosted_tiledb_array": - self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str) - self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str) - self.user_annotations = AnnotationsHostedTileDB( - directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory, - db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri), - ) - else: - raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array') - else: - if self.user_annotations__type == "local_file_csv": - dirname = self.user_annotations__local_file_csv__directory - filename = self.user_annotations__local_file_csv__file - if filename is not None: - context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.") - if dirname is not None: - context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.") - - if self.user_annotations__ontology__enable: - context["messagefn"]( - "Warning: --experimental-annotations-ontology" " ignored as annotations are disabled." - ) - if self.user_annotations__ontology__obo_location is not None: - context["messagefn"]( - "Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled." - ) - - def handle_embeddings(self, context): - self.check_attr("embeddings__names", list) - self.check_attr("embeddings__enable_reembedding", bool) - - server_config = self.app_config.server_config - if self.embeddings__enable_reembedding: - if server_config.single_dataset__datapath: - matrix_data_loader = MatrixDataLoader( - server_config.single_dataset__datapath, app_config=self.app_config - ) - if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD: - raise ConfigurationError("enable-reembedding is only supported with H5AD files.") - if server_config.adaptor__anndata_adaptor__backed: - raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.") - - try: - server.compute.scanpy.get_scanpy_module() - except NotImplementedError: - raise ConfigurationError("Please install scanpy to enable UMAP re-embedding") - - def handle_diffexp(self, context): - self.check_attr("diffexp__enable", bool) - self.check_attr("diffexp__lfc_cutoff", float) - self.check_attr("diffexp__top_n", int) - - server_config = self.app_config.server_config - if server_config.single_dataset__datapath: - with server_config.matrix_data_cache_manager.data_adaptor( - self.tag, server_config.single_dataset__datapath, self.app_config - ) as data_adaptor: - if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): - context["messagefn"]( - "CAUTION: due to the size of your dataset, " - "running differential expression may take longer or fail." - ) diff --git a/server/common/aws_secret_utils.py b/server/common/aws_secret_utils.py index 47f7690b..070ed160 100644 --- a/server/common/aws_secret_utils.py +++ b/server/common/aws_secret_utils.py @@ -1,72 +1,11 @@ import logging -import os -import sys import boto3 from flask import json -from server.common.data_locator import discover_s3_region_name from server.common.errors import SecretKeyRetrievalError -def handle_config_from_secret(app_config): - """Update configuration from the secret manager""" - secret_name = os.getenv("CXG_AWS_SECRET_NAME") - if not secret_name: - return - - # need to find the secret manager region. - # 1. from CXG_AWS_SECRET_REGION_NAME - # 2. discover from dataroot location (if on s3) - # 3. discover from config file location (if on s3) - secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME") - if secret_region_name is None: - secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) - if not secret_region_name: - from server.eb.app import config_file - secret_region_name = discover_s3_region_name(config_file) - if not secret_region_name: - logging.error("Could not determine the AWS Secret Manager region") - sys.exit(1) - - secrets = get_secret_key(secret_region_name, secret_name) - - if not secrets: - return - - server_attrs = ( - ("flask_secret_key", "app__flask_secret_key"), - ("oauth_client_secret", "authentication__params_oauth__client_secret"), - ) - default_dataset_attrs = ( - ("db_uri", "user_annotations__hosted_tiledb_array__db_uri"), - ) - - # update server configuration attributes - for key, attr in server_attrs: - cur_val = getattr(app_config.server_config, attr) - if cur_val: - continue - - # replace the attr with the secret if it is not set - val = secrets.get(key) - if val: - logging.info(f"set {attr} from secret") - app_config.update_server_config(**{attr : val}) - - # update default dataset configuration attributes - for key, attr in default_dataset_attrs: - cur_val = getattr(app_config.default_dataset_config, attr) - if cur_val: - continue - - # replace the attr with the secret if it is not set - val = secrets.get(key) - if val: - logging.info(f"set {attr} from secret") - app_config.update_default_dataset_config(**{attr : val}) - - def get_secret_key(region_name, secret_name): session = boto3.session.Session() client = session.client(service_name="secretsmanager", region_name=region_name) diff --git a/server/common/config/__init__.py b/server/common/config/__init__.py new file mode 100644 index 00000000..fb2439e7 --- /dev/null +++ b/server/common/config/__init__.py @@ -0,0 +1,66 @@ +import logging +import os +import sys + +from server.common.aws_secret_utils import get_secret_key +from server.common.data_locator import discover_s3_region_name + +DEFAULT_SERVER_PORT = 5005 +BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB + + +def handle_config_from_secret(app_config): + """Update configuration from the secret manager""" + secret_name = os.getenv("CXG_AWS_SECRET_NAME") + if not secret_name: + return + + # need to find the secret manager region. + # 1. from CXG_AWS_SECRET_REGION_NAME + # 2. discover from dataroot location (if on s3) + # 3. discover from config file location (if on s3) + secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME") + if secret_region_name is None: + secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) + if not secret_region_name: + from server.eb.app import config_file + + secret_region_name = discover_s3_region_name(config_file) + if not secret_region_name: + logging.error("Could not determine the AWS Secret Manager region") + sys.exit(1) + + secrets = get_secret_key(secret_region_name, secret_name) + + if not secrets: + return + + server_attrs = ( + ("flask_secret_key", "app__flask_secret_key"), + ("oauth_client_secret", "authentication__params_oauth__client_secret"), + ) + default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),) + + # update server configuration attributes + for key, attr in server_attrs: + cur_val = getattr(app_config.server_config, attr) + if cur_val: + continue + + # replace the attr with the secret if it is not set + val = secrets.get(key) + if val: + logging.info(f"set {attr} from secret") + app_config.update_server_config(**{attr: val}) + + # update default dataset configuration attributes + for key, attr in default_dataset_attrs: + cur_val = getattr(app_config.default_dataset_config, attr) + if cur_val: + continue + + # replace the attr with the secret if it is not set + val = secrets.get(key) + if val: + logging.info(f"set {attr} from secret") + app_config.update_default_dataset_config(**{attr: val}) diff --git a/server/common/config/app_config.py b/server/common/config/app_config.py new file mode 100644 index 00000000..422dcb8e --- /dev/null +++ b/server/common/config/app_config.py @@ -0,0 +1,183 @@ +import yaml +from flatten_dict import unflatten + +from server.default_config import get_default_config +from server.common.config.dataset_config import DatasetConfig +from server.common.config.server_config import ServerConfig +from server.common.errors import ConfigurationError + + +class AppConfig(object): + """ + AppConfig stores all the configuration for cellxgene. + AppConfig contains one or more DatasetConfig(s) and one ServerConfig. + The server_config contains attributes that refer to the server process as a whole. + The default_dataset_config refers to attributes that are associated with the features and + presentations of a dataset. + The dataset config attributes can be overridden depending on the url by which the + dataset was accessed. These are stored in dataroot_config. + AppConfig has methods to initialize, modify, and access the configuration. + """ + + def __init__(self): + + # the default configuration (see default_config.py) + # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as + # defaults within the config class? + self.default_config = get_default_config() + # the server configuration + self.server_config = ServerConfig(self, self.default_config["server"]) + # the dataset config, unless overridden by an entry in dataroot_config + self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"]) + # a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot + # attribute of the server_config. The default dataset config will apply to all datasets unless a different set + # of config vars was passed for a specific dataset under the multidataset config. For example: + """ + per_dataset_config: + d1: + user_annotations: + enable: false + d2: + user_annotations: + enable: true + """ + # dataroot config + self.dataroot_config = {} + + # Set to true when config_completed is called + self.is_completed = False + + def get_dataset_config(self, dataroot_key): + if self.server_config.single_dataset__datapath: + return self.default_dataset_config + else: + return self.dataroot_config.get(dataroot_key, self.default_dataset_config) + + def check_config(self): + """Verify all the attributes in the config have been type checked""" + if not self.is_completed: + raise ConfigurationError("The configuration has not been completed") + self.server_config.check_config() + self.default_dataset_config.check_config() + for dataset_config in self.dataroot_config.values(): + dataset_config.check_config() + + def update_server_config(self, **kw): + self.server_config.update(**kw) + self.is_complete = False + + def update_default_dataset_config(self, **kw): + self.default_dataset_config.update(**kw) + # update all the other dataset configs, if any + for value in self.dataroot_config.values(): + value.update(**kw) + self.is_complete = False + + def update_from_config_file(self, config_file): + try: + with open(config_file) as yml_file: + config = yaml.safe_load(yml_file) + except yaml.YAMLError as e: + raise ConfigurationError(f"The specified config file contained an error: {e}") + except OSError as e: + raise ConfigurationError(f"Issue retrieving the specified config file: {e}") + + if config.get("server"): + self.server_config.update_from_config(config["server"], "server") + if config.get("dataset"): + self.default_dataset_config.update_from_config(config["dataset"], "dataset") + + per_dataset_config = config.get("per_dataset_config", {}) + for key, dataroot_config in per_dataset_config.items(): + # first create and initialize the dataroot with the default config + self.add_dataroot_config(key, **config["dataset"]) + # then apply the per dataset configuration + self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}") + + self.is_complete = False + + def write_config(self, config_file): + """output the config to a yaml file""" + server = self.server_config.create_mapping(self.server_config.default_config) + dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) + config = dict(server={}, dataset={}) + for attrname in server.keys(): + config["server__" + attrname] = getattr(self.server_config, attrname) + for attrname in dataset.keys(): + config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname) + if self.dataroot_config: + config["per_dataset_config"] = {} + for dataroot_tag, dataroot_config in self.dataroot_config.items(): + dataset = dataroot_config.create_mapping(dataroot_config.default_config) + for attrname in dataset.keys(): + config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname) + + config = unflatten(config, splitter=lambda key: key.split("__")) + yaml.dump(config, open(config_file, "w")) + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + diff_server = self.server_config.changes_from_default() + diff_dataset = self.default_dataset_config.changes_from_default() + diff = dict(server=diff_server, dataset=diff_dataset) + return diff + + def add_dataroot_config(self, dataroot_tag, **kw): + """Create a new dataset config object based on the default dataset config, and kw parameters""" + if dataroot_tag in self.dataroot_config: + raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}") + if type(self.server_config.multi_dataset__dataroot) != dict: + raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary") + if dataroot_tag not in self.server_config.multi_dataset__dataroot: + raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot") + + self.is_completed = False + self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"]) + flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config) + config = {key: value[1] for key, value in flat_config.items()} + self.dataroot_config[dataroot_tag].update(**config) + self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag) + + def complete_config(self, messagefn=None): + """The configure options are checked, and any additional setup based on the config + parameters is done""" + + if messagefn is None: + + def noop(message): + pass + + messagefn = noop + + # TODO: to give better error messages we can add a mapping between where each config + # attribute originated (e.g. command line argument or config file), then in the error + # messages we can give correct context for attributes with bad value. + context = dict(messagefn=messagefn) + + self.server_config.complete_config(context) + self.default_dataset_config.complete_config(context) + for dataroot_config in self.dataroot_config.values(): + dataroot_config.complete_config(context) + + self.is_completed = True + self.check_config() + + def get_matrix_data_cache_manager(self): + return self.server_config.matrix_data_cache_manager + + def is_multi_dataset(self): + return self.server_config.multi_dataset__dataroot is not None + + def get_title(self, data_adaptor): + return ( + self.server_config.single_dataset__title + if self.server_config.single_dataset__title + else data_adaptor.get_title() + ) + + def get_about(self, data_adaptor): + return ( + self.server_config.single_dataset__about + if self.server_config.single_dataset__about + else data_adaptor.get_about() + ) diff --git a/server/common/config/base_config.py b/server/common/config/base_config.py new file mode 100644 index 00000000..a0b46f1b --- /dev/null +++ b/server/common/config/base_config.py @@ -0,0 +1,113 @@ +import copy + +from flatten_dict import flatten +from server.common.errors import ConfigurationError + + +class BaseConfig(object): + """ + This class handles the mechanics of updating and checking attributes. + Derived classes are expected to store the actual attributes + Currently DatasetConfig and ServerConfig both inherit from BaseConfig. + """ + + def __init__(self, app_config, default_config, dictval_cases={}): + # reference back to the app_config + self.app_config = app_config + # the complete set of attributes and their default values (unflattened) + self.default_config = default_config + # attributes where the value may be a dict (and therefore are not flattened) + self.dictval_cases = dictval_cases + # used to make sure every attribute value is checked + self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()} + + def create_mapping(self, config): + """ + Create a dictionary where the keys are the name of attributes (using double underscore convention) + For example: authentication__type + + The values are a tuple, + - the first item of the tuple is a tuple of path elements (location in config 'tree') + - the second item is the value of the config parameter + + For example: (('authentication', 'type'), 'session')) + """ + config_copy = copy.deepcopy(config) + mapping = {} + + # special cases where the value could be a dict. + # If its value is not None, the entry is added to the mapping, and not included + # in the flattening below. + for dictval_case in self.dictval_cases: + cur = config_copy + for part in dictval_case[:-1]: + cur = cur.get(part, {}) + val = cur.get(dictval_case[-1]) + if val is not None: + key = "__".join(dictval_case) + mapping[key] = (dictval_case, val) + del cur[dictval_case[-1]] + + flat_config = flatten(config_copy) + for key, value in flat_config.items(): + # name of the attribute + attr = "__".join(key) + mapping[attr] = (key, value) + + return mapping + + def validate_correct_type_of_configuration_attribute(self, attrname, vtype): + val = getattr(self, attrname) + if type(vtype) in (list, tuple): + if type(val) not in vtype: + tnames = ",".join([x.__name__ for x in vtype]) + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}" + ) + else: + if type(val) != vtype: + raise ConfigurationError( + f"Invalid type for attribute: {attrname}, " + f"expected type {vtype.__name__}, got {type(val).__name__}" + ) + + self.attr_checked[attrname] = True + + def check_config(self): + mapping = self.create_mapping(self.default_config) + for key in mapping.keys(): + if not self.attr_checked[key]: + raise ConfigurationError(f"The attr '{key}' has not been checked") + + def update(self, **kw): + for key, value in kw.items(): + if not hasattr(self, key): + raise ConfigurationError(f"unknown config parameter {key}.") + try: + if type(value) == tuple: + # convert tuple values to list values + value = list(value) + setattr(self, key, value) + except KeyError: + raise ConfigurationError(f"Unable to set config parameter {key}.") + + self.attr_checked[key] = False + + def update_from_config(self, config, prefix): + mapping = self.create_mapping(config) + for attr, (key, value) in mapping.items(): + if not hasattr(self, attr): + raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}") + setattr(self, attr, value) + + self.attr_checked[attr] = False + + def changes_from_default(self): + """Return all the attribute that are different from the default""" + mapping = self.create_mapping(self.default_config) + diff = [] + for attrname, (key, defval) in mapping.items(): + curval = getattr(self, attrname) + if curval != defval: + diff.append((attrname, curval, defval)) + return diff diff --git a/server/common/config/client_config.py b/server/common/config/client_config.py new file mode 100644 index 00000000..9fec9042 --- /dev/null +++ b/server/common/config/client_config.py @@ -0,0 +1,125 @@ +from server import display_version as cellxgene_display_version + + +def get_client_config(app_config, data_adaptor): + """ + Return the configuration as required by the /config REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + annotation = dataset_config.user_annotations + auth = server_config.auth + + # FIXME The current set of config is not consistently presented: + # we have camalCase, hyphen-text, and underscore_text + + # make sure the configuration has been checked. + app_config.check_config() + + # features + features = [f.todict() for f in data_adaptor.get_features(annotation)] + + # display_names + title = app_config.get_title(data_adaptor) + about = app_config.get_about(data_adaptor) + + display_names = dict(engine=data_adaptor.get_name(), dataset=title) + + # library_versions + library_versions = {} + library_versions.update(data_adaptor.get_library_versions()) + library_versions["cellxgene"] = cellxgene_display_version + + # links + links = {"about-dataset": about} + + # parameters + parameters = { + "layout": dataset_config.embeddings__names, + "max-category-items": dataset_config.presentation__max_categories, + "obs_names": server_config.single_dataset__obs_names, + "var_names": server_config.single_dataset__var_names, + "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff, + "backed": server_config.adaptor__anndata_adaptor__backed, + "disable-diffexp": not dataset_config.diffexp__enable, + "enable-reembedding": dataset_config.embeddings__enable_reembedding, + "annotations": False, + "annotations_file": None, + "annotations_dir": None, + "annotations_cell_ontology_enabled": False, + "annotations_cell_ontology_obopath": None, + "annotations_cell_ontology_terms": None, + "custom_colors": dataset_config.presentation__custom_colors, + "diffexp-may-be-slow": False, + "about_legal_tos": dataset_config.app__about_legal_tos, + "about_legal_privacy": dataset_config.app__about_legal_privacy, + } + + # corpora dataset_props + # TODO/Note: putting info from the dataset into the /config is not ideal. + # However, it is definitely not part of /schema, and we do not have a top-level + # route for data properties. Consider creating one at some point. + corpora_props = data_adaptor.get_corpora_props() + if corpora_props and "default_embedding" in corpora_props: + default_embedding = corpora_props["default_embedding"] + if isinstance(default_embedding, str) and default_embedding.startswith("X_"): + default_embedding = default_embedding[2:] # drop X_ prefix + if default_embedding in data_adaptor.get_embedding_names(): + parameters["default_embedding"] = default_embedding + + data_adaptor.update_parameters(parameters) + if annotation: + annotation.update_parameters(parameters, data_adaptor) + + # gather it all together + client_config = {} + config = client_config["config"] = {} + config["features"] = features + config["displayNames"] = display_names + config["library_versions"] = library_versions + config["links"] = links + config["parameters"] = parameters + config["corpora_props"] = corpora_props + config["limits"] = { + "column_request_max": server_config.limits__column_request_max, + "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max, + } + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + config["authentication"] = { + "requires_client_login": auth.requires_client_login(), + } + if auth.requires_client_login(): + config["authentication"].update( + { + # Todo why are these stored on the data_adaptor? + "login": auth.get_login_url(data_adaptor), + "logout": auth.get_logout_url(data_adaptor), + } + ) + + return client_config + + +def get_client_userinfo(app_config, data_adaptor): + """ + Return the userinfo as required by the /userinfo REST route + """ + + server_config = app_config.server_config + dataset_config = data_adaptor.dataset_config + auth = server_config.auth + + # make sure the configuration has been checked. + app_config.check_config() + + if dataset_config.app__authentication_enable and auth.is_valid_authentication_type(): + userinfo = {} + userinfo["userinfo"] = { + "is_authenticated": auth.is_user_authenticated(), + "username": auth.get_user_name(), + "user_id": auth.get_user_id(), + "email": auth.get_user_email(), + } + return userinfo diff --git a/server/common/config/dataset_config.py b/server/common/config/dataset_config.py new file mode 100644 index 00000000..8c8231ac --- /dev/null +++ b/server/common/config/dataset_config.py @@ -0,0 +1,234 @@ +import os +from os.path import splitext, isdir + +from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB +from server.common.annotations.local_file_csv import AnnotationsLocalFile +from server.common.config.base_config import BaseConfig +from server.common.errors import ConfigurationError, OntologyLoadFailure +from server.compute.scanpy import get_scanpy_module +from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType +from server.db.db_utils import DbUtils + + +class DatasetConfig(BaseConfig): + """Manages the config attribute associated with a dataset.""" + + def __init__(self, tag, app_config, default_config): + super().__init__(app_config, default_config) + self.tag = tag + try: + self.app__scripts = default_config["app"]["scripts"] + self.app__inline_scripts = default_config["app"]["inline_scripts"] + self.app__about_legal_tos = default_config["app"]["about_legal_tos"] + self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"] + self.app__authentication_enable = default_config["app"]["authentication_enable"] + + self.presentation__max_categories = default_config["presentation"]["max_categories"] + self.presentation__custom_colors = default_config["presentation"]["custom_colors"] + + self.user_annotations__enable = default_config["user_annotations"]["enable"] + self.user_annotations__type = default_config["user_annotations"]["type"] + self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][ + "directory" + ] # noqa E501 + self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"] + self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"] + self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][ + "obo_location" + ] # noqa E501 + self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][ + "hosted_tiledb_array" + ][ + "db_uri" + ] # noqa E501 + self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][ + "hosted_tiledb_array" + ][ + "hosted_file_directory" + ] # noqa E501 + + self.embeddings__names = default_config["embeddings"]["names"] + self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"] + + self.diffexp__enable = default_config["diffexp"]["enable"] + self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"] + self.diffexp__top_n = default_config["diffexp"]["top_n"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + # The annotation object is created during complete_config and stored here. + self.user_annotations = None + + def complete_config(self, context): + self.handle_app() + self.handle_presentation() + self.handle_user_annotations(context) + self.handle_embeddings() + self.handle_diffexp(context) + + def handle_app(self): + self.validate_correct_type_of_configuration_attribute("app__scripts", list) + self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list) + self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool) + + # scripts can be string (filename) or dict (attributes). Convert string to dict. + scripts = [] + for script in self.app__scripts: + try: + if isinstance(script, str): + scripts.append({"src": script}) + elif isinstance(script, dict) and isinstance(script["src"], str): + scripts.append(script) + else: + raise Exception + except Exception as e: + raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}") + + self.app__scripts = scripts + + def handle_presentation(self): + self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int) + self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool) + + def handle_user_annotations(self, context): + self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool) + self.validate_correct_type_of_configuration_attribute("user_annotations__type", str) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__directory", (type(None), str) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute( + "user_annotations__local_file_csv__file", (type(None), str) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__ontology__obo_location", (type(None), str) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute( + "user_annotations__hosted_tiledb_array__db_uri", (type(None), str) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute( + "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str) + ) # noqa E501 + if self.user_annotations__enable: + server_config = self.app_config.server_config + if not self.app__authentication_enable: + raise ConfigurationError("user annotations requires authentication to be enabled") + if not server_config.auth.is_valid_authentication_type(): + auth_type = server_config.authentication__type + raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations") + + if self.user_annotations__type == "local_file_csv": + self.handle_local_file_csv_annotations() + elif self.user_annotations__type == "hosted_tiledb_array": + self.handle_hosted_tiledb_annotations() + else: + raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array') + if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location: + try: + self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location) + except OntologyLoadFailure as e: + raise ConfigurationError("Unable to load ontology terms\n" + str(e)) + else: + self.check_annotation_config_vars_not_set(context) + + def handle_local_file_csv_annotations(self): + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + if filename is not None and dirname is not None: + raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.") + + if filename is not None: + lf_name, lf_ext = splitext(filename) + if lf_ext and lf_ext != ".csv": + raise ConfigurationError(f"annotation file type must be .csv: {filename}") + + if dirname is not None and not isdir(dirname): + try: + os.mkdir(dirname) + except OSError: + raise ConfigurationError("Unable to create directory specified by --annotations-dir") + + self.user_annotations = AnnotationsLocalFile(dirname, filename) + + # if the user has specified a fixed label file, go ahead and validate it + # so that we can remove errors early in the process. + server_config = self.app_config.server_config + if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file: + with server_config.matrix_data_cache_manager.data_adaptor( + self.tag, server_config.single_dataset__datapath, self.app_config + ) as data_adaptor: + data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor)) + + def handle_hosted_tiledb_annotations(self): + self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str) + self.validate_correct_type_of_configuration_attribute( + "user_annotations__hosted_tiledb_array__hosted_file_directory", str + ) # noqa E501 + self.user_annotations = AnnotationsHostedTileDB( + directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory, + db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri), + ) + + def check_annotation_config_vars_not_set(self, context): + if self.user_annotations__type is not None: + dirname = self.user_annotations__local_file_csv__directory + filename = self.user_annotations__local_file_csv__file + db_uri = self.user_annotations__hosted_tiledb_array__db_uri + hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory + if filename is not None: + context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.") + if dirname is not None: + context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.") + if db_uri is not None: + context["messagefn"]("Warning: db_uri ignored as annotations are disabled.") + if hosted_file_dirname is not None: + context["messagefn"]( + "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled." + ) + + if self.user_annotations__ontology__enable: + context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.") + if self.user_annotations__ontology__obo_location is not None: + context["messagefn"]( + "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled." + ) + + def handle_embeddings(self): + self.validate_correct_type_of_configuration_attribute("embeddings__names", list) + self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool) + + server_config = self.app_config.server_config + if self.embeddings__enable_reembedding: + if server_config.single_dataset__datapath: + matrix_data_loader = MatrixDataLoader( + server_config.single_dataset__datapath, app_config=self.app_config + ) + if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD: + raise ConfigurationError("enable-reembedding is only supported with H5AD files.") + if server_config.adaptor__anndata_adaptor__backed: + raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.") + + try: + get_scanpy_module() + except NotImplementedError: + # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported + raise ConfigurationError("Please install scanpy to enable UMAP re-embedding") + + def handle_diffexp(self, context): + self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool) + self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float) + self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int) + + server_config = self.app_config.server_config + if server_config.single_dataset__datapath: + with server_config.matrix_data_cache_manager.data_adaptor( + self.tag, server_config.single_dataset__datapath, self.app_config + ) as data_adaptor: + if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False): + context["messagefn"]( + "CAUTION: due to the size of your dataset, " + "running differential expression may take longer or fail." + ) diff --git a/server/common/config/server_config.py b/server/common/config/server_config.py new file mode 100644 index 00000000..508b3d4f --- /dev/null +++ b/server/common/config/server_config.py @@ -0,0 +1,390 @@ +import os +import sys +import warnings +from os.path import basename +from urllib.parse import urlparse, quote_plus + +from server.auth.auth import AuthTypeFactory +from server.common.config.base_config import BaseConfig +from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD +from server.common.errors import ConfigurationError, DatasetAccessError +from server.common.data_locator import discover_s3_region_name +from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning +from server.compute import diffexp_cxg as diffexp_tiledb +from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType + + +class ServerConfig(BaseConfig): + """Manages the config attribute associated with the server.""" + + def __init__(self, app_config, default_config): + dictval_cases = [ + ("app", "csp_directives"), + ("authentication", "params_oauth", "cookie"), + ("authentication", "params_oauth", "jwt_decode_options"), + ("adaptor", "cxg_adaptor", "tiledb_ctx"), + ("multi_dataset", "dataroot"), + ] + super().__init__(app_config, default_config, dictval_cases) + + try: + self.app__verbose = default_config["app"]["verbose"] + self.app__debug = default_config["app"]["debug"] + self.app__host = default_config["app"]["host"] + self.app__port = default_config["app"]["port"] + self.app__open_browser = default_config["app"]["open_browser"] + self.app__force_https = default_config["app"]["force_https"] + self.app__flask_secret_key = default_config["app"]["flask_secret_key"] + self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"] + self.app__server_timing_headers = default_config["app"]["server_timing_headers"] + self.app__csp_directives = default_config["app"]["csp_directives"] + self.app__api_base_url = default_config["app"]["api_base_url"] + self.app__web_base_url = default_config["app"]["web_base_url"] + + self.authentication__type = default_config["authentication"]["type"] + self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][ + "oauth_api_base_url" + ] # noqa E501 + self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"] + self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][ + "client_secret" + ] # noqa E501 + self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][ + "jwt_decode_options" + ] # noqa E501 + self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][ + "session_cookie" + ] # noqa E501 + self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"] + + self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"] + self.multi_dataset__index = default_config["multi_dataset"]["index"] + self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"] + self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][ + "max_datasets" + ] # noqa E501 + self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][ + "timelimit_s" + ] # noqa E501 + + self.single_dataset__datapath = default_config["single_dataset"]["datapath"] + self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"] + self.single_dataset__var_names = default_config["single_dataset"]["var_names"] + self.single_dataset__about = default_config["single_dataset"]["about"] + self.single_dataset__title = default_config["single_dataset"]["title"] + + self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"] + self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"] + self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"] + + self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"] + + self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"] + self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"] + + self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"] + self.limits__column_request_max = default_config["limits"]["column_request_max"] + + except KeyError as e: + raise ConfigurationError(f"Unexpected config: {str(e)}") + + # The matrix data cache manager is created during the complete_config and stored here. + self.matrix_data_cache_manager = None + + # The authentication object + self.auth = None + + def complete_config(self, context): + self.handle_app(context) + self.handle_data_source() + self.handle_authentication() + self.handle_data_locator() + self.handle_adaptor() # may depend on data_locator + self.handle_single_dataset(context) # may depend on adaptor + self.handle_multi_dataset() # may depend on adaptor + self.handle_diffexp() + self.handle_limits() + + self.check_config() + + def handle_app(self, context): + self.validate_correct_type_of_configuration_attribute("app__verbose", bool) + self.validate_correct_type_of_configuration_attribute("app__debug", bool) + self.validate_correct_type_of_configuration_attribute("app__host", str) + self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("app__open_browser", bool) + self.validate_correct_type_of_configuration_attribute("app__force_https", bool) + self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool) + self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool) + self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict)) + self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str)) + self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str)) + + if self.app__port: + try: + if not is_port_available(self.app__host, self.app__port): + raise ConfigurationError( + f"The port selected {self.app__port} is in use, please configure an open port." + ) + except OverflowError: + raise ConfigurationError(f"Invalid port: {self.app__port}") + else: + try: + default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT)) + except ValueError: + raise ConfigurationError( + "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT") + ) + try: + self.app__port = find_available_port(self.app__host, default_server_port) + except OverflowError: + raise ConfigurationError(f"Invalid port: {default_server_port}") + + if self.app__debug: + context["messagefn"]("in debug mode, setting verbose=True and open_browser=False") + self.app__verbose = True + self.app__open_browser = False + else: + warnings.formatwarning = custom_format_warning + + if not self.app__verbose: + sys.tracebacklimit = 0 + + # secret key: + # first, from CXG_SECRET_KEY environment variable + # second, from config file + self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key) + + # CSP Directives are a dict of string: list(string) or string: string + if self.app__csp_directives is not None: + for k, v in self.app__csp_directives.items(): + if not isinstance(k, str): + raise ConfigurationError("CSP directive names must be a string.") + if isinstance(v, list): + for policy in v: + if not isinstance(policy, str): + raise ConfigurationError("CSP directive value must be a string or list of strings.") + elif not isinstance(v, str): + raise ConfigurationError("CSP directive value must be a string or list of strings.") + + if self.app__web_base_url is None: + self.app__web_base_url = self.app__api_base_url + + def handle_authentication(self): + self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str)) + + # oauth + ptypes = str if self.authentication__type == "oauth" else (type(None), str) + self.validate_correct_type_of_configuration_attribute( + "authentication__params_oauth__oauth_api_base_url", ptypes + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes) + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes) + self.validate_correct_type_of_configuration_attribute( + "authentication__params_oauth__jwt_decode_options", (type(None), dict) + ) # noqa E501 + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool) + + if self.authentication__params_oauth__session_cookie: + self.validate_correct_type_of_configuration_attribute( + "authentication__params_oauth__cookie", (type(None), dict) + ) # noqa E501 + else: + self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict) + # secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable + # second, from config file + self.authentication__params_oauth__client_secret = os.environ.get( + "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret + ) + + self.auth = AuthTypeFactory.create(self.authentication__type, self) + if self.auth is None: + raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}") + + def handle_data_locator(self): + self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str)) + if self.data_locator__s3__region_name is True: + path = self.single_dataset__datapath or self.multi_dataset__dataroot + + if type(path) == dict: + # if multi_dataset__dataroot is a dict, then use the first key + # that is in s3. NOTE: it is not supported to have dataroots + # in different regions. + paths = [val.get("dataroot") for val in path.values()] + for path in paths: + if path.startswith("s3://"): + break + if path.startswith("s3://"): + region_name = discover_s3_region_name(path) + if region_name is None: + raise ConfigurationError(f"Unable to discover s3 region name from {path}") + else: + region_name = None + self.data_locator__s3__region_name = region_name + + def handle_data_source(self): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str)) + + if self.single_dataset__datapath and self.multi_dataset__dataroot: + raise ConfigurationError( + "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both" + ) + if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None: + raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets") + + def handle_single_dataset(self, context): + self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None))) + self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None))) + + if self.single_dataset__datapath is None: + return + + # create the matrix data cache manager: + if self.matrix_data_cache_manager is None: + self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None) + + # preload this data set + matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config) + try: + matrix_data_loader.pre_load_validation() + except DatasetAccessError as e: + raise ConfigurationError(str(e)) + + file_size = matrix_data_loader.file_size() + file_basename = basename(self.single_dataset__datapath) + if file_size > BIG_FILE_SIZE_THRESHOLD: + context["messagefn"](f"Loading data from {file_basename}, this may take a while...") + else: + context["messagefn"](f"Loading data from {file_basename}.") + + if self.single_dataset__about: + + def url_check(url): + try: + result = urlparse(url) + if all([result.scheme, result.netloc]): + return True + else: + return False + except ValueError: + return False + + if not url_check(self.single_dataset__about): + raise ConfigurationError( + "Must provide an absolute URL for --about. (Example format: http://example.com)" + ) + + def handle_multi_dataset(self): + self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str)) + self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str)) + self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list) + self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int) + self.validate_correct_type_of_configuration_attribute( + "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float) + ) # noqa E501 + + if self.multi_dataset__dataroot is None: + return + + if type(self.multi_dataset__dataroot) == str: + default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot) + self.multi_dataset__dataroot = dict(d=default_dict) + + for tag, dataroot_dict in self.multi_dataset__dataroot.items(): + if "base_url" not in dataroot_dict: + raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}") + if "dataroot" not in dataroot_dict: + raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}") + + base_url = dataroot_dict["base_url"] + + # sanity check for well formed base urls + bad = False + if type(base_url) != str: + bad = True + elif os.path.normpath(base_url) != base_url: + bad = True + else: + base_url_parts = base_url.split("/") + if [quote_plus(part) for part in base_url_parts] != base_url_parts: + bad = True + if ".." in base_url_parts: + bad = True + if bad: + raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}") + + # verify all the base_urls are unique + base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()] + if len(base_urls) > len(set(base_urls)): + raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique") + + # error checking + for mtype in self.multi_dataset__allowed_matrix_types: + try: + MatrixDataType(mtype) + except ValueError: + raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}') + + # create the matrix data cache manager: + if self.matrix_data_cache_manager is None: + self.matrix_data_cache_manager = MatrixDataCacheManager( + max_cached=self.multi_dataset__matrix_cache__max_datasets, + timelimit_s=self.multi_dataset__matrix_cache__timelimit_s, + ) + + def handle_diffexp(self): + self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int)) + self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int) + self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int) + + max_workers = self.diffexp__alg_cxg__max_workers + cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier + cpu_count = os.cpu_count() + max_workers = min(max_workers, cpu_multiplier * cpu_count) + diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit) + + def handle_adaptor(self): + # cxg + self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict) + regionkey = "vfs.s3.region" + if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx: + if type(self.data_locator__s3__region_name) == str: + self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name + + from server.data_cxg.cxg_adaptor import CxgAdaptor + + CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx) + + # anndata + self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool) + + def handle_limits(self): + self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int)) + self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int)) + + def exceeds_limit(self, limit_name, value): + limit_value = getattr(self, "limits__" + limit_name, None) + if limit_value is None: # disabled + return False + return value > limit_value + + def get_api_base_url(self): + if self.app__api_base_url == "local": + return f"http://{self.app__host}:{self.app__port}" + if self.app__api_base_url and self.app__api_base_url.endswith("/"): + return self.app__api_base_url[:-1] + return self.app__api_base_url + + def get_web_base_url(self): + if self.app__web_base_url == "local": + return f"http://{self.app__host}:{self.app__port}" + if self.app__web_base_url is None: + return self.get_api_base_url() + if self.app__web_base_url.endswith("/"): + return self.app__web_base_url[:-1] + return self.app__web_base_url diff --git a/server/common/errors.py b/server/common/errors.py index 5e8281e6..ca339bee 100644 --- a/server/common/errors.py +++ b/server/common/errors.py @@ -42,14 +42,14 @@ define_request_exception( define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota") define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format") define_request_exception( - "AuthenticationError", - "Raised when there is an authentication error", - default_status_code=HTTPStatus.UNAUTHORIZED) + "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED +) define_request_exception( "AnnotationCategoryNameError", "Raised when an annotation category name cant be saved", - default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY) + default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY, +) define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails") define_exception("ConfigurationError", "Raised when checking configuration errors") diff --git a/server/common/rest.py b/server/common/rest.py index be099f94..9baeca4c 100644 --- a/server/common/rest.py +++ b/server/common/rest.py @@ -6,6 +6,7 @@ from http import HTTPStatus from flask import make_response, jsonify, current_app, abort from werkzeug.urls import url_unquote +from server.common.config.client_config import get_client_config, get_client_userinfo from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg from server.common.errors import ( FilterError, @@ -117,12 +118,12 @@ def schema_get(data_adaptor): def config_get(app_config, data_adaptor): - config = app_config.get_client_config(data_adaptor) + config = get_client_config(app_config, data_adaptor) return make_response(jsonify(config), HTTPStatus.OK) def userinfo_get(app_config, data_adaptor): - config = app_config.get_client_userinfo(data_adaptor) + config = get_client_userinfo(app_config, data_adaptor) return make_response(jsonify(config), HTTPStatus.OK) diff --git a/server/common/utils/cxg_generation_utils.py b/server/common/utils/cxg_generation_utils.py index f29f4bd0..d3ec2b40 100644 --- a/server/common/utils/cxg_generation_utils.py +++ b/server/common/utils/cxg_generation_utils.py @@ -111,7 +111,7 @@ def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx): def convert_matrix_to_cxg_array( - matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None + matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None ): """ Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array` diff --git a/server/common/utils/matrix_utils.py b/server/common/utils/matrix_utils.py index 3eeddc10..60dfb19b 100644 --- a/server/common/utils/matrix_utils.py +++ b/server/common/utils/matrix_utils.py @@ -41,16 +41,19 @@ def is_matrix_sparse(matrix: np.ndarray, sparse_threshold): number_of_non_zero_elements += np.count_nonzero(matrix_subset) if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix: if end_row_index != total_number_of_rows: - percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / ( - end_row_index * total_number_of_columns) + percentage_of_non_zero_elements = ( + 100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns) + ) logging.info( f"Matrix is not sparse. Percentage of non-zero elements (estimate): " - f"{percentage_of_non_zero_elements:6.2f}") + f"{percentage_of_non_zero_elements:6.2f}" + ) else: percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements logging.info( f"Matrix is not sparse. Percentage of non-zero elements (exact): " - f"{percentage_of_non_zero_elements:6.2f}") + f"{percentage_of_non_zero_elements:6.2f}" + ) return False is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold diff --git a/server/common/utils/type_conversion_utils.py b/server/common/utils/type_conversion_utils.py index ac6b3fe4..4bc6bf88 100644 --- a/server/common/utils/type_conversion_utils.py +++ b/server/common/utils/type_conversion_utils.py @@ -9,8 +9,10 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame): schema_type_hints_by_column_name = {} for column_name, column_values in dataframe.items(): - dtypes_by_column_name[column_name], schema_type_hints_by_column_name[column_name] = \ - get_dtype_and_schema_of_array(column_values) + ( + dtypes_by_column_name[column_name], + schema_type_hints_by_column_name[column_name], + ) = get_dtype_and_schema_of_array(column_values) return dtypes_by_column_name, schema_type_hints_by_column_name @@ -24,8 +26,10 @@ def get_schema_type_hint_of_array(array: pd.Series): def get_dtype_and_schema_of_array(array: pd.Series): - return (get_dtype_from_dtype(array.dtype, array_values=array), - get_schema_type_hint_from_dtype(array.dtype, array_values=array)) + return ( + get_dtype_from_dtype(array.dtype, array_values=array), + get_schema_type_hint_from_dtype(array.dtype, array_values=array), + ) def get_dtype_from_dtype(dtype, array_values=None): @@ -133,9 +137,11 @@ def can_cast_to_int32(dtype, array_values=None): if np.can_cast(dtype, np.int32): return True ii32 = np.iinfo(np.int32) - if not ordered_array_values.empty and ( - ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \ - ordered_array_values.empty: + if ( + not ordered_array_values.empty + and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) + or ordered_array_values.empty + ): return True return False diff --git a/server/converters/h5ad_data_file.py b/server/converters/h5ad_data_file.py index c8223822..c79dd785 100644 --- a/server/converters/h5ad_data_file.py +++ b/server/converters/h5ad_data_file.py @@ -24,14 +24,14 @@ class H5ADDataFile: another format (currently just CXG is supported). """ def __init__( - self, - input_filename, - backed=False, - dataset_title=None, - dataset_about=None, - obs_index_column_name=None, - vars_index_column_name=None, - use_corpora_schema=True, + self, + input_filename, + backed=False, + dataset_title=None, + dataset_about=None, + obs_index_column_name=None, + vars_index_column_name=None, + use_corpora_schema=True, ): self.input_filename = input_filename self.backed = backed diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index 20dafdd0..3945ea7a 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -5,7 +5,7 @@ import numpy as np import pandas as pd from server_timing import Timing as ServerTiming -from server.common.app_config import AppFeature, AppConfig +from server.common.config.app_config import AppConfig from server.common.constants import Axis from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError from server.common.utils.utils import jsonify_numpy @@ -173,7 +173,7 @@ class DataAdaptor(metaclass=ABCMeta): mask = np.zeros((count,), dtype=np.bool) for i in filter: if type(i) == list: - mask[i[0]: i[1]] = True + mask[i[0] : i[1]] = True else: mask[i] = True return mask @@ -314,7 +314,7 @@ class DataAdaptor(metaclass=ABCMeta): top_n = self.dataset_config.diffexp__top_n if self.server_config.exceeds_limit( - "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) + "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B) ): raise ExceedsLimitError("Diffexp request exceeds max cell count limit") @@ -388,3 +388,17 @@ class DataAdaptor(metaclass=ABCMeta): except RuntimeError: lastmod = None return lastmod + + +class AppFeature(object): + def __init__(self, path, available=False, method="POST", extra={}): + self.path = path + self.available = available + self.method = method + self.extra = extra + [setattr(self, key, value) for key, value in extra.items()] + + def todict(self): + d = dict(available=self.available, method=self.method, path=self.path) + d.update(self.extra) + return d diff --git a/server/db/cellxgene_orm.py b/server/db/cellxgene_orm.py index f2209b63..2860f6b1 100644 --- a/server/db/cellxgene_orm.py +++ b/server/db/cellxgene_orm.py @@ -1,11 +1,6 @@ import uuid -from sqlalchemy import ( - Column, - DateTime, - ForeignKey, - String, - func, JSON) +from sqlalchemy import Column, DateTime, ForeignKey, String, func, JSON from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship diff --git a/server/db/db_utils.py b/server/db/db_utils.py index 1664303f..b3030bf2 100644 --- a/server/db/db_utils.py +++ b/server/db/db_utils.py @@ -42,9 +42,9 @@ class DbUtils: def get_or_create_dataset(self, dataset_name): try: - dataset_id = self.query( - table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name] - )[0].id + dataset_id = self.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])[ + 0 + ].id except IndexError: dataset_id = uuid.uuid4() dataset = CellxGeneDataset(id=dataset_id, name=dataset_name) @@ -54,9 +54,7 @@ class DbUtils: def get_or_create_user(self, user_id): try: - user_id = self.query( - table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id] - )[0].id + user_id = self.query(table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id])[0].id except IndexError: user = CellxGeneUser(id=user_id) self.session.add(user) diff --git a/server/common/default_config.py b/server/default_config.py similarity index 99% rename from server/common/default_config.py rename to server/default_config.py index 16e0c27d..88e96fe5 100644 --- a/server/common/default_config.py +++ b/server/default_config.py @@ -204,7 +204,6 @@ dataset: enable: true lfc_cutoff: 0.01 top_n: 10 - """ diff --git a/server/eb/app.py b/server/eb/app.py index 7a5e834c..0844f599 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -9,7 +9,7 @@ from flask import json import logging from flask_talisman import Talisman from flask_cors import CORS -from server.common.aws_secret_utils import handle_config_from_secret +from server.common.config import handle_config_from_secret from server.common.errors import SecretKeyRetrievalError @@ -26,7 +26,7 @@ SERVERDIR = os.path.dirname(os.path.realpath(__file__)) sys.path.append(SERVERDIR) try: - from server.common.app_config import AppConfig + from server.common.config.app_config import AppConfig from server.app.app import Server from server.common.data_locator import DataLocator, discover_s3_region_name except Exception: @@ -61,8 +61,7 @@ class WSGIServer(Server): csp = { "default-src": ["'self'"], "connect-src": ["'self'"] + extra_connect_src, - "script-src": ["'self'", "'unsafe-eval'"] - + obsolete_browser_script_hash + script_hashes, + "script-src": ["'self'", "'unsafe-eval'"] + obsolete_browser_script_hash + script_hashes, "style-src": ["'self'", "'unsafe-inline'"], "img-src": ["'self'", "https://cellxgene.cziscience.com", "data:"], "object-src": ["'none'"], @@ -104,7 +103,7 @@ class WSGIServer(Server): if len(script_hashes) == 0: logging.error("Content security policy hashes are missing, falling back to unsafe-inline policy") - return (script_hashes) + return script_hashes @staticmethod def compute_inline_csp_hashes(app, app_config): @@ -173,9 +172,7 @@ try: sys.exit(1) # features are unsupported in the current hosted server - app_config.update_default_dataset_config( - embeddings__enable_reembedding=False, - ) + app_config.update_default_dataset_config(embeddings__enable_reembedding=False,) app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],) app_config.complete_config(logging.info) diff --git a/server/test/__init__.py b/server/test/__init__.py index 8ed6e3e7..bbb1de0b 100644 --- a/server/test/__init__.py +++ b/server/test/__init__.py @@ -13,7 +13,8 @@ import requests from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB from server.common.annotations.local_file_csv import AnnotationsLocalFile -from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT +from server.common.config.app_config import AppConfig +from server.common.config import DEFAULT_SERVER_PORT from server.common.data_locator import DataLocator from server.common.utils.utils import find_available_port from server.data_common.fbs.matrix import encode_matrix_fbs @@ -33,8 +34,7 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType): data_locator = DataLocator(fname) config = AppConfig() config.update_server_config( - multi_dataset__dataroot=data_locator.path, - authentication__type="test", + multi_dataset__dataroot=data_locator.path, authentication__type="test", ) config.update_default_dataset_config( embeddings__names=["umap"], @@ -42,16 +42,13 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType): diffexp__lfc_cutoff=0.01, user_annotations__type="hosted_tiledb_array", user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432", - user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir + user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir, ) config.complete_config() data = MatrixDataLoader(data_locator.abspath()).open(config) - annotations = AnnotationsHostedTileDB( - tmp_dir, - DbUtils("postgresql://postgres:test_pw@localhost:5432"), - ) + annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),) return data, tmp_dir, annotations diff --git a/server/test/fixtures/database/__init__.py b/server/test/fixtures/database/__init__.py index 5a7fa984..0088f6d4 100644 --- a/server/test/fixtures/database/__init__.py +++ b/server/test/fixtures/database/__init__.py @@ -29,34 +29,26 @@ class TestDatabase: def _create_test_user(self): user = CellxGeneUser(id="test_user_id") - user2 = CellxGeneUser(id='1234') + user2 = CellxGeneUser(id="1234") self.db.session.add(user) self.db.session.add(user2) self.db.session.commit() def _create_test_dataset(self): - dataset = CellxGeneDataset( - name="test_dataset", - ) + dataset = CellxGeneDataset(name="test_dataset",) self.db.session.add(dataset) self.db.session.commit() def _create_test_annotation(self): - dataset = self.db.query([CellxGeneDataset], - [CellxGeneDataset.name == "test_dataset"], - )[0] - annotation = Annotation( - tiledb_uri="tiledb_uri", - user_id="test_user_id", - dataset_id=str(dataset.id) - ) + dataset = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == "test_dataset"],)[0] + annotation = Annotation(tiledb_uri="tiledb_uri", user_id="test_user_id", dataset_id=str(dataset.id)) self.db.session.add(annotation) self.db.session.commit() @staticmethod def get_random_string(): letters = string.ascii_lowercase - return ''.join(random.choice(letters) for i in range(12)) + return "".join(random.choice(letters) for i in range(12)) def _create_test_users(self, user_count: int = 10): users = [] @@ -80,10 +72,8 @@ class TestDatabase: for i in range(annotation_count): dataset = self.order_by_random(CellxGeneDataset) user = self.order_by_random(CellxGeneUser) - annotations.append(Annotation( - tiledb_uri=self.get_random_string(), - user_id=user.id, - dataset_id=str(dataset.id) - )) + annotations.append( + Annotation(tiledb_uri=self.get_random_string(), user_id=user.id, dataset_id=str(dataset.id)) + ) self.db.session.add_all(annotations) self.db.session.commit() diff --git a/server/test/fixtures/dataset_config_outline.py b/server/test/fixtures/dataset_config_outline.py new file mode 100644 index 00000000..a3d99d78 --- /dev/null +++ b/server/test/fixtures/dataset_config_outline.py @@ -0,0 +1,37 @@ +f""" +dataset: + app: + scripts: {scripts} #list of strs (filenames) or dicts containing keys + inline_scripts: {inline_scripts} #list of strs (filenames) + + about_legal_tos: {about_legal_tos} + about_legal_privacy: {about_legal_privacy} + + authentication_enable: {authentication_enable} + + presentation: + max_categories: {max_categories} + custom_colors: {custom_colors} + + user_annotations: + enable: {enable_users_annotations} + type: {annotation_type} + hosted_tiledb_array: + db_uri: {db_uri} + hosted_file_directory: {hosted_file_directory} + local_file_csv: + directory: {local_file_csv_directory} + file: {local_file_csv_file} + ontology: + enable: {ontology_enabled} + obo_location: {obo_location} + + embeddings: + names: {embedding_names} + enable_reembedding: {enable_reembedding} + + diffexp: + enable: {enable_difexp} + lfc_cutoff: {lfc_cutoff} + top_n: {top_n} +""" diff --git a/server/test/fixtures/server_config_outline.py b/server/test/fixtures/server_config_outline.py new file mode 100644 index 00000000..4bbb4675 --- /dev/null +++ b/server/test/fixtures/server_config_outline.py @@ -0,0 +1,62 @@ +f"""server: + app: + verbose: {verbose} + debug: {debug} + host: {host} + port: {port} + open_browser: {open_browser} + force_https: {force_https} + flask_secret_key: {flask_secret_key} + generate_cache_control_headers: {generate_cache_control_headers} + server_timing_headers: {server_timing_headers} + csp_directives: {csp_directives} + api_base_url: {api_base_url} + web_base_url: {web_base_url} + authentication: + type: {auth_type} + params_oauth: + oauth_api_base_url: {oauth_api_base_url} + client_id: {client_id} + client_secret: {client_secret} + jwt_decode_options: {jwt_decode_options} + session_cookie: {session_cookie} + cookie: {cookie} + + multi_dataset: + dataroot: {dataroot} + index: {index} + allowed_matrix_types: {allowed_matrix_types} + matrix_cache: + max_datasets: {max_cached_datasets} + timelimit_s: {timelimit_s} + + single_dataset: + datapath: {dataset_datapath} + obs_names: {obs_names} + var_names: {var_names} + about: {about} + title: {title} + + diffexp: + alg_cxg: # number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count) + max_workers: {diffexp_max_workers} + cpu_multiplier: {cpu_multiplier} + target_workunit: {target_workunit} # The target number of matrix elements that are evaluated in one thread. + + data_locator: + s3: + region_name: {data_locater_region_name} + + adaptor: + cxg_adaptor: + tiledb_ctx: + sm.tile_cache_size: {cxg_tile_cache_size} + sm.num_reader_threads: {cxg_num_reader_threads} + + anndata_adaptor: + backed: {anndata_backed} + + limits: + column_request_max: {column_request_max} + diffexp_cellcount_max: {diffexp_cellcount_max} +""" diff --git a/server/test/performance/run_diffexp.py b/server/test/performance/run_diffexp.py index bbbb3094..5cd1fbef 100644 --- a/server/test/performance/run_diffexp.py +++ b/server/test/performance/run_diffexp.py @@ -7,7 +7,7 @@ import numpy as np import server.compute.diffexp_cxg as diffexp_cxg import server.compute.diffexp_generic as diffexp_generic -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.data_common.matrix_loader import MatrixDataLoader from server.data_cxg.cxg_adaptor import CxgAdaptor diff --git a/server/test/test_database/test_database.py b/server/test/test_database/test_database.py index 4ecb4f27..91798fdb 100644 --- a/server/test/test_database/test_database.py +++ b/server/test/test_database/test_database.py @@ -16,45 +16,48 @@ class DatabaseTest(unittest.TestCase): del cls.db def test_user_creation(self): - one_user = self.db.get(table=CellxGeneUser, entity_id='test_user_id') - self.assertEqual(one_user.id, 'test_user_id') + one_user = self.db.get(table=CellxGeneUser, entity_id="test_user_id") + self.assertEqual(one_user.id, "test_user_id") user_count = self.db.session.query(CellxGeneUser).count() self.assertGreater(user_count, 10) def test_dataset_creation(self): - one_dataset = self.db.query(table_args=[CellxGeneDataset], - filter_args=[CellxGeneDataset.name == 'test_dataset']) - self.assertEqual(one_dataset[0].name, 'test_dataset') + one_dataset = self.db.query( + table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"] + ) + self.assertEqual(one_dataset[0].name, "test_dataset") dataset_count = self.db.session.query(CellxGeneDataset).count() self.assertGreater(dataset_count, 10) def test_annotation_creation(self): - one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0] - self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri') + one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == "tiledb_uri"])[0] + self.assertEqual(one_annotation.tiledb_uri, "tiledb_uri") annotation_count = self.db.session.query(Annotation).count() self.assertGreater(annotation_count, 10) def test_get_most_recent_annotation_for_user_dataset(self): - dataset_id = str(self.db.query(table_args=[CellxGeneDataset], - filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id) + dataset_id = str( + self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"])[0].id + ) # have to commit separately because created_at time written on the db server - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_0")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_1")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_2")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_3")) self.db.session.commit() - self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4')) + self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_4")) self.db.session.commit() - most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id, - Annotation.user_id == 'test_user_id']) + most_recent_annotation = self.db.query_for_most_recent( + Annotation, [Annotation.dataset_id == dataset_id, Annotation.user_id == "test_user_id"] + ) - self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4') + self.assertEqual(most_recent_annotation.tiledb_uri, "tiledb_uri_4") diff --git a/server/test/unit/auth/test_auth.py b/server/test/unit/auth/test_auth.py index b07e38d3..f7f6f358 100644 --- a/server/test/unit/auth/test_auth.py +++ b/server/test/unit/auth/test_auth.py @@ -2,7 +2,7 @@ import unittest import requests -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.test import FIXTURES_ROOT, test_server @@ -12,9 +12,7 @@ class AuthTest(unittest.TestCase): def test_auth_none(self): c = AppConfig() - c.update_server_config( - authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot - ) + c.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot) c.update_default_dataset_config(user_annotations__enable=False) c.complete_config() @@ -28,9 +26,7 @@ class AuthTest(unittest.TestCase): def test_auth_session(self): c = AppConfig() - c.update_server_config( - authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot - ) + c.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot) c.update_default_dataset_config(user_annotations__enable=True) c.complete_config() @@ -107,8 +103,8 @@ class AuthTest(unittest.TestCase): def test_auth_test_single(self): c = AppConfig() c.update_server_config( - authentication__type="test", - single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg") + authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg" + ) c.complete_config() diff --git a/server/test/unit/auth/test_oauth.py b/server/test/unit/auth/test_oauth.py index 728fccfe..8b250ed6 100644 --- a/server/test/unit/auth/test_oauth.py +++ b/server/test/unit/auth/test_oauth.py @@ -9,7 +9,7 @@ from flask import Flask, jsonify, make_response, request, redirect from multiprocessing import Process import jose -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.test import FIXTURES_ROOT, test_server # This tests the oauth authentication type. @@ -46,7 +46,7 @@ def token(): "scope": "openid profile email", "expires_in": TOKEN_EXPIRES, "token_type": "Bearer", - "expires_at": expires_at + "expires_at": expires_at, } return make_response(jsonify(r)) @@ -89,9 +89,8 @@ class AuthTest(unittest.TestCase): authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}", authentication__params_oauth__client_id="mock_client_id", authentication__params_oauth__client_secret="mock_client_secret", - authentication__params_oauth__jwt_decode_options={ - "verify_signature": False, "verify_iss": False - }) + authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False}, + ) app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot) app_config.complete_config() @@ -161,9 +160,7 @@ class AuthTest(unittest.TestCase): def test_auth_oauth_session(self): # test with session cookies app_config = AppConfig() - app_config.update_server_config( - authentication__params_oauth__session_cookie=True, - ) + app_config.update_server_config(authentication__params_oauth__session_cookie=True,) self.auth_flow(app_config) def test_auth_oauth_cookie(self): diff --git a/server/test/unit/cli/test_launch.py b/server/test/unit/cli/test_launch.py new file mode 100644 index 00000000..ac388cc0 --- /dev/null +++ b/server/test/unit/cli/test_launch.py @@ -0,0 +1,28 @@ +import filecmp +import os +import shutil +import unittest + +import yaml + +from server.default_config import default_config +from server.test import FIXTURES_ROOT + + +class CLIPLaunchTests(unittest.TestCase): + tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs") + + @classmethod + def setUpClass(cls) -> None: + os.mkdir(cls.tmp_dir) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_dir) + + +def test_dump_default_config(self): + os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt") + with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config: + expected_config.write(yaml.dump(default_config)) + filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt") diff --git a/server/test/unit/common/config/__init__.py b/server/test/unit/common/config/__init__.py new file mode 100644 index 00000000..fa730df2 --- /dev/null +++ b/server/test/unit/common/config/__init__.py @@ -0,0 +1,246 @@ +import os +import shutil +import unittest +import random +from unittest import mock + +from server.test import FIXTURES_ROOT + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class ConfigTests(unittest.TestCase): + tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir") + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp_fixtures_directory) + + @classmethod + def setUpClass(cls) -> None: + os.makedirs(cls.tmp_fixtures_directory) + + def custom_server_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="null", + generate_cache_control_headers="false", + server_timing_headers="false", + csp_directives="null", + api_base_url="null", + web_base_url="null", + auth_type="session", + oauth_api_base_url="null", + client_id="null", + client_secret="null", + jwt_decode_options="null", + session_cookie="true", + cookie="null", + dataroot="null", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + diffexp_max_workers=64, + cpu_multiplier=4, + target_workunit="16_000_000", + data_locater_region_name="us-east-1", + cxg_tile_cache_size=8589934592, + cxg_num_reader_threads=32, + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + config_file_name="server_config.yaml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py") + with open(server_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + server_config = eval(config) + with open(configfile, "w") as server_config_file: + server_config_file.write(server_config) + return configfile + + def custom_app_config( + self, + verbose="false", + debug="false", + host="localhost", + port="null", + open_browser="false", + force_https="false", + flask_secret_key="null", + generate_cache_control_headers="false", + server_timing_headers="false", + csp_directives="null", + api_base_url="null", + web_base_url="null", + auth_type="session", + oauth_api_base_url="null", + client_id="null", + client_secret="null", + jwt_decode_options="null", + session_cookie="true", + cookie="null", + dataroot="null", + index="false", + allowed_matrix_types=[], + max_cached_datasets=5, + timelimit_s=5, + dataset_datapath="null", + obs_names="null", + var_names="null", + about="null", + title="null", + diffexp_max_workers=64, + cpu_multiplier=4, + target_workunit="16_000_000", + data_locater_region_name="us-east-1", + cxg_tile_cache_size=8589934592, + cxg_num_reader_threads=32, + anndata_backed="false", + column_request_max=32, + diffexp_cellcount_max="null", + scripts=[], + inline_scripts=[], + about_legal_tos="null", + about_legal_privacy="null", + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + ontology_enabled="false", + obo_location="null", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + config_file_name="app_config.yml", + ): + random_num = random.randrange(999999) + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + server_config = self.custom_server_config( + verbose=verbose, + debug=debug, + host=host, + port=port, + open_browser=open_browser, + force_https=force_https, + flask_secret_key=flask_secret_key, + generate_cache_control_headers=generate_cache_control_headers, + server_timing_headers=server_timing_headers, + csp_directives=csp_directives, + api_base_url=api_base_url, + web_base_url=web_base_url, + auth_type=auth_type, + oauth_api_base_url=oauth_api_base_url, + client_id=client_id, + client_secret=client_secret, + jwt_decode_options=jwt_decode_options, + session_cookie=session_cookie, + cookie=cookie, + dataroot=dataroot, + index=index, + allowed_matrix_types=allowed_matrix_types, + max_cached_datasets=max_cached_datasets, + timelimit_s=timelimit_s, + dataset_datapath=dataset_datapath, + obs_names=obs_names, + var_names=var_names, + about=about, + title=title, + diffexp_max_workers=diffexp_max_workers, + cpu_multiplier=cpu_multiplier, + target_workunit=target_workunit, + data_locater_region_name=data_locater_region_name, + cxg_tile_cache_size=cxg_tile_cache_size, + cxg_num_reader_threads=cxg_num_reader_threads, + anndata_backed=anndata_backed, + column_request_max=column_request_max, + diffexp_cellcount_max=diffexp_cellcount_max, + config_file_name=f"temp_server_config_{random_num}.yml", + ) + dataset_config = self.custom_dataset_config( + scripts=scripts, + inline_scripts=inline_scripts, + about_legal_tos=about_legal_tos, + about_legal_privacy=about_legal_privacy, + authentication_enable=authentication_enable, + max_categories=max_categories, + custom_colors=custom_colors, + enable_users_annotations=enable_users_annotations, + annotation_type=annotation_type, + db_uri=db_uri, + hosted_file_directory=hosted_file_directory, + local_file_csv_directory=local_file_csv_directory, + local_file_csv_file=local_file_csv_file, + ontology_enabled=ontology_enabled, + obo_location=obo_location, + embedding_names=embedding_names, + enable_reembedding=enable_reembedding, + enable_difexp=enable_difexp, + lfc_cutoff=lfc_cutoff, + top_n=top_n, + config_file_name=f"temp_dataset_config_{random_num}.yml", + ) + with open(server_config) as server_config: + with open(dataset_config) as dataset_config: + with open(configfile, "w") as app_config_file: + for line in server_config: + app_config_file.write(line) + for line in dataset_config: + app_config_file.write(line) + + return configfile + + def custom_dataset_config( + self, + scripts=[], + inline_scripts=[], + about_legal_tos="null", + about_legal_privacy="null", + authentication_enable="true", + max_categories=1000, + custom_colors="true", + enable_users_annotations="true", + annotation_type="local_file_csv", + db_uri="null", + hosted_file_directory="null", + local_file_csv_directory="null", + local_file_csv_file="null", + ontology_enabled="false", + obo_location="null", + embedding_names=[], + enable_reembedding="false", + enable_difexp="true", + lfc_cutoff=0.01, + top_n=10, + config_file_name="dataset_config.yml", + ): + configfile = os.path.join(self.tmp_fixtures_directory, config_file_name) + dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py") + with open(dataset_config_outline_path, "r") as config_skeleton: + config = config_skeleton.read() + dataset_config = eval(config) + with open(configfile, "w") as dataset_config_file: + dataset_config_file.write(dataset_config) + + return configfile diff --git a/server/test/unit/common/config/test_app_config.py b/server/test/unit/common/config/test_app_config.py new file mode 100644 index 00000000..726ff86f --- /dev/null +++ b/server/test/unit/common/config/test_app_config.py @@ -0,0 +1,140 @@ +import os +import tempfile +import unittest + +import yaml + +from server.default_config import default_config +from server.common.config.app_config import AppConfig +from server.test.unit.common.config import ConfigTests +from server.common.errors import ConfigurationError +from server.test import FIXTURES_ROOT + + +class AppConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_get_default_config_correctly_reads_default_config_file(self): + app_default_config = AppConfig().default_config + + expected_config = yaml.load(default_config, Loader=yaml.Loader) + + server_config = app_default_config['server'] + dataset_config = app_default_config['dataset'] + + expected_server_config = expected_config['server'] + expected_dataset_config = expected_config['dataset'] + + self.assertDictEqual(app_default_config, expected_config) + self.assertDictEqual(server_config, expected_server_config) + self.assertDictEqual(dataset_config, expected_dataset_config) + + def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self): + datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad" + file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name) + config = AppConfig() + config.update_from_config_file(file_name) + + self.assertEqual(config.get_dataset_config(""), config.default_dataset_config) + + def test_update_server_config_updates_server_config_and_config_status(self): + config = self.get_config() + config.complete_config() + config.check_config() + config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + with self.assertRaises(ConfigurationError): + config.server_config.check_config() + + def test_write_config_outputs_yaml_with_all_config_vars(self): + config = self.get_config() + config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml") + with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config: + default_config_yml = yaml.safe_load(default_config) + + with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config: + output_config_yml = yaml.safe_load(output_config) + self.maxDiff = None + self.assertEqual(default_config_yml, output_config_yml) + + def test_update_app_config(self): + config = AppConfig() + config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir") + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)]) + + config = AppConfig() + config.update_default_dataset_config(app__scripts=(), app__inline_scripts=()) + vars = config.server_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[]) + vars = config.default_dataset_config.changes_from_default() + self.assertCountEqual(vars, []) + + config = AppConfig() + config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) + vars = config.default_dataset_config.changes_from_default() + self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) + + def test_configfile_no_dataset_section(self): + # test a config file without a dataset section + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + server: + multi_dataset: + dataroot: test_dataroot + + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + server_changes = app_config.server_config.changes_from_default() + dataset_changes = app_config.default_dataset_config.changes_from_default() + self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)]) + self.assertEqual(dataset_changes, []) + + def test_configfile_no_server_section(self): + # test a config file without a dataset section + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + dataset: + user_annotations: + enable: false + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + server_changes = app_config.server_config.changes_from_default() + dataset_changes = app_config.default_dataset_config.changes_from_default() + self.assertEqual(server_changes, []) + self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)]) diff --git a/server/test/unit/common/config/test_base_config.py b/server/test/unit/common/config/test_base_config.py new file mode 100644 index 00000000..d41082cf --- /dev/null +++ b/server/test/unit/common/config/test_base_config.py @@ -0,0 +1,63 @@ +import unittest + +from server.common.config.app_config import AppConfig +from server.test import FIXTURES_ROOT +from server.test.unit.common.config import ConfigTests +from server.common.errors import ConfigurationError + + +class BaseConfigTest(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_mapping_creation_returns_map_of_server_and_dataset_config(self): + config = AppConfig() + mapping = config.default_dataset_config.create_mapping(config.default_config) + self.assertIsNotNone(mapping["server__app__verbose"]) + self.assertIsNotNone(mapping["dataset__presentation__max_categories"]) + self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"]) + self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"]) + + def test_changes_from_default_returns_list_of_nondefault_config_values(self): + config = self.get_config(verbose="true", lfc_cutoff=0.05) + server_changes = config.server_config.changes_from_default() + dataset_changes = config.default_dataset_config.changes_from_default() + + self.assertEqual( + server_changes, + [ + ("app__verbose", True, False), + ("multi_dataset__dataroot", FIXTURES_ROOT, None), + ("multi_dataset__matrix_cache__timelimit_s", 5, 30), + ("data_locator__s3__region_name", "us-east-1", True), + ], + ) + self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)]) + + def test_check_config_throws_error_if_attr_has_not_been_checked(self): + config = self.get_config(verbose="true") + config.complete_config() + config.check_config() + config.update_server_config(app__verbose=False) + with self.assertRaises(ConfigurationError): + config.check_config() diff --git a/server/test/unit/common/config/test_dataset_config.py b/server/test/unit/common/config/test_dataset_config.py new file mode 100644 index 00000000..a32d1f87 --- /dev/null +++ b/server/test/unit/common/config/test_dataset_config.py @@ -0,0 +1,260 @@ +import os +import tempfile + +import requests +import unittest +from unittest.mock import patch + +from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB +from server.common.annotations.local_file_csv import AnnotationsLocalFile +from server.common.config.app_config import AppConfig +from server.common.config.base_config import BaseConfig +from server.test import test_server, PROJECT_ROOT, FIXTURES_ROOT + +from server.common.errors import ConfigurationError +from server.test.unit.common.config import ConfigTests + + +class TestDatasetConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.dataset_config = self.config.default_dataset_config + self.config.complete_config() + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_datatset_config_sets_vars_from_default_config(self): + config = AppConfig() + self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000) + self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv") + self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01) + self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location) + + @patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.dataset_config.complete_config(self.context) + self.assertEqual(mock_check_attrs.call_count, 21) + + def test_app_sets_script_vars(self): + config = self.get_config(scripts=["path/to/script"]) + config.default_dataset_config.handle_app() + + self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}]) + + config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}]) + config.default_dataset_config.handle_app() + self.assertEqual( + config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}] + ) + + config = self.get_config(scripts=["path/to/script", "different/script/path"]) + config.default_dataset_config.handle_app() + # TODO @madison -- is this the desired functionality? + self.assertEqual( + config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}] + ) + + config = self.get_config(scripts=[{"more": "different/script/path"}]) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_app() + + def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self): + config = self.get_config(enable_users_annotations="true", authentication_enable="false") + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_user_annotations(self.context) + + config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend") + with self.assertRaises(ConfigurationError): + config.server_config.complete_config(self.context) + + def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self): + config = self.get_config( + enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set" + ) + config.default_dataset_config.handle_user_annotations(self.context) + + self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."]) + + @patch("server.common.config.dataset_config.DbUtils") + def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils): + mock_db_utils.return_value = "123" + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_user_annotations(self.context) + self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile) + + config = self.get_config( + enable_users_annotations="true", + authentication_enable="true", + annotation_type="hosted_tiledb_array", + db_uri="gotta/set/this", + hosted_file_directory="and/this", + ) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_user_annotations(self.context) + self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB) + + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL" + ) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_user_annotations(self.context) + + def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self): + config = self.get_config( + enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv" + ) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_local_file_csv_annotations() + self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile) + cwd = os.getcwd() + self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd) + + def test_handle_embeddings__checks_data_file_types(self): + file_name = self.custom_app_config( + embedding_names=["name1", "name2"], + enable_reembedding="true", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", + anndata_backed="true", + config_file_name=self.config_file_name, + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.complete_config(self.context) + with self.assertRaises(ConfigurationError): + config.default_dataset_config.handle_embeddings() + + def test_handle_diffexp__raises_warning_for_large_datasets(self): + config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15) + config.server_config.complete_config(self.context) + config.default_dataset_config.handle_diffexp(self.context) + self.assertEqual(len(self.context["messages"]), 0) + + def test_multi_dataset(self): + config = AppConfig() + # test for illegal url_dataroots + for illegal in ("../b", "!$*", "\\n", "", "(bad)"): + config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} + ) + with self.assertRaises(ConfigurationError): + config.complete_config() + + # test for legal url_dataroots + for legal in ("d", "this.is-okay_", "a/b"): + config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} + ) + config.complete_config() + + # test that multi dataroots work end to end + config.update_server_config( + multi_dataset__dataroot=dict( + s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), + s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), + s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), + ) + ) + + # Change this default to test if the dataroot overrides below work. + config.update_default_dataset_config(app__about_legal_tos="tos_default.html") + + # specialize the configs for set1 + config.add_dataroot_config( + "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html" + ) + + # specialize the configs for set2 + config.add_dataroot_config( + "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html" + ) + + # no specializations for set3 (they get the default dataset config) + config.complete_config() + + with test_server(app_config=config) as server: + session = requests.Session() + + response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is False + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html" + + response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html" + + response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html" + + response = session.get(f"{server}/health") + assert response.json()["status"] == "pass" + + def test_configfile_with_specialization(self): + # test that per_dataset_config config load the default config, then the specialized config + + with tempfile.TemporaryDirectory() as tempdir: + configfile = os.path.join(tempdir, "config.yaml") + with open(configfile, "w") as fconfig: + config = """ + server: + multi_dataset: + dataroot: + test: + base_url: test + dataroot: fake_dataroot + + dataset: + user_annotations: + enable: false + type: hosted_tiledb_array + hosted_tiledb_array: + db_uri: fake_db_uri + hosted_file_directory: fake_dir + + per_dataset_config: + test: + user_annotations: + enable: true + """ + fconfig.write(config) + + app_config = AppConfig() + app_config.update_from_config_file(configfile) + + test_config = app_config.dataroot_config["test"] + + # test config from default + self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array") + self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri") + + # test config from specialization + self.assertTrue(test_config.user_annotations__enable) diff --git a/server/test/unit/common/config/test_server_config.py b/server/test/unit/common/config/test_server_config.py new file mode 100644 index 00000000..78ee9570 --- /dev/null +++ b/server/test/unit/common/config/test_server_config.py @@ -0,0 +1,335 @@ +import os +import unittest +from unittest import mock +from unittest.mock import patch + +from server.common.config.base_config import BaseConfig +from server.common.utils.utils import find_available_port +from server.test import PROJECT_ROOT, FIXTURES_ROOT + +import requests + +from server.common.config.app_config import AppConfig +from server.common.errors import ConfigurationError +from server.test import test_server +from server.test.unit.common.config import ConfigTests + + +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + + +class TestServerConfig(ConfigTests): + def setUp(self): + self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml" + self.config = AppConfig() + self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT) + self.server_config = self.config.server_config + self.config.complete_config() + + message_list = [] + + def noop(message): + message_list.append(message) + + messagefn = noop + self.context = dict(messagefn=messagefn, messages=message_list) + + def get_config(self, **kwargs): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs + ) + config = AppConfig() + config.update_from_config_file(file_name) + return config + + def test_init_raises_error_if_default_config_is_invalid(self): + invalid_config = self.get_config(port="not_valid") + with self.assertRaises(ConfigurationError): + invalid_config.complete_config() + + @patch("server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute") + def test_complete_config_checks_all_attr(self, mock_check_attrs): + mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute() + self.server_config.complete_config(self.context) + self.assertEqual(mock_check_attrs.call_count, 40) + + def test_handle_app__throws_error_if_port_doesnt_exist(self): + config = self.get_config(port=99999999) + with self.assertRaises(ConfigurationError): + config.server_config.handle_app(self.context) + + @patch("server.common.config.server_config.discover_s3_region_name") + def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name): + mock_discover_region_name.return_value = None + # Default config + self.assertEqual(self.config.server_config.data_locator__s3__region_name, None) + # hard coded + config = self.get_config() + self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1") + # incorrectly formatted + dataroot = { + "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"}, + "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"}, + } + file_name = self.custom_app_config( + dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true" + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_locator() + + @patch("server.common.config.server_config.discover_s3_region_name") + def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name): + mock_discover_region_name.return_value = "us-west-2" + dataroot = { + "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"}, + "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"}, + } + file_name = self.custom_app_config( + dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true" + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.handle_data_locator() + self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2") + mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev") + + def test_handle_app___can_use_envar_port(self): + config = self.get_config(port=24) + self.assertEqual(config.server_config.app__port, 24) + + # Note if the port is set in the config file it will NOT be overwritten by a different envvar + os.environ["CXG_SERVER_PORT"] = "4008" + self.config = AppConfig() + self.config.server_config.handle_app(self.context) + self.assertEqual(self.config.server_config.app__port, 4008) + + def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self): + config = self.get_config(flask_secret_key="KEY_FROM_FILE") + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE") + + os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV" + config.server_config.handle_app(self.context) + self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV") + + def test_handle_app__sets_web_base_url(self): + config = self.get_config(web_base_url="anything.com") + self.assertEqual(config.server_config.app__web_base_url, "anything.com") + + def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self): + config = self.get_config(client_secret="KEY_FROM_FILE") + config.server_config.handle_authentication() + self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE") + + os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV" + config.server_config.handle_authentication() + + self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV") + + def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self): + file_name = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", + config_file_name="two_data_roots.yml", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_source() + + file_name = self.custom_app_config(config_file_name="zero_roots.yml") + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_data_source() + + def test_get_api_base_url_works(self): + + # test the api_base_url feature, and that it can contain a path + config = AppConfig() + backend_port = find_available_port("localhost", 10000) + config.update_server_config( + app__api_base_url=f"http://localhost:{backend_port}/additional/path", + multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset", + ) + + config.complete_config() + + with test_server(["-p", str(backend_port)], app_config=config) as server: + session = requests.Session() + self.assertEqual(server, f"http://localhost:{backend_port}") + response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config") + self.assertEqual(response.status_code, 200) + data_config = response.json() + self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k") + + # test the health check at the correct url + response = session.get(f"{server}/additional/path/health") + assert response.json()["status"] == "pass" + + # also check that the old URL still works. + # NOTE: this old URL location will soon be deprecated, and when that happens + # this check can be removed. + response = session.get(f"{server}/health") + assert response.json()["status"] == "pass" + + def test_get_web_base_url_works(self): + config = self.get_config(web_base_url="www.thisisawebsite.com") + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "www.thisisawebsite.com") + + config = self.get_config(web_base_url="local", port=12) + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "http://localhost:12") + + config = self.get_config(web_base_url="www.thisisawebsite.com/") + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "www.thisisawebsite.com") + + config = self.get_config(api_base_url="www.api_base.com/") + web_base_url = config.server_config.get_web_base_url() + self.assertEqual(web_base_url, "www.api_base.com") + + def test_config_for_single_dataset(self): + file_name = self.custom_app_config( + config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg" + ) + config = AppConfig() + config.update_from_config_file(file_name) + config.server_config.handle_single_dataset(self.context) + self.assertIsNotNone(config.server_config.matrix_data_cache_manager) + + file_name = self.custom_app_config( + config_file_name="single_dataset_with_about.yml", + about="www.cziscience.com", + dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg", + ) + config = AppConfig() + config.update_from_config_file(file_name) + with self.assertRaises(ConfigurationError): + config.server_config.handle_single_dataset(self.context) + + def test_multi_dataset_raises_error_for_illegal_routes(self): + # test for illegal url_dataroots + for illegal in ("../b", "!$*", "\\n", "", "(bad)"): + self.config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} + ) + with self.assertRaises(ConfigurationError): + self.config.complete_config() + + def test_multidataset_works_for_legal_routes(self): + # test for legal url_dataroots + for legal in ("d", "this.is-okay_", "a/b"): + self.config.update_server_config( + multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} + ) + self.config.complete_config() + + def test_mulitdatasets_work_e2e(self): + # test that multi dataroots work end to end + self.config.update_server_config( + multi_dataset__dataroot=dict( + s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), + s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), + s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), + ) + ) + + # Change this default to test if the dataroot overrides below work. + self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html") + + # specialize the configs for set1 + self.config.add_dataroot_config( + "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html" + ) + + # specialize the configs for set2 + self.config.add_dataroot_config( + "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html" + ) + + # no specializations for set3 (they get the default dataset config) + self.config.complete_config() + + with test_server(app_config=self.config) as server: + session = requests.Session() + + response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is False + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html" + + response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html" + + response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") + data_config = response.json() + assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" + assert data_config["config"]["parameters"]["annotations"] is True + assert data_config["config"]["parameters"]["disable-diffexp"] is False + assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html" + + response = session.get(f"{server}/health") + assert response.json()["status"] == "pass" + + @patch("server.common.config.server_config.diffexp_tiledb.set_config") + def test_handle_diffexp(self, mock_tiledb_config): + custom_config_file = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", + cpu_multiplier=3, + diffexp_max_workers=1, + target_workunit=4, + config_file_name=self.config_file_name, + ) + config = AppConfig() + config.update_from_config_file(custom_config_file) + config.server_config.handle_diffexp() + # called with the min of diffexp_max_workers and cpus*cpu_multiplier + mock_tiledb_config.assert_called_once_with(1, 4) + + @patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context") + def test_handle_adaptor(self, mock_tiledb_context): + custom_config = self.custom_app_config( + dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2 + ) + config = AppConfig() + config.update_from_config_file(custom_config) + config.server_config.handle_adaptor() + mock_tiledb_context.assert_called_once_with( + {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"} + ) + + @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION") + @patch("server.common.config.get_secret_key") + def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key): + mock_get_secret_key.return_value = { + "flask_secret_key": "mock_flask_secret", + "oauth_client_secret": "mock_oauth_secret", + "db_uri": "mock_db_uri", + } + + config = AppConfig() + + with self.assertLogs(level="INFO") as logger: + from server.common.config import handle_config_from_secret + + # should not throw error + # "AttributeError: 'XConfig' object has no attribute 'x'" + handle_config_from_secret(config) + + # should log 3 lines (one for each var set from a secret) + self.assertEqual(len(logger.output), 3) + self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0]) + self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1]) + self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2]) + self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret") + self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret") + self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri") diff --git a/server/test/unit/common/test_api.py b/server/test/unit/common/test_api.py index ee979b1f..7f524f7d 100644 --- a/server/test/unit/common/test_api.py +++ b/server/test/unit/common/test_api.py @@ -8,8 +8,14 @@ import requests import server.test.unit.decode_fbs as decode_fbs from server.data_common.matrix_loader import MatrixDataType -from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server, - stop_test_server) +from server.test import ( + data_with_tmp_annotations, + make_fbs, + PROJECT_ROOT, + FIXTURES_ROOT, + start_test_server, + stop_test_server, +) from server.test.fixtures.fixtures import pbmc3k_colors BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}} @@ -381,11 +387,14 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): @classmethod def setUpClass(cls): - cls._setupClass(cls, [ - f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", - "--disable-annotations", - "--experimental-enable-reembedding", - ]) + cls._setupClass( + cls, + [ + f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + "--disable-annotations", + "--experimental-enable-reembedding", + ], + ) @classmethod def tearDownClass(cls): @@ -403,10 +412,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints): @classmethod def setUpClass(cls): - cls._setupClass(cls, [ - f"{FIXTURES_ROOT}/pbmc3k.cxg", - "--disable-annotations", - ]) + cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"]) @classmethod def tearDownClass(cls): @@ -423,7 +429,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations): cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations( MatrixDataType.H5AD, annotations_fixture=True ) - cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ]) + cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()]) @classmethod def tearDownClass(cls): @@ -439,11 +445,7 @@ class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations): @classmethod def setUpClass(cls): cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True) - cls._setupClass(cls, [ - "--annotations-file", - cls.annotations.output_file, - cls.data.get_location(), - ]) + cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()]) @classmethod def tearDownClass(cls): diff --git a/server/test/unit/common/test_app_config.py b/server/test/unit/common/test_app_config.py deleted file mode 100644 index c3dd1436..00000000 --- a/server/test/unit/common/test_app_config.py +++ /dev/null @@ -1,249 +0,0 @@ -import os -import unittest -from unittest import mock -from unittest.mock import patch -import tempfile - -import requests - -from server.common.app_config import AppConfig -from server.common.errors import ConfigurationError -from server.common.utils.utils import find_available_port -from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT - - -# NOTE, there are more tests that should be written for AppConfig. -# this is just a start. - -def mockenv(**envvars): - return mock.patch.dict(os.environ, envvars) - - -class AppConfigTest(unittest.TestCase): - def test_update(self): - config = AppConfig() - config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir") - vars = config.server_config.changes_from_default() - self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)]) - - config = AppConfig() - config.update_default_dataset_config(app__scripts=(), app__inline_scripts=()) - vars = config.server_config.changes_from_default() - self.assertCountEqual(vars, []) - - config = AppConfig() - config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[]) - vars = config.default_dataset_config.changes_from_default() - self.assertCountEqual(vars, []) - - config = AppConfig() - config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"]) - vars = config.default_dataset_config.changes_from_default() - self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])]) - - def test_multi_dataset(self): - - config = AppConfig() - # test for illegal url_dataroots - for illegal in ("../b", "!$*", "\\n", "", "(bad)"): - config.update_server_config( - multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} - ) - with self.assertRaises(ConfigurationError): - config.complete_config() - - # test for legal url_dataroots - for legal in ("d", "this.is-okay_", "a/b"): - config.update_server_config( - multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}} - ) - config.complete_config() - - # test that multi dataroots work end to end - config.update_server_config( - multi_dataset__dataroot=dict( - s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"), - s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"), - s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"), - ) - ) - - # Change this default to test if the dataroot overrides below work. - config.update_default_dataset_config(app__about_legal_tos="tos_default.html") - - # specialize the configs for set1 - config.add_dataroot_config( - "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html" - ) - - # specialize the configs for set2 - config.add_dataroot_config( - "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html" - ) - - # no specializations for set3 (they get the default dataset config) - config.complete_config() - - with test_server(app_config=config) as server: - session = requests.Session() - - response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config") - data_config = response.json() - assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" - assert data_config["config"]["parameters"]["annotations"] is False - assert data_config["config"]["parameters"]["disable-diffexp"] is False - assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html" - - response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config") - data_config = response.json() - assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" - assert data_config["config"]["parameters"]["annotations"] is True - assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html" - - response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config") - data_config = response.json() - assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k" - assert data_config["config"]["parameters"]["annotations"] is True - assert data_config["config"]["parameters"]["disable-diffexp"] is False - assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html" - - response = session.get(f"{server}/health") - assert response.json()["status"] == "pass" - - @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION") - @patch('server.common.aws_secret_utils.get_secret_key') - def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key): - mock_get_secret_key.return_value = { - "flask_secret_key": "mock_flask_secret", - "oauth_client_secret": "mock_oauth_secret", - "db_uri": "mock_db_uri" - } - - config = AppConfig() - - with self.assertLogs(level="INFO") as logger: - from server.common.aws_secret_utils import handle_config_from_secret - # should not throw error - # "AttributeError: 'XConfig' object has no attribute 'x'" - handle_config_from_secret(config) - - # should log 3 lines (one for each var set from a secret) - self.assertEqual(len(logger.output), 3) - self.assertIn('INFO:root:set app__flask_secret_key from secret', logger.output[0]) - self.assertIn('INFO:root:set authentication__params_oauth__client_secret from secret', logger.output[1]) - self.assertIn('INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret', logger.output[2]) - self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret") - self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret") - self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri") - - def test_api_base_url(self): - - # test the api_base_url feature, and that it can contain a path - config = AppConfig() - backend_port = find_available_port("localhost", 10000) - config.update_server_config( - app__api_base_url=f"http://localhost:{backend_port}/additional/path", - multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset" - ) - - config.complete_config() - - with test_server(["-p", str(backend_port)], app_config=config) as server: - session = requests.Session() - self.assertEqual(server, f"http://localhost:{backend_port}") - response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config") - self.assertEqual(response.status_code, 200) - data_config = response.json() - self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k") - - # test the health check at the correct url - response = session.get(f"{server}/additional/path/health") - assert response.json()["status"] == "pass" - - # also check that the old URL still works. - # NOTE: this old URL location will soon be deprecated, and when that happens - # this check can be removed. - response = session.get(f"{server}/health") - assert response.json()["status"] == "pass" - - def test_configfile_with_specialization(self): - # test that per_dataset_config config load the default config, then the specialized config - - with tempfile.TemporaryDirectory() as tempdir: - configfile = os.path.join(tempdir, "config.yaml") - with open(configfile, "w") as fconfig: - config = """ - server: - multi_dataset: - dataroot: - test: - base_url: test - dataroot: fake_dataroot - - dataset: - user_annotations: - enable: false - type: hosted_tiledb_array - hosted_tiledb_array: - db_uri: fake_db_uri - hosted_file_directory: fake_dir - - per_dataset_config: - test: - user_annotations: - enable: true - """ - fconfig.write(config) - - app_config = AppConfig() - app_config.update_from_config_file(configfile) - - test_config = app_config.dataroot_config["test"] - - # test config from default - self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array") - self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri") - - # test config from specialization - self.assertTrue(test_config.user_annotations__enable) - - def test_configfile_no_dataset_section(self): - # test a config file without a dataset section - - with tempfile.TemporaryDirectory() as tempdir: - configfile = os.path.join(tempdir, "config.yaml") - with open(configfile, "w") as fconfig: - config = """ - server: - multi_dataset: - dataroot: test_dataroot - - """ - fconfig.write(config) - - app_config = AppConfig() - app_config.update_from_config_file(configfile) - server_changes = app_config.server_config.changes_from_default() - dataset_changes = app_config.default_dataset_config.changes_from_default() - self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)]) - self.assertEqual(dataset_changes, []) - - def test_configfile_no_server_section(self): - # test a config file without a dataset section - - with tempfile.TemporaryDirectory() as tempdir: - configfile = os.path.join(tempdir, "config.yaml") - with open(configfile, "w") as fconfig: - config = """ - dataset: - user_annotations: - enable: false - """ - fconfig.write(config) - - app_config = AppConfig() - app_config.update_from_config_file(configfile) - server_changes = app_config.server_config.changes_from_default() - dataset_changes = app_config.default_dataset_config.changes_from_default() - self.assertEqual(server_changes, []) - self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)]) diff --git a/server/test/unit/common/test_corpora.py b/server/test/unit/common/test_corpora.py index 1a0222dc..5f26dae9 100644 --- a/server/test/unit/common/test_corpora.py +++ b/server/test/unit/common/test_corpora.py @@ -87,24 +87,17 @@ class CorporaRESTAPITest(unittest.TestCase): def setCorporaFields(cls, path): adata = anndata.read_h5ad(path) corpora_props = { - "version": { - "corpora_schema_version": "1.0.0", - "corpora_encoding_version": "0.1.0" - }, + "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}, "title": "PBMC3K", - "contributors": json.dumps([ - {"name": "name"} - ]), - "layer_descriptions": { - "X": "raw counts" - }, + "contributors": json.dumps([{"name": "name"}]), + "layer_descriptions": {"X": "raw counts"}, "organism": "human", "organism_ontology_term_id": "unknown", "project_name": "test project", "project_description": "test description", - "project_links": json.dumps([ - {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"} - ]), + "project_links": json.dumps( + [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}] + ), "default_embedding": "X_tsne", } adata.uns.update(corpora_props) diff --git a/server/test/unit/common/test_writable_annotation.py b/server/test/unit/common/test_writable_annotation.py index f650fecf..9d2f5f1d 100644 --- a/server/test/unit/common/test_writable_annotation.py +++ b/server/test/unit/common/test_writable_annotation.py @@ -27,7 +27,7 @@ class auth(object): class WritableTileDBStoredAnnotationTest(unittest.TestCase): def setUp(self): - self.user_id = '1234' + self.user_id = "1234" self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD) self.data.dataset_config.user_annotations = self.annotations self.db = self.annotations.db @@ -38,7 +38,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): } self.fbs = make_fbs(self.test_dict) self.df = pd.DataFrame(self.test_dict) - self.app = Flask('fake_app') + self.app = Flask("fake_app") self.app.__setattr__("auth", auth) def tearDown(self): @@ -65,8 +65,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): self.annotations.write_labels(self.df, self.data) dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id annotation = self.db.query_for_most_recent( - Annotation, - [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)] + Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)] ) # retrieve tiledb array df = tiledb.open(annotation.tiledb_uri) @@ -78,7 +77,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self): with self.app.test_request_context(): - new_name = 'new_dataset/location' + new_name = "new_dataset/location" self.data.get_location = MagicMock(return_value=new_name) num_datasets = len(self.db.query([CellxGeneDataset])) self.annotation_put_fbs(self.fbs) @@ -130,15 +129,14 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase): with self.assertRaises(KeyError): self.annotation_put_fbs(fbs_bad) - @patch('server.common.annotations.hosted_tiledb.current_app') + @patch("server.common.annotations.hosted_tiledb.current_app") def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id): - mock_user_id.auth.get_user_id.return_value = '1234' + mock_user_id.auth.get_user_id.return_value = "1234" self.annotations.write_labels(self.df, self.data) # get uri dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id annotation = self.db.query_for_most_recent( - Annotation, - [Annotation.user_id == '1234', Annotation.dataset_id == str(dataset_id)] + Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)] ) df = tiledb.open(annotation.tiledb_uri) diff --git a/server/test/unit/common/utils/test_cxg_generation_utils.py b/server/test/unit/common/utils/test_cxg_generation_utils.py index 57893913..b9043c33 100644 --- a/server/test/unit/common/utils/test_cxg_generation_utils.py +++ b/server/test/unit/common/utils/test_cxg_generation_utils.py @@ -8,8 +8,12 @@ import numpy as np import tiledb from pandas import Series, DataFrame -from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array, - convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array) +from server.common.utils.cxg_generation_utils import ( + convert_dictionary_to_cxg_group, + convert_dataframe_to_cxg_array, + convert_ndarray_to_cxg_dense_array, + convert_matrix_to_cxg_array, +) PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() @@ -28,8 +32,9 @@ class TestCxgGenerationUtils(unittest.TestCase): dictionary_name = "favorite_desserts" expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}" - convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary, - group_metadata_name=dictionary_name) + convert_dictionary_to_cxg_group( + self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name + ) array = tiledb.open(expected_array_directory) actual_stored_metadata = dict(array.meta.items()) @@ -44,13 +49,16 @@ class TestCxgGenerationUtils(unittest.TestCase): random_dataframe_name = f"random_dataframe_{uuid4()}" random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category}) - convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, - "int_category", tiledb.Ctx()) + convert_dataframe_to_cxg_array( + self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx() + ) expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}" expected_array_metadata = { - "cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, - "index": "int_category"})} + "cxg_schema": json.dumps( + {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"} + ) + } actual_stored_dataframe_array = tiledb.open(expected_array_directory) actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items()) @@ -95,7 +103,7 @@ class TestCxgGenerationUtils(unittest.TestCase): self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[:, :][''].size == 0) + self.assertTrue(actual_stored_array[:, :][""].size == 0) def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self): matrix = np.zeros([3, 3]) @@ -110,10 +118,10 @@ class TestCxgGenerationUtils(unittest.TestCase): self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[0, 0][''] == 1) - self.assertTrue(actual_stored_array[1, 1][''] == 1) - self.assertTrue(actual_stored_array[2, 2][''] == 2) - self.assertTrue(actual_stored_array[:, :][''].size == 3) + self.assertTrue(actual_stored_array[0, 0][""] == 1) + self.assertTrue(actual_stored_array[1, 1][""] == 1) + self.assertTrue(actual_stored_array[2, 2][""] == 2) + self.assertTrue(actual_stored_array[:, :][""].size == 3) def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self): matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}" @@ -122,14 +130,15 @@ class TestCxgGenerationUtils(unittest.TestCase): # a matrix of zeros which is sparse. column_shift = np.ones((3, 2)) - convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(), - column_shift_for_sparse_encoding=column_shift) + convert_matrix_to_cxg_array( + matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift + ) actual_stored_array = tiledb.open(matrix_name) self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[:, :][''].size == 0) + self.assertTrue(actual_stored_array[:, :][""].size == 0) def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self): matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}" @@ -137,13 +146,14 @@ class TestCxgGenerationUtils(unittest.TestCase): # Only column shift the first column of ones. column_shift = np.array([[1, 0], [1, 0]]) - convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(), - column_shift_for_sparse_encoding=column_shift) + convert_matrix_to_cxg_array( + matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift + ) actual_stored_array = tiledb.open(matrix_name) self.assertTrue(path.isdir(matrix_name)) self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray)) - self.assertTrue(actual_stored_array[0, 1][''] == 1) - self.assertTrue(actual_stored_array[1, 1][''] == 1) - self.assertTrue(actual_stored_array[:, :][''].size == 2) + self.assertTrue(actual_stored_array[0, 1][""] == 1) + self.assertTrue(actual_stored_array[1, 1][""] == 1) + self.assertTrue(actual_stored_array[:, :][""].size == 2) diff --git a/server/test/unit/common/utils/test_matrix_utils.py b/server/test/unit/common/utils/test_matrix_utils.py index ffda1045..9cc9daf5 100644 --- a/server/test/unit/common/utils/test_matrix_utils.py +++ b/server/test/unit/common/utils/test_matrix_utils.py @@ -6,7 +6,6 @@ from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_ class TestMatrixUtils(unittest.TestCase): - def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self): matrix = np.array([1, 2, 3]) diff --git a/server/test/unit/common/utils/test_sanitization_utils.py b/server/test/unit/common/utils/test_sanitization_utils.py index 8ef04218..e209be95 100644 --- a/server/test/unit/common/utils/test_sanitization_utils.py +++ b/server/test/unit/common/utils/test_sanitization_utils.py @@ -4,7 +4,6 @@ from server.common.utils.sanitization_utils import sanitize_values_in_list, sani class TestSanitizationUtils(unittest.TestCase): - def test__sanitize_values_in_list__not_strings_raises_exception(self): keys_to_sanitize = [1, 2, 3] diff --git a/server/test/unit/common/utils/test_type_conversion_utils.py b/server/test/unit/common/utils/test_type_conversion_utils.py index f1653697..ade2d999 100644 --- a/server/test/unit/common/utils/test_type_conversion_utils.py +++ b/server/test/unit/common/utils/test_type_conversion_utils.py @@ -5,12 +5,17 @@ from unittest.mock import patch import numpy as np from pandas import Series, DataFrame -from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \ - get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe, convert_pandas_series_to_numpy +from server.common.utils.type_conversion_utils import ( + can_cast_to_float32, + can_cast_to_int32, + get_dtype_of_array, + get_schema_type_hint_of_array, + get_dtypes_and_schemas_of_dataframe, + convert_pandas_series_to_numpy, +) class TestTypeConversionUtils(unittest.TestCase): - def test__can_cast_to_float32__string_is_false(self): array_to_convert = Series(data=["1", "2", "3"], dtype=str) @@ -97,8 +102,9 @@ class TestTypeConversionUtils(unittest.TestCase): expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode] for test_type_index in range(len(types)): - with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", - i=test_type_index): + with self.subTest( + f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) @@ -123,8 +129,9 @@ class TestTypeConversionUtils(unittest.TestCase): expected_dtypes = [np.float32, np.int32] for test_type_index in range(len(types)): - with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", - i=test_type_index): + with self.subTest( + f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) @@ -141,8 +148,9 @@ class TestTypeConversionUtils(unittest.TestCase): expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}] for test_type_index in range(len(types)): - with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", - i=test_type_index): + with self.subTest( + f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) @@ -160,8 +168,9 @@ class TestTypeConversionUtils(unittest.TestCase): for test_type_index in range(len(types)): with self.subTest( - f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", - i=test_type_index): + f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", + i=test_type_index, + ): array = Series(data=[], dtype=types[test_type_index]) self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) @@ -171,8 +180,10 @@ class TestTypeConversionUtils(unittest.TestCase): dataframe = DataFrame({"float_array": float_array, "category_array": category_array}) expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode} - expected_schema_type_hints_dict = {"float_array": {"type": "float32"}, - "category_array": {"type": "categorical", "categories": ["a", "b"]}} + expected_schema_type_hints_dict = { + "float_array": {"type": "float32"}, + "category_array": {"type": "categorical", "categories": ["a", "b"]}, + } actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe) @@ -201,5 +212,6 @@ class TestTypeConversionUtils(unittest.TestCase): with self.assertLogs(level="ERROR") as logger: convert_pandas_series_to_numpy(int_series, np.int32) - self.assertIn("Cannot convert a pandas Series object to an integer dtype if it contains NaNs", - logger.output[0]) + self.assertIn( + "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0] + ) diff --git a/server/test/unit/converters/test_h5ad_data_file.py b/server/test/unit/converters/test_h5ad_data_file.py index f8587ebd..99ad40af 100644 --- a/server/test/unit/converters/test_h5ad_data_file.py +++ b/server/test/unit/converters/test_h5ad_data_file.py @@ -16,7 +16,6 @@ PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip() class TestH5ADDataFile(unittest.TestCase): - def setUp(self): self.sample_anndata = self._create_sample_anndata_dataset() self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata) @@ -40,8 +39,12 @@ class TestH5ADDataFile(unittest.TestCase): def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self): with self.assertLogs(level="WARN") as logger: - H5ADDataFile(self.sample_h5ad_filename, dataset_title="My Awesome Dataset", - dataset_about="http://www.awesomedataset.com", use_corpora_schema=False) + H5ADDataFile( + self.sample_h5ad_filename, + dataset_title="My Awesome Dataset", + dataset_about="http://www.awesomedataset.com", + use_corpora_schema=False, + ) self.assertIn("will override any metadata that is extracted", logger.output[0]) @@ -49,10 +52,12 @@ class TestH5ADDataFile(unittest.TestCase): h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False) self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all()) - self.assertEqual(h5ad_file.anndata.obs.sort_index(inplace=True), - self.sample_anndata.obs.sort_index(inplace=True)) - self.assertEqual(h5ad_file.anndata.var.sort_index(inplace=True), - self.sample_anndata.var.sort_index(inplace=True)) + self.assertEqual( + h5ad_file.anndata.obs.sort_index(inplace=True), self.sample_anndata.obs.sort_index(inplace=True) + ) + self.assertEqual( + h5ad_file.anndata.var.sort_index(inplace=True), self.sample_anndata.var.sort_index(inplace=True) + ) for key in h5ad_file.anndata.obsm.keys(): self.assertIn(key, self.sample_anndata.obsm.keys()) @@ -73,8 +78,12 @@ class TestH5ADDataFile(unittest.TestCase): self.assertIn("name_0", h5ad_file.var.columns) def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self): - h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False, - obs_index_column_name="float_category", vars_index_column_name="int_category") + h5ad_file = H5ADDataFile( + self.sample_h5ad_filename, + use_corpora_schema=False, + obs_index_column_name="float_category", + vars_index_column_name="int_category", + ) self.assertNotIn("name_0", h5ad_file.obs.columns) self.assertNotIn("name_0", h5ad_file.var.columns) @@ -82,15 +91,23 @@ class TestH5ADDataFile(unittest.TestCase): def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self): with self.assertRaises(Exception) as exception_context: - H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False, - obs_index_column_name="float_category", vars_index_column_name="bool_category") + H5ADDataFile( + self.sample_h5ad_filename, + use_corpora_schema=False, + obs_index_column_name="float_category", + vars_index_column_name="bool_category", + ) self.assertIn("Please prepare data to contain unique values", str(exception_context.exception)) def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self): with self.assertRaises(Exception) as exception_context: - H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False, - obs_index_column_name="unknown_category", vars_index_column_name="i_dont_exist") + H5ADDataFile( + self.sample_h5ad_filename, + use_corpora_schema=False, + obs_index_column_name="unknown_category", + vars_index_column_name="i_dont_exist", + ) self.assertIn("does not exist", str(exception_context.exception)) @@ -101,8 +118,9 @@ class TestH5ADDataFile(unittest.TestCase): self.assertEqual(h5ad_file.dataset_about, "www.link.com") def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self): - h5ad_file = H5ADDataFile(self.sample_h5ad_filename, dataset_about="override_about", - dataset_title="override_title") + h5ad_file = H5ADDataFile( + self.sample_h5ad_filename, dataset_about="override_about", dataset_title="override_title" + ) self.assertEqual(h5ad_file.dataset_title, "override_title") self.assertEqual(h5ad_file.dataset_about, "override_about") @@ -145,8 +163,11 @@ class TestH5ADDataFile(unittest.TestCase): remove(sparse_with_column_shift_filename) def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False): - expected_directories, expected_obs_files, expected_var_files = \ - self._get_expected_generated_list_of_tiledb_files() + ( + expected_directories, + expected_obs_files, + expected_var_files, + ) = self._get_expected_generated_list_of_tiledb_files() for directory in expected_directories: self.assertTrue(path.isdir(directory)) @@ -187,8 +208,18 @@ class TestH5ADDataFile(unittest.TestCase): var_files.append("bool_category.tdb") var_files.append("int_category.tdb") - return [metadata_directory, main_x_directory, overall_embedding_directory, specific_embedding_directory, - obs_directory, var_directory], obs_files, var_files + return ( + [ + metadata_directory, + main_x_directory, + overall_embedding_directory, + specific_embedding_directory, + obs_directory, + var_directory, + ], + obs_files, + var_files, + ) def _write_anndata_to_file(self, anndata): temporary_filename = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}.h5ad" @@ -204,7 +235,8 @@ class TestH5ADDataFile(unittest.TestCase): random_string_category = Series(data=["a", "b", "b"], dtype="category") random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32) obs_dataframe = DataFrame( - data={"string_category": random_string_category, "float_category": random_float_category}) + data={"string_category": random_string_category, "float_category": random_float_category} + ) obs = obs_dataframe # Create vars @@ -230,6 +262,7 @@ class TestH5ADDataFile(unittest.TestCase): # Set project links to be a dictionary uns["project_links"] = json.dumps( - [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}]) + [{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}] + ) return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns) diff --git a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py index 35ccb886..b3db3cb3 100644 --- a/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py +++ b/server/test/unit/data_anndata/test_anndata_adaptor_data_load.py @@ -3,7 +3,7 @@ import json from server.data_anndata.anndata_adaptor import AnndataAdaptor from server.common.data_locator import DataLocator -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.test import PROJECT_ROOT diff --git a/server/test/unit/data_common/test_matrix_loader.py b/server/test/unit/data_common/test_matrix_loader.py index d365b001..9631de32 100644 --- a/server/test/unit/data_common/test_matrix_loader.py +++ b/server/test/unit/data_common/test_matrix_loader.py @@ -4,7 +4,7 @@ import tempfile import time import unittest -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from server.common.errors import DatasetAccessError from server.data_common.matrix_loader import MatrixDataCacheManager from server.test import FIXTURES_ROOT @@ -38,7 +38,7 @@ class MatrixCacheTest(unittest.TestCase): result = {} for k, v in datasets.items(): # filter out the dirname and the .cxg from the name - newk = int(k[1][len(dirname) + 1: -4]) + newk = int(k[1][len(dirname) + 1 : -4]) result[newk] = v return result diff --git a/server/test/unit/eb/test_eb.py b/server/test/unit/eb/test_eb.py index b43fda24..c148f6bc 100644 --- a/server/test/unit/eb/test_eb.py +++ b/server/test/unit/eb/test_eb.py @@ -3,7 +3,7 @@ import tempfile import requests import subprocess from server.test import PROJECT_ROOT, FIXTURES_ROOT -from server.common.app_config import AppConfig +from server.common.config.app_config import AppConfig from contextlib import contextmanager import time @@ -36,9 +36,7 @@ class Elastic_Beanstalk_Test(unittest.TestCase): c = AppConfig() # test that eb works - c.update_server_config( - multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame" - ) + c.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame") c.complete_config() c.write_config(f"{tempdirname}/config.yaml")