mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-02 00:38:12 +08:00
Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
@@ -1,4 +0,0 @@
|
||||
from server.common.aws_secret_utils import get_secret_key # noqa F504
|
||||
|
||||
DEFAULT_SERVER_PORT = 5005
|
||||
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
|
||||
@@ -1,247 +0,0 @@
|
||||
import yaml
|
||||
from flatten_dict import unflatten
|
||||
|
||||
from server.default_config import get_default_config
|
||||
from server.common.config.dataset_config import DatasetConfig
|
||||
from server.common.config.server_config import ServerConfig
|
||||
from server.common.config.external_config import ExternalConfig
|
||||
from server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
class AppConfig(object):
|
||||
"""
|
||||
AppConfig stores all the configuration for cellxgene.
|
||||
AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
|
||||
The server_config contains attributes that refer to the server process as a whole.
|
||||
The default_dataset_config refers to attributes that are associated with the features and
|
||||
presentations of a dataset.
|
||||
The dataset config attributes can be overridden depending on the url by which the
|
||||
dataset was accessed. These are stored in dataroot_config.
|
||||
AppConfig has methods to initialize, modify, and access the configuration.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
|
||||
# the default configuration (see default_config.py)
|
||||
# TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
|
||||
# defaults within the config class?
|
||||
self.default_config = get_default_config()
|
||||
# the server configuration
|
||||
self.server_config = ServerConfig(self, self.default_config["server"])
|
||||
# the dataset config, unless overridden by an entry in dataroot_config
|
||||
self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
|
||||
# a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
|
||||
# attribute of the server_config. The default dataset config will apply to all datasets unless a different set
|
||||
# of config vars was passed for a specific dataset under the multidataset config. For example:
|
||||
"""
|
||||
per_dataset_config:
|
||||
d1:
|
||||
user_annotations:
|
||||
enable: false
|
||||
d2:
|
||||
user_annotations:
|
||||
enable: true
|
||||
"""
|
||||
# dataroot config
|
||||
self.dataroot_config = {}
|
||||
|
||||
# external config
|
||||
self.external_config = ExternalConfig(self, self.default_config["external"])
|
||||
|
||||
# Set to true when config_completed is called
|
||||
self.is_completed = False
|
||||
|
||||
def get_dataset_config(self, dataroot_key):
|
||||
if self.server_config.single_dataset__datapath:
|
||||
return self.default_dataset_config
|
||||
else:
|
||||
return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
|
||||
|
||||
def check_config(self):
|
||||
"""Verify all the attributes in the config have been type checked"""
|
||||
if not self.is_completed:
|
||||
raise ConfigurationError("The configuration has not been completed")
|
||||
self.server_config.check_config()
|
||||
self.default_dataset_config.check_config()
|
||||
for dataset_config in self.dataroot_config.values():
|
||||
dataset_config.check_config()
|
||||
self.external_config.check_config()
|
||||
|
||||
def update_server_config(self, **kw):
|
||||
self.server_config.update(**kw)
|
||||
self.is_complete = False
|
||||
|
||||
def update_default_dataset_config(self, **kw):
|
||||
self.default_dataset_config.update(**kw)
|
||||
# update all the other dataset configs, if any
|
||||
for value in self.dataroot_config.values():
|
||||
value.update(**kw)
|
||||
self.is_complete = False
|
||||
|
||||
def update_single_config_from_path_and_value(self, path, value):
|
||||
"""Update a single config parameter with the value.
|
||||
Path is a list of string, that gives a path to the config parameter to be updated.
|
||||
For example, path may be ["server","app","port"].
|
||||
"""
|
||||
self.is_complete = False
|
||||
if not isinstance(path, list):
|
||||
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
|
||||
for part in path:
|
||||
if not isinstance(part, str):
|
||||
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
|
||||
|
||||
if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
|
||||
raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
|
||||
|
||||
if path[0] == "server":
|
||||
attr = "__".join(path[1:])
|
||||
try:
|
||||
self.update_server_config(**{attr: value})
|
||||
except ConfigurationError:
|
||||
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
|
||||
elif path[0] == "dataset":
|
||||
attr = "__".join(path[1:])
|
||||
try:
|
||||
self.update_default_dataset_config(**{attr: value})
|
||||
except ConfigurationError:
|
||||
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
|
||||
|
||||
elif path[0] == "per_dataset_config":
|
||||
if len(path) < 2:
|
||||
raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
|
||||
dataroot = path[1]
|
||||
if dataroot not in self.dataroot_config:
|
||||
dataroots = str(list(self.dataroot_config.keys()))
|
||||
raise ConfigurationError(
|
||||
f"unknown dataroot when using per_dataset_config: got '{path}',"
|
||||
f" dataroots specified in config are {dataroots}"
|
||||
)
|
||||
|
||||
attr = "__".join(path[2:])
|
||||
try:
|
||||
self.dataroot_config[dataroot].update(**{attr: value})
|
||||
except ConfigurationError:
|
||||
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
|
||||
|
||||
def update_from_config_file(self, config_file):
|
||||
try:
|
||||
with open(config_file) as yml_file:
|
||||
config = yaml.safe_load(yml_file)
|
||||
except yaml.YAMLError as e:
|
||||
raise ConfigurationError(f"The specified config file contained an error: {e}")
|
||||
except OSError as e:
|
||||
raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
|
||||
|
||||
if config.get("server"):
|
||||
self.server_config.update_from_config(config["server"], "server")
|
||||
if config.get("dataset"):
|
||||
self.default_dataset_config.update_from_config(config["dataset"], "dataset")
|
||||
|
||||
per_dataset_config = config.get("per_dataset_config", {})
|
||||
for key, dataroot_config in per_dataset_config.items():
|
||||
# first create and initialize the dataroot with the default config
|
||||
self.add_dataroot_config(key, **config["dataset"])
|
||||
# then apply the per dataset configuration
|
||||
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
|
||||
|
||||
if config.get("external"):
|
||||
self.external_config.update_from_config(config["external"], "external")
|
||||
|
||||
self.is_complete = False
|
||||
|
||||
def config_to_dict(self):
|
||||
"""return the configuration as an unflattened dict"""
|
||||
server = self.server_config.create_mapping(self.server_config.default_config)
|
||||
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
|
||||
external = self.external_config.create_mapping(self.external_config.default_config)
|
||||
config = dict(server={}, dataset={})
|
||||
for attrname in server.keys():
|
||||
config["server__" + attrname] = getattr(self.server_config, attrname)
|
||||
for attrname in dataset.keys():
|
||||
config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
|
||||
if self.dataroot_config:
|
||||
config["per_dataset_config"] = {}
|
||||
for dataroot_tag, dataroot_config in self.dataroot_config.items():
|
||||
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
|
||||
for attrname in dataset.keys():
|
||||
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
|
||||
for attrname in external.keys():
|
||||
config["external__" + attrname] = getattr(self.external_config, attrname)
|
||||
|
||||
config = unflatten(config, splitter=lambda key: key.split("__"))
|
||||
return config
|
||||
|
||||
def write_config(self, config_file):
|
||||
"""output the config to a yaml file"""
|
||||
config = self.config_to_dict()
|
||||
yaml.dump(config, open(config_file, "w"))
|
||||
|
||||
def changes_from_default(self):
|
||||
"""Return all the attribute that are different from the default"""
|
||||
diff_server = self.server_config.changes_from_default()
|
||||
diff_dataset = self.default_dataset_config.changes_from_default()
|
||||
diff_external = self.external.changes_from_default()
|
||||
diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
|
||||
return diff
|
||||
|
||||
def add_dataroot_config(self, dataroot_tag, **kw):
|
||||
"""Create a new dataset config object based on the default dataset config, and kw parameters"""
|
||||
if dataroot_tag in self.dataroot_config:
|
||||
raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
|
||||
if type(self.server_config.multi_dataset__dataroot) != dict:
|
||||
raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
|
||||
if dataroot_tag not in self.server_config.multi_dataset__dataroot:
|
||||
raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
|
||||
|
||||
self.is_completed = False
|
||||
self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
|
||||
flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
|
||||
config = {key: value[1] for key, value in flat_config.items()}
|
||||
self.dataroot_config[dataroot_tag].update(**config)
|
||||
self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
|
||||
|
||||
def complete_config(self, messagefn=None):
|
||||
"""The configure options are checked, and any additional setup based on the config
|
||||
parameters is done"""
|
||||
|
||||
if messagefn is None:
|
||||
|
||||
def noop(message):
|
||||
pass
|
||||
|
||||
messagefn = noop
|
||||
|
||||
# TODO: to give better error messages we can add a mapping between where each config
|
||||
# attribute originated (e.g. command line argument or config file), then in the error
|
||||
# messages we can give correct context for attributes with bad value.
|
||||
context = dict(messagefn=messagefn)
|
||||
|
||||
# complete config for external_config first, since this may update values in the other sections
|
||||
self.external_config.complete_config(context)
|
||||
self.server_config.complete_config(context)
|
||||
self.default_dataset_config.complete_config(context)
|
||||
for dataroot_config in self.dataroot_config.values():
|
||||
dataroot_config.complete_config(context)
|
||||
|
||||
self.is_completed = True
|
||||
self.check_config()
|
||||
|
||||
def get_matrix_data_cache_manager(self):
|
||||
return self.server_config.matrix_data_cache_manager
|
||||
|
||||
def is_multi_dataset(self):
|
||||
return self.server_config.multi_dataset__dataroot is not None
|
||||
|
||||
def get_title(self, data_adaptor):
|
||||
return (
|
||||
self.server_config.single_dataset__title
|
||||
if self.server_config.single_dataset__title
|
||||
else data_adaptor.get_title()
|
||||
)
|
||||
|
||||
def get_about(self, data_adaptor):
|
||||
return (
|
||||
self.server_config.single_dataset__about
|
||||
if self.server_config.single_dataset__about
|
||||
else data_adaptor.get_about()
|
||||
)
|
||||
@@ -1,132 +0,0 @@
|
||||
import copy
|
||||
|
||||
from flatten_dict import flatten
|
||||
from server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
class BaseConfig(object):
|
||||
"""
|
||||
This class handles the mechanics of updating and checking attributes.
|
||||
Derived classes are expected to store the actual attributes
|
||||
Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
|
||||
"""
|
||||
|
||||
def __init__(self, app_config, default_config, dictval_cases={}):
|
||||
# reference back to the app_config
|
||||
self.app_config = app_config
|
||||
# the complete set of attributes and their default values (unflattened)
|
||||
self.default_config = default_config
|
||||
# attributes where the value may be a dict (and therefore are not flattened)
|
||||
self.dictval_cases = dictval_cases
|
||||
# used to make sure every attribute value is checked
|
||||
self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
|
||||
|
||||
def create_mapping(self, config):
|
||||
"""
|
||||
Create a dictionary where the keys are the name of attributes (using double underscore convention)
|
||||
For example: authentication__type
|
||||
|
||||
The values are a tuple,
|
||||
- the first item of the tuple is a tuple of path elements (location in config 'tree')
|
||||
- the second item is the value of the config parameter
|
||||
|
||||
For example: (('authentication', 'type'), 'session'))
|
||||
"""
|
||||
config_copy = copy.deepcopy(config)
|
||||
mapping = {}
|
||||
|
||||
# special cases where the value could be a dict.
|
||||
# If its value is not None, the entry is added to the mapping, and not included
|
||||
# in the flattening below.
|
||||
for dictval_case in self.dictval_cases:
|
||||
cur = config_copy
|
||||
for part in dictval_case[:-1]:
|
||||
cur = cur.get(part, {})
|
||||
val = cur.get(dictval_case[-1])
|
||||
if val is not None:
|
||||
key = "__".join(dictval_case)
|
||||
mapping[key] = (dictval_case, val)
|
||||
del cur[dictval_case[-1]]
|
||||
|
||||
flat_config = flatten(config_copy)
|
||||
for key, value in flat_config.items():
|
||||
# name of the attribute
|
||||
attr = "__".join(key)
|
||||
mapping[attr] = (key, value)
|
||||
|
||||
return mapping
|
||||
|
||||
def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
|
||||
val = getattr(self, attrname)
|
||||
if type(vtype) in (list, tuple):
|
||||
if type(val) not in vtype:
|
||||
tnames = ",".join([x.__name__ for x in vtype])
|
||||
raise ConfigurationError(
|
||||
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
|
||||
)
|
||||
else:
|
||||
if type(val) != vtype:
|
||||
raise ConfigurationError(
|
||||
f"Invalid type for attribute: {attrname}, "
|
||||
f"expected type {vtype.__name__}, got {type(val).__name__}"
|
||||
)
|
||||
|
||||
self.attr_checked[attrname] = True
|
||||
|
||||
def check_config(self):
|
||||
mapping = self.create_mapping(self.default_config)
|
||||
for key in mapping.keys():
|
||||
if not self.attr_checked[key]:
|
||||
raise ConfigurationError(f"The attr '{key}' has not been checked")
|
||||
|
||||
def update(self, **kw):
|
||||
"""Update the attributes defined in kw with their new values."""
|
||||
for key, value in kw.items():
|
||||
if not hasattr(self, key):
|
||||
|
||||
# check if the key is setting into a dictval entry.
|
||||
found_dictval = False
|
||||
for dictval in self.dictval_cases:
|
||||
dictvalname = "__".join(dictval)
|
||||
if dictvalname + "__" in key:
|
||||
dictkey = key[len(dictvalname) + 2 :]
|
||||
curdictval = getattr(self, dictvalname)
|
||||
if curdictval is None:
|
||||
setattr(self, dictvalname, dict(dictkey=value))
|
||||
else:
|
||||
curdictval[dictkey] = value
|
||||
|
||||
found_dictval = True
|
||||
break
|
||||
|
||||
if found_dictval:
|
||||
continue
|
||||
raise ConfigurationError(f"unknown config parameter {key}.")
|
||||
try:
|
||||
if type(value) == tuple:
|
||||
# convert tuple values to list values
|
||||
value = list(value)
|
||||
setattr(self, key, value)
|
||||
except KeyError:
|
||||
raise ConfigurationError(f"Unable to set config parameter {key}.")
|
||||
|
||||
self.attr_checked[key] = False
|
||||
|
||||
def update_from_config(self, config, prefix):
|
||||
mapping = self.create_mapping(config)
|
||||
for attr, (key, value) in mapping.items():
|
||||
if not hasattr(self, attr):
|
||||
raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
|
||||
setattr(self, attr, value)
|
||||
|
||||
self.attr_checked[attr] = False
|
||||
|
||||
def changes_from_default(self):
|
||||
"""Return all the attribute that are different from the default"""
|
||||
mapping = self.create_mapping(self.default_config)
|
||||
diff = []
|
||||
for attrname, (key, defval) in mapping.items():
|
||||
curval = getattr(self, attrname)
|
||||
if curval != defval:
|
||||
diff.append((attrname, curval, defval))
|
||||
return diff
|
||||
@@ -1,122 +0,0 @@
|
||||
from server import display_version as cellxgene_display_version
|
||||
|
||||
|
||||
def get_client_config(app_config, data_adaptor):
|
||||
"""
|
||||
Return the configuration as required by the /config REST route
|
||||
"""
|
||||
|
||||
server_config = app_config.server_config
|
||||
dataset_config = data_adaptor.dataset_config
|
||||
annotation = dataset_config.user_annotations
|
||||
auth = server_config.auth
|
||||
|
||||
# FIXME The current set of config is not consistently presented:
|
||||
# we have camalCase, hyphen-text, and underscore_text
|
||||
|
||||
# make sure the configuration has been checked.
|
||||
app_config.check_config()
|
||||
|
||||
# display_names
|
||||
title = app_config.get_title(data_adaptor)
|
||||
about = app_config.get_about(data_adaptor)
|
||||
|
||||
display_names = dict(engine=data_adaptor.get_name(), dataset=title)
|
||||
|
||||
# library_versions
|
||||
library_versions = {}
|
||||
library_versions.update(data_adaptor.get_library_versions())
|
||||
library_versions["cellxgene"] = cellxgene_display_version
|
||||
|
||||
# links
|
||||
links = {"about-dataset": about}
|
||||
|
||||
# parameters
|
||||
parameters = {
|
||||
"layout": dataset_config.embeddings__names,
|
||||
"max-category-items": dataset_config.presentation__max_categories,
|
||||
"obs_names": server_config.single_dataset__obs_names,
|
||||
"var_names": server_config.single_dataset__var_names,
|
||||
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
|
||||
"backed": server_config.adaptor__anndata_adaptor__backed,
|
||||
"disable-diffexp": not dataset_config.diffexp__enable,
|
||||
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
|
||||
"annotations": False,
|
||||
"annotations_file": None,
|
||||
"annotations_dir": None,
|
||||
"annotations_cell_ontology_enabled": False,
|
||||
"annotations_cell_ontology_obopath": None,
|
||||
"annotations_cell_ontology_terms": None,
|
||||
"custom_colors": dataset_config.presentation__custom_colors,
|
||||
"diffexp-may-be-slow": False,
|
||||
"about_legal_tos": dataset_config.app__about_legal_tos,
|
||||
"about_legal_privacy": dataset_config.app__about_legal_privacy,
|
||||
}
|
||||
|
||||
# corpora dataset_props
|
||||
# TODO/Note: putting info from the dataset into the /config is not ideal.
|
||||
# However, it is definitely not part of /schema, and we do not have a top-level
|
||||
# route for data properties. Consider creating one at some point.
|
||||
corpora_props = data_adaptor.get_corpora_props()
|
||||
if corpora_props and "default_embedding" in corpora_props:
|
||||
default_embedding = corpora_props["default_embedding"]
|
||||
if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
|
||||
default_embedding = default_embedding[2:] # drop X_ prefix
|
||||
if default_embedding in data_adaptor.get_embedding_names():
|
||||
parameters["default_embedding"] = default_embedding
|
||||
|
||||
data_adaptor.update_parameters(parameters)
|
||||
if annotation:
|
||||
annotation.update_parameters(parameters, data_adaptor)
|
||||
|
||||
# gather it all together
|
||||
client_config = {}
|
||||
config = client_config["config"] = {}
|
||||
config["displayNames"] = display_names
|
||||
config["library_versions"] = library_versions
|
||||
config["links"] = links
|
||||
config["parameters"] = parameters
|
||||
config["corpora_props"] = corpora_props
|
||||
config["limits"] = {
|
||||
"column_request_max": server_config.limits__column_request_max,
|
||||
"diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
|
||||
}
|
||||
|
||||
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
|
||||
config["authentication"] = {
|
||||
"requires_client_login": auth.requires_client_login(),
|
||||
}
|
||||
if auth.requires_client_login():
|
||||
config["authentication"].update(
|
||||
{
|
||||
# Todo why are these stored on the data_adaptor?
|
||||
"login": auth.get_login_url(data_adaptor),
|
||||
"logout": auth.get_logout_url(data_adaptor),
|
||||
}
|
||||
)
|
||||
|
||||
return client_config
|
||||
|
||||
|
||||
def get_client_userinfo(app_config, data_adaptor):
|
||||
"""
|
||||
Return the userinfo as required by the /userinfo REST route
|
||||
"""
|
||||
|
||||
server_config = app_config.server_config
|
||||
dataset_config = data_adaptor.dataset_config
|
||||
auth = server_config.auth
|
||||
|
||||
# make sure the configuration has been checked.
|
||||
app_config.check_config()
|
||||
|
||||
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
|
||||
userinfo = {}
|
||||
userinfo["userinfo"] = {
|
||||
"is_authenticated": auth.is_user_authenticated(),
|
||||
"username": auth.get_user_name(),
|
||||
"user_id": auth.get_user_id(),
|
||||
"email": auth.get_user_email(),
|
||||
"picture": auth.get_user_picture(),
|
||||
}
|
||||
return userinfo
|
||||
@@ -1,230 +0,0 @@
|
||||
import os
|
||||
from os.path import splitext, isdir
|
||||
|
||||
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
|
||||
from server.common.annotations.local_file_csv import AnnotationsLocalFile
|
||||
from server.common.config.base_config import BaseConfig
|
||||
from server.common.errors import ConfigurationError, OntologyLoadFailure
|
||||
from server.compute.scanpy import get_scanpy_module
|
||||
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
|
||||
from server.db.db_utils import DbUtils
|
||||
|
||||
|
||||
class DatasetConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with a dataset."""
|
||||
|
||||
def __init__(self, tag, app_config, default_config):
|
||||
super().__init__(app_config, default_config)
|
||||
self.tag = tag
|
||||
try:
|
||||
self.app__scripts = default_config["app"]["scripts"]
|
||||
self.app__inline_scripts = default_config["app"]["inline_scripts"]
|
||||
self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
|
||||
self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
|
||||
self.app__authentication_enable = default_config["app"]["authentication_enable"]
|
||||
|
||||
self.presentation__max_categories = default_config["presentation"]["max_categories"]
|
||||
self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
|
||||
|
||||
self.user_annotations__enable = default_config["user_annotations"]["enable"]
|
||||
self.user_annotations__type = default_config["user_annotations"]["type"]
|
||||
self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
|
||||
"directory"
|
||||
]
|
||||
self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
|
||||
self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
|
||||
self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
|
||||
"obo_location"
|
||||
]
|
||||
self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
|
||||
"hosted_tiledb_array"
|
||||
]["db_uri"]
|
||||
self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
|
||||
"hosted_tiledb_array"
|
||||
]["hosted_file_directory"]
|
||||
|
||||
self.embeddings__names = default_config["embeddings"]["names"]
|
||||
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
|
||||
|
||||
self.diffexp__enable = default_config["diffexp"]["enable"]
|
||||
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
|
||||
self.diffexp__top_n = default_config["diffexp"]["top_n"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
# The annotation object is created during complete_config and stored here.
|
||||
self.user_annotations = None
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_app()
|
||||
self.handle_presentation()
|
||||
self.handle_user_annotations(context)
|
||||
self.handle_embeddings()
|
||||
self.handle_diffexp(context)
|
||||
|
||||
def handle_app(self):
|
||||
self.validate_correct_type_of_configuration_attribute("app__scripts", list)
|
||||
self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
|
||||
self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
|
||||
|
||||
# scripts can be string (filename) or dict (attributes). Convert string to dict.
|
||||
scripts = []
|
||||
for script in self.app__scripts:
|
||||
try:
|
||||
if isinstance(script, str):
|
||||
scripts.append({"src": script})
|
||||
elif isinstance(script, dict) and isinstance(script["src"], str):
|
||||
scripts.append(script)
|
||||
else:
|
||||
raise Exception
|
||||
except Exception as e:
|
||||
raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
|
||||
|
||||
self.app__scripts = scripts
|
||||
|
||||
def handle_presentation(self):
|
||||
self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
|
||||
self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
|
||||
|
||||
def handle_user_annotations(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__local_file_csv__directory", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__local_file_csv__file", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__ontology__obo_location", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
|
||||
)
|
||||
if self.user_annotations__enable:
|
||||
server_config = self.app_config.server_config
|
||||
if not self.app__authentication_enable:
|
||||
raise ConfigurationError("user annotations requires authentication to be enabled")
|
||||
if not server_config.auth.is_valid_authentication_type():
|
||||
auth_type = server_config.authentication__type
|
||||
raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
|
||||
|
||||
if self.user_annotations__type == "local_file_csv":
|
||||
self.handle_local_file_csv_annotations()
|
||||
elif self.user_annotations__type == "hosted_tiledb_array":
|
||||
self.handle_hosted_tiledb_annotations()
|
||||
else:
|
||||
raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
|
||||
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
|
||||
try:
|
||||
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
|
||||
except OntologyLoadFailure as e:
|
||||
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
|
||||
else:
|
||||
self.check_annotation_config_vars_not_set(context)
|
||||
|
||||
def handle_local_file_csv_annotations(self):
|
||||
dirname = self.user_annotations__local_file_csv__directory
|
||||
filename = self.user_annotations__local_file_csv__file
|
||||
if filename is not None and dirname is not None:
|
||||
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
|
||||
|
||||
if filename is not None:
|
||||
lf_name, lf_ext = splitext(filename)
|
||||
if lf_ext and lf_ext != ".csv":
|
||||
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
|
||||
|
||||
if dirname is not None and not isdir(dirname):
|
||||
try:
|
||||
os.mkdir(dirname)
|
||||
except OSError:
|
||||
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
|
||||
|
||||
self.user_annotations = AnnotationsLocalFile(dirname, filename)
|
||||
|
||||
# if the user has specified a fixed label file, go ahead and validate it
|
||||
# so that we can remove errors early in the process.
|
||||
server_config = self.app_config.server_config
|
||||
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
|
||||
with server_config.matrix_data_cache_manager.data_adaptor(
|
||||
self.tag, server_config.single_dataset__datapath, self.app_config
|
||||
) as data_adaptor:
|
||||
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
|
||||
|
||||
def handle_hosted_tiledb_annotations(self):
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__hosted_tiledb_array__hosted_file_directory", str
|
||||
)
|
||||
self.user_annotations = AnnotationsHostedTileDB(
|
||||
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
|
||||
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
|
||||
)
|
||||
|
||||
def check_annotation_config_vars_not_set(self, context):
|
||||
if self.user_annotations__type is not None:
|
||||
dirname = self.user_annotations__local_file_csv__directory
|
||||
filename = self.user_annotations__local_file_csv__file
|
||||
db_uri = self.user_annotations__hosted_tiledb_array__db_uri
|
||||
hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
|
||||
if filename is not None:
|
||||
context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
|
||||
if dirname is not None:
|
||||
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
|
||||
if db_uri is not None:
|
||||
context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
|
||||
if hosted_file_dirname is not None:
|
||||
context["messagefn"](
|
||||
"Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
|
||||
)
|
||||
|
||||
if self.user_annotations__ontology__enable:
|
||||
context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
|
||||
if self.user_annotations__ontology__obo_location is not None:
|
||||
context["messagefn"](
|
||||
"Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
|
||||
)
|
||||
|
||||
def handle_embeddings(self):
|
||||
self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
|
||||
self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
|
||||
|
||||
server_config = self.app_config.server_config
|
||||
if self.embeddings__enable_reembedding:
|
||||
if server_config.single_dataset__datapath:
|
||||
matrix_data_loader = MatrixDataLoader(
|
||||
server_config.single_dataset__datapath, app_config=self.app_config
|
||||
)
|
||||
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
|
||||
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
|
||||
if server_config.adaptor__anndata_adaptor__backed:
|
||||
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
|
||||
|
||||
try:
|
||||
get_scanpy_module()
|
||||
except NotImplementedError:
|
||||
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
|
||||
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
|
||||
|
||||
def handle_diffexp(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
|
||||
|
||||
server_config = self.app_config.server_config
|
||||
if server_config.single_dataset__datapath:
|
||||
with server_config.matrix_data_cache_manager.data_adaptor(
|
||||
self.tag, server_config.single_dataset__datapath, self.app_config
|
||||
) as data_adaptor:
|
||||
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
|
||||
context["messagefn"](
|
||||
"CAUTION: due to the size of your dataset, "
|
||||
"running differential expression may take longer or fail."
|
||||
)
|
||||
@@ -1,96 +0,0 @@
|
||||
import os
|
||||
|
||||
from server.common.config.base_config import BaseConfig
|
||||
from server.common.errors import ConfigurationError
|
||||
from server.common.config import get_secret_key
|
||||
from server.common.errors import SecretKeyRetrievalError
|
||||
from server.common.utils.type_conversion_utils import convert_string_to_value
|
||||
|
||||
|
||||
class ExternalConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with external configuration sources, such as
|
||||
environment variables or the AWS Secrets Manager."""
|
||||
|
||||
def __init__(self, app_config, default_config):
|
||||
super().__init__(app_config, default_config)
|
||||
try:
|
||||
self.environment = default_config["environment"]
|
||||
self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
|
||||
self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_environment(context)
|
||||
self.handle_aws_secrets_manager(context)
|
||||
|
||||
def handle_environment(self, context):
|
||||
"""For each environment variable defined, get the value (if it is set),
|
||||
and set the specified config parameter"""
|
||||
self.validate_correct_type_of_configuration_attribute("environment", list)
|
||||
for envdict in self.environment:
|
||||
name = envdict.get("name")
|
||||
if name is None:
|
||||
raise ConfigurationError("environment: 'name' is missing")
|
||||
required = envdict.get("required", False)
|
||||
if type(required) != bool:
|
||||
raise ConfigurationError("environment: 'required' must be a bool")
|
||||
path = envdict.get("path")
|
||||
if path is None:
|
||||
raise ConfigurationError("environment: 'path' is missing")
|
||||
|
||||
value = os.environ.get(name)
|
||||
if value is None:
|
||||
if required:
|
||||
raise ConfigurationError(f"required environment variable '{name}' not set")
|
||||
else:
|
||||
value = convert_string_to_value(value)
|
||||
self.app_config.update_single_config_from_path_and_value(path, value)
|
||||
|
||||
def handle_aws_secrets_manager(self, context):
|
||||
"""For each aws secret defined, get the key/values, and set the specified config parameter"""
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
|
||||
|
||||
if not self.aws_secrets_manager__secrets:
|
||||
return
|
||||
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
|
||||
|
||||
for secret in self.aws_secrets_manager__secrets:
|
||||
secret_name = secret.get("name")
|
||||
if secret_name is None:
|
||||
raise ConfigurationError("aws_secrets_manager: 'name' is missing")
|
||||
if not isinstance(secret_name, str):
|
||||
raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
|
||||
|
||||
try:
|
||||
secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
|
||||
except SecretKeyRetrievalError as e:
|
||||
raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
|
||||
|
||||
values = secret.get("values")
|
||||
if values is None:
|
||||
raise ConfigurationError("aws_secrets_manager: 'values' is missing")
|
||||
if not isinstance(values, list):
|
||||
raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
|
||||
|
||||
for value in values:
|
||||
key = value.get("key")
|
||||
if key is None:
|
||||
raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
|
||||
path = value.get("path")
|
||||
if path is None:
|
||||
raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
|
||||
required = value.get("required", False)
|
||||
if type(required) != bool:
|
||||
raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
|
||||
|
||||
secret_value = secret_dict.get(key)
|
||||
if secret_value is None:
|
||||
if required:
|
||||
raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
|
||||
else:
|
||||
secret_value = convert_string_to_value(secret_value)
|
||||
self.app_config.update_single_config_from_path_and_value(path, secret_value)
|
||||
@@ -1,387 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from os.path import basename
|
||||
from urllib.parse import urlparse, quote_plus
|
||||
|
||||
from server.auth.auth import AuthTypeFactory
|
||||
from server.common.config.base_config import BaseConfig
|
||||
from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
|
||||
from server.common.errors import ConfigurationError, DatasetAccessError
|
||||
from server.common.data_locator import discover_s3_region_name
|
||||
from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
|
||||
from server.compute import diffexp_cxg as diffexp_tiledb
|
||||
from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
|
||||
|
||||
|
||||
class ServerConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with the server."""
|
||||
|
||||
def __init__(self, app_config, default_config):
|
||||
dictval_cases = [
|
||||
("app", "csp_directives"),
|
||||
("authentication", "params_oauth", "cookie"),
|
||||
("authentication", "params_oauth", "jwt_decode_options"),
|
||||
("adaptor", "cxg_adaptor", "tiledb_ctx"),
|
||||
("multi_dataset", "dataroot"),
|
||||
]
|
||||
super().__init__(app_config, default_config, dictval_cases)
|
||||
|
||||
try:
|
||||
self.app__verbose = default_config["app"]["verbose"]
|
||||
self.app__debug = default_config["app"]["debug"]
|
||||
self.app__host = default_config["app"]["host"]
|
||||
self.app__port = default_config["app"]["port"]
|
||||
self.app__open_browser = default_config["app"]["open_browser"]
|
||||
self.app__force_https = default_config["app"]["force_https"]
|
||||
self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
|
||||
self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
|
||||
self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
|
||||
self.app__csp_directives = default_config["app"]["csp_directives"]
|
||||
self.app__api_base_url = default_config["app"]["api_base_url"]
|
||||
self.app__web_base_url = default_config["app"]["web_base_url"]
|
||||
|
||||
self.authentication__type = default_config["authentication"]["type"]
|
||||
self.authentication__insecure_test_environment = default_config["authentication"][
|
||||
"insecure_test_environment"
|
||||
]
|
||||
self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
|
||||
"oauth_api_base_url"
|
||||
]
|
||||
self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
|
||||
self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
|
||||
"client_secret"
|
||||
]
|
||||
self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
|
||||
"jwt_decode_options"
|
||||
]
|
||||
self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
|
||||
"session_cookie"
|
||||
]
|
||||
self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
|
||||
|
||||
self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
|
||||
self.multi_dataset__index = default_config["multi_dataset"]["index"]
|
||||
self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
|
||||
self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
|
||||
"max_datasets"
|
||||
]
|
||||
self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
|
||||
"timelimit_s"
|
||||
]
|
||||
|
||||
self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
|
||||
self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
|
||||
self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
|
||||
self.single_dataset__about = default_config["single_dataset"]["about"]
|
||||
self.single_dataset__title = default_config["single_dataset"]["title"]
|
||||
|
||||
self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
|
||||
self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
|
||||
self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
|
||||
|
||||
self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
|
||||
|
||||
self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
|
||||
self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
|
||||
|
||||
self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
|
||||
self.limits__column_request_max = default_config["limits"]["column_request_max"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
# The matrix data cache manager is created during the complete_config and stored here.
|
||||
self.matrix_data_cache_manager = None
|
||||
|
||||
# The authentication object
|
||||
self.auth = None
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_app(context)
|
||||
self.handle_data_source()
|
||||
self.handle_authentication()
|
||||
self.handle_data_locator()
|
||||
self.handle_adaptor() # may depend on data_locator
|
||||
self.handle_single_dataset(context) # may depend on adaptor
|
||||
self.handle_multi_dataset() # may depend on adaptor
|
||||
self.handle_diffexp()
|
||||
self.handle_limits()
|
||||
|
||||
self.check_config()
|
||||
|
||||
def handle_app(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__debug", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__host", str)
|
||||
self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
|
||||
self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
|
||||
self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
|
||||
self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
|
||||
|
||||
if self.app__port:
|
||||
try:
|
||||
if not is_port_available(self.app__host, self.app__port):
|
||||
raise ConfigurationError(
|
||||
f"The port selected {self.app__port} is in use, please configure an open port."
|
||||
)
|
||||
except OverflowError:
|
||||
raise ConfigurationError(f"Invalid port: {self.app__port}")
|
||||
else:
|
||||
try:
|
||||
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
|
||||
except ValueError:
|
||||
raise ConfigurationError(
|
||||
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
|
||||
)
|
||||
try:
|
||||
self.app__port = find_available_port(self.app__host, default_server_port)
|
||||
except OverflowError:
|
||||
raise ConfigurationError(f"Invalid port: {default_server_port}")
|
||||
|
||||
if self.app__debug:
|
||||
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
|
||||
self.app__verbose = True
|
||||
self.app__open_browser = False
|
||||
else:
|
||||
warnings.formatwarning = custom_format_warning
|
||||
|
||||
if not self.app__verbose:
|
||||
sys.tracebacklimit = 0
|
||||
|
||||
# CSP Directives are a dict of string: list(string) or string: string
|
||||
if self.app__csp_directives is not None:
|
||||
for k, v in self.app__csp_directives.items():
|
||||
if not isinstance(k, str):
|
||||
raise ConfigurationError("CSP directive names must be a string.")
|
||||
if isinstance(v, list):
|
||||
for policy in v:
|
||||
if not isinstance(policy, str):
|
||||
raise ConfigurationError("CSP directive value must be a string or list of strings.")
|
||||
elif not isinstance(v, str):
|
||||
raise ConfigurationError("CSP directive value must be a string or list of strings.")
|
||||
|
||||
if self.app__web_base_url is None:
|
||||
self.app__web_base_url = self.app__api_base_url
|
||||
|
||||
def handle_authentication(self):
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
|
||||
|
||||
if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
|
||||
raise ConfigurationError("Test auth can only be used in an insecure test environment")
|
||||
|
||||
# oauth
|
||||
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"authentication__params_oauth__oauth_api_base_url", ptypes
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"authentication__params_oauth__jwt_decode_options", (type(None), dict)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
|
||||
|
||||
if self.authentication__params_oauth__session_cookie:
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"authentication__params_oauth__cookie", (type(None), dict)
|
||||
)
|
||||
else:
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
|
||||
|
||||
self.auth = AuthTypeFactory.create(self.authentication__type, self)
|
||||
if self.auth is None:
|
||||
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
|
||||
|
||||
def handle_data_locator(self):
|
||||
self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
|
||||
if self.data_locator__s3__region_name is True:
|
||||
path = self.single_dataset__datapath or self.multi_dataset__dataroot
|
||||
|
||||
if type(path) == dict:
|
||||
# if multi_dataset__dataroot is a dict, then use the first key
|
||||
# that is in s3. NOTE: it is not supported to have dataroots
|
||||
# in different regions.
|
||||
paths = [val.get("dataroot") for val in path.values()]
|
||||
for path in paths:
|
||||
if path.startswith("s3://"):
|
||||
break
|
||||
if path.startswith("s3://"):
|
||||
region_name = discover_s3_region_name(path)
|
||||
if region_name is None:
|
||||
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
|
||||
else:
|
||||
region_name = None
|
||||
self.data_locator__s3__region_name = region_name
|
||||
|
||||
def handle_data_source(self):
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
|
||||
|
||||
if self.single_dataset__datapath and self.multi_dataset__dataroot:
|
||||
raise ConfigurationError(
|
||||
"You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
|
||||
)
|
||||
if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
|
||||
raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
|
||||
|
||||
def handle_single_dataset(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
|
||||
|
||||
if self.single_dataset__datapath is None:
|
||||
return
|
||||
|
||||
# create the matrix data cache manager:
|
||||
if self.matrix_data_cache_manager is None:
|
||||
self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
|
||||
|
||||
# preload this data set
|
||||
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
|
||||
try:
|
||||
matrix_data_loader.pre_load_validation()
|
||||
except DatasetAccessError as e:
|
||||
raise ConfigurationError(str(e))
|
||||
|
||||
file_size = matrix_data_loader.file_size()
|
||||
file_basename = basename(self.single_dataset__datapath)
|
||||
if file_size > BIG_FILE_SIZE_THRESHOLD:
|
||||
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
|
||||
else:
|
||||
context["messagefn"](f"Loading data from {file_basename}.")
|
||||
|
||||
if self.single_dataset__about:
|
||||
|
||||
def url_check(url):
|
||||
try:
|
||||
result = urlparse(url)
|
||||
if all([result.scheme, result.netloc]):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
if not url_check(self.single_dataset__about):
|
||||
raise ConfigurationError(
|
||||
"Must provide an absolute URL for --about. (Example format: http://example.com)"
|
||||
)
|
||||
|
||||
def handle_multi_dataset(self):
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
|
||||
self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
|
||||
)
|
||||
|
||||
if self.multi_dataset__dataroot is None:
|
||||
return
|
||||
|
||||
if type(self.multi_dataset__dataroot) == str:
|
||||
default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
|
||||
self.multi_dataset__dataroot = dict(d=default_dict)
|
||||
|
||||
for tag, dataroot_dict in self.multi_dataset__dataroot.items():
|
||||
if "base_url" not in dataroot_dict:
|
||||
raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
|
||||
if "dataroot" not in dataroot_dict:
|
||||
raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
|
||||
|
||||
base_url = dataroot_dict["base_url"]
|
||||
|
||||
# sanity check for well formed base urls
|
||||
bad = False
|
||||
if type(base_url) != str:
|
||||
bad = True
|
||||
elif os.path.normpath(base_url) != base_url:
|
||||
bad = True
|
||||
else:
|
||||
base_url_parts = base_url.split("/")
|
||||
if [quote_plus(part) for part in base_url_parts] != base_url_parts:
|
||||
bad = True
|
||||
if ".." in base_url_parts:
|
||||
bad = True
|
||||
if bad:
|
||||
raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
|
||||
|
||||
# verify all the base_urls are unique
|
||||
base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
|
||||
if len(base_urls) > len(set(base_urls)):
|
||||
raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
|
||||
|
||||
# error checking
|
||||
for mtype in self.multi_dataset__allowed_matrix_types:
|
||||
try:
|
||||
MatrixDataType(mtype)
|
||||
except ValueError:
|
||||
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
|
||||
|
||||
# create the matrix data cache manager:
|
||||
if self.matrix_data_cache_manager is None:
|
||||
self.matrix_data_cache_manager = MatrixDataCacheManager(
|
||||
max_cached=self.multi_dataset__matrix_cache__max_datasets,
|
||||
timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
|
||||
)
|
||||
|
||||
def handle_diffexp(self):
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
|
||||
|
||||
max_workers = self.diffexp__alg_cxg__max_workers
|
||||
cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
|
||||
cpu_count = os.cpu_count()
|
||||
max_workers = min(max_workers, cpu_multiplier * cpu_count)
|
||||
diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
|
||||
|
||||
def handle_adaptor(self):
|
||||
# cxg
|
||||
self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
|
||||
regionkey = "vfs.s3.region"
|
||||
if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
|
||||
if type(self.data_locator__s3__region_name) == str:
|
||||
self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
|
||||
|
||||
from server.data_cxg.cxg_adaptor import CxgAdaptor
|
||||
|
||||
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
|
||||
|
||||
# anndata
|
||||
self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
|
||||
|
||||
def handle_limits(self):
|
||||
self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
|
||||
self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
|
||||
|
||||
def exceeds_limit(self, limit_name, value):
|
||||
limit_value = getattr(self, "limits__" + limit_name, None)
|
||||
if limit_value is None: # disabled
|
||||
return False
|
||||
return value > limit_value
|
||||
|
||||
def get_api_base_url(self):
|
||||
if self.app__api_base_url == "local":
|
||||
return f"http://{self.app__host}:{self.app__port}"
|
||||
if self.app__api_base_url and self.app__api_base_url.endswith("/"):
|
||||
return self.app__api_base_url[:-1]
|
||||
return self.app__api_base_url
|
||||
|
||||
def get_web_base_url(self):
|
||||
if self.app__web_base_url == "local":
|
||||
return f"http://{self.app__host}:{self.app__port}"
|
||||
if self.app__web_base_url is None:
|
||||
return self.get_api_base_url()
|
||||
if self.app__web_base_url.endswith("/"):
|
||||
return self.app__web_base_url[:-1]
|
||||
return self.app__web_base_url
|
||||
Reference in New Issue
Block a user