mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-04 06:38:11 +08:00
Split out the local backend (#2052)
This splits the backend into two parts: the local backend for desktop cellxgene and the AWS backend for hosted cellxgene. The local backend is in local_server while the hosted remains in server. The general idea is to copy everything from server to local_server, pull unneeded stuff out of local_server, and keep server as-is for this PR. Not touching server means all the infra and deployment code will continue working just as it did before so we can make those changes incrementally.
This commit is contained in:
Marcus Kinsella
@@ -0,0 +1,4 @@
|
||||
from local_server.common.aws_secret_utils import get_secret_key # noqa F504
|
||||
|
||||
DEFAULT_SERVER_PORT = 5005
|
||||
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
|
||||
@@ -0,0 +1,171 @@
|
||||
import yaml
|
||||
from flatten_dict import unflatten
|
||||
|
||||
from local_server.default_config import get_default_config
|
||||
from local_server.common.config.dataset_config import DatasetConfig
|
||||
from local_server.common.config.server_config import ServerConfig
|
||||
from local_server.common.config.external_config import ExternalConfig
|
||||
from local_server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
class AppConfig(object):
|
||||
"""
|
||||
AppConfig stores all the configuration for cellxgene.
|
||||
AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
|
||||
The server_config contains attributes that refer to the server process as a whole.
|
||||
The dataset_config refers to attributes that are associated with the features and
|
||||
presentations of a dataset.
|
||||
AppConfig has methods to initialize, modify, and access the configuration.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
|
||||
# the default configuration (see default_config.py)
|
||||
# TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
|
||||
# defaults within the config class?
|
||||
self.default_config = get_default_config()
|
||||
# the server configuration
|
||||
self.server_config = ServerConfig(self, self.default_config["server"])
|
||||
# the dataset config
|
||||
self.dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
|
||||
# external config
|
||||
self.external_config = ExternalConfig(self, self.default_config["external"])
|
||||
|
||||
# Set to true when config_completed is called
|
||||
self.is_completed = False
|
||||
|
||||
def get_dataset_config(self):
|
||||
return self.dataset_config
|
||||
|
||||
def check_config(self):
|
||||
"""Verify all the attributes in the config have been type checked"""
|
||||
if not self.is_completed:
|
||||
raise ConfigurationError("The configuration has not been completed")
|
||||
self.server_config.check_config()
|
||||
self.dataset_config.check_config()
|
||||
self.external_config.check_config()
|
||||
|
||||
def update_server_config(self, **kw):
|
||||
self.server_config.update(**kw)
|
||||
self.is_complete = False
|
||||
|
||||
def update_dataset_config(self, **kw):
|
||||
self.dataset_config.update(**kw)
|
||||
self.is_complete = False
|
||||
|
||||
def update_single_config_from_path_and_value(self, path, value):
|
||||
"""Update a single config parameter with the value.
|
||||
Path is a list of string, that gives a path to the config parameter to be updated.
|
||||
For example, path may be ["server","app","port"].
|
||||
"""
|
||||
self.is_complete = False
|
||||
if not isinstance(path, list):
|
||||
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
|
||||
for part in path:
|
||||
if not isinstance(part, str):
|
||||
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
|
||||
|
||||
if len(path) < 1 or path[0] not in ("server", "dataset"):
|
||||
raise ConfigurationError("path must start with 'server', or 'dataset'")
|
||||
|
||||
if path[0] == "server":
|
||||
attr = "__".join(path[1:])
|
||||
try:
|
||||
self.update_server_config(**{attr: value})
|
||||
except ConfigurationError:
|
||||
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
|
||||
elif path[0] == "dataset":
|
||||
attr = "__".join(path[1:])
|
||||
try:
|
||||
self.update_dataset_config(**{attr: value})
|
||||
except ConfigurationError:
|
||||
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
|
||||
|
||||
def update_from_config_file(self, config_file):
|
||||
try:
|
||||
with open(config_file) as yml_file:
|
||||
config = yaml.safe_load(yml_file)
|
||||
except yaml.YAMLError as e:
|
||||
raise ConfigurationError(f"The specified config file contained an error: {e}")
|
||||
except OSError as e:
|
||||
raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
|
||||
|
||||
if config.get("server"):
|
||||
self.server_config.update_from_config(config["server"], "server")
|
||||
if config.get("dataset"):
|
||||
self.dataset_config.update_from_config(config["dataset"], "dataset")
|
||||
|
||||
if config.get("external"):
|
||||
self.external_config.update_from_config(config["external"], "external")
|
||||
|
||||
self.is_complete = False
|
||||
|
||||
def config_to_dict(self):
|
||||
"""return the configuration as an unflattened dict"""
|
||||
server = self.server_config.create_mapping(self.server_config.default_config)
|
||||
dataset = self.dataset_config.create_mapping(self.dataset_config.default_config)
|
||||
external = self.external_config.create_mapping(self.external_config.default_config)
|
||||
config = dict(server={}, dataset={})
|
||||
for attrname in server.keys():
|
||||
config["server__" + attrname] = getattr(self.server_config, attrname)
|
||||
for attrname in dataset.keys():
|
||||
config["dataset__" + attrname] = getattr(self.dataset_config, attrname)
|
||||
for attrname in external.keys():
|
||||
config["external__" + attrname] = getattr(self.external_config, attrname)
|
||||
|
||||
config = unflatten(config, splitter=lambda key: key.split("__"))
|
||||
return config
|
||||
|
||||
def write_config(self, config_file):
|
||||
"""output the config to a yaml file"""
|
||||
config = self.config_to_dict()
|
||||
yaml.dump(config, open(config_file, "w"))
|
||||
|
||||
def changes_from_default(self):
|
||||
"""Return all the attribute that are different from the default"""
|
||||
diff_server = self.server_config.changes_from_default()
|
||||
diff_dataset = self.dataset_config.changes_from_default()
|
||||
diff_external = self.external.changes_from_default()
|
||||
diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
|
||||
return diff
|
||||
|
||||
def complete_config(self, messagefn=None):
|
||||
"""The configure options are checked, and any additional setup based on the config
|
||||
parameters is done"""
|
||||
|
||||
if messagefn is None:
|
||||
|
||||
def noop(message):
|
||||
pass
|
||||
|
||||
messagefn = noop
|
||||
|
||||
# TODO: to give better error messages we can add a mapping between where each config
|
||||
# attribute originated (e.g. command line argument or config file), then in the error
|
||||
# messages we can give correct context for attributes with bad value.
|
||||
context = dict(messagefn=messagefn)
|
||||
|
||||
# complete config for external_config first, since this may update values in the other sections
|
||||
self.external_config.complete_config(context)
|
||||
self.server_config.complete_config(context)
|
||||
self.dataset_config.complete_config(context)
|
||||
|
||||
self.is_completed = True
|
||||
self.check_config()
|
||||
|
||||
def get_matrix_data_cache_manager(self):
|
||||
return self.server_config.matrix_data_cache_manager
|
||||
|
||||
def get_title(self, data_adaptor):
|
||||
return (
|
||||
self.server_config.single_dataset__title
|
||||
if self.server_config.single_dataset__title
|
||||
else data_adaptor.get_title()
|
||||
)
|
||||
|
||||
def get_about(self, data_adaptor):
|
||||
return (
|
||||
self.server_config.single_dataset__about
|
||||
if self.server_config.single_dataset__about
|
||||
else data_adaptor.get_about()
|
||||
)
|
||||
@@ -0,0 +1,99 @@
|
||||
import copy
|
||||
|
||||
from flatten_dict import flatten
|
||||
from local_server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
class BaseConfig(object):
|
||||
"""
|
||||
This class handles the mechanics of updating and checking attributes.
|
||||
Derived classes are expected to store the actual attributes
|
||||
Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
|
||||
"""
|
||||
|
||||
def __init__(self, app_config, default_config):
|
||||
# reference back to the app_config
|
||||
self.app_config = app_config
|
||||
# the complete set of attributes and their default values (unflattened)
|
||||
self.default_config = default_config
|
||||
# used to make sure every attribute value is checked
|
||||
self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
|
||||
|
||||
def create_mapping(self, config):
|
||||
"""
|
||||
Create a dictionary where the keys are the name of attributes (using double underscore convention)
|
||||
For example: authentication__type
|
||||
|
||||
The values are a tuple,
|
||||
- the first item of the tuple is a tuple of path elements (location in config 'tree')
|
||||
- the second item is the value of the config parameter
|
||||
|
||||
For example: (('authentication', 'type'), 'session'))
|
||||
"""
|
||||
config_copy = copy.deepcopy(config)
|
||||
mapping = {}
|
||||
|
||||
flat_config = flatten(config_copy)
|
||||
for key, value in flat_config.items():
|
||||
# name of the attribute
|
||||
attr = "__".join(key)
|
||||
mapping[attr] = (key, value)
|
||||
|
||||
return mapping
|
||||
|
||||
def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
|
||||
val = getattr(self, attrname)
|
||||
if type(vtype) in (list, tuple):
|
||||
if type(val) not in vtype:
|
||||
tnames = ",".join([x.__name__ for x in vtype])
|
||||
raise ConfigurationError(
|
||||
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
|
||||
)
|
||||
else:
|
||||
if type(val) != vtype:
|
||||
raise ConfigurationError(
|
||||
f"Invalid type for attribute: {attrname}, "
|
||||
f"expected type {vtype.__name__}, got {type(val).__name__}"
|
||||
)
|
||||
|
||||
self.attr_checked[attrname] = True
|
||||
|
||||
def check_config(self):
|
||||
mapping = self.create_mapping(self.default_config)
|
||||
for key in mapping.keys():
|
||||
if not self.attr_checked[key]:
|
||||
raise ConfigurationError(f"The attr '{key}' has not been checked")
|
||||
|
||||
def update(self, **kw):
|
||||
"""Update the attributes defined in kw with their new values."""
|
||||
for key, value in kw.items():
|
||||
if not hasattr(self, key):
|
||||
raise ConfigurationError(f"unknown config parameter {key}.")
|
||||
try:
|
||||
if type(value) == tuple:
|
||||
# convert tuple values to list values
|
||||
value = list(value)
|
||||
setattr(self, key, value)
|
||||
except KeyError:
|
||||
raise ConfigurationError(f"Unable to set config parameter {key}.")
|
||||
|
||||
self.attr_checked[key] = False
|
||||
|
||||
def update_from_config(self, config, prefix):
|
||||
mapping = self.create_mapping(config)
|
||||
for attr, (key, value) in mapping.items():
|
||||
if not hasattr(self, attr):
|
||||
raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
|
||||
setattr(self, attr, value)
|
||||
|
||||
self.attr_checked[attr] = False
|
||||
|
||||
def changes_from_default(self):
|
||||
"""Return all the attribute that are different from the default"""
|
||||
mapping = self.create_mapping(self.default_config)
|
||||
diff = []
|
||||
for attrname, (key, defval) in mapping.items():
|
||||
curval = getattr(self, attrname)
|
||||
if curval != defval:
|
||||
diff.append((attrname, curval, defval))
|
||||
return diff
|
||||
@@ -0,0 +1,120 @@
|
||||
from local_server import display_version as cellxgene_display_version
|
||||
|
||||
|
||||
def get_client_config(app_config, data_adaptor):
|
||||
"""
|
||||
Return the configuration as required by the /config REST route
|
||||
"""
|
||||
|
||||
server_config = app_config.server_config
|
||||
dataset_config = data_adaptor.dataset_config
|
||||
annotation = dataset_config.user_annotations
|
||||
auth = server_config.auth
|
||||
|
||||
# FIXME The current set of config is not consistently presented:
|
||||
# we have camalCase, hyphen-text, and underscore_text
|
||||
|
||||
# make sure the configuration has been checked.
|
||||
app_config.check_config()
|
||||
|
||||
# display_names
|
||||
title = app_config.get_title(data_adaptor)
|
||||
about = app_config.get_about(data_adaptor)
|
||||
|
||||
display_names = dict(engine=data_adaptor.get_name(), dataset=title)
|
||||
|
||||
# library_versions
|
||||
library_versions = {}
|
||||
library_versions.update(data_adaptor.get_library_versions())
|
||||
library_versions["cellxgene"] = cellxgene_display_version
|
||||
|
||||
# links
|
||||
links = {"about-dataset": about}
|
||||
|
||||
# parameters
|
||||
parameters = {
|
||||
"layout": dataset_config.embeddings__names,
|
||||
"max-category-items": dataset_config.presentation__max_categories,
|
||||
"obs_names": server_config.single_dataset__obs_names,
|
||||
"var_names": server_config.single_dataset__var_names,
|
||||
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
|
||||
"backed": server_config.adaptor__anndata_adaptor__backed,
|
||||
"disable-diffexp": not dataset_config.diffexp__enable,
|
||||
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
|
||||
"annotations": False,
|
||||
"annotations_file": None,
|
||||
"annotations_dir": None,
|
||||
"annotations_cell_ontology_enabled": False,
|
||||
"annotations_cell_ontology_obopath": None,
|
||||
"annotations_cell_ontology_terms": None,
|
||||
"custom_colors": dataset_config.presentation__custom_colors,
|
||||
"diffexp-may-be-slow": False,
|
||||
}
|
||||
|
||||
# corpora dataset_props
|
||||
# TODO/Note: putting info from the dataset into the /config is not ideal.
|
||||
# However, it is definitely not part of /schema, and we do not have a top-level
|
||||
# route for data properties. Consider creating one at some point.
|
||||
corpora_props = data_adaptor.get_corpora_props()
|
||||
if corpora_props and "default_embedding" in corpora_props:
|
||||
default_embedding = corpora_props["default_embedding"]
|
||||
if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
|
||||
default_embedding = default_embedding[2:] # drop X_ prefix
|
||||
if default_embedding in data_adaptor.get_embedding_names():
|
||||
parameters["default_embedding"] = default_embedding
|
||||
|
||||
data_adaptor.update_parameters(parameters)
|
||||
if annotation:
|
||||
annotation.update_parameters(parameters, data_adaptor)
|
||||
|
||||
# gather it all together
|
||||
client_config = {}
|
||||
config = client_config["config"] = {}
|
||||
config["displayNames"] = display_names
|
||||
config["library_versions"] = library_versions
|
||||
config["links"] = links
|
||||
config["parameters"] = parameters
|
||||
config["corpora_props"] = corpora_props
|
||||
config["limits"] = {
|
||||
"column_request_max": server_config.limits__column_request_max,
|
||||
"diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
|
||||
}
|
||||
|
||||
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
|
||||
config["authentication"] = {
|
||||
"requires_client_login": auth.requires_client_login(),
|
||||
}
|
||||
if auth.requires_client_login():
|
||||
config["authentication"].update(
|
||||
{
|
||||
# Todo why are these stored on the data_adaptor?
|
||||
"login": auth.get_login_url(data_adaptor),
|
||||
"logout": auth.get_logout_url(data_adaptor),
|
||||
}
|
||||
)
|
||||
|
||||
return client_config
|
||||
|
||||
|
||||
def get_client_userinfo(app_config, data_adaptor):
|
||||
"""
|
||||
Return the userinfo as required by the /userinfo REST route
|
||||
"""
|
||||
|
||||
server_config = app_config.server_config
|
||||
dataset_config = data_adaptor.dataset_config
|
||||
auth = server_config.auth
|
||||
|
||||
# make sure the configuration has been checked.
|
||||
app_config.check_config()
|
||||
|
||||
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
|
||||
userinfo = {}
|
||||
userinfo["userinfo"] = {
|
||||
"is_authenticated": auth.is_user_authenticated(),
|
||||
"username": auth.get_user_name(),
|
||||
"user_id": auth.get_user_id(),
|
||||
"email": auth.get_user_email(),
|
||||
"picture": auth.get_user_picture(),
|
||||
}
|
||||
return userinfo
|
||||
@@ -0,0 +1,191 @@
|
||||
import os
|
||||
from os.path import splitext, isdir
|
||||
|
||||
from local_server.common.annotations.local_file_csv import AnnotationsLocalFile
|
||||
from local_server.common.config.base_config import BaseConfig
|
||||
from local_server.common.errors import ConfigurationError, OntologyLoadFailure
|
||||
from local_server.compute.scanpy import get_scanpy_module
|
||||
from local_server.data_common.matrix_loader import MatrixDataLoader
|
||||
|
||||
|
||||
class DatasetConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with a dataset."""
|
||||
|
||||
def __init__(self, tag, app_config, default_config):
|
||||
super().__init__(app_config, default_config)
|
||||
self.tag = tag
|
||||
try:
|
||||
self.app__scripts = default_config["app"]["scripts"]
|
||||
self.app__inline_scripts = default_config["app"]["inline_scripts"]
|
||||
self.app__authentication_enable = default_config["app"]["authentication_enable"]
|
||||
|
||||
self.presentation__max_categories = default_config["presentation"]["max_categories"]
|
||||
self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
|
||||
|
||||
self.user_annotations__enable = default_config["user_annotations"]["enable"]
|
||||
self.user_annotations__type = default_config["user_annotations"]["type"]
|
||||
self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
|
||||
"directory"
|
||||
]
|
||||
self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
|
||||
self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
|
||||
self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
|
||||
"obo_location"
|
||||
]
|
||||
|
||||
self.embeddings__names = default_config["embeddings"]["names"]
|
||||
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
|
||||
|
||||
self.diffexp__enable = default_config["diffexp"]["enable"]
|
||||
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
|
||||
self.diffexp__top_n = default_config["diffexp"]["top_n"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
# The annotation object is created during complete_config and stored here.
|
||||
self.user_annotations = None
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_app()
|
||||
self.handle_presentation()
|
||||
self.handle_user_annotations(context)
|
||||
self.handle_embeddings()
|
||||
self.handle_diffexp(context)
|
||||
|
||||
def get_data_adaptor(self):
|
||||
server_config = self.app_config.server_config
|
||||
if not server_config.data_adaptor:
|
||||
matrix_data_loader = MatrixDataLoader(
|
||||
server_config.single_dataset__datapath, app_config=self.app_config
|
||||
)
|
||||
server_config.data_adaptor = matrix_data_loader.open(self.app_config)
|
||||
|
||||
return server_config.data_adaptor
|
||||
|
||||
def handle_app(self):
|
||||
self.validate_correct_type_of_configuration_attribute("app__scripts", list)
|
||||
self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
|
||||
self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
|
||||
|
||||
# scripts can be string (filename) or dict (attributes). Convert string to dict.
|
||||
scripts = []
|
||||
for script in self.app__scripts:
|
||||
try:
|
||||
if isinstance(script, str):
|
||||
scripts.append({"src": script})
|
||||
elif isinstance(script, dict) and isinstance(script["src"], str):
|
||||
scripts.append(script)
|
||||
else:
|
||||
raise Exception
|
||||
except Exception as e:
|
||||
raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
|
||||
|
||||
self.app__scripts = scripts
|
||||
|
||||
def handle_presentation(self):
|
||||
self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
|
||||
self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
|
||||
|
||||
def handle_user_annotations(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__local_file_csv__directory", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__local_file_csv__file", (type(None), str)
|
||||
)
|
||||
self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
|
||||
self.validate_correct_type_of_configuration_attribute(
|
||||
"user_annotations__ontology__obo_location", (type(None), str)
|
||||
)
|
||||
if self.user_annotations__enable:
|
||||
server_config = self.app_config.server_config
|
||||
if not self.app__authentication_enable:
|
||||
raise ConfigurationError("user annotations requires authentication to be enabled")
|
||||
if not server_config.auth.is_valid_authentication_type():
|
||||
auth_type = server_config.authentication__type
|
||||
raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
|
||||
|
||||
if self.user_annotations__type == "local_file_csv":
|
||||
self.handle_local_file_csv_annotations()
|
||||
else:
|
||||
raise ConfigurationError('The only annotation type support is "local_file_csv"')
|
||||
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
|
||||
try:
|
||||
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
|
||||
except OntologyLoadFailure as e:
|
||||
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
|
||||
else:
|
||||
self.check_annotation_config_vars_not_set(context)
|
||||
|
||||
def handle_local_file_csv_annotations(self):
|
||||
dirname = self.user_annotations__local_file_csv__directory
|
||||
filename = self.user_annotations__local_file_csv__file
|
||||
if filename is not None and dirname is not None:
|
||||
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
|
||||
|
||||
if filename is not None:
|
||||
lf_name, lf_ext = splitext(filename)
|
||||
if lf_ext and lf_ext != ".csv":
|
||||
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
|
||||
|
||||
if dirname is not None and not isdir(dirname):
|
||||
try:
|
||||
os.mkdir(dirname)
|
||||
except OSError:
|
||||
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
|
||||
|
||||
self.user_annotations = AnnotationsLocalFile(dirname, filename)
|
||||
|
||||
# if the user has specified a fixed label file, go ahead and validate it
|
||||
# so that we can remove errors early in the process.
|
||||
server_config = self.app_config.server_config
|
||||
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
|
||||
data_adaptor = self.get_data_adaptor()
|
||||
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
|
||||
|
||||
def check_annotation_config_vars_not_set(self, context):
|
||||
if self.user_annotations__type is not None:
|
||||
dirname = self.user_annotations__local_file_csv__directory
|
||||
filename = self.user_annotations__local_file_csv__file
|
||||
if filename is not None:
|
||||
context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
|
||||
if dirname is not None:
|
||||
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
|
||||
|
||||
if self.user_annotations__ontology__enable:
|
||||
context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
|
||||
if self.user_annotations__ontology__obo_location is not None:
|
||||
context["messagefn"](
|
||||
"Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
|
||||
)
|
||||
|
||||
def handle_embeddings(self):
|
||||
self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
|
||||
self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
|
||||
|
||||
server_config = self.app_config.server_config
|
||||
if self.embeddings__enable_reembedding:
|
||||
if server_config.single_dataset__datapath:
|
||||
if server_config.adaptor__anndata_adaptor__backed:
|
||||
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
|
||||
|
||||
try:
|
||||
get_scanpy_module()
|
||||
except NotImplementedError:
|
||||
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
|
||||
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
|
||||
|
||||
def handle_diffexp(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
|
||||
self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
|
||||
|
||||
data_adaptor = self.get_data_adaptor()
|
||||
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
|
||||
context["messagefn"](
|
||||
"CAUTION: due to the size of your dataset, "
|
||||
"running differential expression may take longer or fail."
|
||||
)
|
||||
@@ -0,0 +1,96 @@
|
||||
import os
|
||||
|
||||
from local_server.common.config.base_config import BaseConfig
|
||||
from local_server.common.errors import ConfigurationError
|
||||
from local_server.common.config import get_secret_key
|
||||
from local_server.common.errors import SecretKeyRetrievalError
|
||||
from local_server.common.utils.type_conversion_utils import convert_string_to_value
|
||||
|
||||
|
||||
class ExternalConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with external configuration sources, such as
|
||||
environment variables or the AWS Secrets Manager."""
|
||||
|
||||
def __init__(self, app_config, default_config):
|
||||
super().__init__(app_config, default_config)
|
||||
try:
|
||||
self.environment = default_config["environment"]
|
||||
self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
|
||||
self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_environment(context)
|
||||
self.handle_aws_secrets_manager(context)
|
||||
|
||||
def handle_environment(self, context):
|
||||
"""For each environment variable defined, get the value (if it is set),
|
||||
and set the specified config parameter"""
|
||||
self.validate_correct_type_of_configuration_attribute("environment", list)
|
||||
for envdict in self.environment:
|
||||
name = envdict.get("name")
|
||||
if name is None:
|
||||
raise ConfigurationError("environment: 'name' is missing")
|
||||
required = envdict.get("required", False)
|
||||
if type(required) != bool:
|
||||
raise ConfigurationError("environment: 'required' must be a bool")
|
||||
path = envdict.get("path")
|
||||
if path is None:
|
||||
raise ConfigurationError("environment: 'path' is missing")
|
||||
|
||||
value = os.environ.get(name)
|
||||
if value is None:
|
||||
if required:
|
||||
raise ConfigurationError(f"required environment variable '{name}' not set")
|
||||
else:
|
||||
value = convert_string_to_value(value)
|
||||
self.app_config.update_single_config_from_path_and_value(path, value)
|
||||
|
||||
def handle_aws_secrets_manager(self, context):
|
||||
"""For each aws secret defined, get the key/values, and set the specified config parameter"""
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
|
||||
|
||||
if not self.aws_secrets_manager__secrets:
|
||||
return
|
||||
|
||||
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
|
||||
|
||||
for secret in self.aws_secrets_manager__secrets:
|
||||
secret_name = secret.get("name")
|
||||
if secret_name is None:
|
||||
raise ConfigurationError("aws_secrets_manager: 'name' is missing")
|
||||
if not isinstance(secret_name, str):
|
||||
raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
|
||||
|
||||
try:
|
||||
secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
|
||||
except SecretKeyRetrievalError as e:
|
||||
raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
|
||||
|
||||
values = secret.get("values")
|
||||
if values is None:
|
||||
raise ConfigurationError("aws_secrets_manager: 'values' is missing")
|
||||
if not isinstance(values, list):
|
||||
raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
|
||||
|
||||
for value in values:
|
||||
key = value.get("key")
|
||||
if key is None:
|
||||
raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
|
||||
path = value.get("path")
|
||||
if path is None:
|
||||
raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
|
||||
required = value.get("required", False)
|
||||
if type(required) != bool:
|
||||
raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
|
||||
|
||||
secret_value = secret_dict.get(key)
|
||||
if secret_value is None:
|
||||
if required:
|
||||
raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
|
||||
else:
|
||||
secret_value = convert_string_to_value(secret_value)
|
||||
self.app_config.update_single_config_from_path_and_value(path, secret_value)
|
||||
@@ -0,0 +1,183 @@
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
from os.path import basename
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from local_server.auth.auth import AuthTypeFactory
|
||||
from local_server.common.config.base_config import BaseConfig
|
||||
from local_server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
|
||||
from local_server.common.errors import ConfigurationError, DatasetAccessError
|
||||
from local_server.common.data_locator import discover_s3_region_name
|
||||
from local_server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
|
||||
from local_server.data_common.matrix_loader import MatrixDataLoader
|
||||
|
||||
|
||||
class ServerConfig(BaseConfig):
|
||||
"""Manages the config attribute associated with the server."""
|
||||
|
||||
def __init__(self, app_config, default_config):
|
||||
super().__init__(app_config, default_config)
|
||||
|
||||
try:
|
||||
self.app__verbose = default_config["app"]["verbose"]
|
||||
self.app__debug = default_config["app"]["debug"]
|
||||
self.app__host = default_config["app"]["host"]
|
||||
self.app__port = default_config["app"]["port"]
|
||||
self.app__open_browser = default_config["app"]["open_browser"]
|
||||
self.app__force_https = default_config["app"]["force_https"]
|
||||
self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
|
||||
|
||||
self.authentication__type = default_config["authentication"]["type"]
|
||||
self.authentication__insecure_test_environment = default_config["authentication"][
|
||||
"insecure_test_environment"
|
||||
]
|
||||
|
||||
self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
|
||||
self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
|
||||
self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
|
||||
self.single_dataset__about = default_config["single_dataset"]["about"]
|
||||
self.single_dataset__title = default_config["single_dataset"]["title"]
|
||||
|
||||
self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
|
||||
|
||||
self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
|
||||
|
||||
self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
|
||||
self.limits__column_request_max = default_config["limits"]["column_request_max"]
|
||||
|
||||
except KeyError as e:
|
||||
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
||||
|
||||
self.data_adaptor = None
|
||||
|
||||
# The authentication object
|
||||
self.auth = None
|
||||
|
||||
def complete_config(self, context):
|
||||
self.handle_app(context)
|
||||
self.handle_data_source()
|
||||
self.handle_authentication()
|
||||
self.handle_data_locator()
|
||||
self.handle_adaptor() # may depend on data_locator
|
||||
self.handle_single_dataset(context) # may depend on adaptor
|
||||
self.handle_limits()
|
||||
|
||||
self.check_config()
|
||||
|
||||
def handle_app(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__debug", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__host", str)
|
||||
self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
|
||||
self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
|
||||
self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
|
||||
|
||||
if self.app__port:
|
||||
try:
|
||||
if not is_port_available(self.app__host, self.app__port):
|
||||
raise ConfigurationError(
|
||||
f"The port selected {self.app__port} is in use, please configure an open port."
|
||||
)
|
||||
except OverflowError:
|
||||
raise ConfigurationError(f"Invalid port: {self.app__port}")
|
||||
else:
|
||||
try:
|
||||
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
|
||||
except ValueError:
|
||||
raise ConfigurationError(
|
||||
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
|
||||
)
|
||||
try:
|
||||
self.app__port = find_available_port(self.app__host, default_server_port)
|
||||
except OverflowError:
|
||||
raise ConfigurationError(f"Invalid port: {default_server_port}")
|
||||
|
||||
if self.app__debug:
|
||||
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
|
||||
self.app__verbose = True
|
||||
self.app__open_browser = False
|
||||
else:
|
||||
warnings.formatwarning = custom_format_warning
|
||||
|
||||
if not self.app__verbose:
|
||||
sys.tracebacklimit = 0
|
||||
|
||||
def handle_authentication(self):
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
|
||||
self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
|
||||
|
||||
if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
|
||||
raise ConfigurationError("Test auth can only be used in an insecure test environment")
|
||||
|
||||
self.auth = AuthTypeFactory.create(self.authentication__type, self)
|
||||
if self.auth is None:
|
||||
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
|
||||
|
||||
def handle_data_locator(self):
|
||||
self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
|
||||
if self.data_locator__s3__region_name is True:
|
||||
path = self.single_dataset__datapath
|
||||
|
||||
if path.startswith("s3://"):
|
||||
region_name = discover_s3_region_name(path)
|
||||
if region_name is None:
|
||||
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
|
||||
else:
|
||||
region_name = None
|
||||
self.data_locator__s3__region_name = region_name
|
||||
|
||||
def handle_data_source(self):
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", str)
|
||||
|
||||
def handle_single_dataset(self, context):
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
|
||||
self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
|
||||
|
||||
# preload this data set
|
||||
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
|
||||
try:
|
||||
matrix_data_loader.pre_load_validation()
|
||||
except DatasetAccessError as e:
|
||||
raise ConfigurationError(str(e))
|
||||
|
||||
file_size = matrix_data_loader.file_size()
|
||||
file_basename = basename(self.single_dataset__datapath)
|
||||
if file_size > BIG_FILE_SIZE_THRESHOLD:
|
||||
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
|
||||
else:
|
||||
context["messagefn"](f"Loading data from {file_basename}.")
|
||||
|
||||
if self.single_dataset__about:
|
||||
|
||||
def url_check(url):
|
||||
try:
|
||||
result = urlparse(url)
|
||||
if all([result.scheme, result.netloc]):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
if not url_check(self.single_dataset__about):
|
||||
raise ConfigurationError(
|
||||
"Must provide an absolute URL for --about. (Example format: http://example.com)"
|
||||
)
|
||||
|
||||
def handle_adaptor(self):
|
||||
self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
|
||||
|
||||
def handle_limits(self):
|
||||
self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
|
||||
self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
|
||||
|
||||
def exceeds_limit(self, limit_name, value):
|
||||
limit_value = getattr(self, "limits__" + limit_name, None)
|
||||
if limit_value is None: # disabled
|
||||
return False
|
||||
return value > limit_value
|
||||
Reference in New Issue
Block a user