fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-01 04:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

config refactor (#1854)

Browse Source 
* split out config

* add tests for base and app config, refactor client config out of app config

* refactor default config retrieval

* create config test class and helper functions

* move default_config into server to fix import issue
This commit is contained in:
Madison Dunitz
2020-09-29 16:42:46 -05:00
committed by GitHub
parent 1145f61c78
commit af3c6e1d8e
57 changed files with 2667 additions and 1599 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/common/annotations/hosted_tiledb.py
+9 -8
View File
@@ -31,7 +31,8 @@ class AnnotationsHostedTileDB(Annotations):
unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
if unsanitary_original_category_names:
raise AnnotationCategoryNameError(
f"{unsanitary_original_category_names} are not valid category names, please resubmit")
f"{unsanitary_original_category_names} are not valid category names, please resubmit"
)
def is_safe_collection_name(self, name):
"""
@@ -68,11 +69,11 @@ class AnnotationsHostedTileDB(Annotations):
index_dims = None
schema_hints = json.loads(schema_hints)
if '__pandas_attribute_repr' in tileDBArray.meta:
if "__pandas_attribute_repr" in tileDBArray.meta:
# backwards compatibility... unsure if necessary at this point
repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr'])
if '__pandas_index_dims' in tileDBArray.meta:
index_dims = json.loads(tileDBArray.meta['__pandas_index_dims'])
repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
if "__pandas_index_dims" in tileDBArray.meta:
index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
data = tileDBArray[:]
indexes = list()
@@ -80,12 +81,12 @@ class AnnotationsHostedTileDB(Annotations):
for col_name, col_val in data.items():
# If the column values are byte literals, decode them
if isinstance(col_val[0], bytes):
col_val = [value.decode('utf-8') for value in col_val]
col_val = [value.decode("utf-8") for value in col_val]
if schema_hints and col_name in schema_hints:
type = schema_hints.get(col_name).get("type")
if type and type == "categorical":
new_col = pd.Series(col_val, dtype='category')
new_col = pd.Series(col_val, dtype="category")
data[col_name] = new_col
elif repr_meta and col_name in repr_meta:
new_col = pd.Series(col_val, dtype=repr_meta[col_name])
@@ -127,7 +128,7 @@ class AnnotationsHostedTileDB(Annotations):
tiledb_uri=uri,
user_id=user_id,
dataset_id=str(dataset_id),
schema_hints=json.dumps(dataframe_schema_type_hints)
schema_hints=json.dumps(dataframe_schema_type_hints),
)
if not df.empty:
self.check_category_names(df)
server/common/app_config.py
-963
View File
@@ -1,963 +0,0 @@
import copy
import os
import sys
import warnings
from os.path import splitext, basename, isdir
from urllib.parse import urlparse, quote_plus
import yaml
from flatten_dict import flatten, unflatten
import server.compute.diffexp_cxg as diffexp_tiledb
import server.compute.scanpy
from server import display_version as cellxgene_display_version
from server.auth.auth import AuthTypeFactory
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from server.common.annotations.local_file_csv import AnnotationsLocalFile
from server.common.data_locator import discover_s3_region_name
from server.common.default_config import get_default_config
from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
from server.db.db_utils import DbUtils
DEFAULT_SERVER_PORT = 5005
# anything bigger than this will generate a special message
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
class AppFeature(object):
def __init__(self, path, available=False, method="POST", extra={}):
self.path = path
self.available = available
self.method = method
self.extra = extra
for k, v in extra.items():
setattr(self, k, v)
def todict(self):
d = dict(available=self.available, method=self.method, path=self.path)
d.update(self.extra)
return d
class AppConfig(object):
"""AppConfig stores all the configuration for cellxgene. The configuration is divided into two main parts:
server attributes, and dataset attributes. The server_config contains attributes that refer to the server process
as a whole. The default_dataset_config referes to attributes that are associated with the features and
presentations of a dataset. The dataset config attributes can be overridden depending on the url by which the
dataset was accessed. These are stored in dataroot_config.
AppConfig has methods to initialize, modify, and access the configuration.
"""
def __init__(self):
# the default configuration (see default_config.py)
self.default_config = get_default_config()
# the server configuration
self.server_config = ServerConfig(self, self.default_config["server"])
# the dataset config, unless overridden by an entry in dataroot_config
self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
# a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
# attribute of the server_config.
self.dataroot_config = {}
# Set to true when config_completed is called
self.is_completed = False
def get_dataset_config(self, dataroot_key):
if self.server_config.single_dataset__datapath:
return self.default_dataset_config
else:
return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
def check_config(self):
"""Verify all the attributes have been checked"""
if not self.is_completed:
raise ConfigurationError("The configuration has not been completed")
self.server_config.check_config()
self.default_dataset_config.check_config()
for dataset_config in self.dataroot_config.values():
dataset_config.check_config()
def update_server_config(self, **kw):
self.server_config.update(**kw)
self.is_complete = False
def update_default_dataset_config(self, **kw):
self.default_dataset_config.update(**kw)
# update all the other dataset configs, if any
for value in self.dataroot_config.values():
value.update(**kw)
self.is_complete = False
def update_from_config_file(self, config_file):
with open(config_file) as fyaml:
config = yaml.load(fyaml, Loader=yaml.FullLoader)
if config.get("server"):
self.server_config.update_from_config(config["server"], "server")
if config.get("dataset"):
self.default_dataset_config.update_from_config(config["dataset"], "dataset")
per_dataset_config = config.get("per_dataset_config", {})
for key, dataroot_config in per_dataset_config.items():
# first create and initialize the dataroot with the default config
self.add_dataroot_config(key, **config["dataset"])
# then apply the per dataset configuration
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
self.is_complete = False
def write_config(self, config_file):
"""output the config to a yaml file"""
server = self.server_config.create_mapping(self.server_config.default_config)
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = dict(server={}, dataset={})
for attrname in server.keys():
config["server__" + attrname] = getattr(self.server_config, attrname)
for attrname in dataset.keys():
config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
if self.dataroot_config:
config["per_dataset_config"] = {}
for dataroot_tag, dataroot_config in self.dataroot_config.items():
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
for attrname in dataset.keys():
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
config = unflatten(config, splitter=lambda key: key.split("__"))
yaml.dump(config, open(config_file, "w"))
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
diff_server = self.server_config.changes_from_default()
diff_dataset = self.default_dataset_config.changes_from_default()
diff = dict(server=diff_server, dataset=diff_dataset)
return diff
def add_dataroot_config(self, dataroot_tag, **kw):
"""Create a new dataset config object based on the default dataset config, and kw parameters"""
if dataroot_tag in self.dataroot_config:
raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
if type(self.server_config.multi_dataset__dataroot) != dict:
raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
if dataroot_tag not in self.server_config.multi_dataset__dataroot:
raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
self.is_completed = False
self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = {key: value[1] for key, value in flat_config.items()}
self.dataroot_config[dataroot_tag].update(**config)
self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
def complete_config(self, messagefn=None):
"""The configure options are checked, and any additional setup based on the config
parameters is done"""
if messagefn is None:
def noop(message):
pass
messagefn = noop
# TODO: to give better error messages we can add a mapping between where each config
# attribute originated (e.g. command line argument or config file), then in the error
# messages we can give correct context for attributes with bad value.
context = dict(messagefn=messagefn)
self.server_config.complete_config(context)
self.default_dataset_config.complete_config(context)
for dataroot_config in self.dataroot_config.values():
dataroot_config.complete_config(context)
self.is_completed = True
self.check_config()
def get_matrix_data_cache_manager(self):
return self.server_config.matrix_data_cache_manager
def is_multi_dataset(self):
return self.server_config.multi_dataset__dataroot is not None
def get_title(self, data_adaptor):
return (
self.server_config.single_dataset__title
if self.server_config.single_dataset__title
else data_adaptor.get_title()
)
def get_about(self, data_adaptor):
return (
self.server_config.single_dataset__about
if self.server_config.single_dataset__about
else data_adaptor.get_about()
)
def get_client_config(self, data_adaptor):
"""
Return the configuration as required by the /config REST route
"""
server_config = self.server_config
dataset_config = data_adaptor.dataset_config
annotation = dataset_config.user_annotations
auth = server_config.auth
# FIXME The current set of config is not consistently presented:
# we have camalCase, hyphen-text, and underscore_text
# make sure the configuration has been checked.
self.check_config()
# features
features = [f.todict() for f in data_adaptor.get_features(annotation)]
# display_names
title = self.get_title(data_adaptor)
about = self.get_about(data_adaptor)
display_names = dict(engine=data_adaptor.get_name(), dataset=title)
# library_versions
library_versions = {}
library_versions.update(data_adaptor.get_library_versions())
library_versions["cellxgene"] = cellxgene_display_version
# links
links = {"about-dataset": about}
# parameters
parameters = {
"layout": dataset_config.embeddings__names,
"max-category-items": dataset_config.presentation__max_categories,
"obs_names": server_config.single_dataset__obs_names,
"var_names": server_config.single_dataset__var_names,
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
"backed": server_config.adaptor__anndata_adaptor__backed,
"disable-diffexp": not dataset_config.diffexp__enable,
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
"annotations_cell_ontology_enabled": False,
"annotations_cell_ontology_obopath": None,
"annotations_cell_ontology_terms": None,
"custom_colors": dataset_config.presentation__custom_colors,
"diffexp-may-be-slow": False,
"about_legal_tos": dataset_config.app__about_legal_tos,
"about_legal_privacy": dataset_config.app__about_legal_privacy,
}
# corpora dataset_props
# TODO/Note: putting info from the dataset into the /config is not ideal.
# However, it is definitely not part of /schema, and we do not have a top-level
# route for data properties. Consider creating one at some point.
corpora_props = data_adaptor.get_corpora_props()
if corpora_props and "default_embedding" in corpora_props:
default_embedding = corpora_props["default_embedding"]
if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
default_embedding = default_embedding[2:] # drop X_ prefix
if default_embedding in data_adaptor.get_embedding_names():
parameters["default_embedding"] = default_embedding
data_adaptor.update_parameters(parameters)
if annotation:
annotation.update_parameters(parameters, data_adaptor)
# gather it all together
c = {}
config = c["config"] = {}
config["features"] = features
config["displayNames"] = display_names
config["library_versions"] = library_versions
config["links"] = links
config["parameters"] = parameters
config["corpora_props"] = corpora_props
config["limits"] = {
"column_request_max": server_config.limits__column_request_max,
"diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
}
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
config["authentication"] = {
"requires_client_login": auth.requires_client_login(),
}
if auth.requires_client_login():
config["authentication"].update({
"login": auth.get_login_url(data_adaptor),
"logout": auth.get_logout_url(data_adaptor),
})
return c
def get_client_userinfo(self, data_adaptor):
"""
Return the userinfo as required by the /userinfo REST route
"""
server_config = self.server_config
dataset_config = data_adaptor.dataset_config
auth = server_config.auth
# make sure the configuration has been checked.
self.check_config()
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
userinfo = {}
userinfo["userinfo"] = {
"is_authenticated": auth.is_user_authenticated(),
"username": auth.get_user_name(),
"user_id": auth.get_user_id(),
"email": auth.get_user_email()
}
return userinfo
else:
return None
class BaseConfig(object):
"""This class handles the mechanics of updating and checking attributes.
Derived classes are expected to store the actual attributes"""
def __init__(self, app_config, default_config, dictval_cases={}):
# reference back to the app_config
self.app_config = app_config
# the complete set of attribute and their default values (unflattened)
self.default_config = default_config
# attributes where the value may be a dict (and therefore are not flattened)
self.dictval_cases = dictval_cases
# used to make sure every attribute value is checked
self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()}
def create_mapping(self, config):
"""Create a mapping from attribute names to (location in the config tree, value)"""
dc = copy.deepcopy(config)
mapping = {}
# special cases where the value could be a dict.
# If its value is not None, the entry is added to the mapping, and not included
# in the flattening below.
for dictval_case in self.dictval_cases:
cur = dc
for part in dictval_case[:-1]:
cur = cur.get(part, {})
val = cur.get(dictval_case[-1])
if val is not None:
key = "__".join(dictval_case)
mapping[key] = (dictval_case, val)
del cur[dictval_case[-1]]
flat_config = flatten(dc)
for key, value in flat_config.items():
# name of the attribute
attr = "__".join(key)
mapping[attr] = (key, value)
return mapping
def check_attr(self, attrname, vtype):
val = getattr(self, attrname)
if type(vtype) in (list, tuple):
if type(val) not in vtype:
tnames = ",".join([x.__name__ for x in vtype])
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
)
else:
if type(val) != vtype:
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, "
f"expected type {vtype.__name__}, got {type(val).__name__}"
)
self.attr_checked[attrname] = True
def check_config(self):
mapping = self.create_mapping(self.default_config)
for key in mapping.keys():
if not self.attr_checked[key]:
raise ConfigurationError(f"The attr '{key}' has not been checked")
def update(self, **kw):
for key, value in kw.items():
if not hasattr(self, key):
raise ConfigurationError(f"unknown config parameter {key}.")
try:
if type(value) == tuple:
# convert tuple values to list values
value = list(value)
setattr(self, key, value)
except KeyError:
raise ConfigurationError(f"Unable to set config parameter {key}.")
self.attr_checked[key] = False
def update_from_config(self, config, prefix):
mapping = self.create_mapping(config)
for attr, (key, value) in mapping.items():
if not hasattr(self, attr):
raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
try:
setattr(self, attr, value)
except KeyError:
raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}")
self.attr_checked[attr] = False
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
mapping = self.create_mapping(self.default_config)
diff = []
for attrname, (key, defval) in mapping.items():
curval = getattr(self, attrname)
if curval != defval:
diff.append((attrname, curval, defval))
return diff
class ServerConfig(BaseConfig):
"""Manages the config attribute associated with the server."""
def __init__(self, app_config, default_config):
dictval_cases = [
("app", "csp_directives"),
("authentication", "params_oauth", "cookie"),
("authentication", "params_oauth", "jwt_decode_options"),
("adaptor", "cxg_adaptor", "tiledb_ctx"),
("multi_dataset", "dataroot"),
]
super().__init__(app_config, default_config, dictval_cases)
dc = default_config
try:
self.app__verbose = dc["app"]["verbose"]
self.app__debug = dc["app"]["debug"]
self.app__host = dc["app"]["host"]
self.app__port = dc["app"]["port"]
self.app__open_browser = dc["app"]["open_browser"]
self.app__force_https = dc["app"]["force_https"]
self.app__flask_secret_key = dc["app"]["flask_secret_key"]
self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"]
self.app__server_timing_headers = dc["app"]["server_timing_headers"]
self.app__csp_directives = dc["app"]["csp_directives"]
self.app__api_base_url = dc["app"]["api_base_url"]
self.app__web_base_url = dc["app"]["web_base_url"]
self.authentication__type = dc["authentication"]["type"]
self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][
"oauth_api_base_url"
]
self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"]
self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"]
self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][
"jwt_decode_options"]
self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"]
self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"]
self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
self.multi_dataset__index = dc["multi_dataset"]["index"]
self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"]
self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"]
self.single_dataset__datapath = dc["single_dataset"]["datapath"]
self.single_dataset__obs_names = dc["single_dataset"]["obs_names"]
self.single_dataset__var_names = dc["single_dataset"]["var_names"]
self.single_dataset__about = dc["single_dataset"]["about"]
self.single_dataset__title = dc["single_dataset"]["title"]
self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"]
self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"]
self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"]
self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"]
self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"]
self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"]
self.limits__column_request_max = dc["limits"]["column_request_max"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The matrix data cache manager is created during the complete_config and stored here.
self.matrix_data_cache_manager = None
# The authentication object
self.auth = None
def complete_config(self, context):
self.handle_app(context)
self.handle_data_source(context)
self.handle_authentication(context)
self.handle_data_locator(context)
self.handle_adaptor(context) # may depend on data_locator
self.handle_single_dataset(context) # may depend on adaptor
self.handle_multi_dataset(context) # may depend on adaptor
self.handle_diffexp(context)
self.handle_limits(context)
self.check_config()
def handle_app(self, context):
self.check_attr("app__verbose", bool)
self.check_attr("app__debug", bool)
self.check_attr("app__host", str)
self.check_attr("app__port", (type(None), int))
self.check_attr("app__open_browser", bool)
self.check_attr("app__force_https", bool)
self.check_attr("app__flask_secret_key", (type(None), str))
self.check_attr("app__generate_cache_control_headers", bool)
self.check_attr("app__server_timing_headers", bool)
self.check_attr("app__csp_directives", (type(None), dict))
self.check_attr("app__api_base_url", (type(None), str))
self.check_attr("app__web_base_url", (type(None), str))
if self.app__port:
try:
if not is_port_available(self.app__host, self.app__port):
raise ConfigurationError(
f"The port selected {self.app__port} is in use, please configure an open port."
)
except OverflowError:
raise ConfigurationError(f"Invalid port: {self.app__port}")
else:
try:
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
except ValueError:
raise ConfigurationError(
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
)
try:
self.app__port = find_available_port(self.app__host, default_server_port)
except OverflowError:
raise ConfigurationError(f"Invalid port: {default_server_port}")
if self.app__debug:
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
self.app__verbose = True
self.app__open_browser = False
else:
warnings.formatwarning = custom_format_warning
if not self.app__verbose:
sys.tracebacklimit = 0
# secret key:
# first, from CXG_SECRET_KEY environment variable
# second, from config file
self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
# CSP Directives are a dict of string: list(string) or string: string
if self.app__csp_directives is not None:
for k, v in self.app__csp_directives.items():
if not isinstance(k, str):
raise ConfigurationError("CSP directive names must be a string.")
if isinstance(v, list):
for policy in v:
if not isinstance(policy, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
elif not isinstance(v, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
if self.app__web_base_url is None:
self.app__web_base_url = self.app__api_base_url
def handle_authentication(self, context):
self.check_attr("authentication__type", (type(None), str))
# oauth
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes)
self.check_attr("authentication__params_oauth__client_id", ptypes)
self.check_attr("authentication__params_oauth__client_secret", ptypes)
self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict))
self.check_attr("authentication__params_oauth__session_cookie", bool)
if self.authentication__params_oauth__session_cookie:
self.check_attr("authentication__params_oauth__cookie", (type(None), dict))
else:
self.check_attr("authentication__params_oauth__cookie", dict)
# secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
# second, from config file
self.authentication__params__oauth__client_secret = os.environ.get(
"CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret)
self.auth = AuthTypeFactory.create(self.authentication__type, self)
if self.auth is None:
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
def handle_data_locator(self, context):
self.check_attr("data_locator__s3__region_name", (type(None), bool, str))
if self.data_locator__s3__region_name is True:
path = self.single_dataset__datapath or self.multi_dataset__dataroot
if type(path) == dict:
# if multi_dataset__dataroot is a dict, then use the first key
# that is in s3. NOTE: it is not supported to have dataroots
# in different regions.
paths = [val.get("dataroot") for val in path.values()]
for path in paths:
if path.startswith("s3://"):
break
if path.startswith("s3://"):
region_name = discover_s3_region_name(path)
if region_name is None:
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
else:
region_name = None
self.data_locator__s3__region_name = region_name
def handle_data_source(self, context):
self.check_attr("single_dataset__datapath", (str, type(None)))
self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
if self.single_dataset__datapath is None:
if self.multi_dataset__dataroot is None:
# TODO: change the error message once dataroot is fully supported
raise ConfigurationError("missing datapath")
return
else:
if self.multi_dataset__dataroot is not None:
raise ConfigurationError("must supply only one of datapath or dataroot")
def handle_single_dataset(self, context):
self.check_attr("single_dataset__datapath", (str, type(None)))
self.check_attr("single_dataset__title", (str, type(None)))
self.check_attr("single_dataset__about", (str, type(None)))
self.check_attr("single_dataset__obs_names", (str, type(None)))
self.check_attr("single_dataset__var_names", (str, type(None)))
if self.single_dataset__datapath is None:
return
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
def handle_multi_dataset(self, context):
self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
self.check_attr("multi_dataset__index", (type(None), bool, str))
self.check_attr("multi_dataset__allowed_matrix_types", list)
self.check_attr("multi_dataset__matrix_cache__max_datasets", int)
self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float))
if self.multi_dataset__dataroot is None:
return
if type(self.multi_dataset__dataroot) == str:
default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
self.multi_dataset__dataroot = dict(d=default_dict)
for tag, dataroot_dict in self.multi_dataset__dataroot.items():
if "base_url" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
if "dataroot" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
base_url = dataroot_dict["base_url"]
# sanity check for well formed base urls
bad = False
if type(base_url) != str:
bad = True
elif os.path.normpath(base_url) != base_url:
bad = True
else:
base_url_parts = base_url.split("/")
if [quote_plus(part) for part in base_url_parts] != base_url_parts:
bad = True
if ".." in base_url_parts:
bad = True
if bad:
raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
# verify all the base_urls are unique
base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
if len(base_urls) > len(set(base_urls)):
raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
# error checking
for mtype in self.multi_dataset__allowed_matrix_types:
try:
MatrixDataType(mtype)
except ValueError:
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(
max_cached=self.multi_dataset__matrix_cache__max_datasets,
timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
)
def handle_diffexp(self, context):
self.check_attr("diffexp__alg_cxg__max_workers", (str, int))
self.check_attr("diffexp__alg_cxg__cpu_multiplier", int)
self.check_attr("diffexp__alg_cxg__target_workunit", int)
max_workers = self.diffexp__alg_cxg__max_workers
cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
cpu_count = os.cpu_count()
max_workers = min(max_workers, cpu_multiplier * cpu_count)
diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
def handle_adaptor(self, context):
# cxg
self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict)
regionkey = "vfs.s3.region"
if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
if type(self.data_locator__s3__region_name) == str:
self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
from server.data_cxg.cxg_adaptor import CxgAdaptor
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
# anndata
self.check_attr("adaptor__anndata_adaptor__backed", bool)
def handle_limits(self, context):
self.check_attr("limits__diffexp_cellcount_max", (type(None), int))
self.check_attr("limits__column_request_max", (type(None), int))
def exceeds_limit(self, limit_name, value):
limit_value = getattr(self, "limits__" + limit_name, None)
if limit_value is None: # disabled
return False
return value > limit_value
def get_api_base_url(self):
if self.app__api_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__api_base_url and self.app__api_base_url.endswith("/"):
return self.app__api_base_url[:-1]
return self.app__api_base_url
def get_web_base_url(self):
if self.app__web_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__web_base_url is None:
return self.get_api_base_url()
if self.app__web_base_url.endswith("/"):
return self.app__web_base_url[:-1]
return self.api__web_base_url
class DatasetConfig(BaseConfig):
"""Manages the config attribute associated with a dataset."""
def __init__(self, tag, app_config, default_config):
super().__init__(app_config, default_config)
self.tag = tag
dc = default_config
try:
self.app__scripts = dc["app"]["scripts"]
self.app__inline_scripts = dc["app"]["inline_scripts"]
self.app__about_legal_tos = dc["app"]["about_legal_tos"]
self.app__about_legal_privacy = dc["app"]["about_legal_privacy"]
self.app__authentication_enable = dc["app"]["authentication_enable"]
self.presentation__max_categories = dc["presentation"]["max_categories"]
self.presentation__custom_colors = dc["presentation"]["custom_colors"]
self.user_annotations__enable = dc["user_annotations"]["enable"]
self.user_annotations__type = dc["user_annotations"]["type"]
self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"]
self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ] # noqa E501
self.embeddings__names = dc["embeddings"]["names"]
self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
self.diffexp__enable = dc["diffexp"]["enable"]
self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = dc["diffexp"]["top_n"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The annotation object is created during complete_config and stored here.
self.user_annotations = None
def complete_config(self, context):
self.handle_app(context)
self.handle_presentation(context)
self.handle_user_annotations(context)
self.handle_embeddings(context)
self.handle_diffexp(context)
def handle_app(self, context):
self.check_attr("app__scripts", list)
self.check_attr("app__inline_scripts", list)
self.check_attr("app__about_legal_tos", (type(None), str))
self.check_attr("app__about_legal_privacy", (type(None), str))
self.check_attr("app__authentication_enable", bool)
# scripts can be string (filename) or dict (attributes). Convert string to dict.
scripts = []
for s in self.app__scripts:
if isinstance(s, str):
scripts.append({"src": s})
elif isinstance(s, dict) and isinstance(s["src"], str):
scripts.append(s)
else:
raise ConfigurationError("Scripts must be string or dict")
self.app__scripts = scripts
def handle_presentation(self, context):
self.check_attr("presentation__max_categories", int)
self.check_attr("presentation__custom_colors", bool)
def handle_user_annotations(self, context):
self.check_attr("user_annotations__enable", bool)
self.check_attr("user_annotations__type", str)
self.check_attr("user_annotations__local_file_csv__directory", (type(None), str))
self.check_attr("user_annotations__local_file_csv__file", (type(None), str))
self.check_attr("user_annotations__ontology__enable", bool)
self.check_attr("user_annotations__ontology__obo_location", (type(None), str))
self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str))
self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str))
if self.user_annotations__enable:
server_config = self.app_config.server_config
if not self.app__authentication_enable:
raise ConfigurationError("user annotations requires authentication to be enabled")
if not server_config.auth.is_valid_authentication_type():
auth_type = server_config.authentication__type
raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
# TODO, replace this with a factory pattern once we have more than one way
# to do annotations. currently only local_file_csv
if self.user_annotations__type == "local_file_csv":
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None and dirname is not None:
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
if filename is not None:
lf_name, lf_ext = splitext(filename)
if lf_ext and lf_ext != ".csv":
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
if dirname is not None and not isdir(dirname):
try:
os.mkdir(dirname)
except OSError:
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
self.user_annotations = AnnotationsLocalFile(dirname, filename)
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
server_config = self.app_config.server_config
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
try:
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
except OntologyLoadFailure as e:
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
elif self.user_annotations__type == "hosted_tiledb_array":
self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str)
self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str)
self.user_annotations = AnnotationsHostedTileDB(
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
)
else:
raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
else:
if self.user_annotations__type == "local_file_csv":
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None:
context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
if dirname is not None:
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
if self.user_annotations__ontology__enable:
context["messagefn"](
"Warning: --experimental-annotations-ontology" " ignored as annotations are disabled."
)
if self.user_annotations__ontology__obo_location is not None:
context["messagefn"](
"Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled."
)
def handle_embeddings(self, context):
self.check_attr("embeddings__names", list)
self.check_attr("embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
matrix_data_loader = MatrixDataLoader(
server_config.single_dataset__datapath, app_config=self.app_config
)
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
try:
server.compute.scanpy.get_scanpy_module()
except NotImplementedError:
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
def handle_diffexp(self, context):
self.check_attr("diffexp__enable", bool)
self.check_attr("diffexp__lfc_cutoff", float)
self.check_attr("diffexp__top_n", int)
server_config = self.app_config.server_config
if server_config.single_dataset__datapath:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
context["messagefn"](
"CAUTION: due to the size of your dataset, "
"running differential expression may take longer or fail."
)
server/common/aws_secret_utils.py
-61
View File
@@ -1,72 +1,11 @@
import logging
import os
import sys
import boto3
from flask import json
from server.common.data_locator import discover_s3_region_name
from server.common.errors import SecretKeyRetrievalError
def handle_config_from_secret(app_config):
"""Update configuration from the secret manager"""
secret_name = os.getenv("CXG_AWS_SECRET_NAME")
if not secret_name:
return
# need to find the secret manager region.
# 1. from CXG_AWS_SECRET_REGION_NAME
# 2. discover from dataroot location (if on s3)
# 3. discover from config file location (if on s3)
secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
if secret_region_name is None:
secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
if not secret_region_name:
from server.eb.app import config_file
secret_region_name = discover_s3_region_name(config_file)
if not secret_region_name:
logging.error("Could not determine the AWS Secret Manager region")
sys.exit(1)
secrets = get_secret_key(secret_region_name, secret_name)
if not secrets:
return
server_attrs = (
("flask_secret_key", "app__flask_secret_key"),
("oauth_client_secret", "authentication__params_oauth__client_secret"),
)
default_dataset_attrs = (
("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),
)
# update server configuration attributes
for key, attr in server_attrs:
cur_val = getattr(app_config.server_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_server_config(**{attr : val})
# update default dataset configuration attributes
for key, attr in default_dataset_attrs:
cur_val = getattr(app_config.default_dataset_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_default_dataset_config(**{attr : val})
def get_secret_key(region_name, secret_name):
session = boto3.session.Session()
client = session.client(service_name="secretsmanager", region_name=region_name)
server/common/config/__init__.py
+66
View File
@@ -0,0 +1,66 @@
import logging
import os
import sys
from server.common.aws_secret_utils import get_secret_key
from server.common.data_locator import discover_s3_region_name
DEFAULT_SERVER_PORT = 5005
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
def handle_config_from_secret(app_config):
"""Update configuration from the secret manager"""
secret_name = os.getenv("CXG_AWS_SECRET_NAME")
if not secret_name:
return
# need to find the secret manager region.
# 1. from CXG_AWS_SECRET_REGION_NAME
# 2. discover from dataroot location (if on s3)
# 3. discover from config file location (if on s3)
secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
if secret_region_name is None:
secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
if not secret_region_name:
from server.eb.app import config_file
secret_region_name = discover_s3_region_name(config_file)
if not secret_region_name:
logging.error("Could not determine the AWS Secret Manager region")
sys.exit(1)
secrets = get_secret_key(secret_region_name, secret_name)
if not secrets:
return
server_attrs = (
("flask_secret_key", "app__flask_secret_key"),
("oauth_client_secret", "authentication__params_oauth__client_secret"),
)
default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),)
# update server configuration attributes
for key, attr in server_attrs:
cur_val = getattr(app_config.server_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_server_config(**{attr: val})
# update default dataset configuration attributes
for key, attr in default_dataset_attrs:
cur_val = getattr(app_config.default_dataset_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_default_dataset_config(**{attr: val})
server/common/config/app_config.py
+183
View File
@@ -0,0 +1,183 @@
import yaml
from flatten_dict import unflatten
from server.default_config import get_default_config
from server.common.config.dataset_config import DatasetConfig
from server.common.config.server_config import ServerConfig
from server.common.errors import ConfigurationError
class AppConfig(object):
"""
AppConfig stores all the configuration for cellxgene.
AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
The server_config contains attributes that refer to the server process as a whole.
The default_dataset_config refers to attributes that are associated with the features and
presentations of a dataset.
The dataset config attributes can be overridden depending on the url by which the
dataset was accessed. These are stored in dataroot_config.
AppConfig has methods to initialize, modify, and access the configuration.
"""
def __init__(self):
# the default configuration (see default_config.py)
# TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
# defaults within the config class?
self.default_config = get_default_config()
# the server configuration
self.server_config = ServerConfig(self, self.default_config["server"])
# the dataset config, unless overridden by an entry in dataroot_config
self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
# a dictionary of keys to DatasetConfig objects. Each key must exist in the multi_dataset__dataroot
# attribute of the server_config. The default dataset config will apply to all datasets unless a different set
# of config vars was passed for a specific dataset under the multidataset config. For example:
"""
per_dataset_config:
d1:
user_annotations:
enable: false
d2:
user_annotations:
enable: true
"""
# dataroot config
self.dataroot_config = {}
# Set to true when config_completed is called
self.is_completed = False
def get_dataset_config(self, dataroot_key):
if self.server_config.single_dataset__datapath:
return self.default_dataset_config
else:
return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
def check_config(self):
"""Verify all the attributes in the config have been type checked"""
if not self.is_completed:
raise ConfigurationError("The configuration has not been completed")
self.server_config.check_config()
self.default_dataset_config.check_config()
for dataset_config in self.dataroot_config.values():
dataset_config.check_config()
def update_server_config(self, **kw):
self.server_config.update(**kw)
self.is_complete = False
def update_default_dataset_config(self, **kw):
self.default_dataset_config.update(**kw)
# update all the other dataset configs, if any
for value in self.dataroot_config.values():
value.update(**kw)
self.is_complete = False
def update_from_config_file(self, config_file):
try:
with open(config_file) as yml_file:
config = yaml.safe_load(yml_file)
except yaml.YAMLError as e:
raise ConfigurationError(f"The specified config file contained an error: {e}")
except OSError as e:
raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
if config.get("server"):
self.server_config.update_from_config(config["server"], "server")
if config.get("dataset"):
self.default_dataset_config.update_from_config(config["dataset"], "dataset")
per_dataset_config = config.get("per_dataset_config", {})
for key, dataroot_config in per_dataset_config.items():
# first create and initialize the dataroot with the default config
self.add_dataroot_config(key, **config["dataset"])
# then apply the per dataset configuration
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
self.is_complete = False
def write_config(self, config_file):
"""output the config to a yaml file"""
server = self.server_config.create_mapping(self.server_config.default_config)
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = dict(server={}, dataset={})
for attrname in server.keys():
config["server__" + attrname] = getattr(self.server_config, attrname)
for attrname in dataset.keys():
config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
if self.dataroot_config:
config["per_dataset_config"] = {}
for dataroot_tag, dataroot_config in self.dataroot_config.items():
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
for attrname in dataset.keys():
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
config = unflatten(config, splitter=lambda key: key.split("__"))
yaml.dump(config, open(config_file, "w"))
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
diff_server = self.server_config.changes_from_default()
diff_dataset = self.default_dataset_config.changes_from_default()
diff = dict(server=diff_server, dataset=diff_dataset)
return diff
def add_dataroot_config(self, dataroot_tag, **kw):
"""Create a new dataset config object based on the default dataset config, and kw parameters"""
if dataroot_tag in self.dataroot_config:
raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
if type(self.server_config.multi_dataset__dataroot) != dict:
raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
if dataroot_tag not in self.server_config.multi_dataset__dataroot:
raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
self.is_completed = False
self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
config = {key: value[1] for key, value in flat_config.items()}
self.dataroot_config[dataroot_tag].update(**config)
self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
def complete_config(self, messagefn=None):
"""The configure options are checked, and any additional setup based on the config
parameters is done"""
if messagefn is None:
def noop(message):
pass
messagefn = noop
# TODO: to give better error messages we can add a mapping between where each config
# attribute originated (e.g. command line argument or config file), then in the error
# messages we can give correct context for attributes with bad value.
context = dict(messagefn=messagefn)
self.server_config.complete_config(context)
self.default_dataset_config.complete_config(context)
for dataroot_config in self.dataroot_config.values():
dataroot_config.complete_config(context)
self.is_completed = True
self.check_config()
def get_matrix_data_cache_manager(self):
return self.server_config.matrix_data_cache_manager
def is_multi_dataset(self):
return self.server_config.multi_dataset__dataroot is not None
def get_title(self, data_adaptor):
return (
self.server_config.single_dataset__title
if self.server_config.single_dataset__title
else data_adaptor.get_title()
)
def get_about(self, data_adaptor):
return (
self.server_config.single_dataset__about
if self.server_config.single_dataset__about
else data_adaptor.get_about()
)
server/common/config/base_config.py
+113
View File
@@ -0,0 +1,113 @@
import copy
from flatten_dict import flatten
from server.common.errors import ConfigurationError
class BaseConfig(object):
"""
This class handles the mechanics of updating and checking attributes.
Derived classes are expected to store the actual attributes
Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
"""
def __init__(self, app_config, default_config, dictval_cases={}):
# reference back to the app_config
self.app_config = app_config
# the complete set of attributes and their default values (unflattened)
self.default_config = default_config
# attributes where the value may be a dict (and therefore are not flattened)
self.dictval_cases = dictval_cases
# used to make sure every attribute value is checked
self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
def create_mapping(self, config):
"""
Create a dictionary where the keys are the name of attributes (using double underscore convention)
For example: authentication__type
The values are a tuple,
- the first item of the tuple is a tuple of path elements (location in config 'tree')
- the second item is the value of the config parameter
For example: (('authentication', 'type'), 'session'))
"""
config_copy = copy.deepcopy(config)
mapping = {}
# special cases where the value could be a dict.
# If its value is not None, the entry is added to the mapping, and not included
# in the flattening below.
for dictval_case in self.dictval_cases:
cur = config_copy
for part in dictval_case[:-1]:
cur = cur.get(part, {})
val = cur.get(dictval_case[-1])
if val is not None:
key = "__".join(dictval_case)
mapping[key] = (dictval_case, val)
del cur[dictval_case[-1]]
flat_config = flatten(config_copy)
for key, value in flat_config.items():
# name of the attribute
attr = "__".join(key)
mapping[attr] = (key, value)
return mapping
def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
val = getattr(self, attrname)
if type(vtype) in (list, tuple):
if type(val) not in vtype:
tnames = ",".join([x.__name__ for x in vtype])
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
)
else:
if type(val) != vtype:
raise ConfigurationError(
f"Invalid type for attribute: {attrname}, "
f"expected type {vtype.__name__}, got {type(val).__name__}"
)
self.attr_checked[attrname] = True
def check_config(self):
mapping = self.create_mapping(self.default_config)
for key in mapping.keys():
if not self.attr_checked[key]:
raise ConfigurationError(f"The attr '{key}' has not been checked")
def update(self, **kw):
for key, value in kw.items():
if not hasattr(self, key):
raise ConfigurationError(f"unknown config parameter {key}.")
try:
if type(value) == tuple:
# convert tuple values to list values
value = list(value)
setattr(self, key, value)
except KeyError:
raise ConfigurationError(f"Unable to set config parameter {key}.")
self.attr_checked[key] = False
def update_from_config(self, config, prefix):
mapping = self.create_mapping(config)
for attr, (key, value) in mapping.items():
if not hasattr(self, attr):
raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
setattr(self, attr, value)
self.attr_checked[attr] = False
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
mapping = self.create_mapping(self.default_config)
diff = []
for attrname, (key, defval) in mapping.items():
curval = getattr(self, attrname)
if curval != defval:
diff.append((attrname, curval, defval))
return diff
server/common/config/client_config.py
+125
View File
@@ -0,0 +1,125 @@
from server import display_version as cellxgene_display_version
def get_client_config(app_config, data_adaptor):
"""
Return the configuration as required by the /config REST route
"""
server_config = app_config.server_config
dataset_config = data_adaptor.dataset_config
annotation = dataset_config.user_annotations
auth = server_config.auth
# FIXME The current set of config is not consistently presented:
# we have camalCase, hyphen-text, and underscore_text
# make sure the configuration has been checked.
app_config.check_config()
# features
features = [f.todict() for f in data_adaptor.get_features(annotation)]
# display_names
title = app_config.get_title(data_adaptor)
about = app_config.get_about(data_adaptor)
display_names = dict(engine=data_adaptor.get_name(), dataset=title)
# library_versions
library_versions = {}
library_versions.update(data_adaptor.get_library_versions())
library_versions["cellxgene"] = cellxgene_display_version
# links
links = {"about-dataset": about}
# parameters
parameters = {
"layout": dataset_config.embeddings__names,
"max-category-items": dataset_config.presentation__max_categories,
"obs_names": server_config.single_dataset__obs_names,
"var_names": server_config.single_dataset__var_names,
"diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
"backed": server_config.adaptor__anndata_adaptor__backed,
"disable-diffexp": not dataset_config.diffexp__enable,
"enable-reembedding": dataset_config.embeddings__enable_reembedding,
"annotations": False,
"annotations_file": None,
"annotations_dir": None,
"annotations_cell_ontology_enabled": False,
"annotations_cell_ontology_obopath": None,
"annotations_cell_ontology_terms": None,
"custom_colors": dataset_config.presentation__custom_colors,
"diffexp-may-be-slow": False,
"about_legal_tos": dataset_config.app__about_legal_tos,
"about_legal_privacy": dataset_config.app__about_legal_privacy,
}
# corpora dataset_props
# TODO/Note: putting info from the dataset into the /config is not ideal.
# However, it is definitely not part of /schema, and we do not have a top-level
# route for data properties. Consider creating one at some point.
corpora_props = data_adaptor.get_corpora_props()
if corpora_props and "default_embedding" in corpora_props:
default_embedding = corpora_props["default_embedding"]
if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
default_embedding = default_embedding[2:] # drop X_ prefix
if default_embedding in data_adaptor.get_embedding_names():
parameters["default_embedding"] = default_embedding
data_adaptor.update_parameters(parameters)
if annotation:
annotation.update_parameters(parameters, data_adaptor)
# gather it all together
client_config = {}
config = client_config["config"] = {}
config["features"] = features
config["displayNames"] = display_names
config["library_versions"] = library_versions
config["links"] = links
config["parameters"] = parameters
config["corpora_props"] = corpora_props
config["limits"] = {
"column_request_max": server_config.limits__column_request_max,
"diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
}
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
config["authentication"] = {
"requires_client_login": auth.requires_client_login(),
}
if auth.requires_client_login():
config["authentication"].update(
{
# Todo why are these stored on the data_adaptor?
"login": auth.get_login_url(data_adaptor),
"logout": auth.get_logout_url(data_adaptor),
}
)
return client_config
def get_client_userinfo(app_config, data_adaptor):
"""
Return the userinfo as required by the /userinfo REST route
"""
server_config = app_config.server_config
dataset_config = data_adaptor.dataset_config
auth = server_config.auth
# make sure the configuration has been checked.
app_config.check_config()
if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
userinfo = {}
userinfo["userinfo"] = {
"is_authenticated": auth.is_user_authenticated(),
"username": auth.get_user_name(),
"user_id": auth.get_user_id(),
"email": auth.get_user_email(),
}
return userinfo
server/common/config/dataset_config.py
+234
View File
@@ -0,0 +1,234 @@
import os
from os.path import splitext, isdir
from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from server.common.annotations.local_file_csv import AnnotationsLocalFile
from server.common.config.base_config import BaseConfig
from server.common.errors import ConfigurationError, OntologyLoadFailure
from server.compute.scanpy import get_scanpy_module
from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
from server.db.db_utils import DbUtils
class DatasetConfig(BaseConfig):
"""Manages the config attribute associated with a dataset."""
def __init__(self, tag, app_config, default_config):
super().__init__(app_config, default_config)
self.tag = tag
try:
self.app__scripts = default_config["app"]["scripts"]
self.app__inline_scripts = default_config["app"]["inline_scripts"]
self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
self.app__authentication_enable = default_config["app"]["authentication_enable"]
self.presentation__max_categories = default_config["presentation"]["max_categories"]
self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
self.user_annotations__enable = default_config["user_annotations"]["enable"]
self.user_annotations__type = default_config["user_annotations"]["type"]
self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
"directory"
] # noqa E501
self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
"obo_location"
] # noqa E501
self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
"hosted_tiledb_array"
][
"db_uri"
] # noqa E501
self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
"hosted_tiledb_array"
][
"hosted_file_directory"
] # noqa E501
self.embeddings__names = default_config["embeddings"]["names"]
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
self.diffexp__enable = default_config["diffexp"]["enable"]
self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
self.diffexp__top_n = default_config["diffexp"]["top_n"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The annotation object is created during complete_config and stored here.
self.user_annotations = None
def complete_config(self, context):
self.handle_app()
self.handle_presentation()
self.handle_user_annotations(context)
self.handle_embeddings()
self.handle_diffexp(context)
def handle_app(self):
self.validate_correct_type_of_configuration_attribute("app__scripts", list)
self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
# scripts can be string (filename) or dict (attributes). Convert string to dict.
scripts = []
for script in self.app__scripts:
try:
if isinstance(script, str):
scripts.append({"src": script})
elif isinstance(script, dict) and isinstance(script["src"], str):
scripts.append(script)
else:
raise Exception
except Exception as e:
raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
self.app__scripts = scripts
def handle_presentation(self):
self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
def handle_user_annotations(self, context):
self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__directory", (type(None), str)
) # noqa E501
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__file", (type(None), str)
) # noqa E501
self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__ontology__obo_location", (type(None), str)
) # noqa E501
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
) # noqa E501
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
) # noqa E501
if self.user_annotations__enable:
server_config = self.app_config.server_config
if not self.app__authentication_enable:
raise ConfigurationError("user annotations requires authentication to be enabled")
if not server_config.auth.is_valid_authentication_type():
auth_type = server_config.authentication__type
raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
if self.user_annotations__type == "local_file_csv":
self.handle_local_file_csv_annotations()
elif self.user_annotations__type == "hosted_tiledb_array":
self.handle_hosted_tiledb_annotations()
else:
raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
try:
self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
except OntologyLoadFailure as e:
raise ConfigurationError("Unable to load ontology terms\n" + str(e))
else:
self.check_annotation_config_vars_not_set(context)
def handle_local_file_csv_annotations(self):
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
if filename is not None and dirname is not None:
raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
if filename is not None:
lf_name, lf_ext = splitext(filename)
if lf_ext and lf_ext != ".csv":
raise ConfigurationError(f"annotation file type must be .csv: {filename}")
if dirname is not None and not isdir(dirname):
try:
os.mkdir(dirname)
except OSError:
raise ConfigurationError("Unable to create directory specified by --annotations-dir")
self.user_annotations = AnnotationsLocalFile(dirname, filename)
# if the user has specified a fixed label file, go ahead and validate it
# so that we can remove errors early in the process.
server_config = self.app_config.server_config
if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
def handle_hosted_tiledb_annotations(self):
self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", str
) # noqa E501
self.user_annotations = AnnotationsHostedTileDB(
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
)
def check_annotation_config_vars_not_set(self, context):
if self.user_annotations__type is not None:
dirname = self.user_annotations__local_file_csv__directory
filename = self.user_annotations__local_file_csv__file
db_uri = self.user_annotations__hosted_tiledb_array__db_uri
hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
if filename is not None:
context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
if dirname is not None:
context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
if db_uri is not None:
context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
if hosted_file_dirname is not None:
context["messagefn"](
"Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
)
if self.user_annotations__ontology__enable:
context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
if self.user_annotations__ontology__obo_location is not None:
context["messagefn"](
"Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
)
def handle_embeddings(self):
self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
server_config = self.app_config.server_config
if self.embeddings__enable_reembedding:
if server_config.single_dataset__datapath:
matrix_data_loader = MatrixDataLoader(
server_config.single_dataset__datapath, app_config=self.app_config
)
if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
if server_config.adaptor__anndata_adaptor__backed:
raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
try:
get_scanpy_module()
except NotImplementedError:
# Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
def handle_diffexp(self, context):
self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
server_config = self.app_config.server_config
if server_config.single_dataset__datapath:
with server_config.matrix_data_cache_manager.data_adaptor(
self.tag, server_config.single_dataset__datapath, self.app_config
) as data_adaptor:
if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
context["messagefn"](
"CAUTION: due to the size of your dataset, "
"running differential expression may take longer or fail."
)
server/common/config/server_config.py
+390
View File
@@ -0,0 +1,390 @@
import os
import sys
import warnings
from os.path import basename
from urllib.parse import urlparse, quote_plus
from server.auth.auth import AuthTypeFactory
from server.common.config.base_config import BaseConfig
from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
from server.common.errors import ConfigurationError, DatasetAccessError
from server.common.data_locator import discover_s3_region_name
from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
from server.compute import diffexp_cxg as diffexp_tiledb
from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
class ServerConfig(BaseConfig):
"""Manages the config attribute associated with the server."""
def __init__(self, app_config, default_config):
dictval_cases = [
("app", "csp_directives"),
("authentication", "params_oauth", "cookie"),
("authentication", "params_oauth", "jwt_decode_options"),
("adaptor", "cxg_adaptor", "tiledb_ctx"),
("multi_dataset", "dataroot"),
]
super().__init__(app_config, default_config, dictval_cases)
try:
self.app__verbose = default_config["app"]["verbose"]
self.app__debug = default_config["app"]["debug"]
self.app__host = default_config["app"]["host"]
self.app__port = default_config["app"]["port"]
self.app__open_browser = default_config["app"]["open_browser"]
self.app__force_https = default_config["app"]["force_https"]
self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
self.app__csp_directives = default_config["app"]["csp_directives"]
self.app__api_base_url = default_config["app"]["api_base_url"]
self.app__web_base_url = default_config["app"]["web_base_url"]
self.authentication__type = default_config["authentication"]["type"]
self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
"oauth_api_base_url"
] # noqa E501
self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
"client_secret"
] # noqa E501
self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
"jwt_decode_options"
] # noqa E501
self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
"session_cookie"
] # noqa E501
self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
self.multi_dataset__index = default_config["multi_dataset"]["index"]
self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
"max_datasets"
] # noqa E501
self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
"timelimit_s"
] # noqa E501
self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
self.single_dataset__about = default_config["single_dataset"]["about"]
self.single_dataset__title = default_config["single_dataset"]["title"]
self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
self.limits__column_request_max = default_config["limits"]["column_request_max"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
# The matrix data cache manager is created during the complete_config and stored here.
self.matrix_data_cache_manager = None
# The authentication object
self.auth = None
def complete_config(self, context):
self.handle_app(context)
self.handle_data_source()
self.handle_authentication()
self.handle_data_locator()
self.handle_adaptor() # may depend on data_locator
self.handle_single_dataset(context) # may depend on adaptor
self.handle_multi_dataset() # may depend on adaptor
self.handle_diffexp()
self.handle_limits()
self.check_config()
def handle_app(self, context):
self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
self.validate_correct_type_of_configuration_attribute("app__debug", bool)
self.validate_correct_type_of_configuration_attribute("app__host", str)
self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
if self.app__port:
try:
if not is_port_available(self.app__host, self.app__port):
raise ConfigurationError(
f"The port selected {self.app__port} is in use, please configure an open port."
)
except OverflowError:
raise ConfigurationError(f"Invalid port: {self.app__port}")
else:
try:
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
except ValueError:
raise ConfigurationError(
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
)
try:
self.app__port = find_available_port(self.app__host, default_server_port)
except OverflowError:
raise ConfigurationError(f"Invalid port: {default_server_port}")
if self.app__debug:
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
self.app__verbose = True
self.app__open_browser = False
else:
warnings.formatwarning = custom_format_warning
if not self.app__verbose:
sys.tracebacklimit = 0
# secret key:
# first, from CXG_SECRET_KEY environment variable
# second, from config file
self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
# CSP Directives are a dict of string: list(string) or string: string
if self.app__csp_directives is not None:
for k, v in self.app__csp_directives.items():
if not isinstance(k, str):
raise ConfigurationError("CSP directive names must be a string.")
if isinstance(v, list):
for policy in v:
if not isinstance(policy, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
elif not isinstance(v, str):
raise ConfigurationError("CSP directive value must be a string or list of strings.")
if self.app__web_base_url is None:
self.app__web_base_url = self.app__api_base_url
def handle_authentication(self):
self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
# oauth
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__oauth_api_base_url", ptypes
) # noqa E501
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__jwt_decode_options", (type(None), dict)
) # noqa E501
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
if self.authentication__params_oauth__session_cookie:
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__cookie", (type(None), dict)
) # noqa E501
else:
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
# secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
# second, from config file
self.authentication__params_oauth__client_secret = os.environ.get(
"CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret
)
self.auth = AuthTypeFactory.create(self.authentication__type, self)
if self.auth is None:
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
def handle_data_locator(self):
self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
if self.data_locator__s3__region_name is True:
path = self.single_dataset__datapath or self.multi_dataset__dataroot
if type(path) == dict:
# if multi_dataset__dataroot is a dict, then use the first key
# that is in s3. NOTE: it is not supported to have dataroots
# in different regions.
paths = [val.get("dataroot") for val in path.values()]
for path in paths:
if path.startswith("s3://"):
break
if path.startswith("s3://"):
region_name = discover_s3_region_name(path)
if region_name is None:
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
else:
region_name = None
self.data_locator__s3__region_name = region_name
def handle_data_source(self):
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
if self.single_dataset__datapath and self.multi_dataset__dataroot:
raise ConfigurationError(
"You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
)
if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
def handle_single_dataset(self, context):
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
if self.single_dataset__datapath is None:
return
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
# preload this data set
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
try:
matrix_data_loader.pre_load_validation()
except DatasetAccessError as e:
raise ConfigurationError(str(e))
file_size = matrix_data_loader.file_size()
file_basename = basename(self.single_dataset__datapath)
if file_size > BIG_FILE_SIZE_THRESHOLD:
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
else:
context["messagefn"](f"Loading data from {file_basename}.")
if self.single_dataset__about:
def url_check(url):
try:
result = urlparse(url)
if all([result.scheme, result.netloc]):
return True
else:
return False
except ValueError:
return False
if not url_check(self.single_dataset__about):
raise ConfigurationError(
"Must provide an absolute URL for --about. (Example format: http://example.com)"
)
def handle_multi_dataset(self):
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
self.validate_correct_type_of_configuration_attribute(
"multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
) # noqa E501
if self.multi_dataset__dataroot is None:
return
if type(self.multi_dataset__dataroot) == str:
default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
self.multi_dataset__dataroot = dict(d=default_dict)
for tag, dataroot_dict in self.multi_dataset__dataroot.items():
if "base_url" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
if "dataroot" not in dataroot_dict:
raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
base_url = dataroot_dict["base_url"]
# sanity check for well formed base urls
bad = False
if type(base_url) != str:
bad = True
elif os.path.normpath(base_url) != base_url:
bad = True
else:
base_url_parts = base_url.split("/")
if [quote_plus(part) for part in base_url_parts] != base_url_parts:
bad = True
if ".." in base_url_parts:
bad = True
if bad:
raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
# verify all the base_urls are unique
base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
if len(base_urls) > len(set(base_urls)):
raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
# error checking
for mtype in self.multi_dataset__allowed_matrix_types:
try:
MatrixDataType(mtype)
except ValueError:
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
# create the matrix data cache manager:
if self.matrix_data_cache_manager is None:
self.matrix_data_cache_manager = MatrixDataCacheManager(
max_cached=self.multi_dataset__matrix_cache__max_datasets,
timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
)
def handle_diffexp(self):
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
max_workers = self.diffexp__alg_cxg__max_workers
cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
cpu_count = os.cpu_count()
max_workers = min(max_workers, cpu_multiplier * cpu_count)
diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
def handle_adaptor(self):
# cxg
self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
regionkey = "vfs.s3.region"
if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
if type(self.data_locator__s3__region_name) == str:
self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
from server.data_cxg.cxg_adaptor import CxgAdaptor
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
# anndata
self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
def handle_limits(self):
self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
def exceeds_limit(self, limit_name, value):
limit_value = getattr(self, "limits__" + limit_name, None)
if limit_value is None: # disabled
return False
return value > limit_value
def get_api_base_url(self):
if self.app__api_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__api_base_url and self.app__api_base_url.endswith("/"):
return self.app__api_base_url[:-1]
return self.app__api_base_url
def get_web_base_url(self):
if self.app__web_base_url == "local":
return f"http://{self.app__host}:{self.app__port}"
if self.app__web_base_url is None:
return self.get_api_base_url()
if self.app__web_base_url.endswith("/"):
return self.app__web_base_url[:-1]
return self.app__web_base_url
server/common/default_config.py
-212
View File
@@ -1,212 +0,0 @@
import yaml
default_config = """
server:
app:
verbose: false
debug: false
host: localhost
port : null
open_browser: false
force_https: false
flask_secret_key: null
generate_cache_control_headers: false
server_timing_headers: false
csp_directives: null
# By default, cellxgene will serve api requests from the same base url as the webpage.
# In general api_base_url and web_base_url will not need to be set.
# There are two reasons to set these parameters:
# 1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
# which then redirects back to the web_base_url. If the web_base_url is not set, it will default to
# the api_base_url. If oauth authentication is used, the api_base_url must be set.
# For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
# set to the string "local".
# 2. The cellxgene deploymnent is in an environment where the webpage and api have
# different base urls. In this case both api_base_url and web_base_url must be set.
# It is up to the server admin to ensure that the networking is setup correctly for this environment.
api_base_url: null
web_base_url: null
authentication:
# The authentication types may be "none", "session", "oauth"
# none: No authentication support, features like user_annotations must not be enabled.
# session: A session based userid is automatically generated. (no params needed)
# oauth: oauth2 is used for authentication; parameters are defined in params_oauth.
type: session
params_oauth:
# url to the oauth server
oauth_api_base_url: null
# client_id of this app
client_id: null
# the client_secret known to the auth server and this app
client_secret: null
# jwt_decode_options, to specify non default decode options define
# jwt_decode_options to be a dictionary with key/values described by
# the options parameter of the jose.jwt.decode function:
# (https://python-jose.readthedocs.io/en/latest/jwt/api.html)
jwt_decode_options: null
# if true, the jwt containing the id_token is stored in a session cookie
session_cookie: true
# if session_cookie is false, then a regular cookie will be used. In that case
# the cookie will be defined by a dictionary of parameters.
# The keys of the dictionary match the parameters of the flask set_cookie api
# (https://flask.palletsprojects.com/en/1.1.x/api/), and with the same meaning.
# legal keys: key, max_age, expires, path, domain, secure, httponly, and samesite.
cookie: null
multi_dataset:
# If dataroot is set, then cellxgene may serve multiple datasets. This parameter is not
# compatible with single_dataset/datapath.
# dataroot may be a string, representing the path to a directory or S3 prefix. In this
# case the datasets in that location are accessed from <server>/d/<datasetname>.
# example:
# dataroot: /path/to/datasets/
# or
# dataroot: s3://bucket/prefix/
#
# As an alternative, dataroot can be a dictionary, where a dataset key is associated with a base_url
# and a dataroot.
# example:
# dataroot:
# d1:
# base_url: set1
# dataroot: /path/to/set1_datasets/
# d2:
# base_url: set2/subdir
# dataroot: /path/to/set2_datasets/
#
# In this case, datasets can be accessed from <server>/set1/<datasetname> or
# <server>/set2/subdir/<datasetname>. It is possible to have different dataset configurations
# for datasets accessed through different dataroots. For example, in one dataroot, the
# user annotations could be enabled, and in another dataroot they could be disabled.
# To specify dataroot configurations, add a new top level dictionary to the config named
# per_dataset_config. Within per_dataset_config create a dictionary for each dataroot to specialize
# ("d1" or "d2" from the example). Each of these dictionaries has the exact same form as the "dataset"
# dictionary (see below).
# When this approach is used, the values for each configuration option are checked in
# this order: per_dataset_config/<key>, dataset, then the default values.
#
# example:
#
# per_dataset_config:
# d1:
# user_annotations:
# enable: false
# d2:
# user_annotations:
# enable: true
dataroot: null
# The index page when in multi-dataset mode:
# false or null: this returns a 404 code
# true: loads a test index page, which links to the datasets that are available in the dataroot
# string/URL: redirect to this URL: flask.redirect(config.multi_dataset__index)
index: false
# A list of allowed matrix types. If an empty list, then all matrix types are allowed
allowed_matrix_types: []
matrix_cache:
# The maximum number of datasets that may be opened at one time. The least recently used dataset
# is evicted from the cache first.
max_datasets: 5
# A matrix is automatically removed from the cache after timelimit_s number of seconds.
# If timelimit_s is set to None, then there is no time limit.
timelimit_s: 30
single_dataset:
# If datapath is set, then cellxgene with serve a single dataset located at datapath. This parameter is not
# compatible with multi_dataset/dataroot.
datapath: null
obs_names: null
var_names: null
about: null
title: null
diffexp:
alg_cxg:
# The number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count).
# Where cpu_count is determined at runtime.
max_workers: 64
cpu_multiplier: 4
# The target number of matrix elements that are evaluated
# together in one thread.
target_workunit: 16_000_000
data_locator:
s3:
# s3 region name.
# if true, then the s3 location is automatically determined from the datapath or dataroot.
# if false/null, then do not set.
# if a string, then use that value (e.g. us-east-1).
region_name: true
adaptor:
cxg_adaptor:
# The key/values under tiledb_ctx will be used to initialize the tiledb Context.
# If 'vfs.s3.region' is not set, then it will automatically use the setting from
# data_locator / s3 / region_name.
tiledb_ctx:
sm.tile_cache_size: 8589934592
sm.num_reader_threads: 32
anndata_adaptor:
backed: false
limits:
column_request_max: 32
diffexp_cellcount_max: null
dataset:
app:
# Scripts can be a list of either file names (string) or dicts containing keys src, integrity and crossorigin.
# these will be injected into the index template as script tags with these attributes set.
scripts: []
# Inline scripts are a list of file names, where the contents of the file will be injected into the index.
inline_scripts: []
about_legal_tos: null
about_legal_privacy: null
# allow authentication support
authentication_enable: true
presentation:
max_categories: 1000
custom_colors: true
user_annotations:
enable: true
type: local_file_csv
hosted_tiledb_array:
db_uri: null
hosted_file_directory: null
local_file_csv:
directory: null
file: null
ontology:
enable: false
obo_location: null
embeddings:
names : []
enable_reembedding: false
diffexp:
enable: true
lfc_cutoff: 0.01
top_n: 10
"""
def get_default_config():
return yaml.load(default_config, Loader=yaml.Loader)
server/common/errors.py
+4 -4
View File
@@ -42,14 +42,14 @@ define_request_exception(
define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
define_request_exception(
"AuthenticationError",
"Raised when there is an authentication error",
default_status_code=HTTPStatus.UNAUTHORIZED)
"AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
)
define_request_exception(
"AnnotationCategoryNameError",
"Raised when an annotation category name cant be saved",
default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY)
default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
)
define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
define_exception("ConfigurationError", "Raised when checking configuration errors")
server/common/rest.py
+3 -2
View File
@@ -6,6 +6,7 @@ from http import HTTPStatus
from flask import make_response, jsonify, current_app, abort
from werkzeug.urls import url_unquote
from server.common.config.client_config import get_client_config, get_client_userinfo
from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
from server.common.errors import (
FilterError,
@@ -117,12 +118,12 @@ def schema_get(data_adaptor):
def config_get(app_config, data_adaptor):
config = app_config.get_client_config(data_adaptor)
config = get_client_config(app_config, data_adaptor)
return make_response(jsonify(config), HTTPStatus.OK)
def userinfo_get(app_config, data_adaptor):
config = app_config.get_client_userinfo(data_adaptor)
config = get_client_userinfo(app_config, data_adaptor)
return make_response(jsonify(config), HTTPStatus.OK)
server/common/utils/cxg_generation_utils.py
+1 -1
View File
@@ -111,7 +111,7 @@ def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
def convert_matrix_to_cxg_array(
matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
):
"""
Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`
server/common/utils/matrix_utils.py
+7 -4
View File
@@ -41,16 +41,19 @@ def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
number_of_non_zero_elements += np.count_nonzero(matrix_subset)
if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
if end_row_index != total_number_of_rows:
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / (
end_row_index * total_number_of_columns)
percentage_of_non_zero_elements = (
100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
)
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
f"{percentage_of_non_zero_elements:6.2f}")
f"{percentage_of_non_zero_elements:6.2f}"
)
else:
percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
logging.info(
f"Matrix is not sparse. Percentage of non-zero elements (exact): "
f"{percentage_of_non_zero_elements:6.2f}")
f"{percentage_of_non_zero_elements:6.2f}"
)
return False
is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold
server/common/utils/type_conversion_utils.py
+13 -7
View File
@@ -9,8 +9,10 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
schema_type_hints_by_column_name = {}
for column_name, column_values in dataframe.items():
dtypes_by_column_name[column_name], schema_type_hints_by_column_name[column_name] = \
get_dtype_and_schema_of_array(column_values)
(
dtypes_by_column_name[column_name],
schema_type_hints_by_column_name[column_name],
) = get_dtype_and_schema_of_array(column_values)
return dtypes_by_column_name, schema_type_hints_by_column_name
@@ -24,8 +26,10 @@ def get_schema_type_hint_of_array(array: pd.Series):
def get_dtype_and_schema_of_array(array: pd.Series):
return (get_dtype_from_dtype(array.dtype, array_values=array),
get_schema_type_hint_from_dtype(array.dtype, array_values=array))
return (
get_dtype_from_dtype(array.dtype, array_values=array),
get_schema_type_hint_from_dtype(array.dtype, array_values=array),
)
def get_dtype_from_dtype(dtype, array_values=None):
@@ -133,9 +137,11 @@ def can_cast_to_int32(dtype, array_values=None):
if np.can_cast(dtype, np.int32):
return True
ii32 = np.iinfo(np.int32)
if not ordered_array_values.empty and (
ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \
ordered_array_values.empty:
if (
not ordered_array_values.empty
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
or ordered_array_values.empty
):
return True
return False