mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-19 02:48:30 +08:00
fb61bd6e9c
This splits the backend into two parts: the local backend for desktop cellxgene and the AWS backend for hosted cellxgene. The local backend is in local_server while the hosted remains in server. The general idea is to copy everything from server to local_server, pull unneeded stuff out of local_server, and keep server as-is for this PR. Not touching server means all the infra and deployment code will continue working just as it did before so we can make those changes incrementally.
388 lines
20 KiB
Python
388 lines
20 KiB
Python
import os
|
|
import sys
|
|
import warnings
|
|
from os.path import basename
|
|
from urllib.parse import urlparse, quote_plus
|
|
|
|
from server.auth.auth import AuthTypeFactory
|
|
from server.common.config.base_config import BaseConfig
|
|
from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
|
|
from server.common.errors import ConfigurationError, DatasetAccessError
|
|
from server.common.data_locator import discover_s3_region_name
|
|
from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
|
|
from server.compute import diffexp_cxg as diffexp_tiledb
|
|
from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
|
|
|
|
|
|
class ServerConfig(BaseConfig):
|
|
"""Manages the config attribute associated with the server."""
|
|
|
|
def __init__(self, app_config, default_config):
|
|
dictval_cases = [
|
|
("app", "csp_directives"),
|
|
("authentication", "params_oauth", "cookie"),
|
|
("authentication", "params_oauth", "jwt_decode_options"),
|
|
("adaptor", "cxg_adaptor", "tiledb_ctx"),
|
|
("multi_dataset", "dataroot"),
|
|
]
|
|
super().__init__(app_config, default_config, dictval_cases)
|
|
|
|
try:
|
|
self.app__verbose = default_config["app"]["verbose"]
|
|
self.app__debug = default_config["app"]["debug"]
|
|
self.app__host = default_config["app"]["host"]
|
|
self.app__port = default_config["app"]["port"]
|
|
self.app__open_browser = default_config["app"]["open_browser"]
|
|
self.app__force_https = default_config["app"]["force_https"]
|
|
self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
|
|
self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
|
|
self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
|
|
self.app__csp_directives = default_config["app"]["csp_directives"]
|
|
self.app__api_base_url = default_config["app"]["api_base_url"]
|
|
self.app__web_base_url = default_config["app"]["web_base_url"]
|
|
|
|
self.authentication__type = default_config["authentication"]["type"]
|
|
self.authentication__insecure_test_environment = default_config["authentication"][
|
|
"insecure_test_environment"
|
|
]
|
|
self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
|
|
"oauth_api_base_url"
|
|
]
|
|
self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
|
|
self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
|
|
"client_secret"
|
|
]
|
|
self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
|
|
"jwt_decode_options"
|
|
]
|
|
self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
|
|
"session_cookie"
|
|
]
|
|
self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
|
|
|
|
self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
|
|
self.multi_dataset__index = default_config["multi_dataset"]["index"]
|
|
self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
|
|
self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
|
|
"max_datasets"
|
|
]
|
|
self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
|
|
"timelimit_s"
|
|
]
|
|
|
|
self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
|
|
self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
|
|
self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
|
|
self.single_dataset__about = default_config["single_dataset"]["about"]
|
|
self.single_dataset__title = default_config["single_dataset"]["title"]
|
|
|
|
self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
|
|
self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
|
|
self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
|
|
|
|
self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
|
|
|
|
self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
|
|
self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
|
|
|
|
self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
|
|
self.limits__column_request_max = default_config["limits"]["column_request_max"]
|
|
|
|
except KeyError as e:
|
|
raise ConfigurationError(f"Unexpected config: {str(e)}")
|
|
|
|
# The matrix data cache manager is created during the complete_config and stored here.
|
|
self.matrix_data_cache_manager = None
|
|
|
|
# The authentication object
|
|
self.auth = None
|
|
|
|
def complete_config(self, context):
|
|
self.handle_app(context)
|
|
self.handle_data_source()
|
|
self.handle_authentication()
|
|
self.handle_data_locator()
|
|
self.handle_adaptor() # may depend on data_locator
|
|
self.handle_single_dataset(context) # may depend on adaptor
|
|
self.handle_multi_dataset() # may depend on adaptor
|
|
self.handle_diffexp()
|
|
self.handle_limits()
|
|
|
|
self.check_config()
|
|
|
|
def handle_app(self, context):
|
|
self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
|
|
self.validate_correct_type_of_configuration_attribute("app__debug", bool)
|
|
self.validate_correct_type_of_configuration_attribute("app__host", str)
|
|
self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
|
|
self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
|
|
self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
|
|
self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
|
|
self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
|
|
self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
|
|
self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
|
|
self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
|
|
self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
|
|
|
|
if self.app__port:
|
|
try:
|
|
if not is_port_available(self.app__host, self.app__port):
|
|
raise ConfigurationError(
|
|
f"The port selected {self.app__port} is in use, please configure an open port."
|
|
)
|
|
except OverflowError:
|
|
raise ConfigurationError(f"Invalid port: {self.app__port}")
|
|
else:
|
|
try:
|
|
default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
|
|
except ValueError:
|
|
raise ConfigurationError(
|
|
"Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
|
|
)
|
|
try:
|
|
self.app__port = find_available_port(self.app__host, default_server_port)
|
|
except OverflowError:
|
|
raise ConfigurationError(f"Invalid port: {default_server_port}")
|
|
|
|
if self.app__debug:
|
|
context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
|
|
self.app__verbose = True
|
|
self.app__open_browser = False
|
|
else:
|
|
warnings.formatwarning = custom_format_warning
|
|
|
|
if not self.app__verbose:
|
|
sys.tracebacklimit = 0
|
|
|
|
# CSP Directives are a dict of string: list(string) or string: string
|
|
if self.app__csp_directives is not None:
|
|
for k, v in self.app__csp_directives.items():
|
|
if not isinstance(k, str):
|
|
raise ConfigurationError("CSP directive names must be a string.")
|
|
if isinstance(v, list):
|
|
for policy in v:
|
|
if not isinstance(policy, str):
|
|
raise ConfigurationError("CSP directive value must be a string or list of strings.")
|
|
elif not isinstance(v, str):
|
|
raise ConfigurationError("CSP directive value must be a string or list of strings.")
|
|
|
|
if self.app__web_base_url is None:
|
|
self.app__web_base_url = self.app__api_base_url
|
|
|
|
def handle_authentication(self):
|
|
self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
|
|
self.validate_correct_type_of_configuration_attribute("authentication__insecure_test_environment", bool)
|
|
|
|
if self.authentication__type == "test" and not self.authentication__insecure_test_environment:
|
|
raise ConfigurationError("Test auth can only be used in an insecure test environment")
|
|
|
|
# oauth
|
|
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
|
|
self.validate_correct_type_of_configuration_attribute(
|
|
"authentication__params_oauth__oauth_api_base_url", ptypes
|
|
)
|
|
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
|
|
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
|
|
self.validate_correct_type_of_configuration_attribute(
|
|
"authentication__params_oauth__jwt_decode_options", (type(None), dict)
|
|
)
|
|
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
|
|
|
|
if self.authentication__params_oauth__session_cookie:
|
|
self.validate_correct_type_of_configuration_attribute(
|
|
"authentication__params_oauth__cookie", (type(None), dict)
|
|
)
|
|
else:
|
|
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
|
|
|
|
self.auth = AuthTypeFactory.create(self.authentication__type, self)
|
|
if self.auth is None:
|
|
raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
|
|
|
|
def handle_data_locator(self):
|
|
self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
|
|
if self.data_locator__s3__region_name is True:
|
|
path = self.single_dataset__datapath or self.multi_dataset__dataroot
|
|
|
|
if type(path) == dict:
|
|
# if multi_dataset__dataroot is a dict, then use the first key
|
|
# that is in s3. NOTE: it is not supported to have dataroots
|
|
# in different regions.
|
|
paths = [val.get("dataroot") for val in path.values()]
|
|
for path in paths:
|
|
if path.startswith("s3://"):
|
|
break
|
|
if path.startswith("s3://"):
|
|
region_name = discover_s3_region_name(path)
|
|
if region_name is None:
|
|
raise ConfigurationError(f"Unable to discover s3 region name from {path}")
|
|
else:
|
|
region_name = None
|
|
self.data_locator__s3__region_name = region_name
|
|
|
|
def handle_data_source(self):
|
|
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
|
|
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
|
|
|
|
if self.single_dataset__datapath and self.multi_dataset__dataroot:
|
|
raise ConfigurationError(
|
|
"You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
|
|
)
|
|
if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
|
|
raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
|
|
|
|
def handle_single_dataset(self, context):
|
|
self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
|
|
self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
|
|
self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
|
|
self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
|
|
self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
|
|
|
|
if self.single_dataset__datapath is None:
|
|
return
|
|
|
|
# create the matrix data cache manager:
|
|
if self.matrix_data_cache_manager is None:
|
|
self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
|
|
|
|
# preload this data set
|
|
matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
|
|
try:
|
|
matrix_data_loader.pre_load_validation()
|
|
except DatasetAccessError as e:
|
|
raise ConfigurationError(str(e))
|
|
|
|
file_size = matrix_data_loader.file_size()
|
|
file_basename = basename(self.single_dataset__datapath)
|
|
if file_size > BIG_FILE_SIZE_THRESHOLD:
|
|
context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
|
|
else:
|
|
context["messagefn"](f"Loading data from {file_basename}.")
|
|
|
|
if self.single_dataset__about:
|
|
|
|
def url_check(url):
|
|
try:
|
|
result = urlparse(url)
|
|
if all([result.scheme, result.netloc]):
|
|
return True
|
|
else:
|
|
return False
|
|
except ValueError:
|
|
return False
|
|
|
|
if not url_check(self.single_dataset__about):
|
|
raise ConfigurationError(
|
|
"Must provide an absolute URL for --about. (Example format: http://example.com)"
|
|
)
|
|
|
|
def handle_multi_dataset(self):
|
|
self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
|
|
self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
|
|
self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
|
|
self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
|
|
self.validate_correct_type_of_configuration_attribute(
|
|
"multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
|
|
)
|
|
|
|
if self.multi_dataset__dataroot is None:
|
|
return
|
|
|
|
if type(self.multi_dataset__dataroot) == str:
|
|
default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
|
|
self.multi_dataset__dataroot = dict(d=default_dict)
|
|
|
|
for tag, dataroot_dict in self.multi_dataset__dataroot.items():
|
|
if "base_url" not in dataroot_dict:
|
|
raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
|
|
if "dataroot" not in dataroot_dict:
|
|
raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
|
|
|
|
base_url = dataroot_dict["base_url"]
|
|
|
|
# sanity check for well formed base urls
|
|
bad = False
|
|
if type(base_url) != str:
|
|
bad = True
|
|
elif os.path.normpath(base_url) != base_url:
|
|
bad = True
|
|
else:
|
|
base_url_parts = base_url.split("/")
|
|
if [quote_plus(part) for part in base_url_parts] != base_url_parts:
|
|
bad = True
|
|
if ".." in base_url_parts:
|
|
bad = True
|
|
if bad:
|
|
raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
|
|
|
|
# verify all the base_urls are unique
|
|
base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
|
|
if len(base_urls) > len(set(base_urls)):
|
|
raise ConfigurationError("error in multi_dataset__dataroot: base_urls must be unique")
|
|
|
|
# error checking
|
|
for mtype in self.multi_dataset__allowed_matrix_types:
|
|
try:
|
|
MatrixDataType(mtype)
|
|
except ValueError:
|
|
raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
|
|
|
|
# create the matrix data cache manager:
|
|
if self.matrix_data_cache_manager is None:
|
|
self.matrix_data_cache_manager = MatrixDataCacheManager(
|
|
max_cached=self.multi_dataset__matrix_cache__max_datasets,
|
|
timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
|
|
)
|
|
|
|
def handle_diffexp(self):
|
|
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
|
|
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
|
|
self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
|
|
|
|
max_workers = self.diffexp__alg_cxg__max_workers
|
|
cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
|
|
cpu_count = os.cpu_count()
|
|
max_workers = min(max_workers, cpu_multiplier * cpu_count)
|
|
diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
|
|
|
|
def handle_adaptor(self):
|
|
# cxg
|
|
self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
|
|
regionkey = "vfs.s3.region"
|
|
if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
|
|
if type(self.data_locator__s3__region_name) == str:
|
|
self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
|
|
|
|
from server.data_cxg.cxg_adaptor import CxgAdaptor
|
|
|
|
CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
|
|
|
|
# anndata
|
|
self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
|
|
|
|
def handle_limits(self):
|
|
self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
|
|
self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
|
|
|
|
def exceeds_limit(self, limit_name, value):
|
|
limit_value = getattr(self, "limits__" + limit_name, None)
|
|
if limit_value is None: # disabled
|
|
return False
|
|
return value > limit_value
|
|
|
|
def get_api_base_url(self):
|
|
if self.app__api_base_url == "local":
|
|
return f"http://{self.app__host}:{self.app__port}"
|
|
if self.app__api_base_url and self.app__api_base_url.endswith("/"):
|
|
return self.app__api_base_url[:-1]
|
|
return self.app__api_base_url
|
|
|
|
def get_web_base_url(self):
|
|
if self.app__web_base_url == "local":
|
|
return f"http://{self.app__host}:{self.app__port}"
|
|
if self.app__web_base_url is None:
|
|
return self.get_api_base_url()
|
|
if self.app__web_base_url.endswith("/"):
|
|
return self.app__web_base_url[:-1]
|
|
return self.app__web_base_url
|