fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-26 11:28:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into release-version-0.16.5

Browse Source
This commit is contained in:
maniarathi
2020-10-21 09:13:35 -07:00
parent 17e0cbeec9 377e4bccaa
commit 0821b38e84
53 changed files with 1682 additions and 620 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/Makefile
+8
View File
@@ -39,3 +39,11 @@ create-test-db:
clean-test-db:
-docker stop test_db
-docker rm test_db
.PHONY: test-annotations-performance
test-annotations-performance:
python test/performance/performance_test_annotations_backend.py
.PHONY: test-annotations-scale
test-annotations-scale:
locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
server/auth/auth.py
+4
View File
@@ -43,6 +43,10 @@ class AuthTypeBase(ABC):
"""Return the name of the user (string)"""
pass
def get_user_picture(self):
"""Return the location to the user's picture"""
return None
class AuthTypeClientBase(AuthTypeBase):
"""Base type for all authentication types that require the client to login"""
server/auth/auth_oauth.py
+7 -9
View File
@@ -146,21 +146,19 @@ class AuthTypeOAuth(AuthTypeClientBase):
def get_user_id(self):
payload = self.get_userinfo()
if payload and payload.get("sub"):
return payload.get("sub")
return None
return payload.get("sub") if payload else None
def get_user_name(self):
payload = self.get_userinfo()
if payload and payload.get("name"):
return payload.get("name")
return None
return payload.get("name") if payload else None
def get_user_email(self):
payload = self.get_userinfo()
if payload and payload.get("email"):
return payload.get("email")
return None
return payload.get("email") if payload else None
def get_user_picture(self):
payload = self.get_userinfo()
return payload.get("picture") if payload else None
def update_response(self, response):
response.cache_control.update(dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
server/auth/auth_test.py
+6
View File
@@ -10,12 +10,14 @@ class AuthTypeTest(AuthTypeClientBase):
CXGUID = "cxguid_test"
CXGUNAME = "cxguname_test"
CXGUEMAIL = "cxguemail_test"
CXGUPICTURE = "cxgupicture_test"
def __init__(self, app_config):
super().__init__()
self.user_name = "test_account"
self.user_id = "id0001"
self.user_email = "test_account@test.com"
self.user_picture = None
def is_valid_authentication_type(self):
return True
@@ -42,12 +44,16 @@ class AuthTypeTest(AuthTypeClientBase):
def get_user_email(self):
return session.get(self.CXGUEMAIL)
def get_user_picture(self):
return session.get(self.CXGUPICTURE)
def login(self):
args = request.args
return_to = args.get("dataset", "/")
session[self.CXGUID] = args.get("userid", self.user_id)
session[self.CXGUNAME] = args.get("username", self.user_name)
session[self.CXGUEMAIL] = args.get("email", self.user_email)
session[self.CXGUPICTURE] = args.get("picture", self.user_picture)
return redirect(return_to)
def logout(self):
server/common/aws_secret_utils.py
+1 -1
View File
@@ -18,6 +18,6 @@ def get_secret_key(region_name, secret_name):
return secret
except Exception as e:
logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
raise SecretKeyRetrievalError
raise SecretKeyRetrievalError(str(e))
return None
server/common/config/__init__.py
+1 -63
View File
@@ -1,66 +1,4 @@
import logging
import os
import sys
from server.common.aws_secret_utils import get_secret_key
from server.common.data_locator import discover_s3_region_name
from server.common.aws_secret_utils import get_secret_key # noqa F504
DEFAULT_SERVER_PORT = 5005
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
def handle_config_from_secret(app_config):
"""Update configuration from the secret manager"""
secret_name = os.getenv("CXG_AWS_SECRET_NAME")
if not secret_name:
return
# need to find the secret manager region.
# 1. from CXG_AWS_SECRET_REGION_NAME
# 2. discover from dataroot location (if on s3)
# 3. discover from config file location (if on s3)
secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
if secret_region_name is None:
secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
if not secret_region_name:
from server.eb.app import config_file
secret_region_name = discover_s3_region_name(config_file)
if not secret_region_name:
logging.error("Could not determine the AWS Secret Manager region")
sys.exit(1)
secrets = get_secret_key(secret_region_name, secret_name)
if not secrets:
return
server_attrs = (
("flask_secret_key", "app__flask_secret_key"),
("oauth_client_secret", "authentication__params_oauth__client_secret"),
)
default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),)
# update server configuration attributes
for key, attr in server_attrs:
cur_val = getattr(app_config.server_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_server_config(**{attr: val})
# update default dataset configuration attributes
for key, attr in default_dataset_attrs:
cur_val = getattr(app_config.default_dataset_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_default_dataset_config(**{attr: val})
server/common/config/app_config.py
+67 -3
View File
@@ -4,6 +4,7 @@ from flatten_dict import unflatten
from server.default_config import get_default_config
from server.common.config.dataset_config import DatasetConfig
from server.common.config.server_config import ServerConfig
from server.common.config.external_config import ExternalConfig
from server.common.errors import ConfigurationError
@@ -44,6 +45,9 @@ class AppConfig(object):
# dataroot config
self.dataroot_config = {}
# external config
self.external_config = ExternalConfig(self, self.default_config["external"])
# Set to true when config_completed is called
self.is_completed = False
@@ -61,6 +65,7 @@ class AppConfig(object):
self.default_dataset_config.check_config()
for dataset_config in self.dataroot_config.values():
dataset_config.check_config()
self.external_config.check_config()
def update_server_config(self, **kw):
self.server_config.update(**kw)
@@ -73,6 +78,51 @@ class AppConfig(object):
value.update(**kw)
self.is_complete = False
def update_single_config_from_path_and_value(self, path, value):
"""Update a single config parameter with the value.
Path is a list of string, that gives a path to the config parameter to be updated.
For example, path may be ["server","app","port"].
"""
self.is_complete = False
if not isinstance(path, list):
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
for part in path:
if not isinstance(part, str):
raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
if path[0] == "server":
attr = "__".join(path[1:])
try:
self.update_server_config(**{attr: value})
except ConfigurationError:
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
elif path[0] == "dataset":
attr = "__".join(path[1:])
try:
self.update_default_dataset_config(**{attr: value})
except ConfigurationError:
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
elif path[0] == "per_dataset_config":
if len(path) < 2:
raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
dataroot = path[1]
if dataroot not in self.dataroot_config:
dataroots = str(list(self.dataroot_config.keys()))
raise ConfigurationError(
f"unknown dataroot when using per_dataset_config: got '{path}',"
f" dataroots specified in config are {dataroots}"
)
attr = "__".join(path[2:])
try:
self.dataroot_config[dataroot].update(**{attr: value})
except ConfigurationError:
raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
def update_from_config_file(self, config_file):
try:
with open(config_file) as yml_file:
@@ -94,12 +144,16 @@ class AppConfig(object):
# then apply the per dataset configuration
self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
if config.get("external"):
self.external_config.update_from_config(config["external"], "external")
self.is_complete = False
def write_config(self, config_file):
"""output the config to a yaml file"""
def config_to_dict(self):
"""return the configuration as an unflattened dict"""
server = self.server_config.create_mapping(self.server_config.default_config)
dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
external = self.external_config.create_mapping(self.external_config.default_config)
config = dict(server={}, dataset={})
for attrname in server.keys():
config["server__" + attrname] = getattr(self.server_config, attrname)
@@ -111,15 +165,23 @@ class AppConfig(object):
dataset = dataroot_config.create_mapping(dataroot_config.default_config)
for attrname in dataset.keys():
config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
for attrname in external.keys():
config["external__" + attrname] = getattr(self.external_config, attrname)
config = unflatten(config, splitter=lambda key: key.split("__"))
return config
def write_config(self, config_file):
"""output the config to a yaml file"""
config = self.config_to_dict()
yaml.dump(config, open(config_file, "w"))
def changes_from_default(self):
"""Return all the attribute that are different from the default"""
diff_server = self.server_config.changes_from_default()
diff_dataset = self.default_dataset_config.changes_from_default()
diff = dict(server=diff_server, dataset=diff_dataset)
diff_external = self.external.changes_from_default()
diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
return diff
def add_dataroot_config(self, dataroot_tag, **kw):
@@ -154,6 +216,8 @@ class AppConfig(object):
# messages we can give correct context for attributes with bad value.
context = dict(messagefn=messagefn)
# complete config for external_config first, since this may update values in the other sections
self.external_config.complete_config(context)
self.server_config.complete_config(context)
self.default_dataset_config.complete_config(context)
for dataroot_config in self.dataroot_config.values():
server/common/config/base_config.py
+19
View File
@@ -80,8 +80,27 @@ class BaseConfig(object):
raise ConfigurationError(f"The attr '{key}' has not been checked")
def update(self, **kw):
"""Update the attributes defined in kw with their new values."""
for key, value in kw.items():
if not hasattr(self, key):
# check if the key is setting into a dictval entry.
found_dictval = False
for dictval in self.dictval_cases:
dictvalname = "__".join(dictval)
if dictvalname + "__" in key:
dictkey = key[len(dictvalname) + 2 :]
curdictval = getattr(self, dictvalname)
if curdictval is None:
setattr(self, dictvalname, dict(dictkey=value))
else:
curdictval[dictkey] = value
found_dictval = True
break
if found_dictval:
continue
raise ConfigurationError(f"unknown config parameter {key}.")
try:
if type(value) == tuple:
server/common/config/client_config.py
+1
View File
@@ -117,5 +117,6 @@ def get_client_userinfo(app_config, data_adaptor):
"username": auth.get_user_name(),
"user_id": auth.get_user_id(),
"email": auth.get_user_email(),
"picture": auth.get_user_picture(),
}
return userinfo
server/common/config/dataset_config.py
+10 -14
View File
@@ -30,22 +30,18 @@ class DatasetConfig(BaseConfig):
self.user_annotations__type = default_config["user_annotations"]["type"]
self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
"directory"
] # noqa E501
]
self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
"obo_location"
] # noqa E501
]
self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
"hosted_tiledb_array"
][
"db_uri"
] # noqa E501
]["db_uri"]
self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
"hosted_tiledb_array"
][
"hosted_file_directory"
] # noqa E501
]["hosted_file_directory"]
self.embeddings__names = default_config["embeddings"]["names"]
self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
@@ -98,20 +94,20 @@ class DatasetConfig(BaseConfig):
self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__directory", (type(None), str)
) # noqa E501
)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__local_file_csv__file", (type(None), str)
) # noqa E501
)
self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__ontology__obo_location", (type(None), str)
) # noqa E501
)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
) # noqa E501
)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
) # noqa E501
)
if self.user_annotations__enable:
server_config = self.app_config.server_config
if not self.app__authentication_enable:
@@ -166,7 +162,7 @@ class DatasetConfig(BaseConfig):
self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
self.validate_correct_type_of_configuration_attribute(
"user_annotations__hosted_tiledb_array__hosted_file_directory", str
) # noqa E501
)
self.user_annotations = AnnotationsHostedTileDB(
directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
server/common/config/external_config.py
+96
View File
@@ -0,0 +1,96 @@
import os
from server.common.config.base_config import BaseConfig
from server.common.errors import ConfigurationError
from server.common.config import get_secret_key
from server.common.errors import SecretKeyRetrievalError
from server.common.utils.type_conversion_utils import convert_string_to_value
class ExternalConfig(BaseConfig):
"""Manages the config attribute associated with external configuration sources, such as
environment variables or the AWS Secrets Manager."""
def __init__(self, app_config, default_config):
super().__init__(app_config, default_config)
try:
self.environment = default_config["environment"]
self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
except KeyError as e:
raise ConfigurationError(f"Unexpected config: {str(e)}")
def complete_config(self, context):
self.handle_environment(context)
self.handle_aws_secrets_manager(context)
def handle_environment(self, context):
"""For each environment variable defined, get the value (if it is set),
and set the specified config parameter"""
self.validate_correct_type_of_configuration_attribute("environment", list)
for envdict in self.environment:
name = envdict.get("name")
if name is None:
raise ConfigurationError("environment: 'name' is missing")
required = envdict.get("required", False)
if type(required) != bool:
raise ConfigurationError("environment: 'required' must be a bool")
path = envdict.get("path")
if path is None:
raise ConfigurationError("environment: 'path' is missing")
value = os.environ.get(name)
if value is None:
if required:
raise ConfigurationError(f"required environment variable '{name}' not set")
else:
value = convert_string_to_value(value)
self.app_config.update_single_config_from_path_and_value(path, value)
def handle_aws_secrets_manager(self, context):
"""For each aws secret defined, get the key/values, and set the specified config parameter"""
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
if not self.aws_secrets_manager__secrets:
return
self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
for secret in self.aws_secrets_manager__secrets:
secret_name = secret.get("name")
if secret_name is None:
raise ConfigurationError("aws_secrets_manager: 'name' is missing")
if not isinstance(secret_name, str):
raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
try:
secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
except SecretKeyRetrievalError as e:
raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
values = secret.get("values")
if values is None:
raise ConfigurationError("aws_secrets_manager: 'values' is missing")
if not isinstance(values, list):
raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
for value in values:
key = value.get("key")
if key is None:
raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
path = value.get("path")
if path is None:
raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
required = value.get("required", False)
if type(required) != bool:
raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
secret_value = secret_dict.get(key)
if secret_value is None:
if required:
raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
else:
secret_value = convert_string_to_value(secret_value)
self.app_config.update_single_config_from_path_and_value(path, secret_value)
server/common/config/server_config.py
+11 -21
View File
@@ -44,17 +44,17 @@ class ServerConfig(BaseConfig):
self.authentication__type = default_config["authentication"]["type"]
self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
"oauth_api_base_url"
] # noqa E501
]
self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
"client_secret"
] # noqa E501
]
self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
"jwt_decode_options"
] # noqa E501
]
self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
"session_cookie"
] # noqa E501
]
self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
@@ -62,10 +62,10 @@ class ServerConfig(BaseConfig):
self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
"max_datasets"
] # noqa E501
]
self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
"timelimit_s"
] # noqa E501
]
self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
@@ -114,7 +114,7 @@ class ServerConfig(BaseConfig):
self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", (type(None), str))
self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
@@ -151,11 +151,6 @@ class ServerConfig(BaseConfig):
if not self.app__verbose:
sys.tracebacklimit = 0
# secret key:
# first, from CXG_SECRET_KEY environment variable
# second, from config file
self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
# CSP Directives are a dict of string: list(string) or string: string
if self.app__csp_directives is not None:
for k, v in self.app__csp_directives.items():
@@ -178,25 +173,20 @@ class ServerConfig(BaseConfig):
ptypes = str if self.authentication__type == "oauth" else (type(None), str)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__oauth_api_base_url", ptypes
) # noqa E501
)
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__jwt_decode_options", (type(None), dict)
) # noqa E501
)
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
if self.authentication__params_oauth__session_cookie:
self.validate_correct_type_of_configuration_attribute(
"authentication__params_oauth__cookie", (type(None), dict)
) # noqa E501
)
else:
self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
# secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
# second, from config file
self.authentication__params_oauth__client_secret = os.environ.get(
"CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret
)
self.auth = AuthTypeFactory.create(self.authentication__type, self)
if self.auth is None:
@@ -286,7 +276,7 @@ class ServerConfig(BaseConfig):
self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
self.validate_correct_type_of_configuration_attribute(
"multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
) # noqa E501
)
if self.multi_dataset__dataroot is None:
return
server/common/utils/type_conversion_utils.py
+21 -16
View File
@@ -88,24 +88,15 @@ def get_schema_type_hint_from_dtype(dtype, array_values=None):
def can_cast_to_float32(dtype, array_values):
"""
A dtype can be cast to float32 if it is a float type and converting it to float32 presents the same output as the
original values. Note that NaNs fail equality (i.e. np.NaN != np.NaN) so we use np.testing.assert_equal to ensure
that the arrays are equal minus NaNs.
Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
a float.
We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
Since NaNs are floating points in numpy, we upcast the integer array to float32.
Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
"""
if dtype.kind == "f":
# Try to convert the array to float32
converted_float32_values = array_values.to_numpy(np.float32)
original_values = array_values.to_numpy()
# Verify that the two arrays are equal except for NaNs (which will equate to be unequal).
if not ((converted_float32_values != original_values) == np.isnan(original_values)).all():
return False
if dtype != np.float32:
if not np.can_cast(dtype, np.float32):
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
return True
@@ -138,9 +129,9 @@ def can_cast_to_int32(dtype, array_values=None):
return True
ii32 = np.iinfo(np.int32)
if (
not ordered_array_values.empty
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
or ordered_array_values.empty
not ordered_array_values.empty
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
or ordered_array_values.empty
):
return True
return False
@@ -151,3 +142,17 @@ def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
return series_to_convert.to_numpy(dtype)
def convert_string_to_value(value: str):
"""convert a string to value with the most appropriate type"""
if value.lower() == "true":
return True
if value.lower() == "false":
return False
if value == "null":
return None
try:
return eval(value)
except: # noqa E722
return value
server/default_config.py
+55
View File
@@ -204,6 +204,61 @@ dataset:
enable: true
lfc_cutoff: 0.01
top_n: 10
external:
# You can retrieve configuration parameters from this config file, the environment,
# the AWS secrets manager, or from the "cellxgene launch" command line arguments.
# They are applied in that order, meaning that if a parameter is defined in more
# than one location, the last one applied takes effect.
# environment variables:
# This section describes how to map environment variables to configuration parameters.
# The format is a list defining an environment variable.
# Each entry in the list is a dictionary with three entries:
# name: the name of the environment variable
# path: the path within the cellxgene configuration to update.
# required: (default=False) a boolean. If true, then it is an error if the environment variable is not set.
environment:
- name: CXG_SECRET_KEY
path: [server, app, flask_secret_key]
required: false
- name: CXG_OAUTH_CLIENT_SECRET
path: [server, authentication, params_oauth, client_secret]
required: false
# AWS Secrets Manager
# This section describes how to map aws secrets to configuration parameters.
# The format is the region for the secrets manager, then a list of secrets.
# each secret has a name, and a list of values.
# Each entry in the list of values is a dictionary with three entries:
# key: the key of the aws secret.
# path: the path within the cellxgene configuration to update.
# required: (default=False) a boolean. If true, then it is an error if the key does not exist in the secret.
#
# example:
# aws_secrets_manager:
# region: us-west-2
# - name: my_first_secret
# values:
# - key: flask_secret_key
# path: [server, app, flask_secret_key]
# required: true
# - key: db_uri
# path: [dataset, user_annotations, hosted_tiledb_array, db_uri]
# required: true
# - name: my_auth_secret
# values:
# - key: client_secret
# path: [server, authentication, params_oauth, client_secret]
# required: true
# - key: client_id
# path: [server, authentication, params_oauth, client_id]
# required: true
aws_secrets_manager:
region: null
secrets: []
"""
server/eb/README.md
+118 -47
View File
@@ -1,12 +1,12 @@
# AWS Elastic Beanstalk
This directory contains script to aid in creating and deploying cellxgene on
an AWS Elastic Beanstalk instance.
This directory contains scripts to aid in creating and deploying cellxgene on
AWS Elastic Beanstalk.
This will result in a variant of cellxgene, running on AWS EC2 instances, serving data from S3.
All datasets must be in the new CXG (tiledb) format - see the converter script cxgtool.py
in server/converters - and located in a single S3 prefix, which is accessible to the instance.
In the current incarnation, no access control or authentication support is available
All datasets must be in the CXG (tiledb) format (see `cellxene convert --help`),
and located under a single S3 prefix, which is accessible to the instance.
In the current incarnation, no access control is available
(outside of anything you configure yourself), so this is most appropriate for public datasets.
This is early development work, and will change significantly in the near future.
@@ -17,10 +17,10 @@ We would love feedback on it, but please assume it will change.
1. Some familiarity with AWS EB, S3, and IAM are needed.
2. Install the awsebcli.
Instruction are here:
https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html
Instruction are here:
https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html
3. In the top level directory, run ```make build-client``` to create the client static assets.
3. In the top level directory, run `make build-client` to create the client static assets.
## Steps
@@ -31,20 +31,21 @@ There are many more options to these commands that may be important or necessary
The following choices are known to work.
* S3 Bucket.
* POSIX filesystem (such as Lustre)
* Lustre filesystem backed by S3
- S3 Bucket.
- POSIX filesystem (such as Lustre)
- Lustre filesystem backed by S3
S3 is convenient and the relatively inexpensive option.
Lustre is higher performance, but more expensive, and slightly more complex to setup and manage.
AWS supports a feature to back the Lustre filesystem with S3, which give an easy to manage and high
AWS supports a feature to back the Lustre filesystem with S3, which gives an easy to manage, high
performance option.
Once the storage is in place, the next step is to copy your matrix files to that location.
Currently cellxgene supports a flat file organization. Each matrix file is located from
the same s3 prefix or filesystem directory. This location is specified in the configuration as the dataroot.
Once the storage is in place, the next step is to copy your data files to that location.
Currently cellxgene supports a flat file organization. Each matrix file is located under
the same s3 prefix or filesystem directory. This location is specified in the configuration
as the dataroot.
### 2. Create an elastic beanstalk application. For example:
### 2. Create an elastic beanstalk application. For example:
```
EB_APP=cellxgene-app
@@ -54,9 +55,9 @@ eb init -p python-3.6 $EB_APP
### 3. Configuring cellxgene
All the cellxgene configuration options can be set from a configuration file.
This file can be generated like this:
A yaml config file containing all of the default configuration options can be generated like this:
```cellxgene launch --dump-default-config > myconfig.yaml```
`cellxgene launch --dump-default-config > myconfig.yaml`
The config file may then be customized before the app is deployed.
@@ -66,18 +67,14 @@ First, if your config file is named "config.yaml" and exists in `customize/confi
then it will be bundled with the application zip file and installed along
side the app on the EB servers.
Second, a potentially more flexible approach is to place your config file in a location accessible to the EB
servers, such as in S3. For example: s3://my-bucket/my-datasets/config.yaml.
Second, a potentially more flexible approach is to place your config file in a location accessible
to the EB servers, such as in S3. For example: s3://my-bucket/my-datasets/config.yaml.
Set the CXG_CONFIG_FILE environment variable to specify this location.
Another option is to set the CXG_DATAROOT environment variable. The dataroot
Another option is to set the CXG_DATAROOT environment variable. The dataroot
is the location where the matrix files are located.
This environment variable will override the dataroot in the config file (if specified).
- Note: Certain features, such as user annotations, are automatically disabled by the EB app,
and cannot be enabled using configuration. They may be enabled manually by modifying app.py, however
this is not supported or recommended at this time.
### 4. Customization
The deployment can be customized in several ways, by adding files to a directory called
@@ -93,15 +90,15 @@ The cellxgene server can serve additional static webpages that will be associate
These include the about_legal_tos (terms of service), and about_legal_privacy, for example.
To use this feature, do the following:
* In this directory, create a sub directory called "customize/deploy/".
* Copy the files you want to serve into this directory
* modify your configuration file to set the location to these file: /static/cellxgene/deploy/<filename>
- In this directory, create a sub directory called "customize/deploy/".
- Copy the files you want to serve into this directory
- modify your configuration file to set the location to these file: /static/cellxgene/deploy/<filename>
Example: you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene.
Example: you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene.
Assume files called "tos.html" and "privacy.html" exist.
```
$ mkdir static
$ mkdir -p customize/deploy
$ cp <source_dir>/tos.html customize/deploy/tos.html
$ cp <source_dir>/privacy.html customize/deploy/privacy.html
@@ -116,14 +113,14 @@ about_legal_privacy: /static/cellxgene/deploy/privacy.html
Additional scripts can be added using the server/inline_scripts config parameters.
To include these scripts in the deployment, use the following steps:
* In this directory, create a sub directory called "customize/inline_scripts".
* Copy the script files into this directory
* modify your configuration file to set the location to these file (leaving off customize/inline_scripts)
- In this directory, create a sub directory called "customize/inline_scripts".
- Copy the script files into this directory
- Modify your configuration file to set the location to these file (leaving off customize/inline_scripts)
For example, to add an inline script called "myscript.js":
```
$ mkdir scripts
$ mkdir -p customize/inline_scripts
$ cp <source_dir>/myscript.js customize/inline_scripts/myscript.js
# edit the config.yaml
$ grep inline_scripts config.yaml
@@ -135,14 +132,14 @@ $ grep inline_scripts config.yaml
Optionally, you can add plugins to the server python code. To include a plugin in the deployment use the following steps:
```
$ mkdir plugins
$ mkdir -p customize/plugins
$ cp <source_dir>/<my_plugin>.py customize/plugins/<my_plugin>.py
```
#### ebextensions
Any additional config files intended for the `.ebextensions` directory of the artifact can be added
to the `customize/ebextensions` directory. Any file found here will be copied over.
to the `customize/ebextensions` directory. Any file found here will be copied over.
#### requirements.txt
@@ -152,6 +149,7 @@ This is useful to ensure that the dependencies do not change from one deployment
Therefore the custom/requirements.txt must all have exact versions specified (e.g. anndata==0.7.1).
This file can be generated the first time using a process like this:
```
# assume you are running in this directory
$ virtualenv temp
@@ -168,6 +166,20 @@ If a future cellxgene version updates its requirements by modifying a module ver
or adding a new dependency, then the `make build` process will detect any
incompatibilities and raise an error.
#### File structure for customizations
The following diagram shows the file structure for the customization directory.
```
customization
+-- config.yaml
+-- deploy/
+-- inline_scripts/
+-- plugins/
+-- ebextensions/
+-- requirements.txt
```
### 5. Create the artifact.zip file for the application
```
@@ -176,19 +188,13 @@ $ make build
### 6. Flask secret key
The application requires as secret key to be provided to flask, the web framework used by cellxgene.
The application requires a secret key to be provided to flask, the web framework used by cellxgene.
There are three ways to provide the secret key:
- In the configuration file: update the server/flask_secret_key attribute.
- In the configuration file, update the server/flask_secret_key attribute.
- In the configuration file, update the external/aws_secrets_manager section to set the
secret name and key that defines the flask secret key.
- An environment variable: `CXG_SECRET_KEY`
- Managed by the AWS Secret Manager
If using the AWS Secret Manager, then the secret name is passed as an environment variable: CXG_AWS_SECRET_NAME.
The secret must contain a key with the name "flask_secret_key".
The region name for the AWS Secret Manager must be specified (e.g. us-east-1).
The most straightforward way is to specified it with the CXG_AWS_SECRET_REGION_NAME environment variable.
If this environment variable is not defined, then the app attempts to determine the region from the
dataroot (if in s3), or the config file location (if in s3).
### 7. Create an environment
@@ -203,7 +209,8 @@ $ EB_INSTANCE=m5.large
$ CXG_DATAROOT=<location to your S3 bucket>
$ CXG_CONFIG_FILE=<location to your config file>
# Potentially also set envvars for the secret key.
# Potentially also set an environment variable for the flask secret key,
# and other environemet variable described in the configuration file.
$ eb create $EB_ENV --instance-type $EB_INSTANCE \
--envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE
@@ -227,3 +234,67 @@ $ eb deploy $EB_ENV
```
$ eb open $EB_ENV
```
## Advanced Features
### Authentication
Authentication can be configured in the configuration file. Authentication is required
for User Annotations (see below). User Annotations is a feature where annotations can be
created by the user
, and
then associated with the user's id.
When the user revisits the site, their annotations will be available.
There are three main authentication modes: null, session, or oauth.
In the configuration file specify the authentication mode by setting
`server / authentication / type`.
#### null
Authentication is disabled: user annotations cannot be enabled.
#### session
The user is associated with their client browser session. This approach is
simple to setup, but not recommended for hosted cellxgene, since the user will not have access to
their annotations when running from a different browser, or if their cookies get cleared.
#### oauth
A user logs into cellxgene using an identity provider (like Google), or logs in using
an email/password. This is the best option, but requires making use of an oauth service and
additional configuration of the cellxgene server.
To see what this looks like, please look at https://cellxgene.cziscience.com/,
and view one of the cellxgene datasets.
For this server, Auth0 (auth0.com) is used for authentication, but there are other options.
There are good sources of documentation online that describe how to use one of these
services.
The `params_oauth` section in the configuration file describes characteristics of the
authentication service, like "client_id" and "client_secret".
For security, the client_secret needs to be protected. One option is to
store it in the AWS Secrets Manager.
### User Annotations
User annotations can be configured in the configuration file both generally and for a specific data route. The annotations feature is only available when Authorization is enabled.
To enable Annotations, it is necessary to create a relational database and add the database uri (typically `postgresql://[user[:password]@][netloc][:port][/dbname]`) to the secrets manager under `DB_URI`.
The hosted version of cellxgene runs on AWS's [Aurora PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.AuroraPostgreSQL.html) but any sqlalchemy compatible relational database should work.
Once the database is set up apply the cellxgene schema to your database by running the following inside the cellxgene repo
`PROJECT_ROOT=$(git rev-parse --show-toplevel)`
`python3`
Inside the python console
`from sqlalchemy import create_engine`
`from server.db.cellxgene_orm import Base`
`uri = "[DB_URI]”`
`engine = create_engine(uri)`
Base.metadata.create_all(engine)`
To check the schema was properly applied (or just to check what is in the database at any point)
ssh into your database. For a postgres database this entails running:
`psql [DB_URI]`
You'll also need to update your IAM policies to allow the instance to write to the s3 bucket.
server/eb/app.py
+3 -17
View File
@@ -9,8 +9,6 @@ from flask import json
import logging
from flask_talisman import Talisman
from flask_cors import CORS
from server.common.config import handle_config_from_secret
from server.common.errors import SecretKeyRetrievalError
if os.path.isdir("/opt/python/log"):
@@ -165,26 +163,14 @@ try:
logging.info("Configuration from CXG_DATAROOT")
app_config.update_server_config(multi_dataset__dataroot=dataroot)
# update from secret manager
try:
handle_config_from_secret(app_config)
except SecretKeyRetrievalError:
sys.exit(1)
# features are unsupported in the current hosted server
# overwrite configuration for the eb app
app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
# complete config
app_config.complete_config(logging.info)
if not app_config.server_config.app__flask_secret_key:
logging.critical(
"flask_secret_key is not provided. Either set in config file, CXG_SECRET_KEY environment variable, "
"or in AWS Secret Manager"
)
sys.exit(1)
server = WSGIServer(app_config)
debug = False
application = server.app
server/eb/check_config.py
+39
View File
@@ -0,0 +1,39 @@
import sys
import argparse
import yaml
from server.common.config.app_config import AppConfig
def main():
parser = argparse.ArgumentParser("A script to check hosted configuration files")
parser.add_argument("config_file", help="the configuration file")
parser.add_argument(
"-s",
"--show",
default=False,
action="store_true",
help="print the configuration. NOTE: this may print secret values to stdout",
)
args = parser.parse_args()
app_config = AppConfig()
try:
app_config.update_from_config_file(args.config_file)
app_config.complete_config()
except Exception as e:
print(f"Error: {str(e)}")
print("FAIL:", args.config_file)
sys.exit(1)
if args.show:
yaml_config = app_config.config_to_dict()
yaml.dump(yaml_config, sys.stdout)
print("PASS:", args.config_file)
sys.exit(0)
if __name__ == "__main__":
main()
server/test/__init__.py
+10 -6
View File
@@ -34,7 +34,7 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
multi_dataset__dataroot=data_locator.path, authentication__type="test",
app__flask_secret_key="secret", multi_dataset__dataroot=data_locator.path, authentication__type="test",
)
config.update_default_dataset_config(
embeddings__names=["umap"],
@@ -64,7 +64,10 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
single_dataset__obs_names=None, single_dataset__var_names=None, single_dataset__datapath=data_locator.path
app__flask_secret_key="secret",
single_dataset__obs_names=None,
single_dataset__var_names=None,
single_dataset__datapath=data_locator.path,
)
config.update_default_dataset_config(
embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
@@ -97,6 +100,7 @@ def skip_if(condition, reason: str):
def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}):
config = AppConfig()
config.update_server_config(
app__flask_secret_key="secret",
single_dataset__obs_names=None,
single_dataset__var_names=None,
adaptor__anndata_adaptor__backed=backed,
@@ -117,7 +121,7 @@ def random_string(n):
return "".join(random.choice(string.ascii_letters) for _ in range(n))
def start_test_server(command_line_args=[], app_config=None):
def start_test_server(command_line_args=[], app_config=None, env=None):
"""
Command line arguments can be passed in, as well as an app_config.
This function is meant to be used like this, for example:
@@ -155,7 +159,7 @@ def start_test_server(command_line_args=[], app_config=None):
command.extend(["-c", config_file])
server = f"http://localhost:{port}"
ps = Popen(command)
ps = Popen(command, env=env)
for _ in range(10):
try:
@@ -178,10 +182,10 @@ def stop_test_server(ps):
@contextmanager
def test_server(command_line_args=[], app_config=None):
def test_server(command_line_args=[], app_config=None, env=None):
"""A context to run the cellxgene server."""
ps, server = start_test_server(command_line_args, app_config)
ps, server = start_test_server(command_line_args, app_config, env)
try:
yield server
finally:
server/test/performance/performance_test_annotations_backend.py
+215
View File
@@ -0,0 +1,215 @@
import json
import string
from contextlib import contextmanager
from timeit import default_timer
import concurrent.futures
import numpy as np
import requests
import sys
from server.data_common.fbs.matrix import encode_matrix_fbs
import pandas as pd
import random
"""
Before running, sign into the dataportal, copy the cookie and paste it below. To test in staging or prod update the
url base below. It is also possible to configure the number of categories created and the number of unique labels per
category.
"""
cookie = ""
test_datasets = {
"smallest": {
"dataset_url": "kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg",
"name": "smallest",
"num_cells": 5270,
},
"10k": {
"dataset_url": "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
"name": "10k",
"num_cells": 9409,
},
"80k": {
"dataset_url": "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg", # noqa E501
"name": "80k",
"num_cells": 81736,
},
"140k": {"dataset_url": "Single_cell_drug_screening_a549-42-remixed.cxg", "name": "140k", "num_cells": 143015},
"largest": {"dataset_url": "human_cell_landscape.cxg", "name": "largest", "num_cells": 599926},
"1million": {"dataset_url": None, "name": "1million", "num_cells": 1000000},
"4million": {"dataset_url": None, "name": "4million", "num_cells": 4000000},
}
url_base = "https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/"
annotations_category_count = [1, 10, 50]
max_labels = [5, 50, 100]
class PerformanceTestingAnnotations:
def __init__(
self,
datasets=test_datasets,
annotations_category_count=annotations_category_count,
max_labels=max_labels,
url_base=url_base,
):
self.test_datasets = datasets
self.annotations_category_count = annotations_category_count
self.max_labels = max_labels
self.url_base = url_base
self.test_notes = self.create_info_dict()
def set_cell_count(self, dataset_name):
dataset_url = self.test_datasets[dataset_name]["dataset_url"]
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
response = self.client.get(f"{self.url_base}{dataset_url}/api/v0.2/schema", headers=headers)
cell_count = json.loads(response._content)["schema"]["dataframe"]["nObs"]
self.test_datasets[dataset_name]["cell_count"] = cell_count
def create_info_dict(self):
request_info = {}
for dataset in self.test_datasets.keys():
request_info[dataset] = {}
for cat_count in self.annotations_category_count:
request_info[dataset][f"num_categories_{cat_count}"] = {}
for unique_labels in self.max_labels:
request_info[dataset][f"num_categories_{cat_count}"][f"max_label_{unique_labels}"] = {}
return request_info
def create_annotations_dict_multi_process(self, dataset_name, category_count, label_max):
annotation_dict = {}
futures = []
categories = [f"Category{i}" for i in range(category_count)]
if not self.test_datasets[dataset_name]["num_cells"]:
self.set_cell_count(dataset_name)
with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
for category in categories:
futures.append(
executor.submit(
self.build_array_for_category,
category,
self.test_datasets[dataset_name]["num_cells"],
label_max,
)
)
for future in concurrent.futures.as_completed(futures):
try:
result = future.result()
category_name, cells = result
annotation_dict[category_name] = pd.Series(cells, dtype="category")
except Exception as e:
print(f"Issue creating the annotations dict: {e}")
return annotation_dict
def build_array_for_category(self, category_name, cell_count, label_max):
unique_label_count = label_max
labels = self.generate_labels(unique_label_count)
cells_per_label = int(cell_count / len(labels))
extra = cell_count % len(labels)
cells = []
for label in labels:
cells.extend([label] * cells_per_label)
cells.extend(["extra"] * extra)
rng = np.random.default_rng()
rng.shuffle(cells)
return category_name, cells
@staticmethod
def convert_to_fbs(annotation_dict):
df = pd.DataFrame(annotation_dict)
return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
@staticmethod
def generate_labels(unique_label_count):
labels = ["undefined"]
for i in range(unique_label_count):
length = random.randrange(10, 20)
labels.append(f"{i}__" + "".join(random.choice(string.ascii_letters) for z in range(length)))
return labels
@contextmanager
def elapsed_timer(self):
start = default_timer()
elapser = lambda: default_timer() - start # noqa E731
yield lambda: elapser()
end = default_timer()
elapser = lambda: end - start # noqa E731
def create_matrix(self, dataset_name, num_cat, max_labels):
with self.elapsed_timer() as elapsed:
annon_dict = self.create_annotations_dict_multi_process(dataset_name, num_cat, max_labels)
dict_size = sum(sys.getsizeof(value) for value in annon_dict.values()) / 1024 ** 2
self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["annotation_dict"] = {
"creation_time": str(elapsed()),
"size": f"{dict_size} mb",
}
df = pd.DataFrame(annon_dict)
df_size = sys.getsizeof(df) / 1024 ** 2
self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["data_frame"] = {
"creation_time": str(elapsed()),
"size": f"{df_size} mb",
}
try:
matrix = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
matrix_size = sys.getsizeof(matrix) / 1024 ** 2
self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["fbs_matrix"] = {
"creation_time": str(elapsed()),
"size": f"{matrix_size} mb",
}
return matrix
except Exception as e:
print(f"Issue creating fbs matrix: {e}, for {dataset_name}")
return []
def send_put_request(self, dataset_url, data):
url = self.url_base + f"{dataset_url}/api/v0.2/annotations/obs"
with self.elapsed_timer() as elapsed:
try:
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
response = requests.put(url=url, data=data, headers=headers)
except Exception as e:
print(f"Issue with put request: {e}")
return None, elapsed()
return response, elapsed()
def test_categories_max_label_matrix(self, dataset_name):
for unique_labels in self.max_labels:
for category_count in self.annotations_category_count:
print(f"Starting dataset: {dataset_name}, categories: {category_count}, labels: {unique_labels}")
fbs_matrix = self.create_matrix(dataset_name, category_count, unique_labels)
if self.test_datasets[dataset_name]["dataset_url"] and fbs_matrix:
response, response_time = self.send_put_request(
self.test_datasets[dataset_name]["dataset_url"], fbs_matrix
)
if response is None:
self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
"put_request"
] = {"response_status": "failed", "request_time": str(response_time)}
else:
self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
"put_request"
] = {"response_status": response.status_code, "request_time": str(response_time)}
def test_all_datasets():
"""
Run time is dependent on number of datasets, dataset size, number of categories/number being tested and number of
unique label counts being tested. However it generally takes a long time. I recommend running this in tmux
"""
perf_test = PerformanceTestingAnnotations()
for dataset_name in perf_test.test_datasets.keys():
print(f"Testing annotation creation for: {dataset_name}")
try:
perf_test.test_categories_max_label_matrix(dataset_name)
except Exception as e:
print(f"something went wrong with {dataset_name}: {e}")
return perf_test.test_notes
def main():
notes = test_all_datasets()
print(notes)
if __name__ == "__main__":
main()
server/test/performance/scale_test_annotations.py
+46
View File
@@ -0,0 +1,46 @@
import time
import random
from locust import HttpUser, between, task
random.seed(time.time())
"""
To run this script sign into cellxgene in the desired environment and grab the returned cookie, update the cookie
variable below with your cookie and run the following command to see results in the terminal:
locust -f server/test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
Or if you want to use the locust gui run:
locust -f server/test/performance/scale_test_annotations.py -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/
If you want to test staging you'll need to substitute staging for dev in the host url
To test prod you'll need to replace dev.single-cell.czi.technology with cziscience.com
If you'd like to test additional datasets you'll need to add them to the dataset_urls array
Todo @mdunitz update script to retrieve different annotation categories -- may need to create them to ensure the
categories are shared across datasets for a given user.
"""
cookie = ""
class WebsiteUser(HttpUser):
wait_time = between(1, 2)
dataset_urls = [
"human_cell_landscape.cxg",
"Single_cell_drug_screening_a549-42-remixed.cxg",
"kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg",
"krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
"Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg",
]
@task
def get_annotations(self):
dataset_url = random.choice(self.dataset_urls)
url = f"{dataset_url}/api/v0.2/annotations/obs?annotation-name=cell_type"
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
self.client.get(url, headers=headers)
@task
def get_schema(self):
dataset_url = random.choice(self.dataset_urls)
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
self.client.get(f"{dataset_url}/api/v0.2/schema", headers=headers)
server/test/unit/auth/test_auth.py
+38 -27
View File
@@ -11,13 +11,14 @@ class AuthTest(unittest.TestCase):
self.dataset_dataroot = FIXTURES_ROOT
def test_auth_none(self):
c = AppConfig()
c.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
c.update_default_dataset_config(user_annotations__enable=False)
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
app_config.update_default_dataset_config(user_annotations__enable=False)
c.complete_config()
app_config.complete_config()
with test_server(app_config=c) as server:
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
@@ -25,12 +26,13 @@ class AuthTest(unittest.TestCase):
self.assertIsNone(userinfo)
def test_auth_session(self):
c = AppConfig()
c.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
c.update_default_dataset_config(user_annotations__enable=True)
c.complete_config()
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
app_config.update_default_dataset_config(user_annotations__enable=True)
app_config.complete_config()
with test_server(app_config=c) as server:
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
@@ -40,9 +42,10 @@ class AuthTest(unittest.TestCase):
self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
def test_auth_test(self):
c = AppConfig()
c.update_server_config(authentication__type="test")
c.update_server_config(
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__type="test")
app_config.update_server_config(
multi_dataset__dataroot=dict(
a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
@@ -50,12 +53,12 @@ class AuthTest(unittest.TestCase):
)
# specialize the configs
c.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
c.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
app_config.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
app_config.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
c.complete_config()
app_config.complete_config()
with test_server(app_config=c) as server:
with test_server(app_config=app_config) as server:
session = requests.Session()
# auth datasets
@@ -82,6 +85,7 @@ class AuthTest(unittest.TestCase):
userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertEqual(userinfo["userinfo"]["picture"], None)
self.assertTrue(config["config"]["parameters"]["annotations"])
r = session.get(f"{server}/{logout_uri}")
@@ -100,15 +104,22 @@ class AuthTest(unittest.TestCase):
self.assertIsNone(userinfo)
self.assertFalse(config["config"]["parameters"]["annotations"])
# login with a picture
session.get(f"{server}/{login_uri}&picture=myimage.png")
userinfo = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/userinfo").json()
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
def test_auth_test_single(self):
c = AppConfig()
c.update_server_config(
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
)
c.complete_config()
app_config.complete_config()
with test_server(app_config=c) as server:
with test_server(app_config=app_config) as server:
session = requests.Session()
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
@@ -123,10 +134,10 @@ class AuthTest(unittest.TestCase):
self.assertEqual(login_uri, "/login")
self.assertEqual(logout_uri, "/logout")
r = session.get(f"{server}/{login_uri}")
response = session.get(f"{server}/{login_uri}")
# check that the login redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/")
self.assertEqual(response.history[0].status_code, 302)
self.assertEqual(response.url, f"{server}/")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
@@ -134,10 +145,10 @@ class AuthTest(unittest.TestCase):
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertTrue(config["config"]["parameters"]["annotations"])
r = session.get(f"{server}/{logout_uri}")
response = session.get(f"{server}/{logout_uri}")
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/")
self.assertEqual(response.history[0].status_code, 302)
self.assertEqual(response.url, f"{server}/")
config = session.get(f"{server}/api/v0.2/config").json()
userinfo = session.get(f"{server}/api/v0.2/userinfo").json()
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
server/test/unit/auth/test_oauth.py
+2
View File
@@ -160,12 +160,14 @@ class AuthTest(unittest.TestCase):
def test_auth_oauth_session(self):
# test with session cookies
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
self.auth_flow(app_config)
def test_auth_oauth_cookie(self):
# test with specified cookie
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__params_oauth__session_cookie=False,
authentication__params_oauth__cookie=dict(key="test_cxguser", httponly=True, max_age=60),
server/test/unit/common/config/__init__.py
+46 -9
View File
@@ -3,6 +3,7 @@ import shutil
import unittest
import random
from unittest import mock
import yaml
from server.test import FIXTURES_ROOT
@@ -30,7 +31,7 @@ class ConfigTests(unittest.TestCase):
port="null",
open_browser="false",
force_https="false",
flask_secret_key="null",
flask_secret_key="secret",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
@@ -81,7 +82,7 @@ class ConfigTests(unittest.TestCase):
port="null",
open_browser="false",
force_https="false",
flask_secret_key="null",
flask_secret_key="secret",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
@@ -133,6 +134,9 @@ class ConfigTests(unittest.TestCase):
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
environment=None,
aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[],
config_file_name="app_config.yml",
):
random_num = random.randrange(999999)
@@ -201,13 +205,17 @@ class ConfigTests(unittest.TestCase):
top_n=top_n,
config_file_name=f"temp_dataset_config_{random_num}.yml",
)
with open(server_config) as server_config:
with open(dataset_config) as dataset_config:
with open(configfile, "w") as app_config_file:
for line in server_config:
app_config_file.write(line)
for line in dataset_config:
app_config_file.write(line)
external_config = self.custom_external_config(
environment=environment,
aws_secrets_manager_region=aws_secrets_manager_region,
aws_secrets_manager_secrets=aws_secrets_manager_secrets,
config_file_name=f"temp_external_config_{random_num}.yml",
)
with open(configfile, "w") as app_config_file:
app_config_file.write(open(server_config).read())
app_config_file.write(open(dataset_config).read())
app_config_file.write(open(external_config).read())
return configfile
@@ -244,3 +252,32 @@ class ConfigTests(unittest.TestCase):
dataset_config_file.write(dataset_config)
return configfile
def custom_external_config(
self,
environment=None,
aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[],
config_file_name="external_config.yaml",
):
# set to the default if environment is None
if environment is None:
environment = [
dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
dict(
name="CXG_OAUTH_CLIENT_SECRET",
path=["server", "authentication", "params_oauth", "client_secret"],
required=False,
),
]
external_config = {
"external": {
"environment": environment,
"aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
}
}
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
with open(configfile, "w") as external_config_file:
yaml.dump(external_config, external_config_file)
return configfile
server/test/unit/common/config/test_app_config.py
+81 -1
View File
@@ -15,6 +15,7 @@ class AppConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
@@ -106,6 +107,8 @@ class AppConfigTest(ConfigTests):
with open(configfile, "w") as fconfig:
config = """
server:
app:
flask_secret_key: secret
multi_dataset:
dataroot: test_dataroot
@@ -116,7 +119,10 @@ class AppConfigTest(ConfigTests):
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
self.assertEqual(
server_changes,
[("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)],
)
self.assertEqual(dataset_changes, [])
def test_configfile_no_server_section(self):
@@ -138,3 +144,77 @@ class AppConfigTest(ConfigTests):
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
def test_simple_update_single_config_from_path_and_value(self):
"""Update a simple config parameter"""
config = AppConfig()
config.server_config.multi_dataset__dataroot = dict(
s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"),
s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"),
)
config.add_dataroot_config("s1")
config.add_dataroot_config("s2")
# test simple value in server
config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
# test simple value in default dataset
config.update_single_config_from_path_and_value(
["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"], "mydburi",
)
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# test simple value in specific dataset
config.update_single_config_from_path_and_value(
["per_dataset_config", "s1", "user_annotations", "hosted_tiledb_array", "db_uri"], "s1dburi"
)
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "s1dburi")
self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# error checking
bad_paths = [
(
["dataset", "does", "not", "exist"],
"unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
),
(["does", "not", "exist"], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
([], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
(["per_dataset_config"], "missing dataroot when using per_dataset_config: got '['per_dataset_config']'"),
(
["per_dataset_config", "unknown"],
"unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']',"
" dataroots specified in config are ['s1', 's2']",
),
([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
("string", "path must be a list of strings, got 'string'"),
]
for bad_path, error_message in bad_paths:
with self.assertRaises(ConfigurationError) as config_error:
config.update_single_config_from_path_and_value(bad_path, "value")
self.assertEqual(config_error.exception.message, error_message)
def test_dict_update_single_config_from_path_and_value(self):
"""Update a config parameter that has a value of dict"""
# the path leads to a dict config param, set the config parameter to the new value
config = AppConfig()
config.update_single_config_from_path_and_value(
["server", "authentication", "params_oauth", "cookie"], dict(key="mykey1", max_age=100)
)
self.assertEqual(config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100))
# the path leads to an entry within a dict config param, the value is simple
config = AppConfig()
config.server_config.authentication__params_oauth__cookie = dict(key="mykey1", max_age=100)
config.update_single_config_from_path_and_value(
["server", "authentication", "params_oauth", "cookie", "httponly"], True,
)
self.assertEqual(
config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100, httponly=True)
)
server/test/unit/common/config/test_base_config.py
+2
View File
@@ -10,6 +10,7 @@ class BaseConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
@@ -47,6 +48,7 @@ class BaseConfigTest(ConfigTests):
server_changes,
[
("app__verbose", True, False),
("app__flask_secret_key", "secret", None),
("multi_dataset__dataroot", FIXTURES_ROOT, None),
("multi_dataset__matrix_cache__timelimit_s", 5, 30),
("data_locator__s3__region_name", "us-east-1", True),
server/test/unit/common/config/test_dataset_config.py
+7 -3
View File
@@ -19,6 +19,7 @@ class TestDatasetConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.dataset_config = self.config.default_dataset_config
self.config.complete_config()
@@ -155,7 +156,8 @@ class TestDatasetConfig(ConfigTests):
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
app__flask_secret_key="secret",
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
)
with self.assertRaises(ConfigurationError):
config.complete_config()
@@ -163,17 +165,19 @@ class TestDatasetConfig(ConfigTests):
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
app__flask_secret_key="secret",
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
)
config.complete_config()
# test that multi dataroots work end to end
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
)
),
)
# Change this default to test if the dataroot overrides below work.
server/test/unit/common/config/test_external_config.py
+231
View File
@@ -0,0 +1,231 @@
import os
from unittest.mock import patch
import requests
from server.common.errors import ConfigurationError
from server.common.config.app_config import AppConfig
from server.test import test_server, FIXTURES_ROOT
from server.common.utils.type_conversion_utils import convert_string_to_value
from server.test.unit.common.config import ConfigTests
class TestExternalConfig(ConfigTests):
def test_type_convert(self):
# The values from environment variables and aws secrets are returned as strings.
# These values need to be converted to the proper types.
self.assertEqual(convert_string_to_value("1"), int(1))
self.assertEqual(convert_string_to_value("1.1"), float(1.1))
self.assertEqual(convert_string_to_value("string"), "string")
self.assertEqual(convert_string_to_value("true"), True)
self.assertEqual(convert_string_to_value("True"), True)
self.assertEqual(convert_string_to_value("false"), False)
self.assertEqual(convert_string_to_value("False"), False)
self.assertEqual(convert_string_to_value("null"), None)
self.assertEqual(convert_string_to_value("None"), None)
self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
def test_environment_variable(self):
configfile = self.custom_external_config(
environment=[
dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
],
config_file_name="environment_external_config.yaml",
)
env = os.environ
env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k.cxg"
env["DIFFEXP"] = "False"
with test_server(command_line_args=["-c", configfile], env=env) as server:
session = requests.Session()
response = session.get(f"{server}/api/v0.2/config")
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
env["DIFFEXP"] = "True"
with test_server(command_line_args=["-c", configfile], env=env) as server:
session = requests.Session()
response = session.get(f"{server}/api/v0.2/config")
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
def test_environment_variable_errors(self):
# no name
app_config = AppConfig()
app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
# required has wrong type
app_config = AppConfig()
app_config.external_config.environment = [
dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
# no path
app_config = AppConfig()
app_config.external_config.environment = [dict(name="myenvar", required=True)]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
# required environment variable is not set
app_config = AppConfig()
app_config.external_config.environment = [
dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
@patch("server.common.config.external_config.get_secret_key")
def test_aws_secrets_manager(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
configfile = self.custom_external_config(
aws_secrets_manager_region="us-west-2",
aws_secrets_manager_secrets=[
dict(
name="my_secret",
values=[
dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=False),
dict(
key="db_uri",
path=["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"],
required=True,
),
dict(
key="oauth_client_secret",
path=["server", "authentication", "params_oauth", "client_secret"],
required=True,
),
],
)
],
config_file_name="secret_external_config.yaml",
)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.server_config.app__flask_secret_key = "original"
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.complete_config()
self.assertEqual(app_config.server_config.app__flask_secret_key, "original")
self.assertEqual(app_config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
self.assertEqual(app_config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
@patch("server.common.config.external_config.get_secret_key")
def test_aws_secrets_manager_error(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
# no region
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = None
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(
config_error.exception.message,
"Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
)
# missing secret name
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
# secret name wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
# missing values name
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
# values wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
# entry missing key
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
# entry required is wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
# entry missing path
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(key="db_uri", required=True)])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
# secret missing required key
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(
name="mysecret",
values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
)
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
server/test/unit/common/config/test_server_config.py
+8 -31
View File
@@ -23,6 +23,7 @@ class TestServerConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
@@ -103,15 +104,17 @@ class TestServerConfig(ConfigTests):
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
os.environ["CXG_SERVER_PORT"] = "4008"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
del os.environ["CXG_SERVER_PORT"]
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
config.server_config.handle_app(self.context)
config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
def test_handle_app__sets_web_base_url(self):
@@ -124,7 +127,7 @@ class TestServerConfig(ConfigTests):
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
config.server_config.handle_authentication()
config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
@@ -151,6 +154,7 @@ class TestServerConfig(ConfigTests):
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
app__flask_secret_key="secret",
app__api_base_url=f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
)
@@ -215,7 +219,7 @@ class TestServerConfig(ConfigTests):
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
self.config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
with self.assertRaises(ConfigurationError):
self.config.complete_config()
@@ -224,7 +228,7 @@ class TestServerConfig(ConfigTests):
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
self.config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
self.config.complete_config()
@@ -306,30 +310,3 @@ class TestServerConfig(ConfigTests):
mock_tiledb_context.assert_called_once_with(
{"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
)
@mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
@patch("server.common.config.get_secret_key")
def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"flask_secret_key": "mock_flask_secret",
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
config = AppConfig()
with self.assertLogs(level="INFO") as logger:
from server.common.config import handle_config_from_secret
# should not throw error
# "AttributeError: 'XConfig' object has no attribute 'x'"
handle_config_from_secret(config)
# should log 3 lines (one for each var set from a secret)
self.assertEqual(len(logger.output), 3)
self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0])
self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1])
self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2])
self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
server/test/unit/eb/test_eb.py
+35 -4
View File
@@ -6,6 +6,7 @@ from server.test import PROJECT_ROOT, FIXTURES_ROOT
from server.common.config.app_config import AppConfig
from contextlib import contextmanager
import time
import os
@contextmanager
@@ -34,12 +35,12 @@ class Elastic_Beanstalk_Test(unittest.TestCase):
tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/server")
tempdirname = tempdir.name
c = AppConfig()
config = AppConfig()
# test that eb works
c.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
c.complete_config()
c.write_config(f"{tempdirname}/config.yaml")
config.complete_config()
config.write_config(f"{tempdirname}/config.yaml")
subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/server/eb", shell=True)
subprocess.check_call(["make", "build"], cwd=tempdirname)
@@ -50,3 +51,33 @@ class Elastic_Beanstalk_Test(unittest.TestCase):
r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
data_config = r.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
def test_config(self):
check_config_script = os.path.join(PROJECT_ROOT, "server", "eb", "check_config.py")
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
app_config = AppConfig()
app_config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}")
app_config.write_config(configfile)
command = ["python", check_config_script, configfile]
# test failure mode (flask_secret_key not set)
env = os.environ.copy()
env.pop("CXG_SECRET_KEY", None)
with self.assertRaises(subprocess.CalledProcessError) as exception_context:
subprocess.check_output(command, env=env)
output = str(exception_context.exception.stdout, "utf-8")
self.assertTrue(
output.startswith(
"Error: Invalid type for attribute: app__flask_secret_key, expected type str, got NoneType"
)
)
self.assertEqual(exception_context.exception.returncode, 1)
# test passing case
env = os.environ.copy()
env["CXG_SECRET_KEY"] = "secret"
output = subprocess.check_output(command, env=env)
output = str(output, "utf-8")
self.assertTrue(output.startswith("PASS"))