fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-04 20:08:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

Browse Source 
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
2021-03-26 00:27:07 -05:00
committed by GitHub
parent e6e358ddc8
commit 78c9d24ed4
425 changed files with 734 additions and 5317 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/test/test_czi_hosted/unit/common/__init__.py
View File
backend/test/test_czi_hosted/unit/common/config/__init__.py
+284
View File
@@ -0,0 +1,284 @@
import os
import shutil
import unittest
import random
from unittest import mock
import yaml
from backend.test import FIXTURES_ROOT
def mockenv(**envvars):
return mock.patch.dict(os.environ, envvars)
class ConfigTests(unittest.TestCase):
tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
@classmethod
def tearDownClass(cls) -> None:
shutil.rmtree(cls.tmp_fixtures_directory)
@classmethod
def setUpClass(cls) -> None:
os.makedirs(cls.tmp_fixtures_directory)
def custom_server_config(
self,
verbose="false",
debug="false",
host="localhost",
port="null",
open_browser="false",
force_https="false",
flask_secret_key="secret",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
api_base_url="null",
web_base_url="null",
auth_type="session",
insecure_test_environment="false",
oauth_api_base_url="null",
client_id="null",
client_secret="null",
jwt_decode_options="null",
session_cookie="true",
cookie="null",
dataroot="null",
index="false",
allowed_matrix_types=[],
max_cached_datasets=5,
timelimit_s=5,
dataset_datapath="null",
obs_names="null",
var_names="null",
about="null",
title="null",
diffexp_max_workers=64,
cpu_multiplier=4,
target_workunit="16_000_000",
data_locater_region_name="us-east-1",
cxg_tile_cache_size=8589934592,
cxg_num_reader_threads=32,
anndata_backed="false",
column_request_max=32,
diffexp_cellcount_max="null",
config_file_name="server_config.yaml",
):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
server_config_outline_path = os.path.join(FIXTURES_ROOT, "czi_hosted_server_config_outline.py")
with open(server_config_outline_path, "r") as config_skeleton:
config = config_skeleton.read()
server_config = eval(config)
with open(configfile, "w") as server_config_file:
server_config_file.write(server_config)
return configfile
def custom_app_config(
self,
verbose="false",
debug="false",
host="localhost",
port="null",
open_browser="false",
force_https="false",
flask_secret_key="secret",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
api_base_url="null",
web_base_url="null",
auth_type="session",
oauth_api_base_url="null",
client_id="null",
client_secret="null",
jwt_decode_options="null",
session_cookie="true",
cookie="null",
dataroot="null",
index="false",
allowed_matrix_types=[],
max_cached_datasets=5,
timelimit_s=5,
dataset_datapath="null",
obs_names="null",
var_names="null",
about="null",
title="null",
diffexp_max_workers=64,
cpu_multiplier=4,
target_workunit="16_000_000",
data_locater_region_name="us-east-1",
cxg_tile_cache_size=8589934592,
cxg_num_reader_threads=32,
anndata_backed="false",
column_request_max=32,
diffexp_cellcount_max="null",
scripts=[],
inline_scripts=[],
about_legal_tos="null",
about_legal_privacy="null",
authentication_enable="true",
max_categories=1000,
custom_colors="true",
enable_users_annotations="true",
annotation_type="local_file_csv",
db_uri="null",
hosted_file_directory="null",
local_file_csv_directory="null",
local_file_csv_file="null",
ontology_enabled="false",
obo_location="null",
embedding_names=[],
enable_reembedding="false",
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
environment=None,
aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[],
config_file_name="app_config.yml",
):
random_num = random.randrange(999999)
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
server_config = self.custom_server_config(
verbose=verbose,
debug=debug,
host=host,
port=port,
open_browser=open_browser,
force_https=force_https,
flask_secret_key=flask_secret_key,
generate_cache_control_headers=generate_cache_control_headers,
server_timing_headers=server_timing_headers,
csp_directives=csp_directives,
api_base_url=api_base_url,
web_base_url=web_base_url,
auth_type=auth_type,
oauth_api_base_url=oauth_api_base_url,
client_id=client_id,
client_secret=client_secret,
jwt_decode_options=jwt_decode_options,
session_cookie=session_cookie,
cookie=cookie,
dataroot=dataroot,
index=index,
allowed_matrix_types=allowed_matrix_types,
max_cached_datasets=max_cached_datasets,
timelimit_s=timelimit_s,
dataset_datapath=dataset_datapath,
obs_names=obs_names,
var_names=var_names,
about=about,
title=title,
diffexp_max_workers=diffexp_max_workers,
cpu_multiplier=cpu_multiplier,
target_workunit=target_workunit,
data_locater_region_name=data_locater_region_name,
cxg_tile_cache_size=cxg_tile_cache_size,
cxg_num_reader_threads=cxg_num_reader_threads,
anndata_backed=anndata_backed,
column_request_max=column_request_max,
diffexp_cellcount_max=diffexp_cellcount_max,
config_file_name=f"temp_server_config_{random_num}.yml",
)
dataset_config = self.custom_dataset_config(
scripts=scripts,
inline_scripts=inline_scripts,
about_legal_tos=about_legal_tos,
about_legal_privacy=about_legal_privacy,
authentication_enable=authentication_enable,
max_categories=max_categories,
custom_colors=custom_colors,
enable_users_annotations=enable_users_annotations,
annotation_type=annotation_type,
db_uri=db_uri,
hosted_file_directory=hosted_file_directory,
local_file_csv_directory=local_file_csv_directory,
local_file_csv_file=local_file_csv_file,
ontology_enabled=ontology_enabled,
obo_location=obo_location,
embedding_names=embedding_names,
enable_reembedding=enable_reembedding,
enable_difexp=enable_difexp,
lfc_cutoff=lfc_cutoff,
top_n=top_n,
config_file_name=f"temp_dataset_config_{random_num}.yml",
)
external_config = self.custom_external_config(
environment=environment,
aws_secrets_manager_region=aws_secrets_manager_region,
aws_secrets_manager_secrets=aws_secrets_manager_secrets,
config_file_name=f"temp_external_config_{random_num}.yml",
)
with open(configfile, "w") as app_config_file:
app_config_file.write(open(server_config).read())
app_config_file.write(open(dataset_config).read())
app_config_file.write(open(external_config).read())
return configfile
def custom_dataset_config(
self,
scripts=[],
inline_scripts=[],
about_legal_tos="null",
about_legal_privacy="null",
authentication_enable="true",
max_categories=1000,
custom_colors="true",
enable_users_annotations="true",
annotation_type="local_file_csv",
db_uri="null",
hosted_file_directory="null",
local_file_csv_directory="null",
local_file_csv_file="null",
ontology_enabled="false",
obo_location="null",
embedding_names=[],
enable_reembedding="false",
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
config_file_name="dataset_config.yml",
):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "czi_hosted_dataset_config_outline.py")
with open(dataset_config_outline_path, "r") as config_skeleton:
config = config_skeleton.read()
dataset_config = eval(config)
with open(configfile, "w") as dataset_config_file:
dataset_config_file.write(dataset_config)
return configfile
def custom_external_config(
self,
environment=None,
aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[],
config_file_name="external_config.yaml",
):
# set to the default if environment is None
if environment is None:
environment = [
dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
dict(
name="CXG_OAUTH_CLIENT_SECRET",
path=["server", "authentication", "params_oauth", "client_secret"],
required=False,
),
]
external_config = {
"external": {
"environment": environment,
"aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
}
}
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
with open(configfile, "w") as external_config_file:
yaml.dump(external_config, external_config_file)
return configfile
backend/test/test_czi_hosted/unit/common/config/test_app_config.py
+220
View File
@@ -0,0 +1,220 @@
import os
import tempfile
import unittest
import yaml
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.errors import ConfigurationError
from backend.czi_hosted.default_config import default_config
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
class AppConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_get_default_config_correctly_reads_default_config_file(self):
app_default_config = AppConfig().default_config
expected_config = yaml.load(default_config, Loader=yaml.Loader)
server_config = app_default_config["server"]
dataset_config = app_default_config["dataset"]
expected_server_config = expected_config["server"]
expected_dataset_config = expected_config["dataset"]
self.assertDictEqual(app_default_config, expected_config)
self.assertDictEqual(server_config, expected_server_config)
self.assertDictEqual(dataset_config, expected_dataset_config)
def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self):
datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
config = AppConfig()
config.update_from_config_file(file_name)
self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
def test_update_server_config_updates_server_config_and_config_status(self):
config = self.get_config()
config.complete_config()
config.check_config()
config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
with self.assertRaises(ConfigurationError):
config.server_config.check_config()
def test_write_config_outputs_yaml_with_all_config_vars(self):
config = self.get_config()
config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
default_config_yml = yaml.safe_load(default_config)
with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
output_config_yml = yaml.safe_load(output_config)
self.maxDiff = None
self.assertEqual(default_config_yml, output_config_yml)
def test_update_app_config(self):
config = AppConfig()
config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
config = AppConfig()
config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
def test_configfile_no_dataset_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
app:
flask_secret_key: secret
multi_dataset:
dataroot: test_dataroot
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(
server_changes,
[("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)],
)
self.assertEqual(dataset_changes, [])
def test_configfile_no_server_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
dataset:
user_annotations:
enable: false
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
def test_simple_update_single_config_from_path_and_value(self):
"""Update a simple config parameter"""
config = AppConfig()
config.server_config.multi_dataset__dataroot = dict(
s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"),
s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"),
)
config.add_dataroot_config("s1")
config.add_dataroot_config("s2")
# test simple value in server
config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
# test simple value in default dataset
config.update_single_config_from_path_and_value(
["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"], "mydburi",
)
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# test simple value in specific dataset
config.update_single_config_from_path_and_value(
["per_dataset_config", "s1", "user_annotations", "hosted_tiledb_array", "db_uri"], "s1dburi"
)
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "s1dburi")
self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# error checking
bad_paths = [
(
["dataset", "does", "not", "exist"],
"unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
),
(["does", "not", "exist"], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
([], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
(["per_dataset_config"], "missing dataroot when using per_dataset_config: got '['per_dataset_config']'"),
(
["per_dataset_config", "unknown"],
"unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']',"
" dataroots specified in config are ['s1', 's2']",
),
([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
("string", "path must be a list of strings, got 'string'"),
]
for bad_path, error_message in bad_paths:
with self.assertRaises(ConfigurationError) as config_error:
config.update_single_config_from_path_and_value(bad_path, "value")
self.assertEqual(config_error.exception.message, error_message)
def test_dict_update_single_config_from_path_and_value(self):
"""Update a config parameter that has a value of dict"""
# the path leads to a dict config param, set the config parameter to the new value
config = AppConfig()
config.update_single_config_from_path_and_value(
["server", "authentication", "params_oauth", "cookie"], dict(key="mykey1", max_age=100)
)
self.assertEqual(config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100))
# the path leads to an entry within a dict config param, the value is simple
config = AppConfig()
config.server_config.authentication__params_oauth__cookie = dict(key="mykey1", max_age=100)
config.update_single_config_from_path_and_value(
["server", "authentication", "params_oauth", "cookie", "httponly"], True,
)
self.assertEqual(
config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100, httponly=True)
)
backend/test/test_czi_hosted/unit/common/config/test_base_config.py
+65
View File
@@ -0,0 +1,65 @@
import unittest
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import FIXTURES_ROOT
from backend.common.errors import ConfigurationError
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
class BaseConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
config = AppConfig()
mapping = config.default_dataset_config.create_mapping(config.default_config)
self.assertIsNotNone(mapping["server__app__verbose"])
self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
def test_changes_from_default_returns_list_of_nondefault_config_values(self):
config = self.get_config(verbose="true", lfc_cutoff=0.05)
server_changes = config.server_config.changes_from_default()
dataset_changes = config.default_dataset_config.changes_from_default()
self.assertEqual(
server_changes,
[
("app__verbose", True, False),
("app__flask_secret_key", "secret", None),
("multi_dataset__dataroot", FIXTURES_ROOT, None),
("multi_dataset__matrix_cache__timelimit_s", 5, 30),
("data_locator__s3__region_name", "us-east-1", True),
],
)
self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
def test_check_config_throws_error_if_attr_has_not_been_checked(self):
config = self.get_config(verbose="true")
config.complete_config()
config.check_config()
config.update_server_config(app__verbose=False)
with self.assertRaises(ConfigurationError):
config.check_config()
backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py
+265
View File
@@ -0,0 +1,265 @@
import os
import tempfile
import requests
import unittest
from unittest.mock import patch
from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.czi_hosted.common.config.base_config import BaseConfig
from backend.test.test_czi_hosted.unit import test_server
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
from backend.common.errors import ConfigurationError
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
class TestDatasetConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.dataset_config = self.config.default_dataset_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_datatset_config_sets_vars_from_default_config(self):
config = AppConfig()
self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location)
@patch("backend.czi_hosted.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
self.dataset_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 21)
def test_app_sets_script_vars(self):
config = self.get_config(scripts=["path/to/script"])
config.default_dataset_config.handle_app()
self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}])
config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
config.default_dataset_config.handle_app()
self.assertEqual(
config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
)
config = self.get_config(scripts=["path/to/script", "different/script/path"])
config.default_dataset_config.handle_app()
# TODO @madison -- is this the desired functionality?
self.assertEqual(
config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
)
config = self.get_config(scripts=[{"more": "different/script/path"}])
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_app()
def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
config = self.get_config(enable_users_annotations="true", authentication_enable="false")
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_user_annotations(self.context)
config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
with self.assertRaises(ConfigurationError):
config.server_config.complete_config(self.context)
def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self):
config = self.get_config(
enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set"
)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."])
@patch("backend.czi_hosted.common.config.dataset_config.DbUtils")
def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils):
mock_db_utils.return_value = "123"
config = self.get_config(
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
config = self.get_config(
enable_users_annotations="true",
authentication_enable="true",
annotation_type="hosted_tiledb_array",
db_uri="gotta/set/this",
hosted_file_directory="and/this",
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB)
config = self.get_config(
enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_user_annotations(self.context)
def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
config = self.get_config(
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_local_file_csv_annotations()
self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
cwd = os.getcwd()
self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
def test_handle_embeddings__checks_data_file_types(self):
file_name = self.custom_app_config(
embedding_names=["name1", "name2"],
enable_reembedding="true",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
anndata_backed="true",
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_embeddings()
def test_handle_diffexp__raises_warning_for_large_datasets(self):
config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_diffexp(self.context)
self.assertEqual(len(self.context["messages"]), 0)
def test_multi_dataset(self):
config = AppConfig()
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
)
with self.assertRaises(ConfigurationError):
config.complete_config()
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
)
config.complete_config()
# test that multi dataroots work end to end
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
),
)
# Change this default to test if the dataroot overrides below work.
config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
# specialize the configs for set1
config.add_dataroot_config(
"s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
)
# specialize the configs for set2
config.add_dataroot_config(
"s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
)
# no specializations for set3 (they get the default dataset config)
config.complete_config()
with test_server(app_config=config) as server:
session = requests.Session()
response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
multi_dataset:
dataroot:
test:
base_url: test
dataroot: fake_dataroot
dataset:
user_annotations:
enable: false
type: hosted_tiledb_array
hosted_tiledb_array:
db_uri: fake_db_uri
hosted_file_directory: fake_dir
per_dataset_config:
test:
user_annotations:
enable: true
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataroot_config["test"]
# test config from default
self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
# test config from specialization
self.assertTrue(test_config.user_annotations__enable)
backend/test/test_czi_hosted/unit/common/config/test_external_config.py
+232
View File
@@ -0,0 +1,232 @@
import os
from unittest.mock import patch
import requests
from backend.common.errors import ConfigurationError
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.utils.type_conversion_utils import convert_string_to_value
from backend.test.test_czi_hosted.unit import test_server
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
class TestExternalConfig(ConfigTests):
def test_type_convert(self):
# The values from environment variables and aws secrets are returned as strings.
# These values need to be converted to the proper types.
self.assertEqual(convert_string_to_value("1"), int(1))
self.assertEqual(convert_string_to_value("1.1"), float(1.1))
self.assertEqual(convert_string_to_value("string"), "string")
self.assertEqual(convert_string_to_value("true"), True)
self.assertEqual(convert_string_to_value("True"), True)
self.assertEqual(convert_string_to_value("false"), False)
self.assertEqual(convert_string_to_value("False"), False)
self.assertEqual(convert_string_to_value("null"), None)
self.assertEqual(convert_string_to_value("None"), None)
self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
def test_environment_variable(self):
configfile = self.custom_external_config(
environment=[
dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
],
config_file_name="environment_external_config.yaml",
)
env = os.environ
env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k.cxg"
env["DIFFEXP"] = "False"
with test_server(command_line_args=["-c", configfile], env=env) as server:
session = requests.Session()
response = session.get(f"{server}/api/v0.2/config")
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
env["DIFFEXP"] = "True"
with test_server(command_line_args=["-c", configfile], env=env) as server:
session = requests.Session()
response = session.get(f"{server}/api/v0.2/config")
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
def test_environment_variable_errors(self):
# no name
app_config = AppConfig()
app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
# required has wrong type
app_config = AppConfig()
app_config.external_config.environment = [
dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
# no path
app_config = AppConfig()
app_config.external_config.environment = [dict(name="myenvar", required=True)]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
# required environment variable is not set
app_config = AppConfig()
app_config.external_config.environment = [
dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
@patch("backend.czi_hosted.common.config.external_config.get_secret_key")
def test_aws_secrets_manager(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
configfile = self.custom_external_config(
aws_secrets_manager_region="us-west-2",
aws_secrets_manager_secrets=[
dict(
name="my_secret",
values=[
dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=False),
dict(
key="db_uri",
path=["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"],
required=True,
),
dict(
key="oauth_client_secret",
path=["server", "authentication", "params_oauth", "client_secret"],
required=True,
),
],
)
],
config_file_name="secret_external_config.yaml",
)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.server_config.app__flask_secret_key = "original"
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.complete_config()
self.assertEqual(app_config.server_config.app__flask_secret_key, "original")
self.assertEqual(app_config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
self.assertEqual(app_config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
@patch("backend.czi_hosted.common.config.external_config.get_secret_key")
def test_aws_secrets_manager_error(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
# no region
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = None
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(
config_error.exception.message,
"Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
)
# missing secret name
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
# secret name wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
# missing values name
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
# values wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
# entry missing key
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
# entry required is wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
# entry missing path
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(key="db_uri", required=True)])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
# secret missing required key
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(
name="mysecret",
values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
)
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
backend/test/test_czi_hosted/unit/common/config/test_server_config.py
+322
View File
@@ -0,0 +1,322 @@
import os
import unittest
from unittest import mock
from unittest.mock import patch
from backend.czi_hosted.common.config.base_config import BaseConfig
from backend.common.utils.utils import find_available_port
from backend.test.test_czi_hosted.unit import test_server
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
import requests
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.errors import ConfigurationError
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
def mockenv(**envvars):
return mock.patch.dict(os.environ, envvars)
class TestServerConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_raises_error_if_default_config_is_invalid(self):
invalid_config = self.get_config(port="not_valid")
with self.assertRaises(ConfigurationError):
invalid_config.complete_config()
@patch("backend.czi_hosted.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
self.server_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 41)
def test_handle_app__throws_error_if_port_doesnt_exist(self):
config = self.get_config(port=99999999)
with self.assertRaises(ConfigurationError):
config.server_config.handle_app(self.context)
@patch("backend.czi_hosted.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
# incorrectly formatted
dataroot = {
"d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
"d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"},
}
file_name = self.custom_app_config(
dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
@patch("backend.czi_hosted.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
mock_discover_region_name.return_value = "us-west-2"
dataroot = {
"d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
"d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
}
file_name = self.custom_app_config(
dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_data_locator()
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
def test_handle_app___can_use_envar_port(self):
config = self.get_config(port=24)
self.assertEqual(config.server_config.app__port, 24)
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
os.environ["CXG_SERVER_PORT"] = "4008"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
del os.environ["CXG_SERVER_PORT"]
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
def test_handle_app__sets_web_base_url(self):
config = self.get_config(web_base_url="anything.com")
self.assertEqual(config.server_config.app__web_base_url, "anything.com")
def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self):
config = self.get_config(client_secret="KEY_FROM_FILE")
config.server_config.handle_authentication()
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name="two_data_roots.yml",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
file_name = self.custom_app_config(config_file_name="zero_roots.yml")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
def test_get_api_base_url_works(self):
# test the api_base_url feature, and that it can contain a path
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
app__flask_secret_key="secret",
app__api_base_url=f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
)
config.complete_config()
with test_server(["-p", str(backend_port)], app_config=config) as server:
session = requests.Session()
self.assertEqual(server, f"http://localhost:{backend_port}")
response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
self.assertEqual(response.status_code, 200)
data_config = response.json()
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
# test the health check at the correct url
response = session.get(f"{server}/additional/path/health")
assert response.json()["status"] == "pass"
def test_get_web_base_url_works(self):
config = self.get_config(web_base_url="www.thisisawebsite.com")
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "www.thisisawebsite.com")
config = self.get_config(web_base_url="local", port=12)
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "http://localhost:12")
config = self.get_config(web_base_url="www.thisisawebsite.com/")
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "www.thisisawebsite.com")
config = self.get_config(api_base_url="www.api_base.com/")
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "www.api_base.com")
def test_config_for_single_dataset(self):
file_name = self.custom_app_config(
config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg"
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_single_dataset(self.context)
self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
file_name = self.custom_app_config(
config_file_name="single_dataset_with_about.yml",
about="www.cziscience.com",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_single_dataset(self.context)
def test_multi_dataset_raises_error_for_illegal_routes(self):
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
self.config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
with self.assertRaises(ConfigurationError):
self.config.complete_config()
def test_multidataset_works_for_legal_routes(self):
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
self.config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
self.config.complete_config()
def test_mulitdatasets_work_e2e(self):
# test that multi dataroots work end to end
self.config.update_server_config(
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
)
)
# Change this default to test if the dataroot overrides below work.
self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
# specialize the configs for set1
self.config.add_dataroot_config(
"s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
)
# specialize the configs for set2
self.config.add_dataroot_config(
"s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
)
# no specializations for set3 (they get the default dataset config)
self.config.complete_config()
with test_server(app_config=self.config) as server:
session = requests.Session()
response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
response = session.get(f"{server}/health")
assert response.json()["status"] == "pass"
# access a dataset (no slash)
response = session.get(f"{server}/set2/pbmc3k.cxg")
self.assertEqual(response.status_code, 200)
# access a dataset (with slash)
response = session.get(f"{server}/set2/pbmc3k.cxg/")
self.assertEqual(response.status_code, 200)
@patch("backend.czi_hosted.common.config.server_config.diffexp_tiledb.set_config")
def test_handle_diffexp(self, mock_tiledb_config):
custom_config_file = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
cpu_multiplier=3,
diffexp_max_workers=1,
target_workunit=4,
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(custom_config_file)
config.server_config.handle_diffexp()
# called with the min of diffexp_max_workers and cpus*cpu_multiplier
mock_tiledb_config.assert_called_once_with(1, 4)
@patch("backend.czi_hosted.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
def test_handle_adaptor(self, mock_tiledb_context):
custom_config = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
)
config = AppConfig()
config.update_from_config_file(custom_config)
config.server_config.handle_adaptor()
mock_tiledb_context.assert_called_once_with(
{"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
)
def test_test_auth_only_in_insecure(self):
config = self.get_config(auth_type="test")
with self.assertRaises(ConfigurationError):
config.complete_config()
config.update_server_config(authentication__insecure_test_environment=True)
config.complete_config()
backend/test/test_czi_hosted/unit/common/test_api.py
+451
View File
@@ -0,0 +1,451 @@
import shutil
import time
import unittest
import zlib
from http import HTTPStatus
import pandas as pd
import requests
from backend.czi_hosted.data_common.matrix_loader import MatrixDataType
from backend.test.test_czi_hosted.unit import (
data_with_tmp_annotations,
make_fbs,
start_test_server,
stop_test_server,
)
from backend.test import PROJECT_ROOT, FIXTURES_ROOT, decode_fbs
from backend.test.fixtures.fixtures import pbmc3k_colors
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
# TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental
class EndPoints(object):
ANNOTATIONS_ENABLED = True
def test_initialize(self):
endpoint = "schema"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
self.assertEqual(
len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5
)
def test_config(self):
endpoint = "config"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertIn("library_versions", result_data["config"])
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
def test_get_layout_fbs(self):
endpoint = "layout/obs"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 8)
self.assertIsNotNone(df["columns"])
self.assertSetEqual(
set(df["col_idx"]),
{"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"},
)
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_put_layout_fbs(self):
# first check that re-embedding is turned on
result = self.session.get(f"{self.URL_BASE}config")
config_data = result.json()
re_embed = config_data["config"]["parameters"]["enable-reembedding"]
if not re_embed:
return
# attempt to reembed with umap over 100 cells.
endpoint = "layout/obs"
url = f"{self.URL_BASE}{endpoint}"
data = {}
data["filter"] = {}
data["filter"]["obs"] = {}
data["filter"]["obs"]["index"] = list(range(100))
data["method"] = "umap"
result = self.session.put(url, json=data)
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
self.assertIsInstance(result_data, dict)
self.assertEqual(result_data["type"], "float32")
self.assertTrue(result_data["name"].startswith("reembed:umap_"))
self.assertIsInstance(result_data["dims"], list)
self.assertEqual(len(result_data["dims"]), 2)
dims = result_data["dims"]
self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
def test_bad_filter(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.put(url, json=BAD_FILTER)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_get_annotations_obs_fbs(self):
endpoint = "annotations/obs"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 6 if self.ANNOTATIONS_ENABLED else 5)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
self.assertCountEqual(
df["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
+ (["cluster-test"] if self.ANNOTATIONS_ENABLED else []),
)
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"])
def test_get_annotations_obs_error(self):
endpoint = "annotations/obs"
query = "annotation-name=notakey"
url = f"{self.URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_diff_exp(self):
endpoint = "diffexp/obs"
url = f"{self.URL_BASE}{endpoint}"
params = {
"mode": "topN",
"set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
"set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
"count": 7,
}
result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 7)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
url = f"{self.URL_BASE}{endpoint}"
params = {
"mode": "topN",
"count": 10,
"set1": {"filter": {"obs": {"index": [[0, 500]]}}},
"set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
}
result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 10)
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var"
query = "annotation-name=n_cells"
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 1)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertCountEqual(df["col_idx"], ["n_cells"])
def test_get_annotations_var_error(self):
endpoint = "annotations/var"
query = "annotation-name=notakey"
url = f"{self.URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_mimetype_error(self):
endpoint = "data/var"
header = {"Accept": "xxx"}
url = f"{self.URL_BASE}{endpoint}"
result = self.session.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
def test_fbs_default(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.put(url)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.session.put(url, json=filter)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
def test_data_put_fbs(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_get_fbs(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_put_filter_fbs(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 3)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
def test_data_get_filter_fbs(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "data/var"
query = f"var:{index_col_name}=SIK1"
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
def test_data_get_unknown_filter_fbs(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "data/var"
query = f"var:{index_col_name}=UNKNOWN"
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 0)
def test_data_put_single_var(self):
endpoint = "data/var"
url = f"{self.URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
def test_colors(self):
endpoint = "colors"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data, pbmc3k_colors)
def test_static(self):
endpoint = "static"
file = "assets/favicon.ico"
url = f"{self.server}/{endpoint}/{file}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
def _setupClass(child_class, command_line):
child_class.ps, child_class.server = start_test_server(command_line)
child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
child_class.session = requests.Session()
for i in range(90):
try:
result = child_class.session.get(f"{child_class.URL_BASE}schema")
child_class.schema = result.json()
except requests.exceptions.ConnectionError:
time.sleep(1)
class EndPointsAnnotations(EndPoints):
def test_get_schema_existing_writable(self):
self._test_get_schema_writable("cluster-test")
def test_get_user_annotations_existing_obs_keys_fbs(self):
self._test_get_user_annotations_obs_keys_fbs(
"cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
)
def test_put_user_annotations_obs_fbs(self):
endpoint = "annotations/obs"
query = "annotation-collection-name=test_annotations"
url = f"{self.URL_BASE}{endpoint}?{query}"
n_rows = self.data.get_shape()[0]
fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")})
result = self.session.put(url, data=zlib.compress(fbs))
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
self.assertEqual(result.json(), {"status": "OK"})
self._test_get_schema_writable("cat_A")
self._test_get_user_annotations_obs_keys_fbs("cat_A", {"label_A"})
def _test_get_user_annotations_obs_keys_fbs(self, annotation_name, columns):
endpoint = "annotations/obs"
query = f"annotation-name={annotation_name}"
url = f"{self.URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertListEqual(df["col_idx"], [annotation_name])
self.assertEqual(set(df["columns"][0]), columns)
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def _test_get_schema_writable(self, cluster_name):
endpoint = "schema"
url = f"{self.URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
columns = result_data["schema"]["annotations"]["obs"]["columns"]
matching_columns = [c for c in columns if c["name"] == cluster_name]
self.assertEqual(len(matching_columns), 1)
self.assertTrue(matching_columns[0]["writable"])
class EndPointsAnndata(unittest.TestCase, EndPoints):
"""Test Case for endpoints"""
ANNOTATIONS_ENABLED = False
@classmethod
def setUpClass(cls):
cls._setupClass(
cls,
[
f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
"--disable-annotations",
"--experimental-enable-reembedding",
],
)
@classmethod
def tearDownClass(cls):
stop_test_server(cls.ps)
@property
def annotations_enabled(self):
return False
class EndPointsCxg(unittest.TestCase, EndPoints):
"""Test Case for endpoints"""
ANNOTATIONS_ENABLED = False
@classmethod
def setUpClass(cls):
cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"])
@classmethod
def tearDownClass(cls):
stop_test_server(cls.ps)
class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
"""Test Case for endpoints"""
ANNOTATIONS_ENABLED = True
@classmethod
def setUpClass(cls):
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
MatrixDataType.H5AD, annotations_fixture=True
)
cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmp_dir)
stop_test_server(cls.ps)
class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
"""Test Case for endpoints"""
ANNOTATIONS_ENABLED = True
@classmethod
def setUpClass(cls):
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.tmp_dir)
stop_test_server(cls.ps)
backend/test/test_czi_hosted/unit/common/test_corpora.py
+165
View File
@@ -0,0 +1,165 @@
import json
import shutil
import tempfile
import unittest
from http import HTTPStatus
import anndata
import requests
from backend.czi_hosted.common.corpora import (
corpora_get_versions_from_anndata,
corpora_is_version_supported,
corpora_get_props_from_anndata,
)
from backend.test.test_czi_hosted.unit import start_test_server, stop_test_server
from backend.test import PROJECT_ROOT
VERSION = "v0.2"
class CorporaAPITest(unittest.TestCase):
def test_corpora_get_versions_from_anndata(self):
adata = self._get_h5ad()
if "version" in adata.uns:
del adata.uns["version"]
self.assertIsNone(corpora_get_versions_from_anndata(adata))
# something bogus
adata.uns["version"] = 99
self.assertIsNone(corpora_get_versions_from_anndata(adata))
# something legit
adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
def test_corpora_is_version_supported(self):
self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
def test_corpora_get_props_from_anndata(self):
adata = self._get_h5ad()
if "version" in adata.uns:
del adata.uns["version"]
self.assertIsNone(corpora_get_props_from_anndata(adata))
# something bogus
adata.uns["version"] = 99
self.assertIsNone(corpora_get_props_from_anndata(adata))
# unsupported version, but missing required values
adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
with self.assertRaises(ValueError):
corpora_get_props_from_anndata(adata)
# legit version, but missing required values
adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
with self.assertRaises(KeyError):
corpora_get_props_from_anndata(adata)
some_fields = {
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
"title": "title",
"layer_descriptions": "layer_descriptions",
"organism": "organism",
"organism_ontology_term_id": "organism_ontology_term_id",
"project_name": "project_name",
"project_description": "project_description",
"contributors": json.dumps([{"contributors": "contributors"}]),
"project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
}
for k in some_fields:
adata.uns[k] = some_fields[k]
some_fields["contributors"] = json.loads(some_fields["contributors"])
some_fields["project_links"] = json.loads(some_fields["project_links"])
self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
def test_corpora_get_props_from_anndata_v110(self):
adata = self._get_h5ad()
if "version" in adata.uns:
del adata.uns["version"]
self.assertIsNone(corpora_get_props_from_anndata(adata))
# legit version, but missing required values
adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
with self.assertRaises(KeyError):
corpora_get_props_from_anndata(adata)
# Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
some_110_fields = {
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
"title": "title",
"layer_descriptions": "layer_descriptions",
"organism": "organism",
"organism_ontology_term_id": "organism_ontology_term_id",
}
for k in some_110_fields:
adata.uns[k] = some_110_fields[k]
self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
def _get_h5ad(self):
return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
class CorporaRESTAPITest(unittest.TestCase):
""" Confirm endpoints reflect Corpora-specific features """
@classmethod
def setCorporaFields(cls, path):
adata = anndata.read_h5ad(path)
corpora_props = {
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
"title": "PBMC3K",
"contributors": json.dumps([{"name": "name"}]),
"layer_descriptions": {"X": "raw counts"},
"organism": "human",
"organism_ontology_term_id": "unknown",
"project_name": "test project",
"project_description": "test description",
"project_links": json.dumps(
[{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
),
"default_embedding": "X_tsne",
}
adata.uns.update(corpora_props)
adata.write(path)
@classmethod
def setUpClass(cls):
cls.tmp_dir = tempfile.TemporaryDirectory()
src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
shutil.copyfile(src, dst)
cls.setCorporaFields(dst)
cls.ps, cls.server = start_test_server([dst])
@classmethod
def tearDownClass(cls):
stop_test_server(cls.ps)
cls.tmp_dir.cleanup()
def setUp(self):
self.session = requests.Session()
self.url_base = f"{self.server}/api/{VERSION}/"
def test_config(self):
endpoint = "config"
url = f"{self.url_base}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertIsInstance(result_data["config"]["corpora_props"], dict)
self.assertIsInstance(result_data["config"]["parameters"], dict)
corpora_props = result_data["config"]["corpora_props"]
parameters = result_data["config"]["parameters"]
self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
self.assertEqual(corpora_props["organism"], "human")
self.assertEqual(parameters["default_embedding"], "tsne")
backend/test/test_czi_hosted/unit/common/test_nan_rest.py
+64
View File
@@ -0,0 +1,64 @@
from http import HTTPStatus
import unittest
import math
import backend.test.decode_fbs as decode_fbs
import requests
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit import start_test_server, stop_test_server
VERSION = "v0.2"
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
class WithNaNs(unittest.TestCase):
"""Test Case for endpoints"""
@classmethod
def setUpClass(cls):
cls.ps, cls.server = start_test_server([f"{FIXTURES_ROOT}/nan.h5ad"])
@classmethod
def tearDownClass(cls):
stop_test_server(cls.ps)
def setUp(self):
self.session = requests.Session()
self.url_base = f"{self.server}/api/{VERSION}/"
def test_initialize(self):
endpoint = "schema"
url = f"{self.url_base}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
def test_data(self):
endpoint = "data/var"
url = f"{self.url_base}{endpoint}"
filter = {"filter": {"var": {"index": [[0, 20]]}}}
result = self.session.put(url, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][3][3]))
def test_annotation_obs(self):
endpoint = "annotations/obs"
url = f"{self.url_base}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][2][0]))
def test_annotation_var(self):
endpoint = "annotations/var"
url = f"{self.url_base}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertTrue(math.isnan(df["columns"][2][0]))
backend/test/test_czi_hosted/unit/common/test_rest.py
+79
View File
@@ -0,0 +1,79 @@
import unittest
from urllib.parse import parse_qs
from werkzeug.datastructures import MultiDict
from backend.common.errors import FilterError
from backend.czi_hosted.common.rest import _query_parameter_to_filter
def _qsparse(qs):
""" emulate what Flask/Werkzeug do to our QS """
return MultiDict(parse_qs(qs))
class FilterParseTests(unittest.TestCase):
""" Test cases for various filter parsing """
def test_queryparam_to_filter_parse(self):
# categories
self.assertEqual(
_query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
{
"obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
"var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
},
)
# ranges
self.assertEqual(
_query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
{
"obs": {
"annotation_value": [
{"name": "A", "min": 1, "max": 99.0},
{"name": "B", "max": 100.0},
{"name": "C", "min": 0.0},
]
},
},
)
# combo
self.assertEqual(
_query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
{
"var": {
"annotation_value": [
{"name": "B", "values": ["YES", "NO"]},
{"name": "A", "min": 1.0, "max": 99.0},
]
},
},
)
def test_queryparam_to_filter_escaping(self):
self.assertEqual(
_query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
{"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
)
def test_queryparam_to_filter_errors(self):
# should raise FilterError
filter_errors = [
"foo=bar", # no axis
"X=&Y=3", # no value
"X&Y=3", # no value
"moo:foo=bar", # bad axis
"obs:x=1,A", # non-numeric range
"var:X=1,2&var:X=3,4", # duplicate ranges
"var:Y=,",
"var:Y=2,",
"var:Y=,5",
"var:Y=*,",
"var:Y=,*",
"var:Y=*,*",
]
for qs in filter_errors:
with self.assertRaises(FilterError):
_query_parameter_to_filter(_qsparse(qs))
backend/test/test_czi_hosted/unit/common/test_writable_annotation.py
+317
View File
@@ -0,0 +1,317 @@
import json
import shutil
import unittest
from os import path, listdir
from unittest.mock import MagicMock
import numpy as np
import pandas as pd
import tiledb
from flask import Flask
from backend.common.errors import AnnotationCategoryNameError
from backend.czi_hosted.common.rest import annotations_put_fbs_helper, schema_get_helper
from backend.czi_hosted.data_common.matrix_loader import MatrixDataType
from backend.czi_hosted.db.cellxgene_orm import CellxGeneDataset, Annotation
from backend.test.fixtures.database import TestDatabase
from backend.test.test_czi_hosted.unit import make_fbs, data_with_tmp_tiledb_annotations, data_with_tmp_annotations
from backend.test import decode_fbs
TestDatabase()
class auth(object):
def get_user_id():
return "1234"
def get_user_name():
return "person name"
class WritableTileDBStoredAnnotationTest(unittest.TestCase):
def setUp(self):
self.user_id = "1234"
self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
self.data.dataset_config.user_annotations = self.annotations
self.db = self.annotations.db
self.n_rows = self.data.get_shape()[0]
self.test_dict = {
"cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
"cat_B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
}
self.fbs = make_fbs(self.test_dict)
self.df = pd.DataFrame(self.test_dict)
self.app = Flask("fake_app")
self.app.__setattr__("auth", auth)
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def annotation_put_fbs(self, fbs):
annotations_put_fbs_helper(self.data, fbs)
res = json.dumps({"status": "OK"})
return res
def test_category_name_throws_errors_for_categories_that_cant_be_converted_to_filenames(self):
with self.app.test_request_context():
bad_category_names = make_fbs(
{
"cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
"cat/B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
}
)
with self.assertRaises(AnnotationCategoryNameError):
self.annotation_put_fbs(bad_category_names)
def test_convert_to_pandas__converts_tiledb_to_pandas_df(self):
with self.app.test_request_context():
self.annotations.write_labels(self.df, self.data)
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
)
# retrieve tiledb array
df = tiledb.open(annotation.tiledb_uri)
self.assertEqual(type(df), tiledb.array.SparseArray)
# convert to pandas df
pandas_df = self.annotations.convert_to_pandas_df(df, annotation.schema_hints)
self.assertEqual(type(pandas_df), pd.DataFrame)
def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
with self.app.test_request_context():
new_name = "new_dataset/location"
self.data.get_location = MagicMock(return_value=new_name)
num_datasets = len(self.db.query([CellxGeneDataset]))
self.annotation_put_fbs(self.fbs)
more_datasets = len(self.db.query([CellxGeneDataset]))
self.assertGreater(more_datasets, num_datasets)
self.assertGreater(len(self.db.query([CellxGeneDataset], [CellxGeneDataset.name == new_name])), 0)
def test_write_labels_links_to_existing_dataset(self):
with self.app.test_request_context():
# add dataset to to db
self.annotation_put_fbs(self.fbs)
num_datasets = len(self.db.query([CellxGeneDataset]))
# create another annotation with the same dataset
self.annotation_put_fbs(self.fbs)
same_num_datasets = len(self.db.query([CellxGeneDataset]))
self.assertEqual(num_datasets, same_num_datasets)
def test_read_labels_returns_pandas_df(self):
with self.app.test_request_context():
self.annotation_put_fbs(self.fbs)
pandas_df = self.annotations.read_labels(self.data)
self.assertEqual(type(pandas_df), pd.DataFrame)
def test_read_labels_returns_df_matching_original(self):
with self.app.test_request_context():
self.annotation_put_fbs(self.fbs)
pandas_df = self.annotations.read_labels(self.data)
self.assertEqual(pandas_df.shape, (self.n_rows, 2))
self.assertEqual(set(pandas_df.columns), {"cat_A", "cat_B"})
self.assertTrue(self.data.original_obs_index.equals(pandas_df.index))
self.assertTrue(np.all(pandas_df["cat_A"] == ["label_A"] * self.n_rows))
self.assertTrue(np.all(pandas_df["cat_B"] == ["label_B"] * self.n_rows))
def test_error_checks(self):
# verify that the expected errors are generated
with self.app.test_request_context():
n_rows = self.data.get_shape()[0]
fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
# ensure we catch attempt to overwrite non-writable data
with self.assertRaises(KeyError):
self.annotation_put_fbs(fbs_bad)
def test_write_labels_stores_df_as_tiledb_array(self):
with self.app.test_request_context():
self.annotations.write_labels(self.df, self.data)
# get uri
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
)
df = tiledb.open(annotation.tiledb_uri)
self.assertEqual(type(df), tiledb.array.SparseArray)
def test_remove_categories(self):
with self.app.test_request_context():
# update empty category data, which is how annotations are removed
empty = make_fbs({})
self.annotation_put_fbs(empty)
# verify that the tiledb uri is an empty string.
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
)
self.assertEqual(annotation.tiledb_uri, "")
# verify that read_labels returns None
df = self.annotations.read_labels(self.data)
self.assertIsNone(df)
class WritableAnnotationTest(unittest.TestCase):
def setUp(self):
self.data, self.tmp_dir, self.annotations = data_with_tmp_annotations(MatrixDataType.H5AD)
self.data.dataset_config.user_annotations = self.annotations
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def annotation_put_fbs(self, fbs):
annotations_put_fbs_helper(self.data, fbs)
res = json.dumps({"status": "OK"})
return res
def test_error_checks(self):
# verify that the expected errors are generated
n_rows = self.data.get_shape()[0]
fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
# ensure we catch attempt to overwrite non-writable data
with self.assertRaises(KeyError):
self.annotation_put_fbs(fbs_bad)
def test_write_to_file(self):
# verify the file is written as expected
n_rows = self.data.get_shape()[0]
fbs = make_fbs(
{
"cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
"cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
}
)
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations.output_file))
df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
self.assertEqual(df.shape, (n_rows, 2))
self.assertEqual(set(df.columns), {"cat_A", "cat_B"})
self.assertTrue(self.data.original_obs_index.equals(df.index))
self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows))
self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows))
# verify complete overwrite on second attempt, AND rotation occurs
fbs = make_fbs(
{
"cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"),
"cat_C": pd.Series(["label_C"] * n_rows, dtype="category"),
}
)
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations.output_file))
df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
self.assertEqual(set(df.columns), {"cat_A", "cat_C"})
self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows))
self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows))
# rotation
name, ext = path.splitext(self.annotations.output_file)
backup_dir = f"{name}-backups"
self.assertTrue(path.isdir(backup_dir))
found_files = listdir(backup_dir)
self.assertEqual(len(found_files), 1)
def test_file_rotation_to_max_9(self):
# verify we stop rotation at 9
n_rows = self.data.get_shape()[0]
fbs = make_fbs(
{
"cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
"cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
}
)
for i in range(0, 11):
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
name, ext = path.splitext(self.annotations.output_file)
backup_dir = f"{name}-backups"
self.assertTrue(path.isdir(backup_dir))
found_files = listdir(backup_dir)
self.assertTrue(len(found_files) <= 9)
def test_put_get_roundtrip(self):
# verify that OBS PUTs (annotation_put_fbs) are accessible via
# GET (annotation_to_fbs_matrix)
n_rows = self.data.get_shape()[0]
fbs = make_fbs(
{
"cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
"cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
}
)
# put
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
# get
labels = self.annotations.read_labels(None)
fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
schema = schema_get_helper(self.data)
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
obs_index_col_name = schema["annotations"]["obs"]["index"]
self.assertEqual(annotations["n_rows"], n_rows)
self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"])
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
)
col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows)
self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows)
# verify the schema was updated
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(
all_col_schema["cat_A"],
{"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
)
self.assertEqual(
all_col_schema["cat_B"],
{"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
)
def test_put_float_data(self):
# verify that OBS PUTs (annotation_put_fbs) are accessible via
# GET (annotation_to_fbs_matrix)
n_rows = self.data.get_shape()[0]
# verifies that floating point with decimals fail.
fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))})
with self.assertRaises(ValueError) as exception_context:
res = self.annotation_put_fbs(fbs)
self.assertEqual(str(exception_context.exception), "Columns may not have floating point types")
# verifies that floating point that can be converted to int passes
fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")})
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
# check read_labels
labels = self.annotations.read_labels(None)
fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
schema = schema_get_helper(self.data)
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
self.assertEqual(annotations["n_rows"], n_rows)
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(
all_col_schema["cat_F_PASS"],
{"name": "cat_F_PASS", "type": "int32", "writable": True},
)
backend/test/test_czi_hosted/unit/common/utils/__init__.py
View File
backend/test/test_czi_hosted/unit/common/utils/test_cxg_generation_utils.py
+158
View File
@@ -0,0 +1,158 @@
import json
import unittest
from os import path, mkdir
from shutil import rmtree
from uuid import uuid4
import numpy as np
import tiledb
from pandas import Series, DataFrame
from backend.czi_hosted.common.utils.cxg_generation_utils import (
convert_dictionary_to_cxg_group,
convert_dataframe_to_cxg_array,
convert_ndarray_to_cxg_dense_array,
convert_matrix_to_cxg_array,
)
from backend.test import FIXTURES_ROOT
class TestCxgGenerationUtils(unittest.TestCase):
def setUp(self):
self.testing_cxg_temp_directory = f"{FIXTURES_ROOT}/{uuid4()}"
mkdir(self.testing_cxg_temp_directory)
def tearDown(self):
if path.isdir(self.testing_cxg_temp_directory):
rmtree(self.testing_cxg_temp_directory)
def test__convert_dictionary_to_cxg_group__writes_successfully(self):
random_dictionary = {"cookies": "chocolate_chip", "brownies": "chocolate", "cake": "double chocolate"}
dictionary_name = "favorite_desserts"
expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
convert_dictionary_to_cxg_group(
self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
)
array = tiledb.open(expected_array_directory)
actual_stored_metadata = dict(array.meta.items())
self.assertTrue(path.isdir(expected_array_directory))
self.assertTrue(isinstance(array, tiledb.DenseArray))
self.assertEqual(random_dictionary, actual_stored_metadata)
def test__convert_dataframe_to_cxg_array__writes_successfully(self):
random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int64)
random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
random_dataframe_name = f"random_dataframe_{uuid4()}"
random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
convert_dataframe_to_cxg_array(
self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
)
expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
expected_array_metadata = {
"cxg_schema": json.dumps(
{"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
)
}
actual_stored_dataframe_array = tiledb.open(expected_array_directory)
actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
self.assertTrue(path.isdir(expected_array_directory))
self.assertTrue(isinstance(actual_stored_dataframe_array, tiledb.DenseArray))
self.assertDictEqual(expected_array_metadata, actual_stored_dataframe_metadata)
self.assertTrue((actual_stored_dataframe_array[0:4]["int_category"] == random_int_category.to_numpy()).all())
self.assertTrue((actual_stored_dataframe_array[0:4]["bool_category"] == random_bool_category.to_numpy()).all())
def test__convert_ndarray_to_cxg_dense_array__writes_successfully(self):
ndarray = np.random.rand(3, 2)
ndarray_name = f"{self.testing_cxg_temp_directory}/awesome_ndarray_{uuid4()}"
convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, tiledb.Ctx())
actual_stored_array = tiledb.open(ndarray_name)
self.assertTrue(path.isdir(ndarray_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
self.assertTrue((actual_stored_array[:, :] == ndarray).all())
def test__convert_matrix_to_cxg_array__dense_array_writes_successfully(self):
matrix = np.float32(np.random.rand(3, 2))
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_matrix_{uuid4()}"
convert_matrix_to_cxg_array(matrix_name, matrix, False, tiledb.Ctx())
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
self.assertTrue((actual_stored_array[:, :] == matrix).all())
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros_empty_array(self):
matrix = np.zeros([3, 2])
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_zero_matrix_{uuid4()}"
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[:, :][""].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
matrix = np.zeros([3, 3])
matrix[0, 0] = 1
matrix[1, 1] = 1
matrix[2, 2] = 2
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_sparse_matrix_{uuid4()}"
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[0, 0][""] == 1)
self.assertTrue(actual_stored_array[1, 1][""] == 1)
self.assertTrue(actual_stored_array[2, 2][""] == 2)
self.assertTrue(actual_stored_array[:, :][""].size == 3)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
matrix = np.ones((3, 2))
# The column shift will be equal to the matrix since subtracting the column shift from the matrix will create
# a matrix of zeros which is sparse.
column_shift = np.ones((3, 2))
convert_matrix_to_cxg_array(
matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
)
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[:, :][""].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
matrix = np.ones((2, 2))
# Only column shift the first column of ones.
column_shift = np.array([[1, 0], [1, 0]])
convert_matrix_to_cxg_array(
matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
)
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[0, 1][""] == 1)
self.assertTrue(actual_stored_array[1, 1][""] == 1)
self.assertTrue(actual_stored_array[:, :][""].size == 2)
backend/test/test_czi_hosted/unit/common/utils/test_matrix_utils.py
+66
View File
@@ -0,0 +1,66 @@
import unittest
import numpy as np
from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
class TestMatrixUtils(unittest.TestCase):
def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
matrix = np.array([1, 2, 3])
self.assertFalse(is_matrix_sparse(matrix, 0))
self.assertTrue(is_matrix_sparse(matrix, 100))
def test__is_matrix_sparse__partially_populated_sparse_matrix_returns_true(self):
matrix = np.zeros([3, 4])
matrix[2][3] = 1.0
matrix[1][1] = 2.2
self.assertTrue(is_matrix_sparse(matrix, 50))
def test__is_matrix_sparse__partially_populated_dense_matrix_returns_false(self):
matrix = np.zeros([2, 2])
matrix[0][0] = 1.0
matrix[0][1] = 2.2
matrix[1][1] = 3.7
self.assertFalse(is_matrix_sparse(matrix, 50))
def test__is_matrix_sparse__giant_matrix_returns_false_early(self):
matrix = np.ones([20000, 20])
with self.assertLogs(level="INFO") as logger:
self.assertFalse(is_matrix_sparse(matrix, 1))
# Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
# non-zero elements in the matrix.
self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
def test__is_matrix_sparse_with_column_shift_encoding__regular_sparse_returns_true(self):
matrix = np.zeros([2, 2])
matrix[0][0] = 1.0
self.assertIsNotNone(get_column_shift_encode_for_matrix(matrix, 50))
def test__is_matrix_sparse_with_column_shift_encoding__column_shift_returns_same_value(self):
matrix = np.ones([2, 2])
expected_column_shift = [1, 1]
actual_column_shift = get_column_shift_encode_for_matrix(matrix, 50)
self.assertTrue((expected_column_shift == actual_column_shift).all())
def test__is_matrix_sparse_with_column_shift_encoding__impossible_column_shift_returns_none(self):
matrix = np.array([[1, 2], [3, 4]])
self.assertIsNone(get_column_shift_encode_for_matrix(matrix, 50))
def test__is_matrix_sparse_with_column_shift_encoding__giant_matrix_returns_false_early(self):
matrix = np.random.rand(20000, 20)
with self.assertLogs(level="INFO") as logger:
self.assertFalse(is_matrix_sparse(matrix, 1))
# Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
# non-zero elements in the matrix.
self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
backend/test/test_czi_hosted/unit/common/utils/test_sanitization_utils.py
+55
View File
@@ -0,0 +1,55 @@
import unittest
from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list, sanitize_keys_in_dictionary
class TestSanitizationUtils(unittest.TestCase):
def test__sanitize_values_in_list__not_strings_raises_exception(self):
keys_to_sanitize = [1, 2, 3]
with self.assertRaises(Exception) as exception_context:
sanitize_values_in_list(keys_to_sanitize)
self.assertIn("must contain all strings", str(exception_context.exception))
def test__sanitize_values_in_list__not_all_strings_raises_exception(self):
keys_to_sanitize = ["1", "2", 3]
with self.assertRaises(Exception) as exception_context:
sanitize_values_in_list(keys_to_sanitize)
self.assertIn("must contain all strings", str(exception_context.exception))
def test__sanitize_values_in_list__replace_non_ascii_character_with_underscore(self):
keys_to_sanitize = ["abc.", "~abc", "a~b/c"]
expected_sanitized_keys_dict = dict(zip(keys_to_sanitize, ["abc_", "_abc", "a_b_c"]))
actual_sanitized_keys_dict = sanitize_values_in_list(keys_to_sanitize)
self.assertEqual(expected_sanitized_keys_dict, actual_sanitized_keys_dict)
def test__sanitize_keys_in_dictionary__replace_non_ascii_character_with_underscore(self):
dictionary_to_sanitize = {"abc.": 3, "~abc": 4, "a~b/c": 5}
expected_sanitized_dict = {"abc_": 3, "_abc": 4, "a_b_c": 5}
actual_sanitized_dict = dictionary_to_sanitize
sanitize_keys_in_dictionary(actual_sanitized_dict)
self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
def test__sanitize_keys_in_dictionary__non_string_key_raises_exception(self):
dictionary_to_sanitize = {4: 3, "~abc": 4, "a~b/c": 5}
with self.assertRaises(Exception) as exception_context:
sanitize_keys_in_dictionary(dictionary_to_sanitize)
self.assertIn("must contain all strings", str(exception_context.exception))
def test__sanitize_keys_in_dictionary__replace_only_some_keys(self):
dictionary_to_sanitize = {"abc": 3, "~abc": 4, "a~b/c": 5}
expected_sanitized_dict = {"abc": 3, "_abc": 4, "a_b_c": 5}
actual_sanitized_dict = dictionary_to_sanitize
sanitize_keys_in_dictionary(actual_sanitized_dict)
self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
backend/test/test_czi_hosted/unit/common/utils/test_utils.py
+34
View File
@@ -0,0 +1,34 @@
import os
import shutil
import unittest
from backend.common.utils.utils import import_plugins
from backend.test import PROJECT_ROOT, random_string
class TestPlugins(unittest.TestCase):
""" Test plugin import functionality """
plugins_dir = f"{PROJECT_ROOT}/backend/test/test_czi_hosted/unit/plugins"
test_plugin_path = f"{plugins_dir}/foo.py"
secret = random_string(8)
@classmethod
def setUpClass(cls) -> None:
if not os.path.isdir(cls.plugins_dir):
os.mkdir(cls.plugins_dir)
with open(cls.test_plugin_path, "w") as fh:
fh.write(f'SECRET = "{cls.secret}"\n')
@classmethod
def tearDownClass(cls) -> None:
if os.path.isdir(cls.plugins_dir):
shutil.rmtree(cls.plugins_dir)
def test_import_plugins(self):
self.assertTrue(os.path.isfile(self.test_plugin_path))
loaded_modules = import_plugins("backend.test.test_czi_hosted.unit.plugins")
# test that import plugins found the file
self.assertEqual(["backend.test.test_czi_hosted.unit.plugins.foo"], [ele.__name__ for ele in loaded_modules])
# test that the module was properly executed
self.assertEqual(self.secret, loaded_modules[0].SECRET)