mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-01 21:48:12 +08:00
Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
@@ -0,0 +1,228 @@
|
||||
import os
|
||||
import shutil
|
||||
import unittest
|
||||
import random
|
||||
from unittest import mock
|
||||
import yaml
|
||||
|
||||
from backend.test import FIXTURES_ROOT
|
||||
|
||||
|
||||
def mockenv(**envvars):
|
||||
return mock.patch.dict(os.environ, envvars)
|
||||
|
||||
|
||||
class ConfigTests(unittest.TestCase):
|
||||
tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
shutil.rmtree(cls.tmp_fixtures_directory)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
os.makedirs(cls.tmp_fixtures_directory)
|
||||
|
||||
def custom_server_config(
|
||||
self,
|
||||
verbose="false",
|
||||
debug="false",
|
||||
host="localhost",
|
||||
port="null",
|
||||
open_browser="false",
|
||||
force_https="false",
|
||||
flask_secret_key="secret",
|
||||
auth_type="session",
|
||||
insecure_test_environment="false",
|
||||
index="false",
|
||||
allowed_matrix_types=[],
|
||||
max_cached_datasets=5,
|
||||
timelimit_s=5,
|
||||
dataset_datapath="null",
|
||||
obs_names="null",
|
||||
var_names="null",
|
||||
about="null",
|
||||
title="null",
|
||||
data_locater_region_name="us-east-1",
|
||||
anndata_backed="false",
|
||||
column_request_max=32,
|
||||
diffexp_cellcount_max="null",
|
||||
config_file_name="server_config.yaml",
|
||||
):
|
||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||
server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py")
|
||||
with open(server_config_outline_path, "r") as config_skeleton:
|
||||
config = config_skeleton.read()
|
||||
server_config = eval(config)
|
||||
with open(configfile, "w") as server_config_file:
|
||||
server_config_file.write(server_config)
|
||||
return configfile
|
||||
|
||||
def custom_app_config(
|
||||
self,
|
||||
verbose="false",
|
||||
debug="false",
|
||||
host="localhost",
|
||||
port="null",
|
||||
open_browser="false",
|
||||
force_https="false",
|
||||
flask_secret_key="secret",
|
||||
auth_type="session",
|
||||
index="false",
|
||||
allowed_matrix_types=[],
|
||||
max_cached_datasets=5,
|
||||
timelimit_s=5,
|
||||
dataset_datapath="null",
|
||||
obs_names="null",
|
||||
var_names="null",
|
||||
about="null",
|
||||
title="null",
|
||||
data_locater_region_name="us-east-1",
|
||||
anndata_backed="false",
|
||||
column_request_max=32,
|
||||
diffexp_cellcount_max="null",
|
||||
scripts=[],
|
||||
inline_scripts=[],
|
||||
authentication_enable="true",
|
||||
max_categories=1000,
|
||||
custom_colors="true",
|
||||
enable_users_annotations="true",
|
||||
annotation_type="local_file_csv",
|
||||
db_uri="null",
|
||||
hosted_file_directory="null",
|
||||
local_file_csv_directory="null",
|
||||
local_file_csv_file="null",
|
||||
local_file_csv_gene_sets_file="null",
|
||||
ontology_enabled="false",
|
||||
obo_location="null",
|
||||
gene_sets_readonly="false",
|
||||
embedding_names=[],
|
||||
enable_reembedding="false",
|
||||
enable_difexp="true",
|
||||
lfc_cutoff=0.01,
|
||||
top_n=10,
|
||||
environment=None,
|
||||
aws_secrets_manager_region=None,
|
||||
aws_secrets_manager_secrets=[],
|
||||
config_file_name="app_config.yml",
|
||||
):
|
||||
random_num = random.randrange(999999)
|
||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||
server_config = self.custom_server_config(
|
||||
verbose=verbose,
|
||||
debug=debug,
|
||||
host=host,
|
||||
port=port,
|
||||
open_browser=open_browser,
|
||||
force_https=force_https,
|
||||
flask_secret_key=flask_secret_key,
|
||||
auth_type=auth_type,
|
||||
index=index,
|
||||
allowed_matrix_types=allowed_matrix_types,
|
||||
max_cached_datasets=max_cached_datasets,
|
||||
timelimit_s=timelimit_s,
|
||||
dataset_datapath=dataset_datapath,
|
||||
obs_names=obs_names,
|
||||
var_names=var_names,
|
||||
about=about,
|
||||
title=title,
|
||||
data_locater_region_name=data_locater_region_name,
|
||||
anndata_backed=anndata_backed,
|
||||
column_request_max=column_request_max,
|
||||
diffexp_cellcount_max=diffexp_cellcount_max,
|
||||
config_file_name=f"temp_server_config_{random_num}.yml",
|
||||
)
|
||||
dataset_config = self.custom_dataset_config(
|
||||
scripts=scripts,
|
||||
inline_scripts=inline_scripts,
|
||||
authentication_enable=authentication_enable,
|
||||
max_categories=max_categories,
|
||||
custom_colors=custom_colors,
|
||||
enable_users_annotations=enable_users_annotations,
|
||||
annotation_type=annotation_type,
|
||||
db_uri=db_uri,
|
||||
hosted_file_directory=hosted_file_directory,
|
||||
local_file_csv_directory=local_file_csv_directory,
|
||||
local_file_csv_file=local_file_csv_file,
|
||||
local_file_csv_gene_sets_file=local_file_csv_gene_sets_file,
|
||||
ontology_enabled=ontology_enabled,
|
||||
obo_location=obo_location,
|
||||
gene_sets_readonly=gene_sets_readonly,
|
||||
embedding_names=embedding_names,
|
||||
enable_reembedding=enable_reembedding,
|
||||
enable_difexp=enable_difexp,
|
||||
lfc_cutoff=lfc_cutoff,
|
||||
top_n=top_n,
|
||||
config_file_name=f"temp_dataset_config_{random_num}.yml",
|
||||
)
|
||||
external_config = self.custom_external_config(
|
||||
environment=environment,
|
||||
aws_secrets_manager_region=aws_secrets_manager_region,
|
||||
aws_secrets_manager_secrets=aws_secrets_manager_secrets,
|
||||
config_file_name=f"temp_external_config_{random_num}.yml",
|
||||
)
|
||||
|
||||
with open(configfile, "w") as app_config_file:
|
||||
app_config_file.write(open(server_config).read())
|
||||
app_config_file.write(open(dataset_config).read())
|
||||
app_config_file.write(open(external_config).read())
|
||||
|
||||
return configfile
|
||||
|
||||
def custom_dataset_config(
|
||||
self,
|
||||
scripts=[],
|
||||
inline_scripts=[],
|
||||
authentication_enable="true",
|
||||
max_categories=1000,
|
||||
custom_colors="true",
|
||||
enable_users_annotations="true",
|
||||
annotation_type="local_file_csv",
|
||||
db_uri="null",
|
||||
hosted_file_directory="null",
|
||||
local_file_csv_directory="null",
|
||||
local_file_csv_file="null",
|
||||
local_file_csv_gene_sets_file="null",
|
||||
ontology_enabled="false",
|
||||
obo_location="null",
|
||||
gene_sets_readonly="false",
|
||||
embedding_names=[],
|
||||
enable_reembedding="false",
|
||||
enable_difexp="true",
|
||||
lfc_cutoff=0.01,
|
||||
top_n=10,
|
||||
config_file_name="dataset_config.yml",
|
||||
):
|
||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||
dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py")
|
||||
with open(dataset_config_outline_path, "r") as config_skeleton:
|
||||
config = config_skeleton.read()
|
||||
dataset_config = eval(config)
|
||||
with open(configfile, "w") as dataset_config_file:
|
||||
dataset_config_file.write(dataset_config)
|
||||
|
||||
return configfile
|
||||
|
||||
def custom_external_config(
|
||||
self,
|
||||
environment=None,
|
||||
aws_secrets_manager_region=None,
|
||||
aws_secrets_manager_secrets=[],
|
||||
config_file_name="external_config.yaml",
|
||||
):
|
||||
# set to the default if environment is None
|
||||
if environment is None:
|
||||
environment = [
|
||||
dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
|
||||
]
|
||||
external_config = {
|
||||
"external": {
|
||||
"environment": environment,
|
||||
"aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
|
||||
}
|
||||
}
|
||||
|
||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||
with open(configfile, "w") as external_config_file:
|
||||
yaml.dump(external_config, external_config_file)
|
||||
return configfile
|
||||
@@ -0,0 +1,153 @@
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import yaml
|
||||
|
||||
from backend.server.default_config import default_config
|
||||
from backend.server.common.config.app_config import AppConfig
|
||||
from backend.common.errors import ConfigurationError
|
||||
from backend.test.test_server.unit.common.config import ConfigTests
|
||||
from backend.test import FIXTURES_ROOT, H5AD_FIXTURE
|
||||
|
||||
|
||||
class AppConfigTest(ConfigTests):
|
||||
def setUp(self):
|
||||
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
self.server_config = self.config.server_config
|
||||
self.config.complete_config()
|
||||
|
||||
message_list = []
|
||||
|
||||
def noop(message):
|
||||
message_list.append(message)
|
||||
|
||||
messagefn = noop
|
||||
self.context = dict(messagefn=messagefn, messages=message_list)
|
||||
|
||||
def get_config(self, **kwargs):
|
||||
file_name = self.custom_app_config(
|
||||
dataset_datapath=H5AD_FIXTURE, config_file_name=self.config_file_name, **kwargs
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
return config
|
||||
|
||||
def test_get_default_config_correctly_reads_default_config_file(self):
|
||||
app_default_config = AppConfig().default_config
|
||||
|
||||
expected_config = yaml.load(default_config, Loader=yaml.Loader)
|
||||
|
||||
server_config = app_default_config["server"]
|
||||
dataset_config = app_default_config["dataset"]
|
||||
|
||||
expected_server_config = expected_config["server"]
|
||||
expected_dataset_config = expected_config["dataset"]
|
||||
|
||||
self.assertDictEqual(app_default_config, expected_config)
|
||||
self.assertDictEqual(server_config, expected_server_config)
|
||||
self.assertDictEqual(dataset_config, expected_dataset_config)
|
||||
|
||||
def test_get_dataset_config_returns_dataset_config_for_single_datasets(self):
|
||||
datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
|
||||
file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
|
||||
self.assertEqual(config.get_dataset_config(), config.dataset_config)
|
||||
|
||||
def test_update_server_config_updates_server_config_and_config_status(self):
|
||||
config = self.get_config()
|
||||
config.complete_config()
|
||||
config.check_config()
|
||||
config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.check_config()
|
||||
|
||||
def test_write_config_outputs_yaml_with_all_config_vars(self):
|
||||
config = self.get_config()
|
||||
config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
|
||||
with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
|
||||
default_config_yml = yaml.safe_load(default_config)
|
||||
|
||||
with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
|
||||
output_config_yml = yaml.safe_load(output_config)
|
||||
self.maxDiff = None
|
||||
self.assertEqual(default_config_yml, output_config_yml)
|
||||
|
||||
def test_update_app_config(self):
|
||||
config = AppConfig()
|
||||
config.update_server_config(app__verbose=True, single_dataset__datapath="datapath")
|
||||
vars = config.server_config.changes_from_default()
|
||||
self.assertCountEqual(vars, [("app__verbose", True, False), ("single_dataset__datapath", "datapath", None)])
|
||||
|
||||
config = AppConfig()
|
||||
config.update_dataset_config(app__scripts=(), app__inline_scripts=())
|
||||
vars = config.server_config.changes_from_default()
|
||||
self.assertCountEqual(vars, [])
|
||||
|
||||
config = AppConfig()
|
||||
config.update_dataset_config(app__scripts=[], app__inline_scripts=[])
|
||||
vars = config.dataset_config.changes_from_default()
|
||||
self.assertCountEqual(vars, [])
|
||||
|
||||
config = AppConfig()
|
||||
config.update_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
|
||||
vars = config.dataset_config.changes_from_default()
|
||||
self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
|
||||
|
||||
def test_configfile_no_server_section(self):
|
||||
# test a config file without a dataset section
|
||||
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
configfile = os.path.join(tempdir, "config.yaml")
|
||||
with open(configfile, "w") as fconfig:
|
||||
config = """
|
||||
dataset:
|
||||
user_annotations:
|
||||
enable: false
|
||||
"""
|
||||
fconfig.write(config)
|
||||
|
||||
app_config = AppConfig()
|
||||
app_config.update_from_config_file(configfile)
|
||||
server_changes = app_config.server_config.changes_from_default()
|
||||
dataset_changes = app_config.dataset_config.changes_from_default()
|
||||
self.assertEqual(server_changes, [])
|
||||
self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
|
||||
|
||||
def test_simple_update_single_config_from_path_and_value(self):
|
||||
"""Update a simple config parameter"""
|
||||
|
||||
config = AppConfig()
|
||||
config.server_config.single_dataset__datapath = "my/data/path"
|
||||
|
||||
# test simple value in server
|
||||
config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
|
||||
self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
|
||||
|
||||
# test simple value in default dataset
|
||||
config.update_single_config_from_path_and_value(
|
||||
["dataset", "user_annotations", "ontology", "obo_location"], "dummy_location",
|
||||
)
|
||||
self.assertEqual(config.dataset_config.user_annotations__ontology__obo_location, "dummy_location")
|
||||
|
||||
# error checking
|
||||
bad_paths = [
|
||||
(
|
||||
["dataset", "does", "not", "exist"],
|
||||
"unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
|
||||
),
|
||||
(["does", "not", "exist"], "path must start with 'server', or 'dataset'"),
|
||||
([], "path must start with 'server', or 'dataset'"),
|
||||
([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
|
||||
("string", "path must be a list of strings, got 'string'"),
|
||||
]
|
||||
for bad_path, error_message in bad_paths:
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
config.update_single_config_from_path_and_value(bad_path, "value")
|
||||
|
||||
self.assertEqual(config_error.exception.message, error_message)
|
||||
@@ -0,0 +1,63 @@
|
||||
import unittest
|
||||
|
||||
from backend.server.common.config.app_config import AppConfig
|
||||
from backend.test import H5AD_FIXTURE
|
||||
from backend.common.errors import ConfigurationError
|
||||
from backend.test.test_server.unit.common.config import ConfigTests
|
||||
|
||||
|
||||
class BaseConfigTest(ConfigTests):
|
||||
def setUp(self):
|
||||
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
self.server_config = self.config.server_config
|
||||
self.config.complete_config()
|
||||
|
||||
message_list = []
|
||||
|
||||
def noop(message):
|
||||
message_list.append(message)
|
||||
|
||||
messagefn = noop
|
||||
self.context = dict(messagefn=messagefn, messages=message_list)
|
||||
|
||||
def get_config(self, **kwargs):
|
||||
file_name = self.custom_app_config(
|
||||
dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
return config
|
||||
|
||||
def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
|
||||
config = AppConfig()
|
||||
mapping = config.dataset_config.create_mapping(config.default_config)
|
||||
self.assertIsNotNone(mapping["server__app__verbose"])
|
||||
self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
|
||||
self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
|
||||
|
||||
def test_changes_from_default_returns_list_of_nondefault_config_values(self):
|
||||
config = self.get_config(verbose="true", lfc_cutoff=0.05)
|
||||
server_changes = config.server_config.changes_from_default()
|
||||
dataset_changes = config.dataset_config.changes_from_default()
|
||||
|
||||
self.assertEqual(
|
||||
server_changes,
|
||||
[
|
||||
("app__verbose", True, False),
|
||||
("app__flask_secret_key", "secret", None),
|
||||
("single_dataset__datapath", H5AD_FIXTURE, None),
|
||||
('data_locator__s3__region_name', 'us-east-1', True)
|
||||
],
|
||||
)
|
||||
self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
|
||||
|
||||
def test_check_config_throws_error_if_attr_has_not_been_checked(self):
|
||||
config = self.get_config(verbose="true")
|
||||
config.complete_config()
|
||||
config.check_config()
|
||||
config.update_server_config(app__verbose=False)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.check_config()
|
||||
@@ -0,0 +1,157 @@
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from backend.server.common.annotations.local_file_csv import AnnotationsLocalFile
|
||||
from backend.server.common.config.app_config import AppConfig
|
||||
from backend.server.common.config.base_config import BaseConfig
|
||||
from backend.test import FIXTURES_ROOT, H5AD_FIXTURE
|
||||
|
||||
from backend.common.errors import ConfigurationError
|
||||
from backend.test.test_server.unit.common.config import ConfigTests
|
||||
|
||||
|
||||
class TestDatasetConfig(ConfigTests):
|
||||
def setUp(self):
|
||||
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
self.dataset_config = self.config.dataset_config
|
||||
self.config.complete_config()
|
||||
message_list = []
|
||||
|
||||
def noop(message):
|
||||
message_list.append(message)
|
||||
|
||||
messagefn = noop
|
||||
self.context = dict(messagefn=messagefn, messages=message_list)
|
||||
|
||||
def get_config(self, **kwargs):
|
||||
file_name = self.custom_app_config(dataset_datapath=H5AD_FIXTURE, **kwargs)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
return config
|
||||
|
||||
def test_init_datatset_config_sets_vars_from_config(self):
|
||||
config = AppConfig()
|
||||
self.assertEqual(config.dataset_config.presentation__max_categories, 1000)
|
||||
self.assertEqual(config.dataset_config.user_annotations__type, "local_file_csv")
|
||||
self.assertEqual(config.dataset_config.diffexp__lfc_cutoff, 0.01)
|
||||
self.assertIsNone(config.dataset_config.user_annotations__ontology__obo_location)
|
||||
|
||||
@patch("backend.server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
|
||||
def test_complete_config_checks_all_attr(self, mock_check_attrs):
|
||||
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
|
||||
self.dataset_config.complete_config(self.context)
|
||||
self.assertIsNotNone(self.config.server_config.data_adaptor)
|
||||
self.assertEqual(mock_check_attrs.call_count, 19)
|
||||
|
||||
def test_app_sets_script_vars(self):
|
||||
config = self.get_config(scripts=["path/to/script"])
|
||||
config.dataset_config.handle_app()
|
||||
|
||||
self.assertEqual(config.dataset_config.app__scripts, [{"src": "path/to/script"}])
|
||||
|
||||
config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
|
||||
config.dataset_config.handle_app()
|
||||
self.assertEqual(
|
||||
config.dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
|
||||
)
|
||||
|
||||
config = self.get_config(scripts=["path/to/script", "different/script/path"])
|
||||
config.dataset_config.handle_app()
|
||||
# TODO @madison -- is this the desired functionality?
|
||||
self.assertEqual(
|
||||
config.dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
|
||||
)
|
||||
|
||||
config = self.get_config(scripts=[{"more": "different/script/path"}])
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.dataset_config.handle_app()
|
||||
|
||||
def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
|
||||
config = self.get_config(enable_users_annotations="true", authentication_enable="false")
|
||||
config.server_config.complete_config(self.context)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.dataset_config.handle_user_annotations(self.context)
|
||||
|
||||
config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.complete_config(self.context)
|
||||
|
||||
def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self):
|
||||
config = self.get_config(
|
||||
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
|
||||
)
|
||||
config.server_config.complete_config(self.context)
|
||||
config.dataset_config.handle_user_annotations(self.context)
|
||||
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
|
||||
|
||||
config = self.get_config(
|
||||
enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
|
||||
)
|
||||
config.server_config.complete_config(self.context)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.dataset_config.handle_user_annotations(self.context)
|
||||
|
||||
def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
|
||||
config = self.get_config(
|
||||
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
|
||||
)
|
||||
config.server_config.complete_config(self.context)
|
||||
config.dataset_config.handle_local_file_csv_annotations(self.context)
|
||||
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
|
||||
cwd = os.getcwd()
|
||||
self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)
|
||||
|
||||
def test_handle_embeddings__checks_data_file_types(self):
|
||||
file_name = self.custom_app_config(
|
||||
embedding_names=["name1", "name2"],
|
||||
enable_reembedding="true",
|
||||
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
|
||||
anndata_backed="true",
|
||||
config_file_name=self.config_file_name,
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
config.server_config.complete_config(self.context)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.dataset_config.handle_embeddings()
|
||||
|
||||
def test_handle_diffexp__raises_warning_for_large_datasets(self):
|
||||
config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
|
||||
config.server_config.complete_config(self.context)
|
||||
config.dataset_config.handle_diffexp(self.context)
|
||||
self.assertEqual(len(self.context["messages"]), 1)
|
||||
|
||||
def test_configfile_with_specialization(self):
|
||||
# test that per_dataset_config config load the default config, then the specialized config
|
||||
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
configfile = os.path.join(tempdir, "config.yaml")
|
||||
with open(configfile, "w") as fconfig:
|
||||
config = """
|
||||
server:
|
||||
single_dataset:
|
||||
datapath: fake_datapath
|
||||
dataset:
|
||||
user_annotations:
|
||||
enable: false
|
||||
type: local_file_csv
|
||||
local_file_csv:
|
||||
file: fake_file
|
||||
directory: fake_dir
|
||||
"""
|
||||
fconfig.write(config)
|
||||
|
||||
app_config = AppConfig()
|
||||
app_config.update_from_config_file(configfile)
|
||||
|
||||
test_config = app_config.dataset_config
|
||||
|
||||
# test config from default
|
||||
self.assertEqual(test_config.user_annotations__type, "local_file_csv")
|
||||
self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file")
|
||||
@@ -0,0 +1,216 @@
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import requests
|
||||
|
||||
from backend.common.errors import ConfigurationError
|
||||
from backend.server.common.config.app_config import AppConfig
|
||||
from backend.test.test_server.unit import test_server
|
||||
from backend.test import FIXTURES_ROOT
|
||||
from backend.common.utils.type_conversion_utils import convert_string_to_value
|
||||
from backend.test.test_server.unit.common.config import ConfigTests
|
||||
|
||||
|
||||
class TestExternalConfig(ConfigTests):
|
||||
def test_type_convert(self):
|
||||
# The values from environment variables and aws secrets are returned as strings.
|
||||
# These values need to be converted to the proper types.
|
||||
|
||||
self.assertEqual(convert_string_to_value("1"), int(1))
|
||||
self.assertEqual(convert_string_to_value("1.1"), float(1.1))
|
||||
self.assertEqual(convert_string_to_value("string"), "string")
|
||||
self.assertEqual(convert_string_to_value("true"), True)
|
||||
self.assertEqual(convert_string_to_value("True"), True)
|
||||
self.assertEqual(convert_string_to_value("false"), False)
|
||||
self.assertEqual(convert_string_to_value("False"), False)
|
||||
self.assertEqual(convert_string_to_value("null"), None)
|
||||
self.assertEqual(convert_string_to_value("None"), None)
|
||||
self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
|
||||
|
||||
def test_environment_variable(self):
|
||||
configfile = self.custom_external_config(
|
||||
environment=[
|
||||
dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
|
||||
dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
|
||||
],
|
||||
config_file_name="environment_external_config.yaml",
|
||||
)
|
||||
|
||||
env = os.environ
|
||||
env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
|
||||
env["DIFFEXP"] = "False"
|
||||
with test_server(command_line_args=["-c", configfile], env=env) as server:
|
||||
session = requests.Session()
|
||||
response = session.get(f"{server}/api/v0.2/config")
|
||||
data_config = response.json()
|
||||
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k-CSC-gz")
|
||||
self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
|
||||
|
||||
env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
|
||||
env["DIFFEXP"] = "True"
|
||||
with test_server(command_line_args=["-c", configfile], env=env) as server:
|
||||
session = requests.Session()
|
||||
response = session.get(f"{server}/api/v0.2/config")
|
||||
data_config = response.json()
|
||||
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
|
||||
self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
|
||||
|
||||
def test_environment_variable_errors(self):
|
||||
|
||||
# no name
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
|
||||
|
||||
# required has wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.environment = [
|
||||
dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
|
||||
|
||||
# no path
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.environment = [dict(name="myenvar", required=True)]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
|
||||
|
||||
# required environment variable is not set
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.environment = [
|
||||
dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
|
||||
|
||||
@patch("backend.server.common.config.external_config.get_secret_key")
|
||||
def test_aws_secrets_manager(self, mock_get_secret_key):
|
||||
mock_get_secret_key.return_value = {
|
||||
"flask_secret_key": "mock_flask_secret_key",
|
||||
}
|
||||
configfile = self.custom_external_config(
|
||||
aws_secrets_manager_region="us-west-2",
|
||||
aws_secrets_manager_secrets=[
|
||||
dict(
|
||||
name="my_secret",
|
||||
values=[
|
||||
dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=True),
|
||||
],
|
||||
)
|
||||
],
|
||||
config_file_name="secret_external_config.yaml",
|
||||
)
|
||||
|
||||
app_config = AppConfig()
|
||||
app_config.update_from_config_file(configfile)
|
||||
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
|
||||
|
||||
app_config.complete_config()
|
||||
|
||||
self.assertEqual(app_config.server_config.app__flask_secret_key, "mock_flask_secret_key")
|
||||
|
||||
@patch("backend.server.common.config.external_config.get_secret_key")
|
||||
def test_aws_secrets_manager_error(self, mock_get_secret_key):
|
||||
mock_get_secret_key.return_value = {
|
||||
"db_uri": "mock_db_uri",
|
||||
}
|
||||
|
||||
# no region
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = None
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(
|
||||
config_error.exception.message,
|
||||
"Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
|
||||
)
|
||||
|
||||
# missing secret name
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
|
||||
|
||||
# secret name wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
|
||||
|
||||
# missing values name
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
|
||||
|
||||
# values wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
|
||||
|
||||
# entry missing key
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
|
||||
|
||||
# entry required is wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
|
||||
|
||||
# entry missing path
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=[dict(key="db_uri", required=True)])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
|
||||
|
||||
# secret missing required key
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(
|
||||
name="mysecret",
|
||||
values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
|
||||
)
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
|
||||
@@ -0,0 +1,122 @@
|
||||
import os
|
||||
import unittest
|
||||
from unittest import mock
|
||||
from unittest.mock import patch
|
||||
|
||||
from backend.server.common.config.base_config import BaseConfig
|
||||
from backend.test import H5AD_FIXTURE
|
||||
|
||||
from backend.server.common.config.app_config import AppConfig
|
||||
from backend.common.errors import ConfigurationError
|
||||
from backend.test.test_server.unit.common.config import ConfigTests
|
||||
|
||||
|
||||
def mockenv(**envvars):
|
||||
return mock.patch.dict(os.environ, envvars)
|
||||
|
||||
|
||||
class TestServerConfig(ConfigTests):
|
||||
def setUp(self):
|
||||
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
self.server_config = self.config.server_config
|
||||
self.config.complete_config()
|
||||
|
||||
message_list = []
|
||||
|
||||
def noop(message):
|
||||
message_list.append(message)
|
||||
|
||||
messagefn = noop
|
||||
self.context = dict(messagefn=messagefn, messages=message_list)
|
||||
|
||||
def get_config(self, **kwargs):
|
||||
file_name = self.custom_app_config(
|
||||
dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
return config
|
||||
|
||||
def test_init_raises_error_if_default_config_is_invalid(self):
|
||||
invalid_config = self.get_config(port="not_valid")
|
||||
with self.assertRaises(ConfigurationError):
|
||||
invalid_config.complete_config()
|
||||
|
||||
@patch("backend.server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
|
||||
def test_complete_config_checks_all_attr(self, mock_check_attrs):
|
||||
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
|
||||
self.server_config.complete_config(self.context)
|
||||
self.assertEqual(mock_check_attrs.call_count, 20)
|
||||
|
||||
def test_handle_app__throws_error_if_port_doesnt_exist(self):
|
||||
config = self.get_config(port=99999999)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.handle_app(self.context)
|
||||
|
||||
@patch("backend.server.common.config.server_config.discover_s3_region_name")
|
||||
def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
|
||||
mock_discover_region_name.return_value = None
|
||||
# Default config
|
||||
self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
|
||||
# hard coded
|
||||
config = self.get_config()
|
||||
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
|
||||
# incorrectly formatted
|
||||
datapath = "s3://shouldnt/work"
|
||||
file_name = self.custom_app_config(
|
||||
dataset_datapath=datapath, config_file_name=self.config_file_name, data_locater_region_name="true"
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.handle_data_locator()
|
||||
|
||||
def test_handle_app___can_use_envar_port(self):
|
||||
config = self.get_config(port=24)
|
||||
self.assertEqual(config.server_config.app__port, 24)
|
||||
|
||||
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
|
||||
os.environ["CXG_SERVER_PORT"] = "4008"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.server_config.handle_app(self.context)
|
||||
self.assertEqual(self.config.server_config.app__port, 4008)
|
||||
del os.environ["CXG_SERVER_PORT"]
|
||||
|
||||
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
|
||||
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
|
||||
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
|
||||
|
||||
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
|
||||
config.external_config.handle_environment(self.context)
|
||||
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
|
||||
|
||||
def test_config_for_single_dataset(self):
|
||||
file_name = self.custom_app_config(
|
||||
config_file_name="single_dataset.yml", dataset_datapath=f"{H5AD_FIXTURE}"
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
config.server_config.handle_single_dataset(self.context)
|
||||
|
||||
file_name = self.custom_app_config(
|
||||
config_file_name="single_dataset_with_about.yml",
|
||||
about="www.cziscience.com",
|
||||
dataset_datapath=f"{H5AD_FIXTURE}",
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.handle_single_dataset(self.context)
|
||||
|
||||
def test_test_auth_only_in_insecure(self):
|
||||
|
||||
config = self.get_config(auth_type="test")
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.complete_config()
|
||||
|
||||
config.update_server_config(authentication__insecure_test_environment=True)
|
||||
config.complete_config()
|
||||
@@ -0,0 +1,823 @@
|
||||
import shutil
|
||||
import time
|
||||
import unittest
|
||||
import zlib
|
||||
from http import HTTPStatus
|
||||
import tempfile
|
||||
from os import path
|
||||
|
||||
import pandas as pd
|
||||
import requests
|
||||
|
||||
import backend.test.decode_fbs as decode_fbs
|
||||
from backend.server.data_common.matrix_loader import MatrixDataType
|
||||
from backend.test.test_server.unit import (
|
||||
data_with_tmp_annotations,
|
||||
make_fbs,
|
||||
start_test_server,
|
||||
stop_test_server,
|
||||
)
|
||||
from backend.test.fixtures.fixtures import pbmc3k_colors
|
||||
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
|
||||
|
||||
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
|
||||
|
||||
|
||||
# TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental
|
||||
|
||||
|
||||
class EndPoints(object):
|
||||
ANNOTATIONS_ENABLED = True
|
||||
GENESETS_READONLY = False
|
||||
|
||||
def test_initialize(self):
|
||||
endpoint = "schema"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
|
||||
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
|
||||
self.assertEqual(
|
||||
len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5
|
||||
)
|
||||
|
||||
def test_config(self):
|
||||
endpoint = "config"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertIn("library_versions", result_data["config"])
|
||||
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
|
||||
self.assertIsNotNone(result_data["config"]["parameters"])
|
||||
|
||||
def test_get_layout_fbs(self):
|
||||
endpoint = "layout/obs"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 8)
|
||||
self.assertIsNotNone(df["columns"])
|
||||
self.assertSetEqual(
|
||||
set(df["col_idx"]),
|
||||
{"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"},
|
||||
)
|
||||
self.assertIsNone(df["row_idx"])
|
||||
self.assertEqual(len(df["columns"]), df["n_cols"])
|
||||
|
||||
def test_put_layout_fbs(self):
|
||||
# first check that re-embedding is turned on
|
||||
result = self.session.get(f"{self.URL_BASE}config")
|
||||
config_data = result.json()
|
||||
re_embed = config_data["config"]["parameters"]["enable-reembedding"]
|
||||
if not re_embed:
|
||||
return
|
||||
# attempt to reembed with umap over 100 cells.
|
||||
endpoint = "layout/obs"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
data = {}
|
||||
data["filter"] = {}
|
||||
data["filter"]["obs"] = {}
|
||||
data["filter"]["obs"]["index"] = list(range(100))
|
||||
data["method"] = "umap"
|
||||
result = self.session.put(url, json=data)
|
||||
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
result_data = result.json()
|
||||
self.assertIsInstance(result_data, dict)
|
||||
self.assertEqual(result_data["type"], "float32")
|
||||
self.assertTrue(result_data["name"].startswith("reembed:umap_"))
|
||||
self.assertIsInstance(result_data["dims"], list)
|
||||
self.assertEqual(len(result_data["dims"]), 2)
|
||||
dims = result_data["dims"]
|
||||
self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
|
||||
self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
|
||||
|
||||
def test_bad_filter(self):
|
||||
endpoint = "data/var"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.put(url, json=BAD_FILTER)
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def test_get_annotations_obs_fbs(self):
|
||||
endpoint = "annotations/obs"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 6 if self.ANNOTATIONS_ENABLED else 5)
|
||||
self.assertIsNotNone(df["columns"])
|
||||
self.assertIsNone(df["row_idx"])
|
||||
self.assertEqual(len(df["columns"]), df["n_cols"])
|
||||
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
|
||||
self.assertCountEqual(
|
||||
df["col_idx"],
|
||||
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
|
||||
+ (["cluster-test"] if self.ANNOTATIONS_ENABLED else []),
|
||||
)
|
||||
|
||||
def test_get_annotations_obs_keys_fbs(self):
|
||||
endpoint = "annotations/obs"
|
||||
query = "annotation-name=n_genes&annotation-name=percent_mito"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 2)
|
||||
self.assertIsNotNone(df["columns"])
|
||||
self.assertIsNone(df["row_idx"])
|
||||
self.assertEqual(len(df["columns"]), df["n_cols"])
|
||||
self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"])
|
||||
|
||||
def test_get_annotations_obs_error(self):
|
||||
endpoint = "annotations/obs"
|
||||
query = "annotation-name=notakey"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def test_get_annotations_var_fbs(self):
|
||||
endpoint = "annotations/var"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 1838)
|
||||
self.assertEqual(df["n_cols"], 2)
|
||||
self.assertIsNotNone(df["columns"])
|
||||
self.assertIsNone(df["row_idx"])
|
||||
self.assertEqual(len(df["columns"]), df["n_cols"])
|
||||
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
|
||||
self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
|
||||
|
||||
def test_get_annotations_var_keys_fbs(self):
|
||||
endpoint = "annotations/var"
|
||||
query = "annotation-name=n_cells"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 1838)
|
||||
self.assertEqual(df["n_cols"], 1)
|
||||
self.assertIsNotNone(df["columns"])
|
||||
self.assertIsNone(df["row_idx"])
|
||||
self.assertEqual(len(df["columns"]), df["n_cols"])
|
||||
self.assertCountEqual(df["col_idx"], ["n_cells"])
|
||||
|
||||
def test_get_annotations_var_error(self):
|
||||
endpoint = "annotations/var"
|
||||
query = "annotation-name=notakey"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def test_data_mimetype_error(self):
|
||||
endpoint = "data/var"
|
||||
header = {"Accept": "xxx"}
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.put(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
|
||||
|
||||
def test_fbs_default(self):
|
||||
endpoint = "data/var"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.put(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
|
||||
result = self.session.put(url, json=filter)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
|
||||
def test_data_put_fbs(self):
|
||||
endpoint = "data/var"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.put(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def test_data_get_fbs(self):
|
||||
endpoint = "data/var"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def test_data_put_filter_fbs(self):
|
||||
endpoint = "data/var"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
|
||||
result = self.session.put(url, headers=header, json=filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 3)
|
||||
self.assertIsNotNone(df["columns"])
|
||||
self.assertIsNone(df["row_idx"])
|
||||
self.assertEqual(len(df["columns"]), df["n_cols"])
|
||||
self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
|
||||
|
||||
def test_data_get_filter_fbs(self):
|
||||
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
|
||||
endpoint = "data/var"
|
||||
query = f"var:{index_col_name}=SIK1"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 1)
|
||||
|
||||
def test_data_get_unknown_filter_fbs(self):
|
||||
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
|
||||
endpoint = "data/var"
|
||||
query = f"var:{index_col_name}=UNKNOWN"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 0)
|
||||
|
||||
def test_data_put_single_var(self):
|
||||
endpoint = "data/var"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
|
||||
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
|
||||
result = self.session.put(url, headers=header, json=var_filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 1)
|
||||
|
||||
def test_colors(self):
|
||||
endpoint = "colors"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(result_data, pbmc3k_colors)
|
||||
|
||||
def test_static(self):
|
||||
endpoint = "static"
|
||||
file = "assets/favicon.ico"
|
||||
url = f"{self.server}/{endpoint}/{file}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
|
||||
def test_genesets_config(self):
|
||||
result = self.session.get(f"{self.URL_BASE}config")
|
||||
config_data = result.json()
|
||||
params = config_data["config"]["parameters"]
|
||||
annotations_genesets = params["annotations_genesets"]
|
||||
annotations_genesets_readonly = params["annotations_genesets_readonly"]
|
||||
annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
|
||||
self.assertTrue(annotations_genesets)
|
||||
self.assertEqual(annotations_genesets_readonly, self.GENESETS_READONLY)
|
||||
self.assertEqual(annotations_genesets_summary_methods, ["mean"])
|
||||
|
||||
def test_get_genesets(self):
|
||||
endpoint = "genesets"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.get(url, headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertIsNotNone(result_data["genesets"])
|
||||
|
||||
def _setupClass(child_class, command_line):
|
||||
child_class.ps, child_class.server = start_test_server(command_line)
|
||||
child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
|
||||
child_class.session = requests.Session()
|
||||
for i in range(90):
|
||||
try:
|
||||
result = child_class.session.get(f"{child_class.URL_BASE}schema")
|
||||
child_class.schema = result.json()
|
||||
except requests.exceptions.ConnectionError:
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
class EndPointsAnnotations(EndPoints):
|
||||
def test_get_schema_existing_writable(self):
|
||||
self._test_get_schema_writable("cluster-test")
|
||||
|
||||
def test_get_user_annotations_existing_obs_keys_fbs(self):
|
||||
self._test_get_user_annotations_obs_keys_fbs(
|
||||
"cluster-test",
|
||||
{"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
|
||||
)
|
||||
|
||||
def test_put_user_annotations_obs_fbs(self):
|
||||
endpoint = "annotations/obs"
|
||||
query = "annotation-collection-name=test_annotations"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
n_rows = self.data.get_shape()[0]
|
||||
fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")})
|
||||
result = self.session.put(url, data=zlib.compress(fbs))
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
self.assertEqual(result.json(), {"status": "OK"})
|
||||
self._test_get_schema_writable("cat_A")
|
||||
self._test_get_user_annotations_obs_keys_fbs("cat_A", {"label_A"})
|
||||
|
||||
def _test_get_user_annotations_obs_keys_fbs(self, annotation_name, columns):
|
||||
endpoint = "annotations/obs"
|
||||
query = f"annotation-name={annotation_name}"
|
||||
url = f"{self.URL_BASE}{endpoint}?{query}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 1)
|
||||
self.assertListEqual(df["col_idx"], [annotation_name])
|
||||
self.assertEqual(set(df["columns"][0]), columns)
|
||||
self.assertIsNone(df["row_idx"])
|
||||
self.assertEqual(len(df["columns"]), df["n_cols"])
|
||||
|
||||
def _test_get_schema_writable(self, cluster_name):
|
||||
endpoint = "schema"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
columns = result_data["schema"]["annotations"]["obs"]["columns"]
|
||||
matching_columns = [c for c in columns if c["name"] == cluster_name]
|
||||
self.assertEqual(len(matching_columns), 1)
|
||||
self.assertTrue(matching_columns[0]["writable"])
|
||||
|
||||
|
||||
class EndPointsAnndata(unittest.TestCase, EndPoints):
|
||||
"""Test Case for endpoints"""
|
||||
|
||||
ANNOTATIONS_ENABLED = False
|
||||
GENESETS_READONLY = True
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls._setupClass(
|
||||
cls,
|
||||
[
|
||||
f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
|
||||
"--disable-annotations",
|
||||
"--disable-gene-sets-save",
|
||||
"--experimental-enable-reembedding",
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
stop_test_server(cls.ps)
|
||||
|
||||
@property
|
||||
def annotations_enabled(self):
|
||||
return False
|
||||
|
||||
def test_diff_exp(self):
|
||||
endpoint = "diffexp/obs"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
params = {
|
||||
"mode": "topN",
|
||||
"set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
|
||||
"set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
|
||||
"count": 7,
|
||||
}
|
||||
result = self.session.post(url, json=params)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(len(result_data), 7)
|
||||
|
||||
def test_diff_exp_indices(self):
|
||||
endpoint = "diffexp/obs"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
params = {
|
||||
"mode": "topN",
|
||||
"count": 10,
|
||||
"set1": {"filter": {"obs": {"index": [[0, 500]]}}},
|
||||
"set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
|
||||
}
|
||||
result = self.session.post(url, json=params)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertEqual(len(result_data), 10)
|
||||
|
||||
|
||||
class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
|
||||
"""Test Case for endpoints"""
|
||||
|
||||
ANNOTATIONS_ENABLED = True
|
||||
GENESETS_READONLY = False
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
|
||||
MatrixDataType.H5AD, annotations_fixture=True
|
||||
)
|
||||
cls._setupClass(cls, ["--annotations-file", cls.annotations.label_output_file, cls.data.get_location()])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
shutil.rmtree(cls.tmp_dir)
|
||||
stop_test_server(cls.ps)
|
||||
|
||||
|
||||
class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
|
||||
ANNOTATIONS_ENABLED = False
|
||||
GENESETS_READONLY = False
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp_dir = tempfile.mkdtemp()
|
||||
genesets_file = path.join(cls.tmp_dir, "test_genesets.csv")
|
||||
shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-genesets.csv", genesets_file)
|
||||
cls._setupClass(
|
||||
cls,
|
||||
[
|
||||
f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
|
||||
"--disable-annotations",
|
||||
"--gene-sets-file",
|
||||
genesets_file,
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
shutil.rmtree(cls.tmp_dir)
|
||||
stop_test_server(cls.ps)
|
||||
|
||||
def test_get_genesets_json(self):
|
||||
endpoint = "genesets"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.get(url, headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
result_data = result.json()
|
||||
self.assertIsNotNone(result_data["genesets"])
|
||||
self.assertIsNotNone(result_data["tid"])
|
||||
|
||||
self.assertEqual(
|
||||
result_data,
|
||||
{
|
||||
"genesets": [
|
||||
{
|
||||
"genes": [
|
||||
{"gene_description": "a gene_description", "gene_symbol": "F5"},
|
||||
{"gene_description": "", "gene_symbol": "SUMO3"},
|
||||
{"gene_description": "", "gene_symbol": "SRM"},
|
||||
],
|
||||
"geneset_description": "a description",
|
||||
"geneset_name": "first gene set name",
|
||||
},
|
||||
{
|
||||
"genes": [
|
||||
{"gene_description": "", "gene_symbol": "RER1"},
|
||||
{"gene_description": "", "gene_symbol": "SIK1"},
|
||||
],
|
||||
"geneset_description": "",
|
||||
"geneset_name": "second gene set",
|
||||
},
|
||||
{"genes": [], "geneset_description": "", "geneset_name": "third gene set"},
|
||||
{"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"},
|
||||
{"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
|
||||
{
|
||||
"genes": [
|
||||
{"gene_description": "", "gene_symbol": "ACD"},
|
||||
{"gene_description": "", "gene_symbol": "AATF"},
|
||||
{"gene_description": "", "gene_symbol": "F5"},
|
||||
{"gene_description": "", "gene_symbol": "PIGU"},
|
||||
],
|
||||
"geneset_description": "",
|
||||
"geneset_name": "summary test",
|
||||
},
|
||||
],
|
||||
"tid": 0,
|
||||
},
|
||||
)
|
||||
|
||||
def test_get_genesets_csv(self):
|
||||
endpoint = "genesets"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
result = self.session.get(url, headers={"Accept": "text/csv"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "text/csv")
|
||||
self.assertEqual(
|
||||
result.text,
|
||||
"""gene_set_name,gene_set_description,gene_symbol,gene_description\r
|
||||
first gene set name,a description,F5,a gene_description\r
|
||||
first gene set name,a description,SUMO3,\r
|
||||
first gene set name,a description,SRM,\r
|
||||
second gene set,,RER1,\r
|
||||
second gene set,,SIK1,\r
|
||||
third gene set,,,\r
|
||||
fourth_gene_set,fourth description,,\r
|
||||
fifth_dataset,,,\r
|
||||
summary test,,ACD,\r
|
||||
summary test,,AATF,\r
|
||||
summary test,,F5,\r
|
||||
summary test,,PIGU,\r
|
||||
""",
|
||||
)
|
||||
|
||||
def test_put_genesets(self):
|
||||
endpoint = "genesets"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
|
||||
# assume we start with TID 0
|
||||
result = self.session.get(url, headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.json()["tid"], 0)
|
||||
|
||||
test1 = {"tid": 3, "genesets": []}
|
||||
result = self.session.put(url, json=test1)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
result = self.session.get(url, headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.json(), test1)
|
||||
|
||||
# stale TID
|
||||
result = self.session.put(url, json=test1)
|
||||
self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
|
||||
|
||||
test2 = {
|
||||
"tid": 4,
|
||||
"genesets": [
|
||||
{"geneset_name": "foobar", "genes": []},
|
||||
{"geneset_name": "contains a space", "genes": []},
|
||||
{"geneset_name": "contains_weird_characters: #$%^&*()_+=-!@<>,./?';:\"[]{}|\\", "genes": []},
|
||||
],
|
||||
}
|
||||
test2_response = {
|
||||
"tid": 4,
|
||||
"genesets": [
|
||||
{"geneset_name": "foobar", "geneset_description": "", "genes": []},
|
||||
{"geneset_name": "contains a space", "geneset_description": "", "genes": []},
|
||||
{
|
||||
"geneset_name": "contains_weird_characters: #$%^&*()_+=-!@<>,./?';:\"[]{}|\\",
|
||||
"geneset_description": "",
|
||||
"genes": [],
|
||||
},
|
||||
],
|
||||
}
|
||||
result = self.session.put(url, json=test2)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
result = self.session.get(url, headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.json(), test2_response)
|
||||
|
||||
test3 = {
|
||||
"tid": 5,
|
||||
"genesets": [
|
||||
{
|
||||
"geneset_name": "foobar",
|
||||
"geneset_description": "",
|
||||
"genes": [
|
||||
{
|
||||
"gene_symbol": "F5",
|
||||
"gene_description": "",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
result = self.session.put(url, json=test3)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
result = self.session.get(url, headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.json(), test3)
|
||||
|
||||
def test_put_genesets_malformed(self):
|
||||
""" test malformed submissions that we expect the backend to catch/tolerate """
|
||||
endpoint = "genesets"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
|
||||
result = self.session.get(url, headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
original_data = result.json()
|
||||
tid = original_data["tid"]
|
||||
|
||||
def test_case(test, expected_code, original_data):
|
||||
""" check for expected error AND that no change was made to the original state """
|
||||
result = self.session.put(url, json=test)
|
||||
self.assertEqual(result.status_code, expected_code)
|
||||
result = self.session.get(url, headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.json(), original_data)
|
||||
|
||||
# missing or malformed genesets
|
||||
test_case(
|
||||
{"tid": tid + 1},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
test_case(
|
||||
{"tid": tid + 1, "genesets": 99},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
|
||||
# illegal geneset_name
|
||||
test_case(
|
||||
{"tid": tid + 1, "genesets": [{"geneset_name": " foo", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
test_case(
|
||||
{"tid": tid + 1, "genesets": [{"geneset_name": "foo ", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
test_case(
|
||||
{"tid": tid + 1, "genesets": [{"geneset_name": "f oo", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
test_case(
|
||||
{"tid": tid + 1, "genesets": [{"geneset_name": "f\too", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
test_case(
|
||||
{"tid": tid + 1, "genesets": [{"geneset_name": "f\roo", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
test_case(
|
||||
{"tid": tid + 1, "genesets": [{"geneset_name": "f\noo", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
test_case(
|
||||
{"tid": tid + 1, "genesets": [{"geneset_name": "f\voo", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
|
||||
# duplicate geneset_name
|
||||
test_case(
|
||||
{
|
||||
"tid": tid + 1,
|
||||
"genesets": [
|
||||
{"geneset_name": "foo", "genes": []},
|
||||
{"geneset_name": "foo", "genes": []},
|
||||
],
|
||||
},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
|
||||
# missing geneset_name
|
||||
test_case(
|
||||
{"tid": tid + 1, "genesets": [{"genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
|
||||
# non-numeric TID
|
||||
test_case(
|
||||
{"tid": [], "genesets": [{"geneset_name": "foo", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
test_case(
|
||||
{"tid": None, "genesets": [{"geneset_name": "foo", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
test_case(
|
||||
{"tid": "not a number", "genesets": [{"geneset_name": "foo", "genes": []}]},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
|
||||
# duplicate gene_symbol
|
||||
test_case(
|
||||
{
|
||||
"tid": "not a number",
|
||||
"genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": "SIK1"}, {"gene_symbol": "SIK1"}]}],
|
||||
},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
|
||||
# gene_symbol is not a string
|
||||
test_case(
|
||||
{
|
||||
"tid": "not a number",
|
||||
"genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": 99}]}],
|
||||
},
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
original_data,
|
||||
)
|
||||
|
||||
def test_get_geneset_summary(self):
|
||||
endpoint = "geneset_summary?geneset_name=summary%20test&method=mean"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 1)
|
||||
self.assertEqual(df["col_idx"], ["summary test"])
|
||||
self.assertAlmostEqual(df["columns"][0][0], -0.19863907)
|
||||
|
||||
def test_get_geneset_summary_default_method(self):
|
||||
endpoint = "geneset_summary?geneset_name=summary%20test"
|
||||
url = f"{self.URL_BASE}{endpoint}"
|
||||
header = {"Accept": "application/octet-stream"}
|
||||
result = self.session.get(url, headers=header)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 1)
|
||||
self.assertEqual(df["col_idx"], ["summary test"])
|
||||
self.assertAlmostEqual(df["columns"][0][0], -0.19863907)
|
||||
|
||||
def test_get_geneset_summary_check_tid(self):
|
||||
# get the TID
|
||||
result = self.session.get(f"{self.URL_BASE}genesets", headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
tid = result.json()["tid"]
|
||||
|
||||
# current tid
|
||||
endpoint = f"geneset_summary?geneset_name=summary%20test&tid={tid}"
|
||||
result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
|
||||
# future tid
|
||||
endpoint = f"geneset_summary?geneset_name=summary%20test&tid={tid+1}"
|
||||
result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
|
||||
|
||||
# past tid
|
||||
endpoint = f"geneset_summary?geneset_name=summary%20test&tid={tid-1}"
|
||||
result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
|
||||
|
||||
# No tid - ie, skip check
|
||||
endpoint = "geneset_summary?geneset_name=summary%20test"
|
||||
result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
|
||||
def test_get_geneset_summary_edge_cases(self):
|
||||
# attempt to summarize _all_ genesets, including edge cases with zero or one gene
|
||||
result = self.session.get(f"{self.URL_BASE}genesets", headers={"Accept": "application/json"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
geneset_names = [gs["geneset_name"] for gs in result.json()["genesets"]]
|
||||
|
||||
for gs in geneset_names:
|
||||
endpoint = f"geneset_summary?geneset_name={gs}"
|
||||
result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertEqual(df["n_rows"], 2638)
|
||||
self.assertEqual(df["n_cols"], 1)
|
||||
self.assertEqual(df["col_idx"], [gs])
|
||||
|
||||
def test_get_geneset_error_handling(self):
|
||||
# no geneset
|
||||
endpoint = "geneset_summary"
|
||||
result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
# unknown geneset
|
||||
endpoint = "geneset_summary?geneset_name=NO_SUCH_GENE_SET"
|
||||
result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
|
||||
# unknown method
|
||||
endpoint = "geneset_summary?geneset_name=summary%20test&method=NO_SUCH_METHOD"
|
||||
result = self.session.get(f"{self.URL_BASE}{endpoint}", headers={"Accept": "application/octet-stream"})
|
||||
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
|
||||
@@ -0,0 +1,165 @@
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from http import HTTPStatus
|
||||
|
||||
import anndata
|
||||
import requests
|
||||
|
||||
from backend.server.common.corpora import (
|
||||
corpora_get_versions_from_anndata,
|
||||
corpora_is_version_supported,
|
||||
corpora_get_props_from_anndata,
|
||||
)
|
||||
from backend.test.test_server.unit import start_test_server, stop_test_server
|
||||
from backend.test import PROJECT_ROOT
|
||||
|
||||
VERSION = "v0.2"
|
||||
|
||||
|
||||
class CorporaAPITest(unittest.TestCase):
|
||||
def test_corpora_get_versions_from_anndata(self):
|
||||
adata = self._get_h5ad()
|
||||
|
||||
if "version" in adata.uns:
|
||||
del adata.uns["version"]
|
||||
self.assertIsNone(corpora_get_versions_from_anndata(adata))
|
||||
|
||||
# something bogus
|
||||
adata.uns["version"] = 99
|
||||
self.assertIsNone(corpora_get_versions_from_anndata(adata))
|
||||
|
||||
# something legit
|
||||
adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
|
||||
self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
|
||||
|
||||
def test_corpora_is_version_supported(self):
|
||||
self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
|
||||
self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
|
||||
self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
|
||||
|
||||
def test_corpora_get_props_from_anndata(self):
|
||||
adata = self._get_h5ad()
|
||||
|
||||
if "version" in adata.uns:
|
||||
del adata.uns["version"]
|
||||
self.assertIsNone(corpora_get_props_from_anndata(adata))
|
||||
|
||||
# something bogus
|
||||
adata.uns["version"] = 99
|
||||
self.assertIsNone(corpora_get_props_from_anndata(adata))
|
||||
|
||||
# unsupported version, but missing required values
|
||||
adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
|
||||
with self.assertRaises(ValueError):
|
||||
corpora_get_props_from_anndata(adata)
|
||||
|
||||
# legit version, but missing required values
|
||||
adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
|
||||
with self.assertRaises(KeyError):
|
||||
corpora_get_props_from_anndata(adata)
|
||||
|
||||
some_fields = {
|
||||
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
|
||||
"title": "title",
|
||||
"layer_descriptions": "layer_descriptions",
|
||||
"organism": "organism",
|
||||
"organism_ontology_term_id": "organism_ontology_term_id",
|
||||
"project_name": "project_name",
|
||||
"project_description": "project_description",
|
||||
"contributors": json.dumps([{"contributors": "contributors"}]),
|
||||
"project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
|
||||
}
|
||||
for k in some_fields:
|
||||
adata.uns[k] = some_fields[k]
|
||||
some_fields["contributors"] = json.loads(some_fields["contributors"])
|
||||
some_fields["project_links"] = json.loads(some_fields["project_links"])
|
||||
self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
|
||||
|
||||
def test_corpora_get_props_from_anndata_v110(self):
|
||||
adata = self._get_h5ad()
|
||||
|
||||
if "version" in adata.uns:
|
||||
del adata.uns["version"]
|
||||
self.assertIsNone(corpora_get_props_from_anndata(adata))
|
||||
|
||||
# legit version, but missing required values
|
||||
adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
|
||||
with self.assertRaises(KeyError):
|
||||
corpora_get_props_from_anndata(adata)
|
||||
|
||||
# Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
|
||||
some_110_fields = {
|
||||
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
|
||||
"title": "title",
|
||||
"layer_descriptions": "layer_descriptions",
|
||||
"organism": "organism",
|
||||
"organism_ontology_term_id": "organism_ontology_term_id",
|
||||
}
|
||||
for k in some_110_fields:
|
||||
adata.uns[k] = some_110_fields[k]
|
||||
self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
|
||||
|
||||
def _get_h5ad(self):
|
||||
return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
|
||||
|
||||
|
||||
class CorporaRESTAPITest(unittest.TestCase):
|
||||
""" Confirm endpoints reflect Corpora-specific features """
|
||||
|
||||
@classmethod
|
||||
def setCorporaFields(cls, path):
|
||||
adata = anndata.read_h5ad(path)
|
||||
corpora_props = {
|
||||
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
|
||||
"title": "PBMC3K",
|
||||
"contributors": json.dumps([{"name": "name"}]),
|
||||
"layer_descriptions": {"X": "raw counts"},
|
||||
"organism": "human",
|
||||
"organism_ontology_term_id": "unknown",
|
||||
"project_name": "test project",
|
||||
"project_description": "test description",
|
||||
"project_links": json.dumps(
|
||||
[{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
|
||||
),
|
||||
"default_embedding": "X_tsne",
|
||||
}
|
||||
adata.uns.update(corpora_props)
|
||||
adata.write(path)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp_dir = tempfile.TemporaryDirectory()
|
||||
src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
|
||||
dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
|
||||
shutil.copyfile(src, dst)
|
||||
cls.setCorporaFields(dst)
|
||||
cls.ps, cls.server = start_test_server([dst])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
stop_test_server(cls.ps)
|
||||
cls.tmp_dir.cleanup()
|
||||
|
||||
def setUp(self):
|
||||
self.session = requests.Session()
|
||||
self.url_base = f"{self.server}/api/{VERSION}/"
|
||||
|
||||
def test_config(self):
|
||||
endpoint = "config"
|
||||
url = f"{self.url_base}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/json")
|
||||
|
||||
result_data = result.json()
|
||||
self.assertIsInstance(result_data["config"]["corpora_props"], dict)
|
||||
self.assertIsInstance(result_data["config"]["parameters"], dict)
|
||||
|
||||
corpora_props = result_data["config"]["corpora_props"]
|
||||
parameters = result_data["config"]["parameters"]
|
||||
|
||||
self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
|
||||
self.assertEqual(corpora_props["organism"], "human")
|
||||
self.assertEqual(parameters["default_embedding"], "tsne")
|
||||
@@ -0,0 +1,61 @@
|
||||
from http import HTTPStatus
|
||||
import unittest
|
||||
import math
|
||||
from backend.test.test_server.unit import start_test_server, stop_test_server
|
||||
from backend.test import FIXTURES_ROOT
|
||||
import backend.test.decode_fbs as decode_fbs
|
||||
|
||||
import requests
|
||||
|
||||
VERSION = "v0.2"
|
||||
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
|
||||
|
||||
|
||||
class WithNaNs(unittest.TestCase):
|
||||
"""Test Case for endpoints"""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.ps, cls.server = start_test_server([f"{FIXTURES_ROOT}/nan.h5ad"])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
stop_test_server(cls.ps)
|
||||
|
||||
def setUp(self):
|
||||
self.session = requests.Session()
|
||||
self.url_base = f"{self.server}/api/{VERSION}/"
|
||||
|
||||
def test_initialize(self):
|
||||
endpoint = "schema"
|
||||
url = f"{self.url_base}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
|
||||
def test_data(self):
|
||||
endpoint = "data/var"
|
||||
url = f"{self.url_base}{endpoint}"
|
||||
filter = {"filter": {"var": {"index": [[0, 20]]}}}
|
||||
result = self.session.put(url, json=filter)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertTrue(math.isnan(df["columns"][3][3]))
|
||||
|
||||
def test_annotation_obs(self):
|
||||
endpoint = "annotations/obs"
|
||||
url = f"{self.url_base}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertTrue(math.isnan(df["columns"][2][0]))
|
||||
|
||||
def test_annotation_var(self):
|
||||
endpoint = "annotations/var"
|
||||
url = f"{self.url_base}{endpoint}"
|
||||
result = self.session.get(url)
|
||||
self.assertEqual(result.status_code, HTTPStatus.OK)
|
||||
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
|
||||
df = decode_fbs.decode_matrix_FBS(result.content)
|
||||
self.assertTrue(math.isnan(df["columns"][2][0]))
|
||||
@@ -0,0 +1,79 @@
|
||||
import unittest
|
||||
from urllib.parse import parse_qs
|
||||
from werkzeug.datastructures import MultiDict
|
||||
from backend.common.errors import FilterError
|
||||
from backend.server.common.rest import _query_parameter_to_filter
|
||||
|
||||
|
||||
def _qsparse(qs):
|
||||
""" emulate what Flask/Werkzeug do to our QS """
|
||||
return MultiDict(parse_qs(qs))
|
||||
|
||||
|
||||
class FilterParseTests(unittest.TestCase):
|
||||
""" Test cases for various filter parsing """
|
||||
|
||||
def test_queryparam_to_filter_parse(self):
|
||||
# categories
|
||||
self.assertEqual(
|
||||
_query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
|
||||
{
|
||||
"obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
|
||||
"var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
|
||||
},
|
||||
)
|
||||
|
||||
# ranges
|
||||
self.assertEqual(
|
||||
_query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
|
||||
{
|
||||
"obs": {
|
||||
"annotation_value": [
|
||||
{"name": "A", "min": 1, "max": 99.0},
|
||||
{"name": "B", "max": 100.0},
|
||||
{"name": "C", "min": 0.0},
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
# combo
|
||||
self.assertEqual(
|
||||
_query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
|
||||
{
|
||||
"var": {
|
||||
"annotation_value": [
|
||||
{"name": "B", "values": ["YES", "NO"]},
|
||||
{"name": "A", "min": 1.0, "max": 99.0},
|
||||
]
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def test_queryparam_to_filter_escaping(self):
|
||||
self.assertEqual(
|
||||
_query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
|
||||
{"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
|
||||
)
|
||||
|
||||
def test_queryparam_to_filter_errors(self):
|
||||
|
||||
# should raise FilterError
|
||||
filter_errors = [
|
||||
"foo=bar", # no axis
|
||||
"X=&Y=3", # no value
|
||||
"X&Y=3", # no value
|
||||
"moo:foo=bar", # bad axis
|
||||
"obs:x=1,A", # non-numeric range
|
||||
"var:X=1,2&var:X=3,4", # duplicate ranges
|
||||
"var:Y=,",
|
||||
"var:Y=2,",
|
||||
"var:Y=,5",
|
||||
"var:Y=*,",
|
||||
"var:Y=,*",
|
||||
"var:Y=*,*",
|
||||
]
|
||||
|
||||
for qs in filter_errors:
|
||||
with self.assertRaises(FilterError):
|
||||
_query_parameter_to_filter(_qsparse(qs))
|
||||
@@ -0,0 +1,168 @@
|
||||
import json
|
||||
import shutil
|
||||
import unittest
|
||||
from os import path, listdir
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
import backend.test.decode_fbs as decode_fbs
|
||||
from backend.server.common.rest import annotations_put_fbs_helper, schema_get_helper
|
||||
from backend.server.data_common.matrix_loader import MatrixDataType
|
||||
from backend.test.test_server.unit import data_with_tmp_annotations, make_fbs
|
||||
|
||||
|
||||
class WritableAnnotationTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.data, self.tmp_dir, self.annotations = data_with_tmp_annotations(MatrixDataType.H5AD)
|
||||
self.data.dataset_config.user_annotations = self.annotations
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp_dir)
|
||||
|
||||
def annotation_put_fbs(self, fbs):
|
||||
annotations_put_fbs_helper(self.data, fbs)
|
||||
res = json.dumps({"status": "OK"})
|
||||
return res
|
||||
|
||||
def test_error_checks(self):
|
||||
# verify that the expected errors are generated
|
||||
n_rows = self.data.get_shape()[0]
|
||||
fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
|
||||
|
||||
# ensure we catch attempt to overwrite non-writable data
|
||||
with self.assertRaises(KeyError):
|
||||
self.annotation_put_fbs(fbs_bad)
|
||||
|
||||
def test_write_to_file(self):
|
||||
# verify the file is written as expected
|
||||
n_rows = self.data.get_shape()[0]
|
||||
fbs = make_fbs(
|
||||
{
|
||||
"cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
|
||||
"cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
|
||||
}
|
||||
)
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(res, json.dumps({"status": "OK"}))
|
||||
self.assertTrue(path.exists(self.annotations.label_output_file))
|
||||
df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#")
|
||||
self.assertEqual(df.shape, (n_rows, 2))
|
||||
self.assertEqual(set(df.columns), {"cat_A", "cat_B"})
|
||||
self.assertTrue(self.data.original_obs_index.equals(df.index))
|
||||
self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows))
|
||||
self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows))
|
||||
|
||||
# verify complete overwrite on second attempt, AND rotation occurs
|
||||
fbs = make_fbs(
|
||||
{
|
||||
"cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"),
|
||||
"cat_C": pd.Series(["label_C"] * n_rows, dtype="category"),
|
||||
}
|
||||
)
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(res, json.dumps({"status": "OK"}))
|
||||
self.assertTrue(path.exists(self.annotations.label_output_file))
|
||||
df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#")
|
||||
self.assertEqual(set(df.columns), {"cat_A", "cat_C"})
|
||||
self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows))
|
||||
self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows))
|
||||
|
||||
# rotation
|
||||
name, ext = path.splitext(self.annotations.label_output_file)
|
||||
backup_dir = f"{name}-backups"
|
||||
self.assertTrue(path.isdir(backup_dir))
|
||||
found_files = listdir(backup_dir)
|
||||
self.assertEqual(len(found_files), 1)
|
||||
|
||||
def test_file_rotation_to_max_9(self):
|
||||
# verify we stop rotation at 9
|
||||
n_rows = self.data.get_shape()[0]
|
||||
fbs = make_fbs(
|
||||
{
|
||||
"cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
|
||||
"cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
|
||||
}
|
||||
)
|
||||
for i in range(0, 11):
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(res, json.dumps({"status": "OK"}))
|
||||
|
||||
name, ext = path.splitext(self.annotations.label_output_file)
|
||||
backup_dir = f"{name}-backups"
|
||||
self.assertTrue(path.isdir(backup_dir))
|
||||
found_files = listdir(backup_dir)
|
||||
self.assertTrue(len(found_files) <= 9)
|
||||
|
||||
def test_put_get_roundtrip(self):
|
||||
# verify that OBS PUTs (annotation_put_fbs) are accessible via
|
||||
# GET (annotation_to_fbs_matrix)
|
||||
|
||||
n_rows = self.data.get_shape()[0]
|
||||
fbs = make_fbs(
|
||||
{
|
||||
"cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
|
||||
"cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
|
||||
}
|
||||
)
|
||||
|
||||
# put
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(res, json.dumps({"status": "OK"}))
|
||||
|
||||
# get
|
||||
labels = self.annotations.read_labels(None)
|
||||
fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
|
||||
schema = schema_get_helper(self.data)
|
||||
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
|
||||
obs_index_col_name = schema["annotations"]["obs"]["index"]
|
||||
self.assertEqual(annotations["n_rows"], n_rows)
|
||||
self.assertEqual(annotations["n_cols"], 7)
|
||||
self.assertIsNone(annotations["row_idx"])
|
||||
self.assertEqual(
|
||||
annotations["col_idx"],
|
||||
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
|
||||
)
|
||||
col_idx = annotations["col_idx"]
|
||||
self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows)
|
||||
self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows)
|
||||
|
||||
# verify the schema was updated
|
||||
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
|
||||
self.assertEqual(
|
||||
all_col_schema["cat_A"],
|
||||
{"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
|
||||
)
|
||||
self.assertEqual(
|
||||
all_col_schema["cat_B"],
|
||||
{"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
|
||||
)
|
||||
|
||||
def test_put_float_data(self):
|
||||
# verify that OBS PUTs (annotation_put_fbs) are accessible via
|
||||
# GET (annotation_to_fbs_matrix)
|
||||
|
||||
n_rows = self.data.get_shape()[0]
|
||||
|
||||
# verifies that floating point with decimals fail.
|
||||
fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))})
|
||||
with self.assertRaises(ValueError) as exception_context:
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(str(exception_context.exception), "Columns may not have floating point types")
|
||||
|
||||
# verifies that floating point that can be converted to int passes
|
||||
fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")})
|
||||
res = self.annotation_put_fbs(fbs)
|
||||
self.assertEqual(res, json.dumps({"status": "OK"}))
|
||||
|
||||
# check read_labels
|
||||
labels = self.annotations.read_labels(None)
|
||||
fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
|
||||
schema = schema_get_helper(self.data)
|
||||
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
|
||||
self.assertEqual(annotations["n_rows"], n_rows)
|
||||
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
|
||||
self.assertEqual(
|
||||
all_col_schema["cat_F_PASS"],
|
||||
{"name": "cat_F_PASS", "type": "int32", "writable": True},
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
import os
|
||||
import shutil
|
||||
import unittest
|
||||
|
||||
from backend.common.utils.utils import import_plugins
|
||||
from backend.test import PROJECT_ROOT, random_string
|
||||
|
||||
|
||||
class TestPlugins(unittest.TestCase):
|
||||
""" Test plugin import functionality """
|
||||
|
||||
plugins_dir = f"{PROJECT_ROOT}/backend/test/plugins"
|
||||
test_plugin_path = f"{plugins_dir}/foo.py"
|
||||
secret = random_string(8)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
if not os.path.isdir(cls.plugins_dir):
|
||||
os.mkdir(cls.plugins_dir)
|
||||
with open(cls.test_plugin_path, "w") as fh:
|
||||
fh.write(f'SECRET = "{cls.secret}"\n')
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
if os.path.isdir(cls.plugins_dir):
|
||||
shutil.rmtree(cls.plugins_dir)
|
||||
|
||||
def test_import_plugins(self):
|
||||
self.assertTrue(os.path.isfile(self.test_plugin_path))
|
||||
loaded_modules = import_plugins("backend.test.plugins")
|
||||
# test that import plugins found the file
|
||||
self.assertEqual(["backend.test.plugins.foo"], [ele.__name__ for ele in loaded_modules])
|
||||
# test that the module was properly executed
|
||||
self.assertEqual(self.secret, loaded_modules[0].SECRET)
|
||||
Reference in New Issue
Block a user