fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-04 22:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Dunitz/2308-remove hosted (#2399)

Browse Source 
* remove code for and references to czi-hosted

* revert client change
This commit is contained in:
Madison Dunitz
2021-09-02 13:18:35 -04:00
committed by GitHub
parent 780852fd49
commit f9c744327a
154 changed files with 6 additions and 15809 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/test/fixtures/czi_hosted_dataset_config_outline.py
Vendored
-35
View File
@@ -1,35 +0,0 @@
f"""
dataset:
app:
scripts: {scripts} #list of strs (filenames) or dicts containing keys
inline_scripts: {inline_scripts} #list of strs (filenames)
about_legal_tos: {about_legal_tos}
about_legal_privacy: {about_legal_privacy}
authentication_enable: {authentication_enable}
presentation:
max_categories: {max_categories}
custom_colors: {custom_colors}
user_annotations:
enable: {enable_users_annotations}
type: {annotation_type}
hosted_tiledb_array:
db_uri: {db_uri}
hosted_file_directory: {hosted_file_directory}
local_file_csv:
directory: {local_file_csv_directory}
file: {local_file_csv_file}
embeddings:
names: {embedding_names}
diffexp:
enable: {enable_difexp}
lfc_cutoff: {lfc_cutoff}
top_n: {top_n}
X_approximate_distribution: {X_approximate_distribution}
"""
backend/test/fixtures/czi_hosted_server_config_outline.py
Vendored
-63
View File
@@ -1,63 +0,0 @@
f"""server:
app:
verbose: {verbose}
debug: {debug}
host: {host}
port: {port}
open_browser: {open_browser}
force_https: {force_https}
flask_secret_key: {flask_secret_key}
generate_cache_control_headers: {generate_cache_control_headers}
server_timing_headers: {server_timing_headers}
csp_directives: {csp_directives}
api_base_url: {api_base_url}
web_base_url: {web_base_url}
authentication:
type: {auth_type}
insecure_test_environment: {insecure_test_environment}
params_oauth:
oauth_api_base_url: {oauth_api_base_url}
client_id: {client_id}
client_secret: {client_secret}
jwt_decode_options: {jwt_decode_options}
session_cookie: {session_cookie}
cookie: {cookie}
multi_dataset:
dataroot: {dataroot}
index: {index}
allowed_matrix_types: {allowed_matrix_types}
matrix_cache:
max_datasets: {max_cached_datasets}
timelimit_s: {timelimit_s}
single_dataset:
datapath: {dataset_datapath}
obs_names: {obs_names}
var_names: {var_names}
about: {about}
title: {title}
diffexp:
alg_cxg: # number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count)
max_workers: {diffexp_max_workers}
cpu_multiplier: {cpu_multiplier}
target_workunit: {target_workunit} # The target number of matrix elements that are evaluated in one thread.
data_locator:
s3:
region_name: {data_locater_region_name}
adaptor:
cxg_adaptor:
tiledb_ctx:
sm.tile_cache_size: {cxg_tile_cache_size}
sm.num_reader_threads: {cxg_num_reader_threads}
anndata_adaptor:
backed: {anndata_backed}
limits:
column_request_max: {column_request_max}
diffexp_cellcount_max: {diffexp_cellcount_max}
"""
backend/test/fixtures/database/__init__.py
Vendored
-79
View File
@@ -1,79 +0,0 @@
import string
import random
from sqlalchemy import func
from backend.czi_hosted.db.cellxgene_orm import CellxGeneUser, CellxGeneDataset, Annotation, Base
from backend.czi_hosted.db.create_db import create_db
from backend.czi_hosted.db.db_utils import DbUtils
class TestDatabase:
def __init__(self):
local_db_uri = "postgresql://postgres:test_pw@localhost:5432"
create_db(local_db_uri)
self.db = DbUtils(local_db_uri)
self._populate_test_data()
self._populate_test_data_many()
def _populate_test_data(self):
self._create_test_user()
self._create_test_dataset()
self._create_test_annotation()
def _populate_test_data_many(self):
self._create_test_users()
self._create_test_datasets()
self._create_test_annotations()
def _create_test_user(self):
user = CellxGeneUser(id="test_user_id")
user2 = CellxGeneUser(id="1234")
self.db.session.add(user)
self.db.session.add(user2)
self.db.session.commit()
def _create_test_dataset(self):
dataset = CellxGeneDataset(name="test_dataset",)
self.db.session.add(dataset)
self.db.session.commit()
def _create_test_annotation(self):
dataset = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == "test_dataset"],)[0]
annotation = Annotation(tiledb_uri="tiledb_uri", user_id="test_user_id", dataset_id=str(dataset.id))
self.db.session.add(annotation)
self.db.session.commit()
@staticmethod
def get_random_string():
letters = string.ascii_lowercase
return "".join(random.choice(letters) for i in range(12))
def _create_test_users(self, user_count: int = 10):
users = []
for i in range(user_count):
users.append(CellxGeneUser(id=self.get_random_string()))
self.db.session.add_all(users)
self.db.session.commit()
def _create_test_datasets(self, dataset_count: int = 10):
datasets = []
for i in range(dataset_count):
datasets.append(CellxGeneDataset(name=self.get_random_string()))
self.db.session.add_all(datasets)
self.db.session.commit()
def order_by_random(self, table: Base):
return self.db.session.query(table).order_by(func.random()).first()
def _create_test_annotations(self, annotation_count: int = 10):
annotations = []
for i in range(annotation_count):
dataset = self.order_by_random(CellxGeneDataset)
user = self.order_by_random(CellxGeneUser)
annotations.append(
Annotation(tiledb_uri=self.get_random_string(), user_id=user.id, dataset_id=str(dataset.id))
)
self.db.session.add_all(annotations)
self.db.session.commit()
backend/test/test_czi_hosted/__init__.py
View File
backend/test/test_czi_hosted/locust/README.md
-39
View File
@@ -1,39 +0,0 @@
# Locust Load Test
This directory contains scripts to load test cellxgene's backend. It
primary simulates initial data loading and expression data fetch, which
are the most common data routes. It currently does not include tests
for differential expression or re-clustering routes.
## Prerequisites
You need:
- Python 3.6+, and pip
- cellxgene installed
- install the locust dependencies in `requirements-locust.txt`
## To test
1. Choose to run cellxgene in either single dataset or data root mode.
2. Edit config.py to indicate which datasets to load:
- in single dataset mode, just set `DataSets=[""]`
- in dataroot (multi-dataset) mode, add the route names, eg, `DataSets=['foo.cxg', 'bar.cxg']`
3. Launch cellxgene in the appropriate mode
4. launch locust, specifying the correct --host argument
5. point your web browser to the locust http server, usually `http://localhost:8089/`
### Single dataset mode
- Edit config.py and set `DataSets=[""]`
- in a shell, run `cellxgene launch somefile.h5ad`
- launch locust in another shell, `locust --host http://localhost:5005/` (or wherever you are running cellxgene)
- point a browser to the locust port, usually http://localhost:8089/
- run test
### Multi-dataset mode
- Edit config.py and set `DataSets=["datapath1", ...]`
- in a shell, run `cellxgene launch --dataroot path`
The remainder of the steps are same as single dataset.
backend/test/test_czi_hosted/locust/__init__.py
View File
backend/test/test_czi_hosted/locust/config.py
-15
View File
@@ -1,15 +0,0 @@
"""
Locust test config
"""
""" Data routes that will be tested """
# single dataset, for non-dataroot tests
# DataSets = [""]
# multi-dataset, for dataroot tests. these are varied in size/shape
DataSets = [
"GSE60361.cxg",
"WongAdultRetina.cxg",
]
backend/test/test_czi_hosted/locust/locustfile.py
-165
View File
@@ -1,165 +0,0 @@
import json
import random
import requests
from config import DataSets
from locust import HttpUser, SequentialTaskSet, task, between, TaskSet
from locust.clients import HttpSession
from requests.packages.urllib3.exceptions import InsecureRequestWarning
import backend.test.decode_fbs as decode_fbs
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
"""
Simple locust stress test definition for cellxgene
"""
API_SUFFIX = "api/v0.2"
class CellXGeneTasks(TaskSet):
"""
Simulate use against a single dataset
"""
def on_start(self):
self.client.verify = False
self.dataset = random.choice(DataSets)
with self.client.get(
f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True
) as schema_response:
if schema_response.status_code == 200:
self.schema = schema_response.json()["schema"]
else:
self.schema = None
with self.client.get(
f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True
) as config_response:
if config_response.status_code == 200:
self.config = config_response.json()["config"]
else:
self.config = None
with self.client.get(
f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.var_index_name()}",
headers={"Accept": "application/octet-stream"},
catch_response=True,
) as var_index_response:
if var_index_response.status_code == 200:
df = decode_fbs.decode_matrix_FBS(var_index_response.content)
gene_names_idx = df["col_idx"].index(self.var_index_name())
self.gene_names = df["columns"][gene_names_idx]
else:
self.gene_names = []
def var_index_name(self):
if self.schema is not None:
return self.schema["annotations"]["var"]["index"]
return None
def obs_annotation_names(self):
if self.schema is not None:
return [col["name"] for col in self.schema["annotations"]["obs"]["columns"]]
return []
def layout_names(self):
if self.schema is not None:
return [layout["name"] for layout in self.schema["layout"]["obs"]]
else:
return []
@task(2)
class InitializeClient(SequentialTaskSet):
"""
Initial loading of cellxgene - when the user hits the main route.
Currently this sequence skips some of the static assets, which are quite small and should be served by the
HTTP server directly.
1. Load index.html, etc.
2. Concurrently load /config, /schema
3. Concurrently load /layout/obs, /annotations/var?annotation-name=<the index>
-- Does initial render --
4. Concurrently load all /annotations/obs and all /layouts/obs
-- Fully initialized --
"""
# Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency.
wait_time = between(0.01, 0.1)
def on_start(self):
self.dataset = self.parent.dataset
self.client.verify = False
self.api_less_client = HttpSession(
base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""),
request_success=self.client.request_success,
request_failure=self.client.request_failure,
)
@task
def index(self):
self.api_less_client.get(f"{self.dataset}", stream=True)
@task
def loadConfigAndSchema(self):
self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True)
self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True)
@task
def loadBootstrapData(self):
self.client.get(
f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True
)
self.client.get(
f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}",
headers={"Accept": "application/octet-stream"},
catch_response=True,
)
@task
def loadObsAnnotationsAndLayouts(self):
obs_names = self.parent.obs_annotation_names()
for name in obs_names:
self.client.get(
f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}",
headers={"Accept": "application/octet-stream"},
stream=True,
)
layouts = self.parent.layout_names()
for name in layouts:
self.client.get(
f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}",
headers={"Accept": "application/octet-stream"},
stream=True,
)
@task
def done(self):
self.interrupt()
@task(1)
def load_expression(self):
"""
Simulate user occasionally loading some expression data for a gene
"""
gene_name = random.choice(self.gene_names)
filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}}
self.client.put(
f"{self.dataset}/{API_SUFFIX}/data/var",
data=json.dumps(filter),
headers={"Content-Type": "application/json", "Accept": "application/octet-stream"},
stream=True,
).close()
class CellxgeneUser(HttpUser):
tasks = [CellXGeneTasks]
# Most ops do not require back-end interaction, so slow cadence for users
wait_time = between(10, 60)
backend/test/test_czi_hosted/locust/requirements-locust.txt
-2
View File
@@ -1,2 +0,0 @@
locust
-r ../../../czi_hosted/requirements.txt
backend/test/test_czi_hosted/performance/__init__.py
View File
backend/test/test_czi_hosted/performance/create_test_matrix.py
-44
View File
@@ -1,44 +0,0 @@
import anndata
import argparse
import random
import scipy
import numpy as np
def main():
parser = argparse.ArgumentParser("A command to generate test h5ad files")
parser.add_argument("output", help="Name of the output file")
parser.add_argument("nobs", type=int, help="Number of observations (rows)")
parser.add_argument("nvar", type=int, help="Number of variables (columns)")
parser.add_argument("-n", "--nnz-percent", type=float, default=100, help="percent of non-zeros")
parser.add_argument("-c", "--col-shift", action="store_true", help="add a random value to each column")
parser.add_argument("--seed", type=int, default=None, help="add a random value to each column")
args = parser.parse_args()
create_test_h5ad(args.output, args.nobs, args.nvar, args.nnz_percent, args.col_shift, args.seed)
def create_test_h5ad(outfile, nobs, nvar, nnz_percent=100, apply_col_shift=False, seed=None):
random.seed(seed)
np.random.seed(seed)
x = create_X_array(nobs, nvar, nnz_percent, apply_col_shift)
obsm = {"X_random": np.random.rand(nobs, 2).astype(np.float32)}
adata = anndata.AnnData(x, obsm=obsm)
adata.write(outfile)
def create_X_array(nobs, nvar, nnz_percent, apply_col_shift):
if nnz_percent < 100:
array = scipy.sparse.random(nobs, nvar, nnz_percent * 0.01, dtype=np.float32, format="csc")
else:
array = np.random.rand(nobs, nvar).astype(np.float32)
if apply_col_shift:
col_shift = np.random.rand((nvar))
array += col_shift
return array
if __name__ == "__main__":
main()
backend/test/test_czi_hosted/performance/performance_test_annotations_backend.py
-216
View File
@@ -1,216 +0,0 @@
import json
import string
from contextlib import contextmanager
from timeit import default_timer
import concurrent.futures
import numpy as np
import requests
import sys
import pandas as pd
import random
from backend.common.fbs.matrix import encode_matrix_fbs
"""
Before running, sign into the dataportal, copy the cookie and paste it below. To test in staging or prod update the
url base below. It is also possible to configure the number of categories created and the number of unique labels per
category.
"""
cookie = ""
test_datasets = {
"smallest": {
"dataset_url": "kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg",
"name": "smallest",
"num_cells": 5270,
},
"10k": {
"dataset_url": "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
"name": "10k",
"num_cells": 9409,
},
"80k": {
"dataset_url": "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg", # noqa E501
"name": "80k",
"num_cells": 81736,
},
"140k": {"dataset_url": "Single_cell_drug_screening_a549-42-remixed.cxg", "name": "140k", "num_cells": 143015},
"largest": {"dataset_url": "human_cell_landscape.cxg", "name": "largest", "num_cells": 599926},
"1million": {"dataset_url": None, "name": "1million", "num_cells": 1000000},
"4million": {"dataset_url": None, "name": "4million", "num_cells": 4000000},
}
url_base = "https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/"
annotations_category_count = [1, 10, 50]
max_labels = [5, 50, 100]
class PerformanceTestingAnnotations:
def __init__(
self,
datasets=test_datasets,
annotations_category_count=annotations_category_count,
max_labels=max_labels,
url_base=url_base,
):
self.test_datasets = datasets
self.annotations_category_count = annotations_category_count
self.max_labels = max_labels
self.url_base = url_base
self.test_notes = self.create_info_dict()
def set_cell_count(self, dataset_name):
dataset_url = self.test_datasets[dataset_name]["dataset_url"]
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
response = self.client.get(f"{self.url_base}{dataset_url}/api/v0.2/schema", headers=headers)
cell_count = json.loads(response._content)["schema"]["dataframe"]["nObs"]
self.test_datasets[dataset_name]["cell_count"] = cell_count
def create_info_dict(self):
request_info = {}
for dataset in self.test_datasets.keys():
request_info[dataset] = {}
for cat_count in self.annotations_category_count:
request_info[dataset][f"num_categories_{cat_count}"] = {}
for unique_labels in self.max_labels:
request_info[dataset][f"num_categories_{cat_count}"][f"max_label_{unique_labels}"] = {}
return request_info
def create_annotations_dict_multi_process(self, dataset_name, category_count, label_max):
annotation_dict = {}
futures = []
categories = [f"Category{i}" for i in range(category_count)]
if not self.test_datasets[dataset_name]["num_cells"]:
self.set_cell_count(dataset_name)
with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
for category in categories:
futures.append(
executor.submit(
self.build_array_for_category,
category,
self.test_datasets[dataset_name]["num_cells"],
label_max,
)
)
for future in concurrent.futures.as_completed(futures):
try:
result = future.result()
category_name, cells = result
annotation_dict[category_name] = pd.Series(cells, dtype="category")
except Exception as e:
print(f"Issue creating the annotations dict: {e}")
return annotation_dict
def build_array_for_category(self, category_name, cell_count, label_max):
unique_label_count = label_max
labels = self.generate_labels(unique_label_count)
cells_per_label = int(cell_count / len(labels))
extra = cell_count % len(labels)
cells = []
for label in labels:
cells.extend([label] * cells_per_label)
cells.extend(["extra"] * extra)
rng = np.random.default_rng()
rng.shuffle(cells)
return category_name, cells
@staticmethod
def convert_to_fbs(annotation_dict):
df = pd.DataFrame(annotation_dict)
return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
@staticmethod
def generate_labels(unique_label_count):
labels = ["undefined"]
for i in range(unique_label_count):
length = random.randrange(10, 20)
labels.append(f"{i}__" + "".join(random.choice(string.ascii_letters) for z in range(length)))
return labels
@contextmanager
def elapsed_timer(self):
start = default_timer()
elapser = lambda: default_timer() - start # noqa E731
yield lambda: elapser()
end = default_timer()
elapser = lambda: end - start # noqa E731
def create_matrix(self, dataset_name, num_cat, max_labels):
with self.elapsed_timer() as elapsed:
annon_dict = self.create_annotations_dict_multi_process(dataset_name, num_cat, max_labels)
dict_size = sum(sys.getsizeof(value) for value in annon_dict.values()) / 1024 ** 2
self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["annotation_dict"] = {
"creation_time": str(elapsed()),
"size": f"{dict_size} mb",
}
df = pd.DataFrame(annon_dict)
df_size = sys.getsizeof(df) / 1024 ** 2
self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["data_frame"] = {
"creation_time": str(elapsed()),
"size": f"{df_size} mb",
}
try:
matrix = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
matrix_size = sys.getsizeof(matrix) / 1024 ** 2
self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["fbs_matrix"] = {
"creation_time": str(elapsed()),
"size": f"{matrix_size} mb",
}
return matrix
except Exception as e:
print(f"Issue creating fbs matrix: {e}, for {dataset_name}")
return []
def send_put_request(self, dataset_url, data):
url = self.url_base + f"{dataset_url}/api/v0.2/annotations/obs"
with self.elapsed_timer() as elapsed:
try:
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
response = requests.put(url=url, data=data, headers=headers)
except Exception as e:
print(f"Issue with put request: {e}")
return None, elapsed()
return response, elapsed()
def test_categories_max_label_matrix(self, dataset_name):
for unique_labels in self.max_labels:
for category_count in self.annotations_category_count:
print(f"Starting dataset: {dataset_name}, categories: {category_count}, labels: {unique_labels}")
fbs_matrix = self.create_matrix(dataset_name, category_count, unique_labels)
if self.test_datasets[dataset_name]["dataset_url"] and fbs_matrix:
response, response_time = self.send_put_request(
self.test_datasets[dataset_name]["dataset_url"], fbs_matrix
)
if response is None:
self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
"put_request"
] = {"response_status": "failed", "request_time": str(response_time)}
else:
self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
"put_request"
] = {"response_status": response.status_code, "request_time": str(response_time)}
def test_all_datasets():
"""
Run time is dependent on number of datasets, dataset size, number of categories/number being tested and number of
unique label counts being tested. However it generally takes a long time. I recommend running this in tmux
"""
perf_test = PerformanceTestingAnnotations()
for dataset_name in perf_test.test_datasets.keys():
print(f"Testing annotation creation for: {dataset_name}")
try:
perf_test.test_categories_max_label_matrix(dataset_name)
except Exception as e:
print(f"something went wrong with {dataset_name}: {e}")
return perf_test.test_notes
def main():
notes = test_all_datasets()
print(notes)
if __name__ == "__main__":
main()
backend/test/test_czi_hosted/performance/run_diffexp.py
-116
View File
@@ -1,116 +0,0 @@
import sys
import argparse
import random
import time
import numpy as np
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.czi_hosted.compute import diffexp_cxg
from backend.common.compute import diffexp_generic
from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
def main():
parser = argparse.ArgumentParser("A command to test diffexp")
parser.add_argument("dataset", help="name of a dataset to load")
parser.add_argument("-na", "--numA", type=int, help="number of rows in group A")
parser.add_argument("-nb", "--numB", type=int, help="number of rows in group B")
parser.add_argument("-va", "--varA", help="obs variable:value to use for group A")
parser.add_argument("-vb", "--varB", help="obs variable:value to use for group B")
parser.add_argument("-t", "--trials", default=1, type=int, help="number of trials")
parser.add_argument(
"-a", "--alg", choices=("default", "generic", "cxg"), default="default", help="algorithm to use"
)
parser.add_argument("-s", "--show", default=False, action="store_true", help="show the results")
parser.add_argument(
"-n", "--new-selection", default=False, action="store_true", help="change the selection between each trial"
)
parser.add_argument("--seed", default=1, type=int, help="set the random seed")
args = parser.parse_args()
app_config = AppConfig()
app_config.update_server_config(single_dataset__datapath=args.dataset)
app_config.update_server_config(app__verbose=True)
app_config.complete_config()
loader = MatrixDataLoader(args.dataset)
adaptor = loader.open(app_config)
if args.show:
if isinstance(adaptor, CxgAdaptor):
adaptor.open_array("X").schema.dump()
random.seed(args.seed)
np.random.seed(args.seed)
rows = adaptor.get_shape()[0]
if args.numA:
filterA = random.sample(range(rows), args.numA)
elif args.varA:
vname, vval = args.varA.split(":")
filterA = get_filter_from_obs(adaptor, vname, vval)
else:
print("must supply numA or varA")
sys.exit(1)
if args.numB:
filterB = random.sample(range(rows), args.numB)
elif args.varB:
vname, vval = args.varB.split(":")
filterB = get_filter_from_obs(adaptor, vname, vval)
else:
print("must supply numB or varB")
sys.exit(1)
for i in range(args.trials):
if args.new_selection:
if args.numA:
filterA = random.sample(range(rows), args.numA)
if args.numB:
filterB = random.sample(range(rows), args.numB)
maskA = np.zeros(rows, dtype=bool)
maskA[filterA] = True
maskB = np.zeros(rows, dtype=bool)
maskB[filterB] = True
t1 = time.time()
if args.alg == "default":
results = adaptor.compute_diffexp_ttest(maskA, maskB)
elif args.alg == "generic":
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB)
elif args.alg == "cxg":
if not isinstance(adaptor, CxgAdaptor):
print("cxg only works with CxgAdaptor")
sys.exit(1)
results = diffexp_cxg.diffexp_ttest(adaptor, maskA, maskB)
t2 = time.time()
print("TIME=", t2 - t1)
if args.show:
for res in results:
print(res)
def get_filter_from_obs(adaptor, obsname, obsval):
attrs = adaptor.get_obs_columns()
if obsname not in attrs:
print(f"Unknown obs attr {obsname}: expected on of {attrs}")
sys.exit(1)
obsvals = adaptor.query_obs_array(obsname)[:]
obsval = type(obsvals[0])(obsval)
vfilter = np.where(obsvals == obsval)[0]
if len(vfilter) == 0:
u = np.unique(obsvals)
print(f"Unknown value in variable {obsname}:{obsval}: expected one of {list(u)}")
sys.exit(1)
return vfilter
if __name__ == "__main__":
main()
backend/test/test_czi_hosted/performance/scale_test_annotations.py
-45
View File
@@ -1,45 +0,0 @@
import time
import random
from locust import HttpUser, between, task
random.seed(time.time())
"""
To run this script sign into cellxgene in the desired environment and grab the returned cookie, update the cookie
variable below with your cookie and run the following command to see results in the terminal:
locust -f backend/test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
Or if you want to use the locust gui run:
locust -f backend/test/test_czi_hosted/performance/scale_test_annotations.py -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/
If you want to test staging you'll need to substitute staging for dev in the host url
To test prod you'll need to replace dev.single-cell.czi.technology with cziscience.com
If you'd like to test additional datasets you'll need to add them to the dataset_urls array
Todo @mdunitz update script to retrieve different annotation categories -- may need to create them to ensure the
categories are shared across datasets for a given user.
"""
cookie = ""
class WebsiteUser(HttpUser):
wait_time = between(1, 2)
dataset_urls = [
"human_cell_landscape.cxg",
"Single_cell_drug_screening_a549-42-remixed.cxg",
"krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
"Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg",
]
@task
def get_annotations(self):
dataset_url = random.choice(self.dataset_urls)
url = f"{dataset_url}/api/v0.2/annotations/obs?annotation-name=cell_type"
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
self.client.get(url, headers=headers)
@task
def get_schema(self):
dataset_url = random.choice(self.dataset_urls)
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
self.client.get(f"{dataset_url}/api/v0.2/schema", headers=headers)
backend/test/test_czi_hosted/test_database/__init__.py
View File
backend/test/test_czi_hosted/test_database/test_database.py
-63
View File
@@ -1,63 +0,0 @@
import unittest
from backend.czi_hosted.db.cellxgene_orm import CellxGeneUser, CellxGeneDataset, Annotation
from backend.czi_hosted.db.db_utils import DbUtils
from backend.test.fixtures.database import TestDatabase
class DatabaseTest(unittest.TestCase):
db = DbUtils("postgresql://postgres:test_pw@localhost:5432")
@classmethod
def setUpClass(cls) -> None:
TestDatabase()
@classmethod
def tearDownClass(cls) -> None:
del cls.db
def test_user_creation(self):
one_user = self.db.get(table=CellxGeneUser, entity_id="test_user_id")
self.assertEqual(one_user.id, "test_user_id")
user_count = self.db.session.query(CellxGeneUser).count()
self.assertGreater(user_count, 10)
def test_dataset_creation(self):
one_dataset = self.db.query(
table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"]
)
self.assertEqual(one_dataset[0].name, "test_dataset")
dataset_count = self.db.session.query(CellxGeneDataset).count()
self.assertGreater(dataset_count, 10)
def test_annotation_creation(self):
one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == "tiledb_uri"])[0]
self.assertEqual(one_annotation.tiledb_uri, "tiledb_uri")
annotation_count = self.db.session.query(Annotation).count()
self.assertGreater(annotation_count, 10)
def test_get_most_recent_annotation_for_user_dataset(self):
dataset_id = str(
self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"])[0].id
)
# have to commit separately because created_at time written on the db server
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_0"))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_1"))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_2"))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_3"))
self.db.session.commit()
self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_4"))
self.db.session.commit()
most_recent_annotation = self.db.query_for_most_recent(
Annotation, [Annotation.dataset_id == dataset_id, Annotation.user_id == "test_user_id"]
)
self.assertEqual(most_recent_annotation.tiledb_uri, "tiledb_uri_4")
backend/test/test_czi_hosted/unit/__init__.py
-180
View File
@@ -1,180 +0,0 @@
import logging
import shutil
import tempfile
import unittest
from os import path
import pandas as pd
from flask_compress import Compress
from flask_cors import CORS
from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.utils.data_locator import DataLocator
from backend.common.fbs.matrix import encode_matrix_fbs
from backend.czi_hosted.data_common.matrix_loader import MatrixDataType, MatrixDataLoader
from backend.czi_hosted.db.db_utils import DbUtils
from backend.czi_hosted.app.app import Server
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
tmp_dir = tempfile.mkdtemp()
fname = {
MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
MatrixDataType.CXG: "test/fixtures/pbmc3k.cxg",
}[ext]
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot=data_locator.path,
authentication__type="test",
authentication__insecure_test_environment=True,
)
config.update_default_dataset_config(
embeddings__names=["umap"],
presentation__max_categories=100,
diffexp__lfc_cutoff=0.01,
user_annotations__type="hosted_tiledb_array",
user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432",
user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsHostedTileDB(
{
"user-annotations": True,
"genesets-save": False,
},
tmp_dir,
DbUtils("postgresql://postgres:test_pw@localhost:5432"),
)
return data, tmp_dir, annotations
def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
tmp_dir = tempfile.mkdtemp()
annotations_file = path.join(tmp_dir, "test_annotations.csv")
if annotations_fixture:
shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-annotations.csv", annotations_file)
fname = {
MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
MatrixDataType.CXG: f"{FIXTURES_ROOT}/pbmc3k.cxg",
}[ext]
data_locator = DataLocator(fname)
config = AppConfig()
config.update_server_config(
app__flask_secret_key="secret",
single_dataset__obs_names=None,
single_dataset__var_names=None,
single_dataset__datapath=data_locator.path,
)
config.update_default_dataset_config(
embeddings__names=["umap"],
presentation__max_categories=100,
diffexp__lfc_cutoff=0.01,
)
config.complete_config()
data = MatrixDataLoader(data_locator.abspath()).open(config)
annotations = AnnotationsLocalFile(
{
"user-annotations": True,
"genesets-save": False,
},
None,
annotations_file,
)
return data, tmp_dir, annotations, config
def make_fbs(data):
df = pd.DataFrame(data)
return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
def skip_if(condition, reason: str):
def decorator(f):
def wraps(self, *args, **kwargs):
if condition(self):
self.skipTest(reason)
else:
f(self, *args, **kwargs)
return wraps
return decorator
def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}):
config = AppConfig()
config.update_server_config(
app__flask_secret_key="secret",
single_dataset__obs_names=None,
single_dataset__var_names=None,
adaptor__anndata_adaptor__backed=backed,
single_dataset__datapath=data_locator,
limits__diffexp_cellcount_max=None,
limits__column_request_max=None,
)
config.update_default_dataset_config(
embeddings__names=["umap", "tsne", "pca"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01
)
config.update_server_config(**extra_server_config)
config.update_default_dataset_config(**extra_dataset_config)
config.complete_config()
return config
class TestServer(Server):
def __init__(self, app_config):
super().__init__(app_config)
@staticmethod
def _before_adding_routes(app, app_config):
app.config["COMPRESS_MIMETYPES"] = [
"text/html",
"text/css",
"text/xml",
"application/json",
"application/javascript",
"application/octet-stream",
]
Compress(app)
if app_config.server_config.app__debug:
CORS(app, supports_credentials=True)
class BaseTest(unittest.TestCase):
@classmethod
def setUpClass(cls, app_config=None):
cls.TEST_URL_BASE = "/d/pbmc3k.cxg/api/v0.2/"
cls.maxDiff = None
cls.app = cls.create_app(app_config)
@classmethod
def create_app(cls, app_config=None):
if not app_config:
app_config = AppConfig()
app_config.update_server_config(
authentication__type="test",
authentication__insecure_test_environment=True,
app__flask_secret_key="testing",
app__debug=True,
multi_dataset__dataroot=f"{FIXTURES_ROOT}",
multi_dataset__index=True,
multi_dataset__allowed_matrix_types=["cxg"]
)
app_config.complete_config(logging.info)
app = TestServer(app_config).app
app.testing = True
app.debug = True
return app
backend/test/test_czi_hosted/unit/auth/__init__.py
View File
backend/test/test_czi_hosted/unit/auth/test_auth.py
-168
View File
@@ -1,168 +0,0 @@
import json
import unittest
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit import BaseTest
class AuthTest(BaseTest):
def setUp(self):
self.dataset_dataroot = FIXTURES_ROOT
def test_auth_none(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
app_config.update_default_dataset_config(user_annotations__enable=False)
app_config.complete_config()
server= self.create_app(app_config)
server.testing = True
session = server.test_client()
config = json.loads(session.get(f"{self.TEST_URL_BASE}config").data)
userinfo = json.loads(session.get(f"{self.TEST_URL_BASE}userinfo").data)
self.assertNotIn("authentication", config["config"])
self.assertIsNone(userinfo)
def test_auth_session(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
app_config.update_default_dataset_config(user_annotations__enable=True)
app_config.complete_config()
server = self.create_app(app_config)
server.auth.is_user_authenticated = lambda: True
server.testing = True
session = server.test_client()
config = json.loads(session.get(f"{self.TEST_URL_BASE}config").data)
userinfo = json.loads(session.get(f"{self.TEST_URL_BASE}userinfo").data)
self.assertFalse(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "anonymous")
def test_auth_test(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__type="test")
app_config.update_server_config(authentication__insecure_test_environment=True)
app_config.update_server_config(
multi_dataset__dataroot=dict(
a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
)
)
# specialize the configs
app_config.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
app_config.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
app_config.complete_config()
server=self.create_app(app_config)
server.testing = True
session = server.test_client()
# auth datasets
config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get(f"/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, "/login?dataset=auth/pbmc3k.cxg")
self.assertEqual(logout_uri, "/logout?dataset=auth/pbmc3k.cxg")
response = session.get(login_uri)
# check that the login redirect worked
self.assertEqual(response.status_code, 302)
self.assertEqual(response.headers['Location'], 'http://localhost/auth/pbmc3k.cxg')
config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertEqual(userinfo["userinfo"]["picture"], None)
self.assertTrue(config["config"]["parameters"]["annotations"])
response = session.get(logout_uri)
# check that the logout redirect worked
self.assertEqual(response.status_code, 302)
config = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
# no-auth datasets
config = json.loads(session.get("/no-auth/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/no-auth/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertIsNone(userinfo)
self.assertFalse(config["config"]["parameters"]["annotations"])
# login with a picture
session.get(f"{login_uri}&picture=myimage.png")
userinfo = json.loads(session.get("/auth/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["picture"], "myimage.png")
def test_auth_test_single(self):
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
)
app_config.update_server_config(authentication__insecure_test_environment=True)
app_config.complete_config()
server = self.create_app(app_config)
server.testing = True
session = server.test_client()
config = json.loads(session.get("/api/v0.2/config").data)
userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, "/login")
self.assertEqual(logout_uri, "/logout")
# check that the login redirect worked
with server.test_client() as session:
response = session.get(login_uri)
self.assertEqual(response.status_code, 302)
self.assertEqual(response.headers['Location'], "http://localhost/")
config = json.loads(session.get("api/v0.2/config").data)
userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "test_account")
self.assertTrue(config["config"]["parameters"]["annotations"])
response = session.get(logout_uri)
# check that the logout redirect worked
self.assertEqual(response.status_code, 302)
self.assertEqual(response.headers['Location'], "http://localhost/")
config = json.loads(session.get("/api/v0.2/config").data)
userinfo = json.loads(session.get("/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
backend/test/test_czi_hosted/unit/auth/test_oauth.py
-229
View File
@@ -1,229 +0,0 @@
import unittest
import random
import time
import base64
import json
import requests
from flask import Flask, jsonify, make_response, request, redirect
from multiprocessing import Process
import jose
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import FIXTURES_ROOT
# This tests the oauth authentication type.
# This test starts a cellxgene server and a mock oauth server.
# API requests to login and logout and get the userinfo are made
# to the cellxgene server, which then sends requests to the mock
# oauth server.
# number of seconds that the oauth token is valid
from backend.test.test_czi_hosted.unit import BaseTest
TOKEN_EXPIRES = 2
# Create a mocked out oauth token, which servers all the endpoints needed by the oauth type.
mock_oauth_app = Flask("mock_oauth_app")
@mock_oauth_app.route("/authorize")
def authorize():
callback = request.args.get("redirect_uri")
state = request.args.get("state")
return redirect(callback + f"?code=fakecode&state={state}")
@mock_oauth_app.route("/oauth/token", methods=["POST"])
def token():
now = time.time()
expires_at = now + TOKEN_EXPIRES
headers = dict(alg="RS256", kid="fake_kid")
payload = dict(name="fake_user", sub="fake_id", email="fake_user@email.com", email_verified=True, exp=expires_at)
jwt = jose.jwt.encode(claims=payload, key="mysecret", algorithm="HS256", headers=headers)
r = {
"access_token": f"access-{now}",
"id_token": jwt,
"refresh_token": f"random-{now}",
"scope": "openid profile email",
"expires_in": TOKEN_EXPIRES,
"token_type": "Bearer",
"expires_at": expires_at,
}
return make_response(jsonify(r))
@mock_oauth_app.route("/v2/logout")
def logout():
return_to = request.args.get("returnTo")
return redirect(return_to)
@mock_oauth_app.route("/.well-known/jwks.json")
def jwks():
data = dict(alg="RS256", kty="RSA", use="sig", kid="fake_kid",)
return make_response(jsonify(dict(keys=[data])))
# function to launch the mock oauth server
def launch_mock_oauth(mock_port):
mock_oauth_app.run(port=mock_port)
class AuthTest(BaseTest):
@classmethod
def setUpClass(cls):
# The port that the mock oauth server will listen on
cls.mock_port = random.randint(10000, 12000)
cls.dataset_dataroot = FIXTURES_ROOT
cls.mock_oauth_process = Process(target=launch_mock_oauth, args=(cls.mock_port,))
cls.mock_oauth_process.start()
# Verify that the mock oauth server is ready (accepting requests) before starting the tests.
# The following lines are polling until the mock server is ready.
# The issue is we are starting a mock oauth server, then we are starting a cellxgene server,
# which will start making requests to the mock oauth server.
# So there is a race condition because the mock oauth server needs to be ready before it gets requests.
# We check to see if it is ready, and if not we wait 1 second, then try again.
# If it gets to 5 seconds, which is shouldn't, we assume something has gone wrong and fail the test.
server_okay = False
for _ in range(5):
try:
response = requests.get(f"http://localhost:{cls.mock_port}/.well-known/jwks.json")
if response.status_code == 200:
server_okay = True
break
except: # noqa: E722
pass
# wait one second and try again
time.sleep(1)
assert(server_okay)
@classmethod
def tearDownClass(cls):
cls.mock_oauth_process.terminate()
def auth_flow(self, app_config, cookie_key=None):
app_config.update_server_config(
app__api_base_url="local",
authentication__type="oauth",
authentication__params_oauth__oauth_api_base_url=f"http://localhost:{self.mock_port}",
authentication__params_oauth__client_id="mock_client_id",
authentication__params_oauth__client_secret="mock_client_secret",
authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False},
)
app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
app_config.complete_config()
server= self.create_app(app_config)
server.testing = True
session = server.test_client()
# auth datasets
config = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["authentication"]["requires_client_login"])
self.assertTrue(config["config"]["parameters"]["annotations"])
login_uri = config["config"]["authentication"]["login"]
logout_uri = config["config"]["authentication"]["logout"]
self.assertEqual(login_uri, "http://localhost:5005/login?dataset=d/pbmc3k.cxg/")
self.assertEqual(logout_uri, "http://localhost:5005/logout?dataset=d/pbmc3k.cxg/")
response = session.get(login_uri)
# check that the login redirect worked
self.assertEqual(response.status_code, 302)
config = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get("/d/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com")
self.assertTrue(config["config"]["parameters"]["annotations"])
if cookie_key:
cookie = session.cookies.get(cookie_key)
token = json.loads(base64.b64decode(cookie))
access_token_before = token.get("access_token")
id_token_before = token.get("id_token")
# let the token expire
time.sleep(TOKEN_EXPIRES + 1)
# check that refresh works
session.get(login_uri)
userinfo = json.loads(session.get(f"/d/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertTrue(userinfo["userinfo"]["is_authenticated"])
self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
cookie = session.cookies.get(cookie_key)
token = json.loads(base64.b64decode(cookie))
access_token_after = token.get("access_token")
id_token_after = token.get("id_token")
self.assertNotEqual(access_token_before, access_token_after)
self.assertNotEqual(id_token_before, id_token_after)
# invalid cookie is rejected
session.cookies.set(cookie_key, "TEST_" + cookie)
self.assertTrue(cookie_key in session.cookies)
response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
# this is not an error, the invalid cookie is just ignored.
self.assertEqual(response.status_code, 200)
userinfo = json.loads(response.data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
# invalid id_token is rejected
test_token = token
test_token["id_token"] = "TEST_" + id_token_after
encoded_cookie = base64.b64encode(json.dumps(test_token).encode()).decode()
session.cookies.set(cookie_key, encoded_cookie)
response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo")
# this is not an error, the invalid id_token is just ignored.
self.assertEqual(response.status_code, 200)
userinfo = json.loads(response.data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
r = session.get(logout_uri)
# check that the logout redirect worked
self.assertEqual(r.history[0].status_code, 302)
self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
config = json.loads(session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").data)
userinfo = json.loads(session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").data)
self.assertFalse(userinfo["userinfo"]["is_authenticated"])
self.assertIsNone(userinfo["userinfo"]["username"])
self.assertTrue(config["config"]["parameters"]["annotations"])
@unittest.skip("turn on when we utilizing auth in the explorer")
def test_auth_oauth_session(self):
# test with session cookies
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
self.auth_flow(app_config)
@unittest.skip("turn on when we utilizing auth in the explorer")
def test_auth_oauth_cookie(self):
# test with specified cookie
app_config = AppConfig()
app_config.update_server_config(app__flask_secret_key="secret")
app_config.update_server_config(
authentication__params_oauth__session_cookie=False,
authentication__params_oauth__cookie=dict(key="test_cxguser", httponly=True, max_age=60),
)
self.auth_flow(app_config, "test_cxguser")
backend/test/test_czi_hosted/unit/cli/__init__.py
View File
backend/test/test_czi_hosted/unit/cli/test_launch.py
-27
View File
@@ -1,27 +0,0 @@
import filecmp
import os
import shutil
import unittest
import yaml
from backend.czi_hosted.default_config import default_config
from backend.test import FIXTURES_ROOT
class CLIPLaunchTests(unittest.TestCase):
tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
@classmethod
def setUpClass(cls) -> None:
os.mkdir(cls.tmp_dir)
@classmethod
def tearDownClass(cls) -> None:
shutil.rmtree(cls.tmp_dir)
def test_dump_default_config(self):
os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
expected_config.write(yaml.dump(default_config))
filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")
backend/test/test_czi_hosted/unit/cli/test_prepare.py
-15
View File
@@ -1,15 +0,0 @@
import unittest
import pandas as pd
from backend.czi_hosted.cli.prepare import make_index_unique
class CLIPrepareTests(unittest.TestCase):
""" Test cases for CLI prepare logic """
def test_make_index_unique(self):
index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])
result = make_index_unique(index)
expected = pd.Index(["SNORD113", "SNORD113-2", "SNORD113-1"])
self.assertTrue(all(left == right for left, right in zip(result.values, expected.values)))
backend/test/test_czi_hosted/unit/cli/test_upgrade.py
-28
View File
@@ -1,28 +0,0 @@
import unittest
from backend.czi_hosted.cli.upgrade import validate_version_str, split_version, version_gt
class CLIUpgradeTests(unittest.TestCase):
""" Test cases for CLI logic """
def test_validate_version_str(self):
self.assertTrue(validate_version_str("0.1.2"))
self.assertTrue(validate_version_str("0.1.2-RC", release_only=False))
self.assertFalse(validate_version_str("0.1"))
self.assertFalse(validate_version_str("0.1.2.3"))
self.assertFalse(validate_version_str("0.1.2-RC"))
def test_split_version_str(self):
self.assertEqual(split_version("0.1.2"), [0, 1, 2])
with self.assertRaises(AttributeError):
split_version("0.1")
def test_assert_verstion_gt(self):
self.assertTrue(version_gt("1.0.0", "0.1.1"))
self.assertTrue(version_gt("0.1.0", "0.0.1"))
self.assertTrue(version_gt("0.0.1", "0.0.0"))
self.assertFalse(version_gt("0.0.0", "0.0.0"))
self.assertFalse(version_gt("0.0.0", "0.0.1"))
self.assertFalse(version_gt("0.0.1", "0.1.0"))
self.assertFalse(version_gt("0.1.1", "1.0.0"))
backend/test/test_czi_hosted/unit/common/__init__.py
View File
backend/test/test_czi_hosted/unit/common/config/__init__.py
-274
View File
@@ -1,274 +0,0 @@
import os
import shutil
import random
import yaml
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit import BaseTest
class ConfigTests(BaseTest):
tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
@classmethod
def tearDownClass(cls) -> None:
shutil.rmtree(cls.tmp_fixtures_directory)
@classmethod
def setUpClass(cls) -> None:
os.makedirs(cls.tmp_fixtures_directory)
def custom_server_config(
self,
verbose="false",
debug="false",
host="localhost",
port="null",
open_browser="false",
force_https="false",
flask_secret_key="secret",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
api_base_url="null",
web_base_url="null",
auth_type="session",
insecure_test_environment="false",
oauth_api_base_url="null",
client_id="null",
client_secret="null",
jwt_decode_options="null",
session_cookie="true",
cookie="null",
dataroot="null",
index="false",
allowed_matrix_types=[],
max_cached_datasets=5,
timelimit_s=5,
dataset_datapath="null",
obs_names="null",
var_names="null",
about="null",
title="null",
diffexp_max_workers=64,
cpu_multiplier=4,
target_workunit="16_000_000",
data_locater_region_name="us-east-1",
cxg_tile_cache_size=8589934592,
cxg_num_reader_threads=32,
anndata_backed="false",
column_request_max=32,
diffexp_cellcount_max="null",
config_file_name="server_config.yaml",
):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
server_config_outline_path = os.path.join(FIXTURES_ROOT, "czi_hosted_server_config_outline.py")
with open(server_config_outline_path, "r") as config_skeleton:
config = config_skeleton.read()
server_config = eval(config)
with open(configfile, "w") as server_config_file:
server_config_file.write(server_config)
return configfile
def custom_app_config(
self,
verbose="false",
debug="false",
host="localhost",
port="null",
open_browser="false",
force_https="false",
flask_secret_key="secret",
generate_cache_control_headers="false",
server_timing_headers="false",
csp_directives="null",
api_base_url="null",
web_base_url="null",
auth_type="session",
oauth_api_base_url="null",
client_id="null",
client_secret="null",
jwt_decode_options="null",
session_cookie="true",
cookie="null",
dataroot="null",
index="false",
allowed_matrix_types=[],
max_cached_datasets=5,
timelimit_s=5,
dataset_datapath="null",
obs_names="null",
var_names="null",
about="null",
title="null",
diffexp_max_workers=64,
cpu_multiplier=4,
target_workunit="16_000_000",
data_locater_region_name="us-east-1",
cxg_tile_cache_size=8589934592,
cxg_num_reader_threads=32,
anndata_backed="false",
column_request_max=32,
diffexp_cellcount_max="null",
scripts=[],
inline_scripts=[],
about_legal_tos="null",
about_legal_privacy="null",
authentication_enable="true",
max_categories=1000,
custom_colors="true",
enable_users_annotations="true",
annotation_type="local_file_csv",
db_uri="null",
hosted_file_directory="null",
local_file_csv_directory="null",
local_file_csv_file="null",
embedding_names=[],
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
environment=None,
aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[],
X_approximate_distribution="normal",
config_file_name="app_config.yml",
):
random_num = random.randrange(999999)
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
server_config = self.custom_server_config(
verbose=verbose,
debug=debug,
host=host,
port=port,
open_browser=open_browser,
force_https=force_https,
flask_secret_key=flask_secret_key,
generate_cache_control_headers=generate_cache_control_headers,
server_timing_headers=server_timing_headers,
csp_directives=csp_directives,
api_base_url=api_base_url,
web_base_url=web_base_url,
auth_type=auth_type,
oauth_api_base_url=oauth_api_base_url,
client_id=client_id,
client_secret=client_secret,
jwt_decode_options=jwt_decode_options,
session_cookie=session_cookie,
cookie=cookie,
dataroot=dataroot,
index=index,
allowed_matrix_types=allowed_matrix_types,
max_cached_datasets=max_cached_datasets,
timelimit_s=timelimit_s,
dataset_datapath=dataset_datapath,
obs_names=obs_names,
var_names=var_names,
about=about,
title=title,
diffexp_max_workers=diffexp_max_workers,
cpu_multiplier=cpu_multiplier,
target_workunit=target_workunit,
data_locater_region_name=data_locater_region_name,
cxg_tile_cache_size=cxg_tile_cache_size,
cxg_num_reader_threads=cxg_num_reader_threads,
anndata_backed=anndata_backed,
column_request_max=column_request_max,
diffexp_cellcount_max=diffexp_cellcount_max,
config_file_name=f"temp_server_config_{random_num}.yml",
)
dataset_config = self.custom_dataset_config(
scripts=scripts,
inline_scripts=inline_scripts,
about_legal_tos=about_legal_tos,
about_legal_privacy=about_legal_privacy,
authentication_enable=authentication_enable,
max_categories=max_categories,
custom_colors=custom_colors,
enable_users_annotations=enable_users_annotations,
annotation_type=annotation_type,
db_uri=db_uri,
hosted_file_directory=hosted_file_directory,
local_file_csv_directory=local_file_csv_directory,
local_file_csv_file=local_file_csv_file,
embedding_names=embedding_names,
enable_difexp=enable_difexp,
lfc_cutoff=lfc_cutoff,
top_n=top_n,
X_approximate_distribution=X_approximate_distribution,
config_file_name=f"temp_dataset_config_{random_num}.yml",
)
external_config = self.custom_external_config(
environment=environment,
aws_secrets_manager_region=aws_secrets_manager_region,
aws_secrets_manager_secrets=aws_secrets_manager_secrets,
config_file_name=f"temp_external_config_{random_num}.yml",
)
with open(configfile, "w") as app_config_file:
app_config_file.write(open(server_config).read())
app_config_file.write(open(dataset_config).read())
app_config_file.write(open(external_config).read())
return configfile
def custom_dataset_config(
self,
scripts=[],
inline_scripts=[],
about_legal_tos="null",
about_legal_privacy="null",
authentication_enable="true",
max_categories=1000,
custom_colors="true",
enable_users_annotations="true",
annotation_type="local_file_csv",
db_uri="null",
hosted_file_directory="null",
local_file_csv_directory="null",
local_file_csv_file="null",
embedding_names=[],
enable_difexp="true",
lfc_cutoff=0.01,
top_n=10,
X_approximate_distribution="normal",
config_file_name="dataset_config.yml",
):
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "czi_hosted_dataset_config_outline.py")
with open(dataset_config_outline_path, "r") as config_skeleton:
config = config_skeleton.read()
dataset_config = eval(config)
with open(configfile, "w") as dataset_config_file:
dataset_config_file.write(dataset_config)
return configfile
def custom_external_config(
self,
environment=None,
aws_secrets_manager_region=None,
aws_secrets_manager_secrets=[],
config_file_name="external_config.yaml",
):
# set to the default if environment is None
if environment is None:
environment = [
dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
dict(
name="CXG_OAUTH_CLIENT_SECRET",
path=["server", "authentication", "params_oauth", "client_secret"],
required=False,
),
]
external_config = {
"external": {
"environment": environment,
"aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
}
}
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
with open(configfile, "w") as external_config_file:
yaml.dump(external_config, external_config_file)
return configfile
backend/test/test_czi_hosted/unit/common/config/test_app_config.py
-220
View File
@@ -1,220 +0,0 @@
import os
import tempfile
import unittest
import yaml
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.errors import ConfigurationError
from backend.czi_hosted.default_config import default_config
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
class AppConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_get_default_config_correctly_reads_default_config_file(self):
app_default_config = AppConfig().default_config
expected_config = yaml.load(default_config, Loader=yaml.Loader)
server_config = app_default_config["server"]
dataset_config = app_default_config["dataset"]
expected_server_config = expected_config["server"]
expected_dataset_config = expected_config["dataset"]
self.assertDictEqual(app_default_config, expected_config)
self.assertDictEqual(server_config, expected_server_config)
self.assertDictEqual(dataset_config, expected_dataset_config)
def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self):
datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
config = AppConfig()
config.update_from_config_file(file_name)
self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
def test_update_server_config_updates_server_config_and_config_status(self):
config = self.get_config()
config.complete_config()
config.check_config()
config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
with self.assertRaises(ConfigurationError):
config.server_config.check_config()
def test_write_config_outputs_yaml_with_all_config_vars(self):
config = self.get_config()
config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
default_config_yml = yaml.safe_load(default_config)
with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
output_config_yml = yaml.safe_load(output_config)
self.maxDiff = None
self.assertEqual(default_config_yml, output_config_yml)
def test_update_app_config(self):
config = AppConfig()
config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
config = AppConfig()
config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
vars = config.server_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [])
config = AppConfig()
config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
vars = config.default_dataset_config.changes_from_default()
self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
def test_configfile_no_dataset_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
app:
flask_secret_key: secret
multi_dataset:
dataroot: test_dataroot
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(
server_changes,
[("app__flask_secret_key", "secret", None), ("multi_dataset__dataroot", "test_dataroot", None)],
)
self.assertEqual(dataset_changes, [])
def test_configfile_no_server_section(self):
# test a config file without a dataset section
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
dataset:
user_annotations:
enable: false
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
server_changes = app_config.server_config.changes_from_default()
dataset_changes = app_config.default_dataset_config.changes_from_default()
self.assertEqual(server_changes, [])
self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
def test_simple_update_single_config_from_path_and_value(self):
"""Update a simple config parameter"""
config = AppConfig()
config.server_config.multi_dataset__dataroot = dict(
s1=dict(dataroot="my_dataroot_s1", base_url="my_baseurl_s1"),
s2=dict(dataroot="my_dataroot_s2", base_url="my_baseurl_s2"),
)
config.add_dataroot_config("s1")
config.add_dataroot_config("s2")
# test simple value in server
config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
# test simple value in default dataset
config.update_single_config_from_path_and_value(
["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"], "mydburi",
)
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# test simple value in specific dataset
config.update_single_config_from_path_and_value(
["per_dataset_config", "s1", "user_annotations", "hosted_tiledb_array", "db_uri"], "s1dburi"
)
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mydburi")
self.assertEqual(config.dataroot_config["s1"].user_annotations__hosted_tiledb_array__db_uri, "s1dburi")
self.assertEqual(config.dataroot_config["s2"].user_annotations__hosted_tiledb_array__db_uri, "mydburi")
# error checking
bad_paths = [
(
["dataset", "does", "not", "exist"],
"unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
),
(["does", "not", "exist"], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
([], "path must start with 'server', 'dataset', or 'per_dataset_config'"),
(["per_dataset_config"], "missing dataroot when using per_dataset_config: got '['per_dataset_config']'"),
(
["per_dataset_config", "unknown"],
"unknown dataroot when using per_dataset_config: got '['per_dataset_config', 'unknown']',"
" dataroots specified in config are ['s1', 's2']",
),
([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
("string", "path must be a list of strings, got 'string'"),
]
for bad_path, error_message in bad_paths:
with self.assertRaises(ConfigurationError) as config_error:
config.update_single_config_from_path_and_value(bad_path, "value")
self.assertEqual(config_error.exception.message, error_message)
def test_dict_update_single_config_from_path_and_value(self):
"""Update a config parameter that has a value of dict"""
# the path leads to a dict config param, set the config parameter to the new value
config = AppConfig()
config.update_single_config_from_path_and_value(
["server", "authentication", "params_oauth", "cookie"], dict(key="mykey1", max_age=100)
)
self.assertEqual(config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100))
# the path leads to an entry within a dict config param, the value is simple
config = AppConfig()
config.server_config.authentication__params_oauth__cookie = dict(key="mykey1", max_age=100)
config.update_single_config_from_path_and_value(
["server", "authentication", "params_oauth", "cookie", "httponly"], True,
)
self.assertEqual(
config.server_config.authentication__params_oauth__cookie, dict(key="mykey1", max_age=100, httponly=True)
)
backend/test/test_czi_hosted/unit/common/config/test_base_config.py
-66
View File
@@ -1,66 +0,0 @@
import unittest
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import FIXTURES_ROOT
from backend.common.errors import ConfigurationError
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
class BaseConfigTest(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
config = AppConfig()
mapping = config.default_dataset_config.create_mapping(config.default_config)
self.assertIsNotNone(mapping["server__app__verbose"])
self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
def test_changes_from_default_returns_list_of_nondefault_config_values(self):
config = self.get_config(verbose="true", lfc_cutoff=0.05)
server_changes = config.server_config.changes_from_default()
dataset_changes = config.default_dataset_config.changes_from_default()
self.assertEqual(
server_changes,
[
("app__verbose", True, False),
("app__flask_secret_key", "secret", None),
('authentication__type', 'session', 'test'),
('authentication__insecure_test_environment', False, True),
("multi_dataset__dataroot", FIXTURES_ROOT, None),
("multi_dataset__matrix_cache__timelimit_s", 5, 30),
("data_locator__s3__region_name", "us-east-1", True),
],
)
self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
def test_check_config_throws_error_if_attr_has_not_been_checked(self):
config = self.get_config(verbose="true")
config.complete_config()
config.check_config()
config.update_server_config(app__verbose=False)
with self.assertRaises(ConfigurationError):
config.check_config()
backend/test/test_czi_hosted/unit/common/config/test_dataset_config.py
-253
View File
@@ -1,253 +0,0 @@
import json
import os
import tempfile
import unittest
from unittest.mock import patch
from backend.czi_hosted.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
from backend.czi_hosted.common.annotations.local_file_csv import AnnotationsLocalFile
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.czi_hosted.common.config.base_config import BaseConfig
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
from backend.common.errors import ConfigurationError
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
class TestDatasetConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.dataset_config = self.config.default_dataset_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_datatset_config_sets_vars_from_default_config(self):
config = AppConfig()
self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
@patch("backend.czi_hosted.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
self.dataset_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 19)
def test_app_sets_script_vars(self):
config = self.get_config(scripts=["path/to/script"])
config.default_dataset_config.handle_app()
self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}])
config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
config.default_dataset_config.handle_app()
self.assertEqual(
config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
)
config = self.get_config(scripts=["path/to/script", "different/script/path"])
config.default_dataset_config.handle_app()
# TODO @madison -- is this the desired functionality?
self.assertEqual(
config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
)
config = self.get_config(scripts=[{"more": "different/script/path"}])
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_app()
def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
config = self.get_config(enable_users_annotations="true", authentication_enable="false")
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_user_annotations(self.context)
config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
with self.assertRaises(ConfigurationError):
config.server_config.complete_config(self.context)
def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self):
config = self.get_config(
enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set"
)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."])
@patch("backend.czi_hosted.common.config.dataset_config.DbUtils")
def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils):
mock_db_utils.return_value = "123"
config = self.get_config(
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
config = self.get_config(
enable_users_annotations="true",
authentication_enable="true",
annotation_type="hosted_tiledb_array",
db_uri="gotta/set/this",
hosted_file_directory="and/this",
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_user_annotations(self.context)
self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB)
config = self.get_config(
enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
)
config.server_config.complete_config(self.context)
with self.assertRaises(ConfigurationError):
config.default_dataset_config.handle_user_annotations(self.context)
def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
config = self.get_config(
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_local_file_csv_annotations()
self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
cwd = os.getcwd()
self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
def test_handle_diffexp__raises_warning_for_large_datasets(self):
config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
config.server_config.complete_config(self.context)
config.default_dataset_config.handle_diffexp(self.context)
self.assertEqual(len(self.context["messages"]), 0)
def test_multi_dataset(self):
config = AppConfig()
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
)
with self.assertRaises(ConfigurationError):
config.complete_config()
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}},
)
config.complete_config()
# test that multi dataroots work end to end
config.update_server_config(
app__flask_secret_key="secret",
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
),
)
# Change this default to test if the dataroot overrides below work.
config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
# specialize the configs for set1
config.add_dataroot_config(
"s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
)
# specialize the configs for set2
config.add_dataroot_config(
"s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
)
# no specializations for set3 (they get the default dataset config)
config.complete_config()
server = self.create_app(config)
server.testing = True
session = server.test_client()
response = session.get("/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get("/set2/pbmc3k.cxg/api/v0.2/config")
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
response = session.get("/set3/pbmc3k.cxg/api/v0.2/config")
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
response = session.get("/health")
assert json.loads(response.data)["status"] == "pass"
def test_configfile_with_specialization(self):
# test that per_dataset_config config load the default config, then the specialized config
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
with open(configfile, "w") as fconfig:
config = """
server:
multi_dataset:
dataroot:
test:
base_url: test
dataroot: fake_dataroot
dataset:
user_annotations:
enable: false
type: hosted_tiledb_array
hosted_tiledb_array:
db_uri: fake_db_uri
hosted_file_directory: fake_dir
per_dataset_config:
test:
user_annotations:
enable: true
"""
fconfig.write(config)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
test_config = app_config.dataroot_config["test"]
# test config from default
self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
# test config from specialization
self.assertTrue(test_config.user_annotations__enable)
backend/test/test_czi_hosted/unit/common/config/test_external_config.py
-243
View File
@@ -1,243 +0,0 @@
import json
import os
from unittest.mock import patch
import yaml
from backend.common.errors import ConfigurationError
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.utils.type_conversion_utils import convert_string_to_value
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
class TestExternalConfig(ConfigTests):
def test_type_convert(self):
# The values from environment variables and aws secrets are returned as strings.
# These values need to be converted to the proper types.
self.assertEqual(convert_string_to_value("1"), int(1))
self.assertEqual(convert_string_to_value("1.1"), float(1.1))
self.assertEqual(convert_string_to_value("string"), "string")
self.assertEqual(convert_string_to_value("true"), True)
self.assertEqual(convert_string_to_value("True"), True)
self.assertEqual(convert_string_to_value("false"), False)
self.assertEqual(convert_string_to_value("False"), False)
self.assertEqual(convert_string_to_value("null"), None)
self.assertEqual(convert_string_to_value("None"), None)
self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
def test_environment_variable(self):
configfile = self.custom_external_config(
environment=[
dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
],
config_file_name="environment_external_config.yaml",
)
env = os.environ
env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k.cxg"
env["DIFFEXP"] = "False"
config = AppConfig()
config.update_from_config_file(configfile)
config.update_server_config(app__flask_secret_key="123 magic")
server = self.create_app(config)
server.testing = True
session = server.test_client()
response = session.get("/api/v0.2/config")
data_config = json.loads(response.data)
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
os.environ["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
os.environ["DIFFEXP"] = "True"
server= self.create_app(config)
server.testing = True
session = server.test_client()
# session = requests.Session()
response = session.get("/api/v0.2/config")
data_config = json.loads(response.data)
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
def test_environment_variable_errors(self):
# no name
app_config = AppConfig()
app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
# required has wrong type
app_config = AppConfig()
app_config.external_config.environment = [
dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
# no path
app_config = AppConfig()
app_config.external_config.environment = [dict(name="myenvar", required=True)]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
# required environment variable is not set
app_config = AppConfig()
app_config.external_config.environment = [
dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
@patch("backend.czi_hosted.common.config.external_config.get_secret_key")
def test_aws_secrets_manager(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
configfile = self.custom_external_config(
aws_secrets_manager_region="us-west-2",
aws_secrets_manager_secrets=[
dict(
name="my_secret",
values=[
dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=False),
dict(
key="db_uri",
path=["dataset", "user_annotations", "hosted_tiledb_array", "db_uri"],
required=True,
),
dict(
key="oauth_client_secret",
path=["server", "authentication", "params_oauth", "client_secret"],
required=True,
),
],
)
],
config_file_name="secret_external_config.yaml",
)
app_config = AppConfig()
app_config.update_from_config_file(configfile)
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.server_config.app__flask_secret_key = "original"
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k.cxg"
app_config.complete_config()
self.assertEqual(app_config.server_config.app__flask_secret_key, "original")
self.assertEqual(app_config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
self.assertEqual(app_config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
@patch("backend.czi_hosted.common.config.external_config.get_secret_key")
def test_aws_secrets_manager_error(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
# no region
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = None
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(
config_error.exception.message,
"Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
)
# missing secret name
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
# secret name wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
# missing values name
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
# values wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
# entry missing key
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
# entry required is wrong type
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
# entry missing path
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(name="mysecret", values=[dict(key="db_uri", required=True)])
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
# secret missing required key
app_config = AppConfig()
app_config.external_config.aws_secrets_manager__region = "us-west-2"
app_config.external_config.aws_secrets_manager__secrets = [
dict(
name="mysecret",
values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
)
]
with self.assertRaises(ConfigurationError) as config_error:
app_config.complete_config()
self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
backend/test/test_czi_hosted/unit/common/config/test_server_config.py
-324
View File
@@ -1,324 +0,0 @@
import json
import os
import unittest
from unittest.mock import patch
from backend.czi_hosted.common.config.base_config import BaseConfig
from backend.common.utils.utils import find_available_port
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.errors import ConfigurationError
from backend.test.test_czi_hosted.unit.common.config import ConfigTests
class TestServerConfig(ConfigTests):
def setUp(self):
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
self.server_config = self.config.server_config
self.config.complete_config()
message_list = []
def noop(message):
message_list.append(message)
messagefn = noop
self.context = dict(messagefn=messagefn, messages=message_list)
def get_config(self, **kwargs):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
)
config = AppConfig()
config.update_from_config_file(file_name)
return config
def test_init_raises_error_if_default_config_is_invalid(self):
invalid_config = self.get_config(port="not_valid")
with self.assertRaises(ConfigurationError):
invalid_config.complete_config()
@patch("backend.czi_hosted.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
def test_complete_config_checks_all_attr(self, mock_check_attrs):
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
self.server_config.complete_config(self.context)
self.assertEqual(mock_check_attrs.call_count, 41)
def test_handle_app__throws_error_if_port_doesnt_exist(self):
config = self.get_config(port=99999999)
with self.assertRaises(ConfigurationError):
config.server_config.handle_app(self.context)
@patch("backend.czi_hosted.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
mock_discover_region_name.return_value = None
# Default config
self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
# hard coded
config = self.get_config()
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
# incorrectly formatted
dataroot = {
"d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
"d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"},
}
file_name = self.custom_app_config(
dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_locator()
@patch("backend.czi_hosted.common.config.server_config.discover_s3_region_name")
def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
mock_discover_region_name.return_value = "us-west-2"
dataroot = {
"d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
"d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
}
file_name = self.custom_app_config(
dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_data_locator()
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
def test_handle_app___can_use_envar_port(self):
config = self.get_config(port=24)
self.assertEqual(config.server_config.app__port, 24)
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
os.environ["CXG_SERVER_PORT"] = "4008"
self.config = AppConfig()
self.config.update_server_config(app__flask_secret_key="secret")
self.config.server_config.handle_app(self.context)
self.assertEqual(self.config.server_config.app__port, 4008)
del os.environ["CXG_SERVER_PORT"]
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
def test_handle_app__sets_web_base_url(self):
config = self.get_config(web_base_url="anything.com")
self.assertEqual(config.server_config.app__web_base_url, "anything.com")
def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self):
config = self.get_config(client_secret="KEY_FROM_FILE")
config.server_config.handle_authentication()
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
config.external_config.handle_environment(self.context)
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
file_name = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
config_file_name="two_data_roots.yml",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
file_name = self.custom_app_config(config_file_name="zero_roots.yml")
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_data_source()
@unittest.skip("skip when running in github action")
def test_get_api_base_url_works(self):
# test the api_base_url feature, and that it can contain a path
config = AppConfig()
backend_port = find_available_port("localhost", 10000)
config.update_server_config(
app__flask_secret_key="secret",
app__api_base_url=f"http://localhost:{backend_port}/additional/path",
multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
multi_dataset__allowed_matrix_types=["cxg"],
)
config.complete_config()
server = self.create_app(config)
server.testing = True
session = server.test_client()
response = session.get(f"/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
self.assertEqual(response.status_code, 200)
data_config = json.loads(response.data)
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
# test the health check at the correct url
response = session.get(f"/additional/path/health")
assert json.loads(response.data)["status"] == "pass"
def test_get_web_base_url_works(self):
config = self.get_config(web_base_url="www.thisisawebsite.com")
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "www.thisisawebsite.com")
config = self.get_config(web_base_url="local", port=12)
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "http://localhost:12")
config = self.get_config(web_base_url="www.thisisawebsite.com/")
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "www.thisisawebsite.com")
config = self.get_config(api_base_url="www.api_base.com/")
web_base_url = config.server_config.get_web_base_url()
self.assertEqual(web_base_url, "www.api_base.com")
def test_config_for_single_dataset(self):
file_name = self.custom_app_config(
config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg"
)
config = AppConfig()
config.update_from_config_file(file_name)
config.server_config.handle_single_dataset(self.context)
self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
file_name = self.custom_app_config(
config_file_name="single_dataset_with_about.yml",
about="www.cziscience.com",
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
)
config = AppConfig()
config.update_from_config_file(file_name)
with self.assertRaises(ConfigurationError):
config.server_config.handle_single_dataset(self.context)
def test_multi_dataset_raises_error_for_illegal_routes(self):
# test for illegal url_dataroots
for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
self.config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
with self.assertRaises(ConfigurationError):
self.config.complete_config()
def test_multidataset_works_for_legal_routes(self):
# test for legal url_dataroots
for legal in ("d", "this.is-okay_", "a/b"):
self.config.update_server_config(
multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": f"{PROJECT_ROOT}/example-dataset"}}
)
self.config.complete_config()
@patch("backend.czi_hosted.app.app.render_template")
def test_mulitdatasets_work_e2e(self, mock_render_template):
mock_render_template.return_value = "something"
# test that multi dataroots work end to end
self.config.update_server_config(
multi_dataset__dataroot=dict(
s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
)
)
# Change this default to test if the dataroot overrides below work.
self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
# specialize the configs for set1
self.config.add_dataroot_config(
"s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
)
# specialize the configs for set2
self.config.add_dataroot_config(
"s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
)
# no specializations for set3 (they get the default dataset config)
self.config.complete_config()
server = self.create_app(self.config)
server.auth.requires_client_login = lambda: False
server.testing = True
session = server.test_client()
response = session.get(f"/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is False
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
response = session.get("/set2/pbmc3k.cxg/api/v0.2/config")
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
response = session.get("/set3/pbmc3k.cxg/api/v0.2/config")
data_config = json.loads(response.data)
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
assert data_config["config"]["parameters"]["annotations"] is True
assert data_config["config"]["parameters"]["disable-diffexp"] is False
assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
response = session.get("/health")
assert json.loads(response.data)["status"] == "pass"
# access a dataset (no slash)
response = session.get("/set2/pbmc3k.cxg")
self.assertEqual(response.status_code, 200)
# access a dataset (with slash)
response = session.get("/set2/pbmc3k.cxg/")
self.assertEqual(response.status_code, 200)
@patch("backend.czi_hosted.common.config.server_config.diffexp_tiledb.set_config")
def test_handle_diffexp(self, mock_tiledb_config):
custom_config_file = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}",
cpu_multiplier=3,
diffexp_max_workers=1,
target_workunit=4,
config_file_name=self.config_file_name,
)
config = AppConfig()
config.update_from_config_file(custom_config_file)
config.server_config.handle_diffexp()
# called with the min of diffexp_max_workers and cpus*cpu_multiplier
mock_tiledb_config.assert_called_once_with(1, 4)
@patch("backend.czi_hosted.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
def test_handle_adaptor(self, mock_tiledb_context):
custom_config = self.custom_app_config(
dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
)
config = AppConfig()
config.update_from_config_file(custom_config)
config.server_config.handle_adaptor()
mock_tiledb_context.assert_called_once_with(
{"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
)
def test_test_auth_only_in_insecure(self):
config = self.get_config(auth_type="test")
with self.assertRaises(ConfigurationError):
config.complete_config()
config.update_server_config(authentication__insecure_test_environment=True)
config.complete_config()
backend/test/test_czi_hosted/unit/common/test_api.py
-505
View File
@@ -1,505 +0,0 @@
import json
import os
import time
from http import HTTPStatus
import hashlib
import requests
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import decode_fbs
from backend.test.fixtures.fixtures import pbmc3k_colors
from backend.test.test_czi_hosted.unit import BaseTest, skip_if
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
class EndPoints(BaseTest):
@classmethod
def setUpClass(cls, app_config=None):
super().setUpClass(app_config)
cls.app.testing = True
cls.client = cls.app.test_client()
os.environ["SKIP_STATIC"] = "True"
for i in range(90):
try:
result = cls.client.get(f"{cls.TEST_URL_BASE}schema")
cls.schema = json.loads(result.data)
except requests.exceptions.ConnectionError:
time.sleep(1)
def test_initialize(self):
endpoint = "schema"
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = json.loads(result.data)
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
self.assertEqual(
len(result_data["schema"]["annotations"]["obs"]["columns"]), 5
)
def test_config(self):
endpoint = "config"
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = json.loads(result.data)
self.assertIn("library_versions", result_data["config"])
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
def test_get_layout_fbs(self):
endpoint = "layout/obs"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 8)
self.assertIsNotNone(df["columns"])
self.assertSetEqual(
set(df["col_idx"]),
{"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"},
)
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
def test_bad_filter(self):
endpoint = "data/var"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.client.put(url, headers=header, json=BAD_FILTER)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_get_annotations_obs_fbs(self):
endpoint = "annotations/obs"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 5)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
self.assertCountEqual(
df["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
)
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"])
def test_get_annotations_obs_error(self):
endpoint = "annotations/obs"
query = "annotation-name=notakey"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
# TEMP: Testing count 15 to match hardcoded values for diffexp
# TODO(#1281): Switch back to dynamic values
def test_diff_exp(self):
endpoint = "diffexp/obs"
url = f"{self.TEST_URL_BASE}{endpoint}"
params = {
"mode": "topN",
"set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
"set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
"count": 15,
}
result = self.client.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = json.loads(result.data)
self.assertEqual(len(result_data['positive']), 15)
self.assertEqual(len(result_data['negative']), 15)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
url = f"{self.TEST_URL_BASE}{endpoint}"
params = {
"mode": "topN",
"count": 15,
"set1": {"filter": {"obs": {"index": [[0, 500]]}}},
"set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
}
result = self.client.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = json.loads(result.data)
self.assertEqual(len(result_data['positive']), 15)
self.assertEqual(len(result_data['negative']), 15)
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 2)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var"
query = "annotation-name=n_cells"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 1838)
self.assertEqual(df["n_cols"], 1)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertCountEqual(df["col_idx"], ["n_cells"])
def test_get_annotations_var_error(self):
endpoint = "annotations/var"
query = "annotation-name=notakey"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_mimetype_error(self):
endpoint = "data/var"
header = {"Accept": "xxx"}
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
def test_fbs_default(self):
endpoint = "data/var"
url = f"{self.TEST_URL_BASE}{endpoint}"
headers = {"Accept": "application/octet-stream"}
result = self.client.put(url, headers=headers)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.client.put(url, headers=headers, json=filter)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
def test_data_put_fbs(self):
endpoint = "data/var"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.client.put(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_get_fbs(self):
endpoint = "data/var"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
def test_data_put_filter_fbs(self):
endpoint = "data/var"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
filter = {"filter": {"var": {"index": [0, 1, 4]}}}
result = self.client.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 3)
self.assertIsNotNone(df["columns"])
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
def test_data_get_filter_fbs(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "data/var"
query = f"var:{index_col_name}=SIK1"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
def test_data_get_unknown_filter_fbs(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "data/var"
query = f"var:{index_col_name}=UNKNOWN"
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 0)
def test_data_put_single_var(self):
endpoint = "data/var"
url = f"{self.TEST_URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
result = self.client.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
def test_colors(self):
endpoint = "colors"
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = json.loads(result.data)
self.assertEqual(result_data, pbmc3k_colors)
@skip_if(lambda x: os.getenv("SKIP_STATIC"), "Skip static test when running locally")
def test_static(self):
endpoint = "static"
file = "assets/favicon.ico"
url = f"{endpoint}/{file}"
result = self.client.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
def test_genesets_config(self):
result = self.client.get(f"{self.TEST_URL_BASE}config")
config_data = json.loads(result.data)
params = config_data["config"]["parameters"]
annotations_genesets = params["annotations_genesets"]
annotations_genesets_readonly = params["annotations_genesets_readonly"]
annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
self.assertTrue(annotations_genesets)
self.assertTrue(annotations_genesets_readonly)
self.assertEqual(annotations_genesets_summary_methods, ["mean"])
def test_get_genesets(self):
endpoint = "genesets"
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url, headers={"Accept": "application/json"})
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = json.loads(result.data)
self.assertIsNotNone(result_data["genesets"])
def test_get_summaryvar(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "summarize/var"
# single column
filter = f"var:{index_col_name}=F5"
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
# multi-column
col_names = ["F5", "BEB3", "SIK1"]
filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.TEST_URL_BASE}{endpoint}?{query}"
header = {"Accept": "application/octet-stream"}
result = self.client.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
def test_post_summaryvar(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
endpoint = "summarize/var"
headers = {"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/octet-stream"}
# single column
filter = f"var:{index_col_name}=F5"
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.TEST_URL_BASE}{endpoint}?key={query_hash}"
result = self.client.post(url, headers=headers, data=query)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
# multi-column
col_names = ["F5", "BEB3", "SIK1"]
filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
query = f"method=mean&{filter}"
query_hash = hashlib.sha1(query.encode()).hexdigest()
url = f"{self.TEST_URL_BASE}{endpoint}?key={query_hash}"
result = self.client.post(url, headers=headers, data=query)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
self.assertEqual(df["col_idx"], [query_hash])
self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
class EndPointsCxg(EndPoints):
"""Test Case for endpoints"""
@classmethod
def setUpClass(cls):
app_config = AppConfig()
app_config.update_default_dataset_config(user_annotations__enable=False)
def test_get_genesets_json(self):
self.app.auth.is_user_authenticated = lambda: True
endpoint = "genesets"
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url, headers={"Accept": "application/json"})
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = json.loads(result.data)
self.assertIsNotNone(result_data["genesets"])
self.assertIsNotNone(result_data["tid"])
self.assertEqual(
result_data,
{
"genesets": [
{
"genes": [
{"gene_description": " a gene_description", "gene_symbol": "F5"},
{"gene_description": "", "gene_symbol": "SUMO3"},
{"gene_description": "", "gene_symbol": "SRM"},
],
"geneset_description": "a description",
"geneset_name": "first gene set name",
},
{
"genes": [
{"gene_description": "", "gene_symbol": "RER1"},
{"gene_description": "", "gene_symbol": "SIK1"},
],
"geneset_description": "",
"geneset_name": "second_gene_set",
},
{"genes": [], "geneset_description": "", "geneset_name": "third gene set"},
{"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"},
{"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
{
"genes": [
{"gene_description": "", "gene_symbol": "ACD"},
{"gene_description": "", "gene_symbol": "AATF"},
{"gene_description": "", "gene_symbol": "F5"},
{"gene_description": "", "gene_symbol": "PIGU"},
],
"geneset_description": "",
"geneset_name": "summary test",
},
{'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_delete'},
{'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_edit'},
{
'genes': [],
'geneset_description': '',
'geneset_name': 'fill_this_geneset'
},
{
'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
'geneset_description': '',
'geneset_name': 'empty_this_geneset'
},
{
'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
'geneset_description': '',
'geneset_name': 'brush_this_gene'
}
],
"tid": 0,
},
)
def test_get_genesets_csv(self):
endpoint = "genesets"
url = f"{self.TEST_URL_BASE}{endpoint}"
self.app.auth.is_user_authenticated = lambda: True
result = self.client.get(url, headers={"Accept": "text/csv"})
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "text/csv")
expected_data = """gene_set_name,gene_set_description,gene_symbol,gene_description\r
first gene set name,a description,F5, a gene_description\r
first gene set name,a description,SUMO3,\r
first gene set name,a description,SRM,\r
second_gene_set,,RER1,\r
second_gene_set,,SIK1,\r
third gene set,,,\r
fourth_gene_set,fourth description,,\r
fifth_dataset,,,\r
summary test,,ACD,\r
summary test,,AATF,\r
summary test,,F5,\r
summary test,,PIGU,\r
geneset_to_delete,,,\r
geneset_to_edit,,,\r
fill_this_geneset,,,\r
empty_this_geneset,,SIK1,\r
brush_this_gene,,SIK1,\r
"""
self.assertEqual(result.data.decode("utf-8"), expected_data)
def test_put_genesets(self):
endpoint = "genesets"
url = f"{self.TEST_URL_BASE}{endpoint}"
result = self.client.get(url, headers={"Accept": "application/json"})
self.assertEqual(result.status_code, HTTPStatus.OK)
test1 = {"tid": 3, "genesets": []}
result = self.client.put(url, json=test1)
self.assertEqual(result.status_code, HTTPStatus.METHOD_NOT_ALLOWED)
backend/test/test_czi_hosted/unit/common/test_corpora.py
-168
View File
@@ -1,168 +0,0 @@
import json
import shutil
import tempfile
import unittest
from http import HTTPStatus
import anndata
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.czi_hosted.common.corpora import (
corpora_get_versions_from_anndata,
corpora_is_version_supported,
corpora_get_props_from_anndata,
)
from backend.test.test_czi_hosted.unit import BaseTest
from backend.test import PROJECT_ROOT
VERSION = "v0.2"
class CorporaAPITest(unittest.TestCase):
def test_corpora_get_versions_from_anndata(self):
adata = self._get_h5ad()
if "version" in adata.uns:
del adata.uns["version"]
self.assertIsNone(corpora_get_versions_from_anndata(adata))
# something bogus
adata.uns["version"] = 99
self.assertIsNone(corpora_get_versions_from_anndata(adata))
# something legit
adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
def test_corpora_is_version_supported(self):
self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
def test_corpora_get_props_from_anndata(self):
adata = self._get_h5ad()
if "version" in adata.uns:
del adata.uns["version"]
self.assertIsNone(corpora_get_props_from_anndata(adata))
# something bogus
adata.uns["version"] = 99
self.assertIsNone(corpora_get_props_from_anndata(adata))
# unsupported version, but missing required values
adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
with self.assertRaises(ValueError):
corpora_get_props_from_anndata(adata)
# legit version, but missing required values
adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
with self.assertRaises(KeyError):
corpora_get_props_from_anndata(adata)
some_fields = {
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
"title": "title",
"layer_descriptions": "layer_descriptions",
"organism": "organism",
"organism_ontology_term_id": "organism_ontology_term_id",
"project_name": "project_name",
"project_description": "project_description",
"contributors": json.dumps([{"contributors": "contributors"}]),
"project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
}
for k in some_fields:
adata.uns[k] = some_fields[k]
some_fields["contributors"] = json.loads(some_fields["contributors"])
some_fields["project_links"] = json.loads(some_fields["project_links"])
self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
def test_corpora_get_props_from_anndata_v110(self):
adata = self._get_h5ad()
if "version" in adata.uns:
del adata.uns["version"]
self.assertIsNone(corpora_get_props_from_anndata(adata))
# legit version, but missing required values
adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
with self.assertRaises(KeyError):
corpora_get_props_from_anndata(adata)
# Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
some_110_fields = {
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
"title": "title",
"layer_descriptions": "layer_descriptions",
"organism": "organism",
"organism_ontology_term_id": "organism_ontology_term_id",
}
for k in some_110_fields:
adata.uns[k] = some_110_fields[k]
self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
def _get_h5ad(self):
return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
class CorporaRESTAPITest(BaseTest):
""" Confirm endpoints reflect Corpora-specific features """
@classmethod
def setCorporaFields(cls, path):
adata = anndata.read_h5ad(path)
corpora_props = {
"version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
"title": "PBMC3K",
"contributors": json.dumps([{"name": "name"}]),
"layer_descriptions": {"X": "raw counts"},
"organism": "human",
"organism_ontology_term_id": "unknown",
"project_name": "test project",
"project_description": "test description",
"project_links": json.dumps(
[{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
),
"default_embedding": "X_tsne",
}
adata.uns.update(corpora_props)
adata.write(path)
@classmethod
def setUpClass(cls, app_config=None):
if not app_config:
app_config = AppConfig()
cls.tmp_dir = tempfile.TemporaryDirectory()
src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
shutil.copyfile(src, dst)
cls.setCorporaFields(dst)
app_config.update_server_config(single_dataset__datapath=dst)
super().setUpClass(app_config)
cls.app.testing = True
cls.client = cls.app.test_client()
def setUp(self):
self.session = self.client
self.url_base = "/api/v0.2/"
def test_config(self):
endpoint = "config"
url = f"{self.url_base}{endpoint}"
header = {"Content-Type": "application/json"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = json.loads(result.data)
self.assertIsInstance(result_data["config"]["corpora_props"], dict)
self.assertIsInstance(result_data["config"]["parameters"], dict)
corpora_props = result_data["config"]["corpora_props"]
parameters = result_data["config"]["parameters"]
self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
self.assertEqual(corpora_props["organism"], "human")
self.assertEqual(parameters["default_embedding"], "tsne")
backend/test/test_czi_hosted/unit/common/test_nan_rest.py
-65
View File
@@ -1,65 +0,0 @@
from http import HTTPStatus
import math
import backend.test.decode_fbs as decode_fbs
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import FIXTURES_ROOT
from backend.test.test_czi_hosted.unit import BaseTest
VERSION = "v0.2"
BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
class WithNaNs(BaseTest):
"""Test Case for endpoints"""
@classmethod
def setUpClass(cls):
app_config = AppConfig()
app_config.update_server_config(single_dataset__datapath=f"{FIXTURES_ROOT}/nan.h5ad")
app_config.update_default_dataset_config(user_annotations__enable=True)
super().setUpClass(app_config)
cls.app.testing = True
cls.client = cls.app.test_client()
def setUp(self):
self.session = self.client
self.url_base = "api/v0.2/"
def test_initialize(self):
endpoint = "schema"
url = f"{self.url_base}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
def test_data(self):
endpoint = "data/var"
url = f"{self.url_base}{endpoint}"
filter = {"filter": {"var": {"index": [[0, 20]]}}}
header = {"Accept": "application/octet-stream"}
result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertTrue(math.isnan(df["columns"][3][3]))
def test_annotation_obs(self):
endpoint = "annotations/obs"
url = f"{self.url_base}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertTrue(math.isnan(df["columns"][2][0]))
def test_annotation_var(self):
endpoint = "annotations/var"
url = f"{self.url_base}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.data)
self.assertTrue(math.isnan(df["columns"][2][0]))
backend/test/test_czi_hosted/unit/common/test_rest.py
-79
View File
@@ -1,79 +0,0 @@
import unittest
from urllib.parse import parse_qs
from werkzeug.datastructures import MultiDict
from backend.common.errors import FilterError
from backend.czi_hosted.common.rest import _query_parameter_to_filter
def _qsparse(qs):
""" emulate what Flask/Werkzeug do to our QS """
return MultiDict(parse_qs(qs))
class FilterParseTests(unittest.TestCase):
""" Test cases for various filter parsing """
def test_queryparam_to_filter_parse(self):
# categories
self.assertEqual(
_query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
{
"obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
"var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
},
)
# ranges
self.assertEqual(
_query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
{
"obs": {
"annotation_value": [
{"name": "A", "min": 1, "max": 99.0},
{"name": "B", "max": 100.0},
{"name": "C", "min": 0.0},
]
},
},
)
# combo
self.assertEqual(
_query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
{
"var": {
"annotation_value": [
{"name": "B", "values": ["YES", "NO"]},
{"name": "A", "min": 1.0, "max": 99.0},
]
},
},
)
def test_queryparam_to_filter_escaping(self):
self.assertEqual(
_query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
{"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
)
def test_queryparam_to_filter_errors(self):
# should raise FilterError
filter_errors = [
"foo=bar", # no axis
"X=&Y=3", # no value
"X&Y=3", # no value
"moo:foo=bar", # bad axis
"obs:x=1,A", # non-numeric range
"var:X=1,2&var:X=3,4", # duplicate ranges
"var:Y=,",
"var:Y=2,",
"var:Y=,5",
"var:Y=*,",
"var:Y=,*",
"var:Y=*,*",
]
for qs in filter_errors:
with self.assertRaises(FilterError):
_query_parameter_to_filter(_qsparse(qs))
backend/test/test_czi_hosted/unit/common/test_writable_annotation.py
-317
View File
@@ -1,317 +0,0 @@
import json
import shutil
import unittest
from os import path, listdir
from unittest.mock import MagicMock
import numpy as np
import pandas as pd
import tiledb
from flask import Flask
from backend.common.errors import AnnotationCategoryNameError
from backend.czi_hosted.common.rest import annotations_put_fbs_helper, schema_get_helper
from backend.czi_hosted.data_common.matrix_loader import MatrixDataType
from backend.czi_hosted.db.cellxgene_orm import CellxGeneDataset, Annotation
from backend.test.fixtures.database import TestDatabase
from backend.test.test_czi_hosted.unit import make_fbs, data_with_tmp_tiledb_annotations, data_with_tmp_annotations
from backend.test import decode_fbs
TestDatabase()
class auth(object):
def get_user_id():
return "1234"
def get_user_name():
return "person name"
class WritableTileDBStoredAnnotationTest(unittest.TestCase):
def setUp(self):
self.user_id = "1234"
self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
self.data.dataset_config.user_annotations = self.annotations
self.db = self.annotations.db
self.n_rows = self.data.get_shape()[0]
self.test_dict = {
"cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
"cat_B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
}
self.fbs = make_fbs(self.test_dict)
self.df = pd.DataFrame(self.test_dict)
self.app = Flask("fake_app")
self.app.__setattr__("auth", auth)
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def annotation_put_fbs(self, fbs):
annotations_put_fbs_helper(self.data, fbs)
res = json.dumps({"status": "OK"})
return res
def test_category_name_throws_errors_for_categories_that_cant_be_converted_to_filenames(self):
with self.app.test_request_context():
bad_category_names = make_fbs(
{
"cat_A": pd.Series(["label_A"] * self.n_rows, dtype="category"),
"cat/B": pd.Series(["label_B"] * self.n_rows, dtype="category"),
}
)
with self.assertRaises(AnnotationCategoryNameError):
self.annotation_put_fbs(bad_category_names)
def test_convert_to_pandas__converts_tiledb_to_pandas_df(self):
with self.app.test_request_context():
self.annotations.write_labels(self.df, self.data)
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
)
# retrieve tiledb array
df = tiledb.open(annotation.tiledb_uri)
self.assertEqual(type(df), tiledb.array.SparseArray)
# convert to pandas df
pandas_df = self.annotations.convert_to_pandas_df(df, annotation.schema_hints)
self.assertEqual(type(pandas_df), pd.DataFrame)
def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
with self.app.test_request_context():
new_name = "new_dataset/location"
self.data.get_location = MagicMock(return_value=new_name)
num_datasets = len(self.db.query([CellxGeneDataset]))
self.annotation_put_fbs(self.fbs)
more_datasets = len(self.db.query([CellxGeneDataset]))
self.assertGreater(more_datasets, num_datasets)
self.assertGreater(len(self.db.query([CellxGeneDataset], [CellxGeneDataset.name == new_name])), 0)
def test_write_labels_links_to_existing_dataset(self):
with self.app.test_request_context():
# add dataset to to db
self.annotation_put_fbs(self.fbs)
num_datasets = len(self.db.query([CellxGeneDataset]))
# create another annotation with the same dataset
self.annotation_put_fbs(self.fbs)
same_num_datasets = len(self.db.query([CellxGeneDataset]))
self.assertEqual(num_datasets, same_num_datasets)
def test_read_labels_returns_pandas_df(self):
with self.app.test_request_context():
self.annotation_put_fbs(self.fbs)
pandas_df = self.annotations.read_labels(self.data)
self.assertEqual(type(pandas_df), pd.DataFrame)
def test_read_labels_returns_df_matching_original(self):
with self.app.test_request_context():
self.annotation_put_fbs(self.fbs)
pandas_df = self.annotations.read_labels(self.data)
self.assertEqual(pandas_df.shape, (self.n_rows, 2))
self.assertEqual(set(pandas_df.columns), {"cat_A", "cat_B"})
self.assertTrue(self.data.original_obs_index.equals(pandas_df.index))
self.assertTrue(np.all(pandas_df["cat_A"] == ["label_A"] * self.n_rows))
self.assertTrue(np.all(pandas_df["cat_B"] == ["label_B"] * self.n_rows))
def test_error_checks(self):
# verify that the expected errors are generated
with self.app.test_request_context():
n_rows = self.data.get_shape()[0]
fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
# ensure we catch attempt to overwrite non-writable data
with self.assertRaises(KeyError):
self.annotation_put_fbs(fbs_bad)
def test_write_labels_stores_df_as_tiledb_array(self):
with self.app.test_request_context():
self.annotations.write_labels(self.df, self.data)
# get uri
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
)
df = tiledb.open(annotation.tiledb_uri)
self.assertEqual(type(df), tiledb.array.SparseArray)
def test_remove_categories(self):
with self.app.test_request_context():
# update empty category data, which is how annotations are removed
empty = make_fbs({})
self.annotation_put_fbs(empty)
# verify that the tiledb uri is an empty string.
dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
annotation = self.db.query_for_most_recent(
Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
)
self.assertEqual(annotation.tiledb_uri, "")
# verify that read_labels returns None
df = self.annotations.read_labels(self.data)
self.assertIsNone(df)
class WritableAnnotationTest(unittest.TestCase):
def setUp(self):
self.data, self.tmp_dir, self.annotations, self.config= data_with_tmp_annotations(MatrixDataType.H5AD)
self.data.dataset_config.user_annotations = self.annotations
def tearDown(self):
shutil.rmtree(self.tmp_dir)
def annotation_put_fbs(self, fbs):
annotations_put_fbs_helper(self.data, fbs)
res = json.dumps({"status": "OK"})
return res
def test_error_checks(self):
# verify that the expected errors are generated
n_rows = self.data.get_shape()[0]
fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
# ensure we catch attempt to overwrite non-writable data
with self.assertRaises(KeyError):
self.annotation_put_fbs(fbs_bad)
def test_write_to_file(self):
# verify the file is written as expected
n_rows = self.data.get_shape()[0]
fbs = make_fbs(
{
"cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
"cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
}
)
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations.output_file))
df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
self.assertEqual(df.shape, (n_rows, 2))
self.assertEqual(set(df.columns), {"cat_A", "cat_B"})
self.assertTrue(self.data.original_obs_index.equals(df.index))
self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows))
self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows))
# verify complete overwrite on second attempt, AND rotation occurs
fbs = make_fbs(
{
"cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"),
"cat_C": pd.Series(["label_C"] * n_rows, dtype="category"),
}
)
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
self.assertTrue(path.exists(self.annotations.output_file))
df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
self.assertEqual(set(df.columns), {"cat_A", "cat_C"})
self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows))
self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows))
# rotation
name, ext = path.splitext(self.annotations.output_file)
backup_dir = f"{name}-backups"
self.assertTrue(path.isdir(backup_dir))
found_files = listdir(backup_dir)
self.assertEqual(len(found_files), 1)
def test_file_rotation_to_max_9(self):
# verify we stop rotation at 9
n_rows = self.data.get_shape()[0]
fbs = make_fbs(
{
"cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
"cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
}
)
for i in range(0, 11):
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
name, ext = path.splitext(self.annotations.output_file)
backup_dir = f"{name}-backups"
self.assertTrue(path.isdir(backup_dir))
found_files = listdir(backup_dir)
self.assertTrue(len(found_files) <= 9)
def test_put_get_roundtrip(self):
# verify that OBS PUTs (annotation_put_fbs) are accessible via
# GET (annotation_to_fbs_matrix)
n_rows = self.data.get_shape()[0]
fbs = make_fbs(
{
"cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
"cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
}
)
# put
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
# get
labels = self.annotations.read_labels(None)
fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
schema = schema_get_helper(self.data)
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
obs_index_col_name = schema["annotations"]["obs"]["index"]
self.assertEqual(annotations["n_rows"], n_rows)
self.assertEqual(annotations["n_cols"], 7)
self.assertIsNone(annotations["row_idx"])
self.assertEqual(
annotations["col_idx"],
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
)
col_idx = annotations["col_idx"]
self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows)
self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows)
# verify the schema was updated
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(
all_col_schema["cat_A"],
{"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
)
self.assertEqual(
all_col_schema["cat_B"],
{"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
)
def test_put_float_data(self):
# verify that OBS PUTs (annotation_put_fbs) are accessible via
# GET (annotation_to_fbs_matrix)
n_rows = self.data.get_shape()[0]
# verifies that floating point with decimals fail.
fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))})
with self.assertRaises(ValueError) as exception_context:
res = self.annotation_put_fbs(fbs)
self.assertEqual(str(exception_context.exception), "Columns may not have floating point types")
# verifies that floating point that can be converted to int passes
fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")})
res = self.annotation_put_fbs(fbs)
self.assertEqual(res, json.dumps({"status": "OK"}))
# check read_labels
labels = self.annotations.read_labels(None)
fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
schema = schema_get_helper(self.data)
annotations = decode_fbs.decode_matrix_FBS(fbsAll)
self.assertEqual(annotations["n_rows"], n_rows)
all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
self.assertEqual(
all_col_schema["cat_F_PASS"],
{"name": "cat_F_PASS", "type": "int32", "writable": True},
)
backend/test/test_czi_hosted/unit/common/utils/__init__.py
View File
backend/test/test_czi_hosted/unit/common/utils/test_cxg_generation_utils.py
-158
View File
@@ -1,158 +0,0 @@
import json
import unittest
from os import path, mkdir
from shutil import rmtree
from uuid import uuid4
import numpy as np
import tiledb
from pandas import Series, DataFrame
from backend.czi_hosted.common.utils.cxg_generation_utils import (
convert_dictionary_to_cxg_group,
convert_dataframe_to_cxg_array,
convert_ndarray_to_cxg_dense_array,
convert_matrix_to_cxg_array,
)
from backend.test import FIXTURES_ROOT
class TestCxgGenerationUtils(unittest.TestCase):
def setUp(self):
self.testing_cxg_temp_directory = f"{FIXTURES_ROOT}/{uuid4()}"
mkdir(self.testing_cxg_temp_directory)
def tearDown(self):
if path.isdir(self.testing_cxg_temp_directory):
rmtree(self.testing_cxg_temp_directory)
def test__convert_dictionary_to_cxg_group__writes_successfully(self):
random_dictionary = {"cookies": "chocolate_chip", "brownies": "chocolate", "cake": "double chocolate"}
dictionary_name = "favorite_desserts"
expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
convert_dictionary_to_cxg_group(
self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
)
array = tiledb.open(expected_array_directory)
actual_stored_metadata = dict(array.meta.items())
self.assertTrue(path.isdir(expected_array_directory))
self.assertTrue(isinstance(array, tiledb.DenseArray))
self.assertEqual(random_dictionary, actual_stored_metadata)
def test__convert_dataframe_to_cxg_array__writes_successfully(self):
random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int64)
random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
random_dataframe_name = f"random_dataframe_{uuid4()}"
random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
convert_dataframe_to_cxg_array(
self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
)
expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
expected_array_metadata = {
"cxg_schema": json.dumps(
{"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
)
}
actual_stored_dataframe_array = tiledb.open(expected_array_directory)
actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
self.assertTrue(path.isdir(expected_array_directory))
self.assertTrue(isinstance(actual_stored_dataframe_array, tiledb.DenseArray))
self.assertDictEqual(expected_array_metadata, actual_stored_dataframe_metadata)
self.assertTrue((actual_stored_dataframe_array[0:4]["int_category"] == random_int_category.to_numpy()).all())
self.assertTrue((actual_stored_dataframe_array[0:4]["bool_category"] == random_bool_category.to_numpy()).all())
def test__convert_ndarray_to_cxg_dense_array__writes_successfully(self):
ndarray = np.random.rand(3, 2)
ndarray_name = f"{self.testing_cxg_temp_directory}/awesome_ndarray_{uuid4()}"
convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, tiledb.Ctx())
actual_stored_array = tiledb.open(ndarray_name)
self.assertTrue(path.isdir(ndarray_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
self.assertTrue((actual_stored_array[:, :] == ndarray).all())
def test__convert_matrix_to_cxg_array__dense_array_writes_successfully(self):
matrix = np.float32(np.random.rand(3, 2))
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_matrix_{uuid4()}"
convert_matrix_to_cxg_array(matrix_name, matrix, False, tiledb.Ctx())
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
self.assertTrue((actual_stored_array[:, :] == matrix).all())
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros_empty_array(self):
matrix = np.zeros([3, 2])
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_zero_matrix_{uuid4()}"
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[:, :][""].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
matrix = np.zeros([3, 3])
matrix[0, 0] = 1
matrix[1, 1] = 1
matrix[2, 2] = 2
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_sparse_matrix_{uuid4()}"
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[0, 0][""] == 1)
self.assertTrue(actual_stored_array[1, 1][""] == 1)
self.assertTrue(actual_stored_array[2, 2][""] == 2)
self.assertTrue(actual_stored_array[:, :][""].size == 3)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
matrix = np.ones((3, 2))
# The column shift will be equal to the matrix since subtracting the column shift from the matrix will create
# a matrix of zeros which is sparse.
column_shift = np.ones((3, 2))
convert_matrix_to_cxg_array(
matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
)
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[:, :][""].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
matrix = np.ones((2, 2))
# Only column shift the first column of ones.
column_shift = np.array([[1, 0], [1, 0]])
convert_matrix_to_cxg_array(
matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
)
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[0, 1][""] == 1)
self.assertTrue(actual_stored_array[1, 1][""] == 1)
self.assertTrue(actual_stored_array[:, :][""].size == 2)
backend/test/test_czi_hosted/unit/common/utils/test_matrix_utils.py
-66
View File
@@ -1,66 +0,0 @@
import unittest
import numpy as np
from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
class TestMatrixUtils(unittest.TestCase):
def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
matrix = np.array([1, 2, 3])
self.assertFalse(is_matrix_sparse(matrix, 0))
self.assertTrue(is_matrix_sparse(matrix, 100))
def test__is_matrix_sparse__partially_populated_sparse_matrix_returns_true(self):
matrix = np.zeros([3, 4])
matrix[2][3] = 1.0
matrix[1][1] = 2.2
self.assertTrue(is_matrix_sparse(matrix, 50))
def test__is_matrix_sparse__partially_populated_dense_matrix_returns_false(self):
matrix = np.zeros([2, 2])
matrix[0][0] = 1.0
matrix[0][1] = 2.2
matrix[1][1] = 3.7
self.assertFalse(is_matrix_sparse(matrix, 50))
def test__is_matrix_sparse__giant_matrix_returns_false_early(self):
matrix = np.ones([20000, 20])
with self.assertLogs(level="INFO") as logger:
self.assertFalse(is_matrix_sparse(matrix, 1))
# Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
# non-zero elements in the matrix.
self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
def test__is_matrix_sparse_with_column_shift_encoding__regular_sparse_returns_true(self):
matrix = np.zeros([2, 2])
matrix[0][0] = 1.0
self.assertIsNotNone(get_column_shift_encode_for_matrix(matrix, 50))
def test__is_matrix_sparse_with_column_shift_encoding__column_shift_returns_same_value(self):
matrix = np.ones([2, 2])
expected_column_shift = [1, 1]
actual_column_shift = get_column_shift_encode_for_matrix(matrix, 50)
self.assertTrue((expected_column_shift == actual_column_shift).all())
def test__is_matrix_sparse_with_column_shift_encoding__impossible_column_shift_returns_none(self):
matrix = np.array([[1, 2], [3, 4]])
self.assertIsNone(get_column_shift_encode_for_matrix(matrix, 50))
def test__is_matrix_sparse_with_column_shift_encoding__giant_matrix_returns_false_early(self):
matrix = np.random.rand(20000, 20)
with self.assertLogs(level="INFO") as logger:
self.assertFalse(is_matrix_sparse(matrix, 1))
# Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
# non-zero elements in the matrix.
self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
backend/test/test_czi_hosted/unit/common/utils/test_sanitization_utils.py
-55
View File
@@ -1,55 +0,0 @@
import unittest
from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list, sanitize_keys_in_dictionary
class TestSanitizationUtils(unittest.TestCase):
def test__sanitize_values_in_list__not_strings_raises_exception(self):
keys_to_sanitize = [1, 2, 3]
with self.assertRaises(Exception) as exception_context:
sanitize_values_in_list(keys_to_sanitize)
self.assertIn("must contain all strings", str(exception_context.exception))
def test__sanitize_values_in_list__not_all_strings_raises_exception(self):
keys_to_sanitize = ["1", "2", 3]
with self.assertRaises(Exception) as exception_context:
sanitize_values_in_list(keys_to_sanitize)
self.assertIn("must contain all strings", str(exception_context.exception))
def test__sanitize_values_in_list__replace_non_ascii_character_with_underscore(self):
keys_to_sanitize = ["abc.", "~abc", "a~b/c"]
expected_sanitized_keys_dict = dict(zip(keys_to_sanitize, ["abc_", "_abc", "a_b_c"]))
actual_sanitized_keys_dict = sanitize_values_in_list(keys_to_sanitize)
self.assertEqual(expected_sanitized_keys_dict, actual_sanitized_keys_dict)
def test__sanitize_keys_in_dictionary__replace_non_ascii_character_with_underscore(self):
dictionary_to_sanitize = {"abc.": 3, "~abc": 4, "a~b/c": 5}
expected_sanitized_dict = {"abc_": 3, "_abc": 4, "a_b_c": 5}
actual_sanitized_dict = dictionary_to_sanitize
sanitize_keys_in_dictionary(actual_sanitized_dict)
self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
def test__sanitize_keys_in_dictionary__non_string_key_raises_exception(self):
dictionary_to_sanitize = {4: 3, "~abc": 4, "a~b/c": 5}
with self.assertRaises(Exception) as exception_context:
sanitize_keys_in_dictionary(dictionary_to_sanitize)
self.assertIn("must contain all strings", str(exception_context.exception))
def test__sanitize_keys_in_dictionary__replace_only_some_keys(self):
dictionary_to_sanitize = {"abc": 3, "~abc": 4, "a~b/c": 5}
expected_sanitized_dict = {"abc": 3, "_abc": 4, "a_b_c": 5}
actual_sanitized_dict = dictionary_to_sanitize
sanitize_keys_in_dictionary(actual_sanitized_dict)
self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
backend/test/test_czi_hosted/unit/common/utils/test_utils.py
-34
View File
@@ -1,34 +0,0 @@
import os
import shutil
import unittest
from backend.common.utils.utils import import_plugins
from backend.test import PROJECT_ROOT, random_string
class TestPlugins(unittest.TestCase):
""" Test plugin import functionality """
plugins_dir = f"{PROJECT_ROOT}/backend/test/test_czi_hosted/unit/plugins"
test_plugin_path = f"{plugins_dir}/foo.py"
secret = random_string(8)
@classmethod
def setUpClass(cls) -> None:
if not os.path.isdir(cls.plugins_dir):
os.mkdir(cls.plugins_dir)
with open(cls.test_plugin_path, "w") as fh:
fh.write(f'SECRET = "{cls.secret}"\n')
@classmethod
def tearDownClass(cls) -> None:
if os.path.isdir(cls.plugins_dir):
shutil.rmtree(cls.plugins_dir)
def test_import_plugins(self):
self.assertTrue(os.path.isfile(self.test_plugin_path))
loaded_modules = import_plugins("backend.test.test_czi_hosted.unit.plugins")
# test that import plugins found the file
self.assertEqual(["backend.test.test_czi_hosted.unit.plugins.foo"], [ele.__name__ for ele in loaded_modules])
# test that the module was properly executed
self.assertEqual(self.secret, loaded_modules[0].SECRET)
backend/test/test_czi_hosted/unit/compute/__init__.py
View File
backend/test/test_czi_hosted/unit/compute/test_diffexp_cxg.py
-183
View File
@@ -1,183 +0,0 @@
import os
import tempfile
import unittest
import numpy as np
from backend.czi_hosted.compute import diffexp_cxg
from backend.common.compute import diffexp_generic
from backend.czi_hosted.compute.diffexp_cxg import diffexp_ttest
from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
from backend.test.test_czi_hosted.performance.create_test_matrix import create_test_h5ad
from backend.test.test_czi_hosted.unit import app_config
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
class DiffExpTest(unittest.TestCase):
"""Tests the diffexp returns the expected results for one test case, using different
adaptor types and different algorithms."""
def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
extra_dataset_config["X_approximate_distribution"] = "normal" # hardwired for now
config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
loader = MatrixDataLoader(path)
adaptor = loader.open(config)
return adaptor
def get_mask(self, adaptor, start, stride):
"""Simple function to return a mask or rows"""
rows = adaptor.get_shape()[0]
sel = list(range(start, rows, stride))
mask = np.zeros(rows, dtype=bool)
mask[sel] = True
return mask
def compare_diffexp_results(self, results, expects):
self.assertEqual(len(results), len(expects))
for result, expect in zip(results, expects):
self.assertEqual(result[0], expect[0])
self.assertTrue(np.isclose(result[1], expect[1], 1e-6, 1e-4))
self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
def check_1_10_2_10(self, results):
"""Checks the results for a specific set of rows selections"""
positive_expects = [
[1712, 0.24104056, 0.0051788902660723345, 1.0],
[1575, 0.2615018, 0.007830310753043345, 1.0],
[693, 0.23106655, 0.008715846769131548, 1.0],
[916, 0.2395215, 0.009080596532247588, 1.0],
[77, 0.22927025, 0.010070392939027756, 1.0],
[782, 0.20581803, 0.010161745218916036, 1.0],
[913, 0.23841085, 0.010782030711612685, 1.0],
[910, 0.21493295, 0.014596411069229197, 1.0],
[1727, 0.21911663, 0.015168372104237176, 1.0],
[1443, 0.19814226, 0.015337080567465522, 1.0],
]
negative_expects = [
[956, -0.29662406, 0.0008649321884808977, 1.0],
[1124, -0.2607333, 0.0011717216548271284, 1.0],
[1809, -0.24854594, 0.0019304405196777848, 1.0],
[1754, -0.24683577, 0.005691734062127954, 1.0],
[948, -0.18708363, 0.006622111055981219, 1.0],
[1810, -0.2172082, 0.007055917428377063, 1.0],
[779, -0.21150622, 0.007202934422407284, 1.0],
[576, -0.19008157, 0.008272092578813124, 1.0],
[538, -0.21803819, 0.01062259019889307, 1.0],
[436, -0.2100364, 0.01127515110543434, 1.0],
]
self.compare_diffexp_results(results['positive'], positive_expects)
self.compare_diffexp_results(results['negative'], negative_expects)
def get_X_col(self, adaptor, cols):
varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
varmask[cols] = True
return adaptor.get_X_array(None, varmask)
def test_anndata_default(self):
"""Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
self.check_1_10_2_10(results)
def test_cxg_default(self):
"""Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)"""
adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
# run it through the adaptor
results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
self.check_1_10_2_10(results)
# run it directly
results = diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
def test_cxg_generic(self):
"""Test a cxg adaptor with the generic adaptor"""
adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg")
maskA = self.get_mask(adaptor, 1, 10)
maskB = self.get_mask(adaptor, 2, 10)
# run it directly
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
self.check_1_10_2_10(results)
def test_cxg_sparse(self):
self.sparse_diffexp(False)
def test_cxg_sparse_col_shift(self):
self.sparse_diffexp(True)
def sparse_diffexp(self, apply_col_shift):
with tempfile.TemporaryDirectory() as dirname:
# create a sparse matrix
h5adfile_path = os.path.join(dirname, "sparse.h5ad")
create_test_h5ad(h5adfile_path, 2000, 2000, 10, apply_col_shift)
h5ad_file_to_convert = H5ADDataFile(h5adfile_path, use_corpora_schema=False)
sparsename = os.path.join(dirname, "sparse.cxg")
h5ad_file_to_convert.to_cxg(sparsename, 11, True)
adaptor_anndata = self.load_dataset(h5adfile_path, extra_dataset_config=dict(embeddings__names=[]))
adaptor_sparse = self.load_dataset(sparsename)
assert adaptor_sparse.open_array("X").schema.sparse
assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift
densename = os.path.join(dirname, "dense.cxg")
h5ad_file_to_convert.to_cxg(densename, True, 0)
adaptor_dense = self.load_dataset(densename)
assert not adaptor_dense.open_array("X").schema.sparse
assert not adaptor_dense.has_array("X_col_shift")
maskA = self.get_mask(adaptor_anndata, 1, 10)
maskB = self.get_mask(adaptor_anndata, 2, 10)
diffexp_results_anndata = diffexp_generic.diffexp_ttest(adaptor_anndata, maskA, maskB, 10)
diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10)
diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10)
self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_sparse['positive'])
self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_sparse['negative'])
self.compare_diffexp_results(diffexp_results_anndata['positive'], diffexp_results_dense['positive'])
self.compare_diffexp_results(diffexp_results_anndata['negative'], diffexp_results_dense['negative'])
topcols_pos = np.array([x[0] for x in diffexp_results_anndata['positive']])
topcols_neg = np.array([x[0] for x in diffexp_results_anndata['negative']])
topcols = np.concatenate((topcols_pos, topcols_neg))
cols_anndata = self.get_X_col(adaptor_anndata, topcols)
cols_sparse = self.get_X_col(adaptor_sparse, topcols)
cols_dense = self.get_X_col(adaptor_dense, topcols)
assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0]
assert cols_anndata.shape[1] == len(diffexp_results_anndata['positive']) + len(diffexp_results_anndata['negative'])
def convert(mat, cols):
return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy()
cols_anndata = convert(cols_anndata, topcols)
cols_sparse = convert(cols_sparse, topcols)
cols_dense = convert(cols_dense, topcols)
x = adaptor_sparse.get_X_array()
assert x.shape == adaptor_sparse.get_shape()
for row in range(cols_anndata.shape[0]):
for col in range(cols_anndata.shape[1]):
vanndata = cols_anndata[row][col]
vsparse = cols_sparse[row][col]
vdense = cols_dense[row][col]
self.assertTrue(np.isclose(vanndata, vsparse, 1e-6, 1e-6))
self.assertTrue(np.isclose(vanndata, vdense, 1e-6, 1e-6))
backend/test/test_czi_hosted/unit/converters/__init__.py
View File
backend/test/test_czi_hosted/unit/converters/schema/__init__.py
View File
backend/test/test_czi_hosted/unit/converters/schema/test_gene_symbol.py
-61
View File
@@ -1,61 +0,0 @@
import os
import unittest
import pandas as pd
from backend.czi_hosted.converters.schema import gene_symbol
from backend.test import FIXTURES_ROOT
class TestHGNCSymbolChecker(unittest.TestCase):
def setUp(self):
self.test_hgnc_path = os.path.join(FIXTURES_ROOT, "hgnc_example.txt.gz")
self.hgnc_checker = gene_symbol.HGNCSymbolChecker.from_hgnc_records(self.test_hgnc_path)
def test_symbol_upgrade(self):
self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1"), "SEPTIN1")
self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2R"), "ADRB2")
self.assertEqual(self.hgnc_checker.upgrade_symbol("BAR"), "ADRB2")
self.assertEqual(self.hgnc_checker.upgrade_symbol("sept1"), "SEPTIN1")
self.assertEqual(self.hgnc_checker.upgrade_symbol("AdRb2R"), "ADRB2")
self.assertEqual(self.hgnc_checker.upgrade_symbol("bar"), "ADRB2")
# Strip off seurat endings when appropriate
self.assertEqual(self.hgnc_checker.upgrade_symbol("SEPT1.1"), "SEPTIN1")
self.assertEqual(self.hgnc_checker.upgrade_symbol("ADRB2-1"), "ADRB2")
# DIFF6 is ambiguous so don't upgrade it
self.assertEqual(self.hgnc_checker.upgrade_symbol("DIFF6"), "DIFF6")
self.assertEqual(self.hgnc_checker.upgrade_symbol("diff6"), "diff6")
# ARG1 is approved
self.assertEqual(self.hgnc_checker.upgrade_symbol("ARG1"), "ARG1")
self.assertEqual(self.hgnc_checker.upgrade_symbol("arg1"), "ARG1")
# HAP1 is both approved and withdrawn
self.assertEqual(self.hgnc_checker.upgrade_symbol("HAP1"), "HAP1")
self.assertEqual(self.hgnc_checker.upgrade_symbol("hap1"), "HAP1")
# Leave unknown symbols alone
self.assertEqual(self.hgnc_checker.upgrade_symbol("NOTASYMBOL"), "NOTASYMBOL")
self.assertEqual(self.hgnc_checker.upgrade_symbol("notasymbol"), "notasymbol")
# Upgrade HGNC ids unless you can't find it
self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:286"), "ADRB2")
self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:4812"), "HAP1")
self.assertEqual(self.hgnc_checker.upgrade_symbol("HGNC:123456"), "HGNC:123456")
def test_check_symbol(self):
self.assertEqual(self.hgnc_checker.check_symbol("SEPT1"), gene_symbol.SymbolStatus.UPGRADABLE)
self.assertEqual(self.hgnc_checker.check_symbol("DIFF6"), gene_symbol.SymbolStatus.AMBIGUOUS)
self.assertEqual(self.hgnc_checker.check_symbol("NOTASYMBOL"), gene_symbol.SymbolStatus.UNKNOWN)
# HAP1 is one of the approved and withdrawn symbols
self.assertEqual(self.hgnc_checker.check_symbol("HAP1"), gene_symbol.SymbolStatus.APPROVED)
def test_upgrade_index(self):
index = pd.Index(["SEPT1", "DIFF6", "NOTASYMBOL", "bar", "SEPTIN1"])
var_df = pd.DataFrame([[0] * len(index)], index=index)
upgraded_index = gene_symbol.get_upgraded_var_index(var_df, hgnc_path=self.test_hgnc_path)
self.assertEqual(upgraded_index.tolist(), ["SEPTIN1", "DIFF6", "NOTASYMBOL", "ADRB2", "SEPTIN1"])
backend/test/test_czi_hosted/unit/converters/schema/test_ontology.py
-128
View File
@@ -1,128 +0,0 @@
import json
import unittest.mock
from backend.czi_hosted.converters.schema import ontology
class TestOntologyParsing(unittest.TestCase):
def setUp(self):
self.curies = ["UBERON:0002048", "HsapDv:0000174", "NCBITaxon:9606", "EFO:0008995"]
self.names = ["UBERON", "HsapDv", "NCBITaxon", "EFO"]
self.values = ["0002048", "0000174", "9606", "0008995"]
self.iris = [
"http://purl.obolibrary.org/obo/UBERON_0002048",
"http://purl.obolibrary.org/obo/HsapDv_0000174",
"http://purl.obolibrary.org/obo/NCBITaxon_9606",
"http://www.ebi.ac.uk/efo/EFO_0008995",
]
URL_ROOT = "http://www.ebi.ac.uk/ols/api/ontologies/"
self.urls = [
URL_ROOT + "UBERON/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FUBERON_0002048",
URL_ROOT + "HsapDv/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FHsapDv_0000174",
URL_ROOT + "NCBITaxon/terms/http%253A%252F%252Fpurl.obolibrary.org%252Fobo%252FNCBITaxon_9606",
URL_ROOT + "EFO/terms/http%253A%252F%252Fwww.ebi.ac.uk%252Fefo%252FEFO_0008995",
]
self.responses = {
"UBERON:0002048": {
"iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
"description": ["Respiration organ that develops as an outpocketing of the esophagus."],
"label": "lung",
},
"HsapDv:0000174": {
"iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
"description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
"label": "1-month-old human stage",
},
"NCBITaxon:9606": {
"iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
"description": None,
"label": "Homo sapiens",
},
"EFO:0008995": {
"iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
"description": [
(
'10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
"gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent "
"of 1 million pipetting steps. Successive versions of the 10x chemistry use different "
"barcode locations to improve the sequencing yield and quality of 10x experiments."
)
],
"label": "10X sequencing",
},
}
def test_ontololgy_name(self):
for curie, expected_name in zip(self.curies, self.names):
self.assertEqual(ontology._ontology_name(curie), expected_name)
def test_ontololgy_value(self):
for curie, expected_value in zip(self.curies, self.values):
self.assertEqual(ontology._ontology_value(curie), expected_value)
def test_iri(self):
for curie, expected_iri in zip(self.curies, self.iris):
self.assertEqual(ontology._iri(curie), expected_iri)
def test_ontology_info_url(self):
for curie, expected_url in zip(self.curies, self.urls):
self.assertEqual(ontology._ontology_info_url(curie), expected_url)
def test_empty_ontology_info_url(self):
self.assertEqual(ontology._ontology_info_url(""), "")
class TestOntologyLookup(unittest.TestCase):
def setUp(self):
self.responses = {
"UBERON:0002048": {
"iri": "http://purl.obolibrary.org/obo/UBERON_0002048",
"description": ["Respiration organ that develops as an outpocketing of the esophagus."],
"label": "lung",
},
"HsapDv:0000174": {
"iri": "http://purl.obolibrary.org/obo/HsapDv_0000174",
"description": ["Infant stage that refers to an infant who is over 1 and under 2 months old."],
"label": "1-month-old human stage",
},
"NCBITaxon:9606": {
"iri": "http://purl.obolibrary.org/obo/NCBITaxon_9606",
"description": None,
"label": "Homo sapiens",
},
"EFO:0008995": {
"iri": "http://www.ebi.ac.uk/efo/EFO_0008995",
"description": [
('10X is a "synthetic long-read" technology and works by capturing a barcoded oligo-coated '
'gel-bead and 0.3x genome copies into a single emulsion droplet, processing the equivalent '
'of 1 million pipetting steps. Successive versions of the 10x chemistry use different barcode '
'locations to improve the sequencing yield and quality of 10x experiments.')
],
"label": "10X sequencing",
},
}
self.labels = {
"UBERON:0002048": "lung",
"HsapDv:0000174": "1-month-old human stage",
"NCBITaxon:9606": "Homo sapiens",
"EFO:0008995": "10X sequencing",
}
@unittest.mock.patch("requests.get")
def test_lookup_label(self, mock_get):
for curie, response in self.responses.items():
mock_get.return_value.content = json.dumps(response)
mock_get.return_value.json.return_value = response
mock_get.return_value.status_code = 200
label = ontology.get_ontology_label(curie)
self.assertEqual(label, self.labels[curie])
backend/test/test_czi_hosted/unit/converters/schema/test_remix.py
-257
View File
@@ -1,257 +0,0 @@
import json
import os
import unittest
import unittest.mock
import anndata
import numpy
import pandas as pd
import scanpy as sc
from backend.czi_hosted.converters.schema import remix
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
class TestApplySchema(unittest.TestCase):
def setUp(self):
self.source_h5ad_path = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
self.output_h5ad_path = f"{FIXTURES_ROOT}/test_remix.h5ad"
self.config_path = f"{FIXTURES_ROOT}/test_config.yaml"
self.bad_config_path = f"{FIXTURES_ROOT}/test_bad_config.yaml"
def tearDown(self):
try:
os.remove(self.output_h5ad_path)
except OSError:
pass
@unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
def test_apply_schema(self, mock_get_ontology_label):
mock_get_ontology_label.return_value = "test label"
remix.apply_schema(self.source_h5ad_path, self.config_path, self.output_h5ad_path)
new_adata = sc.read_h5ad(self.output_h5ad_path)
self.assertIn("cell_type", new_adata.obs.columns)
self.assertListEqual(["test label"], new_adata.obs["cell_type"].unique().tolist())
self.assertListEqual(
["CL:00001", "CL:00002", "CL:00003", "CL:00004", "CL:00005", "CL:00006", "CL:00007", "CL:00008"],
sorted(new_adata.obs["cell_type_ontology_term_id"].unique().tolist())
)
self.assertIn("version", new_adata.uns_keys())
@unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
def test_apply_bad_schema(self, mock_get_ontology_label):
mock_get_ontology_label.return_value = "test label"
remix.apply_schema(self.source_h5ad_path, self.bad_config_path, self.output_h5ad_path)
new_adata = sc.read_h5ad(self.output_h5ad_path)
# Should refuse to write the version
self.assertNotIn("version", new_adata.uns_keys())
class TestFieldParsing(unittest.TestCase):
def test_is_curie(self):
self.assertTrue(remix.is_curie("EFO:00001"))
self.assertTrue(remix.is_curie("UBERON:123456"))
self.assertTrue(remix.is_curie("HsapDv:0001"))
self.assertFalse(remix.is_curie("UBERON"))
self.assertFalse(remix.is_curie("UBERON:"))
self.assertFalse(remix.is_curie("123456"))
def test_is_ontology_field(self):
self.assertTrue(remix.is_ontology_field("tissue_ontology_term_id"))
self.assertTrue(remix.is_ontology_field("cell_type_ontology_term_id"))
self.assertFalse(remix.is_ontology_field("cell_ontology"))
self.assertFalse(remix.is_ontology_field("method"))
def test_get_label_field_name(self):
self.assertEqual("tissue", remix.get_label_field_name("tissue_ontology_term_id"))
self.assertEqual("cell_type", remix.get_label_field_name("cell_type_ontology_term_id"))
def test_split_suffix(self):
self.assertEqual(("UBERON:1234", " (organoid)"), remix.split_suffix("UBERON:1234 (organoid)"))
self.assertEqual(("UBERON:1234", " (cell culture)"), remix.split_suffix("UBERON:1234 (cell culture)"))
self.assertEqual(("UBERON:1234", ""), remix.split_suffix("UBERON:1234"))
self.assertEqual(("UBERON:1234 (something)", ""), remix.split_suffix("UBERON:1234 (something)"))
@unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
def test_get_curie_and_label(self, mock_get_ontology_label):
mock_get_ontology_label.return_value = "test label"
self.assertEqual(
remix.get_curie_and_label("UBERON:1234"),
("UBERON:1234", "test label")
)
self.assertEqual(
remix.get_curie_and_label("UBERON:1234 (cell culture)"),
("UBERON:1234 (cell culture)", "test label (cell culture)")
)
self.assertEqual(
remix.get_curie_and_label("whatever"),
("", "whatever")
)
class TestManipulateAnndata(unittest.TestCase):
def setUp(self):
self.cell_count = 20
self.gene_count = 200
X = numpy.random.randint(0, 1000, (self.cell_count, self.gene_count))
uns = {"organism": "monkey", "experiment": "monkey experiment"}
obs = pd.DataFrame(
index=[f"Cell{d}" for d in range(self.cell_count)],
columns=["tissue", "CellType"],
data=[["lung", "epithelial"]] * (self.cell_count // 2) + [["lung", "endothelial"]] * (self.cell_count // 2)
)
var = pd.DataFrame(index=[f"SEPT{d}" for d in range(self.gene_count)])
self.adata = anndata.AnnData(X=X, obs=obs, var=var, uns=uns)
def test_safe_add_field(self):
remix.safe_add_field(self.adata.obs, "tissue", ["monkey lung"] * self.cell_count)
self.assertEqual(self.adata.obs["tissue_original"].tolist(), ["lung"] * self.cell_count)
self.assertEqual(self.adata.obs["tissue"].tolist(), ["monkey lung"] * self.cell_count)
remix.safe_add_field(self.adata.uns, "contributors", [{"name": "contributor1"}, {"name": "contributor2"}])
self.assertEqual(
self.adata.uns["contributors"],
json.dumps([{"name": "contributor1"}, {"name": "contributor2"}])
)
@unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
def test_remix_uns(self, mock_get_ontology_label):
mock_get_ontology_label.return_value = "Pan troglodytes"
uns_config = {
"version": {
"corpora_schema_version": "1.0.0",
"corpora_encoding_version": "0.1.0"
},
"organism_ontology_term_id": "NCBITaxon:9598",
"contributors": [
{
"name": "scientist",
"email": "scientist@science.com"
}
]
}
remix.remix_uns(self.adata, uns_config)
self.assertEqual(
sorted(self.adata.uns_keys()),
sorted(["organism_original", "organism", "organism_ontology_term_id",
"contributors", "version", "experiment"])
)
self.assertEqual(self.adata.uns['organism'], "Pan troglodytes")
self.assertEqual(self.adata.uns['organism_original'], "monkey")
self.assertEqual(self.adata.uns['organism_ontology_term_id'], "NCBITaxon:9598")
self.assertEqual(self.adata.uns['contributors'],
json.dumps([{"name": "scientist", "email": "scientist@science.com"}]))
@unittest.mock.patch("backend.czi_hosted.converters.schema.ontology.get_ontology_label")
def test_remix_obs(self, mock_get_ontology_label):
mock_get_ontology_label.return_value = "lung (in a monkey)"
obs_config = {
"tissue_ontology_term_id": {
"tissue": {
"lung": "UBERON:00000"
}
},
"cell_color": {
"CellType": {
"epithelial": "fuschia",
"endothelial": "khaki"
}
},
"sex": "male"
}
remix.remix_obs(self.adata, obs_config)
self.assertEqual(
sorted(self.adata.obs_keys()),
sorted(["tissue", "tissue_ontology_term_id", "tissue_original", "CellType", "cell_color", "sex"])
)
self.assertTrue(all(v == "lung" for v in self.adata.obs.tissue_original))
self.assertTrue(all(v == "UBERON:00000" for v in self.adata.obs.tissue_ontology_term_id))
self.assertTrue(all(v == "lung (in a monkey)" for v in self.adata.obs.tissue))
self.assertTrue(all(v == "male" for v in self.adata.obs.sex))
self.assertTrue(all(v in (("epithelial", "fuschia"), ("endothelial", "khaki"))
for v in zip(self.adata.obs.CellType, self.adata.obs.cell_color)))
class TestFixupGeneSymbols(unittest.TestCase):
def setUp(self):
self.seurat_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/seurat_tutorial.h5ad"
self.seurat_merged_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/seurat_tutorial_merged.h5ad"
self.sctransform_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/sctransform.h5ad"
self.sctransform_merged_path = f"{PROJECT_ROOT}/czi_hosted/test/fixtures/schema_test_data/sctransform_merged.h5ad"
# There's lots of MALAT1, but it doesn't collide with any other names,
# so it shouldn't change during merging.
self.stable_gene = "MALAT1"
def test_fixup_gene_symbols_seurat(self):
if not os.path.isfile(self.seurat_path):
return unittest.skip(
"Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
"run czi_hosted/test/fixtures/schema_test_data/generate_test_data.sh"
)
original_adata = sc.read_h5ad(self.seurat_path)
merged_adata = sc.read_h5ad(self.seurat_merged_path)
fixup_config = {"X": "log1p", "counts": "raw", "scale.data": "log1p"}
fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
self.assertEqual(
merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum(),
fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
)
self.assertAlmostEqual(
merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum()
)
self.assertAlmostEqual(
merged_adata.layers["scale.data"][:, merged_adata.var.index == self.stable_gene].sum(),
fixed_adata.layers["scale.data"][:, fixed_adata.var.index == self.stable_gene].sum()
)
def test_fixup_gene_symbols_sctransform(self):
if not os.path.isfile(self.sctransform_path):
return unittest.skip(
"Skipping gene symbol conversion tests because test h5ads are not present. To create them, "
"run czi_hosted/test/fixtures/schema_test_data/generate_test_data.sh"
)
original_adata = sc.read_h5ad(self.sctransform_path)
merged_adata = sc.read_h5ad(self.sctransform_merged_path)
fixup_config = {"X": "log1p", "counts": "raw"}
fixed_adata = remix.fixup_gene_symbols(original_adata, fixup_config)
# sctransform does a bunch of stuff, including slightly modifying the
# raw counts. So we can't assert for exact equality the way we do with
# the vanilla seurat tutorial. But, the results should still be very
# close.
merged_raw_stable = merged_adata.layers["counts"][:, merged_adata.var.index == self.stable_gene].sum()
fixed_raw_stable = fixed_adata.raw.X[:, fixed_adata.var.index == self.stable_gene].sum()
self.assertLess(abs(merged_raw_stable - fixed_raw_stable), .001 * merged_raw_stable)
self.assertAlmostEqual(
merged_adata.X[:, merged_adata.var.index == self.stable_gene].sum(),
fixed_adata.X[:, fixed_adata.var.index == self.stable_gene].sum(),
0
)
backend/test/test_czi_hosted/unit/converters/schema/test_validate.py
-434
View File
@@ -1,434 +0,0 @@
import json
import unittest
import pandas as pd
import scanpy as sc
from backend.czi_hosted.converters.schema import validate
from backend.test import PROJECT_ROOT
class TestFieldValidation(unittest.TestCase):
def test_validate_stringified_list_of_dicts(self):
good = json.dumps([{"a": 1}, {2: "x", "z": "y"}])
not_stringified = [{"a": 1}, {2: "x", "z": "y"}]
not_a_list = json.dumps({"bad": "dict"})
not_json = "oh hey!"
self.assertTrue(validate._validate_stringified_list_of_dicts(good))
self.assertFalse(validate._validate_stringified_list_of_dicts(not_stringified))
self.assertFalse(validate._validate_stringified_list_of_dicts(not_a_list))
self.assertFalse(validate._validate_stringified_list_of_dicts(not_json))
def test_validate_human_readable_string(self):
good = "oh hey!"
curie = "EFO:0001"
ensg = "ENSG000001234"
enst = "ENST000005678"
self.assertTrue(validate._validate_human_readable_string(good))
self.assertFalse(validate._validate_human_readable_string(curie))
self.assertFalse(validate._validate_human_readable_string(ensg))
self.assertFalse(validate._validate_human_readable_string(enst))
def test_validate_curie(self):
self.assertTrue(validate._validate_curie("UBERON:00001", ["UBERON", "EFO"]))
self.assertTrue(validate._validate_curie("HsapDv:00002", ["HsapDv"]))
self.assertFalse(validate._validate_curie("HsapDv:00002", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_curie("EFO:00002 (organoid)", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_curie("EFO:00002 extra", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_curie("UBERON:ABCD", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_curie("Uberon:00002", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_curie("UBERON:", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_curie("UBERON", ["UBERON", "EFO"]))
def test_validate_suffixed_curie(self):
self.assertTrue(validate._validate_suffixed_curie("EFO:00001", ["UBERON", "EFO"]))
self.assertTrue(validate._validate_suffixed_curie("UBERON:00001 (cell culture)", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002 (organoid)", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002(organoid)", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_suffixed_curie("HsapDv:00002", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_suffixed_curie("EFO:00002 extra", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_suffixed_curie("UBERON:ABCD", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_suffixed_curie("Uberon:00002", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_suffixed_curie("UBERON:", ["UBERON", "EFO"]))
self.assertFalse(validate._validate_suffixed_curie("UBERON", ["UBERON", "EFO"]))
class TestColumnValidation(unittest.TestCase):
def test_validate_unique(self):
unique = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
index=["X", "Y", "Z"], columns=["col1", "col2"])
duped = pd.DataFrame([["abc", "def"], ["ghi", "qrs"], ["abc", "qrs"]],
index=["X", "Y", "X"], columns=["col1", "col2"])
schema_def = {"unique": True}
errors = validate._validate_column(unique.index, "index", "unique_df", schema_def)
self.assertFalse(errors)
errors = validate._validate_column(duped.index, "index", "duped_df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("is not unique", errors[0])
errors = validate._validate_column(unique["col1"], "col1", "unique_df", schema_def)
self.assertFalse(errors)
errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("is not unique", errors[0])
schema_def = {"unique": False}
errors = validate._validate_column(duped["col1"], "col1", "duped_df", schema_def)
self.assertFalse(errors)
def test_validate_nullable(self):
non_null = pd.DataFrame([["abc", "def"], ["ghi", "jkl"], ["mnop", "qrs"]],
index=["X", "Y", "Z"], columns=["col1", "col2"])
has_null = pd.DataFrame([["abc", "", None], ["ghi", "jkl", 1], ["mnop", "qrs", 2]],
index=["X", "Y", "Z"], columns=["col1", "col2", "col3"])
schema_def = {"nullable": False}
errors = validate._validate_column(non_null["col1"], "col1", "nonnull_df", schema_def)
self.assertFalse(errors)
errors = validate._validate_column(has_null["col1"], "col1", "hasnull_df", schema_def)
self.assertFalse(errors)
errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("contains empty values", errors[0])
errors = validate._validate_column(has_null["col3"], "col3", "hasnull_df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("contains empty values", errors[0])
schema_def = {"nullable": True}
errors = validate._validate_column(has_null["col2"], "col2", "hasnull_df", schema_def)
self.assertFalse(errors)
def test_human_readable(self):
hr_df = pd.DataFrame(
[["for you, a human", "UBERON:12345", "UBERON:1234 (thundercat)"],
["hope you're well", "bit of lungs", "brain"]],
index=["ENSG00001", "ENSG00002"],
columns=["good", "curie", "suffixed_curie"])
schema_def = {"type": "human-readable string"}
errors = validate._validate_column(hr_df["good"], "good", "hr", schema_def)
self.assertFalse(errors)
errors = validate._validate_column(hr_df["curie"], "curie", "hr", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("non-human-readable", errors[0])
errors = validate._validate_column(hr_df["suffixed_curie"], "suffixed_curie", "hr", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("non-human-readable", errors[0])
errors = validate._validate_column(hr_df.index, "ensg", "hr", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("non-human-readable", errors[0])
def test_curie(self):
curie_df = pd.DataFrame(
[["EFO:00001", "HsapDv:00001 (cell culture)", "EFO:", "MONDO:0001 cell culture"],
["UBERON:00002", "HsapDv:00002 (organoid)", "EFO:12345", "MONDO:0002 (baba yaga)"],
["EFO:0000000005", "HsapDv:000004 (humanzee)", "EFO:000002", "MONDO:0004 (TMNT)"]],
index=["X", "Y", "Z"],
columns=["good", "good_suffix", "bad", "bad_suffix"])
# Good
schema_def = {"type": "curie", "prefixes": ["EFO", "UBERON"]}
errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
self.assertFalse(errors)
# Good suffix
schema_def = {"type": "suffixed curie", "prefixes": ["HsapDv", "WHATEVER"]}
errors = validate._validate_column(curie_df["good_suffix"], "good_suffix", "curie_df", schema_def)
self.assertFalse(errors)
# Bad prefix
schema_def = {"type": "curie", "prefixes": ["EFO"]}
errors = validate._validate_column(curie_df["good"], "good", "curie_df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("invalid ontology", errors[0])
self.assertIn("must be curies from one of these", errors[0])
# Bad curies
schema_def = {"type": "curie", "prefixes": ["EFO"]}
errors = validate._validate_column(curie_df["bad"], "bad", "curie_df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("invalid ontology", errors[0])
# Bad suffixes
schema_def = {"type": "suffixed curie", "prefixes": ["EFO"]}
errors = validate._validate_column(curie_df["bad_suffix"], "bad_suffix", "curie_df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("invalid ontology", errors[0])
def test_enum(self):
enum_df = pd.DataFrame(
[["abc", "ghi"],
["def", "jkl"]],
index=["X", "Y"],
columns=["col1", "col2"])
# All match
schema_def = {"type": "string", "enum": ["abc", "def", "xyz"]}
errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
self.assertFalse(errors)
# Missing value
schema_def = {"type": "string", "enum": ["abc", "xyz"]}
errors = validate._validate_column(enum_df["col1"], "col1", "enum_df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("unpermitted values", errors[0])
class TestDictValidations(unittest.TestCase):
def test_key_presence(self):
schema_def = {"keys": {"abc": None, "def": None}}
dict_ = {"abc": "123", "def": "456"}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertFalse(errors)
# Missing keys are bad
dict_ = {"abc": "123"}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("missing key", errors[0])
# Extra keys are okay
dict_ = {"abc": "123", "def": "456", "xyz": "789"}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertFalse(errors)
# Better not be empty come on
dict_ = {}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertEqual(len(errors), 2)
def test_nullable(self):
schema_def = {"keys": {"abc": {"type": "string", "nullable": False},
"def": {"type": "string", "nullable": True}}}
dict_ = {"abc": "xyz", "def": ""}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertFalse(errors)
dict_ = {"abc": "", "def": ""}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("empty value", errors[0])
def test_recurse(self):
schema_def = {
"keys": {
"subdict": {
"type": "dict",
"keys": {
"subdict_key1": None,
"subdict_key2": None
}
},
"ontology": {
"type": "curie",
"prefixes": ["ONTOLOGY"]
},
"blob": {
"type": "stringified list of dicts"
}
}
}
dict_ = {
"subdict": {"subdict_key1": "any", "subdict_key2": "any"},
"ontology": "ONTOLOGY:123456",
"blob": json.dumps([{"abc": 123}, {"def": 456}])
}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertFalse(errors)
dict_ = {
"subdict": {"subdict_key1": "any"},
"ontology": "ONTOLOGY:123456",
"blob": json.dumps([{"abc": 123}, {"def": 456}])
}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("missing key", errors[0])
dict_ = {
"subdict": {"subdict_key1": "any", "subdict_key2": "any"},
"ontology": "oh no not an ontology term",
"blob": json.dumps([{"abc": 123}, {"def": 456}])
}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("invalid ontology", errors[0])
dict_ = {
"subdict": {"subdict_key1": "any", "subdict_key2": "any"},
"ontology": "ONTOLOGY:123456",
"blob": [{"abc": 123}, {"def": 456}]
}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("JSON-encoded list of dicts", errors[0])
# Multiple errors
dict_ = {
"subdict": {"subdict_key1": "any"},
"ontology": "oh no not an ontology term",
"blob": json.dumps([{"abc": 123}, {"def": 456}])
}
errors = validate._validate_dict(dict_, "d", schema_def)
self.assertEqual(len(errors), 2)
class TestDataframeValidation(unittest.TestCase):
def test_column_presence(self):
df = pd.DataFrame(
[["abc", "EFO:123"],
["def", "UBERON:456"]],
columns=["hr_string", "ontology"],
index=["X", "Y"]
)
schema_def = {
"columns": {
"hr_string": {"type": "human-readable string"},
"ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
}
}
errors = validate._validate_dataframe(df, "df", schema_def)
self.assertFalse(errors)
schema_def = {
"columns": {
"hr_string": {"type": "human-readable string"},
"another_hr_string": {"type": "human-readable string"},
"ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
}
}
errors = validate._validate_dataframe(df, "df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("missing column", errors[0])
# Extra is okay
df = pd.DataFrame(
[["abc", "EFO:123", "extra"],
["def", "UBERON:456", "extra"]],
columns=["hr_string", "ontology", "extra"],
index=["X", "Y"]
)
schema_def = {
"columns": {
"hr_string": {"type": "human-readable string"},
"ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
}
}
errors = validate._validate_dataframe(df, "df", schema_def)
self.assertFalse(errors)
def test_index(self):
df = pd.DataFrame(
[["abc", "123"],
["def", "456"]],
columns=["col1", "col2"],
index=["ENSG0001", "ENSG0002"]
)
schema_def = {"index": {"unique": True}}
errors = validate._validate_dataframe(df, "df", schema_def)
self.assertFalse(errors)
schema_def = {"index": {"type": "human-readable string"}}
errors = validate._validate_dataframe(df, "df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("non-human-readable", errors[0])
df = pd.DataFrame(
[["abc", "123"],
["def", "456"]],
columns=["col1", "col2"],
index=["ENSG0001", "ENSG0001"]
)
schema_def = {"index": {"unique": True}}
errors = validate._validate_dataframe(df, "df", schema_def)
self.assertEqual(len(errors), 1)
self.assertIn("is not unique", errors[0])
def test_recurse(self):
df = pd.DataFrame(
[["abc", "HsapDv:0001"],
["EFO:123", "UBERON:456"]],
columns=["hr_string", "ontology"],
index=["X", "Y"]
)
schema_def = {
"columns": {
"hr_string": {"type": "human-readable string"},
"ontology": {"type": "curie", "prefixes": ["EFO", "UBERON"]}
}
}
errors = validate._validate_dataframe(df, "df", schema_def)
self.assertEqual(len(errors), 2)
self.assertEqual(len([e for e in errors if "non-human-readable" in e]), 1)
self.assertEqual(len([e for e in errors if "invalid ontology" in e]), 1)
class TestGetSchema(unittest.TestCase):
def test_get_schema(self):
self.assertIsInstance(validate.get_schema_definition("1.0.0"), dict)
with self.assertRaises(ValueError):
validate.get_schema_definition("10.1.5")
class TestValidate(unittest.TestCase):
def setUp(self):
self.source_h5ad_path = f"{PROJECT_ROOT}/backend/test/fixtures/pbmc3k-CSC-gz.h5ad"
def test_shallow(self):
adata = sc.read_h5ad(self.source_h5ad_path)
self.assertFalse(validate.validate_adata(adata, True))
adata.uns["version"] = {
"corpora_schema_version": "1.0.0",
"corpora_encoding_version": "0.1.0"
}
self.assertTrue(validate.validate_adata(adata, True))
def test_deep(self):
adata = sc.read_h5ad(self.source_h5ad_path)
self.assertFalse(validate.validate_adata(adata, False))
adata.uns["version"] = {
"corpora_schema_version": "1.0.0",
"corpora_encoding_version": "0.1.0"
}
self.assertFalse(validate.validate_adata(adata, False))
backend/test/test_czi_hosted/unit/converters/test_h5ad_data_file.py
-284
View File
@@ -1,284 +0,0 @@
import json
import unittest
from glob import glob
from os import remove, path
from shutil import rmtree
from uuid import uuid4
import anndata
import numpy as np
from pandas import Series, DataFrame
import tiledb
from backend.czi_hosted.common.corpora import CorporaConstants
from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile
from backend.test import PROJECT_ROOT
class TestH5ADDataFile(unittest.TestCase):
def setUp(self):
self.sample_anndata = self._create_sample_anndata_dataset()
self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata)
self.sample_output_directory = path.splitext(self.sample_h5ad_filename)[0] + ".cxg"
def tearDown(self):
if self.sample_h5ad_filename:
remove(self.sample_h5ad_filename)
if path.isdir(self.sample_output_directory):
rmtree(self.sample_output_directory)
def test__create_h5ad_data_file__non_h5ad_raises_exception(self):
non_h5ad_filename = "my_fancy_dataset.csv"
with self.assertRaises(Exception) as exception_context:
H5ADDataFile(non_h5ad_filename)
self.assertIn("File must be an H5AD", str(exception_context.exception))
def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self):
with self.assertLogs(level="WARN") as logger:
H5ADDataFile(
self.sample_h5ad_filename,
dataset_title="My Awesome Dataset",
dataset_about="http://www.awesomedataset.com",
use_corpora_schema=False,
)
self.assertIn("will override any metadata that is extracted", logger.output[0])
def test__create_h5ad_data_file__reads_anndata_successfully(self):
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all())
self.assertEqual(
h5ad_file.anndata.obs.sort_index(inplace=True), self.sample_anndata.obs.sort_index(inplace=True)
)
self.assertEqual(
h5ad_file.anndata.var.sort_index(inplace=True), self.sample_anndata.var.sort_index(inplace=True)
)
for key in h5ad_file.anndata.obsm.keys():
self.assertIn(key, self.sample_anndata.obsm.keys())
self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
for key in self.sample_anndata.obsm.keys():
self.assertIn(key, h5ad_file.anndata.obsm.keys())
self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
def test__create_h5ad_data_file__copies_index_of_obs_and_var_to_column(self):
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
# The automatic name chosen for the index should be "name_0"
self.assertNotIn("name_0", self.sample_anndata.obs.columns)
self.assertIn("name_0", h5ad_file.obs.columns)
self.assertNotIn("name_0", self.sample_anndata.var.columns)
self.assertIn("name_0", h5ad_file.var.columns)
def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self):
h5ad_file = H5ADDataFile(
self.sample_h5ad_filename,
use_corpora_schema=False,
obs_index_column_name="float_category",
vars_index_column_name="int_category",
)
self.assertNotIn("name_0", h5ad_file.obs.columns)
self.assertNotIn("name_0", h5ad_file.var.columns)
def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self):
with self.assertRaises(Exception) as exception_context:
H5ADDataFile(
self.sample_h5ad_filename,
use_corpora_schema=False,
obs_index_column_name="float_category",
vars_index_column_name="bool_category",
)
self.assertIn("Please prepare data to contain unique values", str(exception_context.exception))
def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self):
with self.assertRaises(Exception) as exception_context:
H5ADDataFile(
self.sample_h5ad_filename,
use_corpora_schema=False,
obs_index_column_name="unknown_category",
vars_index_column_name="i_dont_exist",
)
self.assertIn("does not exist", str(exception_context.exception))
def test__create_h5ad_data_file__extract_about_and_title_from_dataset(self):
h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
self.assertEqual(h5ad_file.dataset_title, "random_link_name")
self.assertEqual(h5ad_file.dataset_about, "www.link.com")
def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self):
h5ad_file = H5ADDataFile(
self.sample_h5ad_filename, dataset_about="override_about", dataset_title="override_title"
)
self.assertEqual(h5ad_file.dataset_title, "override_title")
self.assertEqual(h5ad_file.dataset_about, "override_about")
def test__to_cxg__simple_anndata_no_corpora_and_sparse(self):
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
h5ad_file.to_cxg(self.sample_output_directory, 100)
self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, True)
def test__to_cxg__simple_anndata_with_corpora_and_sparse(self):
h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
h5ad_file.to_cxg(self.sample_output_directory, 100)
self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, True)
def test__to_cxg__simple_anndata_no_corpora_and_dense(self):
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
h5ad_file.to_cxg(self.sample_output_directory, 0)
self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, False)
def test__to_cxg__simple_anndata_with_corpora_and_dense(self):
h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
h5ad_file.to_cxg(self.sample_output_directory, 0)
self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, False)
def test__to_cxg__with_sparse_column_encoding(self):
anndata = self._create_sample_anndata_dataset()
anndata.X = np.ones((3, 4))
sparse_with_column_shift_filename = self._write_anndata_to_file(anndata)
h5ad_file = H5ADDataFile(sparse_with_column_shift_filename)
h5ad_file.to_cxg(self.sample_output_directory, 50)
self._validate_cxg_and_h5ad_content_match(
sparse_with_column_shift_filename, self.sample_output_directory, False, has_column_encoding=True
)
# Clean up
remove(sparse_with_column_shift_filename)
def _validate_cxg_and_h5ad_content_match(self, h5ad_filename, cxg_directory, is_sparse, has_column_encoding=False):
anndata_object = anndata.read_h5ad(h5ad_filename)
# Array locations
metadata_array_location = f"{cxg_directory}/cxg_group_metadata"
main_x_array_location = f"{cxg_directory}/X"
embedding_array_location = f"{cxg_directory}/emb"
specific_embedding_array_location = f"{self.sample_output_directory}/emb/awesome_embedding"
obs_array_location = f"{cxg_directory}/obs"
var_array_location = f"{cxg_directory}/var"
x_col_shift_array_location = f"{cxg_directory}/X_col_shift"
# Assert CXG structure
self.assertEqual(tiledb.object_type(cxg_directory), "group")
self.assertEqual(tiledb.object_type(obs_array_location), "array")
self.assertEqual(tiledb.object_type(var_array_location), "array")
self.assertEqual(tiledb.object_type(main_x_array_location), "array")
self.assertEqual(tiledb.object_type(embedding_array_location), "group")
self.assertEqual(tiledb.object_type(specific_embedding_array_location), "array")
if has_column_encoding:
self.assertEqual(tiledb.object_type(x_col_shift_array_location), "array")
# Validate metadata
metadata_array = tiledb.DenseArray(metadata_array_location, mode="r")
self.assertIn("cxg_version", metadata_array.meta)
# Validate obs index
obs_array = tiledb.DenseArray(obs_array_location, mode="r")
expected_index_data = anndata_object.obs.index.to_numpy()
index_name = json.loads(obs_array.meta["cxg_schema"])["index"]
actual_index_data = obs_array.query(attrs=[index_name])[:][index_name]
self.assertTrue(np.array_equal(expected_index_data, actual_index_data))
# Validate obs columns
expected_columns = list(anndata_object.obs.columns.values)
for column_name in expected_columns:
expected_data = anndata_object.obs[column_name].to_numpy()
actual_data = obs_array.query(attrs=[column_name])[:][column_name]
self.assertTrue(np.array_equal(expected_data, actual_data))
# Validate var index
var_array = tiledb.DenseArray(var_array_location, mode="r")
expected_index_data = anndata_object.var.index.to_numpy()
index_name = json.loads(var_array.meta["cxg_schema"])["index"]
actual_index_data = var_array.query(attrs=[index_name])[:][index_name]
self.assertTrue(np.array_equal(expected_index_data, actual_index_data))
# Validate var columns
expected_columns = anndata_object.var.columns.values
for column_name in expected_columns:
expected_data = anndata_object.var[column_name].to_numpy()
actual_data = var_array.query(attrs=[column_name])[:][column_name]
self.assertTrue(np.array_equal(expected_data, actual_data))
# Validate embedding
expected_embedding_data = anndata_object.obsm.get("X_awesome_embedding")
embedding_array = tiledb.DenseArray(specific_embedding_array_location, mode="r")
actual_embedding_data = embedding_array[:, 0:2]
self.assertTrue(np.array_equal(expected_embedding_data, actual_embedding_data))
# Validate X matrix if not column shifted
if not has_column_encoding:
expected_x_data = anndata_object.X
if is_sparse:
x_array = tiledb.SparseArray(main_x_array_location, mode="r")
actual_x_data = np.reshape(x_array[:, :][""], expected_x_data.shape)
else:
x_array = tiledb.DenseArray(main_x_array_location, mode="r")
actual_x_data = x_array[:, :]
self.assertTrue(np.array_equal(expected_x_data, actual_x_data))
def _write_anndata_to_file(self, anndata):
temporary_filename = f"{PROJECT_ROOT}/backend/test/fixtures/{uuid4()}.h5ad"
anndata.write(temporary_filename)
return temporary_filename
def _create_sample_anndata_dataset(self):
# Create X
X = np.random.rand(3, 4)
# Create obs
random_string_category = Series(data=["a", "b", "b"], dtype="category")
random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32)
obs_dataframe = DataFrame(
data={"string_category": random_string_category, "float_category": random_float_category}
)
obs = obs_dataframe
# Create vars
random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int32)
random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
var_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
var = var_dataframe
# Create embeddings
random_embedding = np.random.rand(3, 2)
obsm = {"X_awesome_embedding": random_embedding}
# Create uns corpora metadata
uns = {}
for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
uns[metadata_field] = "random"
for metadata_field in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
uns[metadata_field] = json.dumps({"random_key": "random_value"})
# Need to carefully set the corpora schema versions in order for tests to pass.
uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
# Set project links to be a dictionary
uns["project_links"] = json.dumps(
[{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}]
)
return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns)
backend/test/test_czi_hosted/unit/data_anndata/__init__.py
View File
backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor.py
-198
View File
@@ -1,198 +0,0 @@
import json
import sys
import time
import unittest
import numpy as np
import pandas as pd
import pytest
from parameterized import parameterized_class
from backend.common.utils.data_locator import DataLocator
from backend.common.errors import FilterError
from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
from backend.test.test_czi_hosted.unit import app_config
from backend.test import PROJECT_ROOT, FIXTURES_ROOT, decode_fbs
from backend.test.fixtures.fixtures import pbmc3k_colors
"""
Test the anndata adaptor using the pbmc3k data set.
"""
@parameterized_class(
("data_locator", "backed"),
[
(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False),
(f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False),
(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True),
(f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True),
],
)
class AdaptorTest(unittest.TestCase):
def setUp(self):
config = app_config(self.data_locator, self.backed)
self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
def test_init(self):
self.assertEqual(self.data.cell_count, 2638)
self.assertEqual(self.data.gene_count, 1838)
epsilon = 0.000_005
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_mandatory_annotations(self):
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
self.assertIn(obs_index_col_name, self.data.data.obs)
self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertIn(var_index_col_name, self.data.data.var)
self.assertEqual(list(self.data.data.var.index), list(range(1838)))
@pytest.mark.filterwarnings("ignore:Anndata data matrix")
def test_data_type(self):
# don't run the test on the more exotic data types, as they don't
# support the astype() interface (used by this test, but not underlying app)
if isinstance(self.data.data.X, np.ndarray):
self.data.data.X = self.data.data.X.astype("float64")
with self.assertWarns(UserWarning):
self.data._validate_data_types()
def test_filter_idx(self):
filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 102)
def test_filter_complex(self):
filter_ = {
"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 91)
def test_obs_and_var_names(self):
self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
def test_get_colors(self):
self.assertEqual(self.data.get_colors(), pbmc3k_colors)
def test_get_schema(self):
with open(f"{FIXTURES_ROOT}/schema.json") as fh:
schema = json.load(fh)
self.assertDictEqual(self.data.get_schema(), schema)
def test_schema_produces_error(self):
self.data.data.obs["time"] = pd.Series(
list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
)
with pytest.raises(TypeError):
self.data._create_schema()
def test_layout(self):
fbs = self.data.layout_to_fbs_matrix(fields=None)
layout = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(layout["n_cols"], 6)
self.assertEqual(layout["n_rows"], 2638)
X = layout["columns"][0]
self.assertTrue((X >= 0).all() and (X <= 1).all())
Y = layout["columns"][1]
self.assertTrue((Y >= 0).all() and (Y <= 1).all())
def test_layout_fields(self):
""" X_pca, X_tsne, X_umap are available """
fbs = self.data.layout_to_fbs_matrix(["pca"])
layout = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(layout["n_cols"], 2)
self.assertEqual(layout["n_rows"], 2638)
self.assertCountEqual(layout["col_idx"], ["pca_0", "pca_1"])
fbs = self.data.layout_to_fbs_matrix(["tsne", "pca"])
layout = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(layout["n_cols"], 4)
self.assertEqual(layout["n_rows"], 2638)
self.assertCountEqual(layout["col_idx"], ["tsne_0", "tsne_1", "pca_0", "pca_1"])
def test_annotations(self):
fbs = self.data.annotation_to_fbs_matrix("obs")
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["n_cols"], 5)
obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
self.assertEqual(
annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
)
fbs = self.data.annotation_to_fbs_matrix("var")
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 1838)
self.assertEqual(annotations["n_cols"], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
def test_annotation_fields(self):
fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 2638)
self.assertEqual(annotations["n_cols"], 2)
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
annotations = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(annotations["n_rows"], 1838)
self.assertEqual(annotations["n_cols"], 1)
def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
def test_data_frame(self):
f1 = {"var": {"index": [[0, 10]]}}
fbs = self.data.data_frame_to_fbs_matrix(f1, "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 10)
with self.assertRaises(ValueError):
self.data.data_frame_to_fbs_matrix(None, "obs")
def test_filtered_data_frame(self):
filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 1040)
filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
with self.assertRaises(FilterError):
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_data_named_gene(self):
var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 1)
self.assertEqual(data["col_idx"], [4])
filter_ = {
"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}}
}
fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
data = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(data["n_rows"], 2638)
self.assertEqual(data["n_cols"], 3)
self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
backend/test/test_czi_hosted/unit/data_anndata/test_anndata_adaptor_data_load.py
-86
View File
@@ -1,86 +0,0 @@
import unittest
import json
from backend.common.utils.data_locator import DataLocator
from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import PROJECT_ROOT
class DataLoadAdaptorTest(unittest.TestCase):
"""
Test file loading, including deferred loading/update.
"""
def setUp(self):
self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
config = AppConfig()
config.update_server_config(single_dataset__datapath=self.data_file.path)
config.update_server_config(app__flask_secret_key="secret")
config.complete_config()
self.data = AnndataAdaptor(self.data_file, config)
def test_delayed_load_data(self):
self.data._create_schema()
self.assertEqual(self.data.cell_count, 2638)
self.assertEqual(self.data.gene_count, 1838)
epsilon = 0.000_005
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_diffexp_topN(self):
f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
self.assertEqual(len(result['positive']), 10)
self.assertEqual(len(result['negative']), 10)
result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
self.assertEqual(len(result['positive']), 20)
self.assertEqual(len(result['negative']), 20)
class DataLocatorAdaptorTest(unittest.TestCase):
"""
Test various types of data locators we expect to consume
"""
def get_basic_config(self):
config = AppConfig()
config.update_server_config(
single_dataset__obs_names=None, single_dataset__var_names=None,
)
config.update_server_config(app__flask_secret_key="secret")
config.update_default_dataset_config(
embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
)
return config
def stdAsserts(self, data):
""" run these each time we load the data """
self.assertIsNotNone(data)
self.assertEqual(data.cell_count, 2638)
self.assertEqual(data.gene_count, 1838)
def test_posix_file(self):
locator = DataLocator("../../example-dataset/pbmc3k.h5ad")
config = self.get_basic_config()
config.update_server_config(single_dataset__datapath=locator.path)
config.complete_config()
data = AnndataAdaptor(locator, config)
self.stdAsserts(data)
def test_url_https(self):
url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
locator = DataLocator(url)
config = self.get_basic_config()
data = AnndataAdaptor(locator, config)
self.stdAsserts(data)
def test_url_http(self):
url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
locator = DataLocator(url)
config = self.get_basic_config()
data = AnndataAdaptor(locator, config)
self.stdAsserts(data)
backend/test/test_czi_hosted/unit/data_anndata/test_nan_anndata_adaptor.py
-64
View File
@@ -1,64 +0,0 @@
import math
import unittest
import warnings
import pytest
from backend.common.utils.data_locator import DataLocator
from backend.common.errors import FilterError
from backend.czi_hosted.data_anndata.anndata_adaptor import AnndataAdaptor
from backend.test.test_czi_hosted.unit import app_config
from backend.test import FIXTURES_ROOT, decode_fbs
class NaNTest(unittest.TestCase):
def setUp(self):
self.data_locator = DataLocator(f"{FIXTURES_ROOT}/nan.h5ad")
self.config = app_config(self.data_locator.path)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
self.data = AnndataAdaptor(self.data_locator, self.config)
self.data._create_schema()
def test_load(self):
with self.assertLogs(level="WARN") as logger:
self.data = AnndataAdaptor(self.data_locator, self.config)
self.assertTrue(logger.output)
def test_init(self):
self.assertEqual(self.data.cell_count, 100)
self.assertEqual(self.data.gene_count, 100)
epsilon = 0.000_005
self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
def test_dataframe(self):
data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var"))
self.assertIsNotNone(data_frame_var)
self.assertEqual(data_frame_var["n_rows"], 100)
self.assertEqual(data_frame_var["n_cols"], 100)
self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
with pytest.raises(FilterError):
self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
with pytest.raises(FilterError):
filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
def test_dataframe_obs_not_implemented(self):
with self.assertRaises(ValueError) as cm:
decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs"))
self.assertIsNotNone(cm.exception)
def test_annotation(self):
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
var_index_col_name = self.data.schema["annotations"]["var"]["index"]
self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
self.assertEqual(annotations["n_rows"], 100)
self.assertTrue(math.isnan(annotations["columns"][2][0]))
backend/test/test_czi_hosted/unit/data_common/__init__.py
View File
backend/test/test_czi_hosted/unit/data_common/fbs/__init__.py
View File
backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py
-82
View File
@@ -1,82 +0,0 @@
import unittest
import pandas as pd
import numpy as np
from scipy import sparse
from backend.test import decode_fbs
from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
class FbsTests(unittest.TestCase):
"""Test Case for Matrix FBS data encode/decode """
def test_encode_boundary(self):
""" test various boundary checks """
# row indexing is unsupported
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
# matrix must be 2D
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
with self.assertRaises(ValueError):
encode_matrix_fbs(matrix=np.ones((10,)))
def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
d = decode_fbs.decode_matrix_FBS(fbs)
self.assertEqual(d["n_rows"], dims[0])
self.assertEqual(d["n_cols"], dims[1])
self.assertIsNone(d["row_idx"])
self.assertEqual(len(d["columns"]), dims[1])
for i in range(0, len(d["columns"])):
self.assertEqual(len(d["columns"][i]), dims[0])
self.assertIsInstance(d["columns"][i], expected_types[i][0])
if expected_types[i][1] is not None:
self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
if expected_column_idx is not None:
self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
def test_encode_DataFrame(self):
df = pd.DataFrame(
data={
"a": np.zeros((10,), dtype=np.float32),
"b": np.ones((10,), dtype=np.int64),
"c": np.array([i for i in range(0, 10)], dtype=np.uint16),
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}
)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None))
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
def test_encode_ndarray(self):
arr = np.zeros((3, 2), dtype=np.float32)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (3, 2), expected_types, None)
def test_encode_sparse(self):
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
self.fbs_checks(fbs, (2, 3), expected_types, None)
def test_roundtrip(self):
dfSrc = pd.DataFrame(
data={
"a": np.zeros((10,), dtype=np.float32),
"b": np.ones((10,), dtype=np.int64),
"c": np.array([i for i in range(0, 10)], dtype=np.uint16),
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}
)
dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
self.assertEqual(dfSrc.shape, dfDst.shape)
self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
for c in dfSrc.columns:
self.assertTrue(c in dfDst.columns)
if isinstance(dfSrc[c], pd.Series):
self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
else:
self.assertEqual(dfSrc[c], dfDst[c])
backend/test/test_czi_hosted/unit/data_common/test_matrix_loader.py
-126
View File
@@ -1,126 +0,0 @@
import os
import shutil
import tempfile
import time
import unittest
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.common.errors import DatasetAccessError
from backend.czi_hosted.data_common.matrix_loader import MatrixDataCacheManager
from backend.test import FIXTURES_ROOT
class MatrixCacheTest(unittest.TestCase):
def setup(self):
pass
def make_temporay_datasets(self, dirname, num):
source = f"{FIXTURES_ROOT}/pbmc3k.cxg"
for i in range(num):
target = os.path.join(dirname, str(i) + ".cxg")
shutil.copytree(source, target)
def use_dataset(self, matrix_cache, dirname, app_config, dataset_index):
with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config) as adaptor:
pass
return adaptor
def use_dataset_with_error(self, matrix_cache, dirname, app_config, dataset_index):
try:
with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config):
raise DatasetAccessError("something bad happened")
except DatasetAccessError:
# the MatrixDataCacheManager rethrows the exception, so catch and ignore
pass
def get_datasets(self, matrix_cache, dirname):
datasets = matrix_cache.datasets
result = {}
for k, v in datasets.items():
# filter out the dirname and the .cxg from the name
newk = int(k[1][len(dirname) + 1 : -4])
result[newk] = v
return result
def check_datasets(self, matrix_cache, dirname, expected):
res = self.get_datasets(matrix_cache, dirname)
actual = res.keys()
self.assertSetEqual(set(actual), set(expected))
def test_basic(self):
with tempfile.TemporaryDirectory() as dirname:
self.make_temporay_datasets(dirname, 5)
app_config = AppConfig()
m = MatrixDataCacheManager(max_cached=3, timelimit_s=None)
# should have only dataset 0
self.use_dataset(m, dirname, app_config, 0)
self.check_datasets(m, dirname, [0])
# should have datasets 0, 1
self.use_dataset(m, dirname, app_config, 1)
self.check_datasets(m, dirname, [0, 1])
# should have datasets 0, 1, 2
self.use_dataset(m, dirname, app_config, 2)
self.check_datasets(m, dirname, [0, 1, 2])
# should have datasets 1, 2, 3
self.use_dataset(m, dirname, app_config, 3)
self.check_datasets(m, dirname, [1, 2, 3])
# use dataset 1, making is more recent than dataset 2
self.use_dataset(m, dirname, app_config, 1)
self.check_datasets(m, dirname, [1, 2, 3])
# use dataset 4, should have 1,3,4
self.use_dataset(m, dirname, app_config, 4)
self.check_datasets(m, dirname, [1, 3, 4])
# use dataset 4 a few more times, get the count to 3
self.use_dataset(m, dirname, app_config, 4)
self.use_dataset(m, dirname, app_config, 4)
datasets = self.get_datasets(m, dirname)
self.assertEqual(datasets[1].num_access, 2)
self.assertEqual(datasets[3].num_access, 1)
self.assertEqual(datasets[4].num_access, 3)
def test_timelimit(self):
with tempfile.TemporaryDirectory() as dirname:
self.make_temporay_datasets(dirname, 2)
app_config = AppConfig()
m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
adaptor = self.use_dataset(m, dirname, app_config, 0)
adaptor1 = self.use_dataset(m, dirname, app_config, 0)
self.assertTrue(adaptor is adaptor1)
# wait until the timelimit expires and check that there is a new adaptor
time.sleep(1.1)
adaptor2 = self.use_dataset(m, dirname, app_config, 0)
self.assertTrue(adaptor is not adaptor2)
self.check_datasets(m, dirname, [0])
# now load a different dataset and see if dataset 0 gets evicted
time.sleep(1.1)
self.use_dataset(m, dirname, app_config, 1)
self.check_datasets(m, dirname, [1])
def test_access_error(self):
with tempfile.TemporaryDirectory() as dirname:
self.make_temporay_datasets(dirname, 1)
app_config = AppConfig()
m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
# use the 0 datasets
self.use_dataset(m, dirname, app_config, 0)
self.check_datasets(m, dirname, [0])
# use the 0 datasets, but this time a DatasetAccessError is raised.
# verify that dataset is removed from the cache.
self.use_dataset_with_error(m, dirname, app_config, 0)
self.check_datasets(m, dirname, [])
backend/test/test_czi_hosted/unit/data_cxg/__init__.py
View File
backend/test/test_czi_hosted/unit/data_cxg/test_cxg_adaptor.py
-20
View File
@@ -1,20 +0,0 @@
import unittest
from backend.common.utils.data_locator import DataLocator
from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
from backend.test.test_czi_hosted.unit import app_config
from backend.test import FIXTURES_ROOT
from backend.test.fixtures.fixtures import pbmc3k_colors
class TestCxgAdaptor(unittest.TestCase):
def test_get_colors(self):
data = self.get_data("pbmc3k.cxg")
self.assertDictEqual(data.get_colors(), pbmc3k_colors)
data = self.get_data("pbmc3k_v0.cxg")
self.assertDictEqual(data.get_colors(), dict())
def get_data(self, fixture):
data_locator = f"{FIXTURES_ROOT}/{fixture}"
config = app_config(data_locator)
return CxgAdaptor(DataLocator(data_locator), config)
backend/test/test_czi_hosted/unit/eb/__init__.py
View File
backend/test/test_czi_hosted/unit/eb/test_eb.py
-82
View File
@@ -1,82 +0,0 @@
import os
from unittest.mock import patch
import requests
import subprocess
import tempfile
import time
import unittest
from contextlib import contextmanager
from backend.czi_hosted.common.config.app_config import AppConfig
from backend.test import PROJECT_ROOT, FIXTURES_ROOT
@contextmanager
def run_eb_app(tempdirname):
ps = subprocess.Popen(["python", "artifact.dir/application.py"], cwd=tempdirname)
server = "http://localhost:5000"
for _ in range(10):
try:
requests.get(f"{server}/health")
break
except requests.exceptions.ConnectionError:
time.sleep(1)
try:
yield server
finally:
try:
ps.terminate()
except ProcessLookupError:
pass
class Elastic_Beanstalk_Test(unittest.TestCase):
def test_run(self):
tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/backend/czi_hosted")
tempdirname = tempdir.name
config = AppConfig()
# test that eb works
config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame")
config.complete_config()
config.write_config(f"{tempdirname}/config.yaml")
subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/backend/czi_hosted/eb",
shell=True)
subprocess.check_call(["make", "build"], cwd=tempdirname)
with run_eb_app(tempdirname) as server:
session = requests.Session()
response = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
data_config = response.json()
assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
def test_config(self):
check_config_script = os.path.join(PROJECT_ROOT, "backend", "czi_hosted", "eb", "check_config.py")
with tempfile.TemporaryDirectory() as tempdir:
configfile = os.path.join(tempdir, "config.yaml")
app_config = AppConfig()
app_config.update_server_config(multi_dataset__dataroot=f"{FIXTURES_ROOT}")
app_config.write_config(configfile)
command = ["python", check_config_script, configfile]
# test failure mode (flask_secret_key not set)
env = os.environ.copy()
env.pop("CXG_SECRET_KEY", None)
with self.assertRaises(subprocess.CalledProcessError) as exception_context:
subprocess.check_output(command, env=env)
output = str(exception_context.exception.stdout, "utf-8")
self.assertTrue(
output.startswith(
"Error: Invalid type for attribute: app__flask_secret_key, expected type str, got NoneType"
)
)
self.assertEqual(exception_context.exception.returncode, 1)
# test passing case
env = os.environ.copy()
env["CXG_SECRET_KEY"] = "secret"
output = subprocess.check_output(command, env=env)
output = str(output, "utf-8")
self.assertTrue(output.startswith("PASS"))