mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-04 18:48:12 +08:00
Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
@@ -0,0 +1,44 @@
|
||||
import anndata
|
||||
import argparse
|
||||
import random
|
||||
import scipy
|
||||
import numpy as np
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser("A command to generate test h5ad files")
|
||||
parser.add_argument("output", help="Name of the output file")
|
||||
parser.add_argument("nobs", type=int, help="Number of observations (rows)")
|
||||
parser.add_argument("nvar", type=int, help="Number of variables (columns)")
|
||||
parser.add_argument("-n", "--nnz-percent", type=float, default=100, help="percent of non-zeros")
|
||||
parser.add_argument("-c", "--col-shift", action="store_true", help="add a random value to each column")
|
||||
parser.add_argument("--seed", type=int, default=None, help="add a random value to each column")
|
||||
|
||||
args = parser.parse_args()
|
||||
create_test_h5ad(args.output, args.nobs, args.nvar, args.nnz_percent, args.col_shift, args.seed)
|
||||
|
||||
|
||||
def create_test_h5ad(outfile, nobs, nvar, nnz_percent=100, apply_col_shift=False, seed=None):
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
x = create_X_array(nobs, nvar, nnz_percent, apply_col_shift)
|
||||
obsm = {"X_random": np.random.rand(nobs, 2).astype(np.float32)}
|
||||
adata = anndata.AnnData(x, obsm=obsm)
|
||||
adata.write(outfile)
|
||||
|
||||
|
||||
def create_X_array(nobs, nvar, nnz_percent, apply_col_shift):
|
||||
if nnz_percent < 100:
|
||||
array = scipy.sparse.random(nobs, nvar, nnz_percent * 0.01, dtype=np.float32, format="csc")
|
||||
else:
|
||||
array = np.random.rand(nobs, nvar).astype(np.float32)
|
||||
|
||||
if apply_col_shift:
|
||||
col_shift = np.random.rand((nvar))
|
||||
array += col_shift
|
||||
|
||||
return array
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,216 @@
|
||||
import json
|
||||
import string
|
||||
from contextlib import contextmanager
|
||||
from timeit import default_timer
|
||||
import concurrent.futures
|
||||
import numpy as np
|
||||
import requests
|
||||
import sys
|
||||
import pandas as pd
|
||||
import random
|
||||
|
||||
from backend.common.fbs.matrix import encode_matrix_fbs
|
||||
|
||||
"""
|
||||
Before running, sign into the dataportal, copy the cookie and paste it below. To test in staging or prod update the
|
||||
url base below. It is also possible to configure the number of categories created and the number of unique labels per
|
||||
category.
|
||||
"""
|
||||
|
||||
cookie = ""
|
||||
|
||||
test_datasets = {
|
||||
"smallest": {
|
||||
"dataset_url": "kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg",
|
||||
"name": "smallest",
|
||||
"num_cells": 5270,
|
||||
},
|
||||
"10k": {
|
||||
"dataset_url": "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
|
||||
"name": "10k",
|
||||
"num_cells": 9409,
|
||||
},
|
||||
"80k": {
|
||||
"dataset_url": "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg", # noqa E501
|
||||
"name": "80k",
|
||||
"num_cells": 81736,
|
||||
},
|
||||
"140k": {"dataset_url": "Single_cell_drug_screening_a549-42-remixed.cxg", "name": "140k", "num_cells": 143015},
|
||||
"largest": {"dataset_url": "human_cell_landscape.cxg", "name": "largest", "num_cells": 599926},
|
||||
"1million": {"dataset_url": None, "name": "1million", "num_cells": 1000000},
|
||||
"4million": {"dataset_url": None, "name": "4million", "num_cells": 4000000},
|
||||
}
|
||||
|
||||
url_base = "https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/"
|
||||
annotations_category_count = [1, 10, 50]
|
||||
max_labels = [5, 50, 100]
|
||||
|
||||
|
||||
class PerformanceTestingAnnotations:
|
||||
def __init__(
|
||||
self,
|
||||
datasets=test_datasets,
|
||||
annotations_category_count=annotations_category_count,
|
||||
max_labels=max_labels,
|
||||
url_base=url_base,
|
||||
):
|
||||
self.test_datasets = datasets
|
||||
self.annotations_category_count = annotations_category_count
|
||||
self.max_labels = max_labels
|
||||
self.url_base = url_base
|
||||
self.test_notes = self.create_info_dict()
|
||||
|
||||
def set_cell_count(self, dataset_name):
|
||||
dataset_url = self.test_datasets[dataset_name]["dataset_url"]
|
||||
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
|
||||
response = self.client.get(f"{self.url_base}{dataset_url}/api/v0.2/schema", headers=headers)
|
||||
cell_count = json.loads(response._content)["schema"]["dataframe"]["nObs"]
|
||||
self.test_datasets[dataset_name]["cell_count"] = cell_count
|
||||
|
||||
def create_info_dict(self):
|
||||
request_info = {}
|
||||
for dataset in self.test_datasets.keys():
|
||||
request_info[dataset] = {}
|
||||
for cat_count in self.annotations_category_count:
|
||||
request_info[dataset][f"num_categories_{cat_count}"] = {}
|
||||
for unique_labels in self.max_labels:
|
||||
request_info[dataset][f"num_categories_{cat_count}"][f"max_label_{unique_labels}"] = {}
|
||||
return request_info
|
||||
|
||||
def create_annotations_dict_multi_process(self, dataset_name, category_count, label_max):
|
||||
annotation_dict = {}
|
||||
futures = []
|
||||
categories = [f"Category{i}" for i in range(category_count)]
|
||||
if not self.test_datasets[dataset_name]["num_cells"]:
|
||||
self.set_cell_count(dataset_name)
|
||||
with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
|
||||
for category in categories:
|
||||
futures.append(
|
||||
executor.submit(
|
||||
self.build_array_for_category,
|
||||
category,
|
||||
self.test_datasets[dataset_name]["num_cells"],
|
||||
label_max,
|
||||
)
|
||||
)
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
try:
|
||||
result = future.result()
|
||||
category_name, cells = result
|
||||
annotation_dict[category_name] = pd.Series(cells, dtype="category")
|
||||
except Exception as e:
|
||||
print(f"Issue creating the annotations dict: {e}")
|
||||
return annotation_dict
|
||||
|
||||
def build_array_for_category(self, category_name, cell_count, label_max):
|
||||
unique_label_count = label_max
|
||||
labels = self.generate_labels(unique_label_count)
|
||||
cells_per_label = int(cell_count / len(labels))
|
||||
extra = cell_count % len(labels)
|
||||
cells = []
|
||||
for label in labels:
|
||||
cells.extend([label] * cells_per_label)
|
||||
cells.extend(["extra"] * extra)
|
||||
rng = np.random.default_rng()
|
||||
rng.shuffle(cells)
|
||||
return category_name, cells
|
||||
|
||||
@staticmethod
|
||||
def convert_to_fbs(annotation_dict):
|
||||
df = pd.DataFrame(annotation_dict)
|
||||
return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
|
||||
|
||||
@staticmethod
|
||||
def generate_labels(unique_label_count):
|
||||
labels = ["undefined"]
|
||||
for i in range(unique_label_count):
|
||||
length = random.randrange(10, 20)
|
||||
labels.append(f"{i}__" + "".join(random.choice(string.ascii_letters) for z in range(length)))
|
||||
return labels
|
||||
|
||||
@contextmanager
|
||||
def elapsed_timer(self):
|
||||
start = default_timer()
|
||||
elapser = lambda: default_timer() - start # noqa E731
|
||||
yield lambda: elapser()
|
||||
end = default_timer()
|
||||
elapser = lambda: end - start # noqa E731
|
||||
|
||||
def create_matrix(self, dataset_name, num_cat, max_labels):
|
||||
with self.elapsed_timer() as elapsed:
|
||||
annon_dict = self.create_annotations_dict_multi_process(dataset_name, num_cat, max_labels)
|
||||
dict_size = sum(sys.getsizeof(value) for value in annon_dict.values()) / 1024 ** 2
|
||||
self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["annotation_dict"] = {
|
||||
"creation_time": str(elapsed()),
|
||||
"size": f"{dict_size} mb",
|
||||
}
|
||||
df = pd.DataFrame(annon_dict)
|
||||
df_size = sys.getsizeof(df) / 1024 ** 2
|
||||
self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["data_frame"] = {
|
||||
"creation_time": str(elapsed()),
|
||||
"size": f"{df_size} mb",
|
||||
}
|
||||
try:
|
||||
matrix = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
|
||||
matrix_size = sys.getsizeof(matrix) / 1024 ** 2
|
||||
self.test_notes[dataset_name][f"num_categories_{num_cat}"][f"max_label_{max_labels}"]["fbs_matrix"] = {
|
||||
"creation_time": str(elapsed()),
|
||||
"size": f"{matrix_size} mb",
|
||||
}
|
||||
return matrix
|
||||
except Exception as e:
|
||||
print(f"Issue creating fbs matrix: {e}, for {dataset_name}")
|
||||
return []
|
||||
|
||||
def send_put_request(self, dataset_url, data):
|
||||
url = self.url_base + f"{dataset_url}/api/v0.2/annotations/obs"
|
||||
with self.elapsed_timer() as elapsed:
|
||||
try:
|
||||
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
|
||||
response = requests.put(url=url, data=data, headers=headers)
|
||||
except Exception as e:
|
||||
print(f"Issue with put request: {e}")
|
||||
return None, elapsed()
|
||||
return response, elapsed()
|
||||
|
||||
def test_categories_max_label_matrix(self, dataset_name):
|
||||
for unique_labels in self.max_labels:
|
||||
for category_count in self.annotations_category_count:
|
||||
print(f"Starting dataset: {dataset_name}, categories: {category_count}, labels: {unique_labels}")
|
||||
fbs_matrix = self.create_matrix(dataset_name, category_count, unique_labels)
|
||||
if self.test_datasets[dataset_name]["dataset_url"] and fbs_matrix:
|
||||
response, response_time = self.send_put_request(
|
||||
self.test_datasets[dataset_name]["dataset_url"], fbs_matrix
|
||||
)
|
||||
if response is None:
|
||||
self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
|
||||
"put_request"
|
||||
] = {"response_status": "failed", "request_time": str(response_time)}
|
||||
else:
|
||||
self.test_notes[dataset_name][f"num_categories_{category_count}"][f"max_label_{unique_labels}"][
|
||||
"put_request"
|
||||
] = {"response_status": response.status_code, "request_time": str(response_time)}
|
||||
|
||||
|
||||
def test_all_datasets():
|
||||
"""
|
||||
Run time is dependent on number of datasets, dataset size, number of categories/number being tested and number of
|
||||
unique label counts being tested. However it generally takes a long time. I recommend running this in tmux
|
||||
"""
|
||||
perf_test = PerformanceTestingAnnotations()
|
||||
for dataset_name in perf_test.test_datasets.keys():
|
||||
print(f"Testing annotation creation for: {dataset_name}")
|
||||
try:
|
||||
perf_test.test_categories_max_label_matrix(dataset_name)
|
||||
except Exception as e:
|
||||
print(f"something went wrong with {dataset_name}: {e}")
|
||||
return perf_test.test_notes
|
||||
|
||||
|
||||
def main():
|
||||
notes = test_all_datasets()
|
||||
print(notes)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,115 @@
|
||||
import sys
|
||||
import argparse
|
||||
import random
|
||||
import time
|
||||
import numpy as np
|
||||
|
||||
from backend.czi_hosted.common.config.app_config import AppConfig
|
||||
from backend.czi_hosted.compute import diffexp_generic, diffexp_cxg
|
||||
from backend.czi_hosted.data_common.matrix_loader import MatrixDataLoader
|
||||
from backend.czi_hosted.data_cxg.cxg_adaptor import CxgAdaptor
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser("A command to test diffexp")
|
||||
parser.add_argument("dataset", help="name of a dataset to load")
|
||||
parser.add_argument("-na", "--numA", type=int, help="number of rows in group A")
|
||||
parser.add_argument("-nb", "--numB", type=int, help="number of rows in group B")
|
||||
parser.add_argument("-va", "--varA", help="obs variable:value to use for group A")
|
||||
parser.add_argument("-vb", "--varB", help="obs variable:value to use for group B")
|
||||
parser.add_argument("-t", "--trials", default=1, type=int, help="number of trials")
|
||||
parser.add_argument(
|
||||
"-a", "--alg", choices=("default", "generic", "cxg"), default="default", help="algorithm to use"
|
||||
)
|
||||
parser.add_argument("-s", "--show", default=False, action="store_true", help="show the results")
|
||||
parser.add_argument(
|
||||
"-n", "--new-selection", default=False, action="store_true", help="change the selection between each trial"
|
||||
)
|
||||
parser.add_argument("--seed", default=1, type=int, help="set the random seed")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
app_config = AppConfig()
|
||||
app_config.update_server_config(single_dataset__datapath=args.dataset)
|
||||
app_config.update_server_config(app__verbose=True)
|
||||
app_config.complete_config()
|
||||
|
||||
loader = MatrixDataLoader(args.dataset)
|
||||
adaptor = loader.open(app_config)
|
||||
|
||||
if args.show:
|
||||
if isinstance(adaptor, CxgAdaptor):
|
||||
adaptor.open_array("X").schema.dump()
|
||||
|
||||
random.seed(args.seed)
|
||||
np.random.seed(args.seed)
|
||||
rows = adaptor.get_shape()[0]
|
||||
|
||||
if args.numA:
|
||||
filterA = random.sample(range(rows), args.numA)
|
||||
elif args.varA:
|
||||
vname, vval = args.varA.split(":")
|
||||
filterA = get_filter_from_obs(adaptor, vname, vval)
|
||||
else:
|
||||
print("must supply numA or varA")
|
||||
sys.exit(1)
|
||||
|
||||
if args.numB:
|
||||
filterB = random.sample(range(rows), args.numB)
|
||||
elif args.varB:
|
||||
vname, vval = args.varB.split(":")
|
||||
filterB = get_filter_from_obs(adaptor, vname, vval)
|
||||
else:
|
||||
print("must supply numB or varB")
|
||||
sys.exit(1)
|
||||
|
||||
for i in range(args.trials):
|
||||
if args.new_selection:
|
||||
if args.numA:
|
||||
filterA = random.sample(range(rows), args.numA)
|
||||
if args.numB:
|
||||
filterB = random.sample(range(rows), args.numB)
|
||||
|
||||
maskA = np.zeros(rows, dtype=bool)
|
||||
maskA[filterA] = True
|
||||
maskB = np.zeros(rows, dtype=bool)
|
||||
maskB[filterB] = True
|
||||
|
||||
t1 = time.time()
|
||||
if args.alg == "default":
|
||||
results = adaptor.compute_diffexp_ttest(maskA, maskB)
|
||||
elif args.alg == "generic":
|
||||
results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB)
|
||||
elif args.alg == "cxg":
|
||||
if not isinstance(adaptor, CxgAdaptor):
|
||||
print("cxg only works with CxgAdaptor")
|
||||
sys.exit(1)
|
||||
results = diffexp_cxg.diffexp_ttest(adaptor, maskA, maskB)
|
||||
|
||||
t2 = time.time()
|
||||
print("TIME=", t2 - t1)
|
||||
|
||||
if args.show:
|
||||
for res in results:
|
||||
print(res)
|
||||
|
||||
|
||||
def get_filter_from_obs(adaptor, obsname, obsval):
|
||||
attrs = adaptor.get_obs_columns()
|
||||
if obsname not in attrs:
|
||||
print(f"Unknown obs attr {obsname}: expected on of {attrs}")
|
||||
sys.exit(1)
|
||||
obsvals = adaptor.query_obs_array(obsname)[:]
|
||||
obsval = type(obsvals[0])(obsval)
|
||||
|
||||
vfilter = np.where(obsvals == obsval)[0]
|
||||
if len(vfilter) == 0:
|
||||
u = np.unique(obsvals)
|
||||
print(f"Unknown value in variable {obsname}:{obsval}: expected one of {list(u)}")
|
||||
sys.exit(1)
|
||||
|
||||
return vfilter
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,45 @@
|
||||
import time
|
||||
import random
|
||||
|
||||
from locust import HttpUser, between, task
|
||||
|
||||
random.seed(time.time())
|
||||
"""
|
||||
To run this script sign into cellxgene in the desired environment and grab the returned cookie, update the cookie
|
||||
variable below with your cookie and run the following command to see results in the terminal:
|
||||
locust -f backend/test/test_czi_hosted/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
|
||||
|
||||
Or if you want to use the locust gui run:
|
||||
locust -f backend/test/test_czi_hosted/performance/scale_test_annotations.py -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/
|
||||
|
||||
If you want to test staging you'll need to substitute staging for dev in the host url
|
||||
To test prod you'll need to replace dev.single-cell.czi.technology with cziscience.com
|
||||
If you'd like to test additional datasets you'll need to add them to the dataset_urls array
|
||||
|
||||
Todo @mdunitz update script to retrieve different annotation categories -- may need to create them to ensure the
|
||||
categories are shared across datasets for a given user.
|
||||
"""
|
||||
cookie = ""
|
||||
|
||||
|
||||
class WebsiteUser(HttpUser):
|
||||
wait_time = between(1, 2)
|
||||
dataset_urls = [
|
||||
"human_cell_landscape.cxg",
|
||||
"Single_cell_drug_screening_a549-42-remixed.cxg",
|
||||
"krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg",
|
||||
"Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg",
|
||||
]
|
||||
|
||||
@task
|
||||
def get_annotations(self):
|
||||
dataset_url = random.choice(self.dataset_urls)
|
||||
url = f"{dataset_url}/api/v0.2/annotations/obs?annotation-name=cell_type"
|
||||
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
|
||||
self.client.get(url, headers=headers)
|
||||
|
||||
@task
|
||||
def get_schema(self):
|
||||
dataset_url = random.choice(self.dataset_urls)
|
||||
headers = {"Content-Type": "application/octet-stream", "Cookie": cookie}
|
||||
self.client.get(f"{dataset_url}/api/v0.2/schema", headers=headers)
|
||||
Reference in New Issue
Block a user