diff --git a/.github/workflows/scale-test.yml b/.github/workflows/scale-test.yml new file mode 100644 index 00000000..6e86de0e --- /dev/null +++ b/.github/workflows/scale-test.yml @@ -0,0 +1,30 @@ +name: "Scale test cellxgene APIs for initial loading" + +on: + schedule: + - cron: "0 0 * * Sun" + +jobs: + locust-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - name: Install dependencies + run: | + pip install -r server/test/locust/requirements-locust.txt + - name: Dev Scale Test + run: | + locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt + - name: Slack success webhook + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + run: | + DEV_STATS=$(tail -n 61 locust_dev_stats.txt) + DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`" + curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK + + diff --git a/server/test/locust/config.py b/server/test/locust/config.py index 9fbd0c60..7b256ee2 100644 --- a/server/test/locust/config.py +++ b/server/test/locust/config.py @@ -10,12 +10,6 @@ Locust test config # multi-dataset, for dataroot tests. these are varied in size/shape DataSets = [ - "/d/pbmc3k.cxg", - "/d/TM_droplet_processed.cxg", - "/d/pancreas.cxg", - "/d/immune_bone_marrow_processed.cxg", - "/d/10X_mouse_13MM_processed.cxg", - "/d/GSE60361.cxg", - "/d/Reprogrammed_Dendritic_Cells.cxg", - "/d/WongAdultRetina.cxg", + "GSE60361.cxg", + "WongAdultRetina.cxg", ] diff --git a/server/test/locust/locustfile.py b/server/test/locust/locustfile.py index 2e6a796f..186371d5 100644 --- a/server/test/locust/locustfile.py +++ b/server/test/locust/locustfile.py @@ -1,143 +1,144 @@ -from locust import HttpLocust, TaskSet, TaskSequence, seq_task, task -from locust.wait_time import between -import random import json -from gevent.pool import Group +import random + +import requests +from config import DataSets +from locust import HttpUser, SequentialTaskSet, task, between, TaskSet +from locust.clients import HttpSession +from requests.packages.urllib3.exceptions import InsecureRequestWarning import server.test.unit.decode_fbs as decode_fbs -from config import DataSets +requests.packages.urllib3.disable_warnings(InsecureRequestWarning) """ -Simple locust stress test defition for cellxgene +Simple locust stress test definition for cellxgene """ -API = "/api/v0.2" + +API_SUFFIX = "api/v0.2" -class ViewDataset(TaskSet): +class CellXGeneTasks(TaskSet): """ Simulate use against a single dataset """ def on_start(self): + self.client.verify = False self.dataset = random.choice(DataSets) - with self.client.get(f"{self.dataset}{API}/config", catch_response=True) as r: - if r.status_code == 200: - self.config = r.json()["config"] - r.success() - else: - self.config = None - r.failure(f"bad response code {r.status_code}") - - with self.client.get(f"{self.dataset}{API}/schema", catch_response=True) as r: - if r.status_code == 200: - self.schema = r.json()["schema"] - r.success() + with self.client.get( + f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True + ) as schema_response: + if schema_response.status_code == 200: + self.schema = schema_response.json()["schema"] else: self.schema = None - r.failure(f"bad response code {r.status_code}") with self.client.get( - f"{self.dataset}{API}/annotations/var?annotation-name={self.var_index_name()}", + f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True + ) as config_response: + if config_response.status_code == 200: + self.config = config_response.json()["config"] + else: + self.config = None + + with self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.var_index_name()}", headers={"Accept": "application/octet-stream"}, catch_response=True, - ) as r: - if r.status_code == 200: - df = decode_fbs.decode_matrix_FBS(r.content) + ) as var_index_response: + if var_index_response.status_code == 200: + df = decode_fbs.decode_matrix_FBS(var_index_response.content) gene_names_idx = df["col_idx"].index(self.var_index_name()) self.gene_names = df["columns"][gene_names_idx] else: - self.gene_names = None - r.failure(f"bad response code {r.status_code}") + self.gene_names = [] def var_index_name(self): - if self.schema is None: - return None - return self.schema["annotations"]["var"]["index"] + if self.schema is not None: + return self.schema["annotations"]["var"]["index"] + return None def obs_annotation_names(self): - if self.schema is None: + if self.schema is not None: + return [col["name"] for col in self.schema["annotations"]["obs"]["columns"]] + return [] + + def layout_names(self): + if self.schema is not None: + return [layout["name"] for layout in self.schema["layout"]["obs"]] + else: return [] - return [col["name"] for col in self.schema["annotations"]["obs"]["columns"]] @task(2) - class InitializeClient(TaskSequence): + class InitializeClient(SequentialTaskSet): """ Initial loading of cellxgene - when the user hits the main route. - Currently this sequence skips some of the static assets, which are quite - small and should be served by the HTTP server directly. + Currently this sequence skips some of the static assets, which are quite small and should be served by the + HTTP server directly. - 1. load index.html, etc. - 2. concurrently load /config, /schema - 3. concurrently load /layout/obs, /annotations/var?annotation-name= - -- does intitial render -- - 4. concurrently load all /annotations/obs - -- fully initialized -- + 1. Load index.html, etc. + 2. Concurrently load /config, /schema + 3. Concurrently load /layout/obs, /annotations/var?annotation-name= + -- Does initial render -- + 4. Concurrently load all /annotations/obs and all /layouts/obs + -- Fully initialized -- """ - # users hit all of the init routes as fast as they can, subject to the ordering constraints - # and network latency + # Users hit all of the init routes as fast as they can, subject to the ordering constraints and network latency. wait_time = between(0.01, 0.1) def on_start(self): self.dataset = self.parent.dataset self.client.verify = False + self.api_less_client = HttpSession( + base_url=self.client.base_url.replace("api.", "").replace("cellxgene/", ""), + request_success=self.client.request_success, + request_failure=self.client.request_failure, + ) - @seq_task(1) + @task def index(self): - self.client.get(f"{self.dataset}/", stream=True).close() + self.api_less_client.get(f"{self.dataset}", stream=True) - @seq_task(2) - def loadConfigSchema(self): - def config(): - self.client.get(f"{self.dataset}{API}/config", stream=True).close() + @task + def loadConfigAndSchema(self): + self.client.get(f"{self.dataset}/{API_SUFFIX}/schema", stream=True, catch_response=True) + self.client.get(f"{self.dataset}/{API_SUFFIX}/config", stream=True, catch_response=True) - def schema(): - self.client.get(f"{self.dataset}{API}/schema", stream=True).close() - - group = Group() - group.spawn(config) - group.spawn(schema) - group.join() - - @seq_task(3) + @task def loadBootstrapData(self): - def layout(): - self.client.get( - f"{self.dataset}{API}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True - ).close() - - def varAnnotationIndex(): - self.client.get( - f"{self.dataset}{API}/annotations/var?annotation-name={self.parent.var_index_name()}", - headers={"Accept": "application/octet-stream"}, - stream=True, - ).close() - - group = Group() - group.spawn(layout) - group.spawn(varAnnotationIndex) - group.join() - - @seq_task(4) - def loadObsAnnotations(self): - def obs_annotation(name): - self.client.get( - f"{self.dataset}{API}/annotations/obs?annotation-name={name}", - headers={"Accept": "application/octet-stream"}, - stream=True, - ).close() + self.client.get( + f"{self.dataset}/{API_SUFFIX}/layout/obs", headers={"Accept": "application/octet-stream"}, stream=True + ) + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/var?annotation-name={self.parent.var_index_name()}", + headers={"Accept": "application/octet-stream"}, + catch_response=True, + ) + @task + def loadObsAnnotationsAndLayouts(self): obs_names = self.parent.obs_annotation_names() - group = Group() for name in obs_names: - group.spawn(obs_annotation, name) - group.join() + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/obs?annotation-name={name}", + headers={"Accept": "application/octet-stream"}, + stream=True, + ) - @seq_task(5) + layouts = self.parent.layolut_names() + for name in layouts: + self.client.get( + f"{self.dataset}/{API_SUFFIX}/annotations/obs?layout-name={name}", + headers={"Accept": "application/octet-stream"}, + stream=True, + ) + + @task def done(self): self.interrupt() @@ -146,19 +147,19 @@ class ViewDataset(TaskSet): """ Simulate user occasionally loading some expression data for a gene """ + gene_name = random.choice(self.gene_names) filter = {"filter": {"var": {"annotation_value": [{"name": self.var_index_name(), "values": [gene_name]}]}}} self.client.put( - f"{self.dataset}{API}/data/var", + f"{self.dataset}/{API_SUFFIX}/data/var", data=json.dumps(filter), headers={"Content-Type": "application/json", "Accept": "application/octet-stream"}, stream=True, ).close() -class CellxgeneUser(HttpLocust): - task_set = ViewDataset +class CellxgeneUser(HttpUser): + tasks = [CellXGeneTasks] - # most ops do not require back-end interaction, so slow cadence - # for users + # Most ops do not require back-end interaction, so slow cadence for users wait_time = between(10, 60) diff --git a/server/test/performance/scale_test_annotations.py b/server/test/performance/scale_test_annotations.py index 5aa2ac6e..a26a16c0 100644 --- a/server/test/performance/scale_test_annotations.py +++ b/server/test/performance/scale_test_annotations.py @@ -27,7 +27,6 @@ class WebsiteUser(HttpUser): dataset_urls = [ "human_cell_landscape.cxg", "Single_cell_drug_screening_a549-42-remixed.cxg", - "kampmann_lab_human_AD_snRNAseq_EC_inhibitoryNeurons-53-remixed.cxg", "krasnow_lab_human_lung_cell_atlas_smartseq2-2-remixed.cxg", "Single_cell_gene_expression_profiling_of_SARS_CoV_2_infected_human_cell_lines_H1299-27-remixed.cxg", ]