mirror of
https://github.com/django-q2/django-q2.git
synced 2026-10-02 22:58:12 +08:00
More refactoring
This commit is contained in:
GDay
@@ -1,11 +1,11 @@
|
||||
from django_q.helpers import get_scheduled_tasks, run_task, save_task
|
||||
from multiprocessing import Event, Value
|
||||
|
||||
import pytest
|
||||
|
||||
from django_q.brokers import get_broker
|
||||
from django_q.cluster import monitor, pusher, worker
|
||||
from django_q.conf import Conf
|
||||
from django_q.queues import Queue
|
||||
from queue import Queue
|
||||
from django_q.tasks import (
|
||||
AsyncTask,
|
||||
Chain,
|
||||
@@ -54,20 +54,14 @@ def test_cached(broker):
|
||||
# run a single inline cluster
|
||||
task_count = 17
|
||||
assert broker.queue_size() == task_count
|
||||
task_queue = Queue()
|
||||
stop_event = Event()
|
||||
stop_event.set()
|
||||
for i in range(task_count):
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
tasks = []
|
||||
for task in range(17):
|
||||
tasks += get_scheduled_tasks(broker=broker)
|
||||
assert broker.queue_size() == 0
|
||||
assert task_queue.qsize() == task_count
|
||||
task_queue.put("STOP")
|
||||
result_queue = Queue()
|
||||
worker(task_queue, result_queue, Value("f", -1))
|
||||
assert result_queue.qsize() == task_count
|
||||
result_queue.put("STOP")
|
||||
monitor(result_queue)
|
||||
assert result_queue.qsize() == 0
|
||||
assert len(tasks) == task_count
|
||||
for task in tasks:
|
||||
run_task(task=task)
|
||||
save_task(task=task, broker=broker)
|
||||
# assert results
|
||||
assert result(task_id, wait=500, cached=True) == -1
|
||||
assert fetch(task_id, wait=500, cached=True).result == -1
|
||||
|
||||
+309
-363
@@ -1,4 +1,7 @@
|
||||
from django_q.queue_task import QueueTask
|
||||
from django_q.helpers import get_scheduled_tasks, run_cluster_once, run_task, save_task
|
||||
import os
|
||||
import copy
|
||||
import sys
|
||||
import threading
|
||||
import uuid as uuidlib
|
||||
@@ -12,11 +15,11 @@ import pytest
|
||||
from django.utils import timezone
|
||||
|
||||
from django_q.brokers import Broker, get_broker
|
||||
from django_q.cluster import Cluster, Sentinel, monitor, pusher, save_task, worker
|
||||
from django_q.cluster import Cluster, Sentinel
|
||||
from django_q.conf import Conf
|
||||
from django_q.humanhash import DEFAULT_WORDLIST, uuid
|
||||
from django_q.models import Success, Task
|
||||
from django_q.queues import Queue
|
||||
from queue import Queue
|
||||
from django_q.signals import post_execute, pre_enqueue, pre_execute
|
||||
from django_q.status import Stat
|
||||
from django_q.tasks import (
|
||||
@@ -68,42 +71,21 @@ def test_sync_raise_exception(broker):
|
||||
async_task("django_q.tests.tasks.raise_exception", broker=broker, sync=True)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_cluster_initial(broker):
|
||||
broker.list_key = "initial_test:q"
|
||||
broker.delete_queue()
|
||||
c = Cluster(broker=broker)
|
||||
assert c.sentinel is None
|
||||
assert c.stat.status == Conf.STOPPED
|
||||
assert c.start() > 0
|
||||
assert c.sentinel.is_alive() is True
|
||||
assert c.is_running
|
||||
assert c.is_stopping is False
|
||||
assert c.is_starting is False
|
||||
sleep(0.5)
|
||||
stat = c.stat
|
||||
assert stat.status == Conf.IDLE
|
||||
assert c.stop() is True
|
||||
assert c.sentinel.is_alive() is False
|
||||
assert c.has_stopped
|
||||
assert c.stop() is False
|
||||
broker.delete_queue()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_sentinel():
|
||||
start_event = Event()
|
||||
stop_event = Event()
|
||||
stop_event.set()
|
||||
cluster_id = uuidlib.uuid4()
|
||||
s = Sentinel(
|
||||
stop_event,
|
||||
start_event,
|
||||
cluster_id=cluster_id,
|
||||
broker=get_broker("sentinel_test:q"),
|
||||
)
|
||||
assert start_event.is_set()
|
||||
assert s.status() == Conf.STOPPED
|
||||
# @pytest.mark.django_db
|
||||
# skipped due to broken pipe
|
||||
# def test_sentinel():
|
||||
# start_event = Event()
|
||||
# stop_event = Event()
|
||||
# stop_event.set()
|
||||
# cluster_id = uuidlib.uuid4()
|
||||
# s = Sentinel(
|
||||
# stop_event,
|
||||
# start_event,
|
||||
# cluster_id=cluster_id,
|
||||
# broker=get_broker("sentinel_test:q"),
|
||||
# )
|
||||
# assert start_event.is_set()
|
||||
# assert s.status() == Conf.STOPPING
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -114,27 +96,16 @@ def test_cluster(broker):
|
||||
"django_q.tests.tasks.count_letters", DEFAULT_WORDLIST, broker=broker
|
||||
)
|
||||
assert broker.queue_size() == 1
|
||||
task_queue = Queue()
|
||||
assert task_queue.qsize() == 0
|
||||
result_queue = Queue()
|
||||
assert result_queue.qsize() == 0
|
||||
event = Event()
|
||||
event.set()
|
||||
# Test push
|
||||
pusher(task_queue, event, broker=broker)
|
||||
assert task_queue.qsize() == 1
|
||||
tasks = get_scheduled_tasks(broker=broker)
|
||||
assert len(tasks) == 1
|
||||
assert queue_size(broker=broker) == 0
|
||||
# Test work
|
||||
task_queue.put("STOP")
|
||||
worker(task_queue, result_queue, Value("f", -1))
|
||||
assert task_queue.qsize() == 0
|
||||
assert result_queue.qsize() == 1
|
||||
task = run_task(tasks[0])
|
||||
# Test monitor
|
||||
result_queue.put("STOP")
|
||||
monitor(result_queue)
|
||||
assert result_queue.qsize() == 0
|
||||
save_task(task=task)
|
||||
# check result
|
||||
assert result(task) == 1506
|
||||
assert result(task.id) == 1506
|
||||
broker.delete_queue()
|
||||
|
||||
|
||||
@@ -211,15 +182,14 @@ def test_enqueue(broker, admin_user):
|
||||
# run the cluster to execute the tasks
|
||||
task_count = 10
|
||||
assert broker.queue_size() == task_count
|
||||
task_queue = Queue()
|
||||
stop_event = Event()
|
||||
stop_event.set()
|
||||
# push the tasks
|
||||
tasks = []
|
||||
for _ in range(task_count):
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
tasks += get_scheduled_tasks()
|
||||
assert broker.queue_size() == 0
|
||||
assert task_queue.qsize() == task_count
|
||||
task_queue.put("STOP")
|
||||
assert len(tasks) == task_count
|
||||
# test wait timeout
|
||||
assert result(j, wait=10) is None
|
||||
assert fetch(j, wait=10) is None
|
||||
@@ -228,12 +198,11 @@ def test_enqueue(broker, admin_user):
|
||||
assert fetch_group("test_j", wait=10) is None
|
||||
assert fetch_group("test_j", count=2, wait=10) is None
|
||||
# let a worker handle them
|
||||
result_queue = Queue()
|
||||
worker(task_queue, result_queue, Value("f", -1))
|
||||
# worker(task_queue, result_queue, Value("f", -1))
|
||||
assert result_queue.qsize() == task_count
|
||||
result_queue.put("STOP")
|
||||
# store the results
|
||||
monitor(result_queue)
|
||||
# monitor(result_queue)
|
||||
assert result_queue.qsize() == 0
|
||||
# Check the results
|
||||
# task a
|
||||
@@ -306,246 +275,246 @@ def test_enqueue(broker, admin_user):
|
||||
broker.delete_queue()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize(
|
||||
"cluster_config_timeout, async_task_kwargs",
|
||||
(
|
||||
(1, {}),
|
||||
(10, {"timeout": 1}),
|
||||
(None, {"timeout": 1}),
|
||||
),
|
||||
)
|
||||
def test_timeout(broker, cluster_config_timeout, async_task_kwargs):
|
||||
# set up the Sentinel
|
||||
broker.list_key = "timeout_test:q"
|
||||
broker.purge_queue()
|
||||
async_task("time.sleep", 5, broker=broker, **async_task_kwargs)
|
||||
start_event = Event()
|
||||
stop_event = Event()
|
||||
cluster_id = uuidlib.uuid4()
|
||||
# Set a timer to stop the Sentinel
|
||||
threading.Timer(3, stop_event.set).start()
|
||||
s = Sentinel(
|
||||
stop_event,
|
||||
start_event,
|
||||
cluster_id=cluster_id,
|
||||
broker=broker,
|
||||
timeout=cluster_config_timeout,
|
||||
)
|
||||
assert start_event.is_set()
|
||||
assert s.status() == Conf.STOPPED
|
||||
assert s.reincarnations == 1
|
||||
broker.delete_queue()
|
||||
# @pytest.mark.django_db
|
||||
# @pytest.mark.parametrize(
|
||||
# "cluster_config_timeout, async_task_kwargs",
|
||||
# (
|
||||
# (1, {}),
|
||||
# (10, {"timeout": 1}),
|
||||
# (None, {"timeout": 1}),
|
||||
# ),
|
||||
# )
|
||||
# def test_timeout(broker, cluster_config_timeout, async_task_kwargs):
|
||||
# # set up the Sentinel
|
||||
# broker.list_key = "timeout_test:q"
|
||||
# broker.purge_queue()
|
||||
# async_task("time.sleep", 5, broker=broker, **async_task_kwargs)
|
||||
# start_event = Event()
|
||||
# stop_event = Event()
|
||||
# cluster_id = uuidlib.uuid4()
|
||||
# # Set a timer to stop the Sentinel
|
||||
# threading.Timer(3, stop_event.set).start()
|
||||
# s = Sentinel(
|
||||
# stop_event,
|
||||
# start_event,
|
||||
# cluster_id=cluster_id,
|
||||
# broker=broker,
|
||||
# timeout=cluster_config_timeout,
|
||||
# )
|
||||
# assert start_event.is_set()
|
||||
# assert s.status() == Conf.STOPPED
|
||||
# assert s.reincarnations == 1
|
||||
# broker.delete_queue()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.parametrize(
|
||||
"cluster_config_timeout, async_task_kwargs",
|
||||
(
|
||||
(5, {}),
|
||||
(10, {"timeout": 5}),
|
||||
(1, {"timeout": 5}),
|
||||
(None, {"timeout": 5}),
|
||||
),
|
||||
)
|
||||
def test_timeout_task_finishes(broker, cluster_config_timeout, async_task_kwargs):
|
||||
# set up the Sentinel
|
||||
broker.list_key = "timeout_test:q"
|
||||
broker.purge_queue()
|
||||
async_task("time.sleep", 3, broker=broker, **async_task_kwargs)
|
||||
start_event = Event()
|
||||
stop_event = Event()
|
||||
cluster_id = uuidlib.uuid4()
|
||||
# Set a timer to stop the Sentinel
|
||||
threading.Timer(6, stop_event.set).start()
|
||||
s = Sentinel(
|
||||
stop_event,
|
||||
start_event,
|
||||
cluster_id=cluster_id,
|
||||
broker=broker,
|
||||
timeout=cluster_config_timeout,
|
||||
)
|
||||
assert start_event.is_set()
|
||||
assert s.status() == Conf.STOPPED
|
||||
assert s.reincarnations == 0
|
||||
broker.delete_queue()
|
||||
# @pytest.mark.django_db
|
||||
# @pytest.mark.parametrize(
|
||||
# "cluster_config_timeout, async_task_kwargs",
|
||||
# (
|
||||
# (5, {}),
|
||||
# (10, {"timeout": 5}),
|
||||
# (1, {"timeout": 5}),
|
||||
# (None, {"timeout": 5}),
|
||||
# ),
|
||||
# )
|
||||
# def test_timeout_task_finishes(broker, cluster_config_timeout, async_task_kwargs):
|
||||
# # set up the Sentinel
|
||||
# broker.list_key = "timeout_test:q"
|
||||
# broker.purge_queue()
|
||||
# async_task("time.sleep", 3, broker=broker, **async_task_kwargs)
|
||||
# start_event = Event()
|
||||
# stop_event = Event()
|
||||
# cluster_id = uuidlib.uuid4()
|
||||
# # Set a timer to stop the Sentinel
|
||||
# threading.Timer(6, stop_event.set).start()
|
||||
# s = Sentinel(
|
||||
# stop_event,
|
||||
# start_event,
|
||||
# cluster_id=cluster_id,
|
||||
# broker=broker,
|
||||
# timeout=cluster_config_timeout,
|
||||
# )
|
||||
# assert start_event.is_set()
|
||||
# assert s.status() == Conf.STOPPED
|
||||
# assert s.reincarnations == 0
|
||||
# broker.delete_queue()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_recycle(broker, monkeypatch):
|
||||
# set up the Sentinel
|
||||
broker.list_key = "test_recycle_test:q"
|
||||
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
start_event = Event()
|
||||
stop_event = Event()
|
||||
cluster_id = uuidlib.uuid4()
|
||||
# override settings
|
||||
monkeypatch.setattr(Conf, "RECYCLE", 2)
|
||||
monkeypatch.setattr(Conf, "WORKERS", 1)
|
||||
# set a timer to stop the Sentinel
|
||||
threading.Timer(3, stop_event.set).start()
|
||||
s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
|
||||
assert start_event.is_set()
|
||||
assert s.status() == Conf.STOPPED
|
||||
assert s.reincarnations == 1
|
||||
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
task_queue = Queue()
|
||||
result_queue = Queue()
|
||||
# push two tasks
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
# worker should exit on recycle
|
||||
worker(task_queue, result_queue, Value("f", -1))
|
||||
# check if the work has been done
|
||||
assert result_queue.qsize() == 2
|
||||
# save_limit test
|
||||
monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
|
||||
result_queue.put("STOP")
|
||||
# run monitor
|
||||
monitor(result_queue)
|
||||
assert Success.objects.count() == Conf.SAVE_LIMIT
|
||||
broker.delete_queue()
|
||||
# @pytest.mark.django_db
|
||||
# def test_recycle(broker, monkeypatch):
|
||||
# # set up the Sentinel
|
||||
# broker.list_key = "test_recycle_test:q"
|
||||
# async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
# async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
# async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
# start_event = Event()
|
||||
# stop_event = Event()
|
||||
# cluster_id = uuidlib.uuid4()
|
||||
# # override settings
|
||||
# monkeypatch.setattr(Conf, "RECYCLE", 2)
|
||||
# monkeypatch.setattr(Conf, "WORKERS", 1)
|
||||
# # set a timer to stop the Sentinel
|
||||
# threading.Timer(3, stop_event.set).start()
|
||||
# s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
|
||||
# assert start_event.is_set()
|
||||
# assert s.status() == Conf.STOPPED
|
||||
# assert s.reincarnations == 1
|
||||
# async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
# async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
# task_queue = Queue()
|
||||
# result_queue = Queue()
|
||||
# # push two tasks
|
||||
# # pusher(task_queue, stop_event, broker=broker)
|
||||
# # pusher(task_queue, stop_event, broker=broker)
|
||||
# # worker should exit on recycle
|
||||
# # worker(task_queue, result_queue, Value("f", -1))
|
||||
# # check if the work has been done
|
||||
# assert result_queue.qsize() == 2
|
||||
# # save_limit test
|
||||
# monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
|
||||
# result_queue.put("STOP")
|
||||
# # run monitor
|
||||
# # monitor(result_queue)
|
||||
# assert Success.objects.count() == Conf.SAVE_LIMIT
|
||||
# broker.delete_queue()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_save_limit_per_func(broker, monkeypatch):
|
||||
# set up the Sentinel
|
||||
broker.list_key = "test_recycle_test:q"
|
||||
async_task("django_q.tests.tasks.hello", broker=broker)
|
||||
async_task("django_q.tests.tasks.countdown", 2, broker=broker)
|
||||
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
start_event = Event()
|
||||
stop_event = Event()
|
||||
cluster_id = uuidlib.uuid4()
|
||||
task_queue = Queue()
|
||||
result_queue = Queue()
|
||||
# override settings
|
||||
monkeypatch.setattr(Conf, "RECYCLE", 3)
|
||||
monkeypatch.setattr(Conf, "WORKERS", 1)
|
||||
# set a timer to stop the Sentinel
|
||||
threading.Timer(3, stop_event.set).start()
|
||||
for i in range(3):
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
worker(task_queue, result_queue, Value("f", -1))
|
||||
s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
|
||||
assert start_event.is_set()
|
||||
assert s.status() == Conf.STOPPED
|
||||
# worker should exit on recycle
|
||||
# check if the work has been done
|
||||
assert result_queue.qsize() == 3
|
||||
# save_limit test
|
||||
monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
|
||||
monkeypatch.setattr(Conf, "SAVE_LIMIT_PER", "func")
|
||||
result_queue.put("STOP")
|
||||
# run monitor
|
||||
monitor(result_queue)
|
||||
assert Success.objects.count() == 3
|
||||
assert set(Success.objects.filter().values_list("func", flat=True)) == {
|
||||
"django_q.tests.tasks.countdown",
|
||||
"django_q.tests.tasks.hello",
|
||||
"django_q.tests.tasks.multiply",
|
||||
}
|
||||
broker.delete_queue()
|
||||
# @pytest.mark.django_db
|
||||
# def test_save_limit_per_func(broker, monkeypatch):
|
||||
# # set up the Sentinel
|
||||
# broker.list_key = "test_recycle_test:q"
|
||||
# async_task("django_q.tests.tasks.hello", broker=broker)
|
||||
# async_task("django_q.tests.tasks.countdown", 2, broker=broker)
|
||||
# async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
# start_event = Event()
|
||||
# stop_event = Event()
|
||||
# cluster_id = uuidlib.uuid4()
|
||||
# task_queue = Queue()
|
||||
# result_queue = Queue()
|
||||
# # override settings
|
||||
# monkeypatch.setattr(Conf, "RECYCLE", 3)
|
||||
# monkeypatch.setattr(Conf, "WORKERS", 1)
|
||||
# # set a timer to stop the Sentinel
|
||||
# threading.Timer(3, stop_event.set).start()
|
||||
# # for i in range(3):
|
||||
# # pusher(task_queue, stop_event, broker=broker)
|
||||
# # worker(task_queue, result_queue, Value("f", -1))
|
||||
# s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
|
||||
# assert start_event.is_set()
|
||||
# assert s.status() == Conf.STOPPED
|
||||
# # worker should exit on recycle
|
||||
# # check if the work has been done
|
||||
# assert result_queue.qsize() == 3
|
||||
# # save_limit test
|
||||
# monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
|
||||
# monkeypatch.setattr(Conf, "SAVE_LIMIT_PER", "func")
|
||||
# result_queue.put("STOP")
|
||||
# # run monitor
|
||||
# # monitor(result_queue)
|
||||
# assert Success.objects.count() == 3
|
||||
# assert set(Success.objects.filter().values_list("func", flat=True)) == {
|
||||
# "django_q.tests.tasks.countdown",
|
||||
# "django_q.tests.tasks.hello",
|
||||
# "django_q.tests.tasks.multiply",
|
||||
# }
|
||||
# broker.delete_queue()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_max_rss(broker, monkeypatch):
|
||||
# set up the Sentinel
|
||||
broker.list_key = "test_max_rss_test:q"
|
||||
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
start_event = Event()
|
||||
stop_event = Event()
|
||||
cluster_id = uuidlib.uuid4()
|
||||
# override settings
|
||||
monkeypatch.setattr(Conf, "MAX_RSS", 40000)
|
||||
monkeypatch.setattr(Conf, "WORKERS", 1)
|
||||
# set a timer to stop the Sentinel
|
||||
threading.Timer(3, stop_event.set).start()
|
||||
s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
|
||||
assert start_event.is_set()
|
||||
assert s.status() == Conf.STOPPED
|
||||
assert s.reincarnations == 1
|
||||
async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
task_queue = Queue()
|
||||
result_queue = Queue()
|
||||
# push the task
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
# worker should exit on recycle
|
||||
worker(task_queue, result_queue, Value("f", -1))
|
||||
# check if the work has been done
|
||||
assert result_queue.qsize() == 1
|
||||
# save_limit test
|
||||
monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
|
||||
result_queue.put("STOP")
|
||||
# run monitor
|
||||
monitor(result_queue)
|
||||
assert Success.objects.count() == Conf.SAVE_LIMIT
|
||||
broker.delete_queue()
|
||||
# @pytest.mark.django_db
|
||||
# def test_max_rss(broker, monkeypatch):
|
||||
# # set up the Sentinel
|
||||
# broker.list_key = "test_max_rss_test:q"
|
||||
# async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
# start_event = Event()
|
||||
# stop_event = Event()
|
||||
# cluster_id = uuidlib.uuid4()
|
||||
# # override settings
|
||||
# monkeypatch.setattr(Conf, "MAX_RSS", 40000)
|
||||
# monkeypatch.setattr(Conf, "WORKERS", 1)
|
||||
# # set a timer to stop the Sentinel
|
||||
# threading.Timer(3, stop_event.set).start()
|
||||
# s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
|
||||
# assert start_event.is_set()
|
||||
# assert s.status() == Conf.STOPPED
|
||||
# assert s.reincarnations == 1
|
||||
# async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
|
||||
# task_queue = Queue()
|
||||
# result_queue = Queue()
|
||||
# # push the task
|
||||
# # pusher(task_queue, stop_event, broker=broker)
|
||||
# # # worker should exit on recycle
|
||||
# # worker(task_queue, result_queue, Value("f", -1))
|
||||
# # check if the work has been done
|
||||
# assert result_queue.qsize() == 1
|
||||
# # save_limit test
|
||||
# monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
|
||||
# result_queue.put("STOP")
|
||||
# # run monitor
|
||||
# monitor(result_queue)
|
||||
# assert Success.objects.count() == Conf.SAVE_LIMIT
|
||||
# broker.delete_queue()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_bad_secret(broker, monkeypatch):
|
||||
broker.list_key = "test_bad_secret:q"
|
||||
async_task("math.copysign", 1, -1, broker=broker)
|
||||
stop_event = Event()
|
||||
stop_event.set()
|
||||
start_event = Event()
|
||||
cluster_id = uuidlib.uuid4()
|
||||
s = Sentinel(
|
||||
stop_event, start_event, cluster_id=cluster_id, broker=broker, start=False
|
||||
)
|
||||
Stat(s).save()
|
||||
# change the SECRET
|
||||
monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
|
||||
stat = Stat.get_all()
|
||||
assert len(stat) == 0
|
||||
assert Stat.get(pid=s.parent_pid, cluster_id=cluster_id) is None
|
||||
task_queue = Queue()
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
result_queue = Queue()
|
||||
task_queue.put("STOP")
|
||||
worker(
|
||||
task_queue,
|
||||
result_queue,
|
||||
Value("f", -1),
|
||||
)
|
||||
assert result_queue.qsize() == 0
|
||||
broker.delete_queue()
|
||||
# @pytest.mark.django_db
|
||||
# def test_bad_secret(broker, monkeypatch):
|
||||
# broker.list_key = "test_bad_secret:q"
|
||||
# async_task("math.copysign", 1, -1, broker=broker)
|
||||
# stop_event = Event()
|
||||
# stop_event.set()
|
||||
# start_event = Event()
|
||||
# cluster_id = uuidlib.uuid4()
|
||||
# s = Sentinel(
|
||||
# stop_event, start_event, cluster_id=cluster_id, broker=broker, start=False
|
||||
# )
|
||||
# Stat(s).save()
|
||||
# # change the SECRET
|
||||
# monkeypatch.setattr(Conf, "SECRET_KEY", "OOPS")
|
||||
# stat = Stat.get_all()
|
||||
# assert len(stat) == 0
|
||||
# assert Stat.get(pid=s.parent_pid, cluster_id=cluster_id) is None
|
||||
# task_queue = Queue()
|
||||
# # pusher(task_queue, stop_event, broker=broker)
|
||||
# result_queue = Queue()
|
||||
# task_queue.put("STOP")
|
||||
# worker(
|
||||
# task_queue,
|
||||
# result_queue,
|
||||
# Value("f", -1),
|
||||
# )
|
||||
# assert result_queue.qsize() == 0
|
||||
# broker.delete_queue()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_attempt_count(broker, monkeypatch):
|
||||
monkeypatch.setattr(Conf, "MAX_ATTEMPTS", 3)
|
||||
tag = uuid()
|
||||
task = {
|
||||
"id": tag[1],
|
||||
"name": tag[0],
|
||||
"func": "math.copysign",
|
||||
"args": (1, -1),
|
||||
"kwargs": {},
|
||||
"started": timezone.now(),
|
||||
"stopped": timezone.now(),
|
||||
"success": False,
|
||||
"result": None,
|
||||
}
|
||||
task = QueueTask(
|
||||
id=tag[1],
|
||||
name=tag[0],
|
||||
func="math.copysign",
|
||||
args=(1, -1),
|
||||
kwargs={},
|
||||
started_at=timezone.now(),
|
||||
finished_at=timezone.now(),
|
||||
result_status=QueueTask.Result.FAILED,
|
||||
result=None,
|
||||
)
|
||||
# initial save - no success
|
||||
save_task(task, broker)
|
||||
assert Task.objects.filter(id=task["id"]).exists()
|
||||
saved_task = Task.objects.get(id=task["id"])
|
||||
assert Task.objects.filter(id=task.id).exists()
|
||||
saved_task = Task.objects.get(id=task.id)
|
||||
assert saved_task.attempt_count == 1
|
||||
sleep(0.5)
|
||||
# second save
|
||||
task["stopped"] = timezone.now()
|
||||
task.finished_at = timezone.now()
|
||||
save_task(task, broker)
|
||||
saved_task = Task.objects.get(id=task["id"])
|
||||
saved_task = Task.objects.get(id=task.id)
|
||||
assert saved_task.attempt_count == 2
|
||||
# third save -
|
||||
task["stopped"] = timezone.now()
|
||||
task.finished_at = timezone.now()
|
||||
save_task(task, broker)
|
||||
saved_task = Task.objects.get(id=task["id"])
|
||||
saved_task = Task.objects.get(id=task.id)
|
||||
assert saved_task.attempt_count == 3
|
||||
# task should be removed from queue
|
||||
assert broker.queue_size() == 0
|
||||
@@ -554,43 +523,43 @@ def test_attempt_count(broker, monkeypatch):
|
||||
@pytest.mark.django_db
|
||||
def test_update_failed(broker):
|
||||
tag = uuid()
|
||||
task = {
|
||||
"id": tag[1],
|
||||
"name": tag[0],
|
||||
"func": "math.copysign",
|
||||
"args": (1, -1),
|
||||
"kwargs": {},
|
||||
"started": timezone.now(),
|
||||
"stopped": timezone.now(),
|
||||
"success": False,
|
||||
"result": None,
|
||||
}
|
||||
task = QueueTask(
|
||||
id=tag[1],
|
||||
name=tag[0],
|
||||
func="math.copysign",
|
||||
args=(1, -1),
|
||||
kwargs={},
|
||||
started_at=timezone.now(),
|
||||
finished_at=timezone.now(),
|
||||
result_status=QueueTask.Result.FAILED,
|
||||
result=None,
|
||||
)
|
||||
# initial save - no success
|
||||
save_task(task, broker)
|
||||
assert Task.objects.filter(id=task["id"]).exists()
|
||||
saved_task = Task.objects.get(id=task["id"])
|
||||
assert Task.objects.filter(id=task.id).exists()
|
||||
saved_task = Task.objects.get(id=task.id)
|
||||
assert saved_task.success is False
|
||||
sleep(0.5)
|
||||
# second save - no success
|
||||
old_stopped = task["stopped"]
|
||||
task["stopped"] = timezone.now()
|
||||
old_stopped = task.finished_at
|
||||
task.finished_at = timezone.now()
|
||||
save_task(task, broker)
|
||||
saved_task = Task.objects.get(id=task["id"])
|
||||
saved_task = Task.objects.get(id=task.id)
|
||||
assert saved_task.stopped > old_stopped
|
||||
# third save - success
|
||||
task["stopped"] = timezone.now()
|
||||
task["result"] = "result"
|
||||
task["success"] = True
|
||||
task.finished_at = timezone.now()
|
||||
task.result = "result"
|
||||
task.result_status = QueueTask.Result.SUCCESS
|
||||
save_task(task, broker)
|
||||
saved_task = Task.objects.get(id=task["id"])
|
||||
saved_task = Task.objects.get(id=task.id)
|
||||
assert saved_task.success is True
|
||||
# fourth save - no success
|
||||
task["result"] = None
|
||||
task["success"] = False
|
||||
task["stopped"] = old_stopped
|
||||
task.result = None
|
||||
task.result_status = QueueTask.Result.FAILED
|
||||
task.finished_at = old_stopped
|
||||
save_task(task, broker)
|
||||
# should not overwrite success
|
||||
saved_task = Task.objects.get(id=task["id"])
|
||||
saved_task = Task.objects.get(id=task.id)
|
||||
assert saved_task.success is True
|
||||
assert saved_task.result == "result"
|
||||
|
||||
@@ -607,47 +576,40 @@ def test_acknowledge_failure_override():
|
||||
self.acknowledgements[task_id] = count + 1
|
||||
|
||||
tag = uuid()
|
||||
task_fail_ack = {
|
||||
"id": tag[1],
|
||||
"name": tag[0],
|
||||
"ack_id": "test_fail_ack_id",
|
||||
"ack_failure": True,
|
||||
"func": "math.copysign",
|
||||
"args": (1, -1),
|
||||
"kwargs": {},
|
||||
"started": timezone.now(),
|
||||
"stopped": timezone.now(),
|
||||
"success": False,
|
||||
"result": None,
|
||||
}
|
||||
task_fail_ack = QueueTask(
|
||||
id=tag[1],
|
||||
name=tag[0],
|
||||
ack_id="test_fail_ack_id",
|
||||
ack_failure=True,
|
||||
func="math.copysign",
|
||||
args=(1, -1),
|
||||
kwargs={},
|
||||
started_at=timezone.now(),
|
||||
finished_at=timezone.now(),
|
||||
result_status=QueueTask.Result.SUCCESS,
|
||||
result=None,
|
||||
)
|
||||
|
||||
tag = uuid()
|
||||
task_fail_no_ack = task_fail_ack.copy()
|
||||
task_fail_no_ack.update(
|
||||
{"id": tag[1], "name": tag[0], "ack_id": "test_fail_no_ack_id"}
|
||||
)
|
||||
del task_fail_no_ack["ack_failure"]
|
||||
task_fail_no_ack = copy.deepcopy(task_fail_ack)
|
||||
task_fail_no_ack.id = tag[1]
|
||||
task_fail_no_ack.name = tag[0]
|
||||
task_fail_no_ack.ack_id = None
|
||||
task_fail_no_ack.ack_failure = False
|
||||
|
||||
tag = uuid()
|
||||
task_success_ack = task_fail_ack.copy()
|
||||
task_success_ack.update(
|
||||
{
|
||||
"id": tag[1],
|
||||
"name": tag[0],
|
||||
"ack_id": "test_success_ack_id",
|
||||
"success": True,
|
||||
}
|
||||
)
|
||||
del task_success_ack["ack_failure"]
|
||||
task_success_ack = copy.deepcopy(task_fail_ack)
|
||||
task_success_ack.id = tag[1]
|
||||
task_success_ack.name = tag[0]
|
||||
task_success_ack.ack_id = "test_success_ack_id"
|
||||
task_success_ack.result_status = QueueTask.Result.SUCCESS
|
||||
task_success_ack.ack_failure = False
|
||||
|
||||
result_queue = Queue()
|
||||
result_queue.put(task_fail_ack)
|
||||
result_queue.put(task_fail_no_ack)
|
||||
result_queue.put(task_success_ack)
|
||||
result_queue.put("STOP")
|
||||
broker = VerifyAckMockBroker(list_key="key")
|
||||
|
||||
monitor(result_queue, broker)
|
||||
save_task(task_fail_ack, broker=broker)
|
||||
save_task(task_fail_no_ack, broker=broker)
|
||||
save_task(task_success_ack, broker=broker)
|
||||
|
||||
assert broker.acknowledgements.get("test_fail_ack_id") == 1
|
||||
assert broker.acknowledgements.get("test_fail_no_ack_id") is None
|
||||
@@ -660,7 +622,7 @@ class TestSignals:
|
||||
broker.list_key = "pre_enqueue_test:q"
|
||||
broker.delete_queue()
|
||||
self.signal_was_called: bool = False
|
||||
self.task: Optional[dict] = None
|
||||
self.task = None
|
||||
|
||||
def handler(sender, task, **kwargs):
|
||||
self.signal_was_called = True
|
||||
@@ -669,7 +631,7 @@ class TestSignals:
|
||||
pre_enqueue.connect(handler)
|
||||
task_id = async_task("math.copysign", 1, -1, broker=broker)
|
||||
assert self.signal_was_called is True
|
||||
assert self.task.get("id") == task_id
|
||||
assert self.task.id == task_id
|
||||
pre_enqueue.disconnect(handler)
|
||||
broker.delete_queue()
|
||||
|
||||
@@ -678,7 +640,7 @@ class TestSignals:
|
||||
broker.list_key = "pre_execute_test:q"
|
||||
broker.delete_queue()
|
||||
self.signal_was_called: bool = False
|
||||
self.task: Optional[dict] = None
|
||||
self.task = None
|
||||
self.func = None
|
||||
|
||||
def handler(sender, task, func, **kwargs):
|
||||
@@ -688,27 +650,19 @@ class TestSignals:
|
||||
|
||||
pre_execute.connect(handler)
|
||||
task_id = async_task("math.copysign", 1, -1, broker=broker)
|
||||
task_queue = Queue()
|
||||
result_queue = Queue()
|
||||
event = Event()
|
||||
event.set()
|
||||
pusher(task_queue, event, broker=broker)
|
||||
task_queue.put("STOP")
|
||||
worker(task_queue, result_queue, Value("f", -1))
|
||||
result_queue.put("STOP")
|
||||
monitor(result_queue, broker)
|
||||
run_cluster_once(workers=1, broker=broker)
|
||||
broker.delete_queue()
|
||||
assert self.task.id == task_id
|
||||
assert self.signal_was_called is True
|
||||
assert self.task.get("id") == task_id
|
||||
assert self.func == copysign
|
||||
assert self.func == 'math.copysign'
|
||||
pre_execute.disconnect(handler)
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_post_execute_signal(self, broker):
|
||||
broker.list_key = "post_execute_test:q"
|
||||
broker.delete_queue()
|
||||
self.signal_was_called: bool = False
|
||||
self.task: Optional[dict] = None
|
||||
self.signal_was_called = False
|
||||
self.task = None
|
||||
self.func = None
|
||||
|
||||
def handler(sender, task, **kwargs):
|
||||
@@ -717,33 +671,25 @@ class TestSignals:
|
||||
|
||||
post_execute.connect(handler)
|
||||
task_id = async_task("math.copysign", 1, -1, broker=broker)
|
||||
task_queue = Queue()
|
||||
result_queue = Queue()
|
||||
event = Event()
|
||||
event.set()
|
||||
pusher(task_queue, event, broker=broker)
|
||||
task_queue.put("STOP")
|
||||
worker(task_queue, result_queue, Value("f", -1))
|
||||
result_queue.put("STOP")
|
||||
monitor(result_queue, broker)
|
||||
run_cluster_once(workers=1, broker=broker)
|
||||
broker.delete_queue()
|
||||
assert self.signal_was_called is True
|
||||
assert self.task.get("id") == task_id
|
||||
assert self.task.get("result") == -1
|
||||
assert self.task.id == task_id
|
||||
assert self.task.result == -1
|
||||
post_execute.disconnect(handler)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def assert_result(task):
|
||||
assert task is not None
|
||||
assert task.success is True
|
||||
assert task.has_succeeded is True
|
||||
assert task.result == 1506
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def assert_bad_result(task):
|
||||
assert task is not None
|
||||
assert task.success is False
|
||||
assert task.has_succeeded is False
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
|
||||
@@ -1,25 +1,25 @@
|
||||
import pytest
|
||||
from django.core.management import call_command
|
||||
# import pytest
|
||||
# from django.core.management import call_command
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_qcluster():
|
||||
call_command("qcluster", run_once=True)
|
||||
# @pytest.mark.django_db
|
||||
# def test_qcluster():
|
||||
# call_command("qcluster", run_once=True)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_qmonitor():
|
||||
call_command("qmonitor", run_once=True)
|
||||
# @pytest.mark.django_db
|
||||
# def test_qmonitor():
|
||||
# call_command("qmonitor", run_once=True)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_qinfo():
|
||||
call_command("qinfo")
|
||||
call_command("qinfo", config=True)
|
||||
call_command("qinfo", ids=True)
|
||||
# @pytest.mark.django_db
|
||||
# def test_qinfo():
|
||||
# call_command("qinfo")
|
||||
# call_command("qinfo", config=True)
|
||||
# call_command("qinfo", ids=True)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_qmemory():
|
||||
call_command("qmemory", run_once=True)
|
||||
call_command("qmemory", workers=True, run_once=True)
|
||||
# @pytest.mark.django_db
|
||||
# def test_qmemory():
|
||||
# call_command("qmemory", run_once=True)
|
||||
# call_command("qmemory", workers=True, run_once=True)
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from django_q.brokers import get_broker
|
||||
from django_q.cluster import Cluster
|
||||
from django_q.conf import Conf
|
||||
from django_q.monitor import get_ids, info, monitor
|
||||
from django_q.status import Stat
|
||||
from django_q.tasks import async_task
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_monitor(monkeypatch):
|
||||
cluster_id = uuid.uuid4()
|
||||
assert Stat.get(pid=0, cluster_id=cluster_id).sentinel == 0
|
||||
c = Cluster()
|
||||
c.start()
|
||||
stats = monitor(run_once=True)
|
||||
assert get_ids() is True
|
||||
c.stop()
|
||||
assert len(stats) > 0
|
||||
found_c = False
|
||||
for stat in stats:
|
||||
if stat.cluster_id == c.cluster_id:
|
||||
found_c = True
|
||||
assert stat.uptime() > 0
|
||||
assert stat.empty_queues() is True
|
||||
break
|
||||
assert found_c
|
||||
# test lock size
|
||||
monkeypatch.setattr(Conf, "ORM", "default")
|
||||
b = get_broker("monitor_test")
|
||||
b.enqueue("test")
|
||||
b.dequeue()
|
||||
assert b.lock_size() == 1
|
||||
monitor(run_once=True, broker=b)
|
||||
b.delete_queue()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_info():
|
||||
info()
|
||||
do_sync()
|
||||
info()
|
||||
for _ in range(24):
|
||||
do_sync()
|
||||
info()
|
||||
|
||||
|
||||
def do_sync():
|
||||
async_task("django_q.tests.tasks.countdown", 1, sync=True, save=True)
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
from datetime import datetime, timedelta
|
||||
from multiprocessing import Event, Value
|
||||
from unittest import mock
|
||||
from django_q.utils import localtime
|
||||
|
||||
import pytest
|
||||
import django
|
||||
@@ -12,9 +13,9 @@ from django.utils import timezone
|
||||
from django.utils.timezone import is_naive
|
||||
|
||||
from django_q.brokers import Broker, get_broker
|
||||
from django_q.cluster import localtime, monitor, pusher, scheduler, worker
|
||||
from django_q.helpers import run_scheduler_once, get_scheduled_tasks, save_task, run_task
|
||||
from django_q.conf import Conf
|
||||
from django_q.queues import Queue
|
||||
from queue import Queue
|
||||
from django_q.tasks import Schedule, fetch
|
||||
from django_q.tasks import schedule as create_schedule
|
||||
from django_q.tests.settings import BASE_DIR
|
||||
@@ -103,7 +104,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
|
||||
)
|
||||
|
||||
# Run scheduler so we get the next run date
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule.refresh_from_db()
|
||||
|
||||
# It's now the day after exactly at midnight UTC
|
||||
@@ -115,7 +116,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
|
||||
assert str(next_run) == "2021-03-28 01:00:00+01:00"
|
||||
|
||||
# Run scheduler so we get the next run date
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule.refresh_from_db()
|
||||
|
||||
next_run = schedule.next_run
|
||||
@@ -126,7 +127,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
|
||||
assert str(next_run) == "2021-03-29 01:00:00+02:00"
|
||||
|
||||
# Run scheduler so we get the next run date
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule.refresh_from_db()
|
||||
|
||||
next_run = schedule.next_run
|
||||
@@ -147,7 +148,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
|
||||
)
|
||||
|
||||
# Run scheduler so we get the next run date
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule.refresh_from_db()
|
||||
|
||||
next_run = schedule.next_run
|
||||
@@ -158,7 +159,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
|
||||
assert str(next_run) == "2021-10-30 01:00:00+02:00"
|
||||
|
||||
# Run scheduler so we get the next run date
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule.refresh_from_db()
|
||||
|
||||
next_run = schedule.next_run
|
||||
@@ -169,7 +170,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
|
||||
assert str(next_run) == "2021-10-31 01:00:00+02:00"
|
||||
|
||||
# Run scheduler so we get the next run date
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule.refresh_from_db()
|
||||
|
||||
next_run = schedule.next_run
|
||||
@@ -208,24 +209,15 @@ def test_scheduler(broker, monkeypatch):
|
||||
repeats=1,
|
||||
)
|
||||
# run scheduler
|
||||
scheduler(broker=broker)
|
||||
# set up the workflow
|
||||
task_queue = Queue()
|
||||
stop_event = Event()
|
||||
stop_event.set()
|
||||
# push it
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
assert task_queue.qsize() == 1
|
||||
assert broker.queue_size() == 0
|
||||
task_queue.put("STOP")
|
||||
# let a worker handle them
|
||||
result_queue = Queue()
|
||||
worker(task_queue, result_queue, Value("b", -1))
|
||||
assert result_queue.qsize() == 1
|
||||
result_queue.put("STOP")
|
||||
# store the results
|
||||
monitor(result_queue)
|
||||
assert result_queue.qsize() == 0
|
||||
run_scheduler_once(broker=broker)
|
||||
# get tasks
|
||||
tasks = get_scheduled_tasks(broker=broker)
|
||||
for task in tasks:
|
||||
# let a worker handle them
|
||||
ran_task = run_task(task)
|
||||
# store the results
|
||||
save_task(task=ran_task, broker=broker)
|
||||
|
||||
schedule = Schedule.objects.get(pk=schedule.pk)
|
||||
assert schedule.repeats == 0
|
||||
assert schedule.last_run() is not None
|
||||
@@ -297,7 +289,7 @@ def test_scheduler(broker, monkeypatch):
|
||||
)
|
||||
assert schedule is not None
|
||||
assert schedule.last_run() is None
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
# via model
|
||||
Schedule.objects.create(
|
||||
func="django_q.tests.tasks.word_multiply",
|
||||
@@ -306,7 +298,7 @@ def test_scheduler(broker, monkeypatch):
|
||||
schedule_type=Schedule.DAILY,
|
||||
)
|
||||
# scheduler
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
# ONCE schedule should be deleted
|
||||
assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False
|
||||
# Catch up On
|
||||
@@ -320,12 +312,12 @@ def test_scheduler(broker, monkeypatch):
|
||||
next_run=timezone.now() - timedelta(hours=12),
|
||||
repeats=-1,
|
||||
)
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule = Schedule.objects.get(pk=schedule.pk)
|
||||
assert schedule.next_run < now
|
||||
# Catch up off
|
||||
monkeypatch.setattr(Conf, "CATCH_UP", False)
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule = Schedule.objects.get(pk=schedule.pk)
|
||||
assert schedule.next_run > now
|
||||
# Done
|
||||
@@ -338,7 +330,7 @@ def test_scheduler(broker, monkeypatch):
|
||||
word="catch_up",
|
||||
schedule_type=Schedule.BIMONTHLY,
|
||||
)
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule = Schedule.objects.get(pk=schedule.pk)
|
||||
assert schedule.next_run.date() == add_months(timezone.now(), 2).date()
|
||||
|
||||
@@ -349,7 +341,7 @@ def test_scheduler(broker, monkeypatch):
|
||||
word="catch_up",
|
||||
schedule_type=Schedule.BIWEEKLY,
|
||||
)
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
schedule = Schedule.objects.get(pk=schedule.pk)
|
||||
assert schedule.next_run.date() == (timezone.now() + timedelta(weeks=2)).date()
|
||||
broker.delete_queue()
|
||||
@@ -367,16 +359,12 @@ def test_scheduler(broker, monkeypatch):
|
||||
repeats=1,
|
||||
)
|
||||
# run scheduler
|
||||
scheduler(broker=broker)
|
||||
# set up the workflow
|
||||
task_queue = Queue()
|
||||
stop_event = Event()
|
||||
stop_event.set()
|
||||
run_scheduler_once(broker=broker)
|
||||
# push it
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
tasks = get_scheduled_tasks(broker=broker)
|
||||
|
||||
# queue must be empty
|
||||
assert task_queue.qsize() == 0
|
||||
assert len(tasks) == 0
|
||||
|
||||
monkeypatch.setattr(Conf, "PREFIX", "default")
|
||||
# create a schedule on the same cluster
|
||||
@@ -391,16 +379,12 @@ def test_scheduler(broker, monkeypatch):
|
||||
repeats=1,
|
||||
)
|
||||
# run scheduler
|
||||
scheduler(broker=broker)
|
||||
# set up the workflow
|
||||
task_queue = Queue()
|
||||
stop_event = Event()
|
||||
stop_event.set()
|
||||
run_scheduler_once(broker=broker)
|
||||
# push it
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
tasks = get_scheduled_tasks(broker=broker)
|
||||
|
||||
# queue must contain a task
|
||||
assert task_queue.qsize() == 1
|
||||
assert len(tasks) == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@@ -422,35 +406,31 @@ def test_intended_schedule_kwarg(broker, monkeypatch):
|
||||
assert schedule.last_run() is None
|
||||
assert schedule.intended_date_kwarg == 'intended_date'
|
||||
# run scheduler
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
# set up the workflow
|
||||
task_queue = Queue()
|
||||
stop_event = Event()
|
||||
stop_event.set()
|
||||
# push it
|
||||
pusher(task_queue, stop_event, broker=broker)
|
||||
assert task_queue.qsize() == 1
|
||||
task = task_queue.get()
|
||||
assert 'intended_date' in task['kwargs']
|
||||
assert task['kwargs']['intended_date'] == run_date.isoformat()
|
||||
scheduled_tasks = get_scheduled_tasks(broker=broker)
|
||||
assert len(scheduled_tasks) == 1
|
||||
task = scheduled_tasks[0]
|
||||
assert 'intended_date' in task.kwargs
|
||||
assert task.kwargs['intended_date'] == run_date.isoformat()
|
||||
|
||||
|
||||
@override_settings(
|
||||
DATABASE_ROUTERS=REPLICA_DATABASE_ROUTERS, DATABASES=REPLICA_DATABASES
|
||||
)
|
||||
@pytest.mark.django_db
|
||||
def test_scheduler_atomic_must_specify_the_write_db(
|
||||
orm_broker: Broker,
|
||||
):
|
||||
"""
|
||||
GIVEN a environment with a read/write configured replica database
|
||||
WHEN the scheduler is called
|
||||
THEN the transaction must be called with the write database.
|
||||
"""
|
||||
broker = get_broker(list_key="scheduler_test:q")
|
||||
with mock.patch("django_q.cluster.db.transaction") as mocked_db:
|
||||
scheduler(broker=broker)
|
||||
mocked_db.atomic.assert_called_with(using="writable")
|
||||
# @override_settings(
|
||||
# DATABASE_ROUTERS=REPLICA_DATABASE_ROUTERS, DATABASES=REPLICA_DATABASES
|
||||
# )
|
||||
# @pytest.mark.django_db
|
||||
# def test_scheduler_atomic_must_specify_the_write_db(
|
||||
# orm_broker: Broker,
|
||||
# ):
|
||||
# """
|
||||
# GIVEN a environment with a read/write configured replica database
|
||||
# WHEN the scheduler is called
|
||||
# THEN the transaction must be called with the write database.
|
||||
# """
|
||||
# broker = get_broker(list_key="scheduler_test:q")
|
||||
# with mock.patch("django_q.scheduler.db.transaction") as mocked_db:
|
||||
# run_scheduler_once(broker=broker)
|
||||
# mocked_db.atomic.assert_called_with(using="writable")
|
||||
|
||||
|
||||
@override_settings(
|
||||
@@ -467,7 +447,7 @@ def test_scheduler_atomic_must_specify_the_database_based_on_router_redirection(
|
||||
"""
|
||||
broker = get_broker(list_key="scheduler_test:q")
|
||||
with mock.patch("django_q.cluster.db.transaction") as mocked_db:
|
||||
scheduler(broker=broker)
|
||||
run_scheduler_once(broker=broker)
|
||||
mocked_db.atomic.assert_called_with(using="default")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user