mirror of
https://github.com/django-q2/django-q2.git
synced 2026-09-15 13:37:56 +08:00
356 lines
10 KiB
Python
356 lines
10 KiB
Python
import os
|
|
from time import sleep
|
|
|
|
import pytest
|
|
|
|
from django_q.brokers import Broker, get_broker
|
|
from django_q.conf import Conf
|
|
from django_q.humanhash import uuid
|
|
from django_q.tests.settings import MONGO_HOST, REDIS_HOST
|
|
|
|
|
|
def test_broker(monkeypatch):
|
|
broker = Broker()
|
|
broker.enqueue("test")
|
|
broker.dequeue()
|
|
broker.queue_size()
|
|
broker.lock_size()
|
|
broker.purge_queue()
|
|
broker.delete("id")
|
|
broker.delete_queue()
|
|
broker.acknowledge("test")
|
|
broker.ping()
|
|
broker.info()
|
|
# stats
|
|
assert broker.get_stat("test_1") is None
|
|
broker.set_stat("test_1", "test", 3)
|
|
assert broker.get_stat("test_1") == "test"
|
|
assert broker.get_stats("test:*")[0] == "test"
|
|
# stats with no cache
|
|
monkeypatch.setattr(Conf, "CACHE", "not_configured")
|
|
broker.cache = broker.get_cache()
|
|
assert broker.get_stat("test_1") is None
|
|
broker.set_stat("test_1", "test", 3)
|
|
assert broker.get_stat("test_1") is None
|
|
assert broker.get_stats("test:*") is None
|
|
|
|
|
|
def test_broker_set_stat_prunes_stale_master_list():
|
|
# regression: set_stat should prune stale master-list entries on register
|
|
broker = Broker()
|
|
a_key = f"{Conf.Q_STAT}:A"
|
|
b_key = f"{Conf.Q_STAT}:B"
|
|
broker.cache.delete(Conf.Q_STAT)
|
|
|
|
broker.set_stat(a_key, "state_a", 3)
|
|
assert a_key in broker.cache.get(Conf.Q_STAT)
|
|
|
|
# Drop A's per-stat value to simulate a dead cluster whose TTL expired,
|
|
# leaving the master-list entry as a stale reference.
|
|
broker.cache.delete(a_key)
|
|
assert broker.get_stat(a_key) is None
|
|
assert a_key in broker.cache.get(Conf.Q_STAT)
|
|
|
|
# Registering B should prune stale A from the master list.
|
|
broker.set_stat(b_key, "state_b", 3)
|
|
key_list = broker.cache.get(Conf.Q_STAT)
|
|
assert a_key not in key_list
|
|
assert b_key in key_list
|
|
assert broker.get_stat(b_key) == "state_b"
|
|
|
|
|
|
def test_broker_set_stat_skips_master_list_write_on_repeat(monkeypatch):
|
|
# set_stat should only write the master list when membership changes
|
|
broker = Broker()
|
|
a_key = f"{Conf.Q_STAT}:A"
|
|
broker.cache.delete(Conf.Q_STAT)
|
|
|
|
writes = []
|
|
orig_set = broker.cache.set
|
|
|
|
def counting_set(key, value, timeout=None, **kw):
|
|
if key == Conf.Q_STAT:
|
|
writes.append(key)
|
|
return orig_set(key, value, timeout, **kw)
|
|
|
|
monkeypatch.setattr(broker.cache, "set", counting_set)
|
|
|
|
# First call adds a new entry, one master-list write expected
|
|
broker.set_stat(a_key, "state_a", 3)
|
|
assert len(writes) == 1
|
|
|
|
# Subsequent calls for the same key must not rewrite the master list
|
|
broker.set_stat(a_key, "state_a", 3)
|
|
broker.set_stat(a_key, "state_a", 3)
|
|
broker.set_stat(a_key, "state_a", 3)
|
|
assert len(writes) == 1
|
|
|
|
|
|
def test_redis(monkeypatch):
|
|
monkeypatch.setattr(Conf, "DJANGO_REDIS", None)
|
|
broker = get_broker()
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
monkeypatch.setattr(Conf, "REDIS", {"host": REDIS_HOST, "port": 7799})
|
|
broker = get_broker()
|
|
with pytest.raises(Exception):
|
|
broker.ping()
|
|
monkeypatch.setattr(Conf, "REDIS", f"redis://{REDIS_HOST}:7799")
|
|
broker = get_broker()
|
|
with pytest.raises(Exception):
|
|
broker.ping()
|
|
|
|
|
|
def test_custom(monkeypatch):
|
|
monkeypatch.setattr(Conf, "BROKER_CLASS", "django_q.brokers.redis_broker.Redis")
|
|
broker = get_broker()
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
assert broker.__class__.__name__ == "Redis"
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
not os.getenv("IRON_MQ_TOKEN"), reason="requires IronMQ credentials"
|
|
)
|
|
def test_ironmq(monkeypatch):
|
|
monkeypatch.setattr(
|
|
Conf,
|
|
"IRON_MQ",
|
|
{
|
|
"token": os.getenv("IRON_MQ_TOKEN"),
|
|
"project_id": os.getenv("IRON_MQ_PROJECT_ID"),
|
|
},
|
|
)
|
|
# check broker
|
|
broker = get_broker(list_key=uuid()[0])
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
# initialize the queue
|
|
broker.enqueue("test")
|
|
# clear before we start
|
|
broker.purge_queue()
|
|
assert broker.queue_size() == 0
|
|
# async_task
|
|
broker.enqueue("test")
|
|
# dequeue
|
|
task = broker.dequeue()[0]
|
|
assert task[1] == "test"
|
|
broker.acknowledge(task[0])
|
|
assert broker.dequeue() is None
|
|
# Retry test
|
|
# monkeypatch.setattr(Conf, 'RETRY', 1)
|
|
# broker.async_task('test')
|
|
# assert broker.dequeue() is not None
|
|
# sleep(3)
|
|
# assert broker.dequeue() is not None
|
|
# task = broker.dequeue()[0]
|
|
# assert len(task) > 0
|
|
# broker.acknowledge(task[0])
|
|
# sleep(3)
|
|
# delete job
|
|
task_id = broker.enqueue("test")
|
|
broker.delete(task_id)
|
|
assert broker.dequeue() is None
|
|
# fail
|
|
task_id = broker.enqueue("test")
|
|
broker.fail(task_id)
|
|
# bulk test
|
|
for _ in range(5):
|
|
broker.enqueue("test")
|
|
monkeypatch.setattr(Conf, "BULK", 5)
|
|
tasks = broker.dequeue()
|
|
for task in tasks:
|
|
assert task is not None
|
|
broker.acknowledge(task[0])
|
|
# duplicate acknowledge
|
|
broker.acknowledge(task[0])
|
|
# delete queue
|
|
broker.enqueue("test")
|
|
broker.enqueue("test")
|
|
broker.purge_queue()
|
|
assert broker.dequeue() is None
|
|
broker.delete_queue()
|
|
|
|
|
|
def test_sqs(monkeypatch):
|
|
monkeypatch.setattr(
|
|
Conf,
|
|
"SQS",
|
|
{
|
|
"aws_region": os.getenv("AWS_REGION"),
|
|
"aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"),
|
|
"aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"),
|
|
"receive_message_wait_time_seconds": 5,
|
|
},
|
|
)
|
|
# check broker
|
|
broker = get_broker(list_key="testing")
|
|
broker.purge_queue()
|
|
assert "receive_message_wait_time_seconds" in Conf.SQS
|
|
assert "aws_region" in Conf.SQS
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
assert broker.queue_size() == 0
|
|
# async_task
|
|
broker.enqueue("test")
|
|
# dequeue
|
|
task = broker.dequeue()[0]
|
|
assert task[1] == "test"
|
|
broker.acknowledge(task[0])
|
|
assert broker.dequeue() is None
|
|
# Retry test
|
|
monkeypatch.setattr(Conf, "RETRY", 1)
|
|
broker.enqueue("test")
|
|
sleep(2)
|
|
# Sometimes SQS is not linear
|
|
task = broker.dequeue()
|
|
if not task:
|
|
pytest.skip("SQS being weird")
|
|
task = task[0]
|
|
assert len(task) > 0
|
|
broker.acknowledge(task[0])
|
|
sleep(2)
|
|
# delete job
|
|
monkeypatch.setattr(Conf, "RETRY", 60)
|
|
broker.enqueue("test")
|
|
sleep(1)
|
|
task = broker.dequeue()
|
|
if not task:
|
|
pytest.skip("SQS being weird")
|
|
task_id = task[0][0]
|
|
broker.delete(task_id)
|
|
assert broker.dequeue() is None
|
|
# fail
|
|
broker.enqueue("test")
|
|
task = None
|
|
while task is None:
|
|
task = broker.dequeue()
|
|
broker.fail(task[0][0])
|
|
# bulk test
|
|
for _ in range(10):
|
|
broker.enqueue("test")
|
|
monkeypatch.setattr(Conf, "BULK", 12)
|
|
tasks = broker.dequeue()
|
|
for task in tasks:
|
|
assert task is not None
|
|
broker.acknowledge(task[0])
|
|
assert broker.lock_size() == 0
|
|
# delete queue
|
|
broker.enqueue("test")
|
|
broker.purge_queue()
|
|
broker.delete_queue()
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_orm(monkeypatch):
|
|
monkeypatch.setattr(Conf, "ORM", "default")
|
|
# check broker
|
|
broker = get_broker(list_key="orm_test")
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
# clear before we start
|
|
broker.delete_queue()
|
|
# async_task
|
|
broker.enqueue("test")
|
|
assert broker.queue_size() == 1
|
|
# dequeue
|
|
task = broker.dequeue()[0]
|
|
assert task[1] == "test"
|
|
broker.acknowledge(task[0])
|
|
assert broker.queue_size() == 0
|
|
# Retry test
|
|
monkeypatch.setattr(Conf, "RETRY", 1)
|
|
broker.enqueue("test")
|
|
assert broker.queue_size() == 1
|
|
broker.dequeue()
|
|
assert broker.queue_size() == 0
|
|
sleep(1.5)
|
|
assert broker.queue_size() == 1
|
|
task = broker.dequeue()[0]
|
|
assert broker.queue_size() == 0
|
|
broker.acknowledge(task[0])
|
|
sleep(1.5)
|
|
assert broker.queue_size() == 0
|
|
# delete job
|
|
task_id = broker.enqueue("test")
|
|
broker.delete(task_id)
|
|
assert broker.dequeue() is None
|
|
# fail
|
|
task_id = broker.enqueue("test")
|
|
broker.fail(task_id)
|
|
# bulk test
|
|
for _ in range(5):
|
|
broker.enqueue("test")
|
|
monkeypatch.setattr(Conf, "BULK", 5)
|
|
tasks = broker.dequeue()
|
|
assert broker.lock_size() == Conf.BULK
|
|
for task in tasks:
|
|
assert task is not None
|
|
broker.acknowledge(task[0])
|
|
# test lock size
|
|
assert broker.lock_size() == 0
|
|
# test duplicate acknowledge
|
|
broker.acknowledge(task[0])
|
|
# delete queue
|
|
broker.enqueue("test")
|
|
broker.enqueue("test")
|
|
broker.delete_queue()
|
|
assert broker.queue_size() == 0
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_mongo(monkeypatch):
|
|
monkeypatch.setattr(Conf, "MONGO", {"host": MONGO_HOST, "port": 27017})
|
|
# check broker
|
|
broker = get_broker(list_key="mongo_test")
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
# clear before we start
|
|
broker.delete_queue()
|
|
# async_task
|
|
broker.enqueue("test")
|
|
assert broker.queue_size() == 1
|
|
# dequeue
|
|
task = broker.dequeue()[0]
|
|
assert task[1] == "test"
|
|
broker.acknowledge(task[0])
|
|
assert broker.queue_size() == 0
|
|
# Retry test
|
|
monkeypatch.setattr(Conf, "RETRY", 1)
|
|
broker.enqueue("test")
|
|
assert broker.queue_size() == 1
|
|
broker.dequeue()
|
|
assert broker.queue_size() == 0
|
|
sleep(1.5)
|
|
assert broker.queue_size() == 1
|
|
task = broker.dequeue()[0]
|
|
assert broker.queue_size() == 0
|
|
broker.acknowledge(task[0])
|
|
sleep(1.5)
|
|
assert broker.queue_size() == 0
|
|
# delete job
|
|
task_id = broker.enqueue("test")
|
|
broker.delete(task_id)
|
|
assert broker.dequeue() is None
|
|
# fail
|
|
task_id = broker.enqueue("test")
|
|
broker.fail(task_id)
|
|
# bulk test
|
|
for _ in range(5):
|
|
broker.enqueue("test")
|
|
tasks = [broker.dequeue()[0] for _ in range(5)]
|
|
assert broker.lock_size() == 5
|
|
for task in tasks:
|
|
assert task is not None
|
|
broker.acknowledge(task[0])
|
|
# test lock size
|
|
assert broker.lock_size() == 0
|
|
# test duplicate acknowledge
|
|
broker.acknowledge(task[0])
|
|
# delete queue
|
|
broker.enqueue("test")
|
|
broker.enqueue("test")
|
|
broker.purge_queue()
|
|
broker.delete_queue()
|
|
assert broker.queue_size() == 0
|