mirror of
https://github.com/django-q2/django-q2.git
synced 2026-09-30 06:28:11 +08:00
Ilan Steemers
1d12107524
* all new tasks start as timeouts. This makes the queries a lot simpler. * In BULK mode, the bulk update for the lock was causing actual database locks. The lock updates are now iterated.
286 lines
7.8 KiB
Python
286 lines
7.8 KiB
Python
from time import sleep
|
|
import pytest
|
|
import os
|
|
import redis
|
|
from django_q.conf import Conf
|
|
from django_q.brokers import get_broker, Broker
|
|
from django_q.humanhash import uuid
|
|
|
|
|
|
def test_broker():
|
|
broker = Broker()
|
|
broker.enqueue('test')
|
|
broker.dequeue()
|
|
broker.queue_size()
|
|
broker.purge_queue()
|
|
broker.delete('id')
|
|
broker.delete_queue()
|
|
broker.acknowledge('test')
|
|
broker.ping()
|
|
broker.info()
|
|
# stats
|
|
assert broker.get_stat('test_1') is None
|
|
broker.set_stat('test_1', 'test', 3)
|
|
assert broker.get_stat('test_1') == 'test'
|
|
assert broker.get_stats('test:*')[0] == 'test'
|
|
# stats with no cache
|
|
Conf.CACHE = 'not_configured'
|
|
broker.cache = broker.get_cache()
|
|
assert broker.get_stat('test_1') is None
|
|
broker.set_stat('test_1', 'test', 3)
|
|
assert broker.get_stat('test_1') is None
|
|
assert broker.get_stats('test:*') is None
|
|
Conf.CACHE = 'default'
|
|
|
|
|
|
def test_redis():
|
|
Conf.DJANGO_REDIS = None
|
|
broker = get_broker()
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
Conf.REDIS = {'host': '127.0.0.1', 'port': 7799}
|
|
broker = get_broker()
|
|
with pytest.raises(Exception):
|
|
broker.ping()
|
|
Conf.REDIS = None
|
|
Conf.DJANGO_REDIS = 'default'
|
|
|
|
|
|
def test_disque():
|
|
Conf.DISQUE_NODES = ['127.0.0.1:7711']
|
|
# check broker
|
|
broker = get_broker(list_key='disque_test')
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
# clear before we start
|
|
broker.delete_queue()
|
|
# enqueue
|
|
broker.enqueue('test')
|
|
assert broker.queue_size() == 1
|
|
# dequeue
|
|
task = broker.dequeue()[0]
|
|
assert task[1] == 'test'
|
|
broker.acknowledge(task[0])
|
|
assert broker.queue_size() == 0
|
|
# Retry test
|
|
Conf.RETRY = 1
|
|
broker.enqueue('test')
|
|
assert broker.queue_size() == 1
|
|
broker.dequeue()
|
|
assert broker.queue_size() == 0
|
|
sleep(1.5)
|
|
assert broker.queue_size() == 1
|
|
task = broker.dequeue()[0]
|
|
assert broker.queue_size() == 0
|
|
broker.acknowledge(task[0])
|
|
sleep(1.5)
|
|
assert broker.queue_size() == 0
|
|
# delete job
|
|
task_id = broker.enqueue('test')
|
|
broker.delete(task_id)
|
|
assert broker.dequeue() is None
|
|
# fail
|
|
task_id = broker.enqueue('test')
|
|
broker.fail(task_id)
|
|
# bulk test
|
|
for i in range(5):
|
|
broker.enqueue('test')
|
|
Conf.BULK = 5
|
|
Conf.DISQUE_FASTACK = True
|
|
tasks = broker.dequeue()
|
|
for task in tasks:
|
|
assert task is not None
|
|
broker.acknowledge(task[0])
|
|
# test duplicate acknowledge
|
|
broker.acknowledge(task[0])
|
|
# delete queue
|
|
broker.enqueue('test')
|
|
broker.enqueue('test')
|
|
broker.delete_queue()
|
|
assert broker.queue_size() == 0
|
|
# connection test
|
|
Conf.DISQUE_NODES = ['127.0.0.1:7798', '127.0.0.1:7799']
|
|
with pytest.raises(redis.exceptions.ConnectionError):
|
|
broker.get_connection()
|
|
# back to django-redis
|
|
Conf.DISQUE_NODES = None
|
|
Conf.DISQUE_FASTACK = False
|
|
|
|
|
|
@pytest.mark.skipif(not os.getenv('IRON_MQ_TOKEN'),
|
|
reason="requires IronMQ credentials")
|
|
def test_ironmq():
|
|
Conf.DISQUE_NODES = None
|
|
Conf.SQS = None
|
|
Conf.IRON_MQ = {'token': os.getenv('IRON_MQ_TOKEN'),
|
|
'project_id': os.getenv('IRON_MQ_PROJECT_ID')}
|
|
# check broker
|
|
broker = get_broker(list_key=uuid()[0])
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
# initialize the queue
|
|
broker.enqueue('test')
|
|
# clear before we start
|
|
broker.purge_queue()
|
|
assert broker.queue_size() == 0
|
|
# enqueue
|
|
broker.enqueue('test')
|
|
# dequeue
|
|
task = broker.dequeue()[0]
|
|
assert task[1] == 'test'
|
|
broker.acknowledge(task[0])
|
|
assert broker.dequeue() is None
|
|
# Retry test
|
|
Conf.RETRY = 1
|
|
broker.enqueue('test')
|
|
assert broker.dequeue() is not None
|
|
sleep(1.5)
|
|
task = broker.dequeue()[0]
|
|
assert len(task) > 0
|
|
broker.acknowledge(task[0])
|
|
sleep(1.5)
|
|
# delete job
|
|
task_id = broker.enqueue('test')
|
|
broker.delete(task_id)
|
|
assert broker.dequeue() is None
|
|
# fail
|
|
task_id = broker.enqueue('test')
|
|
broker.fail(task_id)
|
|
# bulk test
|
|
for i in range(5):
|
|
broker.enqueue('test')
|
|
Conf.BULK = 5
|
|
tasks = broker.dequeue()
|
|
for task in tasks:
|
|
assert task is not None
|
|
broker.acknowledge(task[0])
|
|
# duplicate acknowledge
|
|
broker.acknowledge(task[0])
|
|
# delete queue
|
|
broker.enqueue('test')
|
|
broker.enqueue('test')
|
|
broker.purge_queue()
|
|
assert broker.dequeue() is None
|
|
broker.delete_queue()
|
|
# back to django-redis
|
|
Conf.IRON_MQ = None
|
|
Conf.DJANGO_REDIS = 'default'
|
|
|
|
|
|
@pytest.mark.skipif(not os.getenv('AWS_ACCESS_KEY_ID'),
|
|
reason="requires AWS credentials")
|
|
def test_sqs():
|
|
Conf.IRON_MQ = None
|
|
Conf.DISQUE_NODES = None
|
|
Conf.SQS = {'aws_region': os.getenv('AWS_REGION'),
|
|
'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
|
|
'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY')}
|
|
# check broker
|
|
broker = get_broker(list_key=uuid()[0])
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
assert broker.queue_size() == 0
|
|
# enqueue
|
|
broker.enqueue('test')
|
|
# dequeue
|
|
task = broker.dequeue()[0]
|
|
assert task[1] == 'test'
|
|
broker.acknowledge(task[0])
|
|
assert broker.dequeue() is None
|
|
# Retry test
|
|
Conf.RETRY = 1
|
|
broker.enqueue('test')
|
|
assert broker.dequeue() is not None
|
|
sleep(1.5)
|
|
task = broker.dequeue()[0]
|
|
assert len(task) > 0
|
|
broker.acknowledge(task[0])
|
|
sleep(1.5)
|
|
# delete job
|
|
broker.enqueue('test')
|
|
task_id = broker.dequeue()[0][0]
|
|
broker.delete(task_id)
|
|
assert broker.dequeue() is None
|
|
# fail
|
|
broker.enqueue('test')
|
|
while task is None:
|
|
task = broker.dequeue()[0]
|
|
broker.fail(task[0])
|
|
# bulk test
|
|
for i in range(10):
|
|
broker.enqueue('test')
|
|
Conf.BULK = 12
|
|
tasks = broker.dequeue()
|
|
for task in tasks:
|
|
assert task is not None
|
|
broker.acknowledge(task[0])
|
|
# duplicate acknowledge
|
|
broker.acknowledge(task[0])
|
|
# delete queue
|
|
broker.enqueue('test')
|
|
broker.purge_queue()
|
|
broker.delete_queue()
|
|
# back to django-redis
|
|
Conf.SQS = None
|
|
Conf.BULK = 1
|
|
Conf.DJANGO_REDIS = 'default'
|
|
|
|
|
|
@pytest.mark.django_db
|
|
def test_orm():
|
|
Conf.ORM = 'default'
|
|
# check broker
|
|
broker = get_broker(list_key='orm_test')
|
|
assert broker.ping() is True
|
|
assert broker.info() is not None
|
|
# clear before we start
|
|
broker.delete_queue()
|
|
# enqueue
|
|
broker.enqueue('test')
|
|
assert broker.queue_size() == 1
|
|
# dequeue
|
|
task = broker.dequeue()[0]
|
|
assert task[1] == 'test'
|
|
broker.acknowledge(task[0])
|
|
assert broker.queue_size() == 0
|
|
# Retry test
|
|
Conf.RETRY = 1
|
|
broker.enqueue('test')
|
|
assert broker.queue_size() == 1
|
|
broker.dequeue()
|
|
assert broker.queue_size() == 0
|
|
sleep(1.5)
|
|
assert broker.queue_size() == 1
|
|
task = broker.dequeue()[0]
|
|
assert broker.queue_size() == 0
|
|
broker.acknowledge(task[0])
|
|
sleep(1.5)
|
|
assert broker.queue_size() == 0
|
|
# delete job
|
|
task_id = broker.enqueue('test')
|
|
broker.delete(task_id)
|
|
assert broker.dequeue() is None
|
|
# fail
|
|
task_id = broker.enqueue('test')
|
|
broker.fail(task_id)
|
|
# bulk test
|
|
for i in range(5):
|
|
broker.enqueue('test')
|
|
Conf.BULK = 5
|
|
tasks = broker.dequeue()
|
|
assert broker.lock_size() == Conf.BULK
|
|
for task in tasks:
|
|
assert task is not None
|
|
broker.acknowledge(task[0])
|
|
# test lock size
|
|
assert broker.lock_size() == 0
|
|
# test duplicate acknowledge
|
|
broker.acknowledge(task[0])
|
|
# delete queue
|
|
broker.enqueue('test')
|
|
broker.enqueue('test')
|
|
broker.delete_queue()
|
|
assert broker.queue_size() == 0
|
|
# back to django-redis
|
|
Conf.ORM = None
|