import os from time import sleep import pytest import redis from django_q.brokers import get_broker, Broker from django_q.conf import Conf from django_q.humanhash import uuid def test_broker(): broker = Broker() broker.enqueue('test') broker.dequeue() broker.queue_size() broker.lock_size() broker.purge_queue() broker.delete('id') broker.delete_queue() broker.acknowledge('test') broker.ping() broker.info() # stats assert broker.get_stat('test_1') is None broker.set_stat('test_1', 'test', 3) assert broker.get_stat('test_1') == 'test' assert broker.get_stats('test:*')[0] == 'test' # stats with no cache Conf.CACHE = 'not_configured' broker.cache = broker.get_cache() assert broker.get_stat('test_1') is None broker.set_stat('test_1', 'test', 3) assert broker.get_stat('test_1') is None assert broker.get_stats('test:*') is None Conf.CACHE = 'default' def test_redis(): Conf.DJANGO_REDIS = None broker = get_broker() assert broker.ping() is True assert broker.info() is not None Conf.REDIS = {'host': '127.0.0.1', 'port': 7799} broker = get_broker() with pytest.raises(Exception): broker.ping() Conf.REDIS = None Conf.DJANGO_REDIS = 'default' def test_custom(): Conf.BROKER_CLASS = 'brokers.redis_broker.Redis' broker = get_broker() assert broker.ping() is True assert broker.info() is not None assert broker.__class__.__name__ == 'Redis' Conf.BROKER_CLASS = None def test_disque(): Conf.DISQUE_NODES = ['127.0.0.1:7711'] # check broker broker = get_broker(list_key='disque_test') assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # enqueue broker.enqueue('test') assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test Conf.RETRY = 1 broker.enqueue('test') assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 sleep(1.5) assert broker.queue_size() == 1 task = broker.dequeue()[0] assert broker.queue_size() == 0 broker.acknowledge(task[0]) sleep(1.5) assert broker.queue_size() == 0 # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') Conf.BULK = 5 Conf.DISQUE_FASTACK = True tasks = broker.dequeue() for task in tasks: assert task is not None broker.acknowledge(task[0]) # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.delete_queue() assert broker.queue_size() == 0 # connection test Conf.DISQUE_NODES = ['127.0.0.1:7798', '127.0.0.1:7799'] with pytest.raises(redis.exceptions.ConnectionError): broker.get_connection() # back to django-redis Conf.DISQUE_NODES = None Conf.DISQUE_FASTACK = False @pytest.mark.skipif(not os.getenv('IRON_MQ_TOKEN'), reason="requires IronMQ credentials") def test_ironmq(): Conf.DISQUE_NODES = None Conf.SQS = None Conf.IRON_MQ = {'token': os.getenv('IRON_MQ_TOKEN'), 'project_id': os.getenv('IRON_MQ_PROJECT_ID')} # check broker broker = get_broker(list_key=uuid()[0]) assert broker.ping() is True assert broker.info() is not None # initialize the queue broker.enqueue('test') # clear before we start broker.purge_queue() assert broker.queue_size() == 0 # enqueue broker.enqueue('test') # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.dequeue() is None # Retry test # Conf.RETRY = 1 # broker.enqueue('test') # assert broker.dequeue() is not None # sleep(3) # assert broker.dequeue() is not None # task = broker.dequeue()[0] # assert len(task) > 0 # broker.acknowledge(task[0]) # sleep(3) # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') Conf.BULK = 5 tasks = broker.dequeue() for task in tasks: assert task is not None broker.acknowledge(task[0]) # duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.purge_queue() assert broker.dequeue() is None broker.delete_queue() # back to django-redis Conf.IRON_MQ = None Conf.DJANGO_REDIS = 'default' @pytest.mark.skipif(not os.getenv('AWS_ACCESS_KEY_ID'), reason="requires AWS credentials") def test_sqs(): Conf.IRON_MQ = None Conf.DISQUE_NODES = None Conf.SQS = {'aws_region': os.getenv('AWS_REGION'), 'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'), 'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY')} # check broker broker = get_broker(list_key=uuid()[0]) assert broker.ping() is True assert broker.info() is not None assert broker.queue_size() == 0 # enqueue broker.enqueue('test') # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.dequeue() is None # Retry test Conf.RETRY = 1 broker.enqueue('test') assert broker.dequeue() is not None sleep(2) task = broker.dequeue()[0] assert len(task) > 0 broker.acknowledge(task[0]) sleep(2) # delete job broker.enqueue('test') task_id = broker.dequeue()[0][0] broker.delete(task_id) assert broker.dequeue() is None # fail broker.enqueue('test') while task is None: task = broker.dequeue()[0] broker.fail(task[0]) # bulk test for i in range(10): broker.enqueue('test') Conf.BULK = 12 tasks = broker.dequeue() for task in tasks: assert task is not None broker.acknowledge(task[0]) # duplicate acknowledge broker.acknowledge(task[0]) assert broker.lock_size() == 0 # delete queue broker.enqueue('test') broker.purge_queue() broker.delete_queue() # back to django-redis Conf.SQS = None Conf.BULK = 1 Conf.DJANGO_REDIS = 'default' @pytest.mark.django_db def test_orm(): Conf.ORM = 'default' # check broker broker = get_broker(list_key='orm_test') assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # enqueue broker.enqueue('test') assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test Conf.RETRY = 1 broker.enqueue('test') assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 sleep(1.5) assert broker.queue_size() == 1 task = broker.dequeue()[0] assert broker.queue_size() == 0 broker.acknowledge(task[0]) sleep(1.5) assert broker.queue_size() == 0 # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') Conf.BULK = 5 tasks = broker.dequeue() assert broker.lock_size() == Conf.BULK for task in tasks: assert task is not None broker.acknowledge(task[0]) # test lock size assert broker.lock_size() == 0 # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.delete_queue() assert broker.queue_size() == 0 # back to django-redis Conf.ORM = None @pytest.mark.django_db def test_mongo(): Conf.MONGO = {'host': '127.0.0.1', 'port': 27017} # check broker broker = get_broker(list_key='mongo_test') assert broker.ping() is True assert broker.info() is not None # clear before we start broker.delete_queue() # enqueue broker.enqueue('test') assert broker.queue_size() == 1 # dequeue task = broker.dequeue()[0] assert task[1] == 'test' broker.acknowledge(task[0]) assert broker.queue_size() == 0 # Retry test Conf.RETRY = 1 broker.enqueue('test') assert broker.queue_size() == 1 broker.dequeue() assert broker.queue_size() == 0 sleep(1.5) assert broker.queue_size() == 1 task = broker.dequeue()[0] assert broker.queue_size() == 0 broker.acknowledge(task[0]) sleep(1.5) assert broker.queue_size() == 0 # delete job task_id = broker.enqueue('test') broker.delete(task_id) assert broker.dequeue() is None # fail task_id = broker.enqueue('test') broker.fail(task_id) # bulk test for i in range(5): broker.enqueue('test') tasks = [] for i in range(5): tasks.append(broker.dequeue()[0]) assert broker.lock_size() == 5 for task in tasks: assert task is not None broker.acknowledge(task[0]) # test lock size assert broker.lock_size() == 0 # test duplicate acknowledge broker.acknowledge(task[0]) # delete queue broker.enqueue('test') broker.enqueue('test') broker.purge_queue() broker.delete_queue() assert broker.queue_size() == 0 # back to django-redis Conf.ORM = None