From 8298f2319fb3cc0838143d8a982dfa16b433dcc0 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 27 Jan 2016 10:32:48 +0100 Subject: [PATCH 1/7] Fixes issue with rollbar configuration --- django_q/conf.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/django_q/conf.py b/django_q/conf.py index 58989d4..3ba9e23 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -1,4 +1,5 @@ import logging +from copy import deepcopy from signal import signal from multiprocessing import cpu_count, Queue @@ -50,6 +51,9 @@ class Conf(object): # ORM broker ORM = conf.get('orm', None) + # Custom broker + BROKER = conf.get('broker', None) + # Database Poll POLL = conf.get('poll', 0.2) @@ -169,7 +173,7 @@ if not logger.handlers: # rollbar if Conf.ROLLBAR: - rollbar_conf = Conf.ROLLBAR + rollbar_conf = deepcopy(Conf.ROLLBAR) try: import rollbar rollbar.init(rollbar_conf.pop('access_token'), environment=rollbar_conf.pop('environment'), **rollbar_conf) @@ -180,9 +184,6 @@ else: rollbar = None - - - # get parent pid compatibility def get_ppid(): if hasattr(os, 'getppid'): From fe8015403cea4ccd9115e1e0cd2462a3a446903c Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 27 Jan 2016 10:33:07 +0100 Subject: [PATCH 2/7] Adds custom broker configuration --- django_q/brokers/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/django_q/brokers/__init__.py b/django_q/brokers/__init__.py index 959d392..6609e5e 100644 --- a/django_q/brokers/__init__.py +++ b/django_q/brokers/__init__.py @@ -1,6 +1,9 @@ -from django_q.conf import Conf +import importlib + from django.core.cache import caches, InvalidCacheBackendError +from django_q.conf import Conf + class Broker(object): def __init__(self, list_key=Conf.PREFIX): @@ -171,6 +174,12 @@ def get_broker(list_key=Conf.PREFIX): elif Conf.MONGO: from brokers import mongo return mongo.Mongo(list_key=list_key) + elif Conf.BROKER: + module, func = Conf.BROKER.rsplit('.', 1) + m = importlib.import_module(module) + broker = getattr(m, func) + return broker(list_key=list) + # default to redis else: from brokers import redis_broker From d2d00e4f948b61b40a7149a470eaecb76c7b743f Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 27 Jan 2016 10:33:19 +0100 Subject: [PATCH 3/7] Adds test for custom broker configuration --- django_q/tests/test_brokers.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/django_q/tests/test_brokers.py b/django_q/tests/test_brokers.py index c0d6b93..a95e17f 100644 --- a/django_q/tests/test_brokers.py +++ b/django_q/tests/test_brokers.py @@ -1,9 +1,11 @@ -from time import sleep -import pytest import os +from time import sleep + +import pytest import redis -from django_q.conf import Conf + from django_q.brokers import get_broker, Broker +from django_q.conf import Conf from django_q.humanhash import uuid @@ -47,6 +49,15 @@ def test_redis(): Conf.DJANGO_REDIS = 'default' +def test_custom(): + Conf.BROKER = 'brokers.redis_broker.Redis' + broker = get_broker() + assert broker.ping() is True + assert broker.info() is not None + assert broker.__class__.__name == 'Redis' + Conf.BROKER = None + + def test_disque(): Conf.DISQUE_NODES = ['127.0.0.1:7711'] # check broker @@ -132,15 +143,15 @@ def test_ironmq(): broker.acknowledge(task[0]) assert broker.dequeue() is None # Retry test - #Conf.RETRY = 1 - #broker.enqueue('test') - #assert broker.dequeue() is not None - #sleep(3) + # Conf.RETRY = 1 + # broker.enqueue('test') # assert broker.dequeue() is not None - #task = broker.dequeue()[0] - #assert len(task) > 0 - #broker.acknowledge(task[0]) - #sleep(3) + # sleep(3) + # assert broker.dequeue() is not None + # task = broker.dequeue()[0] + # assert len(task) > 0 + # broker.acknowledge(task[0]) + # sleep(3) # delete job task_id = broker.enqueue('test') broker.delete(task_id) From b1b4a33c51638fbbe09f9dcf618b3bff485a3a11 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 27 Jan 2016 10:40:45 +0100 Subject: [PATCH 4/7] Fixes typo --- django_q/tests/test_brokers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/tests/test_brokers.py b/django_q/tests/test_brokers.py index a95e17f..6f0c43a 100644 --- a/django_q/tests/test_brokers.py +++ b/django_q/tests/test_brokers.py @@ -54,7 +54,7 @@ def test_custom(): broker = get_broker() assert broker.ping() is True assert broker.info() is not None - assert broker.__class__.__name == 'Redis' + assert broker.__class__.__name__ == 'Redis' Conf.BROKER = None From 3ac7f26de510052014f88bc05b33033f0fe3464e Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 27 Jan 2016 11:11:29 +0100 Subject: [PATCH 5/7] Moves custom broker class to top --- django_q/brokers/__init__.py | 40 ++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/django_q/brokers/__init__.py b/django_q/brokers/__init__.py index 6609e5e..c7c592b 100644 --- a/django_q/brokers/__init__.py +++ b/django_q/brokers/__init__.py @@ -158,28 +158,32 @@ def get_broker(list_key=Conf.PREFIX): :type list_key: str :return: """ - # disque - if Conf.DISQUE_NODES: - from brokers import disque - return disque.Disque(list_key=list_key) - elif Conf.IRON_MQ: - from brokers import ironmq - return ironmq.IronMQBroker(list_key=list_key) - elif Conf.SQS: - from brokers import aws_sqs - return aws_sqs.Sqs(list_key=list_key) - elif Conf.ORM: - from brokers import orm - return orm.ORM(list_key=list_key) - elif Conf.MONGO: - from brokers import mongo - return mongo.Mongo(list_key=list_key) - elif Conf.BROKER: + # custom + if Conf.BROKER: module, func = Conf.BROKER.rsplit('.', 1) m = importlib.import_module(module) broker = getattr(m, func) return broker(list_key=list) - + # disque + elif Conf.DISQUE_NODES: + from brokers import disque + return disque.Disque(list_key=list_key) + # Iron MQ + elif Conf.IRON_MQ: + from brokers import ironmq + return ironmq.IronMQBroker(list_key=list_key) + # SQS + elif Conf.SQS: + from brokers import aws_sqs + return aws_sqs.Sqs(list_key=list_key) + # ORM + elif Conf.ORM: + from brokers import orm + return orm.ORM(list_key=list_key) + # Mongo + elif Conf.MONGO: + from brokers import mongo + return mongo.Mongo(list_key=list_key) # default to redis else: from brokers import redis_broker From c17468ef8666e0537c75484d21201ce2ead06574 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 27 Jan 2016 12:06:49 +0100 Subject: [PATCH 6/7] docs: Adds Custom broker class documentation Also changes some of the variable and settings names, to make them less ambiguous. --- django_q/brokers/__init__.py | 4 ++-- django_q/conf.py | 4 ++-- django_q/tests/test_brokers.py | 4 ++-- docs/brokers.rst | 30 ++++++++++++++++++++++++++++++ docs/configure.rst | 17 +++++++++++++++++ 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/django_q/brokers/__init__.py b/django_q/brokers/__init__.py index c7c592b..32eb6fa 100644 --- a/django_q/brokers/__init__.py +++ b/django_q/brokers/__init__.py @@ -159,8 +159,8 @@ def get_broker(list_key=Conf.PREFIX): :return: """ # custom - if Conf.BROKER: - module, func = Conf.BROKER.rsplit('.', 1) + if Conf.BROKER_CLASS: + module, func = Conf.BROKER_CLASS.rsplit('.', 1) m = importlib.import_module(module) broker = getattr(m, func) return broker(list_key=list) diff --git a/django_q/conf.py b/django_q/conf.py index 3ba9e23..f5cb7df 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -51,8 +51,8 @@ class Conf(object): # ORM broker ORM = conf.get('orm', None) - # Custom broker - BROKER = conf.get('broker', None) + # Custom broker class + BROKER_CLASS = conf.get('broker_class', None) # Database Poll POLL = conf.get('poll', 0.2) diff --git a/django_q/tests/test_brokers.py b/django_q/tests/test_brokers.py index 6f0c43a..8c63a98 100644 --- a/django_q/tests/test_brokers.py +++ b/django_q/tests/test_brokers.py @@ -50,12 +50,12 @@ def test_redis(): def test_custom(): - Conf.BROKER = 'brokers.redis_broker.Redis' + Conf.BROKER_CLASS = 'brokers.redis_broker.Redis' broker = get_broker() assert broker.ping() is True assert broker.info() is not None assert broker.__class__.__name__ == 'Redis' - Conf.BROKER = None + Conf.BROKER_CLASS = None def test_disque(): diff --git a/docs/brokers.rst b/docs/brokers.rst index 60604a2..7345e98 100644 --- a/docs/brokers.rst +++ b/docs/brokers.rst @@ -103,6 +103,36 @@ However for a medium message rate and scheduled tasks, this is the most convenie * Queue editable in Django Admin * See the :ref:`orm_configuration` configuration on how to set it up. + + +Custom Broker +------------- +You can override the :class:`Broker` or any of its existing derived broker types. + +.. code-block:: python + + # example Custom broker.py + from django_q.brokers import Broker + + class CustomBroker(Broker): + def info(self): + return 'My Custom Broker' + +Using the :ref:`broker_class` configuration setting you can then instruct Django Q to use this instead of one of the existing brokers: + +.. code-block:: python + + # example Custom broker class connection + + Q_CLUSTER = { + 'name': 'Custom', + 'workers': 8, + 'timeout': 60, + 'broker_class: 'myapp.broker.CustomBroker' + } + +If you do write a custom broker for one of the many message queueing servers out there we don't support yet, please consider contributing it to the project. + Reference --------- The :class:`Broker` class is used internally to communicate with the different types of brokers. diff --git a/docs/configure.rst b/docs/configure.rst index 56453e5..fbe5601 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -292,6 +292,23 @@ mongo_db When using the MongoDB broker you can optionally provide a database name to use for the queues. Defaults to default database if available, otherwise ``django-q`` +.. _custom_broker: + +broker_class +~~~~~~~~~~~~ +You can use a custom broker class for your cluster workers:: + + # example Custom broker class connection + + Q_CLUSTER = { + 'name': 'Custom', + 'workers': 8, + 'timeout': 60, + 'broker_class: 'myapp.broker.CustomBroker' + } + +Make sure your ``CustomBroker`` class inherits from either the base :class:`Broker` class or one of its children. + .. _bulk: bulk From e1c05732f9451cf54ca33eb97b0d025571a98fea Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Wed, 27 Jan 2016 12:14:46 +0100 Subject: [PATCH 7/7] docs: fixes broken ref --- docs/configure.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configure.rst b/docs/configure.rst index fbe5601..5e063c5 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -292,7 +292,7 @@ mongo_db When using the MongoDB broker you can optionally provide a database name to use for the queues. Defaults to default database if available, otherwise ``django-q`` -.. _custom_broker: +.. _broker_class: broker_class ~~~~~~~~~~~~