removed SQS and IronMQ brokers for now

Both brokers are not atomic and cause all kinds of problems during testing. Needs more time.
Meanwhile I want to release the pluggable broker backend.
This commit is contained in:
Ilan Steemers
2015-09-04 14:49:17 +02:00
parent 1ccfc66254
commit 3f75e7fa05
12 changed files with 36 additions and 239 deletions
+2 -7
View File
@@ -149,15 +149,10 @@ def get_broker(list_key=Conf.PREFIX):
:type list_key: str
:return:
"""
if Conf.IRONMQ:
from brokers import iron_mq
return iron_mq.IronMQBroker(list_key=list_key)
elif Conf.DISQUE_NODES:
# disque
if Conf.DISQUE_NODES:
from brokers import disque
return disque.Disque(list_key=list_key)
elif Conf.SQS:
from brokers import aws_sqs
return aws_sqs.Sqs(list_key=list_key)
# default to redis
else:
from brokers import redis_broker
-63
View File
@@ -1,63 +0,0 @@
import os
from django_q.conf import Conf
from django_q.brokers import Broker
import boto.sqs
from boto.sqs.message import Message
class Sqs(Broker):
def __init__(self, list_key=Conf.PREFIX):
super(Sqs, self).__init__(list_key)
self.queue = self.get_queue()
def enqueue(self, task):
m = Message()
m.set_body(task)
self.queue.write(m)
return m.id
def dequeue(self):
rs = self.queue.get_messages(visibility_timeout=Conf.RETRY or 30)
if rs:
m = rs[0]
return m.receipt_handle, m.get_body()
def acknowledge(self, task_id):
return self.delete(task_id)
def queue_size(self):
return self.queue.count()
def delete(self, task_id):
m = Message()
m.receipt_handle = task_id
return self.queue.delete_message(m)
def fail(self, task_id):
self.delete(task_id)
def delete_queue(self):
self.connection.delete_queue(self.queue)
def purge_queue(self):
self.queue.purge()
def ping(self):
try:
self.connection.get_all_queues()
return True
except Exception as e:
raise e
def info(self):
return 'AWS SQS'
@staticmethod
def get_connection(list_key=Conf.PREFIX):
conn = boto.sqs.connect_to_region(Conf.SQS['region'],
aws_access_key_id=Conf.SQS['aws_access_key_id'],
aws_secret_access_key=Conf.SQS['aws_secret_access_key'])
return conn
def get_queue(self):
return self.connection.create_queue(self.list_key)
+3 -1
View File
@@ -33,7 +33,9 @@ class Disque(Broker):
def delete_queue(self):
jobs = self.connection.execute_command('JSCAN QUEUE {}'.format(self.list_key))[1]
if jobs:
self.connection.execute_command('DELJOB {}'.format(' '.join(map(str, jobs))))
job_ids = ' '.join(jid.decode() for jid in jobs)
self.connection.execute_command('DELJOB {}'.format(job_ids))
return len(jobs)
def info(self):
info = self.connection.info('server')
-44
View File
@@ -1,44 +0,0 @@
from django_q.conf import Conf
from django_q.brokers import Broker
from iron_mq import IronMQ
class IronMQBroker(Broker):
def enqueue(self, task):
return self.connection.post(task)['ids'][0]
def dequeue(self):
timeout = Conf.RETRY or None
task = self.connection.get(timeout=timeout, wait=1)['messages']
if task:
return task[0]['id'], task[0]['body']
def ping(self):
return self.connection.name == self.list_key
def info(self):
return 'IronMQ'
def queue_size(self):
return self.connection.size()
def delete_queue(self):
return self.connection.delete_queue()['msg']
def purge_queue(self):
return self.connection.clear()['msg']
def delete(self, task_id):
return self.connection.delete(task_id)['msg']
def fail(self, task_id):
self.delete(task_id)
def acknowledge(self, task_id):
return self.delete(task_id)
@staticmethod
def get_connection(list_key=Conf.PREFIX):
ironmq = IronMQ(name=None, **Conf.IRONMQ)
return ironmq.queue(queue_name=list_key)
+3
View File
@@ -27,6 +27,9 @@ class Redis(Broker):
def delete_queue(self):
return self.connection.delete(self.list_key)
def purge_queue(self):
return self.connection.ltrim(self.list_key, 1, 0)
def ping(self):
try:
return self.connection.ping()