Moves bulk iteration to pusher

django_q/brokers/__init__.py

@@ -7,7 +7,6 @@ class Broker(object):
         self.connection = self.get_connection(list_key)
         self.list_key = list_key
         self.cache = self.get_cache()
-        self.task_cache = []
 
     def enqueue(self, task):
         """

django_q/brokers/aws_sqs.py

@@ -17,17 +17,9 @@ class Sqs(Broker):
         # sqs supports max 10 messages in bulk
         if Conf.BULK > 10:
             Conf.BULK = 10
-        t = None
-        if len(self.task_cache) > 0:
-            t = self.task_cache.pop()
-        else:
-            tasks = self.queue.receive_messages(MaxNumberOfMessages=Conf.BULK, VisibilityTimeout=Conf.RETRY)
-            if tasks:
-                t = tasks.pop()
-                if tasks:
-                    self.task_cache = tasks
-        if t:
-            return t.receipt_handle, t.body
+        tasks = self.queue.receive_messages(MaxNumberOfMessages=Conf.BULK, VisibilityTimeout=Conf.RETRY)
+        if tasks:
+            return [(t.receipt_handle, t.body) for t in tasks]
 
     def acknowledge(self, task_id):
         return self.delete(task_id)

django_q/brokers/disque.py

@@ -11,18 +11,10 @@ class Disque(Broker):
             'ADDJOB {} {} 500 RETRY {}'.format(self.list_key, task, retry)).decode()
 
     def dequeue(self):
-        t = None
-        if len(self.task_cache) > 0:
-            t = self.task_cache.pop()
-        else:
             tasks = self.connection.execute_command(
                 'GETJOB COUNT {} TIMEOUT 1000 FROM {}'.format(Conf.BULK, self.list_key))
             if tasks:
-                t = tasks.pop()
-                if tasks:
-                    self.task_cache = tasks
-        if t:
-            return t[1].decode(), t[2].decode()
+                return [(t[1].decode(), t[2].decode()) for t in tasks]
 
     def queue_size(self):
         return self.connection.execute_command('QLEN {}'.format(self.list_key))

django_q/brokers/ironmq.py

@@ -9,18 +9,10 @@ class IronMQBroker(Broker):
         return self.connection.post(task)['ids'][0]
 
     def dequeue(self):
-        t = None
-        if len(self.task_cache) > 0:
-            t = self.task_cache.pop()
-        else:
             timeout = Conf.RETRY or None
             tasks = self.connection.get(timeout=timeout, wait=1, max=Conf.BULK)['messages']
             if tasks:
-                t = tasks.pop()
-                if tasks:
-                    self.task_cache = tasks
-        if t:
-            return t['id'], t['body']
+                return [(t['id'], t['body']) for t in tasks]
 
     def ping(self):
         return self.connection.name == self.list_key

django_q/brokers/orm.py

@@ -31,24 +31,13 @@ class ORM(Broker):
         return package.pk
 
     def dequeue(self):
-        if len(self.task_cache) > 0:
-            t = self.task_cache.pop()
-            return t.pk, t.payload
-        else:
-            # Get new and timed out tasks
             tasks = OrmQ.objects.using(Conf.ORM).filter(
                 Q(key=self.list_key, lock__isnull=True) |
                 Q(key=self.list_key, lock__lte=timezone.now() - timedelta(seconds=Conf.RETRY)))[:Conf.BULK]
             if tasks:
                 # lock them
                 OrmQ.objects.using(Conf.ORM).filter(pk__in=tasks).update(lock=timezone.now())
-                tasks = [t for t in tasks]
-                # pop one task
-                t = tasks.pop()
-                if tasks:
-                    # add remainder to cache
-                    self.task_cache = [t for t in tasks]
-                return t.pk, t.payload
+                return [(t.pk, t.payload) for t in tasks]
             # empty queue, spare the cpu
             sleep(0.2)
 

django_q/brokers/redis_broker.py

@@ -19,7 +19,7 @@ class Redis(Broker):
     def dequeue(self):
         task = self.connection.blpop(self.list_key, 1)
         if task:
-            return None, task[1]
+            return [(None, task[1])]
 
     def queue_size(self):
         return self.connection.llen(self.list_key)