fork/django-q2
1
0
Fork 0
mirror of https://github.com/django-q2/django-q2.git synced 2026-10-08 14:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Adds async_iter command

Browse Source 
With async iter you can quickly run the same function on an iterable set of arguments. The results are held in the cache until all are done and collated into a database result.
This commit is contained in:
Ilan Steemers
2015-10-03 18:07:57 +02:00
parent 9243731a9c
commit f73ae66c0f
2 changed files with 68 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
django_q/cluster.py
+28 -11
View File
@@ -326,7 +326,7 @@ def monitor(result_queue, broker=None):
broker.acknowledge(ack_id)
# save the result
if task.get('cached', False):
save_cache(task, broker)
save_cached(task, broker)
else:
save_task(task)
# log the result
@@ -413,22 +413,39 @@ def save_task(task):
logger.error(e)
def save_cache(task, broker):
key = 'django_q:{}:results'.format(broker.list_key)
def save_cached(task, broker):
task_key = '{}:{}'.format(broker.list_key, task['id'])
timeout = task['cached']
if timeout is True:
timeout = None
try:
task_package = signing.SignedPackage.dumps(task)
group = task.get('group', False)
iter_count = task.get('iter_count', None)
# if it's a group append to the group list
if group:
group_list = broker.cache.get('{}:{}'.format(key, group)) or []
group_list.append(task_package)
broker.cache.set('{}:{}'.format(key, group), group_list, timeout)
else:
broker.cache.set('{}:{}'.format(key, task['id']),
task_package,
timeout)
task_key = '{}:{}:{}'.format(broker.list_key, group, task['id'])
group_key = '{}:{}:keys'.format(broker.list_key, group)
group_list = broker.cache.get(group_key) or []
# if it's an inter group, check if we are ready
if iter_count and len(group_list) == iter_count-1:
group_args = '{}:{}:args'.format(broker.list_key, group)
# collate the results into a Task result
results = [signing.SignedPackage.loads(broker.cache.get(k))['result'] for k in group_list]
results.append(task['result'])
task['result'] = results
task['id'] = group
task['args'] = signing.SignedPackage.loads(broker.cache.get(group_args))
save_task(task)
broker.cache.delete_many(group_list)
broker.cache.delete_many([group_key, group_args])
return
# save the group list
group_list.append(task_key)
broker.cache.set(group_key, group_list)
# save the task
broker.cache.set(task_key,
signing.SignedPackage.dumps(task),
timeout)
except Exception as e:
logger.error(e)