From 72f913ab97cf24f010fb75235b6c8a709419998c Mon Sep 17 00:00:00 2001 From: Ilan Date: Sun, 19 Jul 2015 14:29:40 +0200 Subject: [PATCH 1/3] switching hook to post_save This might delay a hook slightly, but it makes more sense if you want to perform hook actions on groups since Pre_save the latest member of the group won't be available yet. --- django_q/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_q/models.py b/django_q/models.py index a8eb818..b9fa003 100644 --- a/django_q/models.py +++ b/django_q/models.py @@ -4,7 +4,7 @@ import importlib from django.core.urlresolvers import reverse from django.utils.translation import ugettext_lazy as _ from django.db import models -from django.db.models.signals import pre_save +from django.db.models.signals import post_save from django.dispatch import receiver from django.utils import timezone from picklefield import PickledObjectField @@ -72,7 +72,7 @@ class Task(models.Model): ordering = ['-stopped'] -@receiver(pre_save, sender=Task) +@receiver(post_save, sender=Task) def call_hook(sender, instance, **kwargs): if instance.hook: logger = logging.getLogger('django-q') From 52e72fde1cca0bffb0243d3fe47d4c8614de6783 Mon Sep 17 00:00:00 2001 From: Ilan Date: Sun, 19 Jul 2015 15:15:57 +0200 Subject: [PATCH 2/3] Adds failed tasks filtering to groups --- django_q/models.py | 13 +++++++--- django_q/tasks.py | 10 ++++--- docs/tasks.rst | 65 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 69 insertions(+), 19 deletions(-) diff --git a/django_q/models.py b/django_q/models.py index b9fa003..c1594c0 100644 --- a/django_q/models.py +++ b/django_q/models.py @@ -32,9 +32,12 @@ class Task(models.Model): return Task.objects.get(name=task_id).result @staticmethod - def get_result_group(group_id): + def get_result_group(group_id, failures=False): # values + decode is 10 times faster than just list comprehension - values = Task.objects.filter(group=group_id).values_list('result', flat=True) + if failures: + values = Task.objects.filter(group=group_id).values_list('result', flat=True) + else: + values = Task.objects.filter(group=group_id).exclude(success=False).values_list('result', flat=True) return [dbsafe_decode(t) for t in values] @staticmethod @@ -58,8 +61,10 @@ class Task(models.Model): return Task.objects.get(name=task_id) @staticmethod - def get_task_group(group_id): - return Task.objects.filter(group=group_id) + def get_task_group(group_id, failures=True): + if failures: + return Task.objects.filter(group=group_id) + return Task.objects.filter(group_id=group_id).exclude(success=False) def time_taken(self): return (self.stopped - self.started).total_seconds() diff --git a/django_q/tasks.py b/django_q/tasks.py index 525ee38..74dd52b 100644 --- a/django_q/tasks.py +++ b/django_q/tasks.py @@ -93,13 +93,14 @@ def result(task_id): return Task.get_result(task_id) -def result_group(group_id): +def result_group(group_id, failures=False): """ returns a list of results for a task group :param str group_id: the group id + :param bool failures: set to True to include failures :return: list or results """ - return Task.get_result_group(group_id) + return Task.get_result_group(group_id, failures) def fetch(task_id): @@ -113,14 +114,15 @@ def fetch(task_id): return Task.get_task(task_id) -def fetch_group(group_id): +def fetch_group(group_id, failures=True): """ Returns a list of Tasks for a task group :param str group_id: the group id + :param bool failures: set to False to exclude failures :return: list of Tasks """ - return Task.get_task_group(group_id) + return Task.get_task_group(group_id, failures) def count_group(group_id, failures=False): diff --git a/docs/tasks.rst b/docs/tasks.rst index 656b2fc..fbdc041 100644 --- a/docs/tasks.rst +++ b/docs/tasks.rst @@ -51,25 +51,36 @@ You can group together results by passing :func:`async` the optional `group` key [(0.0, 0.0), (0.0, 1.0), (0.0, 2.0), (0.0, 3.0)] -Take care to not limit your results database too much and that the group identifier is unique for each run. +Take care to not limit your results database too much and call :func:`delete_group` before each run, unless you want your results to keep adding up. Instead of :func:`result_group` you can also use :func:`fetch_group` to return a queryset of :class:`Task` objects.: .. code-block:: python # fetch group example - from django_q import fetch_group + from django_q import fetch_group, count_group, result_group # count the number of failures - failure_count = fetch_group('modf').filter(success=False).count() + failure_count = count_group('modf', failures=True) - # or print only the successful results - successes = fetch_group('modf').exclude(success=False) + # only use the successes + results = fetch_group('modf') + if failure_count: + results.exclude(success=False) results = [task.result for task in successes] - print(results) + + # this is the same as + results = fetch_group('modf', failures=False) + results = [task.result for task in successes] + + # and the same as + results = result_group('modf') # filters failures by default + + +Getting results by using :func:`result_group` is of course much faster than using :func:`fetch_group`, but it doesn't offer the benefits of Django's queryset functions. .. note:: - Although :func:`fetch_group` returns a queryset, due to the nature of the PickleField , `Queryset.values` will return a list of encoded results. + Although :func:`fetch_group` returns a queryset, due to the nature of the PickleField , calling `Queryset.values` on it will return a list of encoded results. Use list comprehension or an iterator instead. Synchronous testing @@ -154,22 +165,42 @@ Reference Renamed from get_task -.. py:function:: result_group(group_id) +.. py:function:: result_group(group_id, failures=False) Returns the results of a task group :param str group_id: the group identifier + :param bool failures: set this to `True` to include failed results :returns: a list of results :rtype: list -.. py:function:: fetch_group(group_id) +.. py:function:: fetch_group(group_id, failures=True) Returns a list of tasks in a group :param str group_id: the group identifier + :param bool failures: set this to `False` to exclude failed tasks :returns: a list of Tasks :rtype: list +.. py:function:: count_group(group_id, failures=False) + + Counts the number of task results in a group. + + :param str group_id: the group identifier + :param bool failures: counts the number of failures if `True` + :returns: the number of tasks or failures in a group + :rtype: int + +.. py:function:: delete_group(group_id, tasks=False) + + Deletes a group label from the database. + + :param str group_id: the group identifier + :param bool tasks: also deletes the associated tasks if `True` + :returns: the numbers of tasks affected + :rtype: int + .. py:class:: Task Database model describing an executed task @@ -233,17 +264,29 @@ Reference Gets a result directly by task uuid or name. - .. py:classmethod:: get_result_group(group_id) + .. py:classmethod:: get_result_group(group_id, failures=False) Returns a list of results from a task group. + Set failures to `True` to include failed results. .. py:classmethod:: get_task(task_id) Fetches a single task object by uuid or name. - .. py:classmethod:: get_task_group(group_id) + .. py:classmethod:: get_task_group(group_id, failures=True) Gets a queryset of tasks with this group id. + Set failures to `False` to exclude failed tasks. + + .. py:classmethod:: get_group_count(group_id, failures=False) + + Returns a count of the number of tasks results in a group. + Returns the number of failures when `failures=True` + + .. py:classmethod:: delete_group(group_id, objects=False) + + Deletes a group label only, by default. + If `objects=True` it will also delete the tasks in this group from the database. .. py:class:: Success From d7ecaee18de4028200e99a6218f01ee73ab826e3 Mon Sep 17 00:00:00 2001 From: Ilan Date: Sun, 19 Jul 2015 15:21:06 +0200 Subject: [PATCH 3/3] Added tests for failures filtering fixed a bug --- django_q/models.py | 2 +- django_q/tests/test_cluster.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/django_q/models.py b/django_q/models.py index c1594c0..43fbff3 100644 --- a/django_q/models.py +++ b/django_q/models.py @@ -64,7 +64,7 @@ class Task(models.Model): def get_task_group(group_id, failures=True): if failures: return Task.objects.filter(group=group_id) - return Task.objects.filter(group_id=group_id).exclude(success=False) + return Task.objects.filter(group=group_id).exclude(success=False) def time_taken(self): return (self.stopped - self.started).total_seconds() diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py index 1245292..2e347fa 100644 --- a/django_q/tests/test_cluster.py +++ b/django_q/tests/test_cluster.py @@ -202,7 +202,9 @@ def test_async(r, admin_user): assert result(result_j.name) == result_j.result # groups assert result_group('test_j') == [result_j.result] + assert result_group('test_j', failures=True) == [result_j.result] assert fetch_group('test_j')[0].id == [result_j][0].id + assert fetch_group('test_j', failures=False)[0].id == [result_j][0].id assert count_group('test_j') == 1 assert count_group('test_j', failures=True) == 0 assert delete_group('test_j') == 1