Merge pull request #24 from Koed00/dev

Adds failure filtering to group functions

django_q/models.py

@@ -4,7 +4,7 @@ import importlib
 from django.core.urlresolvers import reverse
 from django.utils.translation import ugettext_lazy as _
 from django.db import models
-from django.db.models.signals import pre_save
+from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.utils import timezone
 from picklefield import PickledObjectField
@@ -32,9 +32,12 @@ class Task(models.Model):
             return Task.objects.get(name=task_id).result
 
     @staticmethod
-    def get_result_group(group_id):
+    def get_result_group(group_id, failures=False):
         # values + decode is 10 times faster than just list comprehension
-        values = Task.objects.filter(group=group_id).values_list('result', flat=True)
+        if failures:
+            values = Task.objects.filter(group=group_id).values_list('result', flat=True)
+        else:
+            values = Task.objects.filter(group=group_id).exclude(success=False).values_list('result', flat=True)
         return [dbsafe_decode(t) for t in values]
 
     @staticmethod
@@ -58,8 +61,10 @@ class Task(models.Model):
             return Task.objects.get(name=task_id)
 
     @staticmethod
-    def get_task_group(group_id):
-        return Task.objects.filter(group=group_id)
+    def get_task_group(group_id, failures=True):
+        if failures:
+            return Task.objects.filter(group=group_id)
+        return Task.objects.filter(group=group_id).exclude(success=False)
 
     def time_taken(self):
         return (self.stopped - self.started).total_seconds()
@@ -72,7 +77,7 @@ class Task(models.Model):
         ordering = ['-stopped']
 
 
-@receiver(pre_save, sender=Task)
+@receiver(post_save, sender=Task)
 def call_hook(sender, instance, **kwargs):
     if instance.hook:
         logger = logging.getLogger('django-q')

django_q/tasks.py

@@ -93,13 +93,14 @@ def result(task_id):
     return Task.get_result(task_id)
 
 
-def result_group(group_id):
+def result_group(group_id, failures=False):
     """
     returns a list of results for a task group
     :param str group_id: the group id
+    :param bool failures: set to True to include failures
     :return: list or results
     """
-    return Task.get_result_group(group_id)
+    return Task.get_result_group(group_id, failures)
 
 
 def fetch(task_id):
@@ -113,14 +114,15 @@ def fetch(task_id):
     return Task.get_task(task_id)
 
 
-def fetch_group(group_id):
+def fetch_group(group_id, failures=True):
     """
     Returns a list of Tasks for a task group
     :param str group_id: the group id
+    :param bool failures: set to False to exclude failures
     :return: list of Tasks
     """
 
-    return Task.get_task_group(group_id)
+    return Task.get_task_group(group_id, failures)
 
 
 def count_group(group_id, failures=False):

django_q/tests/test_cluster.py

@@ -202,7 +202,9 @@ def test_async(r, admin_user):
     assert result(result_j.name) == result_j.result
     # groups
     assert result_group('test_j') == [result_j.result]
+    assert result_group('test_j', failures=True) == [result_j.result]
     assert fetch_group('test_j')[0].id == [result_j][0].id
+    assert fetch_group('test_j', failures=False)[0].id == [result_j][0].id
     assert count_group('test_j') == 1
     assert count_group('test_j', failures=True) == 0
     assert delete_group('test_j') == 1

docs/tasks.rst

@@ -51,25 +51,36 @@ You can group together results by passing :func:`async` the optional `group` key
 
     [(0.0, 0.0), (0.0, 1.0), (0.0, 2.0), (0.0, 3.0)]
 
-Take care to not limit your results database too much and that the group identifier is unique for each run.
+Take care to not limit your results database too much and call :func:`delete_group` before each run, unless you want your results to keep adding up.
 Instead of :func:`result_group` you can also use :func:`fetch_group` to return a queryset of :class:`Task` objects.:
 
 .. code-block:: python
 
     # fetch group example
-    from django_q import fetch_group
+    from django_q import fetch_group, count_group, result_group
 
     # count the number of failures
-    failure_count = fetch_group('modf').filter(success=False).count()
+    failure_count = count_group('modf', failures=True)
 
-    # or print only the successful results
-    successes = fetch_group('modf').exclude(success=False)
+    # only use the successes
+    results = fetch_group('modf')
+    if failure_count:
+        results.exclude(success=False)
     results =  [task.result for task in successes]
-    print(results)
+
+    # this is the same as
+    results = fetch_group('modf', failures=False)
+    results =  [task.result for task in successes]
+
+    # and the same as
+    results = result_group('modf') # filters failures by default
+
+
+Getting results by using :func:`result_group` is of course much faster than using :func:`fetch_group`, but it doesn't offer the benefits of Django's queryset functions.
 
 .. note::
 
-   Although :func:`fetch_group` returns a queryset, due to the nature of the PickleField , `Queryset.values` will return a list of encoded results.
+   Although :func:`fetch_group` returns a queryset, due to the nature of the PickleField , calling `Queryset.values` on it will return a list of encoded results.
    Use list comprehension or an iterator instead.
 
 Synchronous testing
@@ -154,22 +165,42 @@ Reference
 
     Renamed from get_task
 
-.. py:function:: result_group(group_id)
+.. py:function:: result_group(group_id, failures=False)
 
     Returns the results of a task group
 
     :param str group_id: the group identifier
+    :param bool failures: set this to `True` to include failed results
     :returns: a list of results
     :rtype: list
 
-.. py:function:: fetch_group(group_id)
+.. py:function:: fetch_group(group_id, failures=True)
 
     Returns a list of tasks in a group
 
     :param str group_id: the group identifier
+    :param bool failures: set this to `False` to exclude failed tasks
     :returns: a list of Tasks
     :rtype: list
 
+.. py:function:: count_group(group_id, failures=False)
+
+    Counts the number of task results in a group.
+
+    :param str group_id: the group identifier
+    :param bool failures: counts the number of failures if `True`
+    :returns: the number of tasks or failures in a group
+    :rtype: int
+
+.. py:function:: delete_group(group_id, tasks=False)
+
+    Deletes a group label from the database.
+
+    :param str group_id: the group identifier
+    :param bool tasks: also deletes the associated tasks if `True`
+    :returns: the numbers of tasks affected
+    :rtype: int
+
 .. py:class:: Task
 
     Database model describing an executed task
@@ -233,17 +264,29 @@ Reference
 
     Gets a result directly by task uuid or name.
 
-    .. py:classmethod:: get_result_group(group_id)
+    .. py:classmethod:: get_result_group(group_id, failures=False)
 
     Returns a list of results from a task group.
+    Set failures to `True` to include failed results.
 
     .. py:classmethod:: get_task(task_id)
 
     Fetches a single task object by uuid or name.
 
-    .. py:classmethod:: get_task_group(group_id)
+    .. py:classmethod:: get_task_group(group_id, failures=True)
 
     Gets a queryset of tasks with this group id.
+    Set failures to `False` to exclude failed tasks.
+
+    .. py:classmethod::  get_group_count(group_id, failures=False)
+
+    Returns a count of the number of tasks results in a group.
+    Returns the number of failures when `failures=True`
+
+    .. py:classmethod:: delete_group(group_id, objects=False)
+
+    Deletes a group label only, by default.
+    If `objects=True` it will also delete the tasks in this group from the database.
 
 .. py:class:: Success