From 3c3395f24b0687699d50accfece774587d7a1894 Mon Sep 17 00:00:00 2001 From: Tim O'Meara Date: Tue, 11 Aug 2020 10:16:31 -0500 Subject: [PATCH 1/7] add attempt_count to task --- django_q/cluster.py | 5 +++++ django_q/conf.py | 3 +++ django_q/models.py | 1 + 3 files changed, 9 insertions(+) diff --git a/django_q/cluster.py b/django_q/cluster.py index 027b6a5..c351ee5 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -478,7 +478,12 @@ def save_task(task, broker: Broker): existing_task.stopped = task["stopped"] existing_task.result = task["result"] existing_task.success = task["success"] + existing_task.attempt_count = existing_task.attempt_count + 1 existing_task.save() + + if 0 < Conf.ATTEMPT_COUNT == existing_task.attempt_count: + broker.acknowledge(task['ack_id']) + else: Task.objects.create( id=task["id"], diff --git a/django_q/conf.py b/django_q/conf.py index 2e9ddc2..964364f 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -164,6 +164,9 @@ class Conf: # Optional error reporting setup ERROR_REPORTER = conf.get("error_reporter", {}) + # Optional attempt count. set to 0 for infinite attempts + ATTEMPT_COUNT = conf.get('attempt_count', 0) + # OSX doesn't implement qsize because of missing sem_getvalue() try: QSIZE = Queue().qsize() == 0 diff --git a/django_q/models.py b/django_q/models.py index 7037f12..7a05693 100644 --- a/django_q/models.py +++ b/django_q/models.py @@ -29,6 +29,7 @@ class Task(models.Model): started = models.DateTimeField(editable=False) stopped = models.DateTimeField(editable=False) success = models.BooleanField(default=True, editable=False) + attempt_count = models.IntegerField(default=0) @staticmethod def get_result(task_id): From 45b0b2cc275ab277b238a5a2cc9c74d81d1efd05 Mon Sep 17 00:00:00 2001 From: Tim O'Meara Date: Tue, 11 Aug 2020 10:42:28 -0500 Subject: [PATCH 2/7] add attempt_count to config docs, set value to 1 in inital task save --- django_q/cluster.py | 1 + django_q/migrations/0013_task_attempt_count.py | 18 ++++++++++++++++++ docs/configure.rst | 9 +++++++++ 3 files changed, 28 insertions(+) create mode 100644 django_q/migrations/0013_task_attempt_count.py diff --git a/django_q/cluster.py b/django_q/cluster.py index c351ee5..d8e6f27 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -497,6 +497,7 @@ def save_task(task, broker: Broker): result=task["result"], group=task.get("group"), success=task["success"], + attempt_count=1 ) except Exception as e: logger.error(e) diff --git a/django_q/migrations/0013_task_attempt_count.py b/django_q/migrations/0013_task_attempt_count.py new file mode 100644 index 0000000..30d03be --- /dev/null +++ b/django_q/migrations/0013_task_attempt_count.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.7 on 2020-08-11 15:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_q', '0012_auto_20200702_1608'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='attempt_count', + field=models.IntegerField(default=0), + ), + ] diff --git a/docs/configure.rst b/docs/configure.rst index 16801b1..afc6aa0 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -75,6 +75,15 @@ ack_failures When set to ``True``, also acknowledge unsuccessful tasks. This causes failed tasks to be considered as successful deliveries, thereby removing them from the task queue. Can also be set per-task by passing the ``ack_failure`` option to :func:`async_task`. Defaults to ``False``. + +.. attempt_count: + +attempt_count +~~~~~~~~~~~~~ + +Limit the number of retries for failed tasks. Set to 0 for infinite retries. Defaults to 0 + + .. _retry: retry From d5a0363af3183af236d5dc9a402d3f7e78a02d63 Mon Sep 17 00:00:00 2001 From: Tim O'Meara Date: Tue, 11 Aug 2020 10:48:46 -0500 Subject: [PATCH 3/7] add attempt_count to admin --- django_q/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/admin.py b/django_q/admin.py index a7484ce..f40118e 100644 --- a/django_q/admin.py +++ b/django_q/admin.py @@ -43,7 +43,7 @@ retry_failed.short_description = _("Resubmit selected tasks to queue") class FailAdmin(admin.ModelAdmin): """model admin for failed tasks.""" - list_display = ("name", "func", "started", "stopped", "short_result") + list_display = ("name", "func", "started", "stopped", "short_result", "attempt_count") def has_add_permission(self, request): """Don't allow adds.""" From f26de929d60964e9cd943d17496985c70c2fa90d Mon Sep 17 00:00:00 2001 From: Tim O'Meara Date: Tue, 11 Aug 2020 11:48:23 -0500 Subject: [PATCH 4/7] adds tests --- django_q/tests/test_admin.py | 6 ++++-- django_q/tests/test_cluster.py | 35 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/django_q/tests/test_admin.py b/django_q/tests/test_admin.py index 52e8d41..5dc8880 100644 --- a/django_q/tests/test_admin.py +++ b/django_q/tests/test_admin.py @@ -21,7 +21,8 @@ def test_admin_views(admin_client, monkeypatch): func='test.fail', started=timezone.now(), stopped=timezone.now(), - success=False) + success=False, + attempt_count=1) tag = uuid() t = Task.objects.create( id=tag[1], @@ -29,7 +30,8 @@ def test_admin_views(admin_client, monkeypatch): func='test.success', started=timezone.now(), stopped=timezone.now(), - success=True) + success=True, + attempt_count=1) q = OrmQ.objects.create( key='test', payload=SignedPackage.dumps({'id': 1, 'func': 'test', 'name': 'test'})) diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py index 1b6661f..efb7089 100644 --- a/django_q/tests/test_cluster.py +++ b/django_q/tests/test_cluster.py @@ -397,6 +397,41 @@ def test_bad_secret(broker, monkeypatch): broker.delete_queue() +@pytest.mark.django_db +def test_attempt_count(broker, monkeypatch): + monkeypatch.setattr(Conf, 'ATTEMPT_COUNT', 3) + tag = uuid() + task = {'id': tag[1], + 'name': tag[0], + 'func': 'math.copysign', + 'args': (1, -1), + 'kwargs': {}, + 'started': timezone.now(), + 'stopped': timezone.now(), + 'success': False, + 'result': None} + # initial save - no success + save_task(task, broker) + assert Task.objects.filter(id=task['id']).exists() + saved_task = Task.objects.get(id=task['id']) + assert saved_task.attempt_count == 1 + sleep(0.5) + # second save + old_stopped = task['stopped'] + task['stopped'] = timezone.now() + save_task(task, broker) + saved_task = Task.objects.get(id=task['id']) + assert saved_task.attempt_count == 2 + # third save - + task['stopped'] = timezone.now() + save_task(task, broker) + saved_task = Task.objects.get(id=task['id']) + assert saved_task.attempt_count == 3 + # task should be removed from queue + assert broker.queue_size() == 0 + + + @pytest.mark.django_db def test_update_failed(broker): tag = uuid() From 3b2f444738104d0a166f5c94884d39ec78855848 Mon Sep 17 00:00:00 2001 From: Tim O'Meara Date: Tue, 11 Aug 2020 17:32:08 -0500 Subject: [PATCH 5/7] version bump --- django_q/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django_q/__init__.py b/django_q/__init__.py index c9bce68..16c5550 100644 --- a/django_q/__init__.py +++ b/django_q/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 2) +VERSION = (1, 3, 3) default_app_config = "django_q.apps.DjangoQConfig" diff --git a/pyproject.toml b/pyproject.toml index 4fcf8c3..45fe9af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "django-q" -version = "1.3.2" +version = "1.3.3" description = "A multiprocessing distributed task queue for Django" authors = ["Ilan Steemers "] license = "MIT" diff --git a/setup.py b/setup.py index 81e6b10..4bc3586 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ class PyTest(Command): setup( name='django-q', - version='1.3.2', + version='1.3.3', author='Ilan Steemers', author_email='koed00@gmail.com', keywords='django multiprocessing worker scheduler queue', From 991cca58442a262f7e5018434fff8e6003f413cd Mon Sep 17 00:00:00 2001 From: Tim O'Meara Date: Tue, 11 Aug 2020 17:32:48 -0500 Subject: [PATCH 6/7] Revert "version bump" This reverts commit 3b2f444738104d0a166f5c94884d39ec78855848. --- django_q/__init__.py | 2 +- pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django_q/__init__.py b/django_q/__init__.py index 16c5550..c9bce68 100644 --- a/django_q/__init__.py +++ b/django_q/__init__.py @@ -1,4 +1,4 @@ -VERSION = (1, 3, 3) +VERSION = (1, 3, 2) default_app_config = "django_q.apps.DjangoQConfig" diff --git a/pyproject.toml b/pyproject.toml index 45fe9af..4fcf8c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "django-q" -version = "1.3.3" +version = "1.3.2" description = "A multiprocessing distributed task queue for Django" authors = ["Ilan Steemers "] license = "MIT" diff --git a/setup.py b/setup.py index 4bc3586..81e6b10 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ class PyTest(Command): setup( name='django-q', - version='1.3.3', + version='1.3.2', author='Ilan Steemers', author_email='koed00@gmail.com', keywords='django multiprocessing worker scheduler queue', From 655f0aadb538c3a97ffda47f84a2b50f4cf4100b Mon Sep 17 00:00:00 2001 From: Tim O'Meara Date: Thu, 13 Aug 2020 11:58:00 -0500 Subject: [PATCH 7/7] rename config var to max_retries, remove attempt_count form admin , add admin ui customization to docs --- django_q/admin.py | 2 +- django_q/cluster.py | 2 +- django_q/conf.py | 2 +- django_q/tests/test_admin.py | 6 ++---- django_q/tests/test_cluster.py | 2 +- docs/admin.rst | 23 +++++++++++++++++++++++ docs/configure.rst | 6 +++--- 7 files changed, 32 insertions(+), 11 deletions(-) diff --git a/django_q/admin.py b/django_q/admin.py index f40118e..a7484ce 100644 --- a/django_q/admin.py +++ b/django_q/admin.py @@ -43,7 +43,7 @@ retry_failed.short_description = _("Resubmit selected tasks to queue") class FailAdmin(admin.ModelAdmin): """model admin for failed tasks.""" - list_display = ("name", "func", "started", "stopped", "short_result", "attempt_count") + list_display = ("name", "func", "started", "stopped", "short_result") def has_add_permission(self, request): """Don't allow adds.""" diff --git a/django_q/cluster.py b/django_q/cluster.py index d8e6f27..1d55663 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -481,7 +481,7 @@ def save_task(task, broker: Broker): existing_task.attempt_count = existing_task.attempt_count + 1 existing_task.save() - if 0 < Conf.ATTEMPT_COUNT == existing_task.attempt_count: + if Conf.MAX_ATTEMPTS > 0 and existing_task.attempt_count >= Conf.MAX_ATTEMPTS: broker.acknowledge(task['ack_id']) else: diff --git a/django_q/conf.py b/django_q/conf.py index 964364f..ba7e370 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -165,7 +165,7 @@ class Conf: ERROR_REPORTER = conf.get("error_reporter", {}) # Optional attempt count. set to 0 for infinite attempts - ATTEMPT_COUNT = conf.get('attempt_count', 0) + MAX_ATTEMPTS = conf.get('max_attempts', 0) # OSX doesn't implement qsize because of missing sem_getvalue() try: diff --git a/django_q/tests/test_admin.py b/django_q/tests/test_admin.py index 5dc8880..52e8d41 100644 --- a/django_q/tests/test_admin.py +++ b/django_q/tests/test_admin.py @@ -21,8 +21,7 @@ def test_admin_views(admin_client, monkeypatch): func='test.fail', started=timezone.now(), stopped=timezone.now(), - success=False, - attempt_count=1) + success=False) tag = uuid() t = Task.objects.create( id=tag[1], @@ -30,8 +29,7 @@ def test_admin_views(admin_client, monkeypatch): func='test.success', started=timezone.now(), stopped=timezone.now(), - success=True, - attempt_count=1) + success=True) q = OrmQ.objects.create( key='test', payload=SignedPackage.dumps({'id': 1, 'func': 'test', 'name': 'test'})) diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py index efb7089..a6fd742 100644 --- a/django_q/tests/test_cluster.py +++ b/django_q/tests/test_cluster.py @@ -399,7 +399,7 @@ def test_bad_secret(broker, monkeypatch): @pytest.mark.django_db def test_attempt_count(broker, monkeypatch): - monkeypatch.setattr(Conf, 'ATTEMPT_COUNT', 3) + monkeypatch.setattr(Conf, 'MAX_ATTEMPTS', 3) tag = uuid() task = {'id': tag[1], 'name': tag[0], diff --git a/docs/admin.rst b/docs/admin.rst index 9eace11..aad6913 100644 --- a/docs/admin.rst +++ b/docs/admin.rst @@ -33,6 +33,29 @@ You can resubmit a failed task back to the queue using the admins action menu. Uses the :class:`Failure` proxy model + + +Customize the admin UI by creating your own ``admin.ModelAdmin`` class and use ``admin.site.unregister`` and ``admin.site.register`` to replace the default +for example: + +.. code-block:: python + from django_q import models as q_models + from django_q import admin as q_admin + + admin.site.unregister([q_models.Failure]) + @admin.register(q_models.Failure) + class ChildClassAdmin(q_admin.FailAdmin): + list_display = ( + 'name', + 'func', + 'result', + 'started', + # add attempt_count to list_display + 'attempt_count' + ) + + + Scheduled tasks --------------- diff --git a/docs/configure.rst b/docs/configure.rst index afc6aa0..18e8d42 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -76,12 +76,12 @@ ack_failures When set to ``True``, also acknowledge unsuccessful tasks. This causes failed tasks to be considered as successful deliveries, thereby removing them from the task queue. Can also be set per-task by passing the ``ack_failure`` option to :func:`async_task`. Defaults to ``False``. -.. attempt_count: +.. _max_attempts: -attempt_count +max_attempts ~~~~~~~~~~~~~ -Limit the number of retries for failed tasks. Set to 0 for infinite retries. Defaults to 0 +Limit the number of retry attempts for failed tasks. Set to 0 for infinite retries. Defaults to 0 .. _retry: