From e3f959c25810eeedab69cf6faaf4d2b5a082ff7e Mon Sep 17 00:00:00 2001 From: Stan Triepels <1939656+GDay@users.noreply.github.com> Date: Fri, 14 Oct 2022 17:13:48 +0200 Subject: [PATCH] Admin improvements (#11) * Add the group column to OrmQ admin page * Add group field to tasks list and search_field * Add search by name to Schedule admin page * Limit succesful task history per schedule with SAVE_LIMIT_GROUP config * Improve ScheduleAdmin list view performance Co-authored-by: Marc Sabatier --- django_q/admin.py | 42 ++++++++++++++++++++++++++++++++++-------- django_q/cluster.py | 22 +++++++++++++++------- django_q/conf.py | 8 ++++++-- django_q/models.py | 6 ++++++ 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/django_q/admin.py b/django_q/admin.py index 53aaa16..9a34115 100644 --- a/django_q/admin.py +++ b/django_q/admin.py @@ -1,16 +1,19 @@ """Admin module for Django.""" +from django.urls import reverse +from django.utils.html import format_html from django.contrib import admin from django.utils.translation import gettext_lazy as _ +from django.db.models.expressions import OuterRef, Subquery from django_q.conf import Conf, croniter -from django_q.models import Failure, OrmQ, Schedule, Success +from django_q.models import Failure, OrmQ, Schedule, Success, Task from django_q.tasks import async_task class TaskAdmin(admin.ModelAdmin): """model admin for success tasks.""" - list_display = ("name", "func", "started", "stopped", "time_taken", "group") + list_display = ("name", "group", "func", "started", "stopped", "time_taken") def has_add_permission(self, request): """Don't allow adds.""" @@ -43,14 +46,14 @@ retry_failed.short_description = _("Resubmit selected tasks to queue") class FailAdmin(admin.ModelAdmin): """model admin for failed tasks.""" - list_display = ("name", "func", "started", "stopped", "short_result") + list_display = ("name", "group", "func", "started", "stopped", "short_result") def has_add_permission(self, request): """Don't allow adds.""" return False actions = [retry_failed] - search_fields = ("name", "func") + search_fields = ("name", "func", "group") list_filter = ("group",) readonly_fields = [] @@ -70,8 +73,8 @@ class ScheduleAdmin(admin.ModelAdmin): "repeats", "cluster", "next_run", - "last_run", - "success", + "get_last_run", + "get_success", ) # optional cron strings @@ -79,14 +82,37 @@ class ScheduleAdmin(admin.ModelAdmin): readonly_fields = ("cron",) list_filter = ("next_run", "schedule_type", "cluster") - search_fields = ("func",) + search_fields = ("name", "func",) list_display_links = ("id", "name") + def get_queryset(self, request): + qs = super().get_queryset(request) + task_query = Task.objects.filter(id=OuterRef('task')).values('id', 'name', 'success') + qs = qs.annotate(task_id=Subquery(task_query.values('id')), task_name=Subquery(task_query.values('name')), + task_success=Subquery(task_query.values('success'))) + return qs + + def get_success(self, obj): + return obj.task_success + get_success.boolean = True + get_success.short_description = _("success") + + def get_last_run(self, obj): + if obj.task_name is not None: + if obj.task_success: + url = reverse("admin:django_q_success_change", args=(obj.task_id,)) + else: + url = reverse("admin:django_q_failure_change", args=(obj.task_id,)) + return format_html(f'[{obj.task_name}]') + return None + get_last_run.allow_tags = True + get_last_run.short_description = _("last_run") + class QueueAdmin(admin.ModelAdmin): """queue admin for ORM broker""" - list_display = ("id", "key", "task_id", "name", "func", "lock") + list_display = ("id", "key", "name", "group", "func", "lock", "task_id") def save_model(self, request, obj, form, change): obj.save(using=Conf.ORM) diff --git a/django_q/cluster.py b/django_q/cluster.py index 87c8077..ee084e0 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -479,12 +479,21 @@ def save_task(task, broker: Broker): # SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning close_old_django_connections() try: - with db.transaction.atomic(): - last = Success.objects.select_for_update().last() - if task["success"] and 0 < Conf.SAVE_LIMIT <= Success.objects.count(): - last.delete() + if task["success"]: + # first apply per group success history limit + if "group" in task: + with db.transaction.atomic(): + qs = Success.objects.filter(group=task["group"]) + last = qs.select_for_update().last() + if Conf.SAVE_LIMIT_PER_GROUP <= qs.count(): + last.delete() + # then apply global success history limit + with db.transaction.atomic(): + last = Success.objects.select_for_update().last() + if Conf.SAVE_LIMIT <= Success.objects.count(): + last.delete() # check if this task has previous results - if Task.objects.filter(id=task["id"], name=task["name"]).exists(): + try: existing_task = Task.objects.get(id=task["id"], name=task["name"]) # only update the result if it hasn't succeeded yet if not existing_task.success: @@ -499,8 +508,7 @@ def save_task(task, broker: Broker): and existing_task.attempt_count >= Conf.MAX_ATTEMPTS ): broker.acknowledge(task["ack_id"]) - - else: + except Task.DoesNotExist: func = task["func"] # convert func to string if inspect.isfunction(func): diff --git a/django_q/conf.py b/django_q/conf.py index 3a8f398..6436eaf 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -86,6 +86,10 @@ class Conf: # Failures are always saved SAVE_LIMIT = conf.get("save_limit", 250) + # Maximum number of successful tasks of the same group kept in the database. 0 saves everything. -1 saves none + # Failures are always saved + SAVE_LIMIT_PER_GROUP = conf.get("save_limit_per_group", 5) + # Guard loop sleep in seconds. Should be between 0 and 60 seconds. GUARD_CYCLE = conf.get("guard_cycle", 0.5) @@ -137,8 +141,8 @@ class Conf: # Verify if retry and timeout settings are correct if not TIMEOUT or (TIMEOUT > RETRY): warn( - """Retry and timeout are misconfigured. Set retry larger than timeout, - failure to do so will cause the tasks to be retriggered before completion. + """Retry and timeout are misconfigured. Set retry larger than timeout, + failure to do so will cause the tasks to be retriggered before completion. See https://django-q.readthedocs.io/en/latest/configure.html#retry for details.""" ) diff --git a/django_q/models.py b/django_q/models.py index d83e039..de9ef27 100644 --- a/django_q/models.py +++ b/django_q/models.py @@ -220,7 +220,10 @@ class Schedule(models.Model): return self.func success.boolean = True + success.short_description = _("success") last_run.allow_tags = True + last_run.short_description = _("last_run") + class Meta: app_label = "django_q" @@ -246,6 +249,9 @@ class OrmQ(models.Model): def name(self): return self.task()["name"] + def group(self): + return self.task().get("group") + class Meta: app_label = "django_q" verbose_name = _("Queued task")