Release v1.7.0

* Add index on task table

* also add models file

* add make commands and limit index

* format

.git-blame-ignore-revs

@@ -1,2 +1,4 @@
 # flake8, black, isort
 b1d000d007f3f77069719523268a0c6256dc0860
+# move to ruff formatting/linting
+ad4d24e17c9424b17cd8ae65c2def7ecc74e63c1

.github/workflows/test.yml

@@ -7,29 +7,32 @@ on:
     branches:
       - master
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/checkout@v3
+        - name: Lint with ruff
+          run: |
+            pipx install ruff==0.4.10
+            ruff format . --check && ruff check .
+
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12.0-rc.3" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
-        django: [ "3.2", "4.1", "4.2", "5.0a1"]
+        django: [ "4.2", "5.0", "5.1"]
         exclude:
+          # django 5.1 does not support 3.8 and 3.9
+          - python-version: "3.8"
+            django: "5.1"
+          - python-version: "3.9"
+            django: "5.1"
           # django 5.0 does not support 3.8 and 3.9
           - python-version: "3.8"
-            django: "5.0a1" 
+            django: "5.0" 
           - python-version: "3.9"
-            django: "5.0a1" 
+            django: "5.0" 
-          # django 4.2 does not support 3.12
-          - python-version: "3.12.0-rc.3"
-            django: "4.2" 
-          # django 4.1 does not support 3.12
-          - python-version: "3.12.0-rc.3"
-            django: "4.1" 
-          # django 3.2 does not support 3.11 and 3.12
-          - python-version: "3.11"
-            django: "3.2" 
-          - python-version: "3.12.0-rc.3"
-            django: "3.2" 
 
     services:
       disque:

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+## [v1.7.0](https://github.com/django-q2/django-q2/tree/v1.7.0) (2024-08-24)
+
+**Merged pull requests:**
+
+- Remove support for Django 3.2 and 4.1 https://github.com/django-q2/django-q2/pull/183
+- Fix max attempts for value 1 https://github.com/django-q2/django-q2/pull/185
+- Replace black/isort with ruff https://github.com/django-q2/django-q2/pull/188
+- Fix repeating task after timeout https://github.com/django-q2/django-q2/pull/184
+- fix: Oracle ORM backend compatibility #180 https://github.com/django-q2/django-q2/pull/186
+- chore: Update CI for Django 4.2 Python 3.12 support https://github.com/django-q2/django-q2/pull/208
+- chore: Add Support Django 5.1 https://github.com/django-q2/django-q2/pull/207
+- Call mark_process_dead on worker pid if prometheus_client is installed https://github.com/django-q2/django-q2/pull/212
+- Add example project https://github.com/django-q2/django-q2/pull/215
+- Add index on succeeded tasks https://github.com/django-q2/django-q2/pull/164
+
 ## [v1.6.2](https://github.com/django-q2/django-q2/tree/v1.6.2) (2024-03-05)
 
 **Merged pull requests:**

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.12
 
 ENV PYTHONUNBUFFERED 1
 RUN mkdir -p /app

Dockerfile.dev

@@ -8,10 +8,10 @@ ENV PYTHONUNBUFFERED 1
 ENV SHELL /bin/bash
 
 # Upgrades pip
-RUN pip install --upgrade pip 
+RUN pip install -U pip setuptools
 
 # Install poetry
-RUN pip install poetry
+RUN pip install poetry==1.8.2
 
 WORKDIR /app
 

Makefile

@@ -0,0 +1,21 @@
+dev:
+	docker compose -f web-docker-compose.yaml up
+
+test:
+	docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run pytest
+
+shell:
+	docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py shell
+
+makemigrations:
+	docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py makemigrations
+
+migrate:
+	docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py migrate
+
+createsuperuser:
+	docker compose -f web-docker-compose.yaml run --rm web python manage.py createsuperuser
+
+format:
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run ruff format .
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run ruff check . --fix

README.rst

@@ -35,10 +35,10 @@ See the `changelog <https://github.com/GDay/django-q2/blob/master/CHANGELOG.md>`
 Requirements
 ~~~~~~~~~~~~
 
--  `Django <https://www.djangoproject.com>`__ > = 3.2
+-  `Django <https://www.djangoproject.com>`__ > = 4.2
 -  `Django-picklefield <https://github.com/gintas/django-picklefield>`__
 
-Tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 3.2.X, 4.1.X, 4.2.X and 5.0.X
+Tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 4.2.X and 5.0.X
 
 Brokers
 ~~~~~~~
@@ -197,6 +197,24 @@ Admin page or directly from your code:
 
 For more info check the `Schedules <https://django-q2.readthedocs.org/en/latest/schedules.html>`__ documentation.
 
+Development
+~~~~~~~
+
+There is an example project that you can use to develop with. Docker (compose) is being used to set everything up.
+Please note that you will have to restart the django-q container when changes have been made to tasks or django-q.
+You can start the example project with:
+
+.. code:: bash
+
+    make dev
+
+Create a superuser with:
+
+.. code:: bash
+
+    make createsuperuser
+
+
 Testing
 ~~~~~~~
 
@@ -204,7 +222,7 @@ Running tests is easy with docker compose, it will also start the necessary data
 
 .. code:: bash
 
-    docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run pytest
+    make test
 
 Locale
 ~~~~~~
@@ -212,12 +230,6 @@ Locale
 Currently available in English, German, Turkish, and French.
 Translation pull requests are always welcome.
 
-Todo
-~~~~
-
--  Better tests and coverage
--  Less dependencies?
-
 Acknowledgements
 ~~~~~~~~~~~~~~~~
 

django_q/__init__.py

@@ -1,6 +1,6 @@
 import django
 
-VERSION = (1, 6, 2)
+VERSION = (1, 7, 0)
 
 if django.VERSION < (3, 2):
     default_app_config = "django_q.apps.DjangoQConfig"

django_q/admin.py

@@ -1,4 +1,5 @@
 """Admin module for Django."""
+
 from django.contrib import admin
 from django.db.models.expressions import OuterRef, Subquery
 from django.urls import reverse

django_q/brokers/__init__.py

@@ -182,7 +182,7 @@ def get_broker(list_key: str = None) -> Broker:
 
         return ironmq.IronMQBroker(list_key=list_key)
     # SQS
-    elif type(Conf.SQS) == dict:
+    elif isinstance(Conf.SQS, dict):
         from django_q.brokers import aws_sqs
 
         return aws_sqs.Sqs(list_key=list_key)

django_q/cluster.py

@@ -21,7 +21,14 @@ from django.utils.translation import gettext_lazy as _
 
 # Local
 from django_q.brokers import Broker, get_broker
-from django_q.conf import Conf, get_ppid, logger, psutil, setproctitle
+from django_q.conf import (
+    Conf,
+    get_ppid,
+    logger,
+    prometheus_multiprocess,
+    psutil,
+    setproctitle,
+)
 from django_q.humanhash import humanize
 from django_q.monitor import monitor
 from django_q.pusher import pusher
@@ -223,6 +230,8 @@ class Sentinel:
                 % {"name": process.name}
             )
         else:
+            if prometheus_multiprocess:
+                prometheus_multiprocess.mark_process_dead(process.pid)
             self.pool.remove(process)
             self.spawn_worker()
             if process.timer.value == 0:

django_q/conf.py

@@ -38,6 +38,11 @@ try:
 except ModuleNotFoundError:
     setproctitle = None
 
+try:
+    from prometheus_client import multiprocess as prometheus_multiprocess
+except ModuleNotFoundError:
+    prometheus_multiprocess = None
+
 
 class Conf:
     """
@@ -263,7 +268,6 @@ if not logger.hasHandlers():
 
 # Error Reporting Interface
 class ErrorReporter:
-
     # initialize with iterator of reporters (better name, targets?)
     def __init__(self, reporters):
         self.targets = [target for target in reporters]

django_q/core_signing.py

@@ -2,17 +2,22 @@ import datetime
 import time
 import zlib
 
-import django
+from django.core.signing import (
-from django.core.signing import BadSignature, JSONSerializer, SignatureExpired
+    BadSignature,
+    JSONSerializer,
+    SignatureExpired,
+    b64_decode,
+    dumps,
+)
 from django.core.signing import Signer as Sgnr
 from django.core.signing import TimestampSigner as TsS
-from django.core.signing import b64_decode, dumps
 
 try:
     from django.core.signing import b62_decode
 except ImportError:
     # fallback for django 3.x
     from django.utils.baseconv import base62
+
     b62_decode = base62.decode
 
 from django.utils.crypto import constant_time_compare

django_q/exceptions.py

@@ -0,0 +1,7 @@
+class TimeoutException(SystemExit):
+    """
+    Exception for when a worker takes too long to complete a task
+    Raising SystemExit will make sure the function terminates gracefully.
+    """
+
+    pass

django_q/humanhash.py

@@ -4,6 +4,7 @@ humanhash: Human-readable representations of digests.
 The simplest ways to use this module are the :func:`humanize` and :func:`uuid`
 functions. For tighter control over the output, see :class:`HumanHasher`.
 """
+
 import operator
 import uuid as uuidlib
 from argparse import ArgumentError
@@ -270,7 +271,6 @@ DEFAULT_WORDLIST = (
 
 
 class HumanHasher:
-
     """
     Transforms hex digests to human-readable strings.
 
@@ -290,7 +290,6 @@ class HumanHasher:
         self.wordlist = wordlist
 
     def humanize(self, hexdigest, words=4, separator="-"):
-
         """
         Humanize a given hexadecimal digest.
 
@@ -314,7 +313,6 @@ class HumanHasher:
 
     @staticmethod
     def compress(bytes, target):
-
         """
         Compress a list of byte values to a fixed target length.
 
@@ -353,7 +351,6 @@ class HumanHasher:
         return checksums
 
     def uuid(self, **params):
-
         """
         Generate a UUID with a human-readable representation.
 

django_q/migrations/0001_initial.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = []
 
     operations = [

django_q/migrations/0002_auto_20150630_1624.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0001_initial"),
     ]

django_q/migrations/0003_auto_20150708_1326.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0002_auto_20150630_1624"),
     ]

django_q/migrations/0004_auto_20150710_1043.py

@@ -2,7 +2,6 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0003_auto_20150708_1326"),
     ]

django_q/migrations/0005_auto_20150718_1506.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0004_auto_20150710_1043"),
     ]

django_q/migrations/0006_auto_20150805_1817.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0005_auto_20150718_1506"),
     ]

django_q/migrations/0007_ormq.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0006_auto_20150805_1817"),
     ]

django_q/migrations/0008_auto_20160224_1026.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0007_ormq"),
     ]

django_q/migrations/0009_auto_20171009_0915.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0008_auto_20160224_1026"),
     ]

django_q/migrations/0010_auto_20200610_0856.py

@@ -3,7 +3,6 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0009_auto_20171009_0915"),
     ]

django_q/migrations/0011_auto_20200628_1055.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0010_auto_20200610_0856"),
     ]

django_q/migrations/0012_auto_20200702_1608.py

@@ -6,7 +6,6 @@ import django_q.models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0011_auto_20200628_1055"),
     ]

django_q/migrations/0013_task_attempt_count.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0012_auto_20200702_1608"),
     ]

django_q/migrations/0014_schedule_cluster.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0013_task_attempt_count"),
     ]

django_q/migrations/0015_alter_schedule_schedule_type.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0014_schedule_cluster"),
     ]

django_q/migrations/0016_schedule_intended_date_kwarg.py

@@ -6,7 +6,6 @@ import django_q.models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0015_alter_schedule_schedule_type"),
     ]

django_q/migrations/0017_task_cluster_alter.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0016_schedule_intended_date_kwarg"),
     ]

django_q/migrations/0018_task_success_index.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.7 on 2024-03-05 17:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("django_q", "0017_task_cluster_alter"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="task",
+            index=models.Index(
+                condition=models.Q(("success", True)),
+                fields=["group", "name", "func"],
+                name="success_index",
+            ),
+        ),
+    ]

django_q/models.py

@@ -5,12 +5,12 @@ from keyword import iskeyword
 from django import get_version
 from django.core.exceptions import ValidationError
 from django.db import models
+from django.db.models import Q
 from django.template.defaultfilters import truncatechars
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.functional import cached_property
 from django.utils.html import format_html
-from django.utils.timezone import is_aware
 from django.utils.translation import gettext_lazy as _
 
 # External
@@ -18,7 +18,7 @@ from picklefield import PickledObjectField
 from picklefield.fields import dbsafe_decode
 
 # Local
-from django_q.conf import Conf, croniter
+from django_q.conf import croniter
 from django_q.signing import SignedPackage
 from django_q.utils import add_months, add_years, localtime
 
@@ -112,6 +112,13 @@ class Task(models.Model):
     class Meta:
         app_label = "django_q"
         ordering = ["-stopped"]
+        indexes = [
+            models.Index(
+                name="success_index",
+                fields=["group", "name", "func"],
+                condition=Q(success=True),
+            ),
+        ]
 
 
 class SuccessManager(models.Manager):

django_q/monitor.py

@@ -2,8 +2,8 @@ from multiprocessing.process import current_process
 from multiprocessing.queues import Queue
 
 from django import core, db
-from django.utils.translation import gettext_lazy as _
 from django.apps.registry import apps
+from django.utils.translation import gettext_lazy as _
 
 try:
     apps.check_apps_ready()
@@ -108,34 +108,28 @@ def save_task(task, broker: Broker):
             filters[Conf.SAVE_LIMIT_PER] = value
 
         with db.transaction.atomic(using=db.router.db_for_write(Success)):
-            last = Success.objects.filter(**filters).select_for_update().last()
+            list(Success.objects.filter(**filters).select_for_update())
             if (
                 task["success"]
                 and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
             ):
-                last.delete()
+                Success.objects.filter(**filters).last().delete()
 
         # check if this task has previous results
         try:
-            existing_task = Task.objects.get(id=task["id"], name=task["name"])
+            task_obj = Task.objects.get(id=task["id"], name=task["name"])
             # only update the result if it hasn't succeeded yet
-            if not existing_task.success:
+            if not task_obj.success:
-                existing_task.stopped = task["stopped"]
+                task_obj.stopped = task["stopped"]
-                existing_task.result = task["result"]
+                task_obj.result = task["result"]
-                existing_task.success = task["success"]
+                task_obj.success = task["success"]
-                existing_task.attempt_count = existing_task.attempt_count + 1
+                task_obj.attempt_count = task_obj.attempt_count + 1
-                existing_task.save()
+                task_obj.save()
-
-            if (
-                Conf.MAX_ATTEMPTS > 0
-                and existing_task.attempt_count >= Conf.MAX_ATTEMPTS
-            ):
-                broker.acknowledge(task["ack_id"])
 
         except Task.DoesNotExist:
             # convert func to string
             func = get_func_repr(task["func"])
-            Task.objects.create(
+            task_obj = Task.objects.create(
                 id=task["id"],
                 name=task["name"],
                 func=func,
@@ -150,6 +144,10 @@ def save_task(task, broker: Broker):
                 success=task["success"],
                 attempt_count=1,
             )
+
+        if Conf.MAX_ATTEMPTS > 0 and task_obj.attempt_count >= Conf.MAX_ATTEMPTS:
+            broker.acknowledge(task["ack_id"])
+
     except Exception:
         logger.exception("Could not save task result")
 

django_q/monitor_terminal.py

@@ -397,9 +397,7 @@ def memory(run_once=False, workers=False, broker=None):
                     2,
                 )
                 # memory available (MB)
-                memory_available = round(
+                memory_available = round(psutil.virtual_memory().available / 1024**2, 2)
-                    psutil.virtual_memory().available / 1024**2, 2
-                )
                 if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
                     MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
                     MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()

django_q/pusher.py

@@ -4,8 +4,8 @@ from multiprocessing.queues import Queue
 from time import sleep
 
 from django import core
-from django.utils.translation import gettext_lazy as _
 from django.apps.registry import apps
+from django.utils.translation import gettext_lazy as _
 
 try:
     apps.check_apps_ready()
@@ -58,9 +58,9 @@ def pusher(task_queue: Queue, event: Event, broker: Broker = None):
                     logger.exception("Failed to push task to queue")
                     broker.fail(ack_id)
                     continue
-                task[
+                task["cluster"] = (
-                    "cluster"
+                    Conf.CLUSTER_NAME
-                ] = Conf.CLUSTER_NAME  # save actual cluster name to orm task table
+                )  # save actual cluster name to orm task table
                 task["ack_id"] = ack_id
                 task_queue.put(task)
             logger.debug(

django_q/queues.py

@@ -2,6 +2,7 @@
 The code is derived from
 https://github.com/althonos/pronto/commit/3384010dfb4fc7c66a219f59276adef3288a886b
 """
+
 import multiprocessing
 import multiprocessing.queues
 import sys

django_q/scheduler.py

@@ -2,11 +2,10 @@ import ast
 from multiprocessing.process import current_process
 
 from django import core, db
+from django.apps.registry import apps
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
-from django.apps.registry import apps
-
 try:
     apps.check_apps_ready()
 except core.exceptions.AppRegistryNotReady:
@@ -86,7 +85,7 @@ def scheduler(broker: Broker = None):
                     # Little Fix for already broken numbers
                     if s.repeats < -1:
                         s.repeats = -1
-                    
+
                     # Check if the value is not zero
                     if s.repeats > 0:
                         s.repeats -= 1

django_q/signing.py

@@ -1,4 +1,5 @@
 """Package signing."""
+
 import pickle
 
 from django_q import core_signing as signing

django_q/tasks.py

@@ -1,4 +1,5 @@
 """Provides task functionality."""
+
 # Standard
 from multiprocessing import Value
 from time import sleep, time
@@ -569,7 +570,9 @@ class Chain:
     A sequential chain of tasks
     """
 
-    def __init__(self, chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC, broker=None):
+    def __init__(
+        self, chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC, broker=None
+    ):
         self.chain = chain or []
         self.group = group or ""
         self.broker = broker or get_broker()
@@ -729,17 +732,14 @@ class AsyncTask:
         return self.id
 
     def result(self, wait=0):
-
         if self.started:
             return result(self.id, wait=wait, cached=self.cached)
 
     def fetch(self, wait=0):
-
         if self.started:
             return fetch(self.id, wait=wait, cached=self.cached)
 
     def result_group(self, failures=False, wait=0, count=None):
-
         if self.started and self.group:
             return result_group(
                 self.group,
@@ -750,7 +750,6 @@ class AsyncTask:
             )
 
     def fetch_group(self, failures=True, wait=0, count=None):
-
         if self.started and self.group:
             return fetch_group(
                 self.group,

django_q/timeout.py

@@ -0,0 +1,38 @@
+import signal
+
+from django.utils.translation import gettext_lazy as _
+
+from django_q.conf import logger
+
+from .exceptions import TimeoutException
+
+
+class TimeoutHandler:
+    def __init__(self, timeout: int):
+        self._timeout = timeout
+
+    def raise_timeout_exception(self, signum, frame):
+        raise TimeoutException(
+            f"Task exceeded maximum timeout value ({self._timeout} seconds)"
+        )
+
+    def __enter__(self):
+        # if the timeout is -1, then there is no timeout and the task will always keep running until it's done or manually killed
+        if self._timeout == -1:
+            return
+        try:
+            signal.signal(signal.SIGALRM, self.raise_timeout_exception)
+        except ValueError:  # ValueError is raised for Windows users
+            logger.debug(_("SIGALARM is not available on your platform"))
+
+        signal.alarm(self._timeout)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self._timeout == -1:
+            return
+        """When getting out of the timeout, reset the alarm, so it won't trigger"""
+        try:
+            signal.alarm(0)
+            signal.signal(signal.SIGALRM, signal.SIG_DFL)
+        except ValueError:  # ValueError is raised for Windows users
+            logger.debug(_("SIGALARM is not available on your platform"))

django_q/worker.py

@@ -5,9 +5,9 @@ from multiprocessing.process import current_process
 from multiprocessing.queues import Queue
 
 from django import core
+from django.apps.registry import apps
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
-from django.apps.registry import apps
 
 try:
     apps.check_apps_ready()
@@ -17,7 +17,9 @@ except core.exceptions.AppRegistryNotReady:
     django.setup()
 
 from django_q.conf import Conf, error_reporter, logger, resource, setproctitle
+from django_q.exceptions import TimeoutException
 from django_q.signals import post_spawn, pre_execute
+from django_q.timeout import TimeoutHandler
 from django_q.utils import close_old_django_connections, get_func_repr
 
 try:
@@ -89,25 +91,37 @@ def worker(
         pre_execute.send(sender="django_q", func=f, task=task)
         # execute the payload
         timer.value = timer_value  # Busy
+        if timer.value != -1:
+            timer.value += 3  # Add buffer so that guard doesn't kill the process on timeout before it gets processed
 
+        timeout_error = False
         try:
             if f is None:
                 # raise a meaningfull error if task["func"] is not a valid function
                 raise ValueError(f"Function {task['func']} is not defined")
-            res = f(*task["args"], **task["kwargs"])
+            with TimeoutHandler(timer_value):
+                res = f(*task["args"], **task["kwargs"])
             result = (res, True)
-        except Exception as e:
+        except (Exception, TimeoutException) as e:
+            if isinstance(e, TimeoutException):
+                timeout_error = True
             result = (f"{e} : {traceback.format_exc()}", False)
             if error_reporter:
                 error_reporter.report()
             if task.get("sync", False):
                 raise
+
         with timer.get_lock():
             # Process result
             task["result"] = result[0]
             task["success"] = result[1]
             task["stopped"] = timezone.now()
             result_queue.put(task)
+            if timeout_error:
+                # force destroy process due to timeout
+                timer.value = 0
+                break
+
             timer.value = -1  # Idle
             if setproctitle:
                 setproctitle.setproctitle(f"qcluster {proc_name} idle")

docs/conf.py

@@ -73,9 +73,9 @@ author = "Ilan Steemers, Stan Triepels"
 # built documents.
 #
 # The short X.Y version.
-version = "1.6"
+version = "1.7"
 # The full version, including alpha/beta/rc tags.
-release = "1.6.2"
+release = "1.7.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/configure.rst

@@ -79,6 +79,8 @@ timeout
 The number of seconds a worker is allowed to spend on a task before it's terminated. Defaults to ``None``, meaning it will never time out.
 Set this to something that makes sense for your project. Can be overridden for individual tasks.
 
+Note: for systems that don't have `SIGALRM` available (e.g. Windows), it will not raise an error properly. It will kill the task, but it will keep retrying until it finishes within the given time.
+
 See :ref:`retry` for details how to set values for timeout and retry.
 
 .. _time_zone:

docs/docker-compose.yaml

@@ -1,4 +1,3 @@
-version: "3.0"
 services:
   docs:
     container_name: djangoq2-docs

docs/index.rst

@@ -27,7 +27,7 @@ Features
 -  Rollbar and Sentry support
 
 
-Django Q2 is tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 3.2.x, 4.1.x, 4.2.x and 5.0.x
+Django Q2 is tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 4.2.x and 5.0.x
 
 Currently available in English, German and French.
 

exampleproject/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for exampleproject project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "exampleproject.settings")
+
+application = get_asgi_application()

exampleproject/settings.py

@@ -0,0 +1,134 @@
+"""
+Django settings for exampleproject project.
+
+Generated by 'django-admin startproject' using Django 5.1.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/5.1/ref/settings/
+"""
+
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = "django-insecure-)oouh93bjg+c=b!l5-*w)7et1l+!3nmp223rr^1r4v#jn&ow3f"
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ["*"]
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "django_q",
+]
+
+MIDDLEWARE = [
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+]
+
+ROOT_URLCONF = "exampleproject.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = "exampleproject.wsgi.application"
+
+
+# Database
+# https://docs.djangoproject.com/en/5.1/ref/settings/#databases
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": BASE_DIR / "db.sqlite3",
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/5.1/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/5.1/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+
+TIME_ZONE = "UTC"
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/5.1/howto/static-files/
+
+STATIC_URL = "static/"
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+Q_CLUSTER = {
+    "name": "DjangORM",
+    "workers": 4,
+    "timeout": 90,
+    "retry": 120,
+    "queue_limit": 50,
+    "bulk": 10,
+    "orm": "default",
+}

exampleproject/urls.py

@@ -0,0 +1,27 @@
+"""
+URL configuration for exampleproject project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/5.1/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+
+from django.contrib import admin
+from django.urls import path
+
+from exampleproject import views
+
+urlpatterns = [
+    path("admin/", admin.site.urls),
+    path("new_task/", views.add_task, name="add_task"),
+    path("result/<slug:task_id>/", views.get_result, name="get_result"),
+]

exampleproject/views.py

@@ -0,0 +1,31 @@
+import time
+
+from django.http import HttpResponse
+from django.urls import reverse
+
+from django_q import tasks
+
+
+# internal function to be called with django_q
+def new_task(run_for_minutes):
+    print("Task started")
+    time.sleep(run_for_minutes)
+    print("Task done")
+    return True
+
+
+def add_task(request):
+    task_id = tasks.async_task(new_task, 5)
+    result_url = reverse("get_result", args=[task_id])
+    return HttpResponse(
+        f"Added async task with <a href='{result_url}'>Go to results</a>"
+    )
+
+
+def get_result(request, task_id):
+    task = tasks.fetch(task_id)
+    if not task:
+        msg = "Task running... please refresh after some time"
+    else:
+        msg = f"Async task result: {task.result}"
+    return HttpResponse(msg)

exampleproject/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for exampleproject project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "exampleproject.settings")
+
+application = get_wsgi_application()

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "django-q2"
-version = "1.6.2"
+version = "1.7.0"
 packages = [
     { include = "django_q" },
 ]
@@ -50,7 +50,7 @@ include = ['CHANGELOG.md']
 
 [tool.poetry.dependencies]
 python = ">=3.8,<4"
-django = { version = ">=3.2, <6", allow-prereleases = true}
+django = ">=4.2, <6"
 django-picklefield = "^3.1"
 
 blessed = { version = "^1.19.1", optional = true }
@@ -73,8 +73,7 @@ pytest = "^7.1.3"
 pytest-django = "^4.5.2"
 Sphinx = "^4.0.2"
 pytest-cov = "^4.0.0"
-black = "^22.10.0"
+ruff = "^0.4.4"
-isort = {extras = ["requirements_deprecated_finder"], version = "^5.10.1"}
 
 [tool.poetry.extras]
 requires = ["poetry_core"]

requirements.txt

@@ -1,40 +1,41 @@
-ansicon==1.89.0; platform_system == "Windows" and python_version >= "2.7" \
+asgiref==3.8.1 ; python_version >= "3.8" and python_version < "4" \
-    --hash=sha256:f1def52d17f65c2c9682cf8370c03f541f410c1752d6a14029f97318e4b9dfec \
+    --hash=sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47 \
-    --hash=sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1
+    --hash=sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590
-arrow==1.1.1; python_version >= "3.6" \
+backports-zoneinfo==0.2.1 ; python_version >= "3.8" and python_version < "3.9" \
-    --hash=sha256:77a60a4db5766d900a2085ce9074c5c7b8e2c99afeaa98ad627637ff6f292510 \
+    --hash=sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf \
-    --hash=sha256:dee7602f6c60e3ec510095b5e301441bc56288cb8f51def14dcb3079f623823a
+    --hash=sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328 \
-asgiref==3.3.4; python_version >= "3.6" \
+    --hash=sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546 \
-    --hash=sha256:92906c611ce6c967347bbfea733f13d6313901d54dcca88195eaeb52b2a8e8ee \
+    --hash=sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6 \
-    --hash=sha256:d1216dfbdfb63826470995d31caed36225dcaf34f182e0fa257a4dd9e86f1b78
+    --hash=sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570 \
-blessed==1.18.1; python_version >= "2.7" \
+    --hash=sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9 \
-    --hash=sha256:dd7c0d33db9a2e7f597b446996484d0ed46e1586239db064fb5025008937dcae \
+    --hash=sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7 \
-    --hash=sha256:8b09936def6bc06583db99b65636b980075733e13550cb6af262ce724a55da23
+    --hash=sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987 \
-django-picklefield==3.0.1; python_version >= "3" \
+    --hash=sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722 \
-    --hash=sha256:15ccba592ca953b9edf9532e64640329cd47b136b7f8f10f2939caa5f9ce4287 \
+    --hash=sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582 \
-    --hash=sha256:3c702a54fde2d322fe5b2f39b8f78d9f655b8f77944ab26f703be6c0ed335a35
+    --hash=sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc \
-django==3.2.4; python_version >= "3.6" \
+    --hash=sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b \
-    --hash=sha256:ea735cbbbb3b2fba6d4da4784a0043d84c67c92f1fdf15ad6db69900e792c10f \
+    --hash=sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1 \
-    --hash=sha256:66c9d8db8cc6fe938a28b7887c1596e42d522e27618562517cc8929eb7e7f296
+    --hash=sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08 \
-jinxed==1.1.0; platform_system == "Windows" and python_version >= "2.7" \
+    --hash=sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac \
-    --hash=sha256:6a61ccf963c16aa885304f27e6e5693783676897cea0c7f223270c8b8e78baf8 \
+    --hash=sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2
-    --hash=sha256:d8f1731f134e9e6b04d95095845ae6c10eb15cb223a5f0cabdea87d4a279c305
+django-picklefield==3.2 ; python_version >= "3.8" and python_version < "4" \
-python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
+    --hash=sha256:aa463f5d79d497dbe789f14b45180f00a51d0d670067d0729f352a3941cdfa4d \
-    --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
+    --hash=sha256:e9a73539d110f69825d9320db18bcb82e5189ff48dbed41821c026a20497764c
-    --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
+django==4.2.11 ; python_version >= "3.8" and python_version < "4" \
-pytz==2021.1; python_version >= "3.6" \
+    --hash=sha256:6e6ff3db2d8dd0c986b4eec8554c8e4f919b5c1ff62a5b4390c17aff2ed6e5c4 \
-    --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 \
+    --hash=sha256:ddc24a0a8280a0430baa37aff11f28574720af05888c62b7cfe71d219f4599d3
-    --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da
+importlib-metadata==7.1.0 ; python_version >= "3.8" and python_version < "3.10" \
-six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
+    --hash=sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570 \
-    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
+    --hash=sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2
-    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926
+sqlparse==0.5.0 ; python_version >= "3.8" and python_version < "4" \
-sqlparse==0.4.1; python_version >= "3.6" \
+    --hash=sha256:714d0a4932c059d16189f58ef5411ec2287a4360f17cdd0edd2d09d4c5087c93 \
-    --hash=sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0 \
+    --hash=sha256:c204494cd97479d0e39f28c93d46c0b2d5959c7b9ab904762ea6c7af211c8663
-    --hash=sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8
+typing-extensions==4.11.0 ; python_version >= "3.8" and python_version < "3.11" \
-typing-extensions==3.10.0.0; python_version < "3.8" and python_version >= "3.6" \
+    --hash=sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0 \
-    --hash=sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497 \
+    --hash=sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a
-    --hash=sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84 \
+tzdata==2024.1 ; python_version >= "3.8" and python_version < "4" and sys_platform == "win32" \
-    --hash=sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342
+    --hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \
-wcwidth==0.2.5; python_version >= "2.7" \
+    --hash=sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252
-    --hash=sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784 \
+zipp==3.18.1 ; python_version >= "3.8" and python_version < "3.10" \
-    --hash=sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83
+    --hash=sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b \
+    --hash=sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715

ruff.toml

@@ -0,0 +1 @@
+lint.extend-select = ["I"]

test-services-docker-compose.yaml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   redis:
     image: redis:latest

web-docker-compose.yaml

@@ -1,11 +1,16 @@
-version: '3'
-
 services:
   web:
     restart: always
-    command: python manage.py runserver 0.0.0.0:8000
+    command: bash -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000"
     ports:
       - "127.0.0.1:8000:8000"
     build: .
     volumes:
       - .:/app
+
+  django-q:
+    restart: always
+    command: bash -c "python manage.py migrate && python manage.py qcluster"
+    build: .
+    volumes:
+      - .:/app