Release v1.9.0

* Remove upper bound on Django version.

This allows it to be installed alongside Django 6 and later.

See https://iscinumpy.dev/post/bound-version-constraints/

* Run tests on Django 6.0 / Python 3.12

.git-blame-ignore-revs

@@ -1,2 +1,4 @@
 # flake8, black, isort
 b1d000d007f3f77069719523268a0c6256dc0860
+# move to ruff formatting/linting
+ad4d24e17c9424b17cd8ae65c2def7ecc74e63c1

.github/workflows/release.yml

@@ -11,15 +11,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
-
+          python-version: 3.11
       - name: Install dependencies
         run: |
           sudo apt-get update

.github/workflows/test.yml

@@ -7,35 +7,58 @@ on:
     branches:
       - master
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/checkout@v3
+        - name: Lint with ruff
+          run: |
+            pipx install ruff==0.4.10
+            ruff format . --check && ruff check .
+        - name: Check twine
+          run: |
+            python -m pip install twine poetry rstcheck
+            poetry build
+            rstcheck README.rst
+            twine check dist/*
+
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12.0-rc.3" ]
-        django: [ "3.2", "4.1", "4.2", "5.0a1"]
+        python-version:
+        - "3.9"
+        - "3.10"
+        - "3.11"
+        - "3.12"
+        django:
+        - "4.2"
+        - "5.0"
+        - "5.1"
+        - "5.2"
+        - "6.0"
         exclude:
-          # django 5.0 does not support 3.8 and 3.9
-          - python-version: "3.8"
-            django: "5.0a1" 
+          # django 5.2 does not support 3.9
           - python-version: "3.9"
-            django: "5.0a1" 
-          # django 4.2 does not support 3.12
-          - python-version: "3.12.0-rc.3"
-            django: "4.2" 
-          # django 4.1 does not support 3.12
-          - python-version: "3.12.0-rc.3"
-            django: "4.1" 
-          # django 3.2 does not support 3.11 and 3.12
+            django: "5.2"
+          # django 5.1 does not support 3.9
+          - python-version: "3.9"
+            django: "5.1"
+          # django 5.0 does not support 3.9
+          - python-version: "3.9"
+            django: "5.0"
+          # django 4.2 does not support 3.13
+          - python-version: "3.13"
+            django: "4.2"
+          # django 6.0 does not support earlier than 3.12
+          - python-version: "3.9"
+            django: "6.0"
+          - python-version: "3.10"
+            django: "6.0"
           - python-version: "3.11"
-            django: "3.2" 
-          - python-version: "3.12.0-rc.3"
-            django: "3.2" 
+            django: "6.0"
 
     services:
-      disque:
-        image: efrecon/disque:1.0-rc1
-        ports:
-          - '7711:7711/tcp'
       mongodb:
         image: mongo
         ports:
@@ -64,8 +87,8 @@ jobs:
         - name: Install dependencies with Django ${{ matrix.django }}
           run: |
             python -m pip install --upgrade pip
-            pip install poetry
-            poetry add "django==${{ matrix.django }}" --python=${{ matrix.python-version }}
+            pip install poetry==1.8.5
+            poetry add "django~=${{ matrix.django }}a1" --python=${{ matrix.python-version }}
             poetry install -E testing
         - name: Run Tests
           run: |
@@ -85,16 +108,12 @@ jobs:
   finish:
     needs: test
     runs-on: ubuntu-latest
-    container: python:3-slim
+    container: python:3.11-bookworm
     steps:
       - name: Upload to coveralls
         run: |
           python -m pip install --upgrade pip
-          python -m pip install coveralls flake8 black
+          python -m pip install coveralls
           coveralls --service=github --finish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Check flake8/black
-        run: |
-          flake8 .
-          black --check .

CHANGELOG.md

@@ -1,5 +1,85 @@
 # Changelog
 
+## [v1.9.0](https://github.com/django-q2/django-q2/tree/v1.9.0) (2025-12-04)
+
+- Django 6.0 support https://github.com/django-q2/django-q2/pull/307
+- Fix to make tests work with redis-py > 5 https://github.com/django-q2/django-q2/pull/282
+
+## [v1.8.0](https://github.com/django-q2/django-q2/tree/v1.8.0) (2025-04-25)
+
+- Delete deprecated imports and unwrapping https://github.com/django-q2/django-q2/pull/261
+- Remove disque from CI https://github.com/django-q2/django-q2/pull/270
+- Avoid creating task run on Scheduler creation https://github.com/django-q2/django-q2/pull/268
+- Update tested versions, add python 3.13 support and django 5.2 support. Drop python 3.8 support https://github.com/django-q2/django-q2/pull/271
+- Move timeout function from ORM broker into class to allow easy customization https://github.com/django-q2/django-q2/pull/274
+
+## [v1.7.6](https://github.com/django-q2/django-q2/tree/v1.7.6) (2025-01-12)
+
+- Make poetry version fixed in CI https://github.com/django-q2/django-q2/pull/260
+
+## [v1.7.5](https://github.com/django-q2/django-q2/tree/v1.7.5) (2025-01-12)
+
+- perf: avoid checking success tasks when save limit is disabled https://github.com/django-q2/django-q2/pull/255
+- Fix install path for CHANGELOG.md https://github.com/django-q2/django-q2/pull/258
+
+## [v1.7.4](https://github.com/django-q2/django-q2/tree/v1.7.4) (2024-11-03)
+
+- Decrease the MAX_RSS set in test_cluster::test_max_rss https://github.com/django-q2/django-q2/pull/240
+- Fix BROKER_CLASS monkeypatch in test_brokers https://github.com/django-q2/django-q2/pull/239
+- Fix 'receive_message_wait_time_seconds' SQS broker management https://github.com/django-q2/django-q2/pull/243
+
+## [v1.7.3](https://github.com/django-q2/django-q2/tree/v1.7.3) (2024-10-15)
+
+- Catch missing SEGALRM with AttributeError instead of ValueError https://github.com/django-q2/django-q2/pull/223
+- Refactor timeout handling to handle AttributeError and ValueError for Windows users https://github.com/django-q2/django-q2/pull/234
+- Fix type check for args in scheduler.py E721 https://github.com/django-q2/django-q2/pull/233
+- Only trigger prometheus if configured https://github.com/django-q2/django-q2/pull/231
+- Fix missing ack_id when finishing task https://github.com/django-q2/django-q2/pull/224
+
+
+## [v1.7.2](https://github.com/django-q2/django-q2/tree/v1.7.2) (2024-09-09)
+
+- Fix twine check
+
+## [v1.7.1](https://github.com/django-q2/django-q2/tree/v1.7.1) (2024-09-08)
+
+- Fixed date of v1.7.0
+- Fixed README.rst formatting which is blocking release of latest version
+
+## [v1.7.0](https://github.com/django-q2/django-q2/tree/v1.7.0) (2024-09-08)
+
+**Merged pull requests:**
+
+- Remove support for Django 3.2 and 4.1 https://github.com/django-q2/django-q2/pull/183
+- Fix max attempts for value 1 https://github.com/django-q2/django-q2/pull/185
+- Replace black/isort with ruff https://github.com/django-q2/django-q2/pull/188
+- Fix repeating task after timeout https://github.com/django-q2/django-q2/pull/184
+- fix: Oracle ORM backend compatibility #180 https://github.com/django-q2/django-q2/pull/186
+- chore: Update CI for Django 4.2 Python 3.12 support https://github.com/django-q2/django-q2/pull/208
+- chore: Add Support Django 5.1 https://github.com/django-q2/django-q2/pull/207
+- Call mark_process_dead on worker pid if prometheus_client is installed https://github.com/django-q2/django-q2/pull/212
+- Add example project https://github.com/django-q2/django-q2/pull/215
+- Add index on succeeded tasks https://github.com/django-q2/django-q2/pull/164
+
+## [v1.6.2](https://github.com/django-q2/django-q2/tree/v1.6.2) (2024-03-05)
+
+**Merged pull requests:**
+
+- Allow different broker on chain https://github.com/django-q2/django-q2/pull/156
+- Fix formatting issues in README.rst https://github.com/django-q2/django-q2/pull/159
+- Update docs to add cluster option to the async_task https://github.com/django-q2/django-q2/pull/157
+- Update release/test dependencies https://github.com/django-q2/django-q2/pull/147
+- Fix for Negative Repeat Count in Scheduler https://github.com/django-q2/django-q2/pull/146
+- Update django.po https://github.com/django-q2/django-q2/pull/138
+- Use importerror for b62_decode and avoid deprecation notification https://github.com/django-q2/django-q2/pull/134
+- Specify build system in pyproject.toml https://github.com/django-q2/django-q2/pull/131
+
+## [v1.6.1](https://github.com/django-q2/django-q2/tree/v1.6.1) (2023-10-13)
+
+**Merged pull requests:**
+
+- Fix strict versions for python/django https://github.com/django-q2/django-q2/pull/130
+
 ## [v1.6.0](https://github.com/django-q2/django-q2/tree/v1.6.0) (2023-10-12)
 
 **Merged pull requests:**

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.12
 
 ENV PYTHONUNBUFFERED 1
 RUN mkdir -p /app

Dockerfile.dev

@@ -7,11 +7,15 @@ ENV PYTHONUNBUFFERED 1
 # Sets the default shell to bash
 ENV SHELL /bin/bash
 
+RUN set -ex \
+    && apt update \
+    && apt-get install gcc python3-dev --yes
+
 # Upgrades pip
-RUN pip install --upgrade pip 
+RUN pip install -U pip setuptools
 
 # Install poetry
-RUN pip install poetry
+RUN pip install poetry==1.8.2
 
 WORKDIR /app
 

Makefile

@@ -0,0 +1,21 @@
+dev:
+	docker compose -f web-docker-compose.yaml up
+
+test:
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run pytest
+
+shell:
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py shell
+
+makemigrations:
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py makemigrations
+
+migrate:
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py migrate
+
+createsuperuser:
+	docker compose -f web-docker-compose.yaml run --rm web python manage.py createsuperuser
+
+format:
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run ruff format .
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run ruff check . --fix

README.rst

@@ -1,11 +1,9 @@
 A multiprocessing distributed task queue for Django
 ---------------------------------------------------
 
-|image0| |image1| |docs| |downloads|
+|image0| |image1| |downloads|
 
-::
-
-    Django Q2 is a fork of Django Q. Big thanks to Ilan Steemers for starting this project. Unfortunately, development has stalled since June 2021. Django Q2 is the new updated version of Django Q, with dependencies updates, docs updates and several bug fixes. Original repository: https://github.com/Koed00/django-q
+Django Q2 is a fork of Django Q. Big thanks to Ilan Steemers for starting this project. Unfortunately, development has stalled since June 2021. Django Q2 is the new updated version of Django Q, with dependencies updates, docs updates and several bug fixes. Original repository: https://github.com/Koed00/django-q
 
 Features
 ~~~~~~~~
@@ -37,10 +35,13 @@ See the `changelog <https://github.com/GDay/django-q2/blob/master/CHANGELOG.md>`
 Requirements
 ~~~~~~~~~~~~
 
--  `Django <https://www.djangoproject.com>`__ > = 3.2
+-  `Django <https://www.djangoproject.com>`__ > = 4.2
 -  `Django-picklefield <https://github.com/gintas/django-picklefield>`__
 
-Tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 3.2.X, 4.1.X, 4.2.X and 5.0.X
+Tested with:
+
+* Python 3.9 to 3.13.
+* Django 4.2 to 6.0.
 
 Brokers
 ~~~~~~~
@@ -73,7 +74,6 @@ Installation
 
 Read the full documentation at `https://django-q2.readthedocs.org <https://django-q2.readthedocs.org>`__
 
-
 Configuration
 ~~~~~~~~~~~~~
 
@@ -104,10 +104,7 @@ For full configuration options, see the `configuration documentation <https://dj
 Management Commands
 ~~~~~~~~~~~~~~~~~~~
 
-::
-
-    For the management commands to work, you will need to install Blessed: <https://github.com/jquast/blessed>
-
+For the management commands to work, you will need to install Blessed: <https://github.com/jquast/blessed>
 
 Start a cluster with::
 
@@ -159,7 +156,6 @@ Use `async_task` from your code to quickly offload tasks:
 
 For more info see `Tasks <https://django-q2.readthedocs.org/en/latest/tasks.html>`__
 
-
 Schedule
 ~~~~~~~~
 
@@ -204,6 +200,22 @@ Admin page or directly from your code:
 
 For more info check the `Schedules <https://django-q2.readthedocs.org/en/latest/schedules.html>`__ documentation.
 
+Development
+~~~~~~~~~~~
+
+There is an example project that you can use to develop with. Docker (compose) is being used to set everything up.
+Please note that you will have to restart the django-q container when changes have been made to tasks or django-q.
+You can start the example project with:
+
+.. code:: bash
+
+    make dev
+
+Create a superuser with:
+
+.. code:: bash
+
+    make createsuperuser
 
 Testing
 ~~~~~~~
@@ -212,7 +224,7 @@ Running tests is easy with docker compose, it will also start the necessary data
 
 .. code:: bash
 
-    docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run pytest
+    make test
 
 Locale
 ~~~~~~
@@ -220,12 +232,6 @@ Locale
 Currently available in English, German, Turkish, and French.
 Translation pull requests are always welcome.
 
-Todo
-~~~~
-
--  Better tests and coverage
--  Less dependencies?
-
 Acknowledgements
 ~~~~~~~~~~~~~~~~
 
@@ -242,9 +248,5 @@ Acknowledgements
    :target: https://github.com/GDay/django-q2/actions?query=workflow%3Atests
 .. |image1| image:: https://coveralls.io/repos/github/GDay/django-q2/badge.svg?branch=master
    :target: https://coveralls.io/github/GDay/django-q2?branch=master
-.. |docs| image:: https://readthedocs.org/projects/docs/badge/?version=latest
-    :alt: Documentation Status
-    :scale: 100
-    :target: https://django-q2.readthedocs.org/
 .. |downloads| image:: https://img.shields.io/pypi/dm/django-q2
    :target: https://img.shields.io/pypi/dm/django-q2

containers/localstack/init.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Note that this file needs to have the executable bit set for it to work with later localstack implementations.
+
+export DEFAULT_REGION=us-west-2
+
+create_sqs() {
+    QUEUE_NAME="$1"
+    TIMEOUT=${2:-60}
+    DL_QUEUE_URL=$(awslocal sqs create-queue --queue-name "dl-$QUEUE_NAME" --query QueueUrl --output text)
+    echo ">>> Created $DL_QUEUE_URL queue!"
+    DL_QUEUE_ARN=$(awslocal sqs get-queue-attributes --queue-url "$DL_QUEUE_URL" --attribute-names QueueArn --query Attributes.QueueArn --output text)
+    awslocal sqs create-queue --queue-name "$QUEUE_NAME" --attributes '{
+        "RedrivePolicy": "{\"deadLetterTargetArn\": \"'"$DL_QUEUE_ARN"'\",\"maxReceiveCount\":\"3\"}",
+        "VisibilityTimeout": "'"$TIMEOUT"'"
+    }'
+}
+
+# Create SQS queues
+create_sqs testing

django_q/__init__.py

@@ -1,8 +1,3 @@
-import django
-
-VERSION = (1, 6, 0)
-
-if django.VERSION < (3, 2):
-    default_app_config = "django_q.apps.DjangoQConfig"
+VERSION = (1, 9, 0)
 
 __all__ = ["conf", "cluster", "models", "tasks"]

django_q/admin.py

@@ -1,4 +1,5 @@
 """Admin module for Django."""
+
 from django.contrib import admin
 from django.db.models.expressions import OuterRef, Subquery
 from django.urls import reverse
@@ -137,7 +138,7 @@ class ScheduleAdmin(admin.ModelAdmin):
                 url = reverse("admin:django_q_success_change", args=(obj.task_id,))
             else:
                 url = reverse("admin:django_q_failure_change", args=(obj.task_id,))
-            return format_html(f'<a href="{url}">[{obj.task_name}]</a>')
+            return format_html('<a href="{}">[{}]</a>', url, obj.task_name)
         return None
 
     get_last_run.allow_tags = True

django_q/brokers/__init__.py

@@ -182,7 +182,7 @@ def get_broker(list_key: str = None) -> Broker:
 
         return ironmq.IronMQBroker(list_key=list_key)
     # SQS
-    elif type(Conf.SQS) == dict:
+    elif isinstance(Conf.SQS, dict):
         from django_q.brokers import aws_sqs
 
         return aws_sqs.Sqs(list_key=list_key)

django_q/brokers/aws_sqs.py

@@ -1,3 +1,5 @@
+import copy
+
 from boto3 import Session
 from botocore.client import ClientError
 
@@ -78,15 +80,15 @@ class Sqs(Broker):
 
     @staticmethod
     def get_connection(list_key: str = None) -> Session:
-        config = Conf.SQS
-        if "aws_region" in config:
-            config["region_name"] = config["aws_region"]
-            del config["aws_region"]
+        config_cloned = copy.deepcopy(Conf.SQS)
+        if "aws_region" in config_cloned:
+            config_cloned["region_name"] = config_cloned["aws_region"]
+            del config_cloned["aws_region"]
 
-        if "receive_message_wait_time_seconds" in config:
-            del config["receive_message_wait_time_seconds"]
+        if "receive_message_wait_time_seconds" in config_cloned:
+            del config_cloned["receive_message_wait_time_seconds"]
 
-        return Session(**config)
+        return Session(**config_cloned)
 
     def get_queue(self):
         self.sqs = self.connection.resource("sqs")

django_q/brokers/orm.py

@@ -10,10 +10,6 @@ from django_q.conf import Conf, logger
 from django_q.models import OrmQ
 
 
-def _timeout():
-    return timezone.now() + timedelta(seconds=Conf.RETRY)
-
-
 class ORM(Broker):
     @staticmethod
     def get_connection(list_key: str = None):
@@ -28,6 +24,9 @@ class ORM(Broker):
             logger.debug("Broker in an atomic transaction")
         return OrmQ.objects.using(Conf.ORM)
 
+    def timeout(self, task):
+        return timezone.now() + timedelta(seconds=Conf.RETRY)
+
     def queue_size(self) -> int:
         return (
             self.get_connection()
@@ -75,7 +74,7 @@ class ORM(Broker):
                 if (
                     self.get_connection()
                     .filter(id=task.id, lock=task.lock)
-                    .update(lock=_timeout())
+                    .update(lock=self.timeout(task))
                 ):
                     task_list.append((task.pk, task.payload))
                 # else don't process, as another cluster has been faster than us on

django_q/cluster.py

@@ -1,4 +1,5 @@
 # Standard
+import os
 import signal
 import socket
 import uuid
@@ -21,7 +22,14 @@ from django.utils.translation import gettext_lazy as _
 
 # Local
 from django_q.brokers import Broker, get_broker
-from django_q.conf import Conf, get_ppid, logger, psutil, setproctitle
+from django_q.conf import (
+    Conf,
+    get_ppid,
+    logger,
+    prometheus_multiprocess,
+    psutil,
+    setproctitle,
+)
 from django_q.humanhash import humanize
 from django_q.monitor import monitor
 from django_q.pusher import pusher
@@ -223,6 +231,14 @@ class Sentinel:
                 % {"name": process.name}
             )
         else:
+            # check if prometheus is proper configurated
+            prometheus_path = os.getenv(
+                "PROMETHEUS_MULTIPROC_DIR", os.getenv("prometheus_multiproc_dir")
+            )
+
+            if prometheus_multiprocess and prometheus_path:
+                prometheus_multiprocess.mark_process_dead(process.pid)
+
             self.pool.remove(process)
             self.spawn_worker()
             if process.timer.value == 0:

django_q/conf.py

@@ -38,6 +38,11 @@ try:
 except ModuleNotFoundError:
     setproctitle = None
 
+try:
+    from prometheus_client import multiprocess as prometheus_multiprocess
+except ModuleNotFoundError:
+    prometheus_multiprocess = None
+
 
 class Conf:
     """
@@ -263,7 +268,6 @@ if not logger.hasHandlers():
 
 # Error Reporting Interface
 class ErrorReporter:
-
     # initialize with iterator of reporters (better name, targets?)
     def __init__(self, reporters):
         self.targets = [target for target in reporters]

django_q/core_signing.py

@@ -2,17 +2,23 @@ import datetime
 import time
 import zlib
 
-import django
-from django.core.signing import BadSignature, JSONSerializer, SignatureExpired
+from django.core.signing import (
+    BadSignature,
+    JSONSerializer,
+    SignatureExpired,
+    b64_decode,
+    dumps,
+)
 from django.core.signing import Signer as Sgnr
 from django.core.signing import TimestampSigner as TsS
-from django.core.signing import b64_decode, dumps
 
-if django.VERSION < (5, 0):
-    from django.utils.baseconv import base62
-    b62_decode = base62.decode
-else:
+try:
     from django.core.signing import b62_decode
+except ImportError:
+    # fallback for django 3.x
+    from django.utils.baseconv import base62
+
+    b62_decode = base62.decode
 
 from django.utils.crypto import constant_time_compare
 from django.utils.encoding import force_bytes, force_str

django_q/exceptions.py

@@ -0,0 +1,7 @@
+class TimeoutException(SystemExit):
+    """
+    Exception for when a worker takes too long to complete a task
+    Raising SystemExit will make sure the function terminates gracefully.
+    """
+
+    pass

django_q/humanhash.py

@@ -4,6 +4,7 @@ humanhash: Human-readable representations of digests.
 The simplest ways to use this module are the :func:`humanize` and :func:`uuid`
 functions. For tighter control over the output, see :class:`HumanHasher`.
 """
+
 import operator
 import uuid as uuidlib
 from argparse import ArgumentError
@@ -270,7 +271,6 @@ DEFAULT_WORDLIST = (
 
 
 class HumanHasher:
-
     """
     Transforms hex digests to human-readable strings.
 
@@ -290,7 +290,6 @@ class HumanHasher:
         self.wordlist = wordlist
 
     def humanize(self, hexdigest, words=4, separator="-"):
-
         """
         Humanize a given hexadecimal digest.
 
@@ -314,7 +313,6 @@ class HumanHasher:
 
     @staticmethod
     def compress(bytes, target):
-
         """
         Compress a list of byte values to a fixed target length.
 
@@ -353,7 +351,6 @@ class HumanHasher:
         return checksums
 
     def uuid(self, **params):
-
         """
         Generate a UUID with a human-readable representation.
 

django_q/locale/fr/LC_MESSAGES/django.po

@@ -303,7 +303,7 @@ msgstr "Bimestriel"
 
 #: models.py:194
 msgid "Quarterly"
-msgstr "Tous les quart-d’heure"
+msgstr "Trimestriel"
 
 #: models.py:195
 msgid "Yearly"

django_q/migrations/0001_initial.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = []
 
     operations = [

django_q/migrations/0002_auto_20150630_1624.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0001_initial"),
     ]

django_q/migrations/0003_auto_20150708_1326.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0002_auto_20150630_1624"),
     ]

django_q/migrations/0004_auto_20150710_1043.py

@@ -2,7 +2,6 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0003_auto_20150708_1326"),
     ]

django_q/migrations/0005_auto_20150718_1506.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0004_auto_20150710_1043"),
     ]

django_q/migrations/0006_auto_20150805_1817.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0005_auto_20150718_1506"),
     ]

django_q/migrations/0007_ormq.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0006_auto_20150805_1817"),
     ]

django_q/migrations/0008_auto_20160224_1026.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0007_ormq"),
     ]

django_q/migrations/0009_auto_20171009_0915.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0008_auto_20160224_1026"),
     ]

django_q/migrations/0010_auto_20200610_0856.py

@@ -3,7 +3,6 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0009_auto_20171009_0915"),
     ]

django_q/migrations/0011_auto_20200628_1055.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0010_auto_20200610_0856"),
     ]

django_q/migrations/0012_auto_20200702_1608.py

@@ -6,7 +6,6 @@ import django_q.models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0011_auto_20200628_1055"),
     ]

django_q/migrations/0013_task_attempt_count.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0012_auto_20200702_1608"),
     ]

django_q/migrations/0014_schedule_cluster.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0013_task_attempt_count"),
     ]

django_q/migrations/0015_alter_schedule_schedule_type.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0014_schedule_cluster"),
     ]

django_q/migrations/0016_schedule_intended_date_kwarg.py

@@ -6,7 +6,6 @@ import django_q.models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0015_alter_schedule_schedule_type"),
     ]

django_q/migrations/0017_task_cluster_alter.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0016_schedule_intended_date_kwarg"),
     ]

django_q/migrations/0018_task_success_index.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.7 on 2024-03-05 17:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("django_q", "0017_task_cluster_alter"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="task",
+            index=models.Index(
+                condition=models.Q(("success", True)),
+                fields=["group", "name", "func"],
+                name="success_index",
+            ),
+        ),
+    ]

django_q/models.py

@@ -2,23 +2,21 @@ from datetime import datetime, timedelta
 from keyword import iskeyword
 
 # Django
-from django import get_version
 from django.core.exceptions import ValidationError
 from django.db import models
+from django.db.models import Q
 from django.template.defaultfilters import truncatechars
 from django.urls import reverse
 from django.utils import timezone
 from django.utils.functional import cached_property
 from django.utils.html import format_html
-from django.utils.timezone import is_aware
 from django.utils.translation import gettext_lazy as _
 
 # External
 from picklefield import PickledObjectField
-from picklefield.fields import dbsafe_decode
 
 # Local
-from django_q.conf import Conf, croniter
+from django_q.conf import croniter
 from django_q.signing import SignedPackage
 from django_q.utils import add_months, add_years, localtime
 
@@ -59,7 +57,7 @@ class Task(models.Model):
                 .exclude(success=False)
                 .values_list("result", flat=True)
             )
-        return decode_results(values)
+        return values
 
     def group_result(self, failures=False):
         if self.group:
@@ -112,6 +110,13 @@ class Task(models.Model):
     class Meta:
         app_label = "django_q"
         ordering = ["-stopped"]
+        indexes = [
+            models.Index(
+                name="success_index",
+                fields=["group", "name", "func"],
+                condition=Q(success=True),
+            ),
+        ]
 
 
 class SuccessManager(models.Manager):
@@ -291,12 +296,18 @@ class Schedule(models.Model):
                 url = reverse("admin:django_q_success_change", args=(task.id,))
             else:
                 url = reverse("admin:django_q_failure_change", args=(task.id,))
-            return format_html(f'<a href="{url}">[{task.name}]</a>')
+            return format_html('<a href="{}">[{}]</a>', url, task.name)
         return None
 
     def __str__(self):
         return self.func
 
+    def save(self, *args, **kwargs):
+        if self.pk is None and self.schedule_type == self.CRON:
+            self.next_run = self.calculate_next_run()
+
+        return super().save(*args, **kwargs)
+
     success.boolean = True
     success.short_description = _("success")
     last_run.allow_tags = True
@@ -349,11 +360,3 @@ class OrmQ(models.Model):
         app_label = "django_q"
         verbose_name = _("Queued task")
         verbose_name_plural = _("Queued tasks")
-
-
-# Backwards compatibility for Django 1.7
-def decode_results(values):
-    if get_version().split(".")[1] == "7":
-        # decode values in 1.7
-        return [dbsafe_decode(v) for v in values]
-    return values

django_q/monitor.py

@@ -2,8 +2,8 @@ from multiprocessing.process import current_process
 from multiprocessing.queues import Queue
 
 from django import core, db
-from django.utils.translation import gettext_lazy as _
 from django.apps.registry import apps
+from django.utils.translation import gettext_lazy as _
 
 try:
     apps.check_apps_ready()
@@ -107,35 +107,32 @@ def save_task(task, broker: Broker):
                 value = get_func_repr(value)
             filters[Conf.SAVE_LIMIT_PER] = value
 
-        with db.transaction.atomic(using=db.router.db_for_write(Success)):
-            last = Success.objects.filter(**filters).select_for_update().last()
-            if (
-                task["success"]
-                and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
-            ):
-                last.delete()
+        # check if we should clean the success tasks
+        if Conf.SAVE_LIMIT > 0:
+            with db.transaction.atomic(using=db.router.db_for_write(Success)):
+                success_tasks_qs = Success.objects.filter(**filters)
+                success_tasks_pks = [
+                    success_task.pk
+                    for success_task in success_tasks_qs.select_for_update()
+                ]
+                if task["success"] and len(success_tasks_pks) >= Conf.SAVE_LIMIT:
+                    success_tasks_qs.last().delete()
 
         # check if this task has previous results
         try:
-            existing_task = Task.objects.get(id=task["id"], name=task["name"])
+            task_obj = Task.objects.get(id=task["id"], name=task["name"])
             # only update the result if it hasn't succeeded yet
-            if not existing_task.success:
-                existing_task.stopped = task["stopped"]
-                existing_task.result = task["result"]
-                existing_task.success = task["success"]
-                existing_task.attempt_count = existing_task.attempt_count + 1
-                existing_task.save()
-
-            if (
-                Conf.MAX_ATTEMPTS > 0
-                and existing_task.attempt_count >= Conf.MAX_ATTEMPTS
-            ):
-                broker.acknowledge(task["ack_id"])
+            if not task_obj.success:
+                task_obj.stopped = task["stopped"]
+                task_obj.result = task["result"]
+                task_obj.success = task["success"]
+                task_obj.attempt_count = task_obj.attempt_count + 1
+                task_obj.save()
 
         except Task.DoesNotExist:
             # convert func to string
             func = get_func_repr(task["func"])
-            Task.objects.create(
+            task_obj = Task.objects.create(
                 id=task["id"],
                 name=task["name"],
                 func=func,
@@ -150,6 +147,14 @@ def save_task(task, broker: Broker):
                 success=task["success"],
                 attempt_count=1,
             )
+
+        if (
+            Conf.MAX_ATTEMPTS > 0
+            and task_obj.attempt_count >= Conf.MAX_ATTEMPTS
+            and task.get("ack_id")
+        ):
+            broker.acknowledge(task["ack_id"])
+
     except Exception:
         logger.exception("Could not save task result")
 

django_q/monitor_terminal.py

@@ -397,9 +397,7 @@ def memory(run_once=False, workers=False, broker=None):
                     2,
                 )
                 # memory available (MB)
-                memory_available = round(
-                    psutil.virtual_memory().available / 1024**2, 2
-                )
+                memory_available = round(psutil.virtual_memory().available / 1024**2, 2)
                 if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
                     MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
                     MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()

django_q/pusher.py

@@ -4,8 +4,8 @@ from multiprocessing.queues import Queue
 from time import sleep
 
 from django import core
-from django.utils.translation import gettext_lazy as _
 from django.apps.registry import apps
+from django.utils.translation import gettext_lazy as _
 
 try:
     apps.check_apps_ready()
@@ -58,9 +58,9 @@ def pusher(task_queue: Queue, event: Event, broker: Broker = None):
                     logger.exception("Failed to push task to queue")
                     broker.fail(ack_id)
                     continue
-                task[
-                    "cluster"
-                ] = Conf.CLUSTER_NAME  # save actual cluster name to orm task table
+                task["cluster"] = (
+                    Conf.CLUSTER_NAME
+                )  # save actual cluster name to orm task table
                 task["ack_id"] = ack_id
                 task_queue.put(task)
             logger.debug(

django_q/queues.py

@@ -2,6 +2,7 @@
 The code is derived from
 https://github.com/althonos/pronto/commit/3384010dfb4fc7c66a219f59276adef3288a886b
 """
+
 import multiprocessing
 import multiprocessing.queues
 import sys

django_q/scheduler.py

@@ -2,11 +2,10 @@ import ast
 from multiprocessing.process import current_process
 
 from django import core, db
+from django.apps.registry import apps
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
-from django.apps.registry import apps
-
 try:
     apps.check_apps_ready()
 except core.exceptions.AppRegistryNotReady:
@@ -66,7 +65,7 @@ def scheduler(broker: Broker = None):
                 if s.args:
                     args = ast.literal_eval(s.args)
                     # single value won't eval to tuple, so:
-                    if type(args) != tuple:
+                    if type(args) is not tuple:
                         args = (args,)
                 q_options = kwargs.get("q_options", {})
                 if s.intended_date_kwarg:
@@ -82,7 +81,15 @@ def scheduler(broker: Broker = None):
                             break
 
                     s.next_run = next_run
-                    s.repeats += -1
+
+                    # Little Fix for already broken numbers
+                    if s.repeats < -1:
+                        s.repeats = -1
+
+                    # Check if the value is not zero
+                    if s.repeats > 0:
+                        s.repeats -= 1
+
                 # send it to the cluster; any cluster name is allowed in multi-queue scenarios
                 # because `broker_name` is confusing, using `cluster` name is recommended and takes precedence
                 q_options["cluster"] = s.cluster or q_options.get(

django_q/signing.py

@@ -1,4 +1,5 @@
 """Package signing."""
+
 import pickle
 
 from django_q import core_signing as signing

django_q/tasks.py

@@ -1,4 +1,5 @@
 """Provides task functionality."""
+
 # Standard
 from multiprocessing import Value
 from time import sleep, time
@@ -569,10 +570,12 @@ class Chain:
     A sequential chain of tasks
     """
 
-    def __init__(self, chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC):
+    def __init__(
+        self, chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC, broker=None
+    ):
         self.chain = chain or []
         self.group = group or ""
-        self.broker = get_broker()
+        self.broker = broker or get_broker()
         self.cached = cached
         self.sync = sync
         self.started = False
@@ -729,17 +732,14 @@ class AsyncTask:
         return self.id
 
     def result(self, wait=0):
-
         if self.started:
             return result(self.id, wait=wait, cached=self.cached)
 
     def fetch(self, wait=0):
-
         if self.started:
             return fetch(self.id, wait=wait, cached=self.cached)
 
     def result_group(self, failures=False, wait=0, count=None):
-
         if self.started and self.group:
             return result_group(
                 self.group,
@@ -750,7 +750,6 @@ class AsyncTask:
             )
 
     def fetch_group(self, failures=True, wait=0, count=None):
-
         if self.started and self.group:
             return fetch_group(
                 self.group,

django_q/tests/settings.py

@@ -116,7 +116,7 @@ CACHES = {
         "LOCATION": f"redis://{REDIS_HOST}:6379/0",
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
-            "PARSER_CLASS": "redis.connection.HiredisParser",
+            "PARSER_CLASS": "redis.connection.DefaultParser",
         },
     }
 }

django_q/tests/test_brokers.py

@@ -51,7 +51,7 @@ def test_redis(monkeypatch):
 
 
 def test_custom(monkeypatch):
-    monkeypatch.setattr(Conf, "BROKER_CLASS", "brokers.redis_broker.Redis")
+    monkeypatch.setattr(Conf, "BROKER_CLASS", "django_q.brokers.redis_broker.Redis")
     broker = get_broker()
     assert broker.ping() is True
     assert broker.info() is not None
@@ -124,7 +124,7 @@ def test_ironmq(monkeypatch):
 @pytest.mark.skipif(
     not os.getenv("AWS_ACCESS_KEY_ID"), reason="requires AWS credentials"
 )
-def canceled_sqs(monkeypatch):
+def test_sqs(monkeypatch):
     monkeypatch.setattr(
         Conf,
         "SQS",
@@ -132,11 +132,13 @@ def canceled_sqs(monkeypatch):
             "aws_region": os.getenv("AWS_REGION"),
             "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"),
             "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"),
-            "receive_message_wait_time_seconds": 20,
+            "receive_message_wait_time_seconds": 5,
         },
     )
     # check broker
-    broker = get_broker(list_key=uuid()[0])
+    broker = get_broker(list_key="testing")
+    assert "receive_message_wait_time_seconds" in Conf.SQS
+    assert "aws_region" in Conf.SQS
     assert broker.ping() is True
     assert broker.info() is not None
     assert broker.queue_size() == 0
@@ -173,7 +175,7 @@ def canceled_sqs(monkeypatch):
     broker.enqueue("test")
     while task is None:
         task = broker.dequeue()[0]
-    broker.fail(task[0])
+    broker.fail(task[0][0])
     # bulk test
     for _ in range(10):
         broker.enqueue("test")

django_q/tests/test_cluster.py

@@ -399,7 +399,7 @@ def test_timeout_task_finishes(broker, cluster_config_timeout, async_task_kwargs
 
 
 @pytest.mark.django_db
-def test_recycle(broker, monkeypatch):
+def test_recycle(broker, monkeypatch, django_assert_num_queries):
     # set up the Sentinel
     broker.list_key = "test_recycle_test:q"
     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
@@ -432,7 +432,8 @@ def test_recycle(broker, monkeypatch):
     monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
     result_queue.put("STOP")
     # run monitor
-    monitor(result_queue)
+    with django_assert_num_queries(12):
+        monitor(result_queue)
     assert Success.objects.count() == Conf.SAVE_LIMIT
     broker.delete_queue()
 
@@ -487,7 +488,7 @@ def test_max_rss(broker, monkeypatch):
     stop_event = Event()
     cluster_id = uuidlib.uuid4()
     # override settings
-    monkeypatch.setattr(Conf, "MAX_RSS", 40000)
+    monkeypatch.setattr(Conf, "MAX_RSS", 20000)
     monkeypatch.setattr(Conf, "WORKERS", 1)
     # set a timer to stop the Sentinel
     threading.Timer(3, stop_event.set).start()

django_q/tests/test_scheduler.py

@@ -473,6 +473,47 @@ def test_scheduler_atomic_must_specify_the_database_based_on_router_redirection(
         mocked_db.atomic.assert_called_with(using="default")
 
 
+@pytest.mark.django_db
+def test_schedule_save_sets_next_run_for_cron():
+    """Ensure Schedule.save() sets next_run correctly for CRON schedules."""
+    cron_expression = "0 12 * * *"  # Executes dialy at 12pm
+    schedule = Schedule(
+        func="math.sqrt",
+        schedule_type=Schedule.CRON,
+        cron=cron_expression,
+    )
+
+    assert schedule.next_run is not None
+    assert schedule.pk is None
+
+    initial_next_run = schedule.next_run
+    schedule.save()
+
+    # After save, next_run must be recalculated based on the CRON expression
+    assert schedule.next_run > initial_next_run
+
+
+@pytest.mark.django_db
+def test_schedule_save_direct_db(broker):
+    """Ensure Schedule.save() updates next_run correctly when created directly in DB."""
+    cron_expression = "0 12 * * *"  # Executes dialy at 12pm
+
+    # Creating schedule directly in database
+    schedule = Schedule.objects.create(
+        func="math.sqrt",
+        schedule_type=Schedule.CRON,
+        cron=cron_expression,
+    )
+
+    # Next run must be defined after execution time
+    assert schedule.next_run is not None
+    assert schedule.next_run > timezone.now()
+
+    scheduler(broker)
+
+    assert broker.queue_size() == 0
+
+
 def test_localtime():
     assert not is_naive(localtime())
 

django_q/timeout.py

@@ -0,0 +1,43 @@
+import signal
+
+from django.utils.translation import gettext_lazy as _
+
+from django_q.conf import logger
+
+from .exceptions import TimeoutException
+
+
+class TimeoutHandler:
+    def __init__(self, timeout: int):
+        self._timeout = timeout
+
+    def raise_timeout_exception(self, signum, frame):
+        raise TimeoutException(
+            f"Task exceeded maximum timeout value ({self._timeout} seconds)"
+        )
+
+    def __enter__(self):
+        # if the timeout is -1, then there is no timeout and the task will always keep running until it's done or manually killed
+        if self._timeout == -1:
+            return
+        try:
+            signal.signal(signal.SIGALRM, self.raise_timeout_exception)
+            signal.alarm(self._timeout)
+        except (
+            ValueError,
+            AttributeError,
+        ):  # AttributeError or ValueError might be raised for Windows users
+            logger.debug(_("SIGALARM is not available on your platform"))
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self._timeout == -1:
+            return
+        """When getting out of the timeout, reset the alarm, so it won't trigger"""
+        try:
+            signal.alarm(0)
+            signal.signal(signal.SIGALRM, signal.SIG_DFL)
+        except (
+            ValueError,
+            AttributeError,
+        ):  # AttributeError or ValueError might be raised for Windows users
+            logger.debug(_("SIGALARM is not available on your platform"))

django_q/worker.py

@@ -5,9 +5,9 @@ from multiprocessing.process import current_process
 from multiprocessing.queues import Queue
 
 from django import core
+from django.apps.registry import apps
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
-from django.apps.registry import apps
 
 try:
     apps.check_apps_ready()
@@ -17,7 +17,9 @@ except core.exceptions.AppRegistryNotReady:
     django.setup()
 
 from django_q.conf import Conf, error_reporter, logger, resource, setproctitle
+from django_q.exceptions import TimeoutException
 from django_q.signals import post_spawn, pre_execute
+from django_q.timeout import TimeoutHandler
 from django_q.utils import close_old_django_connections, get_func_repr
 
 try:
@@ -89,25 +91,37 @@ def worker(
         pre_execute.send(sender="django_q", func=f, task=task)
         # execute the payload
         timer.value = timer_value  # Busy
+        if timer.value != -1:
+            timer.value += 3  # Add buffer so that guard doesn't kill the process on timeout before it gets processed
 
+        timeout_error = False
         try:
             if f is None:
                 # raise a meaningfull error if task["func"] is not a valid function
                 raise ValueError(f"Function {task['func']} is not defined")
-            res = f(*task["args"], **task["kwargs"])
+            with TimeoutHandler(timer_value):
+                res = f(*task["args"], **task["kwargs"])
             result = (res, True)
-        except Exception as e:
+        except (Exception, TimeoutException) as e:
+            if isinstance(e, TimeoutException):
+                timeout_error = True
             result = (f"{e} : {traceback.format_exc()}", False)
             if error_reporter:
                 error_reporter.report()
             if task.get("sync", False):
                 raise
+
         with timer.get_lock():
             # Process result
             task["result"] = result[0]
             task["success"] = result[1]
             task["stopped"] = timezone.now()
             result_queue.put(task)
+            if timeout_error:
+                # force destroy process due to timeout
+                timer.value = 0
+                break
+
             timer.value = -1  # Idle
             if setproctitle:
                 setproctitle.setproctitle(f"qcluster {proc_name} idle")

docs/chain.rst

@@ -49,8 +49,9 @@ Reference
     :param str group: an optional group name.
     :param bool cached: run this against the cache backend
     :param bool sync: execute this inline instead of asynchronous
+    :param broker: an optional broker instance
 
-.. py:class:: Chain(chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC)
+.. py:class:: Chain(chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC, broker=None)
 
     A sequential chain of tasks. Acts as a convenient wrapper for :func:`async_chain`
     You can pass the task chain at construction or you can append individual tasks before running them.
@@ -59,6 +60,8 @@ Reference
         :param str group: an optional group name.
         :param bool cached: run this against the cache backend
         :param bool sync: execute this inline instead of asynchronous
+        :param bool sync: execute this inline instead of asynchronous
+        :param broker: an optional broker instance
 
 
     .. py:method:: append(func, *args, **kwargs)

docs/conf.py

@@ -73,9 +73,9 @@ author = "Ilan Steemers, Stan Triepels"
 # built documents.
 #
 # The short X.Y version.
-version = "1.6"
+version = "1.9"
 # The full version, including alpha/beta/rc tags.
-release = "1.6.0"
+release = "1.9.0"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/configure.rst

@@ -79,6 +79,8 @@ timeout
 The number of seconds a worker is allowed to spend on a task before it's terminated. Defaults to ``None``, meaning it will never time out.
 Set this to something that makes sense for your project. Can be overridden for individual tasks.
 
+Note: for systems that don't have `SIGALRM` available (e.g. Windows), it will not raise an error properly. It will kill the task, but it will keep retrying until it finishes within the given time.
+
 See :ref:`retry` for details how to set values for timeout and retry.
 
 .. _time_zone:

docs/docker-compose.yaml

@@ -1,4 +1,3 @@
-version: "3.0"
 services:
   docs:
     container_name: djangoq2-docs

docs/index.rst

@@ -27,7 +27,7 @@ Features
 -  Rollbar and Sentry support
 
 
-Django Q2 is tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 3.2.x, 4.1.x, 4.2.x and 5.0.x
+Django Q2 is tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 4.2.x and 5.0.x
 
 Currently available in English, German and French.
 

docs/tasks.rst

@@ -75,6 +75,10 @@ broker
 """"""
 A broker instance, in case you want to control your own connections.
 
+cluster
+""""""
+The name of the cluster. Only useful if you are using [alternative queues](https://django-q2.readthedocs.io/en/master/cluster.html#multiple-queues).
+
 task_name
 """""""""
 
@@ -241,7 +245,7 @@ Reference
 ---------
 
 .. py:function:: async_task(func, *args, hook=None, group=None, timeout=None,\
-    save=None, sync=False, cached=False, broker=None, q_options=None, **kwargs)
+    save=None, sync=False, cached=False, broker=None, cluster=None, q_options=None, **kwargs)
 
     Puts a task in the cluster queue
 
@@ -255,6 +259,7 @@ Reference
    :param bool sync: If set to True, async_task will simulate a task execution
    :param cached: Output the result to the cache backend. Bool or timeout in seconds
    :param broker: Optional broker connection from :func:`brokers.get_broker`
+   :param cluster: Optional cluster name if using alternative queues
    :param dict q_options: Options dict, overrides option keywords
    :param dict kwargs: Keyword arguments for the task function
    :returns: The uuid of the task

exampleproject/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for exampleproject project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "exampleproject.settings")
+
+application = get_asgi_application()

exampleproject/settings.py

@@ -0,0 +1,134 @@
+"""
+Django settings for exampleproject project.
+
+Generated by 'django-admin startproject' using Django 5.1.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/5.1/ref/settings/
+"""
+
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = "django-insecure-)oouh93bjg+c=b!l5-*w)7et1l+!3nmp223rr^1r4v#jn&ow3f"
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ["*"]
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "django_q",
+]
+
+MIDDLEWARE = [
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+]
+
+ROOT_URLCONF = "exampleproject.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = "exampleproject.wsgi.application"
+
+
+# Database
+# https://docs.djangoproject.com/en/5.1/ref/settings/#databases
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": BASE_DIR / "db.sqlite3",
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/5.1/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/5.1/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+
+TIME_ZONE = "UTC"
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/5.1/howto/static-files/
+
+STATIC_URL = "static/"
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+Q_CLUSTER = {
+    "name": "DjangORM",
+    "workers": 4,
+    "timeout": 90,
+    "retry": 120,
+    "queue_limit": 50,
+    "bulk": 10,
+    "orm": "default",
+}

exampleproject/urls.py

@@ -0,0 +1,27 @@
+"""
+URL configuration for exampleproject project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/5.1/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+
+from django.contrib import admin
+from django.urls import path
+
+from exampleproject import views
+
+urlpatterns = [
+    path("admin/", admin.site.urls),
+    path("new_task/", views.add_task, name="add_task"),
+    path("result/<slug:task_id>/", views.get_result, name="get_result"),
+]

exampleproject/views.py

@@ -0,0 +1,31 @@
+import time
+
+from django.http import HttpResponse
+from django.urls import reverse
+
+from django_q import tasks
+
+
+# internal function to be called with django_q
+def new_task(run_for_minutes):
+    print("Task started")
+    time.sleep(run_for_minutes)
+    print("Task done")
+    return True
+
+
+def add_task(request):
+    task_id = tasks.async_task(new_task, 5)
+    result_url = reverse("get_result", args=[task_id])
+    return HttpResponse(
+        f"Added async task with <a href='{result_url}'>Go to results</a>"
+    )
+
+
+def get_result(request, task_id):
+    task = tasks.fetch(task_id)
+    if not task:
+        msg = "Task running... please refresh after some time"
+    else:
+        msg = f"Async task result: {task.result}"
+    return HttpResponse(msg)

exampleproject/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for exampleproject project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "exampleproject.settings")
+
+application = get_wsgi_application()

pyproject.toml

@@ -1,6 +1,10 @@
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
 [tool.poetry]
 name = "django-q2"
-version = "1.6.0"
+version = "1.9.0"
 packages = [
     { include = "django_q" },
 ]
@@ -26,16 +30,19 @@ classifiers = [
     'Operating System :: MacOS',
     'Programming Language :: Python',
     'Programming Language :: Python :: 3',
-    'Programming Language :: Python :: 3.8',
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
     'Programming Language :: Python :: 3.12',
+    'Programming Language :: Python :: 3.13',
     'Topic :: Internet :: WWW/HTTP',
     'Topic :: System :: Distributed Computing',
     'Topic :: Software Development :: Libraries :: Python Modules',
 ]
-include = ['CHANGELOG.md']
+
+include = [
+ { path = "CHANGELOG.md", format = "sdist" },
+]
 
 [tool.poetry.plugins] # Optional super table
 
@@ -45,11 +52,8 @@ include = ['CHANGELOG.md']
 
 
 [tool.poetry.dependencies]
-python = ">=3.8,<4"
-django = [
-    { version = ">=3.2, <5", python = ">= 3.8" },
-    { version = "^5.0a1", python = ">= 3.10", allow-prereleases = true }
-]
+python = ">=3.9,<4"
+django = ">=4.2"
 django-picklefield = "^3.1"
 
 blessed = { version = "^1.19.1", optional = true }
@@ -72,8 +76,7 @@ pytest = "^7.1.3"
 pytest-django = "^4.5.2"
 Sphinx = "^4.0.2"
 pytest-cov = "^4.0.0"
-black = "^22.10.0"
-isort = {extras = ["requirements_deprecated_finder"], version = "^5.10.1"}
+ruff = "^0.4.4"
 
 [tool.poetry.extras]
 requires = ["poetry_core"]

requirements.txt

@@ -1,40 +1,41 @@
-ansicon==1.89.0; platform_system == "Windows" and python_version >= "2.7" \
-    --hash=sha256:f1def52d17f65c2c9682cf8370c03f541f410c1752d6a14029f97318e4b9dfec \
-    --hash=sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1
-arrow==1.1.1; python_version >= "3.6" \
-    --hash=sha256:77a60a4db5766d900a2085ce9074c5c7b8e2c99afeaa98ad627637ff6f292510 \
-    --hash=sha256:dee7602f6c60e3ec510095b5e301441bc56288cb8f51def14dcb3079f623823a
-asgiref==3.3.4; python_version >= "3.6" \
-    --hash=sha256:92906c611ce6c967347bbfea733f13d6313901d54dcca88195eaeb52b2a8e8ee \
-    --hash=sha256:d1216dfbdfb63826470995d31caed36225dcaf34f182e0fa257a4dd9e86f1b78
-blessed==1.18.1; python_version >= "2.7" \
-    --hash=sha256:dd7c0d33db9a2e7f597b446996484d0ed46e1586239db064fb5025008937dcae \
-    --hash=sha256:8b09936def6bc06583db99b65636b980075733e13550cb6af262ce724a55da23
-django-picklefield==3.0.1; python_version >= "3" \
-    --hash=sha256:15ccba592ca953b9edf9532e64640329cd47b136b7f8f10f2939caa5f9ce4287 \
-    --hash=sha256:3c702a54fde2d322fe5b2f39b8f78d9f655b8f77944ab26f703be6c0ed335a35
-django==3.2.4; python_version >= "3.6" \
-    --hash=sha256:ea735cbbbb3b2fba6d4da4784a0043d84c67c92f1fdf15ad6db69900e792c10f \
-    --hash=sha256:66c9d8db8cc6fe938a28b7887c1596e42d522e27618562517cc8929eb7e7f296
-jinxed==1.1.0; platform_system == "Windows" and python_version >= "2.7" \
-    --hash=sha256:6a61ccf963c16aa885304f27e6e5693783676897cea0c7f223270c8b8e78baf8 \
-    --hash=sha256:d8f1731f134e9e6b04d95095845ae6c10eb15cb223a5f0cabdea87d4a279c305
-python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
-    --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
-    --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
-pytz==2021.1; python_version >= "3.6" \
-    --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 \
-    --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da
-six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
-    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
-    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926
-sqlparse==0.4.1; python_version >= "3.6" \
-    --hash=sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0 \
-    --hash=sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8
-typing-extensions==3.10.0.0; python_version < "3.8" and python_version >= "3.6" \
-    --hash=sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497 \
-    --hash=sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84 \
-    --hash=sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342
-wcwidth==0.2.5; python_version >= "2.7" \
-    --hash=sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784 \
-    --hash=sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83
+asgiref==3.8.1 ; python_version >= "3.8" and python_version < "4" \
+    --hash=sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47 \
+    --hash=sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590
+backports-zoneinfo==0.2.1 ; python_version >= "3.8" and python_version < "3.9" \
+    --hash=sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf \
+    --hash=sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328 \
+    --hash=sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546 \
+    --hash=sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6 \
+    --hash=sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570 \
+    --hash=sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9 \
+    --hash=sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7 \
+    --hash=sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987 \
+    --hash=sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722 \
+    --hash=sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582 \
+    --hash=sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc \
+    --hash=sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b \
+    --hash=sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1 \
+    --hash=sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08 \
+    --hash=sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac \
+    --hash=sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2
+django-picklefield==3.2 ; python_version >= "3.8" and python_version < "4" \
+    --hash=sha256:aa463f5d79d497dbe789f14b45180f00a51d0d670067d0729f352a3941cdfa4d \
+    --hash=sha256:e9a73539d110f69825d9320db18bcb82e5189ff48dbed41821c026a20497764c
+django==4.2.11 ; python_version >= "3.8" and python_version < "4" \
+    --hash=sha256:6e6ff3db2d8dd0c986b4eec8554c8e4f919b5c1ff62a5b4390c17aff2ed6e5c4 \
+    --hash=sha256:ddc24a0a8280a0430baa37aff11f28574720af05888c62b7cfe71d219f4599d3
+importlib-metadata==7.1.0 ; python_version >= "3.8" and python_version < "3.10" \
+    --hash=sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570 \
+    --hash=sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2
+sqlparse==0.5.0 ; python_version >= "3.8" and python_version < "4" \
+    --hash=sha256:714d0a4932c059d16189f58ef5411ec2287a4360f17cdd0edd2d09d4c5087c93 \
+    --hash=sha256:c204494cd97479d0e39f28c93d46c0b2d5959c7b9ab904762ea6c7af211c8663
+typing-extensions==4.11.0 ; python_version >= "3.8" and python_version < "3.11" \
+    --hash=sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0 \
+    --hash=sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a
+tzdata==2024.1 ; python_version >= "3.8" and python_version < "4" and sys_platform == "win32" \
+    --hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \
+    --hash=sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252
+zipp==3.18.1 ; python_version >= "3.8" and python_version < "3.10" \
+    --hash=sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b \
+    --hash=sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715

ruff.toml

@@ -0,0 +1 @@
+lint.extend-select = ["I"]

test-services-docker-compose.yaml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   redis:
     image: redis:latest
@@ -15,15 +13,41 @@ services:
     networks:
       - main
 
+  aws:
+    container_name: aws
+    image: localstack/localstack:3.4.0
+    ports:
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559"  # External services port range
+    environment:
+      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-west-2}
+      DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-west-2}
+      SQS_ENDPOINT_STRATEGY: path
+      SERVICES: sqs
+      LOCALSTACK_HOST: aws
+      DEBUG: 1
+      LS_LOG: trace
+    volumes:
+      - ./containers/localstack:/etc/localstack/init/ready.d
+    networks:
+      - main
+
   django-q2:
     build:
       dockerfile: ./Dockerfile.dev
       context: .
+    environment:
+      AWS_ENDPOINT_URL: http://aws:4566
+      AWS_REGION: ${AWS_REGION:-us-west-2}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-test}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-test}
+      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-west-2}
     volumes:
       - .:/app
     depends_on:
       - redis
       - mongo
+      - aws
     networks:
       - main
 

web-docker-compose.yaml

@@ -1,11 +1,16 @@
-version: '3'
-
 services:
   web:
     restart: always
-    command: python manage.py runserver 0.0.0.0:8000
+    command: bash -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000"
     ports:
       - "127.0.0.1:8000:8000"
     build: .
     volumes:
       - .:/app
+
+  django-q:
+    restart: always
+    command: bash -c "python manage.py migrate && python manage.py qcluster"
+    build: .
+    volumes:
+      - .:/app