Release v1.7.3

* Update monitor.py

* format

---------

Co-authored-by: Stan Triepels <1939656+GDay@users.noreply.github.com>

.git-blame-ignore-revs

@@ -1,2 +1,4 @@
 # flake8, black, isort
 b1d000d007f3f77069719523268a0c6256dc0860
+# move to ruff formatting/linting
+ad4d24e17c9424b17cd8ae65c2def7ecc74e63c1

.github/workflows/release.yml

@@ -7,19 +7,18 @@ on:
 
 jobs:
   build:
-    if: github.repository == 'GDay/django-q2'
+    if: github.repository == 'django-q2/django-q2'
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.8
-
+          python-version: 3.11
       - name: Install dependencies
         run: |
           sudo apt-get update

.github/workflows/test.yml

@@ -7,12 +7,38 @@ on:
     branches:
       - master
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/checkout@v3
+        - name: Lint with ruff
+          run: |
+            pipx install ruff==0.4.10
+            ruff format . --check && ruff check .
+        - name: Check twine
+          run: |
+            python -m pip install twine poetry rstcheck
+            poetry build
+            rstcheck README.rst
+            twine check dist/*
+
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
-        django: [ "3.2", "4.1" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
+        django: [ "4.2", "5.0", "5.1"]
+        exclude:
+          # django 5.1 does not support 3.8 and 3.9
+          - python-version: "3.8"
+            django: "5.1"
+          - python-version: "3.9"
+            django: "5.1"
+          # django 5.0 does not support 3.8 and 3.9
+          - python-version: "3.8"
+            django: "5.0" 
+          - python-version: "3.9"
+            django: "5.0" 
 
     services:
       disque:
@@ -39,16 +65,16 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
     steps:
-        - uses: actions/checkout@v2
+        - uses: actions/checkout@v3
         - name: Set up Python ${{ matrix.python-version }}
-          uses: actions/setup-python@v2
+          uses: actions/setup-python@v4
           with:
             python-version: ${{ matrix.python-version }}
         - name: Install dependencies with Django ${{ matrix.django }}
           run: |
             python -m pip install --upgrade pip
             pip install poetry
-            poetry add "django==${{ matrix.django }}"
+            poetry add "django==${{ matrix.django }}" --python=${{ matrix.python-version }}
             poetry install -E testing
         - name: Run Tests
           run: |
@@ -68,16 +94,12 @@ jobs:
   finish:
     needs: test
     runs-on: ubuntu-latest
-    container: python:3-slim
+    container: python:3.11-bookworm
     steps:
       - name: Upload to coveralls
         run: |
           python -m pip install --upgrade pip
-          python -m pip install coveralls flake8 black
+          python -m pip install coveralls
           coveralls --service=github --finish
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: Check flake8/black
-        run: |
-          flake8 .
-          black --check .

.readthedocs.yaml

@@ -15,6 +15,8 @@ build:
 sphinx:
    configuration: docs/conf.py
 
+formats: all
+
 python:
    install:
    - requirements: docs/requirements.txt

CHANGELOG.md

@@ -1,6 +1,142 @@
 # Changelog
 
-## [Unreleased](https://github.com/GDay/django-q2/tree/HEAD)
+## [v1.7.3](https://github.com/django-q2/django-q2/tree/v1.7.3) (2024-10-15)
+
+- Catch missing SEGALRM with AttributeError instead of ValueError https://github.com/django-q2/django-q2/pull/223
+- Refactor timeout handling to handle AttributeError and ValueError for Windows users https://github.com/django-q2/django-q2/pull/234
+- Fix type check for args in scheduler.py E721 https://github.com/django-q2/django-q2/pull/233
+- Only trigger prometheus if configured https://github.com/django-q2/django-q2/pull/231
+- Fix missing ack_id when finishing task https://github.com/django-q2/django-q2/pull/224
+
+
+## [v1.7.2](https://github.com/django-q2/django-q2/tree/v1.7.2) (2024-09-09)
+
+- Fix twine check
+
+## [v1.7.1](https://github.com/django-q2/django-q2/tree/v1.7.1) (2024-09-08)
+
+- Fixed date of v1.7.0
+- Fixed README.rst formatting which is blocking release of latest version
+
+## [v1.7.0](https://github.com/django-q2/django-q2/tree/v1.7.0) (2024-09-08)
+
+**Merged pull requests:**
+
+- Remove support for Django 3.2 and 4.1 https://github.com/django-q2/django-q2/pull/183
+- Fix max attempts for value 1 https://github.com/django-q2/django-q2/pull/185
+- Replace black/isort with ruff https://github.com/django-q2/django-q2/pull/188
+- Fix repeating task after timeout https://github.com/django-q2/django-q2/pull/184
+- fix: Oracle ORM backend compatibility #180 https://github.com/django-q2/django-q2/pull/186
+- chore: Update CI for Django 4.2 Python 3.12 support https://github.com/django-q2/django-q2/pull/208
+- chore: Add Support Django 5.1 https://github.com/django-q2/django-q2/pull/207
+- Call mark_process_dead on worker pid if prometheus_client is installed https://github.com/django-q2/django-q2/pull/212
+- Add example project https://github.com/django-q2/django-q2/pull/215
+- Add index on succeeded tasks https://github.com/django-q2/django-q2/pull/164
+
+## [v1.6.2](https://github.com/django-q2/django-q2/tree/v1.6.2) (2024-03-05)
+
+**Merged pull requests:**
+
+- Allow different broker on chain https://github.com/django-q2/django-q2/pull/156
+- Fix formatting issues in README.rst https://github.com/django-q2/django-q2/pull/159
+- Update docs to add cluster option to the async_task https://github.com/django-q2/django-q2/pull/157
+- Update release/test dependencies https://github.com/django-q2/django-q2/pull/147
+- Fix for Negative Repeat Count in Scheduler https://github.com/django-q2/django-q2/pull/146
+- Update django.po https://github.com/django-q2/django-q2/pull/138
+- Use importerror for b62_decode and avoid deprecation notification https://github.com/django-q2/django-q2/pull/134
+- Specify build system in pyproject.toml https://github.com/django-q2/django-q2/pull/131
+
+## [v1.6.1](https://github.com/django-q2/django-q2/tree/v1.6.1) (2023-10-13)
+
+**Merged pull requests:**
+
+- Fix strict versions for python/django https://github.com/django-q2/django-q2/pull/130
+
+## [v1.6.0](https://github.com/django-q2/django-q2/tree/v1.6.0) (2023-10-12)
+
+**Merged pull requests:**
+
+- Add support for Django 5 and python 12 https://github.com/django-q2/django-q2/pull/120
+- Fix for "apps not ready" in Windows and Mac https://github.com/django-q2/django-q2/pull/116
+- Update broken MongoClient link in Docs https://github.com/django-q2/django-q2/pull/127
+- Fix German Translation Typo https://github.com/django-q2/django-q2/pull/124
+- Update Add-ons install command in install.rst https://github.com/django-q2/django-q2/pull/115
+- DOCS: Correct health check import in examples.rst https://github.com/django-q2/django-q2/pull/110
+
+## [v1.5.5](https://github.com/django-q2/django-q2/tree/v1.5.5) (2023-09-01)
+
+**Merged pull requests:**
+
+- Add documentation to migrate from django-q to django-q2 https://github.com/django-q2/django-q2/pull/108
+- Fix not picking up result from falsy result https://github.com/django-q2/django-q2/pull/107
+- Remove deprecated usage pkg_resources https://github.com/django-q2/django-q2/pull/103
+- Move worker, scheduler, pusher and monitor to separate files https://github.com/django-q2/django-q2/pull/100
+
+## [v1.5.4](https://github.com/GDay/django-q2/tree/v1.5.4) (2023-06-29)
+
+**Merged pull requests:**
+
+- Rerun successful tasks https://github.com/django-q2/django-q2/pull/99
+
+## [v1.5.3](https://github.com/GDay/django-q2/tree/v1.5.3) (2023-05-14)
+
+**Merged pull requests:**
+
+- Add post_spawn signal. https://github.com/django-q2/django-q2/pull/93
+- Post spawn docs https://github.com/django-q2/django-q2/pull/95
+- Make processes identifiable with uuid4 https://github.com/django-q2/django-q2/pull/91
+
+## [v1.5.2](https://github.com/GDay/django-q2/tree/v1.5.2) (2023-04-13)
+
+**Merged pull requests:**
+
+- Added Django 4.2 to the test matrix, fixed deprecation warning https://github.com/GDay/django-q2/pull/89
+- Updated docs to show support for 4.2
+
+## [v1.5.1](https://github.com/GDay/django-q2/tree/v1.5.1) (2023-04-02)
+
+- Fix release to pipy due to changed org name
+
+## [v1.5.0](https://github.com/GDay/django-q2/tree/v1.5.0) (2023-04-02)
+
+**Merged pull requests:**
+
+- Multiple queue, multiple cluster in one site https://github.com/GDay/django-q2/pull/71
+- Allow building docs all formats https://github.com/GDay/django-q2/pull/77
+- Remove version locking on `poetry_core` to fix regex error https://github.com/GDay/django-q2/pull/87
+- Update dependencies (2023-04-02) https://github.com/GDay/django-q2/pull/88
+
+## [v1.4.11](https://github.com/GDay/django-q2/tree/v1.4.11) (2023-01-30)
+
+**Merged pull requests:**
+
+- Fix missing setup file for "No matching distribution" error https://github.com/GDay/django-q2/pull/69
+- Remove custom build (revert to auto create setup file) https://github.com/GDay/django-q2/pull/70
+
+## [v1.4.10](https://github.com/GDay/django-q2/tree/v1.4.10) (2023-01-26)
+
+**Merged pull requests:**
+
+- Adding translation mo files automatically on build https://github.com/GDay/django-q2/pull/65
+- Update all dependencies https://github.com/GDay/django-q2/pull/64
+- Bump translations to latest changes https://github.com/GDay/django-q2/pull/63
+- Add meaningfull process titles with currently running task name https://github.com/GDay/django-q2/pull/57
+- Fix use of database router for write queries and remove Conf.HAS_REPLICA https://github.com/GDay/django-q2/pull/61 
+- Add intended_date_kwarg field to Schedule https://github.com/GDay/django-q2/pull/62 
+- Change task timeout logic to have now() as execution time https://github.com/GDay/django-q2/pull/58
+- More explicit log messages in exception handling https://github.com/GDay/django-q2/pull/59
+
+## [v1.4.9](https://github.com/GDay/django-q2/tree/v1.4.9) (2022-12-22)
+
+**Merged pull requests:**
+
+- Fix DST timezone change (move from DST to normal jump) https://github.com/GDay/django-q2/pull/56
+
+## [v1.4.8](https://github.com/GDay/django-q2/tree/v1.4.8) (2022-12-21)
+
+**Merged pull requests:**
+
+- Fix: allow both ZoneInfo and Pytz depending on django version https://github.com/GDay/django-q2/pull/55
 
 ## [v1.4.7](https://github.com/GDay/django-q2/tree/v1.4.7) (2022-12-21)
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.12
 
 ENV PYTHONUNBUFFERED 1
 RUN mkdir -p /app

Dockerfile.dev

@@ -8,10 +8,10 @@ ENV PYTHONUNBUFFERED 1
 ENV SHELL /bin/bash
 
 # Upgrades pip
-RUN pip install --upgrade pip 
+RUN pip install -U pip setuptools
 
 # Install poetry
-RUN pip install poetry
+RUN pip install poetry==1.8.2
 
 WORKDIR /app
 

Makefile

@@ -0,0 +1,21 @@
+dev:
+	docker compose -f web-docker-compose.yaml up
+
+test:
+	docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run pytest
+
+shell:
+	docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py shell
+
+makemigrations:
+	docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py makemigrations
+
+migrate:
+	docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py migrate
+
+createsuperuser:
+	docker compose -f web-docker-compose.yaml run --rm web python manage.py createsuperuser
+
+format:
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run ruff format .
+	docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run ruff check . --fix

README.rst

@@ -1,11 +1,9 @@
 A multiprocessing distributed task queue for Django
 ---------------------------------------------------
 
-|image0| |image1| |docs| |downloads|
+|image0| |image1| |downloads|
 
-::
-
-    Django Q2 is a fork of Django Q. Big thanks to Ilan Steemers for starting this project. Unfortunately, development has stalled since June 2021. Django Q2 is the new updated version of Django Q, with dependencies updates, docs updates and several bug fixes. Original repository: https://github.com/Koed00/django-q
+Django Q2 is a fork of Django Q. Big thanks to Ilan Steemers for starting this project. Unfortunately, development has stalled since June 2021. Django Q2 is the new updated version of Django Q, with dependencies updates, docs updates and several bug fixes. Original repository: https://github.com/Koed00/django-q
 
 Features
 ~~~~~~~~
@@ -27,7 +25,7 @@ Changes compared to the original Django-Q:
 - Dropped support for Disque (hasn't been updated in a long time)
 - Dropped Redis, Arrow and Blessed dependencies
 - Updated all current dependencies
-- Added tests for Django 4.x
+- Added tests for Django 4.x and 5.x
 - Added Turkish language
 - Improved admin area
 - Fixed a lot of issues
@@ -37,10 +35,10 @@ See the `changelog <https://github.com/GDay/django-q2/blob/master/CHANGELOG.md>`
 Requirements
 ~~~~~~~~~~~~
 
--  `Django <https://www.djangoproject.com>`__ > = 3.2
+-  `Django <https://www.djangoproject.com>`__ > = 4.2
 -  `Django-picklefield <https://github.com/gintas/django-picklefield>`__
 
-Tested with: Python 3.8, 3.9, 3.10, 3.11 Django 3.2.X and 4.1.X
+Tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 4.2.X and 5.0.X
 
 Brokers
 ~~~~~~~
@@ -73,7 +71,6 @@ Installation
 
 Read the full documentation at `https://django-q2.readthedocs.org <https://django-q2.readthedocs.org>`__
 
-
 Configuration
 ~~~~~~~~~~~~~
 
@@ -104,10 +101,7 @@ For full configuration options, see the `configuration documentation <https://dj
 Management Commands
 ~~~~~~~~~~~~~~~~~~~
 
-::
-
-    For the management commands to work, you will need to install Blessed: <https://github.com/jquast/blessed>
-
+For the management commands to work, you will need to install Blessed: <https://github.com/jquast/blessed>
 
 Start a cluster with::
 
@@ -159,7 +153,6 @@ Use `async_task` from your code to quickly offload tasks:
 
 For more info see `Tasks <https://django-q2.readthedocs.org/en/latest/tasks.html>`__
 
-
 Schedule
 ~~~~~~~~
 
@@ -204,6 +197,22 @@ Admin page or directly from your code:
 
 For more info check the `Schedules <https://django-q2.readthedocs.org/en/latest/schedules.html>`__ documentation.
 
+Development
+~~~~~~~~~~~
+
+There is an example project that you can use to develop with. Docker (compose) is being used to set everything up.
+Please note that you will have to restart the django-q container when changes have been made to tasks or django-q.
+You can start the example project with:
+
+.. code:: bash
+
+    make dev
+
+Create a superuser with:
+
+.. code:: bash
+
+    make createsuperuser
 
 Testing
 ~~~~~~~
@@ -212,7 +221,7 @@ Running tests is easy with docker compose, it will also start the necessary data
 
 .. code:: bash
 
-    docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run pytest
+    make test
 
 Locale
 ~~~~~~
@@ -220,12 +229,6 @@ Locale
 Currently available in English, German, Turkish, and French.
 Translation pull requests are always welcome.
 
-Todo
-~~~~
-
--  Better tests and coverage
--  Less dependencies?
-
 Acknowledgements
 ~~~~~~~~~~~~~~~~
 
@@ -242,9 +245,5 @@ Acknowledgements
    :target: https://github.com/GDay/django-q2/actions?query=workflow%3Atests
 .. |image1| image:: https://coveralls.io/repos/github/GDay/django-q2/badge.svg?branch=master
    :target: https://coveralls.io/github/GDay/django-q2?branch=master
-.. |docs| image:: https://readthedocs.org/projects/docs/badge/?version=latest
-    :alt: Documentation Status
-    :scale: 100
-    :target: https://django-q2.readthedocs.org/
 .. |downloads| image:: https://img.shields.io/pypi/dm/django-q2
    :target: https://img.shields.io/pypi/dm/django-q2

django_compilemessages.py

@@ -0,0 +1,9 @@
+import subprocess
+
+
+def generate_mo_files():
+    subprocess.run(["django-admin", "compilemessages"])
+
+
+if __name__ == "__main__":
+    generate_mo_files()

django_q/__init__.py

@@ -1,6 +1,6 @@
 import django
 
-VERSION = (1, 4, 7)
+VERSION = (1, 7, 3)
 
 if django.VERSION < (3, 2):
     default_app_config = "django_q.apps.DjangoQConfig"

django_q/admin.py

@@ -1,4 +1,5 @@
 """Admin module for Django."""
+
 from django.contrib import admin
 from django.db.models.expressions import OuterRef, Subquery
 from django.urls import reverse
@@ -10,10 +11,37 @@ from django_q.models import Failure, OrmQ, Schedule, Success, Task
 from django_q.tasks import async_task
 
 
+def resubmit_task(model_admin, request, queryset):
+    """Submit selected tasks back to the queue."""
+    for task in queryset:
+        async_task(
+            task.func,
+            *task.args or (),
+            hook=task.hook,
+            group=task.group,
+            cluster=task.cluster,
+            **task.kwargs or {},
+        )
+        if isinstance(model_admin, FailAdmin):
+            task.delete()
+
+
+resubmit_task.short_description = _("Resubmit selected tasks to queue")
+
+
 class TaskAdmin(admin.ModelAdmin):
     """model admin for success tasks."""
 
-    list_display = ("name", "group", "func", "started", "stopped", "time_taken")
+    list_display = (
+        "name",
+        "group",
+        "func",
+        "cluster",
+        "started",
+        "stopped",
+        "time_taken",
+    )
+    actions = [resubmit_task]
 
     def has_add_permission(self, request):
         """Don't allow adds."""
@@ -26,35 +54,33 @@ class TaskAdmin(admin.ModelAdmin):
 
     search_fields = ("name", "func", "group")
     readonly_fields = []
-    list_filter = ("group",)
+    list_filter = ("group", "cluster")
 
     def get_readonly_fields(self, request, obj=None):
         """Set all fields readonly."""
         return list(self.readonly_fields) + [field.name for field in obj._meta.fields]
 
 
-def retry_failed(FailAdmin, request, queryset):
-    """Submit selected tasks back to the queue."""
-    for task in queryset:
-        async_task(task.func, *task.args or (), hook=task.hook, **task.kwargs or {})
-        task.delete()
-
-
-retry_failed.short_description = _("Resubmit selected tasks to queue")
-
-
 class FailAdmin(admin.ModelAdmin):
     """model admin for failed tasks."""
 
-    list_display = ("name", "group", "func", "started", "stopped", "short_result")
+    list_display = (
+        "name",
+        "group",
+        "func",
+        "cluster",
+        "started",
+        "stopped",
+        "short_result",
+    )
 
     def has_add_permission(self, request):
         """Don't allow adds."""
         return False
 
-    actions = [retry_failed]
+    actions = [resubmit_task]
     search_fields = ("name", "func", "group")
-    list_filter = ("group",)
+    list_filter = ("group", "cluster")
     readonly_fields = []
 
     def get_readonly_fields(self, request, obj=None):
@@ -123,6 +149,18 @@ class QueueAdmin(admin.ModelAdmin):
     """queue admin for ORM broker"""
 
     list_display = ("id", "key", "name", "group", "func", "lock", "task_id")
+    fields = (
+        "key",
+        "lock",
+        "task_id",
+        "name",
+        "group",
+        "func",
+        "args",
+        "kwargs",
+        "q_options",
+    )
+    readonly_fields = fields[2:]
 
     def save_model(self, request, obj, form, change):
         obj.save(using=Conf.ORM)

django_q/brokers/__init__.py

@@ -7,7 +7,9 @@ from django_q.conf import Conf
 
 
 class Broker:
-    def __init__(self, list_key: str = Conf.PREFIX):
+    def __init__(self, list_key: str = None):
+        # With same BROKER_CLASS, `list_key` is just a synonym for `queue_name` except for RedisBroker
+        list_key = list_key or Conf.CLUSTER_NAME
         self.connection = self.get_connection(list_key)
         self.list_key = list_key
         self.cache = self.get_cache()
@@ -151,7 +153,7 @@ class Broker:
             return None
 
     @staticmethod
-    def get_connection(list_key: str = Conf.PREFIX):
+    def get_connection(list_key: str = None):
         """
         Gets a connection to the broker
         :param list_key: Optional queue name
@@ -160,13 +162,14 @@ class Broker:
         return 0
 
 
-def get_broker(list_key: str = Conf.PREFIX) -> Broker:
+def get_broker(list_key: str = None) -> Broker:
     """
     Gets the configured broker type
     :param list_key: optional queue name
     :type list_key: str
     :return: a broker instance
     """
+    list_key = list_key or Conf.CLUSTER_NAME
     # custom
     if Conf.BROKER_CLASS:
         module, func = Conf.BROKER_CLASS.rsplit(".", 1)
@@ -179,7 +182,7 @@ def get_broker(list_key: str = Conf.PREFIX) -> Broker:
 
         return ironmq.IronMQBroker(list_key=list_key)
     # SQS
-    elif type(Conf.SQS) == dict:
+    elif isinstance(Conf.SQS, dict):
         from django_q.brokers import aws_sqs
 
         return aws_sqs.Sqs(list_key=list_key)

django_q/brokers/aws_sqs.py

@@ -8,7 +8,7 @@ QUEUE_DOES_NOT_EXIST = "AWS.SimpleQueueService.NonExistentQueue"
 
 
 class Sqs(Broker):
-    def __init__(self, list_key: str = Conf.PREFIX):
+    def __init__(self, list_key: str = None):
         self.sqs = None
         super(Sqs, self).__init__(list_key)
         self.queue = self.get_queue()
@@ -77,7 +77,7 @@ class Sqs(Broker):
         return "AWS SQS"
 
     @staticmethod
-    def get_connection(list_key: str = Conf.PREFIX) -> Session:
+    def get_connection(list_key: str = None) -> Session:
         config = Conf.SQS
         if "aws_region" in config:
             config["region_name"] = config["aws_region"]

django_q/brokers/ironmq.py

@@ -46,6 +46,7 @@ class IronMQBroker(Broker):
         return self.delete(task_id)
 
     @staticmethod
-    def get_connection(list_key: str = Conf.PREFIX) -> Queue:
+    def get_connection(list_key: str = None) -> Queue:
+        list_key = list_key or Conf.CLUSTER_NAME
         ironmq = IronMQ(name=None, **Conf.IRON_MQ)
         return ironmq.queue(queue_name=list_key)

django_q/brokers/mongo.py

@@ -15,7 +15,7 @@ def _timeout():
 
 
 class Mongo(Broker):
-    def __init__(self, list_key=Conf.PREFIX):
+    def __init__(self, list_key: str = None):
         super(Mongo, self).__init__(list_key)
         self.collection = self.get_collection()
 
@@ -24,7 +24,7 @@ class Mongo(Broker):
         self.collection = self.get_collection()
 
     @staticmethod
-    def get_connection(list_key: str = Conf.PREFIX) -> MongoClient:
+    def get_connection(list_key: str = None) -> MongoClient:
         return MongoClient(**Conf.MONGO)
 
     def get_collection(self):

django_q/brokers/orm.py

@@ -11,12 +11,12 @@ from django_q.models import OrmQ
 
 
 def _timeout():
-    return timezone.now() - timedelta(seconds=Conf.RETRY)
+    return timezone.now() + timedelta(seconds=Conf.RETRY)
 
 
 class ORM(Broker):
     @staticmethod
-    def get_connection(list_key: str = Conf.PREFIX):
+    def get_connection(list_key: str = None):
         if transaction.get_autocommit(
             using=Conf.ORM
         ):  # Only True when not in an atomic block
@@ -31,13 +31,15 @@ class ORM(Broker):
     def queue_size(self) -> int:
         return (
             self.get_connection()
-            .filter(key=self.list_key, lock__lte=_timeout())
+            .filter(key=self.list_key, lock__lte=timezone.now())
             .count()
         )
 
     def lock_size(self) -> int:
         return (
-            self.get_connection().filter(key=self.list_key, lock__gt=_timeout()).count()
+            self.get_connection()
+            .filter(key=self.list_key, lock__gt=timezone.now())
+            .count()
         )
 
     def purge_queue(self):
@@ -55,13 +57,16 @@ class ORM(Broker):
         self.delete(task_id)
 
     def enqueue(self, task):
+        # list_key might be null (e.g. in a test setup) but OrmQ.key has not-null constraint
         package = self.get_connection().create(
-            key=self.list_key, payload=task, lock=_timeout()
+            key=self.list_key or Conf.CLUSTER_NAME, payload=task, lock=timezone.now()
         )
         return package.pk
 
     def dequeue(self):
-        tasks = self.get_connection().filter(key=self.list_key, lock__lt=_timeout())[
+        tasks = self.get_connection().filter(
+            key=self.list_key, lock__lt=timezone.now()
+        )[
             0 : Conf.BULK  # noqa: E203
         ]
         if tasks:
@@ -70,7 +75,7 @@ class ORM(Broker):
                 if (
                     self.get_connection()
                     .filter(id=task.id, lock=task.lock)
-                    .update(lock=timezone.now())
+                    .update(lock=_timeout())
                 ):
                     task_list.append((task.pk, task.payload))
                 # else don't process, as another cluster has been faster than us on

django_q/brokers/redis_broker.py

@@ -11,7 +11,8 @@ except ImportError:
 
 
 class Redis(Broker):
-    def __init__(self, list_key: str = Conf.PREFIX):
+    def __init__(self, list_key: str = None):
+        list_key = list_key or Conf.CLUSTER_NAME
         super(Redis, self).__init__(list_key=f"django_q:{list_key}:q")
 
     def enqueue(self, task):
@@ -57,7 +58,7 @@ class Redis(Broker):
             return self.connection.mget(keys)
 
     @staticmethod
-    def get_connection(list_key: str = Conf.PREFIX) -> Redis:
+    def get_connection(list_key: str = None) -> Redis:
         if django_redis and Conf.DJANGO_REDIS:
             return django_redis.get_redis_connection(Conf.DJANGO_REDIS)
         if isinstance(Conf.REDIS, str):

django_q/cluster.py

@@ -1,12 +1,8 @@
 # Standard
-import ast
-import pydoc
+import os
 import signal
 import socket
-import traceback
 import uuid
-from datetime import datetime, timedelta
-from pytz import timezone as pytz_timezone
 from multiprocessing import Event, Process, Value, current_process
 from time import sleep
 
@@ -21,51 +17,52 @@ except core.exceptions.AppRegistryNotReady:
 
     django.setup()
 
-from django.conf import settings
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
 # Local
-import django_q.tasks
 from django_q.brokers import Broker, get_broker
 from django_q.conf import (
     Conf,
-    croniter,
-    error_reporter,
     get_ppid,
     logger,
+    prometheus_multiprocess,
     psutil,
-    resource,
+    setproctitle,
 )
 from django_q.humanhash import humanize
-from django_q.models import Schedule, Success, Task
+from django_q.monitor import monitor
+from django_q.pusher import pusher
 from django_q.queues import Queue
-from django_q.signals import post_execute, pre_execute
-from django_q.signing import BadSignature, SignedPackage
+from django_q.scheduler import scheduler
 from django_q.status import Stat, Status
-
-from .utils import get_func_repr, localtime
+from django_q.worker import worker
 
 
 class Cluster:
     def __init__(self, broker: Broker = None):
-        self.broker = broker or get_broker()
+        # Cluster do not need an init or default broker except for testing,
+        # The sentinel will create a broker for cluster and utilize ALT_CLUSTERS config in Conf.
+        self.broker = broker  # DON'T USE get_broker() to set a default broker here.
         self.sentinel = None
         self.stop_event = None
         self.start_event = None
         self.pid = current_process().pid
         self.cluster_id = uuid.uuid4()
         self.host = socket.gethostname()
-        self.timeout = Conf.TIMEOUT
+        self.timeout = None
         signal.signal(signal.SIGTERM, self.sig_handler)
         signal.signal(signal.SIGINT, self.sig_handler)
 
     def start(self) -> int:
+        if setproctitle:
+            setproctitle.setproctitle(f"qcluster {current_process().name} {self.name}")
         # Start Sentinel
         self.stop_event = Event()
         self.start_event = Event()
         self.sentinel = Process(
             target=Sentinel,
+            name=f"Process-{uuid.uuid4().hex}",
             args=(
                 self.stop_event,
                 self.start_event,
@@ -140,7 +137,7 @@ class Sentinel:
         start_event,
         cluster_id,
         broker=None,
-        timeout=Conf.TIMEOUT,
+        timeout=None,
         start=True,
     ):
         # Make sure we catch signals for the pool
@@ -157,7 +154,7 @@ class Sentinel:
         self.start_event = start_event
         self.pool_size = Conf.WORKERS
         self.pool = []
-        self.timeout = timeout
+        self.timeout = timeout or Conf.TIMEOUT
         self.task_queue = (
             Queue(maxsize=Conf.QUEUE_LIMIT) if Conf.QUEUE_LIMIT else Queue()
         )
@@ -168,6 +165,10 @@ class Sentinel:
         if start:
             self.start()
 
+    def queue_name(self):
+        # multi-queue: cluster name is (broker's) queue_name
+        return self.broker.list_key if self.broker else "--"
+
     def start(self):
         self.broker.ping()
         self.spawn_cluster()
@@ -189,7 +190,7 @@ class Sentinel:
         """
         :type target: function or class
         """
-        p = Process(target=target, args=args)
+        p = Process(target=target, args=args, name=f"Process-{uuid.uuid4().hex}")
         p.daemon = True
         if target == worker:
             p.daemon = Conf.DAEMONIZE_WORKERS
@@ -219,31 +220,56 @@ class Sentinel:
             db.connections.close_all()
         if process == self.monitor:
             self.monitor = self.spawn_monitor()
-            logger.error(
+            logger.critical(
                 _("reincarnated monitor %(name)s after sudden death")
                 % {"name": process.name}
             )
         elif process == self.pusher:
             self.pusher = self.spawn_pusher()
-            logger.error(
+            logger.critical(
                 _("reincarnated pusher %(name)s after sudden death")
                 % {"name": process.name}
             )
         else:
+            # check if prometheus is proper configurated
+            prometheus_path = os.getenv(
+                "PROMETHEUS_MULTIPROC_DIR", os.getenv("prometheus_multiproc_dir")
+            )
+
+            if prometheus_multiprocess and prometheus_path:
+                prometheus_multiprocess.mark_process_dead(process.pid)
+
             self.pool.remove(process)
             self.spawn_worker()
             if process.timer.value == 0:
                 # only need to terminate on timeout, otherwise we risk destabilizing
                 # the queues
+                task_name = ""
+                if psutil:
+                    try:
+                        process_name = psutil.Process(process.pid).name()
+                        name_splits = process_name.split(" ")
+                        task_name = (
+                            name_splits[3]
+                            if len(name_splits) >= 4 and name_splits[2] == "processing"
+                            else ""
+                        )
+                    except psutil.NoSuchProcess:
+                        pass
                 process.terminate()
-                logger.warning(
-                    _("reincarnated worker %(name)s after timeout")
-                    % {"name": process.name}
-                )
+                if task_name:
+                    msg = _(
+                        "reincarnated worker %(name)s after timeout while processing task %(task_name)s"
+                    ) % {"name": process.name, "task_name": task_name}
+                else:
+                    msg = _("reincarnated worker %(name)s after timeout") % {
+                        "name": process.name
+                    }
+                logger.critical(msg)
             elif int(process.timer.value) == -2:
                 logger.info(_("recycled worker %(name)s") % {"name": process.name})
             else:
-                logger.error(
+                logger.critical(
                     _("reincarnated worker %(name)s after death")
                     % {"name": process.name}
                 )
@@ -271,14 +297,18 @@ class Sentinel:
             _("%(name)s guarding cluster %(cluster_name)s")
             % {
                 "name": current_process().name,
-                "cluster_name": humanize(self.cluster_id.hex),
+                "cluster_name": humanize(self.cluster_id.hex)
+                + f" [{self.queue_name()}]",
             }
         )
         self.start_event.set()
         Stat(self).save()
         logger.info(
             _("Q Cluster %(cluster_name)s running.")
-            % {"cluster_name": humanize(self.cluster_id.hex)}
+            % {
+                "cluster_name": humanize(self.cluster_id.hex)
+                + f" [{self.queue_name()}]"
+            }
         )
         counter = 0
         cycle = Conf.GUARD_CYCLE  # guard loop sleep in seconds
@@ -351,416 +381,6 @@ class Sentinel:
         Stat(self).save()
 
 
-def pusher(task_queue: Queue, event: Event, broker: Broker = None):
-    """
-    Pulls tasks of the broker and puts them in the task queue
-    :type broker:
-    :type task_queue: multiprocessing.Queue
-    :type event: multiprocessing.Event
-    """
-    if not broker:
-        broker = get_broker()
-    logger.info(
-        _("%(process_name)s pushing tasks at %(id)s")
-        % {"process_name": current_process().name, "id": current_process().pid}
-    )
-    while True:
-        try:
-            task_set = broker.dequeue()
-        except Exception as e:
-            logger.exception("Failed to pull task from broker")
-            # broker probably crashed. Let the sentinel handle it.
-            sleep(10)
-            break
-        if task_set:
-            for task in task_set:
-                ack_id = task[0]
-                # unpack the task
-                try:
-                    task = SignedPackage.loads(task[1])
-                except (TypeError, BadSignature) as e:
-                    logger.exception("Failed to push task to queue")
-                    broker.fail(ack_id)
-                    continue
-                task["ack_id"] = ack_id
-                task_queue.put(task)
-            logger.debug(
-                _("queueing from %(list_key)s") % {"list_key": broker.list_key}
-            )
-        if event.is_set():
-            break
-    logger.info(_("%(name)s stopped pushing tasks") % {"name": current_process().name})
-
-
-def monitor(result_queue: Queue, broker: Broker = None):
-    """
-    Gets finished tasks from the result queue and saves them to Django
-    :type broker: brokers.Broker
-    :type result_queue: multiprocessing.Queue
-    """
-    if not broker:
-        broker = get_broker()
-    name = current_process().name
-    logger.info(
-        _("%(name)s monitoring at %(id)s") % {"name": name, "id": current_process().pid}
-    )
-    for task in iter(result_queue.get, "STOP"):
-        # save the result
-        if task.get("cached", False):
-            save_cached(task, broker)
-        else:
-            save_task(task, broker)
-        # acknowledge result
-        ack_id = task.pop("ack_id", False)
-        if ack_id and (task["success"] or task.get("ack_failure", False)):
-            broker.acknowledge(ack_id)
-        # signal execution done
-        post_execute.send(sender="django_q", task=task)
-        # log the result
-        info_name = get_func_repr(task["func"])
-        if task["success"]:
-            # log success
-            logger.info(
-                _("Processed '%(info_name)s' (%(task_name)s)")
-                % {"info_name": info_name, "task_name": task["name"]}
-            )
-        else:
-            # log failure
-            logger.error(
-                _("Failed '%(info_name)s' (%(task_name)s) - %(task_result)s")
-                % {
-                    "info_name": info_name,
-                    "task_name": task["name"],
-                    "task_result": task["result"],
-                }
-            )
-    logger.info(_("%(name)s stopped monitoring results") % {"name": name})
-
-
-def worker(
-    task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
-):
-    """
-    Takes a task from the task queue, tries to execute it and puts the result back in
-    the result queue
-    :param timeout: number of seconds wait for a worker to finish.
-    :type task_queue: multiprocessing.Queue
-    :type result_queue: multiprocessing.Queue
-    :type timer: multiprocessing.Value
-    """
-    proc_name = current_process().name
-    logger.info(
-        _("%(proc_name)s ready for work at %(id)s")
-        % {"proc_name": proc_name, "id": current_process().pid}
-    )
-    task_count = 0
-    if timeout is None:
-        timeout = -1
-    # Start reading the task queue
-    for task in iter(task_queue.get, "STOP"):
-        result = None
-        timer.value = -1  # Idle
-        task_count += 1
-        # Get the function from the task
-        func = task["func"]
-        func_name = get_func_repr(func)
-        logger.info(
-            _("%(proc_name)s processing '%(func_name)s' (%(task_name)s)")
-            % {
-                "proc_name": proc_name,
-                "func_name": func_name,
-                "task_name": task["name"],
-            }
-        )
-        f = task["func"]
-        # if it's not an instance try to get it from the string
-        if not callable(task["func"]):
-            f = pydoc.locate(f)
-        close_old_django_connections()
-        timer_value = task.pop("timeout", timeout)
-        # signal execution
-        pre_execute.send(sender="django_q", func=f, task=task)
-        # execute the payload
-        timer.value = timer_value  # Busy
-        try:
-            res = f(*task["args"], **task["kwargs"])
-            result = (res, True)
-        except Exception as e:
-            result = (f"{e} : {traceback.format_exc()}", False)
-            if error_reporter:
-                error_reporter.report()
-            if task.get("sync", False):
-                raise
-        with timer.get_lock():
-            # Process result
-            task["result"] = result[0]
-            task["success"] = result[1]
-            task["stopped"] = timezone.now()
-            result_queue.put(task)
-            timer.value = -1  # Idle
-            # Recycle
-            if task_count == Conf.RECYCLE or rss_check():
-                timer.value = -2  # Recycled
-                break
-    logger.info(_("%(proc_name)s stopped doing work") % {"proc_name": proc_name})
-
-
-def save_task(task, broker: Broker):
-    """
-    Saves the task package to Django or the cache
-    :param task: the task package
-    :type broker: brokers.Broker
-    """
-    # SAVE LIMIT < 0 : Don't save success
-    if not task.get("save", Conf.SAVE_LIMIT >= 0) and task["success"]:
-        return
-    # enqueues next in a chain
-    if task.get("chain", None):
-        django_q.tasks.async_chain(
-            task["chain"],
-            group=task["group"],
-            cached=task["cached"],
-            sync=task["sync"],
-            broker=broker,
-        )
-    # SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
-    close_old_django_connections()
-
-    try:
-        filters = {}
-        if (
-            Conf.SAVE_LIMIT_PER
-            and Conf.SAVE_LIMIT_PER in {"group", "name", "func"}
-            and Conf.SAVE_LIMIT_PER in task
-        ):
-            value = task[Conf.SAVE_LIMIT_PER]
-            if Conf.SAVE_LIMIT_PER == "func":
-                value = get_func_repr(value)
-            filters[Conf.SAVE_LIMIT_PER] = value
-
-        database_to_use = (
-            {"using": Conf.ORM if Conf.ORM else Schedule.objects.db}
-            if not Conf.HAS_REPLICA
-            else {}
-        )
-        with db.transaction.atomic(**database_to_use):
-            last = Success.objects.filter(**filters).select_for_update().last()
-            if (
-                task["success"]
-                and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
-            ):
-                last.delete()
-
-        # check if this task has previous results
-        try:
-            existing_task = Task.objects.get(id=task["id"], name=task["name"])
-            # only update the result if it hasn't succeeded yet
-            if not existing_task.success:
-                existing_task.stopped = task["stopped"]
-                existing_task.result = task["result"]
-                existing_task.success = task["success"]
-                existing_task.attempt_count = existing_task.attempt_count + 1
-                existing_task.save()
-
-            if (
-                Conf.MAX_ATTEMPTS > 0
-                and existing_task.attempt_count >= Conf.MAX_ATTEMPTS
-            ):
-                broker.acknowledge(task["ack_id"])
-
-        except Task.DoesNotExist:
-            # convert func to string
-            func = get_func_repr(task["func"])
-            Task.objects.create(
-                id=task["id"],
-                name=task["name"],
-                func=func,
-                hook=task.get("hook"),
-                args=task["args"],
-                kwargs=task["kwargs"],
-                started=task["started"],
-                stopped=task["stopped"],
-                result=task["result"],
-                group=task.get("group"),
-                success=task["success"],
-                attempt_count=1,
-            )
-    except Exception as e:
-        logger.error(e)
-
-
-def save_cached(task, broker: Broker):
-    task_key = f'{broker.list_key}:{task["id"]}'
-    timeout = task["cached"]
-    if timeout is True:
-        timeout = None
-    try:
-        group = task.get("group", None)
-        iter_count = task.get("iter_count", 0)
-        # if it's a group append to the group list
-        if group:
-            group_key = f"{broker.list_key}:{group}:keys"
-            group_list = broker.cache.get(group_key) or []
-            # if it's an iter group, check if we are ready
-            if iter_count and len(group_list) == iter_count - 1:
-                group_args = f"{broker.list_key}:{group}:args"
-                # collate the results into a Task result
-                results = [
-                    SignedPackage.loads(broker.cache.get(k))["result"]
-                    for k in group_list
-                ]
-                results.append(task["result"])
-                task["result"] = results
-                task["id"] = group
-                task["args"] = SignedPackage.loads(broker.cache.get(group_args))
-                task.pop("iter_count", None)
-                task.pop("group", None)
-                if task.get("iter_cached", None):
-                    task["cached"] = task.pop("iter_cached", None)
-                    save_cached(task, broker=broker)
-                else:
-                    save_task(task, broker)
-                broker.cache.delete_many(group_list)
-                broker.cache.delete_many([group_key, group_args])
-                return
-            # save the group list
-            group_list.append(task_key)
-            broker.cache.set(group_key, group_list, timeout)
-            # async_task next in a chain
-            if task.get("chain", None):
-                django_q.tasks.async_chain(
-                    task["chain"],
-                    group=group,
-                    cached=task["cached"],
-                    sync=task["sync"],
-                    broker=broker,
-                )
-        # save the task
-        broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
-    except Exception as e:
-        logger.error(e)
-
-
-def scheduler(broker: Broker = None):
-    """
-    Creates a task from a schedule at the scheduled time and schedules next run
-    """
-    if not broker:
-        broker = get_broker()
-    close_old_django_connections()
-    try:
-        database_to_use = (
-            {"using": Conf.ORM if Conf.ORM else Schedule.objects.db}
-            if not Conf.HAS_REPLICA
-            else {}
-        )
-        with db.transaction.atomic(**database_to_use):
-            for s in (
-                Schedule.objects.select_for_update()
-                .exclude(repeats=0)
-                .filter(next_run__lt=timezone.now())
-                .filter(
-                    db.models.Q(cluster__isnull=True) | db.models.Q(cluster=Conf.PREFIX)
-                )
-            ):
-                args = ()
-                kwargs = {}
-                # get args, kwargs and hook
-                if s.kwargs:
-                    try:
-                        # first try the dict syntax
-                        kwargs = ast.literal_eval(s.kwargs)
-                    except (SyntaxError, ValueError):
-                        # else use the kwargs syntax
-                        try:
-                            parsed_kwargs = (
-                                ast.parse(f"f({s.kwargs})").body[0].value.keywords
-                            )
-                            kwargs = {
-                                kwarg.arg: ast.literal_eval(kwarg.value)
-                                for kwarg in parsed_kwargs
-                            }
-                        except (SyntaxError, ValueError):
-                            kwargs = {}
-                if s.args:
-                    args = ast.literal_eval(s.args)
-                    # single value won't eval to tuple, so:
-                    if type(args) != tuple:
-                        args = (args,)
-                q_options = kwargs.get("q_options", {})
-                if s.hook:
-                    q_options["hook"] = s.hook
-                # set up the next run time
-                if s.schedule_type != s.ONCE:
-                    next_run = s.next_run
-                    while True:
-                        next_run = s.calculate_next_run(next_run)
-                        if Conf.CATCH_UP or next_run > localtime():
-                            break
-
-                    s.next_run = next_run
-                    s.repeats += -1
-                # send it to the cluster
-                scheduled_broker = broker
-                try:
-                    scheduled_broker = get_broker(q_options["broker_name"])
-                except:  # noqa: E722
-                    # invalid broker_name or non existing broker with broker_name
-                    pass
-                q_options["broker"] = scheduled_broker
-                q_options["group"] = q_options.get("group", s.name or s.id)
-                kwargs["q_options"] = q_options
-                s.task = django_q.tasks.async_task(s.func, *args, **kwargs)
-                # log it
-                if not s.task:
-                    logger.error(
-                        _(
-                            "%(process_name)s failed to create a task from schedule "
-                            "[%(schedule)s]"
-                        )
-                        % {
-                            "process_name": current_process().name,
-                            "schedule": s.name or s.id,
-                        }
-                    )
-                else:
-                    logger.info(
-                        _(
-                            "%(process_name)s created a task from schedule "
-                            "[%(schedule)s]"
-                        )
-                        % {
-                            "process_name": current_process().name,
-                            "schedule": s.name or s.id,
-                        }
-                    )
-                # default behavior is to delete a ONCE schedule
-                if s.schedule_type == s.ONCE:
-                    if s.repeats < 0:
-                        s.delete()
-                        continue
-                    # but not if it has a positive repeats
-                    s.repeats = 0
-                # save the schedule
-                s.save()
-    except Exception as e:
-        logger.error(e)
-
-
-def close_old_django_connections():
-    """
-    Close django connections unless running with sync=True.
-    """
-    if Conf.SYNC:
-        logger.warning(
-            "Preserving django database connections because sync=True. Beware "
-            "that tasks are now injected in the calling context/transactions "
-            "which may result in unexpected behaviour."
-        )
-    else:
-        db.close_old_connections()
-
-
 def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING):
     """
     Sets the cpu affinity for the supplied processes.
@@ -801,12 +421,3 @@ def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING)
                 _("%(pid)s will use cpu %(affinity)s")
                 % {"pid": pid, "affinity": affinity}
             )
-
-
-def rss_check():
-    if Conf.MAX_RSS:
-        if resource:
-            return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
-        elif psutil:
-            return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
-    return False

django_q/conf.py

@@ -1,16 +1,22 @@
 import logging
 import os
+import sys
 from copy import deepcopy
 from multiprocessing import cpu_count
 from signal import signal
 from warnings import warn
 
-import pkg_resources
 from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 
 from django_q.queues import Queue
 
+# The "selectable" entry points were introduced in importlib_metadata 3.6 and Python 3.10.
+if sys.version_info < (3, 10):
+    from importlib_metadata import entry_points
+else:
+    from importlib.metadata import entry_points
+
 # optional
 try:
     import psutil
@@ -27,6 +33,16 @@ try:
 except ModuleNotFoundError:
     resource = None
 
+try:
+    import setproctitle
+except ModuleNotFoundError:
+    setproctitle = None
+
+try:
+    from prometheus_client import multiprocess as prometheus_multiprocess
+except ModuleNotFoundError:
+    prometheus_multiprocess = None
+
 
 class Conf:
     """
@@ -34,10 +50,25 @@ class Conf:
     """
 
     try:
-        conf = settings.Q_CLUSTER
+        conf = settings.Q_CLUSTER.copy()
     except AttributeError:
         conf = {}
 
+    _Q_CLUSTER_NAME = os.getenv("Q_CLUSTER_NAME")
+    if (
+        _Q_CLUSTER_NAME
+        and _Q_CLUSTER_NAME != conf.get("name")
+        and _Q_CLUSTER_NAME != conf.get("cluster_name")
+    ):
+        conf["cluster_name"] = _Q_CLUSTER_NAME
+        alt_conf = conf.pop("ALT_CLUSTERS")
+        if isinstance(alt_conf, dict):
+            alt_conf = alt_conf.get(_Q_CLUSTER_NAME)
+            if isinstance(alt_conf, dict):
+                alt_conf.pop("name", None)
+                alt_conf.pop("cluster_name", None)
+                conf.update(alt_conf)
+
     # Redis server configuration . Follows standard redis keywords
     REDIS = conf.get("redis", {})
 
@@ -54,9 +85,6 @@ class Conf:
     # ORM broker
     ORM = conf.get("orm", None)
 
-    # ORM support for read/write replicas
-    HAS_REPLICA = conf.get("has_replica", False)
-
     # Custom broker class
     BROKER_CLASS = conf.get("broker_class", None)
 
@@ -68,8 +96,14 @@ class Conf:
     MONGO_DB = conf.get("mongo_db", None)
 
     # Name of the cluster or site. For when you run multiple sites on one redis server
+    # It's also the `salt` for signing OrmQ, and part of the Redis stats caching key
+    # For all clusters in one site, PREFIX should be the same value to be able to decrypt payloads
     PREFIX = conf.get("name", "default")
 
+    # Support alternative cluster name to use multiple queues in one site.
+    #   cluster name and queue name are interchangeable, same thing.
+    CLUSTER_NAME = conf.get("cluster_name", PREFIX)
+
     # Log output level
     LOG_LEVEL = conf.get("log_level", "INFO")
 
@@ -234,7 +268,6 @@ if not logger.hasHandlers():
 
 # Error Reporting Interface
 class ErrorReporter:
-
     # initialize with iterator of reporters (better name, targets?)
     def __init__(self, reporters):
         self.targets = [target for target in reporters]
@@ -253,9 +286,7 @@ if Conf.ERROR_REPORTER:
         # iterate through the configured error reporters,
         # and instantiate an ErrorReporter using the provided config
         for name, conf in error_conf.items():
-            for entry in pkg_resources.iter_entry_points(
-                "djangoq.errorreporters", name
-            ):
+            for entry in entry_points(group="djangoq.errorreporters", name=name):
                 Reporter = entry.load()
                 reporters.append(Reporter(**conf))
         error_reporter = ErrorReporter(reporters)

django_q/core_signing.py

@@ -2,16 +2,24 @@ import datetime
 import time
 import zlib
 
-from django.core.signing import BadSignature, JSONSerializer, SignatureExpired
+from django.core.signing import (
+    BadSignature,
+    JSONSerializer,
+    SignatureExpired,
+    b64_decode,
+    dumps,
+)
 from django.core.signing import Signer as Sgnr
 from django.core.signing import TimestampSigner as TsS
-from django.core.signing import b64_decode, dumps
 
 try:
-    from django.core.signing import base62
+    from django.core.signing import b62_decode
 except ImportError:
-    # For django < 4.0
+    # fallback for django 3.x
     from django.utils.baseconv import base62
+
+    b62_decode = base62.decode
+
 from django.utils.crypto import constant_time_compare
 from django.utils.encoding import force_bytes, force_str
 
@@ -37,7 +45,9 @@ def loads(
     """
     # TimestampSigner.unsign() returns str but base64 and zlib compression
     # operate on bytes.
-    base64d = force_bytes(TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
+    base64d = force_bytes(
+        TimestampSigner(key=key, salt=salt).unsign(s, max_age=max_age)
+    )
     decompress = False
     if base64d[:1] == b".":
         # It's compressed; uncompress it first
@@ -74,7 +84,7 @@ class TimestampSigner(Signer, TsS):
         """
         result = super(TimestampSigner, self).unsign(value)
         value, timestamp = result.rsplit(self.sep, 1)
-        timestamp = base62.decode(timestamp)
+        timestamp = b62_decode(timestamp)
         if max_age is not None:
             if isinstance(max_age, datetime.timedelta):
                 max_age = max_age.total_seconds()

django_q/exceptions.py

@@ -0,0 +1,7 @@
+class TimeoutException(SystemExit):
+    """
+    Exception for when a worker takes too long to complete a task
+    Raising SystemExit will make sure the function terminates gracefully.
+    """
+
+    pass

django_q/humanhash.py

@@ -4,6 +4,7 @@ humanhash: Human-readable representations of digests.
 The simplest ways to use this module are the :func:`humanize` and :func:`uuid`
 functions. For tighter control over the output, see :class:`HumanHasher`.
 """
+
 import operator
 import uuid as uuidlib
 from argparse import ArgumentError
@@ -270,7 +271,6 @@ DEFAULT_WORDLIST = (
 
 
 class HumanHasher:
-
     """
     Transforms hex digests to human-readable strings.
 
@@ -290,7 +290,6 @@ class HumanHasher:
         self.wordlist = wordlist
 
     def humanize(self, hexdigest, words=4, separator="-"):
-
         """
         Humanize a given hexadecimal digest.
 
@@ -314,7 +313,6 @@ class HumanHasher:
 
     @staticmethod
     def compress(bytes, target):
-
         """
         Compress a list of byte values to a fixed target length.
 
@@ -353,7 +351,6 @@ class HumanHasher:
         return checksums
 
     def uuid(self, **params):
-
         """
         Generate a UUID with a human-readable representation.
 

django_q/locale/de/LC_MESSAGES/django.po

@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: \n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2022-11-12 01:47+0000\n"
+"POT-Creation-Date: 2023-01-26 01:38+0000\n"
 "PO-Revision-Date: 2018-08-05 18:28+0200\n"
 "Last-Translator: Jonas Winkler\n"
 "Language-Team: \n"
@@ -17,177 +17,176 @@ msgstr ""
 "X-Generator: Poedit 2.1.1\n"
 "Plural-Forms: nplurals=2; plural=(n > 1);\n"
 
-#: django_q/admin.py:43
+#: admin.py:43
 msgid "Resubmit selected tasks to queue"
 msgstr "Ausgewählte Aufgaben erneut ausführen"
 
-#: django_q/admin.py:98 django_q/models.py:228
+#: admin.py:107 models.py:293
 #, fuzzy
 #| msgid "Success"
 msgid "success"
 msgstr "erfolg"
 
-#: django_q/admin.py:109 django_q/models.py:230
+#: admin.py:119 models.py:295
 msgid "last_run"
 msgstr ""
 
-#: django_q/cluster.py:77
+#: cluster.py:79
 #, python-format
 msgid "Q Cluster %(name)s starting."
 msgstr "Q-Cluster %(name)s wird gestartet."
 
-#: django_q/cluster.py:85
+#: cluster.py:87
 #, fuzzy, python-format
 #| msgid "Q Cluster-{} stopping."
 msgid "Q Cluster %(name)s stopping."
 msgstr "Q-Cluster {name} wird gestoppt."
 
-#: django_q/cluster.py:88
+#: cluster.py:90
 #, python-format
 msgid "Q Cluster %(name)s has stopped."
 msgstr "Q-Cluster %(name)s wurde gestoppt."
 
-#: django_q/cluster.py:96
+#: cluster.py:97
 #, python-format
 msgid "%(name)s got signal %(signal)s"
 msgstr "%(name)s erhielt das Signal %(signal)s"
 
-#: django_q/cluster.py:219
+#: cluster.py:224
 #, python-format
 msgid "reincarnated monitor %(name)s after sudden death"
 msgstr "Monitor %(name)s wurde nach unerwartetem Absturz neu gestartet"
 
-#: django_q/cluster.py:222
+#: cluster.py:230
 #, python-format
 msgid "reincarnated pusher %(name)s after sudden death"
 msgstr "Pusher %(name)s wurde nach unerwartetem Absturz neu gestartet"
 
-#: django_q/cluster.py:229
+#: cluster.py:250
+#, fuzzy, python-format
+#| msgid "reincarnated worker %(name)s after timeout"
+msgid ""
+"reincarnated worker %(name)s after timeout while processing task "
+"%(task_name)s"
+msgstr "Worker %(name)s wurde nach Zeitüberschreitung neu gestartet"
+
+#: cluster.py:255
 #, python-format
 msgid "reincarnated worker %(name)s after timeout"
 msgstr "Worker %(name)s wurde nach Zeitüberschreitung neu gestartet"
 
-#: django_q/cluster.py:231
+#: cluster.py:260
 #, python-format
 msgid "recycled worker %(name)s"
 msgstr "Worker %(name)s wurde wiederverwendet"
 
-#: django_q/cluster.py:233
+#: cluster.py:263
 #, python-format
 msgid "reincarnated worker %(name)s after death"
 msgstr "Worker %(name)s wurde nach unerwartetem Absturz neu gestartet"
 
-#: django_q/cluster.py:256
+#: cluster.py:287
 #, python-format
 msgid "%(name)s guarding cluster %(cluster_name)s"
 msgstr "%(name)s beschützt das Cluster %(cluster_name)s"
 
-#: django_q/cluster.py:261
+#: cluster.py:296
 #, python-format
 msgid "Q Cluster %(cluster_name)s running."
 msgstr "Q-Cluster %(cluster_name)s läuft."
 
-#: django_q/cluster.py:295
+#: cluster.py:332
 #, python-format
 msgid "%(name)s stopping cluster processes"
 msgstr "%(name)s hält Cluster-Prozesse an"
 
-#: django_q/cluster.py:320
+#: cluster.py:357
 #, python-format
 msgid "%(name)s waiting for the monitor."
 msgstr "%(name)s wartet auf den Monitor."
 
-#: django_q/cluster.py:342
-#, python-format
-msgid "%(process_name)s pushing tasks at %(id)s"
+#: cluster.py:383
+#, fuzzy, python-format
+#| msgid "%(process_name)s pushing tasks at %(id)s"
+msgid "%(name)s pushing tasks at %(id)s"
 msgstr "%(process_name)s veröffentlicht Aufagaben auf %(id)s"
 
-#: django_q/cluster.py:363
+#: cluster.py:407
 #, python-format
 msgid "queueing from %(list_key)s"
 msgstr "Einreihen von %(list_key)s"
 
-#: django_q/cluster.py:366
+#: cluster.py:411
 #, python-format
 msgid "%(name)s stopped pushing tasks"
 msgstr "%(name)s veröffentlicht keine Aufgaben mehr"
 
-#: django_q/cluster.py:378
+#: cluster.py:426
 #, python-format
 msgid "%(name)s monitoring at %(id)s"
 msgstr "%(name)s beobachtet auf %(id)s"
 
-#: django_q/cluster.py:395
+#: cluster.py:445
 #, python-format
 msgid "Processed '%(info_name)s' (%(task_name)s)"
 msgstr "[%(task_name)s] - '%(info_name)s' wurde verarbeitet"
 
-#: django_q/cluster.py:398
+#: cluster.py:451
 #, python-format
 msgid "Failed '%(info_name)s' (%(task_name)s) - %(task_result)s"
 msgstr "'%(info_name)s' (%(task_name)s) ist fehlgeschlagen - %(task_result)s"
 
-#: django_q/cluster.py:399
+#: cluster.py:458
 #, python-format
 msgid "%(name)s stopped monitoring results"
 msgstr "%(name)s überwacht keine Ergebnisse mehr"
 
-#: django_q/cluster.py:413
+#: cluster.py:474
 #, python-format
 msgid "%(proc_name)s ready for work at %(id)s"
 msgstr "%(proc_name)s ist bereit für Arbeit auf %(id)s"
 
-#: django_q/cluster.py:425
-#, python-format
-msgid "%(proc_name)s processing '%(func_name)s' (%(task_name)s)"
+#: cluster.py:494
+#, fuzzy, python-format
+#| msgid "%(proc_name)s processing '%(func_name)s' (%(task_name)s)"
+msgid "%(proc_name)s processing %(task_name)s '%(func_name)s'"
 msgstr "%(proc_name)s verarbeitet '%(func_name)s' (%(task_name)s)"
 
-#: django_q/cluster.py:440
-#, python-format
-msgid ""
-"Could not process '%(func_name)s'. Check the location of the function and "
-"the args/kwargs."
-msgstr ""
-"Konnte '%(func_name)s' nicht verarbeiten. Überprüfen Sie den Ort der "
-"Funktion und die args/kwargs."
-
-#: django_q/cluster.py:456
+#: cluster.py:546
 #, python-format
 msgid "%(proc_name)s stopped doing work"
 msgstr "%(proc_name)s hat die Arbeit beendet"
 
-#: django_q/cluster.py:649 django_q/models.py:144
-msgid "Please install croniter to enable cron expressions"
-msgstr "Bitte installieren Sie croniter, um Cron-Ausdrücke zu aktivieren"
-
-#: django_q/cluster.py:672
+#: cluster.py:751
 #, python-format
 msgid "%(process_name)s failed to create a task from schedule [%(schedule)s]"
 msgstr ""
 "%(process_name)s konnte keine Aufgabe von Zeitplan [%(schedule)s] erstellen"
 
-#: django_q/cluster.py:678
-#, python-format
-msgid "%(process_name)s created a task from schedule [%(schedule)s]"
+#: cluster.py:762
+#, fuzzy, python-format
+#| msgid "%(process_name)s created a task from schedule [%(schedule)s]"
+msgid ""
+"%(process_name)s created task %(task_name)s from schedule [%(schedule)s]"
 msgstr ""
 "%(process_name)s hat eine Aufgabe des Zeitplans [%(schedule)s] erstellt"
 
-#: django_q/cluster.py:718
+#: cluster.py:808
 msgid "Skipping cpu affinity because psutil was not found."
 msgstr "Cpu-Affinität wird übersprungen, da psutil nicht gefunden wurde."
 
-#: django_q/cluster.py:723
+#: cluster.py:813
 msgid "Faking cpu affinity because it is not supported on this platform"
 msgstr ""
 "Vortäuschen von CPU-Affinität, da diese auf dieser Plattform nicht "
 "unterstützt wird"
 
-#: django_q/cluster.py:744
+#: cluster.py:835
 #, python-format
 msgid "%(pid)s will use cpu %(affinity)s"
 msgstr "%(pid)s wird CPU %(affinity)s benutzen"
 
-#: django_q/conf.py:85
+#: conf.py:90
 #, python-format
 msgid ""
 "SAVE_LIMIT_PER (%(option)s) is not a valid option. Options are: 'group', "
@@ -197,291 +196,307 @@ msgstr ""
 "'group', 'name', 'func' und None. Standard ist None."
 
 #. Translators: Cluster status descriptions
-#: django_q/conf.py:194
+#: conf.py:207
 msgid "Starting"
 msgstr "Wird gestartet"
 
-#: django_q/conf.py:195
+#: conf.py:208
 msgid "Working"
 msgstr "Arbeitet"
 
-#: django_q/conf.py:196
+#: conf.py:209
 msgid "Idle"
 msgstr "Leerlauf"
 
-#: django_q/conf.py:197
+#: conf.py:210
 msgid "Stopped"
 msgstr "Gestoppt"
 
-#: django_q/conf.py:198
+#: conf.py:211
 msgid "Stopping"
 msgstr "Wird gestoppt"
 
 #. Translators: help text for qcluster management command
-#: django_q/management/commands/qcluster.py:9
+#: management/commands/qcluster.py:9
 msgid "Starts a Django Q Cluster."
 msgstr "Startet ein Django-Q-Cluster."
 
 #. Translators: help text for qinfo management command
-#: django_q/management/commands/qinfo.py:11
+#: management/commands/qinfo.py:11
 msgid "General information over all clusters."
 msgstr "Allgemeine Informationen über alle Cluster"
 
 #. Translators: help text for qmemory management command
-#: django_q/management/commands/qmemory.py:9
+#: management/commands/qmemory.py:9
 msgid "Monitors Q Cluster memory usage"
 msgstr "Überwacht die Speichernutzung von Q Cluster"
 
 #. Translators: help text for qmonitor management command
-#: django_q/management/commands/qmonitor.py:9
+#: management/commands/qmonitor.py:9
 msgid "Monitors Q Cluster activity"
 msgstr "Q-Cluster aktiv überwachen"
 
-#: django_q/models.py:119
+#: models.py:125
 msgid "Successful task"
 msgstr "Erfolgreiche Aufgabe"
 
-#: django_q/models.py:120
+#: models.py:126
 msgid "Successful tasks"
 msgstr "Erfolgreiche Aufgaben"
 
-#: django_q/models.py:135
+#: models.py:141
 msgid "Failed task"
 msgstr "Fehlgeschlagene Aufgabe"
 
-#: django_q/models.py:136
+#: models.py:142
 msgid "Failed tasks"
 msgstr "Fehlgeschlagene Aufgaben"
 
-#: django_q/models.py:160
+#: models.py:150 models.py:234
+msgid "Please install croniter to enable cron expressions"
+msgstr "Bitte installieren Sie croniter, um Cron-Ausdrücke zu aktivieren"
+
+#: models.py:170
 msgid "e.g. 1, 2, 'John'"
 msgstr "zum Beispiel 1, 2, 'John'"
 
-#: django_q/models.py:162
+#: models.py:172
 msgid "e.g. x=1, y=2, name='John'"
 msgstr "zum Beispiel x=1, y=2, name='John'"
 
-#: django_q/models.py:176
+#: models.py:186
 msgid "Once"
 msgstr "Einmal"
 
-#: django_q/models.py:177
+#: models.py:187
 msgid "Minutes"
 msgstr "Minuten"
 
-#: django_q/models.py:178
+#: models.py:188
 msgid "Hourly"
 msgstr "Stündlich"
 
-#: django_q/models.py:179
+#: models.py:189
 msgid "Daily"
 msgstr "Täglich"
 
-#: django_q/models.py:180
+#: models.py:190
 msgid "Weekly"
 msgstr "Wöchentlich"
 
-#: django_q/models.py:181
+#: models.py:191
 msgid "Biweekly"
 msgstr "Zweiwöchentlich"
 
-#: django_q/models.py:182
+#: models.py:192
 msgid "Monthly"
 msgstr "Monatlich"
 
-#: django_q/models.py:183
+#: models.py:193
 msgid "Bimonthly"
 msgstr "Zweimonatlich"
 
-#: django_q/models.py:184
+#: models.py:194
 msgid "Quarterly"
 msgstr "Vierteljährlich"
 
-#: django_q/models.py:185
+#: models.py:195
 msgid "Yearly"
 msgstr "Jährlich"
 
-#: django_q/models.py:186
+#: models.py:196
 msgid "Cron"
 msgstr "Cron"
 
-#: django_q/models.py:189
+#: models.py:199
 msgid "Schedule Type"
 msgstr "Zeitplan-Typ"
 
-#: django_q/models.py:192
+#: models.py:202
 msgid "Number of minutes for the Minutes type"
 msgstr "Anzahl Minuten für den Typ 'Minuten'"
 
-#: django_q/models.py:195
+#: models.py:205
 msgid "Repeats"
-msgstr "Wiederhohlungen"
+msgstr "Wiederholungen"
 
-#: django_q/models.py:195
+#: models.py:205
 msgid "n = n times, -1 = forever"
 msgstr "n = n mal, -1 = für immer"
 
-#: django_q/models.py:198
+#: models.py:208
 msgid "Next Run"
 msgstr "Nächste Ausführung"
 
-#: django_q/models.py:205
+#: models.py:215
 msgid "Cron expression"
 msgstr "Cron-Ausdruck"
 
-#: django_q/models.py:235
+#: models.py:224
+msgid "Name of kwarg to pass intended schedule date"
+msgstr "Name des zu passierenden Kwargs vorgesehenes Datum"
+
+#: models.py:299
 msgid "Scheduled task"
 msgstr "Geplante Aufgabe"
 
-#: django_q/models.py:236
+#: models.py:300
 msgid "Scheduled tasks"
 msgstr "Geplante Aufgaben"
 
-#: django_q/models.py:262
+#: models.py:326
 msgid "Queued task"
 msgstr "Eingereihte Aufgabe"
 
-#: django_q/models.py:263
+#: models.py:327
 msgid "Queued tasks"
 msgstr "Eingereihte Aufgaben"
 
-#: django_q/monitor.py:62 django_q/monitor.py:339
+#: monitor.py:64 monitor.py:348
 msgid "Host"
 msgstr "Host"
 
-#: django_q/monitor.py:66 django_q/monitor.py:343 django_q/monitor.py:450
+#: monitor.py:68 monitor.py:352 monitor.py:459
 msgid "Id"
 msgstr "Id"
 
-#: django_q/monitor.py:70
+#: monitor.py:72
 msgid "State"
 msgstr "Status"
 
-#: django_q/monitor.py:74
+#: monitor.py:76
 msgid "Pool"
 msgstr "Pool"
 
-#: django_q/monitor.py:78
+#: monitor.py:80
 msgid "TQ"
 msgstr "TQ"
 
-#: django_q/monitor.py:82
+#: monitor.py:84
 msgid "RQ"
 msgstr "RQ"
 
-#: django_q/monitor.py:86
+#: monitor.py:88
 msgid "RC"
 msgstr "RC"
 
-#: django_q/monitor.py:90
+#: monitor.py:92
 msgid "Up"
 msgstr "Up"
 
-#: django_q/monitor.py:170 django_q/monitor.py:279
+#: monitor.py:172 monitor.py:286
 msgid "Queued"
 msgstr "Eingereiht"
 
-#: django_q/monitor.py:178
+#: monitor.py:180
 msgid "Success"
 msgstr "Erfolg"
 
-#: django_q/monitor.py:188 django_q/monitor.py:287
+#: monitor.py:190 monitor.py:294
 msgid "Failures"
 msgstr "Fehlschläge"
 
-#: django_q/monitor.py:199 django_q/monitor.py:485
+#: monitor.py:201 monitor.py:498
 msgid "[Press q to quit]"
 msgstr "[Drücken Sie q zum Beenden]"
 
-#: django_q/monitor.py:223
+#: monitor.py:227
 msgid "day"
 msgstr "Tag"
 
-#: django_q/monitor.py:244
+#: monitor.py:248
 msgid "second"
 msgstr "Sekunde"
 
-#: django_q/monitor.py:247
+#: monitor.py:251
 msgid "minute"
 msgstr "Minute"
 
-#: django_q/monitor.py:250
+#: monitor.py:254
 msgid "hour"
 msgstr "Stunde"
 
-#: django_q/monitor.py:260
+#: monitor.py:263
 #, python-format
 msgid "-- %(prefix)s %(version)s on %(info)s --"
 msgstr "-- %(prefix)s %(version)s auf %(info)s --"
 
-#: django_q/monitor.py:266
+#: monitor.py:273
 msgid "Clusters"
 msgstr "Cluster"
 
-#: django_q/monitor.py:270
+#: monitor.py:277
 msgid "Workers"
 msgstr "Arbeiter"
 
-#: django_q/monitor.py:274
+#: monitor.py:281
 msgid "Restarts"
 msgstr "Neustarts"
 
-#: django_q/monitor.py:283
+#: monitor.py:290
 msgid "Successes"
 msgstr "Erfolge"
 
-#: django_q/monitor.py:292
+#: monitor.py:299
 msgid "Schedules"
 msgstr "Zeitpläne"
 
-#: django_q/monitor.py:296
+#: monitor.py:303
 #, python-format
 msgid "Tasks/%(per)s"
 msgstr "Aufgaben/%(per)s"
 
-#: django_q/monitor.py:300
+#: monitor.py:307
 msgid "Avg time"
 msgstr "Durchschnittl. Zeit"
 
-#: django_q/monitor.py:348
+#: monitor.py:357
 msgid "Available (%)"
 msgstr "Verfügbar (%)"
 
-#: django_q/monitor.py:354
+#: monitor.py:363
 msgid "Available (MB)"
 msgstr "Verfügbar (MB)"
 
-#: django_q/monitor.py:359
+#: monitor.py:368
 msgid "Total (MB)"
 msgstr "Insgesamt (MB)"
 
-#: django_q/monitor.py:364
+#: monitor.py:373
 msgid "Sentinel (MB)"
 msgstr "Sentinel (MB)"
 
-#: django_q/monitor.py:370
+#: monitor.py:379
 msgid "Monitor (MB)"
 msgstr "Monitor (MB)"
 
-#: django_q/monitor.py:376
+#: monitor.py:385
 msgid "Workers (MB)"
 msgstr "Arbeiter (MB)"
 
-#: django_q/monitor.py:478
+#: monitor.py:487
 #, python-format
 msgid "Available lowest (): %(memory_percent)s ((at)s)"
 msgstr "Niedrigste verfügbar (): %(memory_percent)s ((at)s)"
 
-#: django_q/monitor.py:496
+#: monitor.py:509
 msgid "No clusters appear to be running."
 msgstr "Es scheinen keine Cluster zu laufen."
 
-#: django_q/signals.py:22
+#: signals.py:22
 #, python-format
 msgid "malformed return hook '%(hook)s' for [%(name)s]"
 msgstr "Ungültiger Return-Hook '%(hook)s' für [%(name)s]"
 
-#: django_q/signals.py:30
+#: signals.py:30
 #, python-format
 msgid "return hook %(hook)s failed on [%(name)s] because %(error)s"
 msgstr "Return-Hook %(hook)s für [%(name)s] ist gescheitert: %(error)s"
+
+#, python-format
+#~ msgid ""
+#~ "Could not process '%(func_name)s'. Check the location of the function and "
+#~ "the args/kwargs."
+#~ msgstr ""
+#~ "Konnte '%(func_name)s' nicht verarbeiten. Überprüfen Sie den Ort der "
+#~ "Funktion und die args/kwargs."

django_q/locale/fr/LC_MESSAGES/django.po

@@ -6,7 +6,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: \n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2022-11-12 01:47+0000\n"
+"POT-Creation-Date: 2023-01-26 01:38+0000\n"
 "PO-Revision-Date: 2018-08-05 18:28+0200\n"
 "Last-Translator: Thierry BOULOGNE <contact@tng-concepts.com>\n"
 "Language-Team: \n"
@@ -17,176 +17,175 @@ msgstr ""
 "X-Generator: Poedit 2.1.1\n"
 "Plural-Forms: nplurals=2; plural=(n > 1);\n"
 
-#: django_q/admin.py:43
+#: admin.py:43
 msgid "Resubmit selected tasks to queue"
 msgstr "Resoumettre les tâches sélectionnées à la file d'attente"
 
-#: django_q/admin.py:98 django_q/models.py:228
+#: admin.py:107 models.py:293
 #, fuzzy
 #| msgid "Success"
 msgid "success"
 msgstr "succès"
 
-#: django_q/admin.py:109 django_q/models.py:230
+#: admin.py:119 models.py:295
 msgid "last_run"
 msgstr ""
 
-#: django_q/cluster.py:77
+#: cluster.py:79
 #, python-format
 msgid "Q Cluster %(name)s starting."
 msgstr "Démarrage de Q Cluster-%(name)s."
 
-#: django_q/cluster.py:85
+#: cluster.py:87
 #, python-format
 msgid "Q Cluster %(name)s stopping."
 msgstr "Arrêt de Q Cluster-%(name)s."
 
-#: django_q/cluster.py:88
+#: cluster.py:90
 #, python-format
 msgid "Q Cluster %(name)s has stopped."
 msgstr "Q Cluster-%(name)s a été arrêté."
 
-#: django_q/cluster.py:96
+#: cluster.py:97
 #, python-format
 msgid "%(name)s got signal %(signal)s"
 msgstr "%(name)s à reçu le signal %(signal)s"
 
-#: django_q/cluster.py:219
+#: cluster.py:224
 #, python-format
 msgid "reincarnated monitor %(name)s after sudden death"
-msgstr "moniteur réintégré %(name)s après un arrêt intempestif"
+msgstr "surveillant %(name)s réincarné après un arrêt intempestif"
 
-#: django_q/cluster.py:222
+#: cluster.py:230
 #, python-format
 msgid "reincarnated pusher %(name)s after sudden death"
-msgstr "pousseur réintégré %(name)s après un arrêt intempestif"
+msgstr "répartiteur %(name)s réincarné après un arrêt intempestif"
 
-#: django_q/cluster.py:229
+#: cluster.py:250
+#, python-format
+msgid ""
+"reincarnated worker %(name)s after timeout while processing task "
+"%(task_name)s"
+msgstr ""
+"processus %(name)s réincarné, délai de traitement dépassé pour la tâche "
+"%(task_name)s"
+
+#: cluster.py:255
 #, python-format
 msgid "reincarnated worker %(name)s after timeout"
-msgstr "processus réintégré %(name)s après un arrêt une attente trop longue"
+msgstr "processus %(name)s réincarné, délai de traitement dépassé"
 
-#: django_q/cluster.py:231
+#: cluster.py:260
 #, python-format
 msgid "recycled worker %(name)s"
 msgstr "processus recyclé %(name)s"
 
-#: django_q/cluster.py:233
+#: cluster.py:263
 #, python-format
 msgid "reincarnated worker %(name)s after death"
 msgstr "processus réintégré %(name)s après arrêt"
 
-#: django_q/cluster.py:256
+#: cluster.py:287
 #, python-format
 msgid "%(name)s guarding cluster %(cluster_name)s"
 msgstr "%(name)s surveillance du cluster à %(cluster_name)s"
 
-#: django_q/cluster.py:261
+#: cluster.py:296
 #, python-format
 msgid "Q Cluster %(cluster_name)s running."
 msgstr "Démarrage de Q Cluster-%(cluster_name)s."
 
-#: django_q/cluster.py:295
+#: cluster.py:332
 #, python-format
 msgid "%(name)s stopping cluster processes"
-msgstr "%(name)s arrêt des processus de cluster"
+msgstr "%(name)s arrêt des processus du cluster"
 
-#: django_q/cluster.py:320
+#: cluster.py:357
 #, python-format
 msgid "%(name)s waiting for the monitor."
-msgstr "%(name)s en attente du moniteur."
+msgstr "%(name)s en attente du surveillant."
 
-#: django_q/cluster.py:342
+#: cluster.py:383
 #, python-format
-msgid "%(process_name)s pushing tasks at %(id)s"
-msgstr "%(process_name)s tâche envoyé à %(id)s"
+msgid "%(name)s pushing tasks at %(id)s"
+msgstr "%(name)s répartit les tâches %(id)s"
 
-#: django_q/cluster.py:363
+#: cluster.py:407
 #, python-format
 msgid "queueing from %(list_key)s"
 msgstr "mise en file d'attente de %(list_key)s"
 
-#: django_q/cluster.py:366
+#: cluster.py:411
 #, python-format
 msgid "%(name)s stopped pushing tasks"
-msgstr "%(name)s a cessé de pousser les tâches"
+msgstr "%(name)s a cessé de répartir les tâches"
 
-#: django_q/cluster.py:378
+#: cluster.py:426
 #, python-format
 msgid "%(name)s monitoring at %(id)s"
-msgstr "%(name)s Surveillance de %(id)s"
+msgstr "%(name)s surveille les résultats %(id)s"
 
-#: django_q/cluster.py:395
+#: cluster.py:445
 #, python-format
 msgid "Processed '%(info_name)s' (%(task_name)s)"
 msgstr "traité '%(info_name)s' (%(task_name)s)"
 
-#: django_q/cluster.py:398
+#: cluster.py:451
 #, python-format
 msgid "Failed '%(info_name)s' (%(task_name)s) - %(task_result)s"
 msgstr "Manqué '%(info_name)s' (%(task_name)s) - %(task_result)s"
 
-#: django_q/cluster.py:399
+#: cluster.py:458
 #, python-format
 msgid "%(name)s stopped monitoring results"
-msgstr "%(name)s arrêt des résultats de surveillance"
+msgstr "%(name)s a cessé de de surveiller les résultats"
 
-#: django_q/cluster.py:413
+#: cluster.py:474
 #, python-format
 msgid "%(proc_name)s ready for work at %(id)s"
 msgstr "%(proc_name)s prêt pour le travail à %(id)s"
 
-#: django_q/cluster.py:425
-#, python-format
-msgid "%(proc_name)s processing '%(func_name)s' (%(task_name)s)"
-msgstr "%(proc_name)s en traitement '%(func_name)s' (%(task_name)s)"
+#: cluster.py:494
+#, fuzzy, python-format
+msgid "%(proc_name)s processing %(task_name)s '%(func_name)s'"
+msgstr "%(proc_name)s exécute %(task_name)s '%(func_name)s'"
 
-#: django_q/cluster.py:440
-#, python-format
-msgid ""
-"Could not process '%(func_name)s'. Check the location of the function and "
-"the args/kwargs."
-msgstr ""
-"Impossible de traiter '%(func_name)s'. Vérifiez l'emplacement de la fonction "
-"et les args/kwargs."
-
-#: django_q/cluster.py:456
+#: cluster.py:546
 #, python-format
 msgid "%(proc_name)s stopped doing work"
-msgstr "%(proc_name)s arrêté de travailler"
+msgstr "%(proc_name)s a cessé de travailler"
 
-#: django_q/cluster.py:649 django_q/models.py:144
-msgid "Please install croniter to enable cron expressions"
-msgstr "Veuillez installer croniter pour activer les expressions croniques."
-
-#: django_q/cluster.py:672
+#: cluster.py:751
 #, python-format
 msgid "%(process_name)s failed to create a task from schedule [%(schedule)s]"
 msgstr ""
 "%(process_name)s Echec de la création d'une tâche à partir de Schedule "
 "[%(schedule)s]"
 
-#: django_q/cluster.py:678
+#: cluster.py:762
 #, python-format
-msgid "%(process_name)s created a task from schedule [%(schedule)s]"
-msgstr "%(process_name)s a créé une tâche à partir de Schedule [%(schedule)s]"
+msgid ""
+"%(process_name)s created task %(task_name)s from schedule [%(schedule)s]"
+msgstr ""
+"%(process_name)s a créé la tâche %(task_name)s à partir de Schedule "
+"[%(schedule)s]"
 
-#: django_q/cluster.py:718
+#: cluster.py:808
 msgid "Skipping cpu affinity because psutil was not found."
-msgstr "Sauter l'affinité du processeur parce que psutil n'a pas été trouvé."
+msgstr "L'affinité cpu ne sera pas définie car psutil n'a pas été trouvé."
 
-#: django_q/cluster.py:723
+#: cluster.py:813
 msgid "Faking cpu affinity because it is not supported on this platform"
 msgstr ""
-"Simulation de l'affinité du processeur parce qu'elle n'est pas supportée sur "
-"cette plateforme."
+"Simulation de l'affinité cpu parce qu'elle n'est pas supportée sur cette "
+"plateforme."
 
-#: django_q/cluster.py:744
+#: cluster.py:835
 #, python-format
 msgid "%(pid)s will use cpu %(affinity)s"
 msgstr "%(pid)s utilisera le CPU %(affinity)s"
 
-#: django_q/conf.py:85
+#: conf.py:90
 #, python-format
 msgid ""
 "SAVE_LIMIT_PER (%(option)s) is not a valid option. Options are: 'group', "
@@ -196,299 +195,315 @@ msgstr ""
 "'group', 'name', 'func' et None. La valeur par défaut est None."
 
 #. Translators: Cluster status descriptions
-#: django_q/conf.py:194
+#: conf.py:207
 msgid "Starting"
 msgstr "Démarrage"
 
-#: django_q/conf.py:195
+#: conf.py:208
 msgid "Working"
 msgstr "Actif"
 
-#: django_q/conf.py:196
+#: conf.py:209
 msgid "Idle"
 msgstr "En attente"
 
-#: django_q/conf.py:197
+#: conf.py:210
 msgid "Stopped"
 msgstr "Arrêté"
 
-#: django_q/conf.py:198
+#: conf.py:211
 msgid "Stopping"
 msgstr "En cours d’arrêt"
 
 #. Translators: help text for qcluster management command
-#: django_q/management/commands/qcluster.py:9
+#: management/commands/qcluster.py:9
 msgid "Starts a Django Q Cluster."
 msgstr "Démarre un cluster Django Q."
 
 #. Translators: help text for qinfo management command
-#: django_q/management/commands/qinfo.py:11
+#: management/commands/qinfo.py:11
 msgid "General information over all clusters."
 msgstr "Informations générales sur tous les clusters."
 
 #. Translators: help text for qmemory management command
-#: django_q/management/commands/qmemory.py:9
+#: management/commands/qmemory.py:9
 #, fuzzy
 #| msgid "Monitors Q Cluster activity"
 msgid "Monitors Q Cluster memory usage"
-msgstr "Surveille l'utilisation de la mémoire du cluster Q"
+msgstr "Surveille l'utilisation mémoire du Q cluster"
 
 #. Translators: help text for qmonitor management command
-#: django_q/management/commands/qmonitor.py:9
+#: management/commands/qmonitor.py:9
 msgid "Monitors Q Cluster activity"
-msgstr "Activité du cluster Moniteur Q"
+msgstr "Surveille l'activité de Q cluster"
 
-#: django_q/models.py:119
+#: models.py:125
 msgid "Successful task"
 msgstr "Tâche réussie"
 
-#: django_q/models.py:120
+#: models.py:126
 msgid "Successful tasks"
 msgstr "Tâches réussies"
 
-#: django_q/models.py:135
+#: models.py:141
 msgid "Failed task"
 msgstr "Tâche échoué"
 
-#: django_q/models.py:136
+#: models.py:142
 msgid "Failed tasks"
 msgstr "Tâches échouées"
 
-#: django_q/models.py:160
+#: models.py:150 models.py:234
+msgid "Please install croniter to enable cron expressions"
+msgstr "Veuillez installer croniter pour activer les expressions cron."
+
+#: models.py:170
 msgid "e.g. 1, 2, 'John'"
 msgstr "ex. 1, 2, ‘Jean’"
 
-#: django_q/models.py:162
+#: models.py:172
 msgid "e.g. x=1, y=2, name='John'"
 msgstr "p. ex. x = 1, y = 2, Nom = ‘Jean’"
 
-#: django_q/models.py:176
+#: models.py:186
 msgid "Once"
 msgstr "Une fois"
 
-#: django_q/models.py:177
+#: models.py:187
 msgid "Minutes"
 msgstr "Minutes"
 
-#: django_q/models.py:178
+#: models.py:188
 msgid "Hourly"
 msgstr "Toutes les heures"
 
-#: django_q/models.py:179
+#: models.py:189
 msgid "Daily"
 msgstr "Quotidien"
 
-#: django_q/models.py:180
+#: models.py:190
 msgid "Weekly"
 msgstr "Hebdomadaire"
 
-#: django_q/models.py:181
+#: models.py:191
 #, fuzzy
 #| msgid "Weekly"
 msgid "Biweekly"
 msgstr "Bihebdomadaire"
 
-#: django_q/models.py:182
+#: models.py:192
 msgid "Monthly"
 msgstr "Mensuel"
 
-#: django_q/models.py:183
+#: models.py:193
 #, fuzzy
 #| msgid "Monthly"
 msgid "Bimonthly"
 msgstr "Bimestriel"
 
-#: django_q/models.py:184
+#: models.py:194
 msgid "Quarterly"
-msgstr "Tous les quart-d’heure"
+msgstr "Trimestriel"
 
-#: django_q/models.py:185
+#: models.py:195
 msgid "Yearly"
 msgstr "Annuel"
 
-#: django_q/models.py:186
+#: models.py:196
 msgid "Cron"
 msgstr "Cron"
 
-#: django_q/models.py:189
+#: models.py:199
 msgid "Schedule Type"
 msgstr "Type de plannification"
 
-#: django_q/models.py:192
+#: models.py:202
 msgid "Number of minutes for the Minutes type"
 msgstr "Nombre de minutes pour le type de minutes"
 
-#: django_q/models.py:195
+#: models.py:205
 msgid "Repeats"
 msgstr "Répéter"
 
-#: django_q/models.py:195
+#: models.py:205
 msgid "n = n times, -1 = forever"
 msgstr "n = n fois,-1 = Toujours"
 
-#: django_q/models.py:198
+#: models.py:208
 msgid "Next Run"
 msgstr "Prochaine exécution"
 
-#: django_q/models.py:205
+#: models.py:215
 msgid "Cron expression"
-msgstr "Expression du Cron"
+msgstr "Expression Cron"
 
-#: django_q/models.py:235
+#: models.py:224
+msgid "Name of kwarg to pass intended schedule date"
+msgstr "Nom du kwarg à passer Date prévue de l'horaire"
+
+#: models.py:299
 msgid "Scheduled task"
 msgstr "Tâche planifiée"
 
-#: django_q/models.py:236
+#: models.py:300
 msgid "Scheduled tasks"
 msgstr "Tâches planifiées"
 
-#: django_q/models.py:262
+#: models.py:326
 msgid "Queued task"
 msgstr "Tâche en file d'attente"
 
-#: django_q/models.py:263
+#: models.py:327
 msgid "Queued tasks"
 msgstr "Tâches en file d'attente"
 
-#: django_q/monitor.py:62 django_q/monitor.py:339
+#: monitor.py:64 monitor.py:348
 msgid "Host"
 msgstr "Hôte"
 
-#: django_q/monitor.py:66 django_q/monitor.py:343 django_q/monitor.py:450
+#: monitor.py:68 monitor.py:352 monitor.py:459
 msgid "Id"
 msgstr "Id"
 
-#: django_q/monitor.py:70
+#: monitor.py:72
 msgid "State"
 msgstr "Statut"
 
-#: django_q/monitor.py:74
+#: monitor.py:76
 msgid "Pool"
 msgstr "Piscine"
 
-#: django_q/monitor.py:78
+#: monitor.py:80
 msgid "TQ"
 msgstr "TQ"
 
-#: django_q/monitor.py:82
+#: monitor.py:84
 msgid "RQ"
 msgstr "RQ"
 
-#: django_q/monitor.py:86
+#: monitor.py:88
 msgid "RC"
 msgstr "RC"
 
-#: django_q/monitor.py:90
+#: monitor.py:92
 msgid "Up"
 msgstr "Haut"
 
-#: django_q/monitor.py:170 django_q/monitor.py:279
+#: monitor.py:172 monitor.py:286
 msgid "Queued"
 msgstr "En file d'attente"
 
-#: django_q/monitor.py:178
+#: monitor.py:180
 msgid "Success"
 msgstr "Succès"
 
-#: django_q/monitor.py:188 django_q/monitor.py:287
+#: monitor.py:190 monitor.py:294
 msgid "Failures"
 msgstr "Défaillances"
 
-#: django_q/monitor.py:199 django_q/monitor.py:485
+#: monitor.py:201 monitor.py:498
 msgid "[Press q to quit]"
 msgstr "[appuyez sur q pour quitter]"
 
-#: django_q/monitor.py:223
+#: monitor.py:227
 msgid "day"
 msgstr "jour"
 
-#: django_q/monitor.py:244
+#: monitor.py:248
 msgid "second"
 msgstr "seconde"
 
-#: django_q/monitor.py:247
+#: monitor.py:251
 msgid "minute"
 msgstr "minute"
 
-#: django_q/monitor.py:250
+#: monitor.py:254
 msgid "hour"
 msgstr "heure"
 
-#: django_q/monitor.py:260
+#: monitor.py:263
 #, python-format
 msgid "-- %(prefix)s %(version)s on %(info)s --"
 msgstr "--%(prefix)s %(version)s sur %(info)s --"
 
-#: django_q/monitor.py:266
+#: monitor.py:273
 msgid "Clusters"
 msgstr "Grappes"
 
-#: django_q/monitor.py:270
+#: monitor.py:277
 msgid "Workers"
 msgstr "Processus"
 
-#: django_q/monitor.py:274
+#: monitor.py:281
 msgid "Restarts"
 msgstr "Redémarrages"
 
-#: django_q/monitor.py:283
+#: monitor.py:290
 msgid "Successes"
 msgstr "Succès"
 
-#: django_q/monitor.py:292
+#: monitor.py:299
 msgid "Schedules"
 msgstr "Planifications"
 
-#: django_q/monitor.py:296
+#: monitor.py:303
 #, python-format
 msgid "Tasks/%(per)s"
 msgstr "Tâches/%(per)s"
 
-#: django_q/monitor.py:300
+#: monitor.py:307
 msgid "Avg time"
 msgstr "Temps Moyen"
 
-#: django_q/monitor.py:348
+#: monitor.py:357
 msgid "Available (%)"
 msgstr ""
 
-#: django_q/monitor.py:354
+#: monitor.py:363
 msgid "Available (MB)"
 msgstr "Disponible sur (MB)"
 
-#: django_q/monitor.py:359
+#: monitor.py:368
 msgid "Total (MB)"
 msgstr "Total (MB)"
 
-#: django_q/monitor.py:364
+#: monitor.py:373
 msgid "Sentinel (MB)"
 msgstr "Sentinel (MB)"
 
-#: django_q/monitor.py:370
+#: monitor.py:379
 msgid "Monitor (MB)"
 msgstr "Monitor (MB)"
 
-#: django_q/monitor.py:376
+#: monitor.py:385
 #, fuzzy
 #| msgid "Workers"
 msgid "Workers (MB)"
 msgstr "Processus (MB)"
 
-#: django_q/monitor.py:478
+#: monitor.py:487
 #, python-format
 msgid "Available lowest (): %(memory_percent)s ((at)s)"
 msgstr "Disponible le plus bas () : %(memory_percent)s ((at)s)"
 
-#: django_q/monitor.py:496
+#: monitor.py:509
 msgid "No clusters appear to be running."
 msgstr "Aucun cluster ne semble être en cours d'exécution."
 
-#: django_q/signals.py:22
+#: signals.py:22
 #, python-format
 msgid "malformed return hook '%(hook)s' for [%(name)s]"
-msgstr "hook de retour mal formé' %(hook)s 'pour [%(name)s]"
+msgstr "hook de retour '%(hook)s' mal formé pour [%(name)s]"
 
-#: django_q/signals.py:30
+#: signals.py:30
 #, python-format
 msgid "return hook %(hook)s failed on [%(name)s] because %(error)s"
-msgstr "le crochet de retour %(hook)s a échoué sur [%(name)s] parce que %(error)s"
+msgstr "hook de retour %(hook)s a échoué sur [%(name)s] à cause de %(error)s"
+
+#, python-format
+#~ msgid ""
+#~ "Could not process '%(func_name)s'. Check the location of the function and "
+#~ "the args/kwargs."
+#~ msgstr ""
+#~ "Impossible de traiter '%(func_name)s'. Vérifiez l'emplacement de la "
+#~ "fonction et les args/kwargs."

django_q/locale/tr/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: \n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2022-11-12 01:47+0000\n"
+"POT-Creation-Date: 2023-01-26 01:38+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: Ethem Güner <ethemguener@gmail.com>\n"
 "Language-Team: \n"
@@ -18,172 +18,171 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n > 1);\n"
 
-#: django_q/admin.py:43
+#: admin.py:43
 msgid "Resubmit selected tasks to queue"
 msgstr "Seçili işleri kuyruğa tekrar gönder"
 
-#: django_q/admin.py:98 django_q/models.py:228
+#: admin.py:107 models.py:293
 #, fuzzy
 #| msgid "Success"
 msgid "success"
 msgstr "başarılı olanlar"
 
-#: django_q/admin.py:109 django_q/models.py:230
+#: admin.py:119 models.py:295
 msgid "last_run"
 msgstr ""
 
-#: django_q/cluster.py:77
+#: cluster.py:79
 #, python-format
 msgid "Q Cluster %(name)s starting."
 msgstr "Q Cluster %(name)s başlatılıyor."
 
-#: django_q/cluster.py:85
+#: cluster.py:87
 #, python-format
 msgid "Q Cluster %(name)s stopping."
 msgstr "Q Cluster %(name)s durduruluyor."
 
-#: django_q/cluster.py:88
+#: cluster.py:90
 #, python-format
 msgid "Q Cluster %(name)s has stopped."
 msgstr "Q Cluster %(name)s durduruldu."
 
-#: django_q/cluster.py:96
+#: cluster.py:97
 #, python-format
 msgid "%(name)s got signal %(signal)s"
 msgstr "%(name)s, %(signal)s pid'inde izleniyor/monitoring yapılıyor."
 
-#: django_q/cluster.py:219
+#: cluster.py:224
 #, python-format
 msgid "reincarnated monitor %(name)s after sudden death"
 msgstr "Monitor %(name)s ani ölüm sonrası tekrar dirildi"
 
-#: django_q/cluster.py:222
+#: cluster.py:230
 #, python-format
 msgid "reincarnated pusher %(name)s after sudden death"
 msgstr "Pusher %(name)s ani ölüm sonrası tekrar dirildi"
 
-#: django_q/cluster.py:229
+#: cluster.py:250
+#, fuzzy, python-format
+#| msgid "reincarnated worker %(name)s after timeout"
+msgid ""
+"reincarnated worker %(name)s after timeout while processing task "
+"%(task_name)s"
+msgstr "Worker %(name)s zaman aşımı sonrası tekrar dirildi"
+
+#: cluster.py:255
 #, python-format
 msgid "reincarnated worker %(name)s after timeout"
 msgstr "Worker %(name)s zaman aşımı sonrası tekrar dirildi"
 
-#: django_q/cluster.py:231
+#: cluster.py:260
 #, python-format
 msgid "recycled worker %(name)s"
 msgstr "Worker %(name)s geri döndürüldü"
 
-#: django_q/cluster.py:233
+#: cluster.py:263
 #, python-format
 msgid "reincarnated worker %(name)s after death"
 msgstr "Worker %(name)s ani ölüm sonrası tekrar dirildi"
 
-#: django_q/cluster.py:256
+#: cluster.py:287
 #, python-format
 msgid "%(name)s guarding cluster %(cluster_name)s"
 msgstr "%(name)s, %(cluster_name)s cluster'ını koruyor"
 
-#: django_q/cluster.py:261
+#: cluster.py:296
 #, python-format
 msgid "Q Cluster %(cluster_name)s running."
 msgstr "Q Cluster %(cluster_name)s başlatılıyor."
 
-#: django_q/cluster.py:295
+#: cluster.py:332
 #, python-format
 msgid "%(name)s stopping cluster processes"
 msgstr "Cluster %(name)s işlemleri durduruluyor."
 
-#: django_q/cluster.py:320
+#: cluster.py:357
 #, python-format
 msgid "%(name)s waiting for the monitor."
 msgstr "%(name)s monitor için bekliyor."
 
-#: django_q/cluster.py:342
-#, python-format
-msgid "%(process_name)s pushing tasks at %(id)s"
+#: cluster.py:383
+#, fuzzy, python-format
+#| msgid "%(process_name)s pushing tasks at %(id)s"
+msgid "%(name)s pushing tasks at %(id)s"
 msgstr "%(process_name)s, işleri %(id)s pid'ine gönderiyor."
 
-#: django_q/cluster.py:363
+#: cluster.py:407
 #, python-format
 msgid "queueing from %(list_key)s"
 msgstr ""
 
-#: django_q/cluster.py:366
+#: cluster.py:411
 #, python-format
 msgid "%(name)s stopped pushing tasks"
 msgstr "%(name)s işleri göndermeyi durdurdu"
 
-#: django_q/cluster.py:378
+#: cluster.py:426
 #, python-format
 msgid "%(name)s monitoring at %(id)s"
 msgstr "%(name)s, %(id)s pid'inde izleniyor/monitoring yapılıyor."
 
-#: django_q/cluster.py:395
+#: cluster.py:445
 #, python-format
 msgid "Processed '%(info_name)s' (%(task_name)s)"
 msgstr "[%(task_name)s] - '%(info_name)s işlendi."
 
-#: django_q/cluster.py:398
+#: cluster.py:451
 #, python-format
 msgid "Failed '%(info_name)s' (%(task_name)s) - %(task_result)s"
 msgstr "[%(task_name)s] - '%(info_name)s' - %(task_result)s başarısız oldu"
 
-#: django_q/cluster.py:399
+#: cluster.py:458
 #, python-format
 msgid "%(name)s stopped monitoring results"
 msgstr "%(name)s sonuçları göstermeyi bıraktı"
 
-#: django_q/cluster.py:413
+#: cluster.py:474
 #, python-format
 msgid "%(proc_name)s ready for work at %(id)s"
 msgstr "%(proc_name)s, %(id)s pid'inde çalışmaya hazır"
 
-#: django_q/cluster.py:425
-#, python-format
-msgid "%(proc_name)s processing '%(func_name)s' (%(task_name)s)"
+#: cluster.py:494
+#, fuzzy, python-format
+#| msgid "%(proc_name)s processing '%(func_name)s' (%(task_name)s)"
+msgid "%(proc_name)s processing %(task_name)s '%(func_name)s'"
 msgstr "%(proc_name)s, '%(func_name)s' [%(task_name)s] işlerini işiyor"
 
-#: django_q/cluster.py:440
-#, python-format
-msgid ""
-"Could not process '%(func_name)s'. Check the location of the function and "
-"the args/kwargs."
-msgstr ""
-"%(func_name)s' işlenemedi. İşlevin konumunu ve args/kwargs öğelerini kontrol "
-"edin."
-
-#: django_q/cluster.py:456
+#: cluster.py:546
 #, python-format
 msgid "%(proc_name)s stopped doing work"
 msgstr "%(proc_name)s çalışmayı bıraktı"
 
-#: django_q/cluster.py:649 django_q/models.py:144
-msgid "Please install croniter to enable cron expressions"
-msgstr "Cron expressions'ları açmak için croniter yükleyin"
-
-#: django_q/cluster.py:672
+#: cluster.py:751
 #, python-format
 msgid "%(process_name)s failed to create a task from schedule [%(schedule)s]"
 msgstr "%(process_name)s programdan bir görev oluşturamadı [%(schedule)s]"
 
-#: django_q/cluster.py:678
-#, python-format
-msgid "%(process_name)s created a task from schedule [%(schedule)s]"
+#: cluster.py:762
+#, fuzzy, python-format
+#| msgid "%(process_name)s created a task from schedule [%(schedule)s]"
+msgid ""
+"%(process_name)s created task %(task_name)s from schedule [%(schedule)s]"
 msgstr "%(process_name)s programdan bir görev oluşturamadı [%(schedule)s]"
 
-#: django_q/cluster.py:718
+#: cluster.py:808
 msgid "Skipping cpu affinity because psutil was not found."
 msgstr "Psutil bulunamadığı için cpu benzeşimi atlanıyor."
 
-#: django_q/cluster.py:723
+#: cluster.py:813
 msgid "Faking cpu affinity because it is not supported on this platform"
 msgstr "Bu platformda desteklenmediği için sahte cpu benzeşimi"
 
-#: django_q/cluster.py:744
+#: cluster.py:835
 #, python-format
 msgid "%(pid)s will use cpu %(affinity)s"
 msgstr "%(pid)s cpu %(affinity)s kullanacaktır"
 
-#: django_q/conf.py:85
+#: conf.py:90
 #, python-format
 msgid ""
 "SAVE_LIMIT_PER (%(option)s) is not a valid option. Options are: 'group', "
@@ -193,295 +192,311 @@ msgstr ""
 "'group', 'name', 'func' ve None. Varsayılan değer None'dır."
 
 #. Translators: Cluster status descriptions
-#: django_q/conf.py:194
+#: conf.py:207
 msgid "Starting"
 msgstr "Başlıyor"
 
-#: django_q/conf.py:195
+#: conf.py:208
 msgid "Working"
 msgstr "Çalışıyor"
 
-#: django_q/conf.py:196
+#: conf.py:209
 msgid "Idle"
 msgstr "Boşta"
 
-#: django_q/conf.py:197
+#: conf.py:210
 msgid "Stopped"
 msgstr "Durdu"
 
-#: django_q/conf.py:198
+#: conf.py:211
 msgid "Stopping"
 msgstr "Durduruluyor"
 
 #. Translators: help text for qcluster management command
-#: django_q/management/commands/qcluster.py:9
+#: management/commands/qcluster.py:9
 msgid "Starts a Django Q Cluster."
 msgstr "Bir Django Q Cluster çalıştırır."
 
 #. Translators: help text for qinfo management command
-#: django_q/management/commands/qinfo.py:11
+#: management/commands/qinfo.py:11
 msgid "General information over all clusters."
 msgstr "Tüm cluster'lar için genel bilgiler."
 
 #. Translators: help text for qmemory management command
-#: django_q/management/commands/qmemory.py:9
+#: management/commands/qmemory.py:9
 msgid "Monitors Q Cluster memory usage"
 msgstr "Q Cluster'ın bellek kullanımını izler"
 
 #. Translators: help text for qmonitor management command
-#: django_q/management/commands/qmonitor.py:9
+#: management/commands/qmonitor.py:9
 msgid "Monitors Q Cluster activity"
 msgstr "Q Cluster'ın aktivitelerini izler"
 
-#: django_q/models.py:119
+#: models.py:125
 msgid "Successful task"
 msgstr "Başarılı iş"
 
-#: django_q/models.py:120
+#: models.py:126
 msgid "Successful tasks"
 msgstr "Başarılı işler"
 
-#: django_q/models.py:135
+#: models.py:141
 msgid "Failed task"
 msgstr "Başarısız iş"
 
-#: django_q/models.py:136
+#: models.py:142
 msgid "Failed tasks"
 msgstr "Başarısız işler"
 
-#: django_q/models.py:160
+#: models.py:150 models.py:234
+msgid "Please install croniter to enable cron expressions"
+msgstr "Cron expressions'ları açmak için croniter yükleyin"
+
+#: models.py:170
 msgid "e.g. 1, 2, 'John'"
 msgstr "Örneğin: 1, 2, 'Melih'"
 
-#: django_q/models.py:162
+#: models.py:172
 msgid "e.g. x=1, y=2, name='John'"
 msgstr "Örneğin: x=1, y=2, name='Melih'"
 
-#: django_q/models.py:176
+#: models.py:186
 msgid "Once"
 msgstr "Bir kere"
 
-#: django_q/models.py:177
+#: models.py:187
 msgid "Minutes"
 msgstr "Dakika"
 
-#: django_q/models.py:178
+#: models.py:188
 msgid "Hourly"
 msgstr "Saatlik"
 
-#: django_q/models.py:179
+#: models.py:189
 msgid "Daily"
 msgstr "Günlük"
 
-#: django_q/models.py:180
+#: models.py:190
 msgid "Weekly"
 msgstr "Haftalık"
 
-#: django_q/models.py:181
+#: models.py:191
 #, fuzzy
 #| msgid "Weekly"
 msgid "Biweekly"
 msgstr "İki haftada bir"
 
-#: django_q/models.py:182
+#: models.py:192
 msgid "Monthly"
 msgstr "Aylık"
 
-#: django_q/models.py:183
+#: models.py:193
 #, fuzzy
 #| msgid "Monthly"
 msgid "Bimonthly"
 msgstr "İki ayda bir"
 
-#: django_q/models.py:184
+#: models.py:194
 msgid "Quarterly"
 msgstr "Bir Çeyrek (3 Ay)"
 
-#: django_q/models.py:185
+#: models.py:195
 msgid "Yearly"
 msgstr "Yıllık"
 
-#: django_q/models.py:186
+#: models.py:196
 msgid "Cron"
 msgstr ""
 
-#: django_q/models.py:189
+#: models.py:199
 msgid "Schedule Type"
 msgstr "Zamanlama Tipi"
 
-#: django_q/models.py:192
+#: models.py:202
 msgid "Number of minutes for the Minutes type"
 msgstr "Dakika tipine göre dakika sayısı"
 
-#: django_q/models.py:195
+#: models.py:205
 msgid "Repeats"
 msgstr "Tekrar eder"
 
-#: django_q/models.py:195
+#: models.py:205
 msgid "n = n times, -1 = forever"
 msgstr "n = n kere, -1 = sonsuza kadar"
 
-#: django_q/models.py:198
+#: models.py:208
 msgid "Next Run"
 msgstr "Bir dahaki çalışma tarihi"
 
-#: django_q/models.py:205
+#: models.py:215
 msgid "Cron expression"
 msgstr ""
 
-#: django_q/models.py:235
+#: models.py:224
+msgid "Name of kwarg to pass intended schedule date"
+msgstr "Geçilecek kwarg'ın adı öngörülen program tarihi"
+
+#: models.py:299
 msgid "Scheduled task"
 msgstr "Zamanlanmış iş"
 
-#: django_q/models.py:236
+#: models.py:300
 msgid "Scheduled tasks"
 msgstr "Zamanlanmış işler"
 
-#: django_q/models.py:262
+#: models.py:326
 msgid "Queued task"
 msgstr "Sıraya alınmış iş"
 
-#: django_q/models.py:263
+#: models.py:327
 msgid "Queued tasks"
 msgstr "Sıraya alınmış işler"
 
-#: django_q/monitor.py:62 django_q/monitor.py:339
+#: monitor.py:64 monitor.py:348
 msgid "Host"
 msgstr ""
 
-#: django_q/monitor.py:66 django_q/monitor.py:343 django_q/monitor.py:450
+#: monitor.py:68 monitor.py:352 monitor.py:459
 msgid "Id"
 msgstr ""
 
-#: django_q/monitor.py:70
+#: monitor.py:72
 msgid "State"
 msgstr "Durum"
 
-#: django_q/monitor.py:74
+#: monitor.py:76
 msgid "Pool"
 msgstr "Havuz"
 
-#: django_q/monitor.py:78
+#: monitor.py:80
 msgid "TQ"
 msgstr ""
 
-#: django_q/monitor.py:82
+#: monitor.py:84
 msgid "RQ"
 msgstr ""
 
-#: django_q/monitor.py:86
+#: monitor.py:88
 msgid "RC"
 msgstr ""
 
-#: django_q/monitor.py:90
+#: monitor.py:92
 msgid "Up"
 msgstr ""
 
-#: django_q/monitor.py:170 django_q/monitor.py:279
+#: monitor.py:172 monitor.py:286
 msgid "Queued"
 msgstr "Sıraya alınmış"
 
-#: django_q/monitor.py:178
+#: monitor.py:180
 msgid "Success"
 msgstr "Başarılı olanlar"
 
-#: django_q/monitor.py:188 django_q/monitor.py:287
+#: monitor.py:190 monitor.py:294
 msgid "Failures"
 msgstr "Başarısız olanlar"
 
-#: django_q/monitor.py:199 django_q/monitor.py:485
+#: monitor.py:201 monitor.py:498
 msgid "[Press q to quit]"
 msgstr "[Çıkmak için q'ya basın]"
 
-#: django_q/monitor.py:223
+#: monitor.py:227
 msgid "day"
 msgstr "gün"
 
-#: django_q/monitor.py:244
+#: monitor.py:248
 msgid "second"
 msgstr "saniye"
 
-#: django_q/monitor.py:247
+#: monitor.py:251
 msgid "minute"
 msgstr "dakika"
 
-#: django_q/monitor.py:250
+#: monitor.py:254
 msgid "hour"
 msgstr "saat"
 
-#: django_q/monitor.py:260
+#: monitor.py:263
 #, python-format
 msgid "-- %(prefix)s %(version)s on %(info)s --"
 msgstr "-- %(prefix)s %(version)s üzerinde %(info)s --"
 
-#: django_q/monitor.py:266
+#: monitor.py:273
 msgid "Clusters"
 msgstr ""
 
-#: django_q/monitor.py:270
+#: monitor.py:277
 msgid "Workers"
 msgstr ""
 
-#: django_q/monitor.py:274
+#: monitor.py:281
 msgid "Restarts"
 msgstr "Yeniden çalıştırmalar"
 
-#: django_q/monitor.py:283
+#: monitor.py:290
 msgid "Successes"
 msgstr "Başarılı olanlar"
 
-#: django_q/monitor.py:292
+#: monitor.py:299
 msgid "Schedules"
 msgstr "Zamanlanmışlar"
 
-#: django_q/monitor.py:296
+#: monitor.py:303
 #, python-format
 msgid "Tasks/%(per)s"
 msgstr "İş/%(per)s"
 
-#: django_q/monitor.py:300
+#: monitor.py:307
 msgid "Avg time"
 msgstr "Ortalama süre"
 
-#: django_q/monitor.py:348
+#: monitor.py:357
 msgid "Available (%)"
 msgstr "Müsait (%) "
 
-#: django_q/monitor.py:354
+#: monitor.py:363
 msgid "Available (MB)"
 msgstr "Müsait (MB)"
 
-#: django_q/monitor.py:359
+#: monitor.py:368
 msgid "Total (MB)"
 msgstr "Toplam (MB)"
 
-#: django_q/monitor.py:364
+#: monitor.py:373
 msgid "Sentinel (MB)"
 msgstr ""
 
-#: django_q/monitor.py:370
+#: monitor.py:379
 msgid "Monitor (MB)"
 msgstr "İzleme (MB)"
 
-#: django_q/monitor.py:376
+#: monitor.py:385
 msgid "Workers (MB)"
 msgstr ""
 
-#: django_q/monitor.py:478
+#: monitor.py:487
 #, python-format
 msgid "Available lowest (): %(memory_percent)s ((at)s)"
 msgstr "Mevcut en düşük (): %(memory_percent)s ((at)s)"
 
-#: django_q/monitor.py:496
+#: monitor.py:509
 msgid "No clusters appear to be running."
 msgstr "Hiçbir küme çalışıyor görünmüyor."
 
-#: django_q/signals.py:22
+#: signals.py:22
 #, python-format
 msgid "malformed return hook '%(hook)s' for [%(name)s]"
 msgstr ""
 
-#: django_q/signals.py:30
+#: signals.py:30
 #, python-format
 msgid "return hook %(hook)s failed on [%(name)s] because %(error)s"
 msgstr ""
+
+#, python-format
+#~ msgid ""
+#~ "Could not process '%(func_name)s'. Check the location of the function and "
+#~ "the args/kwargs."
+#~ msgstr ""
+#~ "%(func_name)s' işlenemedi. İşlevin konumunu ve args/kwargs öğelerini "
+#~ "kontrol edin."

django_q/management/commands/qcluster.py

@@ -1,3 +1,5 @@
+import os
+
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
 
@@ -16,8 +18,21 @@ class Command(BaseCommand):
             default=False,
             help="Run once and then stop.",
         )
+        parser.add_argument(
+            "-n",
+            "--name",
+            dest="cluster_name",
+            default=None,
+            help="Set alternative cluster name instead of the name in Q_CLUSTER settings (for multi-queue setup). "
+            "On Linux you should set name through `Q_CLUSTER_NAME=cluster_name python manage.py qcluster` instead.",
+        )
 
     def handle(self, *args, **options):
+        # Set alternative cluster_name before creating the cluster (cluster_name is broker's queue_name, too)
+        cluster_name = options.get("cluster_name")
+        if cluster_name:
+            os.environ["Q_CLUSTER_NAME"] = cluster_name
+
         q = Cluster()
         q.start()
         if options.get("run_once", False):

django_q/management/commands/qinfo.py

@@ -3,7 +3,7 @@ from django.utils.translation import gettext as _
 
 from django_q import VERSION
 from django_q.conf import Conf
-from django_q.monitor import get_ids, info
+from django_q.monitor_terminal import get_ids, info
 
 
 class Command(BaseCommand):

django_q/management/commands/qmemory.py

@@ -1,7 +1,7 @@
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
 
-from django_q.monitor import memory
+from django_q.monitor_terminal import memory
 
 
 class Command(BaseCommand):

django_q/management/commands/qmonitor.py

@@ -1,7 +1,7 @@
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
 
-from django_q.monitor import monitor
+from django_q.monitor_terminal import monitor
 
 
 class Command(BaseCommand):

django_q/migrations/0001_initial.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = []
 
     operations = [

django_q/migrations/0002_auto_20150630_1624.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0001_initial"),
     ]

django_q/migrations/0003_auto_20150708_1326.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0002_auto_20150630_1624"),
     ]

django_q/migrations/0004_auto_20150710_1043.py

@@ -2,7 +2,6 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0003_auto_20150708_1326"),
     ]

django_q/migrations/0005_auto_20150718_1506.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0004_auto_20150710_1043"),
     ]

django_q/migrations/0006_auto_20150805_1817.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0005_auto_20150718_1506"),
     ]

django_q/migrations/0007_ormq.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0006_auto_20150805_1817"),
     ]

django_q/migrations/0008_auto_20160224_1026.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0007_ormq"),
     ]

django_q/migrations/0009_auto_20171009_0915.py

@@ -2,7 +2,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0008_auto_20160224_1026"),
     ]

django_q/migrations/0010_auto_20200610_0856.py

@@ -3,7 +3,6 @@ from django.db import migrations
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0009_auto_20171009_0915"),
     ]

django_q/migrations/0011_auto_20200628_1055.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0010_auto_20200610_0856"),
     ]

django_q/migrations/0012_auto_20200702_1608.py

@@ -6,7 +6,6 @@ import django_q.models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0011_auto_20200628_1055"),
     ]

django_q/migrations/0013_task_attempt_count.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0012_auto_20200702_1608"),
     ]

django_q/migrations/0014_schedule_cluster.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0013_task_attempt_count"),
     ]

django_q/migrations/0015_alter_schedule_schedule_type.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("django_q", "0014_schedule_cluster"),
     ]

django_q/migrations/0016_schedule_intended_date_kwarg.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.2 on 2023-01-15 22:34
+
+from django.db import migrations, models
+
+import django_q.models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("django_q", "0015_alter_schedule_schedule_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="schedule",
+            name="intended_date_kwarg",
+            field=models.CharField(
+                blank=True,
+                help_text="Name of kwarg to pass intended schedule date",
+                max_length=100,
+                null=True,
+                validators=[django_q.models.validate_kwarg],
+            ),
+        ),
+    ]

django_q/migrations/0017_task_cluster_alter.py

@@ -0,0 +1,42 @@
+# Generated by Django 4.1.5 on 2023-03-07 12:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("django_q", "0016_schedule_intended_date_kwarg"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="task",
+            name="cluster",
+            field=models.CharField(blank=True, default=None, max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name="ormq",
+            name="key",
+            field=models.CharField(
+                help_text="Name of the target cluster", max_length=100
+            ),
+        ),
+        migrations.AlterField(
+            model_name="ormq",
+            name="lock",
+            field=models.DateTimeField(
+                help_text="Prevent any cluster from pulling until", null=True
+            ),
+        ),
+        migrations.AlterField(
+            model_name="schedule",
+            name="cluster",
+            field=models.CharField(
+                blank=True,
+                default=None,
+                help_text="Name of the target cluster",
+                max_length=100,
+                null=True,
+            ),
+        ),
+    ]

django_q/migrations/0018_task_success_index.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.7 on 2024-03-05 17:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("django_q", "0017_task_cluster_alter"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="task",
+            index=models.Index(
+                condition=models.Q(("success", True)),
+                fields=["group", "name", "func"],
+                name="success_index",
+            ),
+        ),
+    ]

django_q/models.py

@@ -1,13 +1,15 @@
 from datetime import datetime, timedelta
+from keyword import iskeyword
 
 # Django
 from django import get_version
 from django.core.exceptions import ValidationError
 from django.db import models
+from django.db.models import Q
 from django.template.defaultfilters import truncatechars
 from django.urls import reverse
 from django.utils import timezone
-from django.utils.timezone import is_aware
+from django.utils.functional import cached_property
 from django.utils.html import format_html
 from django.utils.translation import gettext_lazy as _
 
@@ -16,9 +18,9 @@ from picklefield import PickledObjectField
 from picklefield.fields import dbsafe_decode
 
 # Local
-from django_q.conf import croniter, Conf
+from django_q.conf import croniter
 from django_q.signing import SignedPackage
-from django_q.utils import localtime, add_months, add_years
+from django_q.utils import add_months, add_years, localtime
 
 from .utils import get_func_repr
 
@@ -32,6 +34,7 @@ class Task(models.Model):
     kwargs = PickledObjectField(null=True, protocol=-1)
     result = PickledObjectField(null=True, protocol=-1)
     group = models.CharField(max_length=100, editable=False, null=True)
+    cluster = models.CharField(max_length=100, default=None, null=True, blank=True)
     started = models.DateTimeField(editable=False)
     stopped = models.DateTimeField(editable=False)
     success = models.BooleanField(default=True, editable=False)
@@ -109,6 +112,13 @@ class Task(models.Model):
     class Meta:
         app_label = "django_q"
         ordering = ["-stopped"]
+        indexes = [
+            models.Index(
+                name="success_index",
+                fields=["group", "name", "func"],
+                condition=Q(success=True),
+            ),
+        ]
 
 
 class SuccessManager(models.Manager):
@@ -153,6 +163,10 @@ def validate_cron(value):
         raise ValidationError(e)
 
 
+def validate_kwarg(value):
+    return value.isidentifier() and not iskeyword(value)
+
+
 class Schedule(models.Model):
     name = models.CharField(max_length=100, null=True, blank=True)
     func = models.CharField(max_length=256, help_text="e.g. module.tasks.function")
@@ -210,7 +224,20 @@ class Schedule(models.Model):
         help_text=_("Cron expression"),
     )
     task = models.CharField(max_length=100, null=True, editable=False)
-    cluster = models.CharField(max_length=100, default=None, null=True, blank=True)
+    cluster = models.CharField(
+        max_length=100,
+        default=None,
+        null=True,
+        blank=True,
+        help_text=_("Name of the target cluster"),
+    )
+    intended_date_kwarg = models.CharField(
+        max_length=100,
+        null=True,
+        blank=True,
+        validators=[validate_kwarg],
+        help_text=_("Name of kwarg to pass intended schedule date"),
+    )
 
     def calculate_next_run(self, next_run=None):
         # next run is always in UTC
@@ -255,13 +282,8 @@ class Schedule(models.Model):
             # based on DST active or not
             extra_diff = (new_next_run - current_next_run) - add
 
-            # if we have one positive hour difference, then subtract it, so we are even
-            # and vice versa. In most cases, this will be 0, as there won't be a
-            # timezone diff
-            if extra_diff > timedelta(hours=0):
-                next_run -= extra_diff
-            else:
-                next_run += extra_diff
+            # subtract difference
+            next_run -= extra_diff
 
         return next_run
 
@@ -295,24 +317,40 @@ class Schedule(models.Model):
 
 
 class OrmQ(models.Model):
-    key = models.CharField(max_length=100)
+    key = models.CharField(max_length=100, help_text=_("Name of the target cluster"))
     payload = models.TextField()
-    lock = models.DateTimeField(null=True)
+    lock = models.DateTimeField(
+        null=True, help_text=_("Prevent any cluster from pulling until")
+    )
 
+    @cached_property
     def task(self):
-        return SignedPackage.loads(self.payload)
+        try:
+            return SignedPackage.loads(self.payload)
+        except Exception as e:
+            return {"id": "*" + e.__class__.__name__}
 
     def func(self):
-        return get_func_repr(self.task()["func"])
+        return get_func_repr(self.task.get("func"))
 
     def task_id(self):
-        return self.task()["id"]
+        return self.task.get("id")
 
     def name(self):
-        return self.task()["name"]
+        return self.task.get("name")
 
     def group(self):
-        return self.task().get("group")
+        return self.task.get("group")
+
+    def args(self):
+        return self.task.get("args")
+
+    def kwargs(self):
+        return self.task.get("kwargs")
+
+    def q_options(self):
+        exclude = {"id", "name", "group", "func", "args", "kwargs"}
+        return {k: v for k, v in self.task.items() if k not in exclude}
 
     class Meta:
         app_label = "django_q"

django_q/monitor.py

@@ -1,510 +1,208 @@
-from datetime import timedelta
+from multiprocessing.process import current_process
+from multiprocessing.queues import Queue
 
-# django
-from django.db import connection
-from django.db.models import F, Sum
-from django.utils import timezone
-from django.utils.translation import gettext as _
+from django import core, db
+from django.apps.registry import apps
+from django.utils.translation import gettext_lazy as _
 
-from django_q import VERSION, models
-from django_q.brokers import get_broker
-
-# local
-from django_q.conf import Conf
-from django_q.status import Stat
-
-# optional
 try:
-    import psutil
-except ImportError:
-    psutil = None
+    apps.check_apps_ready()
+except core.exceptions.AppRegistryNotReady:
+    import django
+
+    django.setup()
+
+from django_q.brokers import Broker, get_broker
+from django_q.conf import Conf, logger, setproctitle
+from django_q.models import Success, Task
+from django_q.signals import post_execute
+from django_q.signing import SignedPackage
+from django_q.tasks import async_chain
+from django_q.utils import close_old_django_connections, get_func_repr
+
+try:
+    import setproctitle
+except ModuleNotFoundError:
+    setproctitle = None
 
 
-def get_process_mb(pid):
-    try:
-        process = psutil.Process(pid)
-        mb_used = round(process.memory_info().rss / 1024**2, 2)
-    except psutil.NoSuchProcess:
-        mb_used = "NO_PROCESS_FOUND"
-    return mb_used
-
-
-BLESSED_INSTALL_MESSAGE = (
-    "Blessed is not installed. Please install blessed to use this: "
-    "https://pypi.org/project/blessed/"
-)
-
-
-def monitor(run_once=False, broker=None):
+def monitor(result_queue: Queue, broker: Broker = None):
+    """
+    Gets finished tasks from the result queue and saves them to Django
+    :type broker: brokers.Broker
+    :type result_queue: multiprocessing.Queue
+    """
     if not broker:
         broker = get_broker()
-    try:
-        from blessed import Terminal
+    proc_name = current_process().name
+    if setproctitle:
+        setproctitle.setproctitle(f"qcluster {proc_name} monitor")
+    logger.info(
+        _("%(name)s monitoring at %(id)s")
+        % {"name": proc_name, "id": current_process().pid}
+    )
+    for task in iter(result_queue.get, "STOP"):
+        # save the result
+        if task.get("cached", False):
+            save_cached(task, broker)
+        else:
+            save_task(task, broker)
+        # acknowledge result
+        ack_id = task.pop("ack_id", False)
+        if ack_id and (task["success"] or task.get("ack_failure", False)):
+            broker.acknowledge(ack_id)
+        # signal execution done
+        post_execute.send(sender="django_q", task=task)
+        # log the result
+        info_name = get_func_repr(task["func"])
+        if task["success"]:
+            # log success
+            logger.info(
+                _("Processed '%(info_name)s' (%(task_name)s)")
+                % {"info_name": info_name, "task_name": task["name"]}
+            )
+        else:
+            # log failure
+            logger.error(
+                _("Failed '%(info_name)s' (%(task_name)s) - %(task_result)s")
+                % {
+                    "info_name": info_name,
+                    "task_name": task["name"],
+                    "task_result": task["result"],
+                }
+            )
+    logger.info(_("%(name)s stopped monitoring results") % {"name": proc_name})
 
-        term = Terminal()
-    except ImportError:
-        print(BLESSED_INSTALL_MESSAGE)
+
+def save_task(task, broker: Broker):
+    """
+    Saves the task package to Django or the cache
+    :param task: the task package
+    :type broker: brokers.Broker
+    """
+    # SAVE LIMIT < 0 : Don't save success
+    if not task.get("save", Conf.SAVE_LIMIT >= 0) and task["success"]:
         return
+    # enqueues next in a chain
+    if task.get("chain", None):
+        async_chain(
+            task["chain"],
+            group=task["group"],
+            cached=task["cached"],
+            sync=task["sync"],
+            broker=broker,
+        )
+    # SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
+    close_old_django_connections()
 
-    broker.ping()
-    with term.fullscreen(), term.hidden_cursor(), term.cbreak():
-        val = None
-        start_width = int(term.width / 8)
-        while val not in (
-            "q",
-            "Q",
+    try:
+        filters = {}
+        if (
+            Conf.SAVE_LIMIT_PER
+            and Conf.SAVE_LIMIT_PER in {"group", "name", "func"}
+            and Conf.SAVE_LIMIT_PER in task
         ):
-            col_width = int(term.width / 8)
-            # In case of resize
-            if col_width != start_width:
-                print(term.clear())
-                start_width = col_width
-            print(
-                term.move(0, 0)
-                + term.black_on_green(term.center(_("Host"), width=col_width - 1))
+            value = task[Conf.SAVE_LIMIT_PER]
+            if Conf.SAVE_LIMIT_PER == "func":
+                value = get_func_repr(value)
+            filters[Conf.SAVE_LIMIT_PER] = value
+
+        with db.transaction.atomic(using=db.router.db_for_write(Success)):
+            list(Success.objects.filter(**filters).select_for_update())
+            if (
+                task["success"]
+                and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
+            ):
+                Success.objects.filter(**filters).last().delete()
+
+        # check if this task has previous results
+        try:
+            task_obj = Task.objects.get(id=task["id"], name=task["name"])
+            # only update the result if it hasn't succeeded yet
+            if not task_obj.success:
+                task_obj.stopped = task["stopped"]
+                task_obj.result = task["result"]
+                task_obj.success = task["success"]
+                task_obj.attempt_count = task_obj.attempt_count + 1
+                task_obj.save()
+
+        except Task.DoesNotExist:
+            # convert func to string
+            func = get_func_repr(task["func"])
+            task_obj = Task.objects.create(
+                id=task["id"],
+                name=task["name"],
+                func=func,
+                hook=task.get("hook"),
+                args=task["args"],
+                kwargs=task["kwargs"],
+                cluster=task.get("cluster"),
+                started=task["started"],
+                stopped=task["stopped"],
+                result=task["result"],
+                group=task.get("group"),
+                success=task["success"],
+                attempt_count=1,
             )
-            print(
-                term.move(0, 1 * col_width)
-                + term.black_on_green(term.center(_("Id"), width=col_width - 1))
-            )
-            print(
-                term.move(0, 2 * col_width)
-                + term.black_on_green(term.center(_("State"), width=col_width - 1))
-            )
-            print(
-                term.move(0, 3 * col_width)
-                + term.black_on_green(term.center(_("Pool"), width=col_width - 1))
-            )
-            print(
-                term.move(0, 4 * col_width)
-                + term.black_on_green(term.center(_("TQ"), width=col_width - 1))
-            )
-            print(
-                term.move(0, 5 * col_width)
-                + term.black_on_green(term.center(_("RQ"), width=col_width - 1))
-            )
-            print(
-                term.move(0, 6 * col_width)
-                + term.black_on_green(term.center(_("RC"), width=col_width - 1))
-            )
-            print(
-                term.move(0, 7 * col_width)
-                + term.black_on_green(term.center(_("Up"), width=col_width - 1))
-            )
-            i = 2
-            stats = Stat.get_all(broker=broker)
-            print(term.clear_eos())
-            for stat in stats:
-                status = stat.status
-                # color status
-                if stat.status == Conf.WORKING:
-                    status = term.green(str(Conf.WORKING))
-                elif stat.status == Conf.STOPPING:
-                    status = term.yellow(str(Conf.STOPPING))
-                elif stat.status == Conf.STOPPED:
-                    status = term.red(str(Conf.STOPPED))
-                elif stat.status == Conf.IDLE:
-                    status = str(Conf.IDLE)
-                # color q's
-                tasks = str(stat.task_q_size)
-                if stat.task_q_size > 0:
-                    tasks = term.cyan(str(stat.task_q_size))
-                    if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
-                        tasks = term.green(str(stat.task_q_size))
-                results = stat.done_q_size
-                if results > 0:
-                    results = term.cyan(str(results))
-                # color workers
-                workers = len(stat.workers)
-                if workers < Conf.WORKERS:
-                    workers = term.yellow(str(workers))
-                # format uptime
-                uptime = (timezone.now() - stat.tob).total_seconds()
-                hours, remainder = divmod(uptime, 3600)
-                minutes, seconds = divmod(remainder, 60)
-                uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
-                # print to the terminal
-                print(
-                    term.move(i, 0)
-                    + term.center(stat.host[: col_width - 1], width=col_width - 1)
-                )
-                print(
-                    term.move(i, 1 * col_width)
-                    + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
-                )
-                print(
-                    term.move(i, 2 * col_width)
-                    + term.center(status, width=col_width - 1)
-                )
-                print(
-                    term.move(i, 3 * col_width)
-                    + term.center(workers, width=col_width - 1)
-                )
-                print(
-                    term.move(i, 4 * col_width)
-                    + term.center(tasks, width=col_width - 1)
-                )
-                print(
-                    term.move(i, 5 * col_width)
-                    + term.center(results, width=col_width - 1)
-                )
-                print(
-                    term.move(i, 6 * col_width)
-                    + term.center(stat.reincarnations, width=col_width - 1)
-                )
-                print(
-                    term.move(i, 7 * col_width)
-                    + term.center(uptime, width=col_width - 1)
-                )
-                i += 1
-            # bottom bar
-            i += 1
-            queue_size = broker.queue_size()
-            lock_size = broker.lock_size()
-            if lock_size:
-                queue_size = f"{queue_size}({lock_size})"
-            print(
-                term.move(i, 0)
-                + term.white_on_cyan(term.center(broker.info(), width=col_width * 2))
-            )
-            print(
-                term.move(i, 2 * col_width)
-                + term.black_on_cyan(term.center(_("Queued"), width=col_width))
-            )
-            print(
-                term.move(i, 3 * col_width)
-                + term.white_on_cyan(term.center(queue_size, width=col_width))
-            )
-            print(
-                term.move(i, 4 * col_width)
-                + term.black_on_cyan(term.center(_("Success"), width=col_width))
-            )
-            print(
-                term.move(i, 5 * col_width)
-                + term.white_on_cyan(
-                    term.center(models.Success.objects.count(), width=col_width)
-                )
-            )
-            print(
-                term.move(i, 6 * col_width)
-                + term.black_on_cyan(term.center(_("Failures"), width=col_width))
-            )
-            print(
-                term.move(i, 7 * col_width)
-                + term.white_on_cyan(
-                    term.center(models.Failure.objects.count(), width=col_width)
-                )
-            )
-            # for testing
-            if run_once:
-                return Stat.get_all(broker=broker)
-            print(term.move(i + 2, 0) + term.center(_("[Press q to quit]")))
-            val = term.inkey(timeout=1)
+
+        if (
+            Conf.MAX_ATTEMPTS > 0
+            and task_obj.attempt_count >= Conf.MAX_ATTEMPTS
+            and task.get("ack_id")
+        ):
+            broker.acknowledge(task["ack_id"])
+
+    except Exception:
+        logger.exception("Could not save task result")
 
 
-def info(broker=None):
-    if not broker:
-        broker = get_broker()
+def save_cached(task, broker: Broker):
+    task_key = f'{broker.list_key}:{task["id"]}'
+    timeout = task["cached"]
+    if timeout is True:
+        timeout = None
     try:
-        from blessed import Terminal
-
-        term = Terminal()
-    except ImportError:
-        print(BLESSED_INSTALL_MESSAGE)
-        return
-
-    broker.ping()
-    stat = Stat.get_all(broker=broker)
-    # general stats
-    clusters = len(stat)
-    workers = 0
-    reincarnations = 0
-    for cluster in stat:
-        workers += len(cluster.workers)
-        reincarnations += cluster.reincarnations
-    # calculate tasks pm and avg exec time
-    tasks_per = 0
-    per = _("day")
-    exec_time = 0
-    last_tasks = models.Success.objects.filter(
-        stopped__gte=timezone.now() - timedelta(hours=24)
-    )
-    tasks_per_day = last_tasks.count()
-    if tasks_per_day > 0:
-        # average execution time over the last 24 hours
-        if connection.vendor != "sqlite":
-            exec_time = last_tasks.aggregate(
-                time_taken=Sum(F("stopped") - F("started"))
-            )
-            exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
-        else:
-            # can't sum timedeltas on sqlite
-            for t in last_tasks:
-                exec_time += t.time_taken()
-            exec_time = exec_time / tasks_per_day
-        # tasks per second/minute/hour/day in the last 24 hours
-        if tasks_per_day > 24 * 60 * 60:
-            tasks_per = tasks_per_day / (24 * 60 * 60)
-            per = _("second")
-        elif tasks_per_day > 24 * 60:
-            tasks_per = tasks_per_day / (24 * 60)
-            per = _("minute")
-        elif tasks_per_day > 24:
-            tasks_per = tasks_per_day / 24
-            per = _("hour")
-        else:
-            tasks_per = tasks_per_day
-    # print to terminal
-    print(term.clear_eos())
-    col_width = int(term.width / 6)
-    print(
-        term.black_on_green(
-            term.center(
-                _("-- %(prefix)s %(version)s on %(info)s --")
-                % {
-                    "prefix": Conf.PREFIX.capitalize(),
-                    "version": ".".join(str(v) for v in VERSION),
-                    "info": broker.info(),
-                }
-            )
-        )
-    )
-    print(
-        term.cyan(_("Clusters"))
-        + term.move_x(1 * col_width)
-        + term.white(str(clusters))
-        + term.move_x(2 * col_width)
-        + term.cyan(_("Workers"))
-        + term.move_x(3 * col_width)
-        + term.white(str(workers))
-        + term.move_x(4 * col_width)
-        + term.cyan(_("Restarts"))
-        + term.move_x(5 * col_width)
-        + term.white(str(reincarnations))
-    )
-    print(
-        term.cyan(_("Queued"))
-        + term.move_x(1 * col_width)
-        + term.white(str(broker.queue_size()))
-        + term.move_x(2 * col_width)
-        + term.cyan(_("Successes"))
-        + term.move_x(3 * col_width)
-        + term.white(str(models.Success.objects.count()))
-        + term.move_x(4 * col_width)
-        + term.cyan(_("Failures"))
-        + term.move_x(5 * col_width)
-        + term.white(str(models.Failure.objects.count()))
-    )
-    print(
-        term.cyan(_("Schedules"))
-        + term.move_x(1 * col_width)
-        + term.white(str(models.Schedule.objects.count()))
-        + term.move_x(2 * col_width)
-        + term.cyan(_("Tasks/%(per)s") % {"per": per})
-        + term.move_x(3 * col_width)
-        + term.white(f"{tasks_per:.2f}")
-        + term.move_x(4 * col_width)
-        + term.cyan(_("Avg time"))
-        + term.move_x(5 * col_width)
-        + term.white(f"{exec_time:.4f}")
-    )
-    return True
-
-
-def memory(run_once=False, workers=False, broker=None):
-    if not broker:
-        broker = get_broker()
-    try:
-        from blessed import Terminal
-
-        term = Terminal()
-    except ImportError:
-        print(BLESSED_INSTALL_MESSAGE)
-        return
-    broker.ping()
-    if not psutil:
-        print(term.clear_eos())
-        print(
-            term.white_on_red(
-                'Cannot start "qmemory" command. Missing "psutil" library.'
-            )
-        )
-        return
-    with term.fullscreen(), term.hidden_cursor(), term.cbreak():
-        MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
-        MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
-        cols = 8
-        val = None
-        start_width = int(term.width / cols)
-        while val not in ["q", "Q"]:
-            col_width = int(term.width / cols)
-            # In case of resize
-            if col_width != start_width:
-                print(term.clear())
-                start_width = col_width
-            # sentinel, monitor and workers memory usage
-            print(
-                term.move(0, 0 * col_width)
-                + term.black_on_green(term.center(_("Host"), width=col_width - 1))
-            )
-            print(
-                term.move(0, 1 * col_width)
-                + term.black_on_green(term.center(_("Id"), width=col_width - 1))
-            )
-            print(
-                term.move(0, 2 * col_width)
-                + term.black_on_green(
-                    term.center(_("Available (%)"), width=col_width - 1)
+        group = task.get("group", None)
+        iter_count = task.get("iter_count", 0)
+        # if it's a group append to the group list
+        if group:
+            group_key = f"{broker.list_key}:{group}:keys"
+            group_list = broker.cache.get(group_key) or []
+            # if it's an iter group, check if we are ready
+            if iter_count and len(group_list) == iter_count - 1:
+                group_args = f"{broker.list_key}:{group}:args"
+                # collate the results into a Task result
+                results = [
+                    SignedPackage.loads(broker.cache.get(k))["result"]
+                    for k in group_list
+                ]
+                results.append(task["result"])
+                task["result"] = results
+                task["id"] = group
+                task["args"] = SignedPackage.loads(broker.cache.get(group_args))
+                task.pop("iter_count", None)
+                task.pop("group", None)
+                if task.get("iter_cached", None):
+                    task["cached"] = task.pop("iter_cached", None)
+                    save_cached(task, broker=broker)
+                else:
+                    save_task(task, broker)
+                broker.cache.delete_many(group_list)
+                broker.cache.delete_many([group_key, group_args])
+                return
+            # save the group list
+            group_list.append(task_key)
+            broker.cache.set(group_key, group_list, timeout)
+            # async_task next in a chain
+            if task.get("chain", None):
+                async_chain(
+                    task["chain"],
+                    group=group,
+                    cached=task["cached"],
+                    sync=task["sync"],
+                    broker=broker,
                 )
-            )
-            print(
-                term.move(0, 3 * col_width)
-                + term.black_on_green(
-                    term.center(_("Available (MB)"), width=col_width - 1)
-                )
-            )
-            print(
-                term.move(0, 4 * col_width)
-                + term.black_on_green(term.center(_("Total (MB)"), width=col_width - 1))
-            )
-            print(
-                term.move(0, 5 * col_width)
-                + term.black_on_green(
-                    term.center(_("Sentinel (MB)"), width=col_width - 1)
-                )
-            )
-            print(
-                term.move(0, 6 * col_width)
-                + term.black_on_green(
-                    term.center(_("Monitor (MB)"), width=col_width - 1)
-                )
-            )
-            print(
-                term.move(0, 7 * col_width)
-                + term.black_on_green(
-                    term.center(_("Workers (MB)"), width=col_width - 1)
-                )
-            )
-            row = 2
-            stats = Stat.get_all(broker=broker)
-            print(term.clear_eos())
-            for stat in stats:
-                # memory available (%)
-                memory_available_percentage = round(
-                    psutil.virtual_memory().available
-                    * 100
-                    / psutil.virtual_memory().total,
-                    2,
-                )
-                # memory available (MB)
-                memory_available = round(
-                    psutil.virtual_memory().available / 1024**2, 2
-                )
-                if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
-                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
-                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
-                print(
-                    term.move(row, 0 * col_width)
-                    + term.center(stat.host[: col_width - 1], width=col_width - 1)
-                )
-                print(
-                    term.move(row, 1 * col_width)
-                    + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
-                )
-                print(
-                    term.move(row, 2 * col_width)
-                    + term.center(memory_available_percentage, width=col_width - 1)
-                )
-                print(
-                    term.move(row, 3 * col_width)
-                    + term.center(memory_available, width=col_width - 1)
-                )
-                print(
-                    term.move(row, 4 * col_width)
-                    + term.center(
-                        round(psutil.virtual_memory().total / 1024**2, 2),
-                        width=col_width - 1,
-                    )
-                )
-                print(
-                    term.move(row, 5 * col_width)
-                    + term.center(get_process_mb(stat.sentinel), width=col_width - 1)
-                )
-                print(
-                    term.move(row, 6 * col_width)
-                    + term.center(
-                        get_process_mb(getattr(stat, "monitor", None)),
-                        width=col_width - 1,
-                    )
-                )
-                workers_mb = 0
-                for worker_pid in stat.workers:
-                    result = get_process_mb(worker_pid)
-                    if isinstance(result, str):
-                        result = 0
-                    workers_mb += result
-                print(
-                    term.move(row, 7 * col_width)
-                    + term.center(
-                        workers_mb or "NO_PROCESSES_FOUND", width=col_width - 1
-                    )
-                )
-                row += 1
-            # each worker's memory usage
-            if workers:
-                row += 2
-                col_width = int(term.width / (1 + Conf.WORKERS))
-                print(
-                    term.move(row, 0 * col_width)
-                    + term.black_on_cyan(term.center(_("Id"), width=col_width - 1))
-                )
-                for worker_num in range(Conf.WORKERS):
-                    print(
-                        term.move(row, (worker_num + 1) * col_width)
-                        + term.black_on_cyan(
-                            term.center(
-                                "Worker #{} (MB)".format(worker_num + 1),
-                                width=col_width - 1,
-                            )
-                        )
-                    )
-                row += 2
-                for stat in stats:
-                    print(
-                        term.move(row, 0 * col_width)
-                        + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
-                    )
-                    for idx, worker_pid in enumerate(stat.workers):
-                        mb_used = get_process_mb(worker_pid)
-                        print(
-                            term.move(row, (idx + 1) * col_width)
-                            + term.center(mb_used, width=col_width - 1)
-                        )
-                    row += 1
-            row += 1
-            print(
-                term.move(row, 0)
-                + _("Available lowest (): %(memory_percent)s ((at)s)")
-                % {
-                    "memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
-                    "at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
-                        "%Y-%m-%d %H:%M:%S+00:00"
-                    ),
-                }
-            )
-            # for testing
-            if run_once:
-                return Stat.get_all(broker=broker)
-            print(term.move(row + 2, 0) + term.center(_("[Press q to quit]")))
-            val = term.inkey(timeout=1)
-
-
-def get_ids():
-    # prints id (PID) of running clusters
-    stat = Stat.get_all()
-    if stat:
-        for s in stat:
-            print(s.cluster_id)
-    else:
-        print(_("No clusters appear to be running."))
-    return True
+        # save the task
+        broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
+    except Exception:
+        logger.exception("Could not save task result")

django_q/monitor_terminal.py

@@ -0,0 +1,508 @@
+from datetime import timedelta
+
+# django
+from django.db import connection
+from django.db.models import F, Sum
+from django.utils import timezone
+from django.utils.translation import gettext as _
+
+from django_q import VERSION, models
+from django_q.brokers import get_broker
+
+# local
+from django_q.conf import Conf
+from django_q.status import Stat
+
+# optional
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+
+def get_process_mb(pid):
+    try:
+        process = psutil.Process(pid)
+        mb_used = round(process.memory_info().rss / 1024**2, 2)
+    except psutil.NoSuchProcess:
+        mb_used = "NO_PROCESS_FOUND"
+    return mb_used
+
+
+BLESSED_INSTALL_MESSAGE = (
+    "Blessed is not installed. Please install blessed to use this: "
+    "https://pypi.org/project/blessed/"
+)
+
+
+def monitor(run_once=False, broker=None):
+    if not broker:
+        broker = get_broker()
+    try:
+        from blessed import Terminal
+
+        term = Terminal()
+    except ImportError:
+        print(BLESSED_INSTALL_MESSAGE)
+        return
+
+    broker.ping()
+    with term.fullscreen(), term.hidden_cursor(), term.cbreak():
+        val = None
+        start_width = int(term.width / 8)
+        while val not in (
+            "q",
+            "Q",
+        ):
+            col_width = int(term.width / 8)
+            # In case of resize
+            if col_width != start_width:
+                print(term.clear())
+                start_width = col_width
+            print(
+                term.move(0, 0)
+                + term.black_on_green(term.center(_("Host"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 1 * col_width)
+                + term.black_on_green(term.center(_("Id"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 2 * col_width)
+                + term.black_on_green(term.center(_("State"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 3 * col_width)
+                + term.black_on_green(term.center(_("Pool"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 4 * col_width)
+                + term.black_on_green(term.center(_("TQ"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 5 * col_width)
+                + term.black_on_green(term.center(_("RQ"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 6 * col_width)
+                + term.black_on_green(term.center(_("RC"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 7 * col_width)
+                + term.black_on_green(term.center(_("Up"), width=col_width - 1))
+            )
+            i = 2
+            stats = Stat.get_all(broker=broker)
+            print(term.clear_eos())
+            for stat in stats:
+                status = stat.status
+                # color status
+                if stat.status == Conf.WORKING:
+                    status = term.green(str(Conf.WORKING))
+                elif stat.status == Conf.STOPPING:
+                    status = term.yellow(str(Conf.STOPPING))
+                elif stat.status == Conf.STOPPED:
+                    status = term.red(str(Conf.STOPPED))
+                elif stat.status == Conf.IDLE:
+                    status = str(Conf.IDLE)
+                # color q's
+                tasks = str(stat.task_q_size)
+                if stat.task_q_size > 0:
+                    tasks = term.cyan(str(stat.task_q_size))
+                    if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
+                        tasks = term.green(str(stat.task_q_size))
+                results = stat.done_q_size
+                if results > 0:
+                    results = term.cyan(str(results))
+                # color workers
+                workers = len(stat.workers)
+                if workers < Conf.WORKERS:
+                    workers = term.yellow(str(workers))
+                # format uptime
+                uptime = (timezone.now() - stat.tob).total_seconds()
+                hours, remainder = divmod(uptime, 3600)
+                minutes, seconds = divmod(remainder, 60)
+                uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
+                # print to the terminal
+                print(
+                    term.move(i, 0)
+                    + term.center(stat.host[: col_width - 1], width=col_width - 1)
+                )
+                print(
+                    term.move(i, 1 * col_width)
+                    + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
+                )
+                print(
+                    term.move(i, 2 * col_width)
+                    + term.center(status, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 3 * col_width)
+                    + term.center(workers, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 4 * col_width)
+                    + term.center(tasks, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 5 * col_width)
+                    + term.center(results, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 6 * col_width)
+                    + term.center(stat.reincarnations, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 7 * col_width)
+                    + term.center(uptime, width=col_width - 1)
+                )
+                i += 1
+            # bottom bar
+            i += 1
+            queue_size = broker.queue_size()
+            lock_size = broker.lock_size()
+            if lock_size:
+                queue_size = f"{queue_size}({lock_size})"
+            print(
+                term.move(i, 0)
+                + term.white_on_cyan(term.center(broker.info(), width=col_width * 2))
+            )
+            print(
+                term.move(i, 2 * col_width)
+                + term.black_on_cyan(term.center(_("Queued"), width=col_width))
+            )
+            print(
+                term.move(i, 3 * col_width)
+                + term.white_on_cyan(term.center(queue_size, width=col_width))
+            )
+            print(
+                term.move(i, 4 * col_width)
+                + term.black_on_cyan(term.center(_("Success"), width=col_width))
+            )
+            print(
+                term.move(i, 5 * col_width)
+                + term.white_on_cyan(
+                    term.center(models.Success.objects.count(), width=col_width)
+                )
+            )
+            print(
+                term.move(i, 6 * col_width)
+                + term.black_on_cyan(term.center(_("Failures"), width=col_width))
+            )
+            print(
+                term.move(i, 7 * col_width)
+                + term.white_on_cyan(
+                    term.center(models.Failure.objects.count(), width=col_width)
+                )
+            )
+            # for testing
+            if run_once:
+                return Stat.get_all(broker=broker)
+            print(term.move(i + 2, 0) + term.center(_("[Press q to quit]")))
+            val = term.inkey(timeout=1)
+
+
+def info(broker=None):
+    if not broker:
+        broker = get_broker()
+    try:
+        from blessed import Terminal
+
+        term = Terminal()
+    except ImportError:
+        print(BLESSED_INSTALL_MESSAGE)
+        return
+
+    broker.ping()
+    stat = Stat.get_all(broker=broker)
+    # general stats
+    clusters = len(stat)
+    workers = 0
+    reincarnations = 0
+    for cluster in stat:
+        workers += len(cluster.workers)
+        reincarnations += cluster.reincarnations
+    # calculate tasks pm and avg exec time
+    tasks_per = 0
+    per = _("day")
+    exec_time = 0
+    last_tasks = models.Success.objects.filter(
+        stopped__gte=timezone.now() - timedelta(hours=24)
+    )
+    tasks_per_day = last_tasks.count()
+    if tasks_per_day > 0:
+        # average execution time over the last 24 hours
+        if connection.vendor != "sqlite":
+            exec_time = last_tasks.aggregate(
+                time_taken=Sum(F("stopped") - F("started"))
+            )
+            exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
+        else:
+            # can't sum timedeltas on sqlite
+            for t in last_tasks:
+                exec_time += t.time_taken()
+            exec_time = exec_time / tasks_per_day
+        # tasks per second/minute/hour/day in the last 24 hours
+        if tasks_per_day > 24 * 60 * 60:
+            tasks_per = tasks_per_day / (24 * 60 * 60)
+            per = _("second")
+        elif tasks_per_day > 24 * 60:
+            tasks_per = tasks_per_day / (24 * 60)
+            per = _("minute")
+        elif tasks_per_day > 24:
+            tasks_per = tasks_per_day / 24
+            per = _("hour")
+        else:
+            tasks_per = tasks_per_day
+    # print to terminal
+    print(term.clear_eos())
+    col_width = int(term.width / 6)
+    print(
+        term.black_on_green(
+            term.center(
+                _("-- %(prefix)s %(version)s on %(info)s --")
+                % {
+                    "prefix": Conf.PREFIX.capitalize(),
+                    "version": ".".join(str(v) for v in VERSION),
+                    "info": broker.info(),
+                }
+            )
+        )
+    )
+    print(
+        term.cyan(_("Clusters"))
+        + term.move_x(1 * col_width)
+        + term.white(str(clusters))
+        + term.move_x(2 * col_width)
+        + term.cyan(_("Workers"))
+        + term.move_x(3 * col_width)
+        + term.white(str(workers))
+        + term.move_x(4 * col_width)
+        + term.cyan(_("Restarts"))
+        + term.move_x(5 * col_width)
+        + term.white(str(reincarnations))
+    )
+    print(
+        term.cyan(_("Queued"))
+        + term.move_x(1 * col_width)
+        + term.white(str(broker.queue_size()))
+        + term.move_x(2 * col_width)
+        + term.cyan(_("Successes"))
+        + term.move_x(3 * col_width)
+        + term.white(str(models.Success.objects.count()))
+        + term.move_x(4 * col_width)
+        + term.cyan(_("Failures"))
+        + term.move_x(5 * col_width)
+        + term.white(str(models.Failure.objects.count()))
+    )
+    print(
+        term.cyan(_("Schedules"))
+        + term.move_x(1 * col_width)
+        + term.white(str(models.Schedule.objects.count()))
+        + term.move_x(2 * col_width)
+        + term.cyan(_("Tasks/%(per)s") % {"per": per})
+        + term.move_x(3 * col_width)
+        + term.white(f"{tasks_per:.2f}")
+        + term.move_x(4 * col_width)
+        + term.cyan(_("Avg time"))
+        + term.move_x(5 * col_width)
+        + term.white(f"{exec_time:.4f}")
+    )
+    return True
+
+
+def memory(run_once=False, workers=False, broker=None):
+    if not broker:
+        broker = get_broker()
+    try:
+        from blessed import Terminal
+
+        term = Terminal()
+    except ImportError:
+        print(BLESSED_INSTALL_MESSAGE)
+        return
+    broker.ping()
+    if not psutil:
+        print(term.clear_eos())
+        print(
+            term.white_on_red(
+                'Cannot start "qmemory" command. Missing "psutil" library.'
+            )
+        )
+        return
+    with term.fullscreen(), term.hidden_cursor(), term.cbreak():
+        MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
+        MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
+        cols = 8
+        val = None
+        start_width = int(term.width / cols)
+        while val not in ["q", "Q"]:
+            col_width = int(term.width / cols)
+            # In case of resize
+            if col_width != start_width:
+                print(term.clear())
+                start_width = col_width
+            # sentinel, monitor and workers memory usage
+            print(
+                term.move(0, 0 * col_width)
+                + term.black_on_green(term.center(_("Host"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 1 * col_width)
+                + term.black_on_green(term.center(_("Id"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 2 * col_width)
+                + term.black_on_green(
+                    term.center(_("Available (%)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 3 * col_width)
+                + term.black_on_green(
+                    term.center(_("Available (MB)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 4 * col_width)
+                + term.black_on_green(term.center(_("Total (MB)"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 5 * col_width)
+                + term.black_on_green(
+                    term.center(_("Sentinel (MB)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 6 * col_width)
+                + term.black_on_green(
+                    term.center(_("Monitor (MB)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 7 * col_width)
+                + term.black_on_green(
+                    term.center(_("Workers (MB)"), width=col_width - 1)
+                )
+            )
+            row = 2
+            stats = Stat.get_all(broker=broker)
+            print(term.clear_eos())
+            for stat in stats:
+                # memory available (%)
+                memory_available_percentage = round(
+                    psutil.virtual_memory().available
+                    * 100
+                    / psutil.virtual_memory().total,
+                    2,
+                )
+                # memory available (MB)
+                memory_available = round(psutil.virtual_memory().available / 1024**2, 2)
+                if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
+                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
+                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
+                print(
+                    term.move(row, 0 * col_width)
+                    + term.center(stat.host[: col_width - 1], width=col_width - 1)
+                )
+                print(
+                    term.move(row, 1 * col_width)
+                    + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
+                )
+                print(
+                    term.move(row, 2 * col_width)
+                    + term.center(memory_available_percentage, width=col_width - 1)
+                )
+                print(
+                    term.move(row, 3 * col_width)
+                    + term.center(memory_available, width=col_width - 1)
+                )
+                print(
+                    term.move(row, 4 * col_width)
+                    + term.center(
+                        round(psutil.virtual_memory().total / 1024**2, 2),
+                        width=col_width - 1,
+                    )
+                )
+                print(
+                    term.move(row, 5 * col_width)
+                    + term.center(get_process_mb(stat.sentinel), width=col_width - 1)
+                )
+                print(
+                    term.move(row, 6 * col_width)
+                    + term.center(
+                        get_process_mb(getattr(stat, "monitor", None)),
+                        width=col_width - 1,
+                    )
+                )
+                workers_mb = 0
+                for worker_pid in stat.workers:
+                    result = get_process_mb(worker_pid)
+                    if isinstance(result, str):
+                        result = 0
+                    workers_mb += result
+                print(
+                    term.move(row, 7 * col_width)
+                    + term.center(
+                        workers_mb or "NO_PROCESSES_FOUND", width=col_width - 1
+                    )
+                )
+                row += 1
+            # each worker's memory usage
+            if workers:
+                row += 2
+                col_width = int(term.width / (1 + Conf.WORKERS))
+                print(
+                    term.move(row, 0 * col_width)
+                    + term.black_on_cyan(term.center(_("Id"), width=col_width - 1))
+                )
+                for worker_num in range(Conf.WORKERS):
+                    print(
+                        term.move(row, (worker_num + 1) * col_width)
+                        + term.black_on_cyan(
+                            term.center(
+                                "Worker #{} (MB)".format(worker_num + 1),
+                                width=col_width - 1,
+                            )
+                        )
+                    )
+                row += 2
+                for stat in stats:
+                    print(
+                        term.move(row, 0 * col_width)
+                        + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
+                    )
+                    for idx, worker_pid in enumerate(stat.workers):
+                        mb_used = get_process_mb(worker_pid)
+                        print(
+                            term.move(row, (idx + 1) * col_width)
+                            + term.center(mb_used, width=col_width - 1)
+                        )
+                    row += 1
+            row += 1
+            print(
+                term.move(row, 0)
+                + _("Available lowest (): %(memory_percent)s ((at)s)")
+                % {
+                    "memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
+                    "at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
+                        "%Y-%m-%d %H:%M:%S+00:00"
+                    ),
+                }
+            )
+            # for testing
+            if run_once:
+                return Stat.get_all(broker=broker)
+            print(term.move(row + 2, 0) + term.center(_("[Press q to quit]")))
+            val = term.inkey(timeout=1)
+
+
+def get_ids():
+    # prints id (PID) of running clusters
+    stat = Stat.get_all()
+    if stat:
+        for s in stat:
+            print(s.cluster_id)
+    else:
+        print(_("No clusters appear to be running."))
+    return True

django_q/pusher.py

@@ -0,0 +1,71 @@
+from multiprocessing import Event
+from multiprocessing.process import current_process
+from multiprocessing.queues import Queue
+from time import sleep
+
+from django import core
+from django.apps.registry import apps
+from django.utils.translation import gettext_lazy as _
+
+try:
+    apps.check_apps_ready()
+except core.exceptions.AppRegistryNotReady:
+    import django
+
+    django.setup()
+
+from django_q.brokers import Broker, get_broker
+from django_q.conf import Conf, logger
+from django_q.signing import BadSignature, SignedPackage
+
+try:
+    import setproctitle
+except ModuleNotFoundError:
+    setproctitle = None
+
+
+def pusher(task_queue: Queue, event: Event, broker: Broker = None):
+    """
+    Pulls tasks of the broker and puts them in the task queue
+    :type broker:
+    :type task_queue: multiprocessing.Queue
+    :type event: multiprocessing.Event
+    """
+    if not broker:
+        broker = get_broker()
+    proc_name = current_process().name
+    if setproctitle:
+        setproctitle.setproctitle(f"qcluster {proc_name} pusher")
+    logger.info(
+        _("%(name)s pushing tasks at %(id)s")
+        % {"name": proc_name, "id": current_process().pid}
+    )
+    while True:
+        try:
+            task_set = broker.dequeue()
+        except Exception:
+            logger.exception("Failed to pull task from broker")
+            # broker probably crashed. Let the sentinel handle it.
+            sleep(10)
+            break
+        if task_set:
+            for task in task_set:
+                ack_id = task[0]
+                # unpack the task
+                try:
+                    task = SignedPackage.loads(task[1])
+                except (TypeError, BadSignature):
+                    logger.exception("Failed to push task to queue")
+                    broker.fail(ack_id)
+                    continue
+                task["cluster"] = (
+                    Conf.CLUSTER_NAME
+                )  # save actual cluster name to orm task table
+                task["ack_id"] = ack_id
+                task_queue.put(task)
+            logger.debug(
+                _("queueing from %(list_key)s") % {"list_key": broker.list_key}
+            )
+        if event.is_set():
+            break
+    logger.info(_("%(name)s stopped pushing tasks") % {"name": current_process().name})

django_q/queues.py

@@ -2,6 +2,7 @@
 The code is derived from
 https://github.com/althonos/pronto/commit/3384010dfb4fc7c66a219f59276adef3288a886b
 """
+
 import multiprocessing
 import multiprocessing.queues
 import sys

django_q/scheduler.py

@@ -0,0 +1,140 @@
+import ast
+from multiprocessing.process import current_process
+
+from django import core, db
+from django.apps.registry import apps
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+
+try:
+    apps.check_apps_ready()
+except core.exceptions.AppRegistryNotReady:
+    import django
+
+    django.setup()
+
+from django_q.brokers import Broker, get_broker
+from django_q.conf import Conf, logger
+from django_q.humanhash import humanize
+from django_q.models import Schedule
+from django_q.tasks import async_task
+from django_q.utils import close_old_django_connections, localtime
+
+
+def scheduler(broker: Broker = None):
+    """
+    Creates a task from a schedule at the scheduled time and schedules next run
+    """
+    if not broker:
+        broker = get_broker()
+    close_old_django_connections()
+    try:
+        # Only default cluster will handler schedule with default(null) cluster
+        Q_default = (
+            db.models.Q(cluster__isnull=True)
+            if Conf.CLUSTER_NAME == Conf.PREFIX
+            else db.models.Q(pk__in=[])
+        )
+
+        with db.transaction.atomic(using=db.router.db_for_write(Schedule)):
+            for s in (
+                Schedule.objects.select_for_update()
+                .exclude(repeats=0)
+                .filter(next_run__lt=timezone.now())
+                .filter(Q_default | db.models.Q(cluster=Conf.CLUSTER_NAME))
+            ):
+                args = ()
+                kwargs = {}
+                # get args, kwargs and hook
+                if s.kwargs:
+                    try:
+                        # first try the dict syntax
+                        kwargs = ast.literal_eval(s.kwargs)
+                    except (SyntaxError, ValueError):
+                        # else use the kwargs syntax
+                        try:
+                            parsed_kwargs = (
+                                ast.parse(f"f({s.kwargs})").body[0].value.keywords
+                            )
+                            kwargs = {
+                                kwarg.arg: ast.literal_eval(kwarg.value)
+                                for kwarg in parsed_kwargs
+                            }
+                        except (SyntaxError, ValueError):
+                            kwargs = {}
+                if s.args:
+                    args = ast.literal_eval(s.args)
+                    # single value won't eval to tuple, so:
+                    if type(args) is not tuple:
+                        args = (args,)
+                q_options = kwargs.get("q_options", {})
+                if s.intended_date_kwarg:
+                    kwargs[s.intended_date_kwarg] = s.next_run.isoformat()
+                if s.hook:
+                    q_options["hook"] = s.hook
+                # set up the next run time
+                if s.schedule_type != s.ONCE:
+                    next_run = s.next_run
+                    while True:
+                        next_run = s.calculate_next_run(next_run)
+                        if Conf.CATCH_UP or next_run > localtime():
+                            break
+
+                    s.next_run = next_run
+
+                    # Little Fix for already broken numbers
+                    if s.repeats < -1:
+                        s.repeats = -1
+
+                    # Check if the value is not zero
+                    if s.repeats > 0:
+                        s.repeats -= 1
+
+                # send it to the cluster; any cluster name is allowed in multi-queue scenarios
+                # because `broker_name` is confusing, using `cluster` name is recommended and takes precedence
+                q_options["cluster"] = s.cluster or q_options.get(
+                    "cluster", q_options.pop("broker_name", None)
+                )
+                if (
+                    q_options["cluster"] is None
+                    or q_options["cluster"] == Conf.CLUSTER_NAME
+                ):
+                    q_options["broker"] = broker
+                q_options["group"] = q_options.get("group", s.name or s.id)
+                kwargs["q_options"] = q_options
+                s.task = async_task(s.func, *args, **kwargs)
+                # log it
+                if not s.task:
+                    logger.error(
+                        _(
+                            "%(process_name)s failed to create a task from schedule "
+                            "[%(schedule)s]"
+                        )
+                        % {
+                            "process_name": current_process().name,
+                            "schedule": s.name or s.id,
+                        }
+                    )
+                else:
+                    logger.info(
+                        _(
+                            "%(process_name)s created task %(task_name)s from schedule "
+                            "[%(schedule)s]"
+                        )
+                        % {
+                            "process_name": current_process().name,
+                            "task_name": humanize(s.task),
+                            "schedule": s.name or s.id,
+                        }
+                    )
+                # default behavior is to delete a ONCE schedule
+                if s.schedule_type == s.ONCE:
+                    if s.repeats < 0:
+                        s.delete()
+                        continue
+                    # but not if it has a positive repeats
+                    s.repeats = 0
+                # save the schedule
+                s.save()
+    except Exception:
+        logger.exception("Could not create task from schedule")

django_q/signals.py

@@ -32,6 +32,9 @@ def call_hook(sender, instance, **kwargs):
             )
 
 
+# args: proc_name
+post_spawn = Signal()
+
 # args: task
 pre_enqueue = Signal()
 

django_q/signing.py

@@ -1,4 +1,5 @@
 """Package signing."""
+
 import pickle
 
 from django_q import core_signing as signing

django_q/tasks.py

@@ -1,4 +1,5 @@
 """Provides task functionality."""
+
 # Standard
 from multiprocessing import Value
 from time import sleep, time
@@ -31,6 +32,7 @@ def async_task(func, *args, **kwargs):
         "iter_cached",
         "chain",
         "broker",
+        "cluster",
         "timeout",
     )
     q_options = keywords.pop("q_options", {})
@@ -52,7 +54,7 @@ def async_task(func, *args, **kwargs):
         elif key in keywords:
             task[key] = keywords.pop(key)
     # don't serialize the broker
-    broker = task.pop("broker", get_broker())
+    broker = task.pop("broker", None) or get_broker(task.get("cluster"))
     # overrides
     if "cached" not in task and Conf.CACHED:
         task["cached"] = Conf.CACHED
@@ -71,7 +73,7 @@ def async_task(func, *args, **kwargs):
         return _sync(pack)
     # push it
     enqueue_id = broker.enqueue(pack)
-    logger.info(f"Enqueued {enqueue_id}")
+    logger.info(f"Enqueued [{broker.list_key}] {enqueue_id}")
     logger.debug(f"Pushed {tag}")
     return task["id"]
 
@@ -90,6 +92,7 @@ def schedule(func, *args, **kwargs):
     :type next_run: datetime.datetime
     :param cluster: optional cluster name.
     :param cron: optional cron expression
+    :param intended_date_kwarg: optional identifier to pass intended schedule date.
     :param kwargs: function keyword arguments.
     :return: the schedule object.
     :rtype: Schedule
@@ -102,6 +105,7 @@ def schedule(func, *args, **kwargs):
     next_run = kwargs.pop("next_run", timezone.now())
     cron = kwargs.pop("cron", None)
     cluster = kwargs.pop("cluster", None)
+    intended_date_kwarg = kwargs.pop("intended_date_kwarg", None)
 
     # check for name duplicates instead of am unique constraint
     if name and Schedule.objects.filter(name=name).exists():
@@ -120,6 +124,7 @@ def schedule(func, *args, **kwargs):
         next_run=next_run,
         cron=cron,
         cluster=cluster,
+        intended_date_kwarg=intended_date_kwarg,
     )
     # make sure we trigger validation
     s.full_clean()
@@ -144,7 +149,7 @@ def result(task_id, wait=0, cached=Conf.CACHED):
     start = time()
     while True:
         r = Task.get_result(task_id)
-        if r:
+        if r is not None:
             return r
         if (time() - start) * 1000 >= wait >= 0:
             break
@@ -270,6 +275,7 @@ def fetch_cached(task_id, wait=0, broker=None):
                 hook=task.get("hook"),
                 args=task["args"],
                 kwargs=task["kwargs"],
+                cluster=task.get("cluster"),
                 started=task["started"],
                 stopped=task["stopped"],
                 result=task["result"],
@@ -340,6 +346,7 @@ def fetch_group_cached(group_id, failures=True, wait=0, count=None, broker=None)
                         hook=task.get("hook"),
                         args=task["args"],
                         kwargs=task["kwargs"],
+                        cluster=task.get("cluster"),
                         started=task["started"],
                         stopped=task["stopped"],
                         result=task["result"],
@@ -563,10 +570,12 @@ class Chain:
     A sequential chain of tasks
     """
 
-    def __init__(self, chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC):
+    def __init__(
+        self, chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC, broker=None
+    ):
         self.chain = chain or []
         self.group = group or ""
-        self.broker = get_broker()
+        self.broker = broker or get_broker()
         self.cached = cached
         self.sync = sync
         self.started = False
@@ -723,17 +732,14 @@ class AsyncTask:
         return self.id
 
     def result(self, wait=0):
-
         if self.started:
             return result(self.id, wait=wait, cached=self.cached)
 
     def fetch(self, wait=0):
-
         if self.started:
             return fetch(self.id, wait=wait, cached=self.cached)
 
     def result_group(self, failures=False, wait=0, count=None):
-
         if self.started and self.group:
             return result_group(
                 self.group,
@@ -744,7 +750,6 @@ class AsyncTask:
             )
 
     def fetch_group(self, failures=True, wait=0, count=None):
-
         if self.started and self.group:
             return fetch_group(
                 self.group,
@@ -757,7 +762,8 @@ class AsyncTask:
 
 def _sync(pack):
     """Simulate a package travelling through the cluster."""
-    from django_q.cluster import monitor, worker
+    from django_q.monitor import monitor
+    from django_q.worker import worker
 
     task_queue = Queue()
     result_queue = Queue()

django_q/tests/tasks.py

@@ -46,6 +46,10 @@ def hello():
     return "hello"
 
 
+def return_falsy_value():
+    return []
+
+
 def result(obj):
     print(f"RESULT HOOK {obj.name} : {obj.result()}")
 

django_q/tests/test_admin.py

@@ -64,7 +64,7 @@ def test_admin_views(admin_client, monkeypatch):
 
     # resubmit the failure
     url = reverse("admin:django_q_failure_changelist")
-    data = {"action": "retry_failed", "_selected_action": [f.pk]}
+    data = {"action": "resubmit_task", "_selected_action": [f.pk]}
     response = admin_client.post(url, data)
     assert response.status_code == 302
     assert Failure.objects.filter(name=f.id).exists() is False
@@ -84,3 +84,10 @@ def test_admin_views(admin_client, monkeypatch):
     data = {"post": "yes"}
     response = admin_client.post(url, data)
     assert response.status_code == 302
+    # Resubmit a successful task.
+    url = reverse("admin:django_q_success_changelist")
+    data = {"action": "resubmit_task", "_selected_action": [t.pk]}
+    initial_queue_count = OrmQ.objects.count()
+    response = admin_client.post(url, data)
+    assert response.status_code == 302
+    assert OrmQ.objects.count() > initial_queue_count

django_q/tests/test_cached.py

@@ -3,8 +3,9 @@ from multiprocessing import Event, Value
 import pytest
 
 from django_q.brokers import get_broker
-from django_q.cluster import monitor, pusher, worker
 from django_q.conf import Conf
+from django_q.monitor import monitor
+from django_q.pusher import pusher
 from django_q.queues import Queue
 from django_q.tasks import (
     AsyncTask,
@@ -21,6 +22,7 @@ from django_q.tasks import (
     result,
     result_group,
 )
+from django_q.worker import worker
 
 
 @pytest.fixture

django_q/tests/test_cluster.py

@@ -12,10 +12,12 @@ import pytest
 from django.utils import timezone
 
 from django_q.brokers import Broker, get_broker
-from django_q.cluster import Cluster, Sentinel, monitor, pusher, save_task, worker
+from django_q.cluster import Cluster, Sentinel
 from django_q.conf import Conf
 from django_q.humanhash import DEFAULT_WORDLIST, uuid
 from django_q.models import Success, Task
+from django_q.monitor import monitor, save_task
+from django_q.pusher import pusher
 from django_q.queues import Queue
 from django_q.signals import post_execute, pre_enqueue, pre_execute
 from django_q.status import Stat
@@ -29,8 +31,9 @@ from django_q.tasks import (
     result,
     result_group,
 )
-from django_q.tests.tasks import multiply, TaskError
+from django_q.tests.tasks import TaskError, multiply
 from django_q.utils import add_months, add_years
+from django_q.worker import worker
 
 myPath = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, myPath + "/../")
@@ -138,6 +141,30 @@ def test_cluster(broker):
     broker.delete_queue()
 
 
+@pytest.mark.django_db
+def test_results(broker):
+    broker.list_key = "cluster_test:q"
+    broker.delete_queue()
+    a = async_task(
+        "django_q.tests.tasks.return_falsy_value",
+        broker=broker,
+    )
+
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
+    pusher(task_queue, stop_event, broker=broker)
+    task_queue.put("STOP")
+    result_queue = Queue()
+    worker(task_queue, result_queue, Value("f", -1))
+    result_queue.put("STOP")
+    monitor(result_queue)
+
+    # should not loop indefinitely when a real value is returned
+    value = result(a, wait=-1)
+    assert value == []
+
+
 @pytest.mark.django_db
 def test_enqueue(broker, admin_user):
     broker.list_key = "cluster_test:q"

django_q/tests/test_monitor.py

@@ -5,7 +5,7 @@ import pytest
 from django_q.brokers import get_broker
 from django_q.cluster import Cluster
 from django_q.conf import Conf
-from django_q.monitor import get_ids, info, monitor
+from django_q.monitor_terminal import get_ids, info, monitor
 from django_q.status import Stat
 from django_q.tasks import async_task
 

django_q/tests/test_scheduler.py

@@ -1,9 +1,9 @@
 import os
-import pytz
 from datetime import datetime, timedelta
 from multiprocessing import Event, Value
 from unittest import mock
 
+import django
 import pytest
 from django.core.exceptions import ValidationError
 from django.db import IntegrityError
@@ -12,9 +12,11 @@ from django.utils import timezone
 from django.utils.timezone import is_naive
 
 from django_q.brokers import Broker, get_broker
-from django_q.cluster import localtime, monitor, pusher, scheduler, worker
 from django_q.conf import Conf
+from django_q.monitor import monitor
+from django_q.pusher import pusher
 from django_q.queues import Queue
+from django_q.scheduler import scheduler
 from django_q.tasks import Schedule, fetch
 from django_q.tasks import schedule as create_schedule
 from django_q.tests.settings import BASE_DIR
@@ -22,7 +24,17 @@ from django_q.tests.testing_utilities.multiple_database_routers import (
     TestingMultipleAppsDatabaseRouter,
     TestingReplicaDatabaseRouter,
 )
-from django_q.utils import add_months
+from django_q.utils import add_months, localtime
+from django_q.worker import worker
+
+if django.VERSION < (4, 0):
+    # pytz is the default in django 3.2. Remove when no support for 3.2
+    from pytz import timezone as ZoneInfo
+else:
+    try:
+        from zoneinfo import ZoneInfo
+    except ImportError:
+        from backports.zoneinfo import ZoneInfo
 
 
 @pytest.fixture
@@ -41,25 +53,11 @@ def orm_broker(monkeypatch) -> None:
     monkeypatch.setattr(Conf, "ORM", "default")
 
 
-@pytest.fixture
-def orm_no_replica_broker(orm_broker, monkeypatch) -> Broker:
-    """Generates a Broker with a disabled read replica database configuration."""
-    monkeypatch.setattr(Conf, "HAS_REPLICA", False)
-    return get_broker(list_key="scheduler_test:q")
-
-
-@pytest.fixture
-def orm_replica_broker(orm_broker, monkeypatch) -> Broker:
-    """Generates a Broker with read replica database configuration."""
-    monkeypatch.setattr(Conf, "HAS_REPLICA", True)
-    return get_broker(list_key="scheduler_test:q")
-
-
 REPLICA_DATABASE_ROUTERS = [
     f"{TestingReplicaDatabaseRouter.__module__}.{TestingReplicaDatabaseRouter.__name__}"
 ]
 REPLICA_DATABASES = {
-    "default": {
+    "writable": {
         "ENGINE": "django.db.backends.sqlite3",
         "NAME": os.path.join(BASE_DIR, "db.sqlite3"),
     },
@@ -90,7 +88,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     # 28th of March 2021 is the day when sunlight saving starts (at 2 am)
 
     monkeypatch.setattr(Conf, "TIME_ZONE", "Europe/Amsterdam")
-    tz = pytz.timezone('Europe/Amsterdam')
+    tz = ZoneInfo("Europe/Amsterdam")
     broker.list_key = "scheduler_test:q"
     # Let's start a schedule at 1 am on the 27th of March. This is in AMS timezone.
     # So, 2021-03-27 00:00:00 when saved (due to TZ being Amsterdam and saved in UTC)
@@ -140,6 +138,51 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     next_run = next_run.astimezone(tz)
     assert str(next_run) == "2021-03-30 01:00:00+02:00"
 
+    # Create second schedule with the next run date on the start date. It will move
+    # one day forward when we run the scheduler
+    start_date = datetime(2021, 10, 29, 1, 0, 0)
+    schedule = create_schedule(
+        "django_q.tests.tasks.word_multiply",
+        2,
+        name="multiply",
+        schedule_type=Schedule.DAILY,
+        next_run=start_date,
+    )
+
+    # Run scheduler so we get the next run date
+    scheduler(broker=broker)
+    schedule.refresh_from_db()
+
+    next_run = schedule.next_run
+
+    assert str(next_run) == "2021-10-29 23:00:00+00:00"
+    # In the Amsterdam timezone, it's 1 hour over midnight (+02)
+    next_run = next_run.astimezone(tz)
+    assert str(next_run) == "2021-10-30 01:00:00+02:00"
+
+    # Run scheduler so we get the next run date
+    scheduler(broker=broker)
+    schedule.refresh_from_db()
+
+    next_run = schedule.next_run
+
+    assert str(next_run) == "2021-10-30 23:00:00+00:00"
+    # In the Amsterdam timezone, it's 1 hour over midnight (+02)
+    next_run = next_run.astimezone(tz)
+    assert str(next_run) == "2021-10-31 01:00:00+02:00"
+
+    # Run scheduler so we get the next run date
+    scheduler(broker=broker)
+    schedule.refresh_from_db()
+
+    next_run = schedule.next_run
+
+    assert str(next_run) == "2021-11-01 00:00:00+00:00"
+    # In the Amsterdam timezone, it's 1 hour over midnight (+01)
+    # Switch of DST
+    next_run = next_run.astimezone(tz)
+    assert str(next_run) == "2021-11-01 01:00:00+01:00"
+
 
 @pytest.mark.django_db
 def test_scheduler(broker, monkeypatch):
@@ -313,7 +356,7 @@ def test_scheduler(broker, monkeypatch):
     assert schedule.next_run.date() == (timezone.now() + timedelta(weeks=2)).date()
     broker.delete_queue()
 
-    monkeypatch.setattr(Conf, "PREFIX", "some_cluster_name")
+    monkeypatch.setattr(Conf, "CLUSTER_NAME", "some_cluster_name")
     # create a schedule on another cluster
     schedule = create_schedule(
         "math.copysign",
@@ -337,7 +380,7 @@ def test_scheduler(broker, monkeypatch):
     # queue must be empty
     assert task_queue.qsize() == 0
 
-    monkeypatch.setattr(Conf, "PREFIX", "default")
+    monkeypatch.setattr(Conf, "CLUSTER_NAME", "default")
     # create a schedule on the same cluster
     schedule = create_schedule(
         "math.copysign",
@@ -362,43 +405,54 @@ def test_scheduler(broker, monkeypatch):
     assert task_queue.qsize() == 1
 
 
-@override_settings(
-    DATABASE_ROUTERS=REPLICA_DATABASE_ROUTERS, DATABASES=REPLICA_DATABASES
-)
 @pytest.mark.django_db
-def test_scheduler_atomic_transaction_must_specify_a_database_when_no_replicas_are_used(
-    orm_no_replica_broker: Broker,
-):
-    """
-    GIVEN a environment without a read replica database
-    WHEN the scheduler is called
-    THEN the transaction atomic must be called using the configured database in the
-    Conf.ORM settings.
-    """
-    broker = orm_no_replica_broker
-    with mock.patch("django_q.cluster.db") as mocked_db:
-        scheduler(broker=broker)
-        # The router should correctly set the database to use!
-        mocked_db.transaction.atomic.assert_called_with(using=broker.connection.db)
+def test_intended_schedule_kwarg(broker, monkeypatch):
+    broker.list_key = "scheduler_test:q"
+    broker.delete_queue()
+    run_date = timezone.now() - timedelta(hours=1)
+    schedule = create_schedule(
+        "math.copysign",
+        1,
+        -1,
+        name="test math",
+        hook="django_q.tests.tasks.result",
+        schedule_type=Schedule.HOURLY,
+        repeats=1,
+        next_run=run_date,
+        intended_date_kwarg="intended_date",
+    )
+    assert schedule.last_run() is None
+    assert schedule.intended_date_kwarg == "intended_date"
+    # run scheduler
+    scheduler(broker=broker)
+    # set up the workflow
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
+    # push it
+    pusher(task_queue, stop_event, broker=broker)
+    assert task_queue.qsize() == 1
+    task = task_queue.get()
+    assert "intended_date" in task["kwargs"]
+    assert task["kwargs"]["intended_date"] == run_date.isoformat()
 
 
 @override_settings(
     DATABASE_ROUTERS=REPLICA_DATABASE_ROUTERS, DATABASES=REPLICA_DATABASES
 )
 @pytest.mark.django_db
-def test_scheduler_atomic_must_specify_no_db_when_read_write_replicas_are_used(
-    orm_replica_broker: Broker,
+def test_scheduler_atomic_must_specify_the_write_db(
+    orm_broker: Broker,
 ):
     """
     GIVEN a environment with a read/write configured replica database
     WHEN the scheduler is called
-    THEN the transaction must be called without a specific database, thus letting the
-    database router pick.
+    THEN the transaction must be called with the write database.
     """
-    with mock.patch("django_q.cluster.db") as mocked_db:
-        scheduler(broker=orm_replica_broker)
-        # No specific databases should be set here, this is the job of the router!
-        mocked_db.transaction.atomic.assert_called_with()
+    broker = get_broker(list_key="scheduler_test:q")
+    with mock.patch("django_q.cluster.db.transaction") as mocked_db:
+        scheduler(broker=broker)
+        mocked_db.atomic.assert_called_with(using="writable")
 
 
 @override_settings(
@@ -406,20 +460,17 @@ def test_scheduler_atomic_must_specify_no_db_when_read_write_replicas_are_used(
 )
 @pytest.mark.django_db
 def test_scheduler_atomic_must_specify_the_database_based_on_router_redirection(
-    orm_no_replica_broker: Broker,
+    orm_broker: Broker,
 ):
     """
     GIVEN a environment without a read replica database
     WHEN the scheduler is called
-    THEN the transaction atomic must be called using the configured database in the
-    Conf.ORM settings.
+    THEN the transaction atomic must be called using the default connection.
     """
-    broker = orm_no_replica_broker
-    with mock.patch("django_q.cluster.db") as mocked_db:
+    broker = get_broker(list_key="scheduler_test:q")
+    with mock.patch("django_q.cluster.db.transaction") as mocked_db:
         scheduler(broker=broker)
-        # The router should correctly set the database to use!
-        assert broker.connection.db == "default"
-        mocked_db.transaction.atomic.assert_called_with(using=broker.connection.db)
+        mocked_db.atomic.assert_called_with(using="default")
 
 
 def test_localtime():

django_q/tests/testing_utilities/multiple_database_routers.py

@@ -12,9 +12,9 @@ class TestingReplicaDatabaseRouter:
 
     def db_for_write(self, model, **hints):
         """
-        Always write to DEFAULT database
+        Always write to WRITABLE database
         """
-        return "default"
+        return "writable"
 
 
 class TestingMultipleAppsDatabaseRouter:

django_q/timeout.py

@@ -0,0 +1,43 @@
+import signal
+
+from django.utils.translation import gettext_lazy as _
+
+from django_q.conf import logger
+
+from .exceptions import TimeoutException
+
+
+class TimeoutHandler:
+    def __init__(self, timeout: int):
+        self._timeout = timeout
+
+    def raise_timeout_exception(self, signum, frame):
+        raise TimeoutException(
+            f"Task exceeded maximum timeout value ({self._timeout} seconds)"
+        )
+
+    def __enter__(self):
+        # if the timeout is -1, then there is no timeout and the task will always keep running until it's done or manually killed
+        if self._timeout == -1:
+            return
+        try:
+            signal.signal(signal.SIGALRM, self.raise_timeout_exception)
+            signal.alarm(self._timeout)
+        except (
+            ValueError,
+            AttributeError,
+        ):  # AttributeError or ValueError might be raised for Windows users
+            logger.debug(_("SIGALARM is not available on your platform"))
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self._timeout == -1:
+            return
+        """When getting out of the timeout, reset the alarm, so it won't trigger"""
+        try:
+            signal.alarm(0)
+            signal.signal(signal.SIGALRM, signal.SIG_DFL)
+        except (
+            ValueError,
+            AttributeError,
+        ):  # AttributeError or ValueError might be raised for Windows users
+            logger.debug(_("SIGALARM is not available on your platform"))

django_q/utils.py

@@ -1,13 +1,23 @@
-from datetime import datetime
-import pytz
 import calendar
 import inspect
-from datetime import date
+from datetime import date, datetime
 
-from django.utils import timezone
+import django
+from django import db
 from django.conf import settings
+from django.utils import timezone
+
+from django_q.conf import Conf, logger
+
+if django.VERSION < (4, 0):
+    # pytz is the default in django 3.2. Remove when no support for 3.2
+    from pytz import timezone as ZoneInfo
+else:
+    try:
+        from zoneinfo import ZoneInfo
+    except ImportError:
+        from backports.zoneinfo import ZoneInfo
 
-from django_q.conf import Conf
 
 # credits: https://stackoverflow.com/a/4131114
 # Made them aware of timezone
@@ -44,14 +54,35 @@ def get_func_repr(func):
             f"{func.__self__.__module__}." f"{func.__self__.__name__}.{func.__name__}"
         )
     else:
-        return str(func)
+        return str(func) if func else None
 
 
 def localtime(value=None) -> datetime:
     """Override for timezone.localtime to deal with naive times and local times"""
     if settings.USE_TZ:
-        return timezone.localtime(value=value, timezone=pytz.timezone(Conf.TIME_ZONE))
+        if django.VERSION >= (4, 0) and getattr(settings, "USE_DEPRECATED_PYTZ", False):
+            import pytz
+
+            convert_to_tz = pytz.timezone(Conf.TIME_ZONE)
+        else:
+            convert_to_tz = ZoneInfo(Conf.TIME_ZONE)
+
+        return timezone.localtime(value=value, timezone=convert_to_tz)
     if value is None:
         return datetime.now()
     else:
         return value
+
+
+def close_old_django_connections():
+    """
+    Close django connections unless running with sync=True.
+    """
+    if Conf.SYNC:
+        logger.warning(
+            "Preserving django database connections because sync=True. Beware "
+            "that tasks are now injected in the calling context/transactions "
+            "which may result in unexpected behaviour."
+        )
+    else:
+        db.close_old_connections()

django_q/worker.py

@@ -0,0 +1,141 @@
+import pydoc
+import traceback
+from multiprocessing import Value
+from multiprocessing.process import current_process
+from multiprocessing.queues import Queue
+
+from django import core
+from django.apps.registry import apps
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+
+try:
+    apps.check_apps_ready()
+except core.exceptions.AppRegistryNotReady:
+    import django
+
+    django.setup()
+
+from django_q.conf import Conf, error_reporter, logger, resource, setproctitle
+from django_q.exceptions import TimeoutException
+from django_q.signals import post_spawn, pre_execute
+from django_q.timeout import TimeoutHandler
+from django_q.utils import close_old_django_connections, get_func_repr
+
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+try:
+    import setproctitle
+except ModuleNotFoundError:
+    setproctitle = None
+
+
+def worker(
+    task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
+):
+    """
+    Takes a task from the task queue, tries to execute it and puts the result back in
+    the result queue
+    :param timeout: number of seconds wait for a worker to finish.
+    :type task_queue: multiprocessing.Queue
+    :type result_queue: multiprocessing.Queue
+    :type timer: multiprocessing.Value
+    """
+    proc_name = current_process().name
+    logger.info(
+        _("%(proc_name)s ready for work at %(id)s")
+        % {"proc_name": proc_name, "id": current_process().pid}
+    )
+    post_spawn.send(sender="django_q", proc_name=proc_name)
+    if setproctitle:
+        setproctitle.setproctitle(f"qcluster {proc_name} idle")
+    task_count = 0
+    if timeout is None:
+        timeout = -1
+    # Start reading the task queue
+    for task in iter(task_queue.get, "STOP"):
+        result = None
+        timer.value = -1  # Idle
+        task_count += 1
+        f = task["func"]
+
+        # Log task creation and set process name
+        # Get the function from the task
+        func_name = get_func_repr(f)
+        task_name = task["name"]
+        task_desc = _("%(proc_name)s processing %(task_name)s '%(func_name)s'") % {
+            "proc_name": proc_name,
+            "func_name": func_name,
+            "task_name": task_name,
+        }
+        if "group" in task:
+            task_desc += f" [{task['group']}]"
+        logger.info(task_desc)
+
+        if setproctitle:
+            proc_title = f"qcluster {proc_name} processing {task_name} '{func_name}'"
+            if "group" in task:
+                proc_title += f" [{task['group']}]"
+            setproctitle.setproctitle(proc_title)
+
+        # if it's not an instance try to get it from the string
+        if not callable(f):
+            # locate() returns None if f cannot be loaded
+            f = pydoc.locate(f)
+        close_old_django_connections()
+        timer_value = task.pop("timeout", timeout)
+        # signal execution
+        pre_execute.send(sender="django_q", func=f, task=task)
+        # execute the payload
+        timer.value = timer_value  # Busy
+        if timer.value != -1:
+            timer.value += 3  # Add buffer so that guard doesn't kill the process on timeout before it gets processed
+
+        timeout_error = False
+        try:
+            if f is None:
+                # raise a meaningfull error if task["func"] is not a valid function
+                raise ValueError(f"Function {task['func']} is not defined")
+            with TimeoutHandler(timer_value):
+                res = f(*task["args"], **task["kwargs"])
+            result = (res, True)
+        except (Exception, TimeoutException) as e:
+            if isinstance(e, TimeoutException):
+                timeout_error = True
+            result = (f"{e} : {traceback.format_exc()}", False)
+            if error_reporter:
+                error_reporter.report()
+            if task.get("sync", False):
+                raise
+
+        with timer.get_lock():
+            # Process result
+            task["result"] = result[0]
+            task["success"] = result[1]
+            task["stopped"] = timezone.now()
+            result_queue.put(task)
+            if timeout_error:
+                # force destroy process due to timeout
+                timer.value = 0
+                break
+
+            timer.value = -1  # Idle
+            if setproctitle:
+                setproctitle.setproctitle(f"qcluster {proc_name} idle")
+            # Recycle
+            if task_count == Conf.RECYCLE or rss_check():
+                timer.value = -2  # Recycled
+                break
+    logger.info(_("%(proc_name)s stopped doing work") % {"proc_name": proc_name})
+
+
+def rss_check():
+    if Conf.MAX_RSS:
+        if resource:
+            return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
+        elif psutil:
+            return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
+    return False

docs/chain.rst

@@ -49,8 +49,9 @@ Reference
     :param str group: an optional group name.
     :param bool cached: run this against the cache backend
     :param bool sync: execute this inline instead of asynchronous
+    :param broker: an optional broker instance
 
-.. py:class:: Chain(chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC)
+.. py:class:: Chain(chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC, broker=None)
 
     A sequential chain of tasks. Acts as a convenient wrapper for :func:`async_chain`
     You can pass the task chain at construction or you can append individual tasks before running them.
@@ -59,6 +60,8 @@ Reference
         :param str group: an optional group name.
         :param bool cached: run this against the cache backend
         :param bool sync: execute this inline instead of asynchronous
+        :param bool sync: execute this inline instead of asynchronous
+        :param broker: an optional broker instance
 
 
     .. py:method:: append(func, *args, **kwargs)

docs/cluster.rst

@@ -11,36 +11,36 @@ Start your cluster using Django's ``manage.py`` command::
 
 You should see the cluster starting ::
 
-    10:57:40 [Q] INFO Q Cluster-31781 starting.
-    10:57:40 [Q] INFO Process-1:1 ready for work at 31784
-    10:57:40 [Q] INFO Process-1:2 ready for work at 31785
-    10:57:40 [Q] INFO Process-1:3 ready for work at 31786
-    10:57:40 [Q] INFO Process-1:4 ready for work at 31787
-    10:57:40 [Q] INFO Process-1:5 ready for work at 31788
-    10:57:40 [Q] INFO Process-1:6 ready for work at 31789
-    10:57:40 [Q] INFO Process-1:7 ready for work at 31790
-    10:57:40 [Q] INFO Process-1:8 ready for work at 31791
-    10:57:40 [Q] INFO Process-1:9 monitoring at 31792
-    10:57:40 [Q] INFO Process-1 guarding cluster at 31783
-    10:57:40 [Q] INFO Process-1:10 pushing tasks at 31793
-    10:57:40 [Q] INFO Q Cluster-31781 running.
+    10:57:40 [Q] INFO Q Cluster freddie-uncle-twenty-ten starting.
+    10:57:40 [Q] INFO Process-ede257774c4444c980ab479f10947acc ready for work at 31784
+    10:57:40 [Q] INFO Process-ed580482da3f42968230baa2e4253e42 ready for work at 31785
+    10:57:40 [Q] INFO Process-8a370dc2bc1d49aa9864e517c9895f74 ready for work at 31786
+    10:57:40 [Q] INFO Process-74912f9844264d1397c6e54476b530c0 ready for work at 31787
+    10:57:40 [Q] INFO Process-b00edb26c6074a6189e5696c60aeb35b ready for work at 31788
+    10:57:40 [Q] INFO Process-b0862965db04479f9784a26639ee51e0 ready for work at 31789
+    10:57:40 [Q] INFO Process-7e8abbb8ca2d4d9bb20a937dd5e2872b ready for work at 31790
+    10:57:40 [Q] INFO Process-b0862965db04479f9784a26639ee51e0 ready for work at 31791
+    10:57:40 [Q] INFO Process-67fa9461ac034736a766cd813f617e62 monitoring at 31792
+    10:57:40 [Q] INFO Process-eac052c646b2459797cee98bdb84c85d guarding cluster at 31783
+    10:57:40 [Q] INFO Process-5d98deb19b1e4b2da2ef1e5bd6824f75 pushing tasks at 31793
+    10:57:40 [Q] INFO Q Cluster freddie-uncle-twenty-ten running.
 
 
 Stopping the cluster with ctrl-c or either the ``SIGTERM`` and ``SIGKILL`` signals, will initiate the :ref:`stop_procedure`::
 
-    16:44:12 [Q] INFO Q Cluster-31781 stopping.
-    16:44:12 [Q] INFO Process-1 stopping cluster processes
-    16:44:13 [Q] INFO Process-1:10 stopped pushing tasks
-    16:44:13 [Q] INFO Process-1:6 stopped doing work
-    16:44:13 [Q] INFO Process-1:4 stopped doing work
-    16:44:13 [Q] INFO Process-1:1 stopped doing work
-    16:44:13 [Q] INFO Process-1:5 stopped doing work
-    16:44:13 [Q] INFO Process-1:7 stopped doing work
-    16:44:13 [Q] INFO Process-1:3 stopped doing work
-    16:44:13 [Q] INFO Process-1:8 stopped doing work
-    16:44:13 [Q] INFO Process-1:2 stopped doing work
-    16:44:14 [Q] INFO Process-1:9 stopped monitoring results
-    16:44:15 [Q] INFO Q Cluster-31781 has stopped.
+    16:44:12 [Q] INFO Q Cluster freddie-uncle-twenty-ten stopping.
+    16:44:12 [Q] INFO Process-eac052c646b2459797cee98bdb84c85d stopping cluster processes
+    16:44:13 [Q] INFO Process-5d98deb19b1e4b2da2ef1e5bd6824f75 stopped pushing tasks
+    16:44:13 [Q] INFO Process-b0862965db04479f9784a26639ee51e0 stopped doing work
+    16:44:13 [Q] INFO Process-7e8abbb8ca2d4d9bb20a937dd5e2872b stopped doing work
+    16:44:13 [Q] INFO Process-b0862965db04479f9784a26639ee51e0 stopped doing work
+    16:44:13 [Q] INFO Process-b00edb26c6074a6189e5696c60aeb35b stopped doing work
+    16:44:13 [Q] INFO Process-74912f9844264d1397c6e54476b530c0 stopped doing work
+    16:44:13 [Q] INFO Process-8a370dc2bc1d49aa9864e517c9895f74 stopped doing work
+    16:44:13 [Q] INFO Process-ed580482da3f42968230baa2e4253e42 stopped doing work
+    16:44:13 [Q] INFO Process-ede257774c4444c980ab479f10947acc stopped doing work
+    16:44:14 [Q] INFO Process-67fa9461ac034736a766cd813f617e62 stopped monitoring results
+    16:44:15 [Q] INFO Q Cluster freddie-uncle-twenty-ten has stopped.
 
 The number of workers, optional timeouts, recycles and cpu_affinity can be controlled via the :doc:`configure` settings.
 
@@ -52,6 +52,30 @@ You can have multiple clusters on multiple machines, working on the same queue a
 - They use the same cluster name. See :doc:`configure`
 - They share the same ``SECRET_KEY`` for Django.
 
+.. _multiple-queues
+
+Multiple Queues
+-----------------
+You can have multiple queues in one Django site, and use multiple cluster to work on each queue.
+Different queues are identified by different queue names which are also cluster names.
+To run an alternate cluster, e.g. to work on the 'long' queue, start your cluster with command::
+
+    # On Linux
+    $ Q_CLUSTER_NAME=long python manage.py qcluster
+
+    # On Windows
+    $ python manage.py qcluster --name long
+
+You can set different Q_CLUSTER options for alternative clusters, such as 'timeout', 'queue_limit'
+and any other options which are valid in :doc:`configure`. See :ref:`alt-clusters`.
+
+.. note::
+
+    To use multiple queue, use the keyword argument `cluster` in async_task() and schedule():
+
+    * if `cluster` is not set (the default), async_task() and schedule() will be handled by the default cluster;
+    * if `cluster` is set, only clusters with matching cluster name will run the task or do the schedule.
+
 Using a Procfile
 ----------------
 If you host on `Heroku <https://heroku.com>`__ or you are using `Honcho <https://github.com/nickstenning/honcho>`__ you can start the cluster from a :file:`Procfile` with an entry like this::

docs/conf.py

@@ -73,9 +73,9 @@ author = "Ilan Steemers, Stan Triepels"
 # built documents.
 #
 # The short X.Y version.
-version = "1.4"
+version = "1.7"
 # The full version, including alpha/beta/rc tags.
-release = "1.4.7"
+release = "1.7.3"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/configure.rst

@@ -20,7 +20,18 @@ Configuration is handled via the ``Q_CLUSTER`` dictionary in your :file:`setting
         'redis': {
             'host': '127.0.0.1',
             'port': 6379,
-            'db': 0, }
+            'db': 0, },
+        'ALT_CLUSTERS': {
+            'long': {
+                'timeout': 3000,
+                'retry': 3600,
+                'max_attempts': 2,
+            },
+            'short': {
+                'timeout': 10,
+                'max_attempts': 1,
+            },
+        }
     }
 
 All configuration settings are optional:
@@ -68,6 +79,8 @@ timeout
 The number of seconds a worker is allowed to spend on a task before it's terminated. Defaults to ``None``, meaning it will never time out.
 Set this to something that makes sense for your project. Can be overridden for individual tasks.
 
+Note: for systems that don't have `SIGALRM` available (e.g. Windows), it will not raise an error properly. It will kill the task, but it will keep retrying until it finishes within the given time.
+
 See :ref:`retry` for details how to set values for timeout and retry.
 
 .. _time_zone:
@@ -317,16 +330,6 @@ Using the Django ORM backend will also enable the Queued Tasks table in the Admi
 If you need better performance , you should consider using a different database backend than the main project.
 Set ``orm`` to the name of that database connection and make sure you run migrations on it using the ``--database`` option.
 
-When using the Django database as a message broker, you can set the ``has_replica`` boolean keyword to ensure Django-Q2 works properly letting a `Database Router <https://docs.djangoproject.com/en/3.2/topics/db/multi-db/>`__. ::
-
-    # example ORM broker connection with replica database
-
-    Q_CLUSTER = {
-        ...
-        'orm': 'default',
-        'has_replica': True
-    }
-
 .. _mongo_configuration:
 
 mongo
@@ -347,7 +350,7 @@ To use MongoDB as a message broker you simply provide the connection information
         }
     }
 
-The ``mongo`` dictionary can contain any of the parameters exposed by pymongo's `MongoClient <https://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`__
+The ``mongo`` dictionary can contain any of the parameters exposed by pymongo's `MongoClient <https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`__
 If you want to use a mongodb uri, you can supply it as the ``host`` parameter.
 
 mongo_db
@@ -469,6 +472,24 @@ As a rule of thumb; cpu_affinity 1 favors repetitive short running tasks, while
 
     *Psutil does not support cpu affinity on OS X at this time.*
 
+.. _alt-clusters:
+
+ALT_CLUSTERS
+~~~~~~~~~~~~
+
+For multiple clusters working on multiple queues to run in one Django site.
+ALT_CLUSTERS should be a dict with cluster_name as its key, and the value is the configuration for the cluster
+with the key as its name. The configuration items are consistent with Q_CLUSTER,
+except for a few items such as name/cluster_name/ALT_CLUSTER, which are not available of course.
+
+See :ref:`multiple-queues`.
+
+.. note::
+
+    For a cluster, if its name is in ALT_CLUSTERS, the config item in ALT_CLUSTER will override
+    the same config item in the Q_CLUSTER root. Other config items in Q_CLUSTER root remain in effect for this cluster.
+
+
 .. py:module:: django_q
 
 .. rubric:: Footnotes

docs/docker-compose.yaml

@@ -1,4 +1,3 @@
-version: "3.0"
 services:
   docs:
     container_name: djangoq2-docs

docs/examples.rst

@@ -328,7 +328,7 @@ Requires cache to be enabled. Save file in your Django project's root directory
     # All django stuff has to come after the setup:
     django.setup()
 
-    from django_q.monitor import Stat
+    from django_q.status import Stat
     from django_q.conf import Conf
 
     # Set host and port settings

docs/index.rst

@@ -27,7 +27,7 @@ Features
 -  Rollbar and Sentry support
 
 
-Django Q2 is tested with: Python 3.8, 3.9 and 3.10, 3.11, Django 3.2.x and 4.1.x
+Django Q2 is tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 4.2.x and 5.0.x
 
 Currently available in English, German and French.
 

docs/install.rst

@@ -24,15 +24,31 @@ Installation
 
     $ python manage.py qcluster
 
+
+Migrate from Django-Q to Django-Q2
+----------------------------------
+
+If you have an application with django-q running right now, you can simply swap the libraries and you should be good to go.::
+
+
+    $ pip uninstall django-q # you might have to uninstall django-q add-ons as well
+    $ pip install django-q2
+
+
+Then migrate the database to get the latest tables/fields::
+
+    $ python manage.py migrate
+
+
 Requirements
 ------------
 
-Django Q2 is tested for Python 3.8, 3.9, 3.10 and 3.11
+Django Q2 is tested for Python 3.8, 3.9, 3.10, 3.11 and 3.12
 
 -  `Django <https://www.djangoproject.com>`__
 
     Django Q2 aims to use as much of Django's standard offerings as possible.
-    The code is tested against Django versions `3.2.x` and `4.1.x`.
+    The code is tested against Django versions `3.2.x`, `4.1.x`, `4.2.x` and `5.0.x`.
 
 -  `Django-picklefield <https://github.com/gintas/django-picklefield>`__
 
@@ -55,6 +71,10 @@ Optional
 
     $ pip install psutil
 
+- `setproctitle <https://github.com/dvarrazzo/py-setproctitle>`__  python module to customize the process title by Daniele Varrazzo', is an optional requirement used to set informative process titles::
+
+    $ pip install setproctitle
+
 -  `Hiredis <https://github.com/redis/hiredis>`__ parser. This C library maintained by the core Redis team is faster than the standard PythonParser during high loads::
 
     $ pip install hiredis
@@ -95,11 +115,11 @@ Add-ons
 -------
 - `django-q-rollbar <https://github.com/danielwelch/django-q-rollbar>`__ is a Rollbar error reporter::
 
-    $ pip install django-q[rollbar]
+    $ pip install django-q2[rollbar]
 
 - `django-q-sentry <https://github.com/danielwelch/django-q-sentry>`__ is a Sentry error reporter::
 
-    $ pip install django-q[sentry]
+    $ pip install django-q2[sentry]
 
 - `django-q-email <https://github.com/joeyespo/django-q-email>`__ is a compatible Django email backend that will automatically async queue your emails.
 
@@ -125,7 +145,7 @@ Other known issues are:
 
 Python
 ~~~~~~
-Current tests are performed with 3.8, 3.9, 3.10 and 3.11
+Current tests are performed with 3.8, 3.9, 3.10, 3.11 and 3.12
 If you do encounter any regressions with earlier versions, please submit an issue on `github <https://github.com/GDay/django-q2>`__
 
 Open-source packages
@@ -135,8 +155,8 @@ You can reference the `requirements <https://github.com/GDay/django-q2/blob/mast
 
 Django
 ~~~~~~
-We strive to be compatible with last two major version of Django.
-At the moment this means we support the 3.2.x and 4.1.x releases.
+We strive to be compatible with the last two major version of Django.
+At the moment this means we support the 3.2.x, 4.1.x, 4.2.x and 5.0.x releases.
 
-Since we are now no longer supporting Python 2, we can also not support older versions of Django that do not support Python >= 3.6
+Since we are now no longer supporting Python 2, we can also not support older versions of Django that do not support Python >= 3.8
 For this you can always use older releases, but they are no longer maintained.

docs/monitor.rst

@@ -111,7 +111,7 @@ You can check the status of your clusters straight from your code with the :clas
 
 .. code:: python
 
-    from django_q.monitor import Stat
+    from django_q.status import Stat
 
     for stat in Stat.get_all():
         print(stat.cluster_id, stat.status)

docs/schedules.rst

@@ -69,6 +69,10 @@ You can change this by setting the :ref:`catch_up` configuration setting to ``Fa
 The scheduler will then skip execution of scheduled events in the past.
 Instead those tasks will run once when the cluster starts again and the scheduler will find the next available slot in the future according to original schedule parameters.
 
+When :ref:`catch_up` is to ``True`` it may be useful for the task to know what was the date and time it was originally intended to run at.
+To achieve this, pass an identifier name to parameter `intended_date_kwarg` when creating the schedule. The intended datetime will then be passed - in isoformat string - as
+a kwarg with that identifier name to the task that has been created.
+
 Management Commands
 -------------------
 
@@ -123,9 +127,10 @@ Reference
     :param int repeats: Number of times to repeat schedule. -1=Always, 0=Never, n =n.
     :param datetime next_run: Next or first scheduled execution datetime.
     :param str cluster: optional cluster name. Task will be executed only on a cluster with a matching :ref:`name`.
+    :param str intended_date_kwarg: optional identifier to pass intended schedule date.
     :param dict q_options: options passed to async_task for this schedule
     :param kwargs: optional keyword arguments for the scheduled function.
-    
+
     .. note::
 
         q_options does not accept the 'broker' key with a broker instance but accepts a 'broker_name' key instead. This can be used to specify the broker connection name to assign the task. If a broker with the specified name does not exist or is not running at the moment of placing the task in queue it fallbacks to the random broker/queue that handled the schedule.
@@ -183,9 +188,13 @@ Reference
     When set to -1, this will keep counting down.
 
     .. py:attribute:: cluster
-    
+
     Task will be executed only on a cluster with a matching :ref:`name`.
 
+    .. py:attribute:: intended_date_kwarg
+
+    Name of kwarg to pass intended schedule date.
+
     .. py:attribute:: next_run
 
     Datetime of the next scheduled execution.

docs/signals.rst

@@ -13,6 +13,12 @@ Before enqueuing a task
 The ``django_q.signals.pre_enqueue`` signal is emitted before a task is
 enqueued. The task dictionary is given as the ``task`` argument.
 
+After spawning a worker process
+"""""""""""""""""""""""""""""""
+
+The ``django_q.signals.post_spawn`` signal is emitted after a worker process has
+spawned. The process name is given as the ``proc_name`` argument (string).
+
 Before executing a task
 """""""""""""""""""""""
 
@@ -37,7 +43,7 @@ Connecting to a Django Q2 signal is done the same as any other Django
 signal::
 
     from django.dispatch import receiver
-    from django_q.signals import pre_enqueue, pre_execute, post_execute
+    from django_q.signals import pre_enqueue, pre_execute, post_execute, post_spawn
 
     @receiver(pre_enqueue)
     def my_pre_enqueue_callback(sender, task, **kwargs):
@@ -51,4 +57,8 @@ signal::
     def my_post_execute_callback(sender, task, **kwargs):
         print(f"Task {task['name']} was executed with result {task['result']}")
 
+    @receiver(post_spawn)
+    def my_post_spawn_callback(sender, proc_name, **kwargs):
+        print(f"Process {proc_name} has spawned")
+
 

docs/tasks.rst

@@ -75,6 +75,10 @@ broker
 """"""
 A broker instance, in case you want to control your own connections.
 
+cluster
+""""""
+The name of the cluster. Only useful if you are using [alternative queues](https://django-q2.readthedocs.io/en/master/cluster.html#multiple-queues).
+
 task_name
 """""""""
 
@@ -121,6 +125,7 @@ Optionally you can use the :class:`AsyncTask` class to instantiate a task and ke
     a.run()
 
     # wait indefinitely for the result and print it
+    # don't let the task return `None` or it will wait indefinitely
     print(a.result(wait=-1))
 
     # change the args
@@ -240,7 +245,7 @@ Reference
 ---------
 
 .. py:function:: async_task(func, *args, hook=None, group=None, timeout=None,\
-    save=None, sync=False, cached=False, broker=None, q_options=None, **kwargs)
+    save=None, sync=False, cached=False, broker=None, cluster=None, q_options=None, **kwargs)
 
     Puts a task in the cluster queue
 
@@ -254,6 +259,7 @@ Reference
    :param bool sync: If set to True, async_task will simulate a task execution
    :param cached: Output the result to the cache backend. Bool or timeout in seconds
    :param broker: Optional broker connection from :func:`brokers.get_broker`
+   :param cluster: Optional cluster name if using alternative queues
    :param dict q_options: Options dict, overrides option keywords
    :param dict kwargs: Keyword arguments for the task function
    :returns: The uuid of the task
@@ -264,7 +270,7 @@ Reference
     Gets the result of a previously executed task
 
     :param str task_id: the uuid or name of the task
-    :param int wait: optional milliseconds to wait for a result. -1 for indefinite
+    :param int wait: optional milliseconds to wait for a result. -1 for indefinite, but be sure the result will not be `None` otherwise it will wait indefinitely!
     :param bool cached: run this against the cache backend.
     :returns: The result of the executed task
 

exampleproject/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for exampleproject project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "exampleproject.settings")
+
+application = get_asgi_application()

exampleproject/settings.py

@@ -0,0 +1,134 @@
+"""
+Django settings for exampleproject project.
+
+Generated by 'django-admin startproject' using Django 5.1.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/5.1/ref/settings/
+"""
+
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = "django-insecure-)oouh93bjg+c=b!l5-*w)7et1l+!3nmp223rr^1r4v#jn&ow3f"
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = ["*"]
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    "django.contrib.admin",
+    "django.contrib.auth",
+    "django.contrib.contenttypes",
+    "django.contrib.sessions",
+    "django.contrib.messages",
+    "django.contrib.staticfiles",
+    "django_q",
+]
+
+MIDDLEWARE = [
+    "django.middleware.security.SecurityMiddleware",
+    "django.contrib.sessions.middleware.SessionMiddleware",
+    "django.middleware.common.CommonMiddleware",
+    "django.middleware.csrf.CsrfViewMiddleware",
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
+    "django.contrib.messages.middleware.MessageMiddleware",
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
+]
+
+ROOT_URLCONF = "exampleproject.urls"
+
+TEMPLATES = [
+    {
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
+        "DIRS": [],
+        "APP_DIRS": True,
+        "OPTIONS": {
+            "context_processors": [
+                "django.template.context_processors.debug",
+                "django.template.context_processors.request",
+                "django.contrib.auth.context_processors.auth",
+                "django.contrib.messages.context_processors.messages",
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = "exampleproject.wsgi.application"
+
+
+# Database
+# https://docs.djangoproject.com/en/5.1/ref/settings/#databases
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": BASE_DIR / "db.sqlite3",
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/5.1/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
+    },
+    {
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/5.1/topics/i18n/
+
+LANGUAGE_CODE = "en-us"
+
+TIME_ZONE = "UTC"
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/5.1/howto/static-files/
+
+STATIC_URL = "static/"
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+
+Q_CLUSTER = {
+    "name": "DjangORM",
+    "workers": 4,
+    "timeout": 90,
+    "retry": 120,
+    "queue_limit": 50,
+    "bulk": 10,
+    "orm": "default",
+}

exampleproject/urls.py

@@ -0,0 +1,27 @@
+"""
+URL configuration for exampleproject project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/5.1/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+
+from django.contrib import admin
+from django.urls import path
+
+from exampleproject import views
+
+urlpatterns = [
+    path("admin/", admin.site.urls),
+    path("new_task/", views.add_task, name="add_task"),
+    path("result/<slug:task_id>/", views.get_result, name="get_result"),
+]

exampleproject/views.py

@@ -0,0 +1,31 @@
+import time
+
+from django.http import HttpResponse
+from django.urls import reverse
+
+from django_q import tasks
+
+
+# internal function to be called with django_q
+def new_task(run_for_minutes):
+    print("Task started")
+    time.sleep(run_for_minutes)
+    print("Task done")
+    return True
+
+
+def add_task(request):
+    task_id = tasks.async_task(new_task, 5)
+    result_url = reverse("get_result", args=[task_id])
+    return HttpResponse(
+        f"Added async task with <a href='{result_url}'>Go to results</a>"
+    )
+
+
+def get_result(request, task_id):
+    task = tasks.fetch(task_id)
+    if not task:
+        msg = "Task running... please refresh after some time"
+    else:
+        msg = f"Async task result: {task.result}"
+    return HttpResponse(msg)

exampleproject/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for exampleproject project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "exampleproject.settings")
+
+application = get_wsgi_application()

pyproject.toml

@@ -1,6 +1,10 @@
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
+
 [tool.poetry]
 name = "django-q2"
-version = "1.4.7"
+version = "1.7.3"
 packages = [
     { include = "django_q" },
 ]
@@ -30,6 +34,7 @@ classifiers = [
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12',
     'Topic :: Internet :: WWW/HTTP',
     'Topic :: System :: Distributed Computing',
     'Topic :: Software Development :: Libraries :: Python Modules',
@@ -44,8 +49,8 @@ include = ['CHANGELOG.md']
 
 
 [tool.poetry.dependencies]
-python = ">=3.8.14, <4"
-django = ">=3.2"
+python = ">=3.8,<4"
+django = ">=4.2, <6"
 django-picklefield = "^3.1"
 
 blessed = { version = "^1.19.1", optional = true }
@@ -55,10 +60,12 @@ django-redis = { version = "^5.2.0", optional = true }
 iron-mq = { version = "^0.9", optional = true }
 boto3 = { version = "^1.24.92", optional = true }
 pymongo = { version = "^4.2.0", optional = true }
-croniter = { version = "^1.3.7", optional = true }
+croniter = { version = "^2.0.1", optional = true }
 django-q-rollbar = {version = ">=0.1", optional = true}
 django-q-sentry = {version = ">=0.1", optional = true}
 redis = {version = "^4.3.4", optional = true}
+setproctitle = {version = "^1.3.2", optional = true}
+importlib-metadata = {version = ">=3.6", python = "<3.10"}
 
 
 [tool.poetry.dev-dependencies]
@@ -66,13 +73,12 @@ pytest = "^7.1.3"
 pytest-django = "^4.5.2"
 Sphinx = "^4.0.2"
 pytest-cov = "^4.0.0"
-black = "^22.10.0"
-isort = {extras = ["requirements_deprecated_finder"], version = "^5.10.1"}
+ruff = "^0.4.4"
 
 [tool.poetry.extras]
-requires = ["poetry_core>=1.0.0"]
+requires = ["poetry_core"]
 build-backend = ["poetry.core.masonry.api"]
-testing = ["django-redis", "croniter", "hiredis", "psutil", "iron-mq", "boto3", "pymongo", "blessed", "redis"]
+testing = ["django-redis", "croniter", "hiredis", "psutil", "iron-mq", "boto3", "pymongo", "blessed", "redis", "setproctitle"]
 rollbar = ["django-q-rollbar"]
 sentry = ["django-q-sentry"]
 

requirements.txt

@@ -1,40 +1,41 @@
-ansicon==1.89.0; platform_system == "Windows" and python_version >= "2.7" \
-    --hash=sha256:f1def52d17f65c2c9682cf8370c03f541f410c1752d6a14029f97318e4b9dfec \
-    --hash=sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1
-arrow==1.1.1; python_version >= "3.6" \
-    --hash=sha256:77a60a4db5766d900a2085ce9074c5c7b8e2c99afeaa98ad627637ff6f292510 \
-    --hash=sha256:dee7602f6c60e3ec510095b5e301441bc56288cb8f51def14dcb3079f623823a
-asgiref==3.3.4; python_version >= "3.6" \
-    --hash=sha256:92906c611ce6c967347bbfea733f13d6313901d54dcca88195eaeb52b2a8e8ee \
-    --hash=sha256:d1216dfbdfb63826470995d31caed36225dcaf34f182e0fa257a4dd9e86f1b78
-blessed==1.18.1; python_version >= "2.7" \
-    --hash=sha256:dd7c0d33db9a2e7f597b446996484d0ed46e1586239db064fb5025008937dcae \
-    --hash=sha256:8b09936def6bc06583db99b65636b980075733e13550cb6af262ce724a55da23
-django-picklefield==3.0.1; python_version >= "3" \
-    --hash=sha256:15ccba592ca953b9edf9532e64640329cd47b136b7f8f10f2939caa5f9ce4287 \
-    --hash=sha256:3c702a54fde2d322fe5b2f39b8f78d9f655b8f77944ab26f703be6c0ed335a35
-django==3.2.4; python_version >= "3.6" \
-    --hash=sha256:ea735cbbbb3b2fba6d4da4784a0043d84c67c92f1fdf15ad6db69900e792c10f \
-    --hash=sha256:66c9d8db8cc6fe938a28b7887c1596e42d522e27618562517cc8929eb7e7f296
-jinxed==1.1.0; platform_system == "Windows" and python_version >= "2.7" \
-    --hash=sha256:6a61ccf963c16aa885304f27e6e5693783676897cea0c7f223270c8b8e78baf8 \
-    --hash=sha256:d8f1731f134e9e6b04d95095845ae6c10eb15cb223a5f0cabdea87d4a279c305
-python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
-    --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
-    --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
-pytz==2021.1; python_version >= "3.6" \
-    --hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 \
-    --hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da
-six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
-    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
-    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926
-sqlparse==0.4.1; python_version >= "3.6" \
-    --hash=sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0 \
-    --hash=sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8
-typing-extensions==3.10.0.0; python_version < "3.8" and python_version >= "3.6" \
-    --hash=sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497 \
-    --hash=sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84 \
-    --hash=sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342
-wcwidth==0.2.5; python_version >= "2.7" \
-    --hash=sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784 \
-    --hash=sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83
+asgiref==3.8.1 ; python_version >= "3.8" and python_version < "4" \
+    --hash=sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47 \
+    --hash=sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590
+backports-zoneinfo==0.2.1 ; python_version >= "3.8" and python_version < "3.9" \
+    --hash=sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf \
+    --hash=sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328 \
+    --hash=sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546 \
+    --hash=sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6 \
+    --hash=sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570 \
+    --hash=sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9 \
+    --hash=sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7 \
+    --hash=sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987 \
+    --hash=sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722 \
+    --hash=sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582 \
+    --hash=sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc \
+    --hash=sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b \
+    --hash=sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1 \
+    --hash=sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08 \
+    --hash=sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac \
+    --hash=sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2
+django-picklefield==3.2 ; python_version >= "3.8" and python_version < "4" \
+    --hash=sha256:aa463f5d79d497dbe789f14b45180f00a51d0d670067d0729f352a3941cdfa4d \
+    --hash=sha256:e9a73539d110f69825d9320db18bcb82e5189ff48dbed41821c026a20497764c
+django==4.2.11 ; python_version >= "3.8" and python_version < "4" \
+    --hash=sha256:6e6ff3db2d8dd0c986b4eec8554c8e4f919b5c1ff62a5b4390c17aff2ed6e5c4 \
+    --hash=sha256:ddc24a0a8280a0430baa37aff11f28574720af05888c62b7cfe71d219f4599d3
+importlib-metadata==7.1.0 ; python_version >= "3.8" and python_version < "3.10" \
+    --hash=sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570 \
+    --hash=sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2
+sqlparse==0.5.0 ; python_version >= "3.8" and python_version < "4" \
+    --hash=sha256:714d0a4932c059d16189f58ef5411ec2287a4360f17cdd0edd2d09d4c5087c93 \
+    --hash=sha256:c204494cd97479d0e39f28c93d46c0b2d5959c7b9ab904762ea6c7af211c8663
+typing-extensions==4.11.0 ; python_version >= "3.8" and python_version < "3.11" \
+    --hash=sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0 \
+    --hash=sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a
+tzdata==2024.1 ; python_version >= "3.8" and python_version < "4" and sys_platform == "win32" \
+    --hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \
+    --hash=sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252
+zipp==3.18.1 ; python_version >= "3.8" and python_version < "3.10" \
+    --hash=sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b \
+    --hash=sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715

ruff.toml

@@ -0,0 +1 @@
+lint.extend-select = ["I"]

test-services-docker-compose.yaml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   redis:
     image: redis:latest

web-docker-compose.yaml

@@ -1,11 +1,16 @@
-version: '3'
-
 services:
   web:
     restart: always
-    command: python manage.py runserver 0.0.0.0:8000
+    command: bash -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000"
     ports:
       - "127.0.0.1:8000:8000"
     build: .
     volumes:
       - .:/app
+
+  django-q:
+    restart: always
+    command: bash -c "python manage.py migrate && python manage.py qcluster"
+    build: .
+    volumes:
+      - .:/app