Release v1.6.1

* Correct health check import in examples.rst

* Update monitor.rst

---------

Co-authored-by: Stan Triepels <1939656+GDay@users.noreply.github.com>

.github/workflows/test.yml

@@ -11,8 +11,25 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
-        django: [ "3.2", "4.1" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12.0-rc.3" ]
+        django: [ "3.2", "4.1", "4.2", "5.0a1"]
+        exclude:
+          # django 5.0 does not support 3.8 and 3.9
+          - python-version: "3.8"
+            django: "5.0a1" 
+          - python-version: "3.9"
+            django: "5.0a1" 
+          # django 4.2 does not support 3.12
+          - python-version: "3.12.0-rc.3"
+            django: "4.2" 
+          # django 4.1 does not support 3.12
+          - python-version: "3.12.0-rc.3"
+            django: "4.1" 
+          # django 3.2 does not support 3.11 and 3.12
+          - python-version: "3.11"
+            django: "3.2" 
+          - python-version: "3.12.0-rc.3"
+            django: "3.2" 
 
     services:
       disque:
@@ -39,16 +56,16 @@ jobs:
           - 6379:6379
         options: --entrypoint redis-server
     steps:
-        - uses: actions/checkout@v2
+        - uses: actions/checkout@v3
         - name: Set up Python ${{ matrix.python-version }}
-          uses: actions/setup-python@v2
+          uses: actions/setup-python@v4
           with:
             python-version: ${{ matrix.python-version }}
         - name: Install dependencies with Django ${{ matrix.django }}
           run: |
             python -m pip install --upgrade pip
             pip install poetry
-            poetry add "django==${{ matrix.django }}"
+            poetry add "django==${{ matrix.django }}" --python=${{ matrix.python-version }}
             poetry install -E testing
         - name: Run Tests
           run: |

CHANGELOG.md

@@ -1,6 +1,51 @@
 # Changelog
 
-## [Unreleased](https://github.com/GDay/django-q2/tree/HEAD)
+## [v1.6.1](https://github.com/django-q2/django-q2/tree/v1.6.1) (2023-10-13)
+
+**Merged pull requests:**
+
+- Fix strict versions for python/django https://github.com/django-q2/django-q2/pull/130
+
+## [v1.6.0](https://github.com/django-q2/django-q2/tree/v1.6.0) (2023-10-12)
+
+**Merged pull requests:**
+
+- Add support for Django 5 and python 12 https://github.com/django-q2/django-q2/pull/120
+- Fix for "apps not ready" in Windows and Mac https://github.com/django-q2/django-q2/pull/116
+- Update broken MongoClient link in Docs https://github.com/django-q2/django-q2/pull/127
+- Fix German Translation Typo https://github.com/django-q2/django-q2/pull/124
+- Update Add-ons install command in install.rst https://github.com/django-q2/django-q2/pull/115
+- DOCS: Correct health check import in examples.rst https://github.com/django-q2/django-q2/pull/110
+
+## [v1.5.5](https://github.com/django-q2/django-q2/tree/v1.5.5) (2023-09-01)
+
+**Merged pull requests:**
+
+- Add documentation to migrate from django-q to django-q2 https://github.com/django-q2/django-q2/pull/108
+- Fix not picking up result from falsy result https://github.com/django-q2/django-q2/pull/107
+- Remove deprecated usage pkg_resources https://github.com/django-q2/django-q2/pull/103
+- Move worker, scheduler, pusher and monitor to separate files https://github.com/django-q2/django-q2/pull/100
+
+## [v1.5.4](https://github.com/GDay/django-q2/tree/v1.5.4) (2023-06-29)
+
+**Merged pull requests:**
+
+- Rerun successful tasks https://github.com/django-q2/django-q2/pull/99
+
+## [v1.5.3](https://github.com/GDay/django-q2/tree/v1.5.3) (2023-05-14)
+
+**Merged pull requests:**
+
+- Add post_spawn signal. https://github.com/django-q2/django-q2/pull/93
+- Post spawn docs https://github.com/django-q2/django-q2/pull/95
+- Make processes identifiable with uuid4 https://github.com/django-q2/django-q2/pull/91
+
+## [v1.5.2](https://github.com/GDay/django-q2/tree/v1.5.2) (2023-04-13)
+
+**Merged pull requests:**
+
+- Added Django 4.2 to the test matrix, fixed deprecation warning https://github.com/GDay/django-q2/pull/89
+- Updated docs to show support for 4.2
 
 ## [v1.5.1](https://github.com/GDay/django-q2/tree/v1.5.1) (2023-04-02)
 

README.rst

@@ -27,7 +27,7 @@ Changes compared to the original Django-Q:
 - Dropped support for Disque (hasn't been updated in a long time)
 - Dropped Redis, Arrow and Blessed dependencies
 - Updated all current dependencies
-- Added tests for Django 4.x
+- Added tests for Django 4.x and 5.x
 - Added Turkish language
 - Improved admin area
 - Fixed a lot of issues
@@ -40,7 +40,7 @@ Requirements
 -  `Django <https://www.djangoproject.com>`__ > = 3.2
 -  `Django-picklefield <https://github.com/gintas/django-picklefield>`__
 
-Tested with: Python 3.8, 3.9, 3.10, 3.11 Django 3.2.X and 4.1.X
+Tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 3.2.X, 4.1.X, 4.2.X and 5.0.X
 
 Brokers
 ~~~~~~~
@@ -104,6 +104,11 @@ For full configuration options, see the `configuration documentation <https://dj
 Management Commands
 ~~~~~~~~~~~~~~~~~~~
 
+::
+
+    For the management commands to work, you will need to install Blessed: <https://github.com/jquast/blessed>
+
+
 Start a cluster with::
 
     $ python manage.py qcluster

django_q/__init__.py

@@ -1,6 +1,6 @@
 import django
 
-VERSION = (1, 5, 1)
+VERSION = (1, 6, 1)
 
 if django.VERSION < (3, 2):
     default_app_config = "django_q.apps.DjangoQConfig"

django_q/admin.py

@@ -10,10 +10,37 @@ from django_q.models import Failure, OrmQ, Schedule, Success, Task
 from django_q.tasks import async_task
 
 
+def resubmit_task(model_admin, request, queryset):
+    """Submit selected tasks back to the queue."""
+    for task in queryset:
+        async_task(
+            task.func,
+            *task.args or (),
+            hook=task.hook,
+            group=task.group,
+            cluster=task.cluster,
+            **task.kwargs or {},
+        )
+        if isinstance(model_admin, FailAdmin):
+            task.delete()
+
+
+resubmit_task.short_description = _("Resubmit selected tasks to queue")
+
+
 class TaskAdmin(admin.ModelAdmin):
     """model admin for success tasks."""
 
-    list_display = ("name", "group", "func", "cluster", "started", "stopped", "time_taken")
+    list_display = (
+        "name",
+        "group",
+        "func",
+        "cluster",
+        "started",
+        "stopped",
+        "time_taken",
+    )
+    actions = [resubmit_task]
 
     def has_add_permission(self, request):
         """Don't allow adds."""
@@ -33,27 +60,24 @@ class TaskAdmin(admin.ModelAdmin):
         return list(self.readonly_fields) + [field.name for field in obj._meta.fields]
 
 
-def retry_failed(FailAdmin, request, queryset):
-    """Submit selected tasks back to the queue."""
-    for task in queryset:
-        async_task(task.func, *task.args or (), hook=task.hook,
-                   group=task.group, cluster=task.cluster, **task.kwargs or {})
-        task.delete()
-
-
-retry_failed.short_description = _("Resubmit selected tasks to queue")
-
-
 class FailAdmin(admin.ModelAdmin):
     """model admin for failed tasks."""
 
-    list_display = ("name", "group", "func", "cluster", "started", "stopped", "short_result")
+    list_display = (
+        "name",
+        "group",
+        "func",
+        "cluster",
+        "started",
+        "stopped",
+        "short_result",
+    )
 
     def has_add_permission(self, request):
         """Don't allow adds."""
         return False
 
-    actions = [retry_failed]
+    actions = [resubmit_task]
     search_fields = ("name", "func", "group")
     list_filter = ("group", "cluster")
     readonly_fields = []
@@ -124,7 +148,17 @@ class QueueAdmin(admin.ModelAdmin):
     """queue admin for ORM broker"""
 
     list_display = ("id", "key", "name", "group", "func", "lock", "task_id")
-    fields = ("key", "lock", "task_id", "name", "group", "func", "args", "kwargs", "q_options")
+    fields = (
+        "key",
+        "lock",
+        "task_id",
+        "name",
+        "group",
+        "func",
+        "args",
+        "kwargs",
+        "q_options",
+    )
     readonly_fields = fields[2:]
 
     def save_model(self, request, obj, form, change):

django_q/brokers/orm.py

@@ -37,7 +37,9 @@ class ORM(Broker):
 
     def lock_size(self) -> int:
         return (
-            self.get_connection().filter(key=self.list_key, lock__gt=timezone.now()).count()
+            self.get_connection()
+            .filter(key=self.list_key, lock__gt=timezone.now())
+            .count()
         )
 
     def purge_queue(self):
@@ -62,7 +64,9 @@ class ORM(Broker):
         return package.pk
 
     def dequeue(self):
-        tasks = self.get_connection().filter(key=self.list_key, lock__lt=timezone.now())[
+        tasks = self.get_connection().filter(
+            key=self.list_key, lock__lt=timezone.now()
+        )[
             0 : Conf.BULK  # noqa: E203
         ]
         if tasks:

django_q/cluster.py

@@ -1,12 +1,8 @@
 # Standard
-from django_q.scheduler import Scheduler
-from django_q.puller import Puller
-from django_q.worker import Pool
 import signal
-from django_q.monitor import Monitor
 import socket
 import uuid
-from multiprocessing import Event, Process, current_process
+from multiprocessing import Event, Process, Value, current_process
 from time import sleep
 
 # Django
@@ -24,17 +20,16 @@ from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
 
 # Local
-import django_q.tasks
 from django_q.brokers import Broker, get_broker
-from django_q.conf import (
-    Conf,
-    get_ppid,
-    logger,
-    psutil,
-    setproctitle,
-)
+from django_q.conf import Conf, get_ppid, logger, psutil, setproctitle
 from django_q.humanhash import humanize
+from django_q.monitor import monitor
+from django_q.pusher import pusher
+from django_q.queues import Queue
+from django_q.scheduler import scheduler
 from django_q.status import Stat, Status
+from django_q.worker import worker
+
 
 class Cluster:
     def __init__(self, broker: Broker = None):
@@ -59,6 +54,7 @@ class Cluster:
         self.start_event = Event()
         self.sentinel = Process(
             target=Sentinel,
+            name=f"Process-{uuid.uuid4().hex}",
             args=(
                 self.stop_event,
                 self.start_event,
@@ -148,17 +144,22 @@ class Sentinel:
         self.tob = timezone.now()
         self.stop_event = stop_event
         self.start_event = start_event
+        self.pool_size = Conf.WORKERS
+        self.pool = []
         self.timeout = timeout or Conf.TIMEOUT
-        self.event_out = Event()
-        logger.info(
-            _("%(name)s main at %(id)s") % {"name": self.name, "id": current_process().pid}
+        self.task_queue = (
+            Queue(maxsize=Conf.QUEUE_LIMIT) if Conf.QUEUE_LIMIT else Queue()
         )
+        self.result_queue = Queue()
+        self.event_out = Event()
+        self.monitor = None
+        self.pusher = None
         if start:
             self.start()
 
     def queue_name(self):
         # multi-queue: cluster name is (broker's) queue_name
-        return self.broker.list_key if self.broker else '--'
+        return self.broker.list_key if self.broker else "--"
 
     def start(self):
         self.broker.ping()
@@ -169,140 +170,199 @@ class Sentinel:
         if not self.start_event.is_set() and not self.stop_event.is_set():
             return Conf.STARTING
         elif self.start_event.is_set() and not self.stop_event.is_set():
-            if self.monitor.is_idle and self.pool.is_done:
+            if self.result_queue.empty() and self.task_queue.empty():
                 return Conf.IDLE
             return Conf.WORKING
         elif self.stop_event.is_set() and self.start_event.is_set():
-            if self.monitor.is_alive or self.puller.is_alive or len(self.pool.workers) > 0:
+            if self.monitor.is_alive() or self.pusher.is_alive() or len(self.pool) > 0:
                 return Conf.STOPPING
             return Conf.STOPPED
 
+    def spawn_process(self, target, *args) -> Process:
+        """
+        :type target: function or class
+        """
+        p = Process(target=target, args=args, name=f"Process-{uuid.uuid4().hex}")
+        p.daemon = True
+        if target == worker:
+            p.daemon = Conf.DAEMONIZE_WORKERS
+            p.timer = args[2]
+            self.pool.append(p)
+        p.start()
+        return p
+
+    def spawn_pusher(self) -> Process:
+        return self.spawn_process(pusher, self.task_queue, self.event_out, self.broker)
+
+    def spawn_worker(self):
+        self.spawn_process(
+            worker, self.task_queue, self.result_queue, Value("f", -1), self.timeout
+        )
+
+    def spawn_monitor(self) -> Process:
+        return self.spawn_process(monitor, self.result_queue, self.broker)
+
+    def reincarnate(self, process):
+        """
+        :param process: the process to reincarnate
+        :type process: Process or None
+        """
+        # close connections before spawning new process
+        if not Conf.SYNC:
+            db.connections.close_all()
+        if process == self.monitor:
+            self.monitor = self.spawn_monitor()
+            logger.critical(
+                _("reincarnated monitor %(name)s after sudden death")
+                % {"name": process.name}
+            )
+        elif process == self.pusher:
+            self.pusher = self.spawn_pusher()
+            logger.critical(
+                _("reincarnated pusher %(name)s after sudden death")
+                % {"name": process.name}
+            )
+        else:
+            self.pool.remove(process)
+            self.spawn_worker()
+            if process.timer.value == 0:
+                # only need to terminate on timeout, otherwise we risk destabilizing
+                # the queues
+                task_name = ""
+                if psutil:
+                    try:
+                        process_name = psutil.Process(process.pid).name()
+                        name_splits = process_name.split(" ")
+                        task_name = (
+                            name_splits[3]
+                            if len(name_splits) >= 4 and name_splits[2] == "processing"
+                            else ""
+                        )
+                    except psutil.NoSuchProcess:
+                        pass
+                process.terminate()
+                if task_name:
+                    msg = _(
+                        "reincarnated worker %(name)s after timeout while processing task %(task_name)s"
+                    ) % {"name": process.name, "task_name": task_name}
+                else:
+                    msg = _("reincarnated worker %(name)s after timeout") % {
+                        "name": process.name
+                    }
+                logger.critical(msg)
+            elif int(process.timer.value) == -2:
+                logger.info(_("recycled worker %(name)s") % {"name": process.name})
+            else:
+                logger.critical(
+                    _("reincarnated worker %(name)s after death")
+                    % {"name": process.name}
+                )
+
+        self.reincarnations += 1
+
     def spawn_cluster(self):
+        self.pool = []
+        Stat(self).save()
         # close connections before spawning new process
         if not Conf.SYNC:
             db.connections.close_all()
         # spawn worker pool
-        self.pool = Pool()
-        self.puller = Puller()
-        self.monitor = Monitor()
-        self.scheduler = Scheduler()
+        for __ in range(self.pool_size):
+            self.spawn_worker()
+        # spawn auxiliary
+        self.monitor = self.spawn_monitor()
+        self.pusher = self.spawn_pusher()
         # set worker cpu affinity if needed
         if psutil and Conf.CPU_AFFINITY:
-            set_cpu_affinity(Conf.CPU_AFFINITY, [w.process.pid for w in self.pool.workers])
-        Stat(self).save()
-
+            set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
 
     def guard(self):
         logger.info(
             _("%(name)s guarding cluster %(cluster_name)s")
             % {
                 "name": current_process().name,
-                "cluster_name": humanize(self.cluster_id.hex) + f" [{self.queue_name()}]",
+                "cluster_name": humanize(self.cluster_id.hex)
+                + f" [{self.queue_name()}]",
             }
         )
         self.start_event.set()
         Stat(self).save()
         logger.info(
             _("Q Cluster %(cluster_name)s running.")
-            % {"cluster_name": humanize(self.cluster_id.hex) + f" [{self.queue_name()}]"}
+            % {
+                "cluster_name": humanize(self.cluster_id.hex)
+                + f" [{self.queue_name()}]"
+            }
         )
         counter = 0
+        cycle = Conf.GUARD_CYCLE  # guard loop sleep in seconds
         # Guard loop. Runs at least once
         while not self.stop_event.is_set() or not counter:
-            # Check if the pool of workers is healthy
-            logger.info("Check if pool is healthy")
-            if not self.pool.is_healthy:
-                # reincarnate workers that died
-                print("reincarnate workers")
-                self.pool.reincarnate_stopped_workers()
-
-            print("Check if puller is healthy")
-            if not self.puller.is_alive:
-                self.puller.reincarnate_process()
-
-            print("Check if monitor is healthy")
-            if not self.monitor.is_alive:
-                self.monitor.reincarnate_process()
-
-            print("Check if scheduler is healthy")
-            if not self.scheduler.is_alive:
-                self.scheduler.reincarnate_process()
-
-
-            print("add tasks and mark workers idle")
-            for worker in self.pool.get_done_workers():
-                # put result in task_queue to be picked up by monitor for processing
-                self.monitor.add_task(worker.get_result())
-                # mark task back to idle or reincarnate to be picked up for a new task
-                if worker.is_recycle:
-                    worker.reincarnate_process()
-                else:
-                    worker.mark_idle()
-
-            # check if monitor has items to process
-            print("run monitor item")
-            self.monitor.run_item()
-
-            print("Add task to worker pool")
-            if self.puller.has_results:
-                self.pool.add_task(self.puller.get_result())
-
-            # delegate tasks to workers that are now available
-            print("delegate tasks")
-            self.pool.delegate_tasks()
-
-            logger.info("sleep")
-            counter += 1
-            sleep(Conf.GUARD_CYCLE)
+            # Check Workers
+            for p in self.pool:
+                with p.timer.get_lock():
+                    # Are you alive?
+                    if not p.is_alive() or p.timer.value == 0:
+                        self.reincarnate(p)
+                        continue
+                    # Decrement timer if work is being done
+                    if p.timer.value > 0:
+                        p.timer.value -= cycle
+            # Check Monitor
+            if not self.monitor.is_alive():
+                self.reincarnate(self.monitor)
+            # Check Pusher
+            if not self.pusher.is_alive():
+                self.reincarnate(self.pusher)
+            # Call scheduler once a minute (or so)
+            counter += cycle
+            if counter >= 30 and Conf.SCHEDULER:
+                counter = 0
+                scheduler(broker=self.broker)
+            # Save current status
             Stat(self).save()
+            sleep(cycle)
         self.stop()
 
     def stop(self):
+        Stat(self).save()
         name = current_process().name
         logger.info(_("%(name)s stopping cluster processes") % {"name": name})
-        # Stopping guard
-        self.stop_event.set()
-        logger.debug(_("Guard has stopped"))
-        # Stop scheduler
-        self.scheduler.stop_scheduler()
-
-        # End all workers gracefully
-        for __ in range(Conf.WORKERS):
-            self.pool.add_task("STOP")
-
-        # make sure the tasks queue in the pool is empty and workers are idle max timeout 20 sec
-        time_passed = 0
-        while not self.pool.is_done and time_passed <= 20:
-            self.monitor.run_item()
-            self.pool.delegate_tasks()
-            time_passed += 0.5
-            sleep(0.5)
-        if time_passed >= 20:
-            logger.error(_("Couldn't terminate tasks within 20 seconds, killing processes now"))
-            for worker in self.pool.workers:
-                worker.process.kill()
-
-        logger.debug(_("All tasks were processed and workers where stopped"))
-
-        self.monitor.add_task("STOP")
-        while not self.monitor.is_done:
-            # in the case the monitor was behind, let's run through all
-            self.monitor.run_item()
-            sleep(0.5)
-
-        logger.debug(_("All tasks were saved"))
-
-        self.puller.stop_puller()
-
-        # make sure all processes are terminated
-        for worker in self.pool.workers:
-            worker.process.terminate()
-
-        self.monitor.process.terminate()
-        self.puller.process.terminate()
-        self.scheduler.process.terminate()
-
-        logger.debug(_("All processes were terminated"))
+        # Stopping pusher
+        self.event_out.set()
+        # Wait for it to stop
+        while self.pusher.is_alive():
+            sleep(0.1)
+            Stat(self).save()
+        # Put poison pills in the queue
+        for __ in range(len(self.pool)):
+            self.task_queue.put("STOP")
+        self.task_queue.close()
+        # wait for the task queue to empty
+        self.task_queue.join_thread()
+        # Wait for all the workers to exit
+        while len(self.pool):
+            for p in self.pool:
+                if not p.is_alive():
+                    self.pool.remove(p)
+            sleep(0.1)
+            Stat(self).save()
+        # Finally stop the monitor
+        self.result_queue.put("STOP")
+        self.result_queue.close()
+        # Wait for the result queue to empty
+        self.result_queue.join_thread()
+        logger.info(_("%(name)s waiting for the monitor.") % {"name": name})
+        # Wait for everything to close or time out
+        count = 0
+        if not self.timeout:
+            self.timeout = 30
+        while self.status() == Conf.STOPPING and count < self.timeout * 10:
+            sleep(0.1)
+            Stat(self).save()
+            count += 1
+        # Final status
+        Stat(self).save()
 
 
 def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING):

django_q/conf.py

@@ -1,15 +1,21 @@
 import logging
 import os
+import sys
 from copy import deepcopy
 from multiprocessing import cpu_count
 from signal import signal
 from warnings import warn
 
-import pkg_resources
 from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 
-from queue import Queue
+from django_q.queues import Queue
+
+# The "selectable" entry points were introduced in importlib_metadata 3.6 and Python 3.10.
+if sys.version_info < (3, 10):
+    from importlib_metadata import entry_points
+else:
+    from importlib.metadata import entry_points
 
 # optional
 try:
@@ -44,15 +50,18 @@ class Conf:
         conf = {}
 
     _Q_CLUSTER_NAME = os.getenv("Q_CLUSTER_NAME")
-    if _Q_CLUSTER_NAME and _Q_CLUSTER_NAME != conf.get("name") and \
-            _Q_CLUSTER_NAME != conf.get("cluster_name"):
+    if (
+        _Q_CLUSTER_NAME
+        and _Q_CLUSTER_NAME != conf.get("name")
+        and _Q_CLUSTER_NAME != conf.get("cluster_name")
+    ):
         conf["cluster_name"] = _Q_CLUSTER_NAME
         alt_conf = conf.pop("ALT_CLUSTERS")
         if isinstance(alt_conf, dict):
             alt_conf = alt_conf.get(_Q_CLUSTER_NAME)
             if isinstance(alt_conf, dict):
-                alt_conf.pop('name', None)
-                alt_conf.pop('cluster_name', None)
+                alt_conf.pop("name", None)
+                alt_conf.pop("cluster_name", None)
                 conf.update(alt_conf)
 
     # Redis server configuration . Follows standard redis keywords
@@ -91,7 +100,7 @@ class Conf:
     CLUSTER_NAME = conf.get("cluster_name", PREFIX)
 
     # Log output level
-    LOG_LEVEL = conf.get("log_level", "DEBUG")
+    LOG_LEVEL = conf.get("log_level", "INFO")
 
     # Maximum number of successful tasks kept in the database. 0 saves everything.
     # -1 saves none
@@ -112,7 +121,7 @@ class Conf:
         )
 
     # Guard loop sleep in seconds. Should be between 0 and 60 seconds.
-    GUARD_CYCLE = conf.get("guard_cycle", 1)
+    GUARD_CYCLE = conf.get("guard_cycle", 0.5)
 
     # Disable the scheduler
     SCHEDULER = conf.get("scheduler", True)
@@ -240,7 +249,6 @@ class Conf:
 # logger
 logger = logging.getLogger("django-q")
 
-
 # Set up standard logging handler in case there is none
 if not logger.hasHandlers():
     logger.setLevel(level=getattr(logging, Conf.LOG_LEVEL))
@@ -274,9 +282,7 @@ if Conf.ERROR_REPORTER:
         # iterate through the configured error reporters,
         # and instantiate an ErrorReporter using the provided config
         for name, conf in error_conf.items():
-            for entry in pkg_resources.iter_entry_points(
-                "djangoq.errorreporters", name
-            ):
+            for entry in entry_points(group="djangoq.errorreporters", name=name):
                 Reporter = entry.load()
                 reporters.append(Reporter(**conf))
         error_reporter = ErrorReporter(reporters)

django_q/core_signing.py

@@ -2,16 +2,18 @@ import datetime
 import time
 import zlib
 
+import django
 from django.core.signing import BadSignature, JSONSerializer, SignatureExpired
 from django.core.signing import Signer as Sgnr
 from django.core.signing import TimestampSigner as TsS
 from django.core.signing import b64_decode, dumps
 
-try:
-    from django.core.signing import base62
-except ImportError:
-    # For django < 4.0
+if django.VERSION < (5, 0):
     from django.utils.baseconv import base62
+    b62_decode = base62.decode
+else:
+    from django.core.signing import b62_decode
+
 from django.utils.crypto import constant_time_compare
 from django.utils.encoding import force_bytes, force_str
 
@@ -37,7 +39,9 @@ def loads(
     """
     # TimestampSigner.unsign() returns str but base64 and zlib compression
     # operate on bytes.
-    base64d = force_bytes(TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
+    base64d = force_bytes(
+        TimestampSigner(key=key, salt=salt).unsign(s, max_age=max_age)
+    )
     decompress = False
     if base64d[:1] == b".":
         # It's compressed; uncompress it first
@@ -74,7 +78,7 @@ class TimestampSigner(Signer, TsS):
         """
         result = super(TimestampSigner, self).unsign(value)
         value, timestamp = result.rsplit(self.sep, 1)
-        timestamp = base62.decode(timestamp)
+        timestamp = b62_decode(timestamp)
         if max_age is not None:
             if isinstance(max_age, datetime.timedelta):
                 max_age = max_age.total_seconds()

django_q/exceptions.py

@@ -1,26 +0,0 @@
-import signal
-from typing import Optional
-
-class TimeoutException(SystemExit):
-    """Exception for when a worker takes too long to complete a task"""
-    pass
-
-
-class TimeoutHandler:
-    def __init__(self, timeout: Optional[int] = None):
-        self._timeout = timeout
-
-    def raise_timeout_exception(self, signum, frame):
-        raise TimeoutException('Task exceeded maximum timeout value '
-                              '({0} seconds)'.format(self._timeout))
-
-    def __enter__(self):
-        if self._timeout is None:
-            return
-        signal.signal(signal.SIGALRM, self.raise_timeout_exception)
-        signal.alarm(self._timeout)
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        """When getting out of the timeout, reset the alarm, so it won't trigger"""
-        signal.alarm(0)
-        signal.signal(signal.SIGALRM, signal.SIG_DFL)

django_q/helpers.py

@@ -1,43 +0,0 @@
-from copy import Error
-from django_q.worker import WorkerProcess
-from django_q.monitor import Monitor
-from django_q.models import Task
-from typing import Optional, Sequence, Tuple
-from django_q.conf import logger
-from django_q.queue_task import QueueTask
-from django_q.puller import Puller
-from django_q.scheduler import Scheduler
-
-
-def run_scheduler_once(broker=None) -> None:
-    Scheduler.schedule_tasks(broker)
-
-def get_scheduled_tasks(broker=None) -> Sequence[QueueTask]:
-    try:
-        return Puller.get_tasks_from_broker(broker=broker)
-    except ValueError:
-        logger.exception("Couldn't get items from broker")
-        return []
-
-def run_task(task=None) -> QueueTask:
-    if task is None:
-
-        scheduled_tasks = get_scheduled_tasks()
-        if not len(scheduled_tasks):
-            raise ValueError("No tasks scheduled and no task given to run for worker")
-        task = scheduled_tasks[0]
-
-    return WorkerProcess.run_task(task)
-
-def save_task(task, broker=None) -> Tuple[QueueTask, Optional[Task]]:
-    return Monitor.save_task(task, broker)
-
-def run_cluster_once(workers, tasks=[], broker=None) -> None:
-    if not len(tasks):
-        run_scheduler_once(broker=broker)
-        tasks = get_scheduled_tasks(broker=broker)
-
-    for idx, worker in enumerate(range(workers)):
-        if len(tasks) >= idx + 1:
-            task = run_task(tasks[idx])
-            save_task(task, broker=broker)

django_q/locale/de/LC_MESSAGES/django.po

@@ -318,7 +318,7 @@ msgstr "Anzahl Minuten für den Typ 'Minuten'"
 
 #: models.py:205
 msgid "Repeats"
-msgstr "Wiederhohlungen"
+msgstr "Wiederholungen"
 
 #: models.py:205
 msgid "n = n times, -1 = forever"

django_q/management/commands/qcluster.py

@@ -1,8 +1,9 @@
+import os
+
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
 
 from django_q.cluster import Cluster
-import os
 
 
 class Command(BaseCommand):
@@ -23,7 +24,7 @@ class Command(BaseCommand):
             dest="cluster_name",
             default=None,
             help="Set alternative cluster name instead of the name in Q_CLUSTER settings (for multi-queue setup). "
-                 "On Linux you should set name through `Q_CLUSTER_NAME=cluster_name python manage.py qcluster` instead."
+            "On Linux you should set name through `Q_CLUSTER_NAME=cluster_name python manage.py qcluster` instead.",
         )
 
     def handle(self, *args, **options):

django_q/management/commands/qinfo.py

@@ -1,15 +1,9 @@
-from datetime import timedelta
-from django_q.brokers import get_broker
-from django_q.models import Failure, Schedule, Success
-from django_q.status import Stat
-from django.db.models import F, Sum
-from django.db import connection
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
-from django.utils import timezone
 
 from django_q import VERSION
 from django_q.conf import Conf
+from django_q.monitor_terminal import get_ids, info
 
 
 class Command(BaseCommand):
@@ -34,13 +28,7 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if options.get("ids", True):
-            stat = Stat.get_all()
-            if not stat:
-                print(_("No clusters appear to be running."))
-
-            for s in stat:
-                print(s.cluster_id)
-
+            get_ids()
         elif options.get("config", False):
             hide = [
                 "conf",
@@ -50,7 +38,6 @@ class Command(BaseCommand):
                 "WORKING",
                 "SIGNAL_NAMES",
                 "STOPPED",
-                "SECRET_KEY",
             ]
             settings = [
                 a for a in dir(Conf) if not a.startswith("__") and a not in hide
@@ -61,69 +48,4 @@ class Command(BaseCommand):
                 if value is not None:
                     self.stdout.write(f"{setting}: {value}")
         else:
-            broker = get_broker()
-
-            broker.ping()
-
-            stats = Stat.get_all(broker=broker)
-            clusters = len(stats)
-            workers = 0
-            reincarnations = 0
-            for cluster in stats:
-                workers += len(cluster.workers)
-                reincarnations += cluster.reincarnations
-
-            # calculate tasks pm and avg exec time
-            tasks_per = 0
-            per = _("day")
-            exec_time = 0
-            last_tasks = Success.objects.filter(
-                stopped__gte=timezone.now() - timedelta(hours=24)
-            )
-            tasks_per_day = last_tasks.count()
-            if tasks_per_day > 0:
-                # average execution time over the last 24 hours
-                if connection.vendor != "sqlite":
-                    exec_time = last_tasks.aggregate(
-                        time_taken=Sum(F("stopped") - F("started"))
-                    )
-                    exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
-                else:
-                    # can't sum timedeltas on sqlite
-                    for t in last_tasks:
-                        exec_time += t.time_taken()
-                    exec_time = exec_time / tasks_per_day
-                # tasks per second/minute/hour/day in the last 24 hours
-                if tasks_per_day > 24 * 60 * 60:
-                    tasks_per = tasks_per_day / (24 * 60 * 60)
-                    per = _("second")
-                elif tasks_per_day > 24 * 60:
-                    tasks_per = tasks_per_day / (24 * 60)
-                    per = _("minute")
-                elif tasks_per_day > 24:
-                    tasks_per = tasks_per_day / 24
-                    per = _("hour")
-                else:
-                    tasks_per = tasks_per_day
-
-            print(
-                _("-- %(prefix)s %(version)s on %(info)s --")
-                % {
-                    "prefix": Conf.PREFIX.capitalize(),
-                    "version": ".".join(str(v) for v in VERSION),
-                    "info": broker.info(),
-                }
-            )
-            print(_("Clusters: %(clusters)s") % {"clusters": clusters})
-            print(_("Workers: %(workers)s") % {"workers": workers})
-            print(_("Restarts: %(restarts)s") % {"restarts": reincarnations})
-
-            print("")
-            print(_("Queued: %(queue_size)s") % {"queue_size": str(broker.queue_size())})
-            print(_("Successes: %(success_count)s") % {"success_count": str(Success.objects.count())})
-            print(_("Failures: %(failure_count)s") % {"failure_count": str(Failure.objects.count())})
-
-            print("")
-            print(_("Schedules: %(schedules_count)s") % {"schedules_count": str(Schedule.objects.count())})
-            print(_("Tasks/%(per)s: %(amount)s") % {"per": per, "amount": f"{tasks_per:.2f}"})
-            print(_("Avg time: %(time)s") % {"time": f"{exec_time:.4f}"})
+            info()

django_q/management/commands/qmemory.py

@@ -1,19 +1,7 @@
-import curses
-from django_q.conf import Conf
-import signal
-
-import time
-from django_q.status import Stat
-from django_q.brokers import get_broker
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
-from django.utils import timezone
-import curses
 
-try:
-    import psutil
-except ImportError:
-    psutil = None
+from django_q.monitor_terminal import memory
 
 
 class Command(BaseCommand):
@@ -37,103 +25,7 @@ class Command(BaseCommand):
         )
 
     def handle(self, *args, **options):
-        memory_stats = MemoryTerminalStats(
+        memory(
             run_once=options.get("run_once", False),
             workers=options.get("workers", False),
         )
-        curses.wrapper(memory_stats.start)
-
-
-def get_process_mb(pid):
-    try:
-        process = psutil.Process(pid)
-        mb_used = round(process.memory_info().rss / 1024**2, 2)
-    except psutil.NoSuchProcess:
-        mb_used = "NO_PROCESS_FOUND"
-    return mb_used
-
-
-class MemoryTerminalStats:
-    stop_writing = False
-
-    def __init__(self, run_once=False, workers=False):
-        self.run_once = run_once
-        self.workers = workers
-
-    def start(self, stdscr):
-        self.show_stats()
-
-    def on_exit(self, signum, frame):
-        # exit clean
-        self.stop_writing = True
-
-    def show_stats(self):
-        signal.signal(signal.SIGTERM, self.on_exit)
-        signal.signal(signal.SIGINT, self.on_exit)
-        scr = curses.initscr()
-
-        if not broker:
-            broker = get_broker()
-
-        broker.ping()
-        if not psutil:
-            scr.addstr(0, 0, 'Cannot start "qmemory" command. Missing "psutil" library.')
-            scr.refresh()
-            return
-
-        MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
-        MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
-
-        stats = Stat.get_all(broker=broker)
-
-        if not stats:
-            scr.addstr(1, 0, "Cluster is not running")
-            scr.refresh()
-        while not self.stop_writing:
-            data = []
-            for stat in stats:
-                # memory available (%)
-                memory_available_percentage = round(
-                    psutil.virtual_memory().available
-                    * 100
-                    / psutil.virtual_memory().total,
-                    2,
-                )
-                # memory available (MB)
-                memory_available = round(
-                    psutil.virtual_memory().available / 1024**2, 2
-                )
-                if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
-                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
-                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
-
-                data.append(f"Host: {str(stat.host)}")
-                data.append(f"ID: {str(stat.cluster_id)[-8:]}")
-                data.append(f"Available (%): {memory_available_percentage}")
-                data.append(f"Available (MB): {memory_available}")
-                data.append(f"Total (MB): {round(psutil.virtual_memory().total / 1024**2, 2)}")
-                data.append(f"Sentinel (MB): {get_process_mb(stat.sentinel)}")
-                data.append(f"Monitor (MB): {get_process_mb(getattr(stat, 'monitor', None))}")
-
-                if self.workers:
-                    data.append("")
-                    for worker_num in range(Conf.WORKERS):
-                        data.append(f"Worker #{worker_num+1} (MB): {get_process_mb(stat.workers[worker_num])}")
-
-                data.append("")
-                data.append(_("Available lowest: %(memory_percent)s (%(at)s)")
-                % {
-                    "memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
-                    "at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
-                        "%Y-%m-%d %H:%M:%S+00:00"
-                    ),
-                })
-
-
-            for idx, item in enumerate(data):
-                scr.addstr(idx, 0, item)
-
-            scr.refresh()
-            time.sleep(0.5)
-            if self.run_once:
-                return

django_q/management/commands/qmonitor.py

@@ -1,14 +1,7 @@
-import curses
-from django_q.brokers import get_broker
-from django_q.models import Failure, Success
-from django_q.conf import Conf
-from django_q.status import Stat
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
-from django.utils import timezone
-import time
 
-import signal
+from django_q.monitor_terminal import monitor
 
 
 class Command(BaseCommand):
@@ -25,109 +18,4 @@ class Command(BaseCommand):
         )
 
     def handle(self, *args, **options):
-        memory_stats = MonitorTerminalStats(
-            run_once=options.get("run_once", False),
-        )
-        curses.wrapper(memory_stats.start)
-
-
-class MonitorTerminalStats:
-    stop_writing = False
-    table_cell_size = 20
-
-    def __init__(self, run_once=False):
-        self.run_once = run_once
-
-    def start(self, stdscr):
-        self.show_stats()
-
-    def on_exit(self, signum, frame):
-        # exit clean
-        self.stop_writing = True
-
-    def get_table_cell(self, data):
-        spaces = self.table_cell_size - len(str(data))
-        return data + " " * spaces + "//  "
-
-    def show_stats(self):
-        signal.signal(signal.SIGTERM, self.on_exit)
-        signal.signal(signal.SIGINT, self.on_exit)
-        scr = curses.initscr()
-
-        broker = get_broker()
-
-        broker.ping()
-
-        stats = Stat.get_all(broker=broker)
-
-        if not stats:
-            scr.addstr(1, 0, "Cluster is not running")
-            scr.refresh()
-
-        while not self.stop_writing:
-            data = []
-            table_headers = [
-                _("Host"),
-                _("Id"),
-                _("State"),
-                _("Pool"),
-                _("TQ"),
-                _("RQ"),
-                _("RC"),
-                _("Up"),
-            ]
-
-            data.append("".join([self.get_table_cell(header) for header in table_headers]))
-
-            for stat in stats:
-                tasks = str(stat.task_q_size)
-
-                if stat.task_q_size > 0:
-                    tasks = str(stat.task_q_size)
-                    if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
-                        tasks += " (at maximum size)"
-                results = stat.done_q_size
-                if results > 0:
-                    results = str(results)
-                # color workers
-                workers = len(stat.workers)
-                # format uptime
-                uptime = (timezone.now() - stat.tob).total_seconds()
-                hours, remainder = divmod(uptime, 3600)
-                minutes, seconds = divmod(remainder, 60)
-                uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
-                # print to the terminal
-                stat_values = [
-                    str(stat.host),
-                    str(stat.cluster_id)[-8:],
-                    str(stat.status),
-                    str(workers),
-                    str(tasks),
-                    str(results),
-                    str(stat.reincarnations),
-                    str(uptime),
-                ]
-
-                data.append("".join([self.get_table_cell(val) for val in stat_values]))
-
-            data.append("")
-            queue_size = broker.queue_size()
-            lock_size = broker.lock_size()
-            if lock_size:
-                queue_size = f"{queue_size}({lock_size})"
-
-
-            data.append("")
-            data.append(_("info: %(broker_info)s") % {"broker_info": broker.info()})
-            data.append("")
-            data.append(_("Queued: %(queue_size)s") % {"queue_size": str(broker.queue_size())})
-            data.append(_("Successes: %(success_count)s") % {"success_count": str(Success.objects.count())})
-            data.append(_("Failures: %(failure_count)s") % {"failure_count": str(Failure.objects.count())})
-
-            for idx, item in enumerate(data):
-                scr.addstr(idx, 0, item)
-
-            scr.refresh()
-            time.sleep(0.5)
-            if self.run_once:
-                return
+        monitor(run_once=options.get("run_once", False))

django_q/migrations/0016_schedule_intended_date_kwarg.py

@@ -1,6 +1,7 @@
 # Generated by Django 4.1.2 on 2023-01-15 22:34
 
 from django.db import migrations, models
+
 import django_q.models
 
 

django_q/models.py

@@ -1,6 +1,5 @@
 from datetime import datetime, timedelta
 from keyword import iskeyword
-import ast
 
 # Django
 from django import get_version
@@ -9,19 +8,21 @@ from django.db import models
 from django.template.defaultfilters import truncatechars
 from django.urls import reverse
 from django.utils import timezone
-from django.utils.timezone import is_aware
-from django.utils.html import format_html
-from django.utils.translation import gettext_lazy as _
 from django.utils.functional import cached_property
+from django.utils.html import format_html
+from django.utils.timezone import is_aware
+from django.utils.translation import gettext_lazy as _
 
 # External
 from picklefield import PickledObjectField
 from picklefield.fields import dbsafe_decode
 
 # Local
-from django_q.conf import croniter, Conf
+from django_q.conf import Conf, croniter
 from django_q.signing import SignedPackage
-from django_q.utils import localtime, add_months, add_years
+from django_q.utils import add_months, add_years, localtime
+
+from .utils import get_func_repr
 
 
 class Task(models.Model):
@@ -217,8 +218,11 @@ class Schedule(models.Model):
     )
     task = models.CharField(max_length=100, null=True, editable=False)
     cluster = models.CharField(
-        max_length=100, default=None, null=True, blank=True,
-        help_text=_("Name of the target cluster")
+        max_length=100,
+        default=None,
+        null=True,
+        blank=True,
+        help_text=_("Name of the target cluster"),
     )
     intended_date_kwarg = models.CharField(
         max_length=100,
@@ -228,34 +232,6 @@ class Schedule(models.Model):
         help_text=_("Name of kwarg to pass intended schedule date"),
     )
 
-    def parse_kwargs(self):
-        if not self.kwargs:
-            return {}
-        try:
-            # first try the dict syntax
-            return ast.literal_eval(self.kwargs)
-        except (SyntaxError, ValueError):
-            # else use the kwargs syntax
-            try:
-                parsed_kwargs = (
-                    ast.parse(f"f({self.kwargs})").body[0].value.keywords
-                )
-                return {
-                    kwarg.arg: ast.literal_eval(kwarg.value)
-                    for kwarg in parsed_kwargs
-                }
-            except (SyntaxError, ValueError):
-                return {}
-
-    def parse_args(self):
-        if not self.args:
-            return tuple()
-        args = ast.literal_eval(self.args)
-        # single value won't eval to tuple, so:
-        if type(args) != tuple:
-            args = (args,)
-        return args
-
     def calculate_next_run(self, next_run=None):
         # next run is always in UTC
         next_run = next_run or self.next_run
@@ -336,8 +312,9 @@ class Schedule(models.Model):
 class OrmQ(models.Model):
     key = models.CharField(max_length=100, help_text=_("Name of the target cluster"))
     payload = models.TextField()
-    lock = models.DateTimeField(null=True, help_text=_("Prevent any cluster from pulling until"))
-
+    lock = models.DateTimeField(
+        null=True, help_text=_("Prevent any cluster from pulling until")
+    )
 
     @cached_property
     def task(self):
@@ -347,24 +324,16 @@ class OrmQ(models.Model):
             return {"id": "*" + e.__class__.__name__}
 
     def func(self):
-        if isinstance(self.task, dict):
-            return self.task.get("func_name", "")
-        return self.task.func_name
+        return get_func_repr(self.task.get("func"))
 
     def task_id(self):
-        if isinstance(self.task, dict):
-            return self.task.get("id", "")
-        return self.task.id
+        return self.task.get("id")
 
     def name(self):
-        if isinstance(self.task, dict):
-            return self.task["name"]
-        return self.task.name
+        return self.task.get("name")
 
     def group(self):
-        if isinstance(self.task, dict):
-            return self.task.get("group", "")
-        return self.task.group
+        return self.task.get("group")
 
     def args(self):
         return self.task.get("args")

django_q/monitor.py

@@ -1,15 +1,24 @@
-from django_q.worker import WorkerProcess
-from django_q.queue_task import QueueTask
-from django_q.models import Task
-from queue import Queue
-from queue import Empty
-from typing import Optional, Tuple
-from django_q.brokers import get_broker
-from django_q.process_manager import ProcessManager
-from django_q.signals import post_execute
-from django_q.conf import logger
+from multiprocessing.process import current_process
+from multiprocessing.queues import Queue
+
+from django import core, db
 from django.utils.translation import gettext_lazy as _
-from multiprocessing import current_process
+from django.apps.registry import apps
+
+try:
+    apps.check_apps_ready()
+except core.exceptions.AppRegistryNotReady:
+    import django
+
+    django.setup()
+
+from django_q.brokers import Broker, get_broker
+from django_q.conf import Conf, logger, setproctitle
+from django_q.models import Success, Task
+from django_q.signals import post_execute
+from django_q.signing import SignedPackage
+from django_q.tasks import async_chain
+from django_q.utils import close_old_django_connections, get_func_repr
 
 try:
     import setproctitle
@@ -17,86 +26,181 @@ except ModuleNotFoundError:
     setproctitle = None
 
 
-class Monitor(ProcessManager):
-    def __init__(self):
-        super().__init__()
-        self.task_queue = Queue()
-
-    @staticmethod
-    def save_task(task, broker=None) -> Tuple[QueueTask, Optional[Task]]:
-        task_db_obj = None
-        if broker is None:
-            broker = get_broker()
-        if task.cached:
-            task.save_cached(broker)
+def monitor(result_queue: Queue, broker: Broker = None):
+    """
+    Gets finished tasks from the result queue and saves them to Django
+    :type broker: brokers.Broker
+    :type result_queue: multiprocessing.Queue
+    """
+    if not broker:
+        broker = get_broker()
+    proc_name = current_process().name
+    if setproctitle:
+        setproctitle.setproctitle(f"qcluster {proc_name} monitor")
+    logger.info(
+        _("%(name)s monitoring at %(id)s")
+        % {"name": proc_name, "id": current_process().pid}
+    )
+    for task in iter(result_queue.get, "STOP"):
+        # save the result
+        if task.get("cached", False):
+            save_cached(task, broker)
         else:
-            print("SAVE TO DB")
-            task_db_obj = task.save_to_db(broker)
+            save_task(task, broker)
         # acknowledge result
-        if task.ack_id and (task.has_succeeded or not task.ack_failure):
-            broker.acknowledge(task.ack_id)
+        ack_id = task.pop("ack_id", False)
+        if ack_id and (task["success"] or task.get("ack_failure", False)):
+            broker.acknowledge(ack_id)
         # signal execution done
         post_execute.send(sender="django_q", task=task)
-        return task, task_db_obj
+        # log the result
+        info_name = get_func_repr(task["func"])
+        if task["success"]:
+            # log success
+            logger.info(
+                _("Processed '%(info_name)s' (%(task_name)s)")
+                % {"info_name": info_name, "task_name": task["name"]}
+            )
+        else:
+            # log failure
+            logger.error(
+                _("Failed '%(info_name)s' (%(task_name)s) - %(task_result)s")
+                % {
+                    "info_name": info_name,
+                    "task_name": task["name"],
+                    "task_result": task["result"],
+                }
+            )
+    logger.info(_("%(name)s stopped monitoring results") % {"name": proc_name})
 
 
-    @property
-    def is_done(self):
-        return self.status.value == self.Status.IDLE.value and self.task_queue.empty()
+def save_task(task, broker: Broker):
+    """
+    Saves the task package to Django or the cache
+    :param task: the task package
+    :type broker: brokers.Broker
+    """
+    # SAVE LIMIT < 0 : Don't save success
+    if not task.get("save", Conf.SAVE_LIMIT >= 0) and task["success"]:
+        return
+    # enqueues next in a chain
+    if task.get("chain", None):
+        async_chain(
+            task["chain"],
+            group=task["group"],
+            cached=task["cached"],
+            sync=task["sync"],
+            broker=broker,
+        )
+    # SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
+    close_old_django_connections()
 
-    def get_target(self):
-        return self.run_monitor
+    try:
+        filters = {}
+        if (
+            Conf.SAVE_LIMIT_PER
+            and Conf.SAVE_LIMIT_PER in {"group", "name", "func"}
+            and Conf.SAVE_LIMIT_PER in task
+        ):
+            value = task[Conf.SAVE_LIMIT_PER]
+            if Conf.SAVE_LIMIT_PER == "func":
+                value = get_func_repr(value)
+            filters[Conf.SAVE_LIMIT_PER] = value
 
-    def run_item(self):
-        if self.is_idle:
-            try:
-                task = self.task_queue.get_nowait()
-            except Empty:
-                # if the queue is empty, then just stop
+        with db.transaction.atomic(using=db.router.db_for_write(Success)):
+            last = Success.objects.filter(**filters).select_for_update().last()
+            if (
+                task["success"]
+                and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
+            ):
+                last.delete()
+
+        # check if this task has previous results
+        try:
+            existing_task = Task.objects.get(id=task["id"], name=task["name"])
+            # only update the result if it hasn't succeeded yet
+            if not existing_task.success:
+                existing_task.stopped = task["stopped"]
+                existing_task.result = task["result"]
+                existing_task.success = task["success"]
+                existing_task.attempt_count = existing_task.attempt_count + 1
+                existing_task.save()
+
+            if (
+                Conf.MAX_ATTEMPTS > 0
+                and existing_task.attempt_count >= Conf.MAX_ATTEMPTS
+            ):
+                broker.acknowledge(task["ack_id"])
+
+        except Task.DoesNotExist:
+            # convert func to string
+            func = get_func_repr(task["func"])
+            Task.objects.create(
+                id=task["id"],
+                name=task["name"],
+                func=func,
+                hook=task.get("hook"),
+                args=task["args"],
+                kwargs=task["kwargs"],
+                cluster=task.get("cluster"),
+                started=task["started"],
+                stopped=task["stopped"],
+                result=task["result"],
+                group=task.get("group"),
+                success=task["success"],
+                attempt_count=1,
+            )
+    except Exception:
+        logger.exception("Could not save task result")
+
+
+def save_cached(task, broker: Broker):
+    task_key = f'{broker.list_key}:{task["id"]}'
+    timeout = task["cached"]
+    if timeout is True:
+        timeout = None
+    try:
+        group = task.get("group", None)
+        iter_count = task.get("iter_count", 0)
+        # if it's a group append to the group list
+        if group:
+            group_key = f"{broker.list_key}:{group}:keys"
+            group_list = broker.cache.get(group_key) or []
+            # if it's an iter group, check if we are ready
+            if iter_count and len(group_list) == iter_count - 1:
+                group_args = f"{broker.list_key}:{group}:args"
+                # collate the results into a Task result
+                results = [
+                    SignedPackage.loads(broker.cache.get(k))["result"]
+                    for k in group_list
+                ]
+                results.append(task["result"])
+                task["result"] = results
+                task["id"] = group
+                task["args"] = SignedPackage.loads(broker.cache.get(group_args))
+                task.pop("iter_count", None)
+                task.pop("group", None)
+                if task.get("iter_cached", None):
+                    task["cached"] = task.pop("iter_cached", None)
+                    save_cached(task, broker=broker)
+                else:
+                    save_task(task, broker)
+                broker.cache.delete_many(group_list)
+                broker.cache.delete_many([group_key, group_args])
                 return
-            try:
-                self.manager_pipe.send(task)
-            except BrokenPipeError:
-                # recycle process if pipe is broken
-                self.status.value = ProcessManager.Status.RECYCLE.value
-
-    def add_task(self, task):
-        self.task_queue.put(task)
-
-    def run_monitor(self, status, pipe) -> None:
-        broker = get_broker()
-        proc_name = current_process().name
-        if setproctitle:
-            setproctitle.setproctitle(f"qcluster {proc_name} monitor")
-        logger.info(
-                _("%(name)s monitoring at %(id)s") % {"name": proc_name, "id": current_process().pid}
+            # save the group list
+            group_list.append(task_key)
+            broker.cache.set(group_key, group_list, timeout)
+            # async_task next in a chain
+            if task.get("chain", None):
+                async_chain(
+                    task["chain"],
+                    group=group,
+                    cached=task["cached"],
+                    sync=task["sync"],
+                    broker=broker,
                 )
-        status.value = self.Status.IDLE.value
-
-        while True:
-            task = pipe.recv()
-            if task == "STOP":
-                logger.info(f"Monitor {proc_name} shut down")
-                break
-            status.value = self.Status.BUSY.value
-            # save the result
-            task, __ = Monitor.save_task(task, broker=broker)
-            # log the result
-            if task.has_succeeded:
-                # log success
-                logger.info(
-                        _("Processed '%(info_name)s' (%(task_name)s)")
-                        % {"info_name": task.func_name, "task_name": task.name}
-                        )
-            else:
-                # log failure
-                logger.error(
-                        _("Failed '%(info_name)s' (%(task_name)s) - %(task_result)s")
-                        % {
-                            "info_name": task.func_name,
-                            "task_name": task.name,
-                            "task_result": task.result,
-                            }
-                        )
-            status.value = self.Status.IDLE.value
-        logger.info(_("%(name)s stopped monitoring results") % {"name": proc_name})
+        # save the task
+        broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
+    except Exception:
+        logger.exception("Could not save task result")

django_q/monitor_terminal.py

@@ -0,0 +1,510 @@
+from datetime import timedelta
+
+# django
+from django.db import connection
+from django.db.models import F, Sum
+from django.utils import timezone
+from django.utils.translation import gettext as _
+
+from django_q import VERSION, models
+from django_q.brokers import get_broker
+
+# local
+from django_q.conf import Conf
+from django_q.status import Stat
+
+# optional
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+
+def get_process_mb(pid):
+    try:
+        process = psutil.Process(pid)
+        mb_used = round(process.memory_info().rss / 1024**2, 2)
+    except psutil.NoSuchProcess:
+        mb_used = "NO_PROCESS_FOUND"
+    return mb_used
+
+
+BLESSED_INSTALL_MESSAGE = (
+    "Blessed is not installed. Please install blessed to use this: "
+    "https://pypi.org/project/blessed/"
+)
+
+
+def monitor(run_once=False, broker=None):
+    if not broker:
+        broker = get_broker()
+    try:
+        from blessed import Terminal
+
+        term = Terminal()
+    except ImportError:
+        print(BLESSED_INSTALL_MESSAGE)
+        return
+
+    broker.ping()
+    with term.fullscreen(), term.hidden_cursor(), term.cbreak():
+        val = None
+        start_width = int(term.width / 8)
+        while val not in (
+            "q",
+            "Q",
+        ):
+            col_width = int(term.width / 8)
+            # In case of resize
+            if col_width != start_width:
+                print(term.clear())
+                start_width = col_width
+            print(
+                term.move(0, 0)
+                + term.black_on_green(term.center(_("Host"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 1 * col_width)
+                + term.black_on_green(term.center(_("Id"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 2 * col_width)
+                + term.black_on_green(term.center(_("State"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 3 * col_width)
+                + term.black_on_green(term.center(_("Pool"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 4 * col_width)
+                + term.black_on_green(term.center(_("TQ"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 5 * col_width)
+                + term.black_on_green(term.center(_("RQ"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 6 * col_width)
+                + term.black_on_green(term.center(_("RC"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 7 * col_width)
+                + term.black_on_green(term.center(_("Up"), width=col_width - 1))
+            )
+            i = 2
+            stats = Stat.get_all(broker=broker)
+            print(term.clear_eos())
+            for stat in stats:
+                status = stat.status
+                # color status
+                if stat.status == Conf.WORKING:
+                    status = term.green(str(Conf.WORKING))
+                elif stat.status == Conf.STOPPING:
+                    status = term.yellow(str(Conf.STOPPING))
+                elif stat.status == Conf.STOPPED:
+                    status = term.red(str(Conf.STOPPED))
+                elif stat.status == Conf.IDLE:
+                    status = str(Conf.IDLE)
+                # color q's
+                tasks = str(stat.task_q_size)
+                if stat.task_q_size > 0:
+                    tasks = term.cyan(str(stat.task_q_size))
+                    if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
+                        tasks = term.green(str(stat.task_q_size))
+                results = stat.done_q_size
+                if results > 0:
+                    results = term.cyan(str(results))
+                # color workers
+                workers = len(stat.workers)
+                if workers < Conf.WORKERS:
+                    workers = term.yellow(str(workers))
+                # format uptime
+                uptime = (timezone.now() - stat.tob).total_seconds()
+                hours, remainder = divmod(uptime, 3600)
+                minutes, seconds = divmod(remainder, 60)
+                uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
+                # print to the terminal
+                print(
+                    term.move(i, 0)
+                    + term.center(stat.host[: col_width - 1], width=col_width - 1)
+                )
+                print(
+                    term.move(i, 1 * col_width)
+                    + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
+                )
+                print(
+                    term.move(i, 2 * col_width)
+                    + term.center(status, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 3 * col_width)
+                    + term.center(workers, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 4 * col_width)
+                    + term.center(tasks, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 5 * col_width)
+                    + term.center(results, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 6 * col_width)
+                    + term.center(stat.reincarnations, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 7 * col_width)
+                    + term.center(uptime, width=col_width - 1)
+                )
+                i += 1
+            # bottom bar
+            i += 1
+            queue_size = broker.queue_size()
+            lock_size = broker.lock_size()
+            if lock_size:
+                queue_size = f"{queue_size}({lock_size})"
+            print(
+                term.move(i, 0)
+                + term.white_on_cyan(term.center(broker.info(), width=col_width * 2))
+            )
+            print(
+                term.move(i, 2 * col_width)
+                + term.black_on_cyan(term.center(_("Queued"), width=col_width))
+            )
+            print(
+                term.move(i, 3 * col_width)
+                + term.white_on_cyan(term.center(queue_size, width=col_width))
+            )
+            print(
+                term.move(i, 4 * col_width)
+                + term.black_on_cyan(term.center(_("Success"), width=col_width))
+            )
+            print(
+                term.move(i, 5 * col_width)
+                + term.white_on_cyan(
+                    term.center(models.Success.objects.count(), width=col_width)
+                )
+            )
+            print(
+                term.move(i, 6 * col_width)
+                + term.black_on_cyan(term.center(_("Failures"), width=col_width))
+            )
+            print(
+                term.move(i, 7 * col_width)
+                + term.white_on_cyan(
+                    term.center(models.Failure.objects.count(), width=col_width)
+                )
+            )
+            # for testing
+            if run_once:
+                return Stat.get_all(broker=broker)
+            print(term.move(i + 2, 0) + term.center(_("[Press q to quit]")))
+            val = term.inkey(timeout=1)
+
+
+def info(broker=None):
+    if not broker:
+        broker = get_broker()
+    try:
+        from blessed import Terminal
+
+        term = Terminal()
+    except ImportError:
+        print(BLESSED_INSTALL_MESSAGE)
+        return
+
+    broker.ping()
+    stat = Stat.get_all(broker=broker)
+    # general stats
+    clusters = len(stat)
+    workers = 0
+    reincarnations = 0
+    for cluster in stat:
+        workers += len(cluster.workers)
+        reincarnations += cluster.reincarnations
+    # calculate tasks pm and avg exec time
+    tasks_per = 0
+    per = _("day")
+    exec_time = 0
+    last_tasks = models.Success.objects.filter(
+        stopped__gte=timezone.now() - timedelta(hours=24)
+    )
+    tasks_per_day = last_tasks.count()
+    if tasks_per_day > 0:
+        # average execution time over the last 24 hours
+        if connection.vendor != "sqlite":
+            exec_time = last_tasks.aggregate(
+                time_taken=Sum(F("stopped") - F("started"))
+            )
+            exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
+        else:
+            # can't sum timedeltas on sqlite
+            for t in last_tasks:
+                exec_time += t.time_taken()
+            exec_time = exec_time / tasks_per_day
+        # tasks per second/minute/hour/day in the last 24 hours
+        if tasks_per_day > 24 * 60 * 60:
+            tasks_per = tasks_per_day / (24 * 60 * 60)
+            per = _("second")
+        elif tasks_per_day > 24 * 60:
+            tasks_per = tasks_per_day / (24 * 60)
+            per = _("minute")
+        elif tasks_per_day > 24:
+            tasks_per = tasks_per_day / 24
+            per = _("hour")
+        else:
+            tasks_per = tasks_per_day
+    # print to terminal
+    print(term.clear_eos())
+    col_width = int(term.width / 6)
+    print(
+        term.black_on_green(
+            term.center(
+                _("-- %(prefix)s %(version)s on %(info)s --")
+                % {
+                    "prefix": Conf.PREFIX.capitalize(),
+                    "version": ".".join(str(v) for v in VERSION),
+                    "info": broker.info(),
+                }
+            )
+        )
+    )
+    print(
+        term.cyan(_("Clusters"))
+        + term.move_x(1 * col_width)
+        + term.white(str(clusters))
+        + term.move_x(2 * col_width)
+        + term.cyan(_("Workers"))
+        + term.move_x(3 * col_width)
+        + term.white(str(workers))
+        + term.move_x(4 * col_width)
+        + term.cyan(_("Restarts"))
+        + term.move_x(5 * col_width)
+        + term.white(str(reincarnations))
+    )
+    print(
+        term.cyan(_("Queued"))
+        + term.move_x(1 * col_width)
+        + term.white(str(broker.queue_size()))
+        + term.move_x(2 * col_width)
+        + term.cyan(_("Successes"))
+        + term.move_x(3 * col_width)
+        + term.white(str(models.Success.objects.count()))
+        + term.move_x(4 * col_width)
+        + term.cyan(_("Failures"))
+        + term.move_x(5 * col_width)
+        + term.white(str(models.Failure.objects.count()))
+    )
+    print(
+        term.cyan(_("Schedules"))
+        + term.move_x(1 * col_width)
+        + term.white(str(models.Schedule.objects.count()))
+        + term.move_x(2 * col_width)
+        + term.cyan(_("Tasks/%(per)s") % {"per": per})
+        + term.move_x(3 * col_width)
+        + term.white(f"{tasks_per:.2f}")
+        + term.move_x(4 * col_width)
+        + term.cyan(_("Avg time"))
+        + term.move_x(5 * col_width)
+        + term.white(f"{exec_time:.4f}")
+    )
+    return True
+
+
+def memory(run_once=False, workers=False, broker=None):
+    if not broker:
+        broker = get_broker()
+    try:
+        from blessed import Terminal
+
+        term = Terminal()
+    except ImportError:
+        print(BLESSED_INSTALL_MESSAGE)
+        return
+    broker.ping()
+    if not psutil:
+        print(term.clear_eos())
+        print(
+            term.white_on_red(
+                'Cannot start "qmemory" command. Missing "psutil" library.'
+            )
+        )
+        return
+    with term.fullscreen(), term.hidden_cursor(), term.cbreak():
+        MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
+        MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
+        cols = 8
+        val = None
+        start_width = int(term.width / cols)
+        while val not in ["q", "Q"]:
+            col_width = int(term.width / cols)
+            # In case of resize
+            if col_width != start_width:
+                print(term.clear())
+                start_width = col_width
+            # sentinel, monitor and workers memory usage
+            print(
+                term.move(0, 0 * col_width)
+                + term.black_on_green(term.center(_("Host"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 1 * col_width)
+                + term.black_on_green(term.center(_("Id"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 2 * col_width)
+                + term.black_on_green(
+                    term.center(_("Available (%)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 3 * col_width)
+                + term.black_on_green(
+                    term.center(_("Available (MB)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 4 * col_width)
+                + term.black_on_green(term.center(_("Total (MB)"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 5 * col_width)
+                + term.black_on_green(
+                    term.center(_("Sentinel (MB)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 6 * col_width)
+                + term.black_on_green(
+                    term.center(_("Monitor (MB)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 7 * col_width)
+                + term.black_on_green(
+                    term.center(_("Workers (MB)"), width=col_width - 1)
+                )
+            )
+            row = 2
+            stats = Stat.get_all(broker=broker)
+            print(term.clear_eos())
+            for stat in stats:
+                # memory available (%)
+                memory_available_percentage = round(
+                    psutil.virtual_memory().available
+                    * 100
+                    / psutil.virtual_memory().total,
+                    2,
+                )
+                # memory available (MB)
+                memory_available = round(
+                    psutil.virtual_memory().available / 1024**2, 2
+                )
+                if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
+                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
+                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
+                print(
+                    term.move(row, 0 * col_width)
+                    + term.center(stat.host[: col_width - 1], width=col_width - 1)
+                )
+                print(
+                    term.move(row, 1 * col_width)
+                    + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
+                )
+                print(
+                    term.move(row, 2 * col_width)
+                    + term.center(memory_available_percentage, width=col_width - 1)
+                )
+                print(
+                    term.move(row, 3 * col_width)
+                    + term.center(memory_available, width=col_width - 1)
+                )
+                print(
+                    term.move(row, 4 * col_width)
+                    + term.center(
+                        round(psutil.virtual_memory().total / 1024**2, 2),
+                        width=col_width - 1,
+                    )
+                )
+                print(
+                    term.move(row, 5 * col_width)
+                    + term.center(get_process_mb(stat.sentinel), width=col_width - 1)
+                )
+                print(
+                    term.move(row, 6 * col_width)
+                    + term.center(
+                        get_process_mb(getattr(stat, "monitor", None)),
+                        width=col_width - 1,
+                    )
+                )
+                workers_mb = 0
+                for worker_pid in stat.workers:
+                    result = get_process_mb(worker_pid)
+                    if isinstance(result, str):
+                        result = 0
+                    workers_mb += result
+                print(
+                    term.move(row, 7 * col_width)
+                    + term.center(
+                        workers_mb or "NO_PROCESSES_FOUND", width=col_width - 1
+                    )
+                )
+                row += 1
+            # each worker's memory usage
+            if workers:
+                row += 2
+                col_width = int(term.width / (1 + Conf.WORKERS))
+                print(
+                    term.move(row, 0 * col_width)
+                    + term.black_on_cyan(term.center(_("Id"), width=col_width - 1))
+                )
+                for worker_num in range(Conf.WORKERS):
+                    print(
+                        term.move(row, (worker_num + 1) * col_width)
+                        + term.black_on_cyan(
+                            term.center(
+                                "Worker #{} (MB)".format(worker_num + 1),
+                                width=col_width - 1,
+                            )
+                        )
+                    )
+                row += 2
+                for stat in stats:
+                    print(
+                        term.move(row, 0 * col_width)
+                        + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
+                    )
+                    for idx, worker_pid in enumerate(stat.workers):
+                        mb_used = get_process_mb(worker_pid)
+                        print(
+                            term.move(row, (idx + 1) * col_width)
+                            + term.center(mb_used, width=col_width - 1)
+                        )
+                    row += 1
+            row += 1
+            print(
+                term.move(row, 0)
+                + _("Available lowest (): %(memory_percent)s ((at)s)")
+                % {
+                    "memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
+                    "at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
+                        "%Y-%m-%d %H:%M:%S+00:00"
+                    ),
+                }
+            )
+            # for testing
+            if run_once:
+                return Stat.get_all(broker=broker)
+            print(term.move(row + 2, 0) + term.center(_("[Press q to quit]")))
+            val = term.inkey(timeout=1)
+
+
+def get_ids():
+    # prints id (PID) of running clusters
+    stat = Stat.get_all()
+    if stat:
+        for s in stat:
+            print(s.cluster_id)
+    else:
+        print(_("No clusters appear to be running."))
+    return True

django_q/process_manager.py

@@ -1,75 +0,0 @@
-from abc import ABC
-from django_q.conf import Conf, logger
-from django_q.humanhash import humanize
-import uuid
-from django_q.conf import Conf
-import enum
-from django import db
-from typing import Callable
-from django.utils.translation import gettext_lazy as _
-import multiprocessing
-from multiprocessing import Process, Value
-
-class ProcessManager(ABC):
-    class Status(enum.IntEnum):
-        IDLE = 1
-        BUSY = 2
-        DONE = 3
-        RECYCLE = 4
-
-    target = None
-
-    def get_target(self) -> Callable:
-        if self.target is None:
-            raise ValueError("Process must have target specified")
-        return self.target
-
-    def __init__(self):
-        self.status = Value("i", self.Status.IDLE.value)
-        self.process = self.spawn_process()
-        self.name = humanize(uuid.uuid4().hex)
-
-    def spawn_process(self) -> Process:
-        self.manager_pipe, process_pipe = multiprocessing.Pipe(duplex=True)
-        p = Process(target=self.get_target(), args=(self.status, process_pipe))
-        p.start()
-        return p
-
-    def reincarnate_process(self):
-        # kill connections before killing the process
-        logger.critical(_("reincarnated worker %(name)s after death") % {"name": self.process.name})
-        if not Conf.SYNC:
-            db.connections.close_all()
-        self.process.kill()
-        self.process = self.spawn_process()
-        self.mark_idle()
-
-    @property
-    def has_results(self):
-        # poll puller pipe for new tasks
-        return self.manager_pipe.poll()
-
-    def get_result(self):
-        # get the puller pipe task object back from the worker
-        return self.manager_pipe.recv()
-
-    @property
-    def is_alive(self):
-        # get the puller status
-        return self.process.is_alive()
-
-    @property
-    def is_done(self):
-        return self.status.value == self.Status.DONE.value
-
-    @property
-    def is_idle(self):
-        return self.status.value == self.Status.IDLE.value
-
-    @property
-    def is_recycle(self):
-        # worker needs to be recycled/reincarnated
-        return self.status.value == self.Status.RECYCLE.value
-
-    def mark_idle(self):
-        self.status.value = self.Status.IDLE.value

django_q/puller.py

@@ -1,85 +0,0 @@
-from django_q.worker import Worker
-from django_q.signing import BadSignature, SignedPackage
-from time import sleep
-from django_q.brokers import get_broker
-import multiprocessing
-from django_q.queue_task import QueueTask
-from django.utils import timezone
-import enum
-import traceback
-
-from multiprocessing import Event, Process, Value, current_process
-from django_q.utils import close_old_django_connections
-from django.utils.translation import gettext_lazy as _
-from django_q.conf import Conf, logger, setproctitle, error_reporter, resource, psutil
-from django_q.exceptions import TimeoutException, TimeoutHandler
-from django_q.process_manager import ProcessManager
-
-
-
-class Puller(ProcessManager):
-    """The Puller is responsible for pulling the tasks from the broker, then return them to be picked up by the
-    guard"""
-
-    @staticmethod
-    def get_tasks_from_broker(broker=None):
-        queued_tasks = []
-        if broker is None:
-            broker = get_broker()
-        try:
-            task_set = broker.dequeue()
-        except Exception:
-            # broker probably crashed. Let the sentinel handle it.
-            raise ValueError("Failed to pull task from broker")
-        if task_set:
-            logger.info(
-                _("Found %(amount_tasks)s tasks") % {"amount_tasks": len(task_set)}
-            )
-            for task in task_set:
-                print(task)
-                logger.info("ONE TASK")
-                ack_id = task[0]
-                # unpack the task
-                try:
-                    queue_task = SignedPackage.loads(task[1])
-                except (TypeError, BadSignature):
-                    logger.exception("Failed to pull task from broker - bad task")
-                    broker.fail(ack_id)
-                    continue
-                queue_task.cluster = Conf.CLUSTER_NAME # save actual cluster name to orm task table
-                queue_task.ack_id = ack_id
-                # send back to main process
-                queued_tasks.append(queue_task)
-            logger.debug(
-                _("queueing from %(list_key)s") % {"list_key": broker.list_key}
-            )
-        return queued_tasks
-
-    def get_target(self):
-        return self.run_puller
-
-    def stop_puller(self):
-        self.status.value = self.Status.DONE.value
-
-    def run_puller(self, status, pipe) -> None:
-        broker = get_broker()
-        proc_name = current_process().name
-        if setproctitle:
-            setproctitle.setproctitle(f"qcluster {proc_name} puller")
-        logger.info(
-            _("%(name)s pulling tasks from broker %(id)s")
-            % {"name": proc_name, "id": current_process().pid}
-        )
-        while True:
-            if status.value == Worker.Status.DONE.value:
-                logger.info("Stopping Puller")
-                break
-            try:
-                queued_tasks = Puller.get_tasks_from_broker(broker=broker)
-            except Exception:
-                logger.exception("Couldn't get items from broker")
-                sleep(10)
-                break
-            for queue_task in queued_tasks:
-                pipe.send(queue_task)
-        logger.info(_("%(name)s stopped pushing tasks") % {"name": current_process().name})

django_q/pusher.py

@@ -0,0 +1,71 @@
+from multiprocessing import Event
+from multiprocessing.process import current_process
+from multiprocessing.queues import Queue
+from time import sleep
+
+from django import core
+from django.utils.translation import gettext_lazy as _
+from django.apps.registry import apps
+
+try:
+    apps.check_apps_ready()
+except core.exceptions.AppRegistryNotReady:
+    import django
+
+    django.setup()
+
+from django_q.brokers import Broker, get_broker
+from django_q.conf import Conf, logger
+from django_q.signing import BadSignature, SignedPackage
+
+try:
+    import setproctitle
+except ModuleNotFoundError:
+    setproctitle = None
+
+
+def pusher(task_queue: Queue, event: Event, broker: Broker = None):
+    """
+    Pulls tasks of the broker and puts them in the task queue
+    :type broker:
+    :type task_queue: multiprocessing.Queue
+    :type event: multiprocessing.Event
+    """
+    if not broker:
+        broker = get_broker()
+    proc_name = current_process().name
+    if setproctitle:
+        setproctitle.setproctitle(f"qcluster {proc_name} pusher")
+    logger.info(
+        _("%(name)s pushing tasks at %(id)s")
+        % {"name": proc_name, "id": current_process().pid}
+    )
+    while True:
+        try:
+            task_set = broker.dequeue()
+        except Exception:
+            logger.exception("Failed to pull task from broker")
+            # broker probably crashed. Let the sentinel handle it.
+            sleep(10)
+            break
+        if task_set:
+            for task in task_set:
+                ack_id = task[0]
+                # unpack the task
+                try:
+                    task = SignedPackage.loads(task[1])
+                except (TypeError, BadSignature):
+                    logger.exception("Failed to push task to queue")
+                    broker.fail(ack_id)
+                    continue
+                task[
+                    "cluster"
+                ] = Conf.CLUSTER_NAME  # save actual cluster name to orm task table
+                task["ack_id"] = ack_id
+                task_queue.put(task)
+            logger.debug(
+                _("queueing from %(list_key)s") % {"list_key": broker.list_key}
+            )
+        if event.is_set():
+            break
+    logger.info(_("%(name)s stopped pushing tasks") % {"name": current_process().name})

django_q/queue_task.py

@@ -1,211 +0,0 @@
-from __future__ import annotations
-from datetime import datetime
-from django_q.brokers import Broker
-from django.utils import timezone
-from django_q.signing import SignedPackage
-from django_q.models import Success, Task
-from django_q.utils import close_old_django_connections
-import enum
-from django_q import tasks
-import inspect
-import pydoc
-from django import db
-
-from dataclasses import dataclass, field
-from typing import Any, Callable, Optional, Union
-
-from django_q.conf import Conf, logger
-
-
-@dataclass
-class QueueTask:
-    class Status(enum.IntEnum):
-        QUEUED = 0
-        SUCCESS = 1
-        FAILED = 2
-        TIMEOUT = 3
-
-    func: Union[Callable, str]
-    name: str
-    group: Optional[str] = None
-    cluster: str = Conf.CLUSTER_NAME
-    queued_at: Optional[datetime] = timezone.now()
-    finished_at: Optional[datetime] = None
-    ack_id: Optional[str] = None
-    started_at: Optional[datetime] = None
-    id: str = "-1"
-    timeout: Optional[int] = Conf.TIMEOUT
-    status: Optional[Status] = None
-    result: Any = None
-    save: bool = Conf.SAVE_LIMIT >= 0
-    chain: Union[str, QueueTask] = ""
-    cached: bool = Conf.CACHED
-    sync: bool = Conf.SYNC
-    hook: Optional[str] = None
-    args: tuple = field(default_factory=tuple)
-    kwargs: dict = field(default_factory=dict)
-    ack_failure: bool = Conf.ACK_FAILURES
-    iter_count: Optional[int] = None
-    iter_cached: Optional[int] = None
-
-    def callable_func(self):
-        func = self.func
-        if not callable(func):
-            func = pydoc.locate(func)
-        return func
-
-    @property
-    def has_succeeded(self):
-        return self.status == self.Status.SUCCESS
-
-    @property
-    def has_timed_out(self):
-        return self.status == self.Status.TIMEOUT
-
-    @property
-    def is_callable(self):
-        return self.callable_func is not None
-
-    @property
-    def func_name(self):
-        if inspect.isfunction(self.func):
-            return f"{self.func.__module__}.{self.func.__name__}"
-        elif inspect.ismethod(self.func) and hasattr(self.func.__self__, "__name__"):
-            return (
-                f"{self.func.__self__.__module__}." f"{self.func.__self__.__name__}.{self.func.__name__}"
-            )
-        else:
-            return str(self.func)
-
-    def save_to_db(self, broker: Broker):
-        """
-        Saves the task package to Django or the cache
-        :param task: the task package
-        :type broker: brokers.Broker
-        """
-        # SAVE LIMIT < 0 : Don't save success
-        if not self.save and self.has_succeeded:
-            return
-        # enqueues next in a chain
-        if self.chain:
-            tasks.async_chain(
-                self.chain,
-                group=self.group,
-                cached=self.cached,
-                sync=self.sync,
-                broker=broker,
-            )
-        close_old_django_connections()
-
-        logger.debug(self.func_name)
-        try:
-            filters = {}
-            if (
-                Conf.SAVE_LIMIT_PER
-                and Conf.SAVE_LIMIT_PER in {"group", "name", "func"}
-                and Conf.SAVE_LIMIT_PER in self
-            ):
-                value = getattr(self, Conf.SAVE_LIMIT_PER)
-                if Conf.SAVE_LIMIT_PER == "func":
-                    value = self.func_name
-                filters[Conf.SAVE_LIMIT_PER] = value
-
-            with db.transaction.atomic(using=db.router.db_for_write(Success)):
-                last = Success.objects.filter(**filters).select_for_update().last()
-                if (
-                    self.has_succeeded
-                    and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
-                ):
-                    # delete the last entry if we are hitting the limit
-                    last.delete()
-
-            # check if this task has previous results
-            existing_task, created = Task.objects.get_or_create(
-                id=self.id,
-                name=self.name,
-                defaults={
-                    'func': self.func_name,
-                    'stopped': self.finished_at,
-                    'hook': self.hook,
-                    'args': self.args,
-                    'kwargs': self.kwargs,
-                    'cluster': self.cluster,
-                    'started': self.started_at,
-                    'result': self.result,
-                    'group': self.group,
-                    'success': self.has_succeeded,
-                    'attempt_count': 1
-                }
-            )
-
-            # only update the result if it hasn't succeeded yet
-            if not created and not existing_task.success:
-                existing_task.stopped = self.finished_at
-                existing_task.result = self.result
-                existing_task.success = self.has_succeeded
-                existing_task.attempt_count += 1
-                existing_task.save()
-
-            if (
-                Conf.MAX_ATTEMPTS > 0
-                and existing_task.attempt_count >= Conf.MAX_ATTEMPTS
-            ):
-                broker.acknowledge(self.ack_id)
-
-            return existing_task
-
-        except Exception:
-            logger.exception("Could not save task result")
-
-
-    def save_cached(self, broker: Broker):
-        task_key = f'{broker.list_key}:{self.id}'
-        timeout = self.cached
-        if timeout is True:
-            timeout = None
-        try:
-            group = self.group
-            iter_count = self.iter_count
-            # if it's a group append to the group list
-            if group:
-                group_key = f"{broker.list_key}:{group}:keys"
-                group_list = broker.cache.get(group_key) or []
-                # if it's an iter group, check if we are ready
-                if iter_count and len(group_list) == iter_count - 1:
-                    group_args = f"{broker.list_key}:{group}:args"
-                    # collate the results into a Task result
-                    results = [
-                        SignedPackage.loads(broker.cache.get(k)).result
-                        for k in group_list
-                    ]
-                    results.append(self.result)
-                    self.result = results
-                    self.id = group
-                    self.args = SignedPackage.loads(broker.cache.get(group_args))
-                    self.iter_count = None
-                    self.group = None
-                    if self.iter_cached:
-                        self.cached = self.iter_cached
-                        self.save_cached(broker=broker)
-                    else:
-                        self.save_to_db(broker)
-                    broker.cache.delete_many(group_list)
-                    broker.cache.delete_many([group_key, group_args])
-                    return
-                # save the group list
-                group_list.append(task_key)
-                broker.cache.set(group_key, group_list, timeout)
-                # async_task next in a chain
-                if self.chain:
-                    tasks.async_chain(
-                        self.chain,
-                        group=group,
-                        cached=self.cached,
-                        sync=self.sync,
-                        broker=broker,
-                    )
-            # save the task
-            broker.cache.set(task_key, SignedPackage.dumps(self), timeout)
-        except Exception:
-            logger.exception("Could not save task result")
-

django_q/queues.py

@@ -0,0 +1,82 @@
+"""
+The code is derived from
+https://github.com/althonos/pronto/commit/3384010dfb4fc7c66a219f59276adef3288a886b
+"""
+import multiprocessing
+import multiprocessing.queues
+import sys
+
+
+class SharedCounter:
+    """A synchronized shared counter.
+
+    The locking done by multiprocessing.Value ensures that only a single
+    process or thread may read or write the in-memory ctypes object. However,
+    in order to do n += 1, Python performs a read followed by a write, so a
+    second process may read the old value before the new one is written by
+    the first process. The solution is to use a multiprocessing.Lock to
+    guarantee the atomicity of the modifications to Value.
+
+    This class comes almost entirely from Eli Bendersky's blog:
+    http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing/
+    """
+
+    def __init__(self, n=0):
+        self.count = multiprocessing.Value("i", n)
+
+    def increment(self, n=1):
+        """Increment the counter by n (default = 1)"""
+        with self.count.get_lock():
+            self.count.value += n
+
+    @property
+    def value(self):
+        """Return the value of the counter"""
+        return self.count.value
+
+
+class Queue(multiprocessing.queues.Queue):
+    """A portable implementation of multiprocessing.Queue.
+
+    Because of multithreading / multiprocessing semantics, Queue.qsize() may
+    raise the NotImplementedError exception on Unix platforms like Mac OS X
+    where sem_getvalue() is not implemented. This subclass addresses this
+    problem by using a synchronized shared counter (initialized to zero) and
+    increasing / decreasing its value every time the put() and get() methods
+    are called, respectively. This not only prevents NotImplementedError from
+    being raised, but also allows us to implement a reliable version of both
+    qsize() and empty().
+    """
+
+    def __init__(self, *args, **kwargs):
+        if sys.version_info < (3, 0):
+            super(Queue, self).__init__(*args, **kwargs)
+        else:
+            super(Queue, self).__init__(
+                *args, ctx=multiprocessing.get_context(), **kwargs
+            )
+        self.size = SharedCounter(0)
+
+    def __getstate__(self):
+        return super(Queue, self).__getstate__() + (self.size,)
+
+    def __setstate__(self, state):
+        super(Queue, self).__setstate__(state[:-1])
+        self.size = state[-1]
+
+    def put(self, *args, **kwargs):
+        super(Queue, self).put(*args, **kwargs)
+        self.size.increment(1)
+
+    def get(self, *args, **kwargs):
+        x = super(Queue, self).get(*args, **kwargs)
+        self.size.increment(-1)
+        return x
+
+    def qsize(self) -> int:
+        """Reliable implementation of multiprocessing.Queue.qsize()"""
+        return self.size.value
+
+    def empty(self) -> bool:
+        """Reliable implementation of multiprocessing.Queue.empty()"""
+        return not self.qsize() > 0

django_q/scheduler.py

@@ -1,39 +1,73 @@
-from django_q.utils import localtime
-import uuid
-from django_q.models import Schedule
-from django_q import tasks
 import ast
-from django_q.humanhash import humanize
-from django import db
-from time import sleep
-from django_q.brokers import get_broker
+from multiprocessing.process import current_process
+
+from django import core, db
 from django.utils import timezone
-
-from multiprocessing import Value, current_process
-from django_q.utils import close_old_django_connections
 from django.utils.translation import gettext_lazy as _
+
+from django.apps.registry import apps
+
+try:
+    apps.check_apps_ready()
+except core.exceptions.AppRegistryNotReady:
+    import django
+
+    django.setup()
+
+from django_q.brokers import Broker, get_broker
 from django_q.conf import Conf, logger
-from django_q.process_manager import ProcessManager
+from django_q.humanhash import humanize
+from django_q.models import Schedule
+from django_q.tasks import async_task
+from django_q.utils import close_old_django_connections, localtime
 
 
+def scheduler(broker: Broker = None):
+    """
+    Creates a task from a schedule at the scheduled time and schedules next run
+    """
+    if not broker:
+        broker = get_broker()
+    close_old_django_connections()
+    try:
+        # Only default cluster will handler schedule with default(null) cluster
+        Q_default = (
+            db.models.Q(cluster__isnull=True)
+            if Conf.CLUSTER_NAME == Conf.PREFIX
+            else db.models.Q(pk__in=[])
+        )
 
-class Scheduler(ProcessManager):
-    """The Scheduler is responsible for scheduling new tasks"""
-
-    @staticmethod
-    def schedule_tasks(broker=None):
-        logger.debug("Start sheduling")
-        if broker is None:
-            broker = get_broker()
-        q_default = db.models.Q(cluster__isnull=True) if Conf.CLUSTER_NAME == Conf.PREFIX else db.models.Q(pk__in=[])
         with db.transaction.atomic(using=db.router.db_for_write(Schedule)):
             for s in (
                 Schedule.objects.select_for_update()
                 .exclude(repeats=0)
-                .filter(db.models.Q(next_run__lt=timezone.now()), q_default | db.models.Q(cluster=Conf.CLUSTER_NAME))
+                .filter(next_run__lt=timezone.now())
+                .filter(Q_default | db.models.Q(cluster=Conf.CLUSTER_NAME))
             ):
-                args = s.parse_args()
-                kwargs = s.parse_kwargs()
+                args = ()
+                kwargs = {}
+                # get args, kwargs and hook
+                if s.kwargs:
+                    try:
+                        # first try the dict syntax
+                        kwargs = ast.literal_eval(s.kwargs)
+                    except (SyntaxError, ValueError):
+                        # else use the kwargs syntax
+                        try:
+                            parsed_kwargs = (
+                                ast.parse(f"f({s.kwargs})").body[0].value.keywords
+                            )
+                            kwargs = {
+                                kwarg.arg: ast.literal_eval(kwarg.value)
+                                for kwarg in parsed_kwargs
+                            }
+                        except (SyntaxError, ValueError):
+                            kwargs = {}
+                if s.args:
+                    args = ast.literal_eval(s.args)
+                    # single value won't eval to tuple, so:
+                    if type(args) != tuple:
+                        args = (args,)
                 q_options = kwargs.get("q_options", {})
                 if s.intended_date_kwarg:
                     kwargs[s.intended_date_kwarg] = s.next_run.isoformat()
@@ -41,23 +75,27 @@ class Scheduler(ProcessManager):
                     q_options["hook"] = s.hook
                 # set up the next run time
                 if s.schedule_type != s.ONCE:
-                    next_run = s.calculate_next_run(s.next_run)
-                    if not Conf.CATCH_UP:
-                        while next_run <= localtime():
-                            next_run = s.calculate_next_run(next_run)
+                    next_run = s.next_run
+                    while True:
+                        next_run = s.calculate_next_run(next_run)
+                        if Conf.CATCH_UP or next_run > localtime():
+                            break
 
                     s.next_run = next_run
                     s.repeats += -1
                 # send it to the cluster; any cluster name is allowed in multi-queue scenarios
-                # because `broker_name` is confusing, using `cluster` name is recommended and take
-                q_options["cluster"] = s.cluster or q_options.get("cluster", q_options.pop("broker_name", None))
-                if q_options['cluster'] is None or q_options['cluster'] == Conf.CLUSTER_NAME:
+                # because `broker_name` is confusing, using `cluster` name is recommended and takes precedence
+                q_options["cluster"] = s.cluster or q_options.get(
+                    "cluster", q_options.pop("broker_name", None)
+                )
+                if (
+                    q_options["cluster"] is None
+                    or q_options["cluster"] == Conf.CLUSTER_NAME
+                ):
                     q_options["broker"] = broker
-
                 q_options["group"] = q_options.get("group", s.name or s.id)
                 kwargs["q_options"] = q_options
-
-                s.task = tasks.async_task(s.func, *args, **kwargs)
+                s.task = async_task(s.func, *args, **kwargs)
                 # log it
                 if not s.task:
                     logger.error(
@@ -91,35 +129,5 @@ class Scheduler(ProcessManager):
                     s.repeats = 0
                 # save the schedule
                 s.save()
-
-    def get_target(self):
-        return self.run_scheduler
-
-    def stop_scheduler(self) -> None:
-        # send stop signal to worker
-        try:
-            self.manager_pipe.send("STOP")
-        except BrokenPipeError:
-            # recycle process if pipe is broken
-            self.status.value = ProcessManager.Status.DONE.value
-
-    def run_scheduler(self, status, pipe) -> None:
-        self.process_name = current_process().name
-        self.process_id = current_process().pid
-        status.value = self.Status.BUSY.value
-        logger.info(
-            _("%(proc_name)s scheduling at %(id)s")
-            % {"proc_name": self.process_name, "id": self.process_id}
-        )
-        while True:
-            if pipe.poll() and pipe.recv() == "STOP":
-                status.value = self.Status.DONE.value
-                break
-            broker = get_broker()
-            close_old_django_connections()
-            try:
-                Scheduler.schedule_tasks(broker=broker)
-            except Exception:
-                logger.exception("Could not create task from schedule")
-            # sleep 60 seconds for next schedule
-            sleep(60)
+    except Exception:
+        logger.exception("Could not create task from schedule")

django_q/signals.py

@@ -32,6 +32,9 @@ def call_hook(sender, instance, **kwargs):
             )
 
 
+# args: proc_name
+post_spawn = Signal()
+
 # args: task
 pre_enqueue = Signal()
 

django_q/status.py

@@ -41,7 +41,14 @@ class Stat(Status):
         self.status = sentinel.status()
         self.done_q_size = 0
         self.task_q_size = 0
-        self.workers = [w.process.pid for w in sentinel.pool.workers]
+        if Conf.QSIZE:
+            self.done_q_size = sentinel.result_queue.qsize()
+            self.task_q_size = sentinel.task_queue.qsize()
+        if sentinel.monitor:
+            self.monitor = sentinel.monitor.pid
+        if sentinel.pusher:
+            self.pusher = sentinel.pusher.pid
+        self.workers = [w.pid for w in sentinel.pool]
 
     def uptime(self) -> float:
         return (timezone.now() - self.tob).total_seconds()

django_q/tasks.py

@@ -1,7 +1,5 @@
 """Provides task functionality."""
 # Standard
-from django_q.helpers import run_cluster_once
-from django_q.queue_task import QueueTask
 from multiprocessing import Value
 from time import sleep, time
 
@@ -14,13 +12,14 @@ from django_q.brokers import get_broker
 from django_q.conf import Conf, logger
 from django_q.humanhash import uuid
 from django_q.models import Schedule, Task
+from django_q.queues import Queue
 from django_q.signals import pre_enqueue
 from django_q.signing import SignedPackage
 
 
 def async_task(func, *args, **kwargs):
     """Queue a task for the cluster."""
-    given_kwargs = kwargs.copy()
+    keywords = kwargs.copy()
     opt_keys = (
         "hook",
         "group",
@@ -35,40 +34,47 @@ def async_task(func, *args, **kwargs):
         "cluster",
         "timeout",
     )
-    q_options = given_kwargs.pop("q_options", {})
+    q_options = keywords.pop("q_options", {})
     # get an id
     tag = uuid()
     # build the task package
-    task = QueueTask(
-        id=tag[1],
-        name=given_kwargs.pop("task_name", None) or q_options.pop("task_name", None) or tag[0],
-        func=func,
-        args=args
-    )
-
-    # don't serialize the broker
-    broker = given_kwargs.pop("broker", None) or q_options.pop("broker", None) or get_broker(task.cluster) or get_broker()
-
-    print(broker.list_key)
-
+    task = {
+        "id": tag[1],
+        "name": keywords.pop("task_name", None)
+        or q_options.pop("task_name", None)
+        or tag[0],
+        "func": func,
+        "args": args,
+    }
     # push optionals
     for key in opt_keys:
-        if key in q_options or key in given_kwargs:
-            setattr(task, key, q_options.pop(key, None) or given_kwargs.pop(key, None))
-
+        if q_options and key in q_options:
+            task[key] = q_options[key]
+        elif key in keywords:
+            task[key] = keywords.pop(key)
+    # don't serialize the broker
+    broker = task.pop("broker", None) or get_broker(task.get("cluster"))
+    # overrides
+    if "cached" not in task and Conf.CACHED:
+        task["cached"] = Conf.CACHED
+    if "sync" not in task and Conf.SYNC:
+        task["sync"] = Conf.SYNC
+    if "ack_failure" not in task and Conf.ACK_FAILURES:
+        task["ack_failure"] = Conf.ACK_FAILURES
     # finalize
-    task.kwargs = given_kwargs
+    task["kwargs"] = keywords
+    task["started"] = timezone.now()
     # signal it
     pre_enqueue.send(sender="django_q", task=task)
     # sign it
     pack = SignedPackage.dumps(task)
-    if task.sync:
+    if task.get("sync", False):
         return _sync(pack)
     # push it
     enqueue_id = broker.enqueue(pack)
     logger.info(f"Enqueued [{broker.list_key}] {enqueue_id}")
     logger.debug(f"Pushed {tag}")
-    return task.id
+    return task["id"]
 
 
 def schedule(func, *args, **kwargs):
@@ -105,7 +111,7 @@ def schedule(func, *args, **kwargs):
         raise IntegrityError("A schedule with the same name already exists.")
 
     # create and return the schedule
-    schedule = Schedule(
+    s = Schedule(
         name=name,
         func=func,
         hook=hook,
@@ -119,9 +125,11 @@ def schedule(func, *args, **kwargs):
         cluster=cluster,
         intended_date_kwarg=intended_date_kwarg,
     )
-    schedule.full_clean()
-    schedule.save()
-    return schedule
+    # make sure we trigger validation
+    s.full_clean()
+    s.save()
+    return s
+
 
 def result(task_id, wait=0, cached=Conf.CACHED):
     """
@@ -140,7 +148,7 @@ def result(task_id, wait=0, cached=Conf.CACHED):
     start = time()
     while True:
         r = Task.get_result(task_id)
-        if r:
+        if r is not None:
             return r
         if (time() - start) * 1000 >= wait >= 0:
             break
@@ -157,7 +165,7 @@ def result_cached(task_id, wait=0, broker=None):
     while True:
         r = broker.cache.get(f"{broker.list_key}:{task_id}")
         if r:
-            return SignedPackage.loads(r).result
+            return SignedPackage.loads(r)["result"]
         if (time() - start) * 1000 >= wait >= 0:
             break
         sleep(0.01)
@@ -216,8 +224,8 @@ def result_group_cached(group_id, failures=False, wait=0, count=None, broker=Non
             result_list = []
             for task_key in group_list:
                 task = SignedPackage.loads(broker.cache.get(task_key))
-                if task.has_succeeded or failures:
-                    result_list.append(task.result)
+                if task["success"] or failures:
+                    result_list.append(task["result"])
             return result_list
         if (time() - start) * 1000 >= wait >= 0:
             break
@@ -260,17 +268,17 @@ def fetch_cached(task_id, wait=0, broker=None):
         if r:
             task = SignedPackage.loads(r)
             return Task(
-                id=task.id,
-                name=task.name,
-                func=task.func,
-                hook=task.hook,
-                args=task.args,
-                kwargs=task.kwargs,
-                cluster=task.cluster,
-                started=task.started_at,
-                stopped=task.finished_at,
-                result=task.result,
-                success=task.has_succeeded,
+                id=task["id"],
+                name=task["name"],
+                func=task["func"],
+                hook=task.get("hook"),
+                args=task["args"],
+                kwargs=task["kwargs"],
+                cluster=task.get("cluster"),
+                started=task["started"],
+                stopped=task["stopped"],
+                result=task["result"],
+                success=task["success"],
             )
         if (time() - start) * 1000 >= wait >= 0:
             break
@@ -329,20 +337,20 @@ def fetch_group_cached(group_id, failures=True, wait=0, count=None, broker=None)
             task_list = []
             for task_key in group_list:
                 task = SignedPackage.loads(broker.cache.get(task_key))
-                if task.has_succeeded or failures:
+                if task["success"] or failures:
                     t = Task(
-                        id=task.id,
-                        name=task.name,
-                        func=task.func,
-                        hook=task.hook,
-                        args=task.args,
-                        kwargs=task.kwargs,
-                        cluster=task.cluster,
-                        started=task.started_at,
-                        stopped=task.finished_at,
-                        result=task.result,
-                        group=task.group,
-                        success=task.has_succeeded,
+                        id=task["id"],
+                        name=task["name"],
+                        func=task["func"],
+                        hook=task.get("hook"),
+                        args=task["args"],
+                        kwargs=task["kwargs"],
+                        cluster=task.get("cluster"),
+                        started=task["started"],
+                        stopped=task["stopped"],
+                        result=task["result"],
+                        group=task.get("group"),
+                        success=task["success"],
                     )
                     task_list.append(t)
             return task_list
@@ -379,7 +387,7 @@ def count_group_cached(group_id, failures=False, broker=None):
         failure_count = 0
         for task_key in group_list:
             task = SignedPackage.loads(broker.cache.get(task_key))
-            if not task.has_succeeded:
+            if not task["success"]:
                 failure_count += 1
         return failure_count
 
@@ -755,7 +763,19 @@ class AsyncTask:
 
 def _sync(pack):
     """Simulate a package travelling through the cluster."""
-    task = SignedPackage.loads(pack)
-    run_cluster_once(workers=1, tasks=[task])
+    from django_q.monitor import monitor
+    from django_q.worker import worker
 
-    return task.id
+    task_queue = Queue()
+    result_queue = Queue()
+    task = SignedPackage.loads(pack)
+    task_queue.put(task)
+    task_queue.put("STOP")
+    worker(task_queue, result_queue, Value("f", -1))
+    result_queue.put("STOP")
+    monitor(result_queue)
+    task_queue.close()
+    task_queue.join_thread()
+    result_queue.close()
+    result_queue.join_thread()
+    return task["id"]

django_q/tests/tasks.py

@@ -46,6 +46,10 @@ def hello():
     return "hello"
 
 
+def return_falsy_value():
+    return []
+
+
 def result(obj):
     print(f"RESULT HOOK {obj.name} : {obj.result()}")
 

django_q/tests/test_admin.py

@@ -64,7 +64,7 @@ def test_admin_views(admin_client, monkeypatch):
 
     # resubmit the failure
     url = reverse("admin:django_q_failure_changelist")
-    data = {"action": "retry_failed", "_selected_action": [f.pk]}
+    data = {"action": "resubmit_task", "_selected_action": [f.pk]}
     response = admin_client.post(url, data)
     assert response.status_code == 302
     assert Failure.objects.filter(name=f.id).exists() is False
@@ -84,3 +84,10 @@ def test_admin_views(admin_client, monkeypatch):
     data = {"post": "yes"}
     response = admin_client.post(url, data)
     assert response.status_code == 302
+    # Resubmit a successful task.
+    url = reverse("admin:django_q_success_changelist")
+    data = {"action": "resubmit_task", "_selected_action": [t.pk]}
+    initial_queue_count = OrmQ.objects.count()
+    response = admin_client.post(url, data)
+    assert response.status_code == 302
+    assert OrmQ.objects.count() > initial_queue_count

django_q/tests/test_cached.py

@@ -1,11 +1,12 @@
-from django_q.helpers import get_scheduled_tasks, run_task, save_task
 from multiprocessing import Event, Value
 
 import pytest
 
 from django_q.brokers import get_broker
 from django_q.conf import Conf
-from queue import Queue
+from django_q.monitor import monitor
+from django_q.pusher import pusher
+from django_q.queues import Queue
 from django_q.tasks import (
     AsyncTask,
     Chain,
@@ -21,6 +22,7 @@ from django_q.tasks import (
     result,
     result_group,
 )
+from django_q.worker import worker
 
 
 @pytest.fixture
@@ -54,14 +56,20 @@ def test_cached(broker):
     # run a single inline cluster
     task_count = 17
     assert broker.queue_size() == task_count
-    tasks = []
-    for task in range(17):
-        tasks += get_scheduled_tasks(broker=broker)
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
+    for i in range(task_count):
+        pusher(task_queue, stop_event, broker=broker)
     assert broker.queue_size() == 0
-    assert len(tasks) == task_count
-    for task in tasks:
-        run_task(task=task)
-        save_task(task=task, broker=broker)
+    assert task_queue.qsize() == task_count
+    task_queue.put("STOP")
+    result_queue = Queue()
+    worker(task_queue, result_queue, Value("f", -1))
+    assert result_queue.qsize() == task_count
+    result_queue.put("STOP")
+    monitor(result_queue)
+    assert result_queue.qsize() == 0
     # assert results
     assert result(task_id, wait=500, cached=True) == -1
     assert fetch(task_id, wait=500, cached=True).result == -1
@@ -155,7 +163,6 @@ def test_chain(broker):
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_asynctask_class(broker, monkeypatch):
     broker.purge_queue()
     broker.cache.clear()

django_q/tests/test_cluster.py

@@ -1,8 +1,4 @@
-from django_q.worker import Worker
-from django_q.queue_task import QueueTask
-from django_q.helpers import get_scheduled_tasks, run_cluster_once, run_task, save_task
 import os
-import copy
 import sys
 import threading
 import uuid as uuidlib
@@ -20,7 +16,9 @@ from django_q.cluster import Cluster, Sentinel
 from django_q.conf import Conf
 from django_q.humanhash import DEFAULT_WORDLIST, uuid
 from django_q.models import Success, Task
-from queue import Queue
+from django_q.monitor import monitor, save_task
+from django_q.pusher import pusher
+from django_q.queues import Queue
 from django_q.signals import post_execute, pre_enqueue, pre_execute
 from django_q.status import Stat
 from django_q.tasks import (
@@ -33,8 +31,9 @@ from django_q.tasks import (
     result,
     result_group,
 )
-from django_q.tests.tasks import multiply, TaskError
+from django_q.tests.tasks import TaskError, multiply
 from django_q.utils import add_months, add_years
+from django_q.worker import worker
 
 myPath = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, myPath + "/../")
@@ -72,21 +71,42 @@ def test_sync_raise_exception(broker):
         async_task("django_q.tests.tasks.raise_exception", broker=broker, sync=True)
 
 
-# @pytest.mark.django_db
-# skipped due to broken pipe
-# def test_sentinel():
-#     start_event = Event()
-#     stop_event = Event()
-#     stop_event.set()
-#     cluster_id = uuidlib.uuid4()
-#     s = Sentinel(
-#         stop_event,
-#         start_event,
-#         cluster_id=cluster_id,
-#         broker=get_broker("sentinel_test:q"),
-#     )
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPING
+@pytest.mark.django_db
+def test_cluster_initial(broker):
+    broker.list_key = "initial_test:q"
+    broker.delete_queue()
+    c = Cluster(broker=broker)
+    assert c.sentinel is None
+    assert c.stat.status == Conf.STOPPED
+    assert c.start() > 0
+    assert c.sentinel.is_alive() is True
+    assert c.is_running
+    assert c.is_stopping is False
+    assert c.is_starting is False
+    sleep(0.5)
+    stat = c.stat
+    assert stat.status == Conf.IDLE
+    assert c.stop() is True
+    assert c.sentinel.is_alive() is False
+    assert c.has_stopped
+    assert c.stop() is False
+    broker.delete_queue()
+
+
+@pytest.mark.django_db
+def test_sentinel():
+    start_event = Event()
+    stop_event = Event()
+    stop_event.set()
+    cluster_id = uuidlib.uuid4()
+    s = Sentinel(
+        stop_event,
+        start_event,
+        cluster_id=cluster_id,
+        broker=get_broker("sentinel_test:q"),
+    )
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
 
 
 @pytest.mark.django_db
@@ -97,19 +117,54 @@ def test_cluster(broker):
         "django_q.tests.tasks.count_letters", DEFAULT_WORDLIST, broker=broker
     )
     assert broker.queue_size() == 1
+    task_queue = Queue()
+    assert task_queue.qsize() == 0
+    result_queue = Queue()
+    assert result_queue.qsize() == 0
+    event = Event()
+    event.set()
     # Test push
-    tasks = get_scheduled_tasks(broker=broker)
-    assert len(tasks) == 1
+    pusher(task_queue, event, broker=broker)
+    assert task_queue.qsize() == 1
     assert queue_size(broker=broker) == 0
     # Test work
-    task = run_task(tasks[0])
+    task_queue.put("STOP")
+    worker(task_queue, result_queue, Value("f", -1))
+    assert task_queue.qsize() == 0
+    assert result_queue.qsize() == 1
     # Test monitor
-    save_task(task=task)
+    result_queue.put("STOP")
+    monitor(result_queue)
+    assert result_queue.qsize() == 0
     # check result
-    assert result(task.id) == 1506
+    assert result(task) == 1506
     broker.delete_queue()
 
 
+@pytest.mark.django_db
+def test_results(broker):
+    broker.list_key = "cluster_test:q"
+    broker.delete_queue()
+    a = async_task(
+        "django_q.tests.tasks.return_falsy_value",
+        broker=broker,
+    )
+
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
+    pusher(task_queue, stop_event, broker=broker)
+    task_queue.put("STOP")
+    result_queue = Queue()
+    worker(task_queue, result_queue, Value("f", -1))
+    result_queue.put("STOP")
+    monitor(result_queue)
+
+    # should not loop indefinitely when a real value is returned
+    value = result(a, wait=-1)
+    assert value == []
+
+
 @pytest.mark.django_db
 def test_enqueue(broker, admin_user):
     broker.list_key = "cluster_test:q"
@@ -183,14 +238,15 @@ def test_enqueue(broker, admin_user):
     # run the cluster to execute the tasks
     task_count = 10
     assert broker.queue_size() == task_count
+    task_queue = Queue()
     stop_event = Event()
     stop_event.set()
     # push the tasks
-    tasks = []
     for _ in range(task_count):
-        tasks += get_scheduled_tasks(broker=broker)
+        pusher(task_queue, stop_event, broker=broker)
     assert broker.queue_size() == 0
-    assert len(tasks) == task_count
+    assert task_queue.qsize() == task_count
+    task_queue.put("STOP")
     # test wait timeout
     assert result(j, wait=10) is None
     assert fetch(j, wait=10) is None
@@ -199,10 +255,13 @@ def test_enqueue(broker, admin_user):
     assert fetch_group("test_j", wait=10) is None
     assert fetch_group("test_j", count=2, wait=10) is None
     # let a worker handle them
-    for task in tasks:
-        run_task(task=task)
-        save_task(task=task)
-
+    result_queue = Queue()
+    worker(task_queue, result_queue, Value("f", -1))
+    assert result_queue.qsize() == task_count
+    result_queue.put("STOP")
+    # store the results
+    monitor(result_queue)
+    assert result_queue.qsize() == 0
     # Check the results
     # task a
     result_a = fetch(a)
@@ -228,11 +287,10 @@ def test_enqueue(broker, admin_user):
     assert result_e.success is True
     assert result(e) is None
     # task f
-    # @TODO: fix this
-    # result_f = fetch(f)
-    # assert result_f is not None
-    # assert result_f.success is True
-    # assert result(f) == 1506
+    result_f = fetch(f)
+    assert result_f is not None
+    assert result_f.success is True
+    assert result(f) == 1506
     # task g
     result_g = fetch(g)
     assert result_g is not None
@@ -275,153 +333,152 @@ def test_enqueue(broker, admin_user):
     broker.delete_queue()
 
 
-# @pytest.mark.django_db
-# @pytest.mark.parametrize(
-#     "cluster_config_timeout, async_task_kwargs",
-#     (
-#         (1, {}),
-#         (10, {"timeout": 1}),
-#         (None, {"timeout": 1}),
-#     ),
-# )
-# def test_timeout(broker, cluster_config_timeout, async_task_kwargs):
-#     # set up the Sentinel
-#     broker.list_key = "timeout_test:q"
-#     broker.purge_queue()
-#     async_task("time.sleep", 5, broker=broker, **async_task_kwargs)
-#     start_event = Event()
-#     stop_event = Event()
-#     cluster_id = uuidlib.uuid4()
-#     # Set a timer to stop the Sentinel
-#     threading.Timer(3, stop_event.set).start()
-#     s = Sentinel(
-#         stop_event,
-#         start_event,
-#         cluster_id=cluster_id,
-#         broker=broker,
-#         timeout=cluster_config_timeout,
-#     )
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPED
-#     assert s.reincarnations == 1
-#     broker.delete_queue()
-
-
-# @pytest.mark.django_db
-# @pytest.mark.parametrize(
-#     "cluster_config_timeout, async_task_kwargs",
-#     (
-#         (5, {}),
-#         (10, {"timeout": 5}),
-#         (1, {"timeout": 5}),
-#         (None, {"timeout": 5}),
-#     ),
-# )
-# def test_timeout_task_finishes(broker, cluster_config_timeout, async_task_kwargs):
-#     # set up the Sentinel
-#     broker.list_key = "timeout_test:q"
-#     broker.purge_queue()
-#     async_task("time.sleep", 3, broker=broker, **async_task_kwargs)
-#     start_event = Event()
-#     stop_event = Event()
-#     cluster_id = uuidlib.uuid4()
-#     # Set a timer to stop the Sentinel
-#     threading.Timer(6, stop_event.set).start()
-#     s = Sentinel(
-#         stop_event,
-#         start_event,
-#         cluster_id=cluster_id,
-#         broker=broker,
-#         timeout=cluster_config_timeout,
-#     )
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPED
-#     assert s.reincarnations == 0
-#     broker.delete_queue()
-
-
-# @pytest.mark.django_db
-# def test_recycle(broker, monkeypatch):
-#     # set up the Sentinel
-#     broker.list_key = "test_recycle_test:q"
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     start_event = Event()
-#     stop_event = Event()
-#     cluster_id = uuidlib.uuid4()
-#     # override settings
-#     monkeypatch.setattr(Conf, "RECYCLE", 2)
-#     monkeypatch.setattr(Conf, "WORKERS", 1)
-#     # set a timer to stop the Sentinel
-#     threading.Timer(3, stop_event.set).start()
-#     s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPED
-#     assert s.reincarnations == 1
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     task_queue = Queue()
-#     result_queue = Queue()
-#     # push two tasks
-#     # pusher(task_queue, stop_event, broker=broker)
-#     # pusher(task_queue, stop_event, broker=broker)
-#     # worker should exit on recycle
-#     # worker(task_queue, result_queue, Value("f", -1))
-#     # check if the work has been done
-#     assert result_queue.qsize() == 2
-#     # save_limit test
-#     monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
-#     result_queue.put("STOP")
-#     # run monitor
-#     # monitor(result_queue)
-#     assert Success.objects.count() == Conf.SAVE_LIMIT
-#     broker.delete_queue()
-
-
-# @pytest.mark.django_db
-# def test_save_limit_per_func(broker, monkeypatch):
-#     # set up the Sentinel
-#     broker.list_key = "test_recycle_test:q"
-#     async_task("django_q.tests.tasks.hello", broker=broker)
-#     async_task("django_q.tests.tasks.countdown", 2, broker=broker)
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     start_event = Event()
-#     stop_event = Event()
-#     cluster_id = uuidlib.uuid4()
-#     task_queue = Queue()
-#     result_queue = Queue()
-#     # override settings
-#     monkeypatch.setattr(Conf, "RECYCLE", 3)
-#     monkeypatch.setattr(Conf, "WORKERS", 1)
-#     # set a timer to stop the Sentinel
-#     threading.Timer(3, stop_event.set).start()
-#     # for i in range(3):
-#     #     pusher(task_queue, stop_event, broker=broker)
-#     # worker(task_queue, result_queue, Value("f", -1))
-#     s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPED
-#     # worker should exit on recycle
-#     # check if the work has been done
-#     assert result_queue.qsize() == 3
-#     # save_limit test
-#     monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
-#     monkeypatch.setattr(Conf, "SAVE_LIMIT_PER", "func")
-#     result_queue.put("STOP")
-#     # run monitor
-#     # monitor(result_queue)
-#     assert Success.objects.count() == 3
-#     assert set(Success.objects.filter().values_list("func", flat=True)) == {
-#         "django_q.tests.tasks.countdown",
-#         "django_q.tests.tasks.hello",
-#         "django_q.tests.tasks.multiply",
-#     }
-#     broker.delete_queue()
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "cluster_config_timeout, async_task_kwargs",
+    (
+        (1, {}),
+        (10, {"timeout": 1}),
+        (None, {"timeout": 1}),
+    ),
+)
+def test_timeout(broker, cluster_config_timeout, async_task_kwargs):
+    # set up the Sentinel
+    broker.list_key = "timeout_test:q"
+    broker.purge_queue()
+    async_task("time.sleep", 5, broker=broker, **async_task_kwargs)
+    start_event = Event()
+    stop_event = Event()
+    cluster_id = uuidlib.uuid4()
+    # Set a timer to stop the Sentinel
+    threading.Timer(3, stop_event.set).start()
+    s = Sentinel(
+        stop_event,
+        start_event,
+        cluster_id=cluster_id,
+        broker=broker,
+        timeout=cluster_config_timeout,
+    )
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
+    assert s.reincarnations == 1
+    broker.delete_queue()
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "cluster_config_timeout, async_task_kwargs",
+    (
+        (5, {}),
+        (10, {"timeout": 5}),
+        (1, {"timeout": 5}),
+        (None, {"timeout": 5}),
+    ),
+)
+def test_timeout_task_finishes(broker, cluster_config_timeout, async_task_kwargs):
+    # set up the Sentinel
+    broker.list_key = "timeout_test:q"
+    broker.purge_queue()
+    async_task("time.sleep", 3, broker=broker, **async_task_kwargs)
+    start_event = Event()
+    stop_event = Event()
+    cluster_id = uuidlib.uuid4()
+    # Set a timer to stop the Sentinel
+    threading.Timer(6, stop_event.set).start()
+    s = Sentinel(
+        stop_event,
+        start_event,
+        cluster_id=cluster_id,
+        broker=broker,
+        timeout=cluster_config_timeout,
+    )
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
+    assert s.reincarnations == 0
+    broker.delete_queue()
+
+
+@pytest.mark.django_db
+def test_recycle(broker, monkeypatch):
+    # set up the Sentinel
+    broker.list_key = "test_recycle_test:q"
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    start_event = Event()
+    stop_event = Event()
+    cluster_id = uuidlib.uuid4()
+    # override settings
+    monkeypatch.setattr(Conf, "RECYCLE", 2)
+    monkeypatch.setattr(Conf, "WORKERS", 1)
+    # set a timer to stop the Sentinel
+    threading.Timer(3, stop_event.set).start()
+    s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
+    assert s.reincarnations == 1
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    task_queue = Queue()
+    result_queue = Queue()
+    # push two tasks
+    pusher(task_queue, stop_event, broker=broker)
+    pusher(task_queue, stop_event, broker=broker)
+    # worker should exit on recycle
+    worker(task_queue, result_queue, Value("f", -1))
+    # check if the work has been done
+    assert result_queue.qsize() == 2
+    # save_limit test
+    monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
+    result_queue.put("STOP")
+    # run monitor
+    monitor(result_queue)
+    assert Success.objects.count() == Conf.SAVE_LIMIT
+    broker.delete_queue()
+
+
+@pytest.mark.django_db
+def test_save_limit_per_func(broker, monkeypatch):
+    # set up the Sentinel
+    broker.list_key = "test_recycle_test:q"
+    async_task("django_q.tests.tasks.hello", broker=broker)
+    async_task("django_q.tests.tasks.countdown", 2, broker=broker)
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    start_event = Event()
+    stop_event = Event()
+    cluster_id = uuidlib.uuid4()
+    task_queue = Queue()
+    result_queue = Queue()
+    # override settings
+    monkeypatch.setattr(Conf, "RECYCLE", 3)
+    monkeypatch.setattr(Conf, "WORKERS", 1)
+    # set a timer to stop the Sentinel
+    threading.Timer(3, stop_event.set).start()
+    for i in range(3):
+        pusher(task_queue, stop_event, broker=broker)
+    worker(task_queue, result_queue, Value("f", -1))
+    s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
+    # worker should exit on recycle
+    # check if the work has been done
+    assert result_queue.qsize() == 3
+    # save_limit test
+    monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
+    monkeypatch.setattr(Conf, "SAVE_LIMIT_PER", "func")
+    result_queue.put("STOP")
+    # run monitor
+    monitor(result_queue)
+    assert Success.objects.count() == 3
+    assert set(Success.objects.filter().values_list("func", flat=True)) == {
+        "django_q.tests.tasks.countdown",
+        "django_q.tests.tasks.hello",
+        "django_q.tests.tasks.multiply",
+    }
+    broker.delete_queue()
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_max_rss(broker, monkeypatch):
     # set up the Sentinel
     broker.list_key = "test_max_rss_test:q"
@@ -436,18 +493,27 @@ def test_max_rss(broker, monkeypatch):
     threading.Timer(3, stop_event.set).start()
     s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
     assert start_event.is_set()
-    assert s.status() == Conf.STOPPING
+    assert s.status() == Conf.STOPPED
     assert s.reincarnations == 1
     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-    for _ in range(2):
-        get_scheduled_tasks(broker=broker)
-    worker = s.pool.workers[0]
-    s.pool.delegate_tasks()
-    assert worker.status == Worker.Status.Idle
+    task_queue = Queue()
+    result_queue = Queue()
+    # push the task
+    pusher(task_queue, stop_event, broker=broker)
+    # worker should exit on recycle
+    worker(task_queue, result_queue, Value("f", -1))
+    # check if the work has been done
+    assert result_queue.qsize() == 1
+    # save_limit test
+    monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
+    result_queue.put("STOP")
+    # run monitor
+    monitor(result_queue)
+    assert Success.objects.count() == Conf.SAVE_LIMIT
+    broker.delete_queue()
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_bad_secret(broker, monkeypatch):
     broker.list_key = "test_bad_secret:q"
     async_task("math.copysign", 1, -1, broker=broker)
@@ -464,8 +530,16 @@ def test_bad_secret(broker, monkeypatch):
     stat = Stat.get_all()
     assert len(stat) == 0
     assert Stat.get(pid=s.parent_pid, cluster_id=cluster_id) is None
-    task = get_scheduled_tasks(broker=broker)
-    assert task == []
+    task_queue = Queue()
+    pusher(task_queue, stop_event, broker=broker)
+    result_queue = Queue()
+    task_queue.put("STOP")
+    worker(
+        task_queue,
+        result_queue,
+        Value("f", -1),
+    )
+    assert result_queue.qsize() == 0
     broker.delete_queue()
 
 
@@ -473,32 +547,32 @@ def test_bad_secret(broker, monkeypatch):
 def test_attempt_count(broker, monkeypatch):
     monkeypatch.setattr(Conf, "MAX_ATTEMPTS", 3)
     tag = uuid()
-    task = QueueTask(
-        id=tag[1],
-        name=tag[0],
-        func="math.copysign",
-        args=(1, -1),
-        kwargs={},
-        started_at=timezone.now(),
-        finished_at=timezone.now(),
-        status=QueueTask.Status.FAILED,
-        result=None,
-    )
+    task = {
+        "id": tag[1],
+        "name": tag[0],
+        "func": "math.copysign",
+        "args": (1, -1),
+        "kwargs": {},
+        "started": timezone.now(),
+        "stopped": timezone.now(),
+        "success": False,
+        "result": None,
+    }
     # initial save - no success
     save_task(task, broker)
-    assert Task.objects.filter(id=task.id).exists()
-    saved_task = Task.objects.get(id=task.id)
+    assert Task.objects.filter(id=task["id"]).exists()
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.attempt_count == 1
     sleep(0.5)
     # second save
-    task.finished_at = timezone.now()
+    task["stopped"] = timezone.now()
     save_task(task, broker)
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.attempt_count == 2
     # third save -
-    task.finished_at = timezone.now()
+    task["stopped"] = timezone.now()
     save_task(task, broker)
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.attempt_count == 3
     # task should be removed from queue
     assert broker.queue_size() == 0
@@ -507,43 +581,43 @@ def test_attempt_count(broker, monkeypatch):
 @pytest.mark.django_db
 def test_update_failed(broker):
     tag = uuid()
-    task = QueueTask(
-        id=tag[1],
-        name=tag[0],
-        func="math.copysign",
-        args=(1, -1),
-        kwargs={},
-        started_at=timezone.now(),
-        finished_at=timezone.now(),
-        status=QueueTask.Status.FAILED,
-        result=None,
-    )
+    task = {
+        "id": tag[1],
+        "name": tag[0],
+        "func": "math.copysign",
+        "args": (1, -1),
+        "kwargs": {},
+        "started": timezone.now(),
+        "stopped": timezone.now(),
+        "success": False,
+        "result": None,
+    }
     # initial save - no success
     save_task(task, broker)
-    assert Task.objects.filter(id=task.id).exists()
-    saved_task = Task.objects.get(id=task.id)
+    assert Task.objects.filter(id=task["id"]).exists()
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.success is False
     sleep(0.5)
     # second save - no success
-    old_stopped = task.finished_at
-    task.finished_at = timezone.now()
+    old_stopped = task["stopped"]
+    task["stopped"] = timezone.now()
     save_task(task, broker)
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.stopped > old_stopped
     # third save - success
-    task.finished_at = timezone.now()
-    task.result = "result"
-    task.status = QueueTask.Status.SUCCESS
+    task["stopped"] = timezone.now()
+    task["result"] = "result"
+    task["success"] = True
     save_task(task, broker)
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.success is True
     # fourth save - no success
-    task.result = None
-    task.status = QueueTask.Status.FAILED
-    task.finished_at = old_stopped
+    task["result"] = None
+    task["success"] = False
+    task["stopped"] = old_stopped
     save_task(task, broker)
     # should not overwrite success
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.success is True
     assert saved_task.result == "result"
 
@@ -560,40 +634,47 @@ def test_acknowledge_failure_override():
             self.acknowledgements[task_id] = count + 1
 
     tag = uuid()
-    task_fail_ack = QueueTask(
-        id=tag[1],
-        name=tag[0],
-        ack_id="test_fail_ack_id",
-        ack_failure=True,
-        func="math.copysign",
-        args=(1, -1),
-        kwargs={},
-        started_at=timezone.now(),
-        finished_at=timezone.now(),
-        status=QueueTask.Status.SUCCESS,
-        result=None,
+    task_fail_ack = {
+        "id": tag[1],
+        "name": tag[0],
+        "ack_id": "test_fail_ack_id",
+        "ack_failure": True,
+        "func": "math.copysign",
+        "args": (1, -1),
+        "kwargs": {},
+        "started": timezone.now(),
+        "stopped": timezone.now(),
+        "success": False,
+        "result": None,
+    }
+
+    tag = uuid()
+    task_fail_no_ack = task_fail_ack.copy()
+    task_fail_no_ack.update(
+        {"id": tag[1], "name": tag[0], "ack_id": "test_fail_no_ack_id"}
     )
+    del task_fail_no_ack["ack_failure"]
 
     tag = uuid()
-    task_fail_no_ack = copy.deepcopy(task_fail_ack)
-    task_fail_no_ack.id = tag[1]
-    task_fail_no_ack.name = tag[0]
-    task_fail_no_ack.ack_id = None
-    task_fail_no_ack.ack_failure = False
-
-    tag = uuid()
-    task_success_ack = copy.deepcopy(task_fail_ack)
-    task_success_ack.id = tag[1]
-    task_success_ack.name = tag[0]
-    task_success_ack.ack_id = "test_success_ack_id"
-    task_success_ack.status = QueueTask.Status.SUCCESS
-    task_success_ack.ack_failure = False
+    task_success_ack = task_fail_ack.copy()
+    task_success_ack.update(
+        {
+            "id": tag[1],
+            "name": tag[0],
+            "ack_id": "test_success_ack_id",
+            "success": True,
+        }
+    )
+    del task_success_ack["ack_failure"]
 
+    result_queue = Queue()
+    result_queue.put(task_fail_ack)
+    result_queue.put(task_fail_no_ack)
+    result_queue.put(task_success_ack)
+    result_queue.put("STOP")
     broker = VerifyAckMockBroker(list_key="key")
 
-    save_task(task_fail_ack, broker=broker)
-    save_task(task_fail_no_ack, broker=broker)
-    save_task(task_success_ack, broker=broker)
+    monitor(result_queue, broker)
 
     assert broker.acknowledgements.get("test_fail_ack_id") == 1
     assert broker.acknowledgements.get("test_fail_no_ack_id") is None
@@ -606,7 +687,7 @@ class TestSignals:
         broker.list_key = "pre_enqueue_test:q"
         broker.delete_queue()
         self.signal_was_called: bool = False
-        self.task = None
+        self.task: Optional[dict] = None
 
         def handler(sender, task, **kwargs):
             self.signal_was_called = True
@@ -615,7 +696,7 @@ class TestSignals:
         pre_enqueue.connect(handler)
         task_id = async_task("math.copysign", 1, -1, broker=broker)
         assert self.signal_was_called is True
-        assert self.task.id == task_id
+        assert self.task.get("id") == task_id
         pre_enqueue.disconnect(handler)
         broker.delete_queue()
 
@@ -624,7 +705,7 @@ class TestSignals:
         broker.list_key = "pre_execute_test:q"
         broker.delete_queue()
         self.signal_was_called: bool = False
-        self.task = None
+        self.task: Optional[dict] = None
         self.func = None
 
         def handler(sender, task, func, **kwargs):
@@ -634,19 +715,27 @@ class TestSignals:
 
         pre_execute.connect(handler)
         task_id = async_task("math.copysign", 1, -1, broker=broker)
-        run_cluster_once(workers=1, broker=broker)
+        task_queue = Queue()
+        result_queue = Queue()
+        event = Event()
+        event.set()
+        pusher(task_queue, event, broker=broker)
+        task_queue.put("STOP")
+        worker(task_queue, result_queue, Value("f", -1))
+        result_queue.put("STOP")
+        monitor(result_queue, broker)
         broker.delete_queue()
-        assert self.task.id == task_id
         assert self.signal_was_called is True
-        assert self.func == 'math.copysign'
+        assert self.task.get("id") == task_id
+        assert self.func == copysign
         pre_execute.disconnect(handler)
 
     @pytest.mark.django_db
     def test_post_execute_signal(self, broker):
         broker.list_key = "post_execute_test:q"
         broker.delete_queue()
-        self.signal_was_called = False
-        self.task = None
+        self.signal_was_called: bool = False
+        self.task: Optional[dict] = None
         self.func = None
 
         def handler(sender, task, **kwargs):
@@ -655,11 +744,19 @@ class TestSignals:
 
         post_execute.connect(handler)
         task_id = async_task("math.copysign", 1, -1, broker=broker)
-        run_cluster_once(workers=1, broker=broker)
+        task_queue = Queue()
+        result_queue = Queue()
+        event = Event()
+        event.set()
+        pusher(task_queue, event, broker=broker)
+        task_queue.put("STOP")
+        worker(task_queue, result_queue, Value("f", -1))
+        result_queue.put("STOP")
+        monitor(result_queue, broker)
         broker.delete_queue()
         assert self.signal_was_called is True
-        assert self.task.id == task_id
-        assert self.task.result == -1
+        assert self.task.get("id") == task_id
+        assert self.task.get("result") == -1
         post_execute.disconnect(handler)
 
 

django_q/tests/test_commands.py

@@ -8,7 +8,6 @@ def test_qcluster():
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_qmonitor():
     call_command("qmonitor", run_once=True)
 
@@ -21,7 +20,6 @@ def test_qinfo():
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_qmemory():
     call_command("qmemory", run_once=True)
     call_command("qmemory", workers=True, run_once=True)

django_q/tests/test_monitor.py

@@ -0,0 +1,52 @@
+import uuid
+
+import pytest
+
+from django_q.brokers import get_broker
+from django_q.cluster import Cluster
+from django_q.conf import Conf
+from django_q.monitor_terminal import get_ids, info, monitor
+from django_q.status import Stat
+from django_q.tasks import async_task
+
+
+@pytest.mark.django_db
+def test_monitor(monkeypatch):
+    cluster_id = uuid.uuid4()
+    assert Stat.get(pid=0, cluster_id=cluster_id).sentinel == 0
+    c = Cluster()
+    c.start()
+    stats = monitor(run_once=True)
+    assert get_ids() is True
+    c.stop()
+    assert len(stats) > 0
+    found_c = False
+    for stat in stats:
+        if stat.cluster_id == c.cluster_id:
+            found_c = True
+            assert stat.uptime() > 0
+            assert stat.empty_queues() is True
+            break
+    assert found_c
+    # test lock size
+    monkeypatch.setattr(Conf, "ORM", "default")
+    b = get_broker("monitor_test")
+    b.enqueue("test")
+    b.dequeue()
+    assert b.lock_size() == 1
+    monitor(run_once=True, broker=b)
+    b.delete_queue()
+
+
+@pytest.mark.django_db
+def test_info():
+    info()
+    do_sync()
+    info()
+    for _ in range(24):
+        do_sync()
+    info()
+
+
+def do_sync():
+    async_task("django_q.tests.tasks.countdown", 1, sync=True, save=True)

django_q/tests/test_scheduler.py

@@ -2,10 +2,9 @@ import os
 from datetime import datetime, timedelta
 from multiprocessing import Event, Value
 from unittest import mock
-from django_q.utils import localtime
 
-import pytest
 import django
+import pytest
 from django.core.exceptions import ValidationError
 from django.db import IntegrityError
 from django.test import override_settings
@@ -13,9 +12,11 @@ from django.utils import timezone
 from django.utils.timezone import is_naive
 
 from django_q.brokers import Broker, get_broker
-from django_q.helpers import run_scheduler_once, get_scheduled_tasks, save_task, run_task
 from django_q.conf import Conf
-from queue import Queue
+from django_q.monitor import monitor
+from django_q.pusher import pusher
+from django_q.queues import Queue
+from django_q.scheduler import scheduler
 from django_q.tasks import Schedule, fetch
 from django_q.tasks import schedule as create_schedule
 from django_q.tests.settings import BASE_DIR
@@ -23,7 +24,8 @@ from django_q.tests.testing_utilities.multiple_database_routers import (
     TestingMultipleAppsDatabaseRouter,
     TestingReplicaDatabaseRouter,
 )
-from django_q.utils import add_months
+from django_q.utils import add_months, localtime
+from django_q.worker import worker
 
 if django.VERSION < (4, 0):
     # pytz is the default in django 3.2. Remove when no support for 3.2
@@ -86,7 +88,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     # 28th of March 2021 is the day when sunlight saving starts (at 2 am)
 
     monkeypatch.setattr(Conf, "TIME_ZONE", "Europe/Amsterdam")
-    tz = ZoneInfo('Europe/Amsterdam')
+    tz = ZoneInfo("Europe/Amsterdam")
     broker.list_key = "scheduler_test:q"
     # Let's start a schedule at 1 am on the 27th of March. This is in AMS timezone.
     # So, 2021-03-27 00:00:00 when saved (due to TZ being Amsterdam and saved in UTC)
@@ -104,7 +106,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     )
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     # It's now the day after exactly at midnight UTC
@@ -116,7 +118,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     assert str(next_run) == "2021-03-28 01:00:00+01:00"
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -127,7 +129,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     assert str(next_run) == "2021-03-29 01:00:00+02:00"
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -148,7 +150,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     )
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -159,7 +161,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     assert str(next_run) == "2021-10-30 01:00:00+02:00"
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -170,7 +172,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     assert str(next_run) == "2021-10-31 01:00:00+02:00"
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -182,7 +184,6 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     assert str(next_run) == "2021-11-01 01:00:00+01:00"
 
 
-
 @pytest.mark.django_db
 def test_scheduler(broker, monkeypatch):
     broker.list_key = "scheduler_test:q"
@@ -209,15 +210,24 @@ def test_scheduler(broker, monkeypatch):
             repeats=1,
         )
     # run scheduler
-    run_scheduler_once(broker=broker)
-    # get tasks
-    tasks = get_scheduled_tasks(broker=broker)
-    for task in tasks:
-        # let a worker handle them
-        ran_task = run_task(task)
-        # store the results
-        save_task(task=ran_task, broker=broker)
-
+    scheduler(broker=broker)
+    # set up the workflow
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
+    # push it
+    pusher(task_queue, stop_event, broker=broker)
+    assert task_queue.qsize() == 1
+    assert broker.queue_size() == 0
+    task_queue.put("STOP")
+    # let a worker handle them
+    result_queue = Queue()
+    worker(task_queue, result_queue, Value("b", -1))
+    assert result_queue.qsize() == 1
+    result_queue.put("STOP")
+    # store the results
+    monitor(result_queue)
+    assert result_queue.qsize() == 0
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.repeats == 0
     assert schedule.last_run() is not None
@@ -289,7 +299,7 @@ def test_scheduler(broker, monkeypatch):
         )
         assert schedule is not None
         assert schedule.last_run() is None
-        run_scheduler_once(broker=broker)
+        scheduler(broker=broker)
     # via model
     Schedule.objects.create(
         func="django_q.tests.tasks.word_multiply",
@@ -298,7 +308,7 @@ def test_scheduler(broker, monkeypatch):
         schedule_type=Schedule.DAILY,
     )
     # scheduler
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     # ONCE schedule should be deleted
     assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False
     # Catch up On
@@ -312,12 +322,12 @@ def test_scheduler(broker, monkeypatch):
         next_run=timezone.now() - timedelta(hours=12),
         repeats=-1,
     )
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.next_run < now
     # Catch up off
     monkeypatch.setattr(Conf, "CATCH_UP", False)
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.next_run > now
     # Done
@@ -330,7 +340,7 @@ def test_scheduler(broker, monkeypatch):
         word="catch_up",
         schedule_type=Schedule.BIMONTHLY,
     )
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.next_run.date() == add_months(timezone.now(), 2).date()
 
@@ -341,7 +351,7 @@ def test_scheduler(broker, monkeypatch):
         word="catch_up",
         schedule_type=Schedule.BIWEEKLY,
     )
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.next_run.date() == (timezone.now() + timedelta(weeks=2)).date()
     broker.delete_queue()
@@ -359,12 +369,16 @@ def test_scheduler(broker, monkeypatch):
         repeats=1,
     )
     # run scheduler
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
+    # set up the workflow
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
     # push it
-    tasks = get_scheduled_tasks(broker=broker)
+    pusher(task_queue, stop_event, broker=broker)
 
     # queue must be empty
-    assert len(tasks) == 0
+    assert task_queue.qsize() == 0
 
     monkeypatch.setattr(Conf, "CLUSTER_NAME", "default")
     # create a schedule on the same cluster
@@ -379,19 +393,23 @@ def test_scheduler(broker, monkeypatch):
         repeats=1,
     )
     # run scheduler
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
+    # set up the workflow
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
     # push it
-    tasks = get_scheduled_tasks(broker=broker)
+    pusher(task_queue, stop_event, broker=broker)
 
     # queue must contain a task
-    assert len(tasks) == 1
+    assert task_queue.qsize() == 1
 
 
 @pytest.mark.django_db
 def test_intended_schedule_kwarg(broker, monkeypatch):
     broker.list_key = "scheduler_test:q"
     broker.delete_queue()
-    run_date = timezone.now()-timedelta(hours=1)
+    run_date = timezone.now() - timedelta(hours=1)
     schedule = create_schedule(
         "math.copysign",
         1,
@@ -401,36 +419,40 @@ def test_intended_schedule_kwarg(broker, monkeypatch):
         schedule_type=Schedule.HOURLY,
         repeats=1,
         next_run=run_date,
-        intended_date_kwarg='intended_date',
+        intended_date_kwarg="intended_date",
     )
     assert schedule.last_run() is None
-    assert schedule.intended_date_kwarg == 'intended_date'
+    assert schedule.intended_date_kwarg == "intended_date"
     # run scheduler
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     # set up the workflow
-    scheduled_tasks = get_scheduled_tasks(broker=broker)
-    assert len(scheduled_tasks) == 1
-    task = scheduled_tasks[0]
-    assert 'intended_date' in task.kwargs
-    assert task.kwargs['intended_date'] == run_date.isoformat()
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
+    # push it
+    pusher(task_queue, stop_event, broker=broker)
+    assert task_queue.qsize() == 1
+    task = task_queue.get()
+    assert "intended_date" in task["kwargs"]
+    assert task["kwargs"]["intended_date"] == run_date.isoformat()
 
 
-# @override_settings(
-#     DATABASE_ROUTERS=REPLICA_DATABASE_ROUTERS, DATABASES=REPLICA_DATABASES
-# )
-# @pytest.mark.django_db
-# def test_scheduler_atomic_must_specify_the_write_db(
-#     orm_broker: Broker,
-# ):
-#     """
-#     GIVEN a environment with a read/write configured replica database
-#     WHEN the scheduler is called
-#     THEN the transaction must be called with the write database.
-#     """
-#     broker = get_broker(list_key="scheduler_test:q")
-#     with mock.patch("django_q.scheduler.db.transaction") as mocked_db:
-#         run_scheduler_once(broker=broker)
-#         mocked_db.atomic.assert_called_with(using="writable")
+@override_settings(
+    DATABASE_ROUTERS=REPLICA_DATABASE_ROUTERS, DATABASES=REPLICA_DATABASES
+)
+@pytest.mark.django_db
+def test_scheduler_atomic_must_specify_the_write_db(
+    orm_broker: Broker,
+):
+    """
+    GIVEN a environment with a read/write configured replica database
+    WHEN the scheduler is called
+    THEN the transaction must be called with the write database.
+    """
+    broker = get_broker(list_key="scheduler_test:q")
+    with mock.patch("django_q.cluster.db.transaction") as mocked_db:
+        scheduler(broker=broker)
+        mocked_db.atomic.assert_called_with(using="writable")
 
 
 @override_settings(
@@ -447,7 +469,7 @@ def test_scheduler_atomic_must_specify_the_database_based_on_router_redirection(
     """
     broker = get_broker(list_key="scheduler_test:q")
     with mock.patch("django_q.cluster.db.transaction") as mocked_db:
-        run_scheduler_once(broker=broker)
+        scheduler(broker=broker)
         mocked_db.atomic.assert_called_with(using="default")
 
 

django_q/utils.py

@@ -1,11 +1,11 @@
-from datetime import datetime
-from django import db
 import calendar
-from datetime import date
+import inspect
+from datetime import date, datetime
 
 import django
-from django.utils import timezone
+from django import db
 from django.conf import settings
+from django.utils import timezone
 
 from django_q.conf import Conf, logger
 
@@ -45,10 +45,22 @@ def add_years(d, years):
         return d.replace(year=new_date.year, month=new_date.month, day=new_date.day)
 
 
+def get_func_repr(func):
+    # convert func to string
+    if inspect.isfunction(func):
+        return f"{func.__module__}.{func.__name__}"
+    elif inspect.ismethod(func) and hasattr(func.__self__, "__name__"):
+        return (
+            f"{func.__self__.__module__}." f"{func.__self__.__name__}.{func.__name__}"
+        )
+    else:
+        return str(func) if func else None
+
+
 def localtime(value=None) -> datetime:
     """Override for timezone.localtime to deal with naive times and local times"""
     if settings.USE_TZ:
-        if django.VERSION >= (4, 0) and settings.USE_DEPRECATED_PYTZ:
+        if django.VERSION >= (4, 0) and getattr(settings, "USE_DEPRECATED_PYTZ", False):
             import pytz
 
             convert_to_tz = pytz.timezone(Conf.TIME_ZONE)
@@ -74,4 +86,3 @@ def close_old_django_connections():
         )
     else:
         db.close_old_connections()
-

django_q/worker.py

@@ -1,221 +1,127 @@
-import multiprocessing
-from queue import Queue
-from queue import Empty
-from typing import Optional, Tuple, Union
-from django_q.queue_task import QueueTask
-from django.utils import timezone
+import pydoc
 import traceback
+from multiprocessing import Value
+from multiprocessing.process import current_process
+from multiprocessing.queues import Queue
 
-from multiprocessing import Process, Value, current_process
-from django_q.utils import close_old_django_connections
+from django import core
+from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
-from django_q.conf import Conf, logger, setproctitle, error_reporter, resource, psutil
-from django_q.signals import pre_execute
-from django_q.exceptions import TimeoutException, TimeoutHandler
-from django_q.process_manager import ProcessManager
+from django.apps.registry import apps
+
+try:
+    apps.check_apps_ready()
+except core.exceptions.AppRegistryNotReady:
+    import django
+
+    django.setup()
+
+from django_q.conf import Conf, error_reporter, logger, resource, setproctitle
+from django_q.signals import post_spawn, pre_execute
+from django_q.utils import close_old_django_connections, get_func_repr
+
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+try:
+    import setproctitle
+except ModuleNotFoundError:
+    setproctitle = None
 
 
+def worker(
+    task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
+):
+    """
+    Takes a task from the task queue, tries to execute it and puts the result back in
+    the result queue
+    :param timeout: number of seconds wait for a worker to finish.
+    :type task_queue: multiprocessing.Queue
+    :type result_queue: multiprocessing.Queue
+    :type timer: multiprocessing.Value
+    """
+    proc_name = current_process().name
+    logger.info(
+        _("%(proc_name)s ready for work at %(id)s")
+        % {"proc_name": proc_name, "id": current_process().pid}
+    )
+    post_spawn.send(sender="django_q", proc_name=proc_name)
+    if setproctitle:
+        setproctitle.setproctitle(f"qcluster {proc_name} idle")
+    task_count = 0
+    if timeout is None:
+        timeout = -1
+    # Start reading the task queue
+    for task in iter(task_queue.get, "STOP"):
+        result = None
+        timer.value = -1  # Idle
+        task_count += 1
+        f = task["func"]
 
-class Worker(ProcessManager):
-
-    def spawn_process(self) -> Process:
-        """
-        :type target: function or class
-        """
-        self.status = Value("i", Worker.Status.IDLE.value)
-        self.manager_pipe, worker_process_pipe = multiprocessing.Pipe(duplex=True)
-        p = WorkerProcess(args=(self.status, worker_process_pipe))
-        p.daemon = Conf.DAEMONIZE_WORKERS
-        p.start()
-        return p
-
-    def start_task(self, task) -> None:
-        # send task to worker
-        try:
-            self.manager_pipe.send(task)
-        except BrokenPipeError:
-            # recycle process if pipe is broken
-            self.status.value = ProcessManager.Status.RECYCLE.value
-
-
-class Pool:
-    """This will manager the individual workers"""
-    def __init__(self, workers=Conf.WORKERS):
-        self.amount_workers = workers
-        self.workers = []
-        self.task_queue = Queue()
-        self.start_workers()
-
-    def start_workers(self):
-        for __ in range(self.amount_workers):
-            self.workers.append(Worker())
-
-    def get_worker(self, id) -> Optional[Worker]:
-        worker = next((worker for worker in self.workers if worker.id == id), None)
-        if worker is None:
-            logger.error("Couldn't find worker")
-            return
-        return worker
-
-    @property
-    def is_healthy(self):
-        """Checks if all workers are still operating"""
-        return all(worker.is_alive for worker in self.workers)
-
-    @property
-    def is_idle(self):
-        """Checks if all workers are idle"""
-        return all(worker.is_idle for worker in self.workers)
-
-    @property
-    def is_done(self):
-        """Checks if all workers are idle and task queue is empty"""
-        return self.is_idle and self.task_queue.empty()
-
-    def reincarnate_stopped_workers(self):
-        """Reincarnates workers that are not alive anymore"""
-        stopped_workers = [worker for worker in self.workers if not worker.is_alive]
-        for worker in stopped_workers:
-            worker.reincarnate_process()
-
-    def add_task(self, task):
-        self.task_queue.put(task)
-
-    def get_done_workers(self):
-        """Worker tasks that have been completed, but need to be saved to cache/db - to be processed by monitor worker"""
-        return [worker for worker in self.workers if worker.is_done]
-
-    def mark_workers_idle(self, worker_ids):
-        """Mark workers idle when they are ready to be used again"""
-        for worker_id in worker_ids:
-            # We are going to process the result, mark them idle, so they can be used for a different task
-            worker = self.get_worker(id=worker_id)
-            if worker is not None:
-                worker.mark_idle()
-
-    def delegate_tasks(self):
-        available_workers = [worker for worker in self.workers if worker.is_idle]
-        for worker in available_workers:
-            try:
-                task = self.task_queue.get_nowait()
-            except Empty:
-                # if the queue is empty, then just stop
-                break
-            worker.start_task(task)
-
-
-class WorkerProcess(Process):
-
-    @staticmethod
-    def run_task(task) -> Tuple[QueueTask, bool]:
-        # signal execution
-        pre_execute.send(sender="django_q", func=task.func, task=task)
-        task.started_at = timezone.now()
-        try:
-            with TimeoutHandler(timeout=task.timeout):
-                func = task.callable_func()
-                res = func(*task.args, **task.kwargs)
-                result = res
-        except (TimeoutException, Exception) as e:
-            if isinstance(e, TimeoutException):
-                task.status = QueueTask.Status.TIMEOUT
-            else:
-                task.status = QueueTask.Status.FAILED
-            result = f"{e} : {traceback.format_exc()}"
-
-            if error_reporter:
-                error_reporter.report()
-            if task.sync:
-                raise
-            return task
-        else:
-            # succeeded
-            task.status = QueueTask.Status.SUCCESS
-        finally:
-            task.result = result
-            task.finished_at = timezone.now()
-        return task
-
-    def __init__(self, group=None, name=None, args=(), kwargs={}, daemon=None):
-        target = self.processing_tasks
-        super().__init__(group=group, target=target, name=name, args=args, kwargs=kwargs, daemon=daemon)
-
-    def mark_ready(self):
-        self.process_name = current_process().name
-        self.process_id = current_process().pid
-        self.task_count = 0
-        logger.info(
-            _("%(proc_name)s ready for work at %(id)s")
-            % {"proc_name": self.process_name, "id": self.process_id}
-        )
-
-
-    def mark_start_task(self, task):
         # Log task creation and set process name
-        task_desc = (
-            _("%(proc_name)s processing %(task_name)s '%(func_name)s'")
-            % {
-                "proc_name": self.process_name,
-                "func_name": task.func_name,
-                "task_name": task.name,
-            }
-        )
-        if task.group is not None:
-            task_desc += f" [{task.group}]"
+        # Get the function from the task
+        func_name = get_func_repr(f)
+        task_name = task["name"]
+        task_desc = _("%(proc_name)s processing %(task_name)s '%(func_name)s'") % {
+            "proc_name": proc_name,
+            "func_name": func_name,
+            "task_name": task_name,
+        }
+        if "group" in task:
+            task_desc += f" [{task['group']}]"
         logger.info(task_desc)
 
         if setproctitle:
-            proc_title = f"qcluster {self.process_name} processing {task.name} '{task.func_name}'"
-            if task.group is not None:
-                proc_title += f" [{task.group}]"
+            proc_title = f"qcluster {proc_name} processing {task_name} '{func_name}'"
+            if "group" in task:
+                proc_title += f" [{task['group']}]"
             setproctitle.setproctitle(proc_title)
 
-    def processing_tasks(self, status: Value, pipe):
-        self.mark_ready()
-        while True:
-            task = pipe.recv()
-            if task == "STOP":
-                logger.info(f"Worker {self.process_name} stopped processing")
-                break
-            # got a new task, let's mark it starting
-            self.mark_start_task(task)
+        # if it's not an instance try to get it from the string
+        if not callable(f):
+            # locate() returns None if f cannot be loaded
+            f = pydoc.locate(f)
+        close_old_django_connections()
+        timer_value = task.pop("timeout", timeout)
+        # signal execution
+        pre_execute.send(sender="django_q", func=f, task=task)
+        # execute the payload
+        timer.value = timer_value  # Busy
 
-            # make sure the function actually exists, before we try to run it
-            try:
-                if not task.is_callable:
-                    raise ValueError(f"Function {task.func_name} is not defined")
-            except Exception as e:
-                result = (f"{e} : {traceback.format_exc()}", False)
-                if error_reporter:
-                    error_reporter.report()
-                if task.sync:
-                    raise
-                # stop here, move on to the next one
-                continue
-
-            close_old_django_connections()
-
-            status.value = ProcessManager.Status.BUSY.value
-            task = WorkerProcess.run_task(task)
-            # Add task towards total
-            self.task_count += 1
-
-            # Set to DONE so main process can pick it up
-            status.value = ProcessManager.Status.DONE.value
+        try:
+            if f is None:
+                # raise a meaningfull error if task["func"] is not a valid function
+                raise ValueError(f"Function {task['func']} is not defined")
+            res = f(*task["args"], **task["kwargs"])
+            result = (res, True)
+        except Exception as e:
+            result = (f"{e} : {traceback.format_exc()}", False)
+            if error_reporter:
+                error_reporter.report()
+            if task.get("sync", False):
+                raise
+        with timer.get_lock():
+            # Process result
+            task["result"] = result[0]
+            task["success"] = result[1]
+            task["stopped"] = timezone.now()
+            result_queue.put(task)
+            timer.value = -1  # Idle
             if setproctitle:
-                setproctitle.setproctitle(f"qcluster {self.process_name} completed with task")
-
-            # Recreate a new process if this task has had the max amount of runs or exceeded resources
-            if self.task_count == Conf.RECYCLE or self.rss_check():
-                status.value = ProcessManager.Status.RECYCLE
+                setproctitle.setproctitle(f"qcluster {proc_name} idle")
+            # Recycle
+            if task_count == Conf.RECYCLE or rss_check():
+                timer.value = -2  # Recycled
                 break
+    logger.info(_("%(proc_name)s stopped doing work") % {"proc_name": proc_name})
 
-            pipe.send(task)
 
-    def rss_check(self):
-        if Conf.MAX_RSS:
-            if resource:
-                return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
-            elif psutil:
-                return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
-        return False
+def rss_check():
+    if Conf.MAX_RSS:
+        if resource:
+            return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
+        elif psutil:
+            return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
+    return False

docs/cluster.rst

@@ -11,36 +11,36 @@ Start your cluster using Django's ``manage.py`` command::
 
 You should see the cluster starting ::
 
-    10:57:40 [Q] INFO Q Cluster-31781 starting.
-    10:57:40 [Q] INFO Process-1:1 ready for work at 31784
-    10:57:40 [Q] INFO Process-1:2 ready for work at 31785
-    10:57:40 [Q] INFO Process-1:3 ready for work at 31786
-    10:57:40 [Q] INFO Process-1:4 ready for work at 31787
-    10:57:40 [Q] INFO Process-1:5 ready for work at 31788
-    10:57:40 [Q] INFO Process-1:6 ready for work at 31789
-    10:57:40 [Q] INFO Process-1:7 ready for work at 31790
-    10:57:40 [Q] INFO Process-1:8 ready for work at 31791
-    10:57:40 [Q] INFO Process-1:9 monitoring at 31792
-    10:57:40 [Q] INFO Process-1 guarding cluster at 31783
-    10:57:40 [Q] INFO Process-1:10 pushing tasks at 31793
-    10:57:40 [Q] INFO Q Cluster-31781 running.
+    10:57:40 [Q] INFO Q Cluster freddie-uncle-twenty-ten starting.
+    10:57:40 [Q] INFO Process-ede257774c4444c980ab479f10947acc ready for work at 31784
+    10:57:40 [Q] INFO Process-ed580482da3f42968230baa2e4253e42 ready for work at 31785
+    10:57:40 [Q] INFO Process-8a370dc2bc1d49aa9864e517c9895f74 ready for work at 31786
+    10:57:40 [Q] INFO Process-74912f9844264d1397c6e54476b530c0 ready for work at 31787
+    10:57:40 [Q] INFO Process-b00edb26c6074a6189e5696c60aeb35b ready for work at 31788
+    10:57:40 [Q] INFO Process-b0862965db04479f9784a26639ee51e0 ready for work at 31789
+    10:57:40 [Q] INFO Process-7e8abbb8ca2d4d9bb20a937dd5e2872b ready for work at 31790
+    10:57:40 [Q] INFO Process-b0862965db04479f9784a26639ee51e0 ready for work at 31791
+    10:57:40 [Q] INFO Process-67fa9461ac034736a766cd813f617e62 monitoring at 31792
+    10:57:40 [Q] INFO Process-eac052c646b2459797cee98bdb84c85d guarding cluster at 31783
+    10:57:40 [Q] INFO Process-5d98deb19b1e4b2da2ef1e5bd6824f75 pushing tasks at 31793
+    10:57:40 [Q] INFO Q Cluster freddie-uncle-twenty-ten running.
 
 
 Stopping the cluster with ctrl-c or either the ``SIGTERM`` and ``SIGKILL`` signals, will initiate the :ref:`stop_procedure`::
 
-    16:44:12 [Q] INFO Q Cluster-31781 stopping.
-    16:44:12 [Q] INFO Process-1 stopping cluster processes
-    16:44:13 [Q] INFO Process-1:10 stopped pushing tasks
-    16:44:13 [Q] INFO Process-1:6 stopped doing work
-    16:44:13 [Q] INFO Process-1:4 stopped doing work
-    16:44:13 [Q] INFO Process-1:1 stopped doing work
-    16:44:13 [Q] INFO Process-1:5 stopped doing work
-    16:44:13 [Q] INFO Process-1:7 stopped doing work
-    16:44:13 [Q] INFO Process-1:3 stopped doing work
-    16:44:13 [Q] INFO Process-1:8 stopped doing work
-    16:44:13 [Q] INFO Process-1:2 stopped doing work
-    16:44:14 [Q] INFO Process-1:9 stopped monitoring results
-    16:44:15 [Q] INFO Q Cluster-31781 has stopped.
+    16:44:12 [Q] INFO Q Cluster freddie-uncle-twenty-ten stopping.
+    16:44:12 [Q] INFO Process-eac052c646b2459797cee98bdb84c85d stopping cluster processes
+    16:44:13 [Q] INFO Process-5d98deb19b1e4b2da2ef1e5bd6824f75 stopped pushing tasks
+    16:44:13 [Q] INFO Process-b0862965db04479f9784a26639ee51e0 stopped doing work
+    16:44:13 [Q] INFO Process-7e8abbb8ca2d4d9bb20a937dd5e2872b stopped doing work
+    16:44:13 [Q] INFO Process-b0862965db04479f9784a26639ee51e0 stopped doing work
+    16:44:13 [Q] INFO Process-b00edb26c6074a6189e5696c60aeb35b stopped doing work
+    16:44:13 [Q] INFO Process-74912f9844264d1397c6e54476b530c0 stopped doing work
+    16:44:13 [Q] INFO Process-8a370dc2bc1d49aa9864e517c9895f74 stopped doing work
+    16:44:13 [Q] INFO Process-ed580482da3f42968230baa2e4253e42 stopped doing work
+    16:44:13 [Q] INFO Process-ede257774c4444c980ab479f10947acc stopped doing work
+    16:44:14 [Q] INFO Process-67fa9461ac034736a766cd813f617e62 stopped monitoring results
+    16:44:15 [Q] INFO Q Cluster freddie-uncle-twenty-ten has stopped.
 
 The number of workers, optional timeouts, recycles and cpu_affinity can be controlled via the :doc:`configure` settings.
 

docs/conf.py

@@ -73,9 +73,9 @@ author = "Ilan Steemers, Stan Triepels"
 # built documents.
 #
 # The short X.Y version.
-version = "1.5"
+version = "1.6"
 # The full version, including alpha/beta/rc tags.
-release = "1.5.1"
+release = "1.6.1"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/configure.rst

@@ -348,7 +348,7 @@ To use MongoDB as a message broker you simply provide the connection information
         }
     }
 
-The ``mongo`` dictionary can contain any of the parameters exposed by pymongo's `MongoClient <https://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`__
+The ``mongo`` dictionary can contain any of the parameters exposed by pymongo's `MongoClient <https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`__
 If you want to use a mongodb uri, you can supply it as the ``host`` parameter.
 
 mongo_db

docs/examples.rst

@@ -328,7 +328,7 @@ Requires cache to be enabled. Save file in your Django project's root directory
     # All django stuff has to come after the setup:
     django.setup()
 
-    from django_q.monitor import Stat
+    from django_q.status import Stat
     from django_q.conf import Conf
 
     # Set host and port settings

docs/index.rst

@@ -27,7 +27,7 @@ Features
 -  Rollbar and Sentry support
 
 
-Django Q2 is tested with: Python 3.8, 3.9, 3.10, and 3.11. Works with Django 3.2.x and 4.1.x.
+Django Q2 is tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 3.2.x, 4.1.x, 4.2.x and 5.0.x
 
 Currently available in English, German and French.
 

docs/install.rst

@@ -24,15 +24,31 @@ Installation
 
     $ python manage.py qcluster
 
+
+Migrate from Django-Q to Django-Q2
+----------------------------------
+
+If you have an application with django-q running right now, you can simply swap the libraries and you should be good to go.::
+
+
+    $ pip uninstall django-q # you might have to uninstall django-q add-ons as well
+    $ pip install django-q2
+
+
+Then migrate the database to get the latest tables/fields::
+
+    $ python manage.py migrate
+
+
 Requirements
 ------------
 
-Django Q2 is tested for Python 3.8, 3.9, 3.10 and 3.11
+Django Q2 is tested for Python 3.8, 3.9, 3.10, 3.11 and 3.12
 
 -  `Django <https://www.djangoproject.com>`__
 
     Django Q2 aims to use as much of Django's standard offerings as possible.
-    The code is tested against Django versions `3.2.x` and `4.1.x`.
+    The code is tested against Django versions `3.2.x`, `4.1.x`, `4.2.x` and `5.0.x`.
 
 -  `Django-picklefield <https://github.com/gintas/django-picklefield>`__
 
@@ -41,6 +57,10 @@ Django Q2 is tested for Python 3.8, 3.9, 3.10 and 3.11
 
 Optional
 ~~~~~~~~
+-  `Blessed <https://github.com/jquast/blessed>`__ is used to display the statistics in the terminal::
+
+    $ pip install blessed
+
 -  `Redis-py <https://github.com/andymccurdy/redis-py>`__ client by Andy McCurdy is used  to interface with both the Redis::
 
     $ pip install redis
@@ -95,11 +115,11 @@ Add-ons
 -------
 - `django-q-rollbar <https://github.com/danielwelch/django-q-rollbar>`__ is a Rollbar error reporter::
 
-    $ pip install django-q[rollbar]
+    $ pip install django-q2[rollbar]
 
 - `django-q-sentry <https://github.com/danielwelch/django-q-sentry>`__ is a Sentry error reporter::
 
-    $ pip install django-q[sentry]
+    $ pip install django-q2[sentry]
 
 - `django-q-email <https://github.com/joeyespo/django-q-email>`__ is a compatible Django email backend that will automatically async queue your emails.
 
@@ -125,7 +145,7 @@ Other known issues are:
 
 Python
 ~~~~~~
-Current tests are performed with 3.8, 3.9, 3.10 and 3.11
+Current tests are performed with 3.8, 3.9, 3.10, 3.11 and 3.12
 If you do encounter any regressions with earlier versions, please submit an issue on `github <https://github.com/GDay/django-q2>`__
 
 Open-source packages
@@ -135,8 +155,8 @@ You can reference the `requirements <https://github.com/GDay/django-q2/blob/mast
 
 Django
 ~~~~~~
-We strive to be compatible with last two major version of Django.
-At the moment this means we support the 3.2.x and 4.1.x releases.
+We strive to be compatible with the last two major version of Django.
+At the moment this means we support the 3.2.x, 4.1.x, 4.2.x and 5.0.x releases.
 
-Since we are now no longer supporting Python 2, we can also not support older versions of Django that do not support Python >= 3.6
+Since we are now no longer supporting Python 2, we can also not support older versions of Django that do not support Python >= 3.8
 For this you can always use older releases, but they are no longer maintained.

docs/monitor.rst

@@ -3,6 +3,10 @@ Monitor
 .. py:currentmodule::django_q.monitor
 
 
+.. warning::
+    Blessed needs to be installed to get this to work! See: https://pypi.org/project/blessed/
+
+
 The cluster monitor shows live information about all the Q clusters connected to your project.
 
 Start the monitor with Django's `manage.py` command::
@@ -107,7 +111,7 @@ You can check the status of your clusters straight from your code with the :clas
 
 .. code:: python
 
-    from django_q.monitor import Stat
+    from django_q.status import Stat
 
     for stat in Stat.get_all():
         print(stat.cluster_id, stat.status)

docs/signals.rst

@@ -13,6 +13,12 @@ Before enqueuing a task
 The ``django_q.signals.pre_enqueue`` signal is emitted before a task is
 enqueued. The task dictionary is given as the ``task`` argument.
 
+After spawning a worker process
+"""""""""""""""""""""""""""""""
+
+The ``django_q.signals.post_spawn`` signal is emitted after a worker process has
+spawned. The process name is given as the ``proc_name`` argument (string).
+
 Before executing a task
 """""""""""""""""""""""
 
@@ -37,7 +43,7 @@ Connecting to a Django Q2 signal is done the same as any other Django
 signal::
 
     from django.dispatch import receiver
-    from django_q.signals import pre_enqueue, pre_execute, post_execute
+    from django_q.signals import pre_enqueue, pre_execute, post_execute, post_spawn
 
     @receiver(pre_enqueue)
     def my_pre_enqueue_callback(sender, task, **kwargs):
@@ -51,4 +57,8 @@ signal::
     def my_post_execute_callback(sender, task, **kwargs):
         print(f"Task {task['name']} was executed with result {task['result']}")
 
+    @receiver(post_spawn)
+    def my_post_spawn_callback(sender, proc_name, **kwargs):
+        print(f"Process {proc_name} has spawned")
+
 

docs/tasks.rst

@@ -121,6 +121,7 @@ Optionally you can use the :class:`AsyncTask` class to instantiate a task and ke
     a.run()
 
     # wait indefinitely for the result and print it
+    # don't let the task return `None` or it will wait indefinitely
     print(a.result(wait=-1))
 
     # change the args
@@ -264,7 +265,7 @@ Reference
     Gets the result of a previously executed task
 
     :param str task_id: the uuid or name of the task
-    :param int wait: optional milliseconds to wait for a result. -1 for indefinite
+    :param int wait: optional milliseconds to wait for a result. -1 for indefinite, but be sure the result will not be `None` otherwise it will wait indefinitely!
     :param bool cached: run this against the cache backend.
     :returns: The result of the executed task
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "django-q2"
-version = "1.5.1"
+version = "1.6.1"
 packages = [
     { include = "django_q" },
 ]
@@ -30,6 +30,7 @@ classifiers = [
     'Programming Language :: Python :: 3.9',
     'Programming Language :: Python :: 3.10',
     'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12',
     'Topic :: Internet :: WWW/HTTP',
     'Topic :: System :: Distributed Computing',
     'Topic :: Software Development :: Libraries :: Python Modules',
@@ -44,8 +45,8 @@ include = ['CHANGELOG.md']
 
 
 [tool.poetry.dependencies]
-python = ">=3.8, <4"
-django = ">=3.2"
+python = ">=3.8,<4"
+django = { version = ">=3.2, <6", allow-prereleases = true}
 django-picklefield = "^3.1"
 
 blessed = { version = "^1.19.1", optional = true }
@@ -55,11 +56,12 @@ django-redis = { version = "^5.2.0", optional = true }
 iron-mq = { version = "^0.9", optional = true }
 boto3 = { version = "^1.24.92", optional = true }
 pymongo = { version = "^4.2.0", optional = true }
-croniter = { version = "^1.3.7", optional = true }
+croniter = { version = "^2.0.1", optional = true }
 django-q-rollbar = {version = ">=0.1", optional = true}
 django-q-sentry = {version = ">=0.1", optional = true}
 redis = {version = "^4.3.4", optional = true}
 setproctitle = {version = "^1.3.2", optional = true}
+importlib-metadata = {version = ">=3.6", python = "<3.10"}
 
 
 [tool.poetry.dev-dependencies]