Post spawn docs

Co-authored-by: David Hoover <hooverd@arizona.edu>

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         python-version: [ "3.8", "3.9", "3.10", "3.11" ]
-        django: [ "3.2", "4.1" ]
+        django: [ "3.2", "4.1", "4.2" ]
 
     services:
       disque:

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## [Unreleased](https://github.com/GDay/django-q2/tree/HEAD)
 
+## [v1.5.2](https://github.com/GDay/django-q2/tree/v1.5.2) (2023-04-13)
+
+**Merged pull requests:**
+
+- Added Django 4.2 to the test matrix, fixed deprecation warning https://github.com/GDay/django-q2/pull/89
+- Updated docs to show support for 4.2
+
 ## [v1.5.1](https://github.com/GDay/django-q2/tree/v1.5.1) (2023-04-02)
 
 - Fix release to pipy due to changed org name

README.rst

@@ -40,7 +40,7 @@ Requirements
 -  `Django <https://www.djangoproject.com>`__ > = 3.2
 -  `Django-picklefield <https://github.com/gintas/django-picklefield>`__
 
-Tested with: Python 3.8, 3.9, 3.10, 3.11 Django 3.2.X and 4.1.X
+Tested with: Python 3.8, 3.9, 3.10 and 3.11. Works with Django 3.2.X, 4.1.X and 4.2.X.
 
 Brokers
 ~~~~~~~
@@ -104,6 +104,11 @@ For full configuration options, see the `configuration documentation <https://dj
 Management Commands
 ~~~~~~~~~~~~~~~~~~~
 
+::
+
+    For the management commands to work, you will need to install Blessed: <https://github.com/jquast/blessed>
+
+
 Start a cluster with::
 
     $ python manage.py qcluster

django_q/__init__.py

@@ -1,6 +1,6 @@
 import django
 
-VERSION = (1, 5, 1)
+VERSION = (1, 5, 2)
 
 if django.VERSION < (3, 2):
     default_app_config = "django_q.apps.DjangoQConfig"

django_q/cluster.py

@@ -1,12 +1,12 @@
 # Standard
-from django_q.scheduler import Scheduler
-from django_q.puller import Puller
-from django_q.worker import Pool
+import ast
+import pydoc
 import signal
-from django_q.monitor import Monitor
 import socket
+import traceback
 import uuid
-from multiprocessing import Event, Process, current_process
+from datetime import datetime, timedelta
+from multiprocessing import Event, Process, Value, current_process
 from time import sleep
 
 # Django
@@ -28,14 +28,24 @@ import django_q.tasks
 from django_q.brokers import Broker, get_broker
 from django_q.conf import (
     Conf,
+    croniter,
+    error_reporter,
     get_ppid,
     logger,
     psutil,
     setproctitle,
+    resource,
 )
 from django_q.humanhash import humanize
+from django_q.models import Schedule, Success, Task
+from django_q.queues import Queue
+from django_q.signals import post_execute, post_spawn, pre_execute
+from django_q.signing import BadSignature, SignedPackage
 from django_q.status import Stat, Status
 
+from .utils import get_func_repr, localtime
+
+
 class Cluster:
     def __init__(self, broker: Broker = None):
         # Cluster do not need an init or default broker except for testing,
@@ -148,11 +158,16 @@ class Sentinel:
         self.tob = timezone.now()
         self.stop_event = stop_event
         self.start_event = start_event
+        self.pool_size = Conf.WORKERS
+        self.pool = []
         self.timeout = timeout or Conf.TIMEOUT
-        self.event_out = Event()
-        logger.info(
-            _("%(name)s main at %(id)s") % {"name": self.name, "id": current_process().pid}
+        self.task_queue = (
+            Queue(maxsize=Conf.QUEUE_LIMIT) if Conf.QUEUE_LIMIT else Queue()
         )
+        self.result_queue = Queue()
+        self.event_out = Event()
+        self.monitor = None
+        self.pusher = None
         if start:
             self.start()
 
@@ -169,28 +184,109 @@ class Sentinel:
         if not self.start_event.is_set() and not self.stop_event.is_set():
             return Conf.STARTING
         elif self.start_event.is_set() and not self.stop_event.is_set():
-            if self.monitor.is_idle and self.pool.is_done:
+            if self.result_queue.empty() and self.task_queue.empty():
                 return Conf.IDLE
             return Conf.WORKING
         elif self.stop_event.is_set() and self.start_event.is_set():
-            if self.monitor.is_alive or self.puller.is_alive or len(self.pool.workers) > 0:
+            if self.monitor.is_alive() or self.pusher.is_alive() or len(self.pool) > 0:
                 return Conf.STOPPING
             return Conf.STOPPED
 
+    def spawn_process(self, target, *args) -> Process:
+        """
+        :type target: function or class
+        """
+        p = Process(target=target, args=args)
+        p.daemon = True
+        if target == worker:
+            p.daemon = Conf.DAEMONIZE_WORKERS
+            p.timer = args[2]
+            self.pool.append(p)
+        p.start()
+        return p
+
+    def spawn_pusher(self) -> Process:
+        return self.spawn_process(pusher, self.task_queue, self.event_out, self.broker)
+
+    def spawn_worker(self):
+        self.spawn_process(
+            worker, self.task_queue, self.result_queue, Value("f", -1), self.timeout
+        )
+
+    def spawn_monitor(self) -> Process:
+        return self.spawn_process(monitor, self.result_queue, self.broker)
+
+    def reincarnate(self, process):
+        """
+        :param process: the process to reincarnate
+        :type process: Process or None
+        """
+        # close connections before spawning new process
+        if not Conf.SYNC:
+            db.connections.close_all()
+        if process == self.monitor:
+            self.monitor = self.spawn_monitor()
+            logger.critical(
+                _("reincarnated monitor %(name)s after sudden death")
+                % {"name": process.name}
+            )
+        elif process == self.pusher:
+            self.pusher = self.spawn_pusher()
+            logger.critical(
+                _("reincarnated pusher %(name)s after sudden death")
+                % {"name": process.name}
+            )
+        else:
+            self.pool.remove(process)
+            self.spawn_worker()
+            if process.timer.value == 0:
+                # only need to terminate on timeout, otherwise we risk destabilizing
+                # the queues
+                task_name = ""
+                if psutil:
+                    try:
+                        process_name = psutil.Process(process.pid).name()
+                        name_splits = process_name.split(" ")
+                        task_name = name_splits[3] if len(name_splits) >= 4 and name_splits[2] == "processing" else ""
+                    except psutil.NoSuchProcess:
+                        pass
+                process.terminate()
+                if task_name:
+                    msg = (
+                        _("reincarnated worker %(name)s after timeout while processing task %(task_name)s")
+                        % {"name": process.name, "task_name": task_name}
+                    )
+                else:
+                    msg = (
+                        _("reincarnated worker %(name)s after timeout")
+                        % {"name": process.name}
+                    )
+                logger.critical(msg)
+            elif int(process.timer.value) == -2:
+                logger.info(_("recycled worker %(name)s") % {"name": process.name})
+            else:
+                logger.critical(
+                    _("reincarnated worker %(name)s after death")
+                    % {"name": process.name}
+                )
+
+        self.reincarnations += 1
+
     def spawn_cluster(self):
+        self.pool = []
+        Stat(self).save()
         # close connections before spawning new process
         if not Conf.SYNC:
             db.connections.close_all()
         # spawn worker pool
-        self.pool = Pool()
-        self.puller = Puller()
-        self.monitor = Monitor()
-        self.scheduler = Scheduler()
+        for __ in range(self.pool_size):
+            self.spawn_worker()
+        # spawn auxiliary
+        self.monitor = self.spawn_monitor()
+        self.pusher = self.spawn_pusher()
         # set worker cpu affinity if needed
         if psutil and Conf.CPU_AFFINITY:
-            set_cpu_affinity(Conf.CPU_AFFINITY, [w.process.pid for w in self.pool.workers])
-        Stat(self).save()
-
+            set_cpu_affinity(Conf.CPU_AFFINITY, [w.pid for w in self.pool])
 
     def guard(self):
         logger.info(
@@ -207,102 +303,506 @@ class Sentinel:
             % {"cluster_name": humanize(self.cluster_id.hex) + f" [{self.queue_name()}]"}
         )
         counter = 0
+        cycle = Conf.GUARD_CYCLE  # guard loop sleep in seconds
         # Guard loop. Runs at least once
         while not self.stop_event.is_set() or not counter:
-            # Check if the pool of workers is healthy
-            logger.info("Check if pool is healthy")
-            if not self.pool.is_healthy:
-                # reincarnate workers that died
-                print("reincarnate workers")
-                self.pool.reincarnate_stopped_workers()
-
-            print("Check if puller is healthy")
-            if not self.puller.is_alive:
-                self.puller.reincarnate_process()
-
-            print("Check if monitor is healthy")
-            if not self.monitor.is_alive:
-                self.monitor.reincarnate_process()
-
-            print("Check if scheduler is healthy")
-            if not self.scheduler.is_alive:
-                self.scheduler.reincarnate_process()
-
-
-            print("add tasks and mark workers idle")
-            for worker in self.pool.get_done_workers():
-                # put result in task_queue to be picked up by monitor for processing
-                self.monitor.add_task(worker.get_result())
-                # mark task back to idle or reincarnate to be picked up for a new task
-                if worker.is_recycle:
-                    worker.reincarnate_process()
-                else:
-                    worker.mark_idle()
-
-            # check if monitor has items to process
-            print("run monitor item")
-            self.monitor.run_item()
-
-            print("Add task to worker pool")
-            if self.puller.has_results:
-                self.pool.add_task(self.puller.get_result())
-
-            # delegate tasks to workers that are now available
-            print("delegate tasks")
-            self.pool.delegate_tasks()
-
-            logger.info("sleep")
-            counter += 1
-            sleep(Conf.GUARD_CYCLE)
+            # Check Workers
+            for p in self.pool:
+                with p.timer.get_lock():
+                    # Are you alive?
+                    if not p.is_alive() or p.timer.value == 0:
+                        self.reincarnate(p)
+                        continue
+                    # Decrement timer if work is being done
+                    if p.timer.value > 0:
+                        p.timer.value -= cycle
+            # Check Monitor
+            if not self.monitor.is_alive():
+                self.reincarnate(self.monitor)
+            # Check Pusher
+            if not self.pusher.is_alive():
+                self.reincarnate(self.pusher)
+            # Call scheduler once a minute (or so)
+            counter += cycle
+            if counter >= 30 and Conf.SCHEDULER:
+                counter = 0
+                scheduler(broker=self.broker)
+            # Save current status
             Stat(self).save()
+            sleep(cycle)
         self.stop()
 
     def stop(self):
+        Stat(self).save()
         name = current_process().name
         logger.info(_("%(name)s stopping cluster processes") % {"name": name})
-        # Stopping guard
-        self.stop_event.set()
-        logger.debug(_("Guard has stopped"))
-        # Stop scheduler
-        self.scheduler.stop_scheduler()
+        # Stopping pusher
+        self.event_out.set()
+        # Wait for it to stop
+        while self.pusher.is_alive():
+            sleep(0.1)
+            Stat(self).save()
+        # Put poison pills in the queue
+        for __ in range(len(self.pool)):
+            self.task_queue.put("STOP")
+        self.task_queue.close()
+        # wait for the task queue to empty
+        self.task_queue.join_thread()
+        # Wait for all the workers to exit
+        while len(self.pool):
+            for p in self.pool:
+                if not p.is_alive():
+                    self.pool.remove(p)
+            sleep(0.1)
+            Stat(self).save()
+        # Finally stop the monitor
+        self.result_queue.put("STOP")
+        self.result_queue.close()
+        # Wait for the result queue to empty
+        self.result_queue.join_thread()
+        logger.info(_("%(name)s waiting for the monitor.") % {"name": name})
+        # Wait for everything to close or time out
+        count = 0
+        if not self.timeout:
+            self.timeout = 30
+        while self.status() == Conf.STOPPING and count < self.timeout * 10:
+            sleep(0.1)
+            Stat(self).save()
+            count += 1
+        # Final status
+        Stat(self).save()
 
-        # End all workers gracefully
-        for __ in range(Conf.WORKERS):
-            self.pool.add_task("STOP")
 
-        # make sure the tasks queue in the pool is empty and workers are idle max timeout 20 sec
-        time_passed = 0
-        while not self.pool.is_done and time_passed <= 20:
-            self.monitor.run_item()
-            self.pool.delegate_tasks()
-            time_passed += 0.5
-            sleep(0.5)
-        if time_passed >= 20:
-            logger.error(_("Couldn't terminate tasks within 20 seconds, killing processes now"))
-            for worker in self.pool.workers:
-                worker.process.kill()
+def pusher(task_queue: Queue, event: Event, broker: Broker = None):
+    """
+    Pulls tasks of the broker and puts them in the task queue
+    :type broker:
+    :type task_queue: multiprocessing.Queue
+    :type event: multiprocessing.Event
+    """
+    if not broker:
+        broker = get_broker()
+    proc_name = current_process().name
+    if setproctitle:
+        setproctitle.setproctitle(f"qcluster {proc_name} pusher")
+    logger.info(
+        _("%(name)s pushing tasks at %(id)s")
+        % {"name": proc_name, "id": current_process().pid}
+    )
+    while True:
+        try:
+            task_set = broker.dequeue()
+        except Exception:
+            logger.exception("Failed to pull task from broker")
+            # broker probably crashed. Let the sentinel handle it.
+            sleep(10)
+            break
+        if task_set:
+            for task in task_set:
+                ack_id = task[0]
+                # unpack the task
+                try:
+                    task = SignedPackage.loads(task[1])
+                except (TypeError, BadSignature):
+                    logger.exception("Failed to push task to queue")
+                    broker.fail(ack_id)
+                    continue
+                task["cluster"] = Conf.CLUSTER_NAME  # save actual cluster name to orm task table
+                task["ack_id"] = ack_id
+                task_queue.put(task)
+            logger.debug(
+                _("queueing from %(list_key)s") % {"list_key": broker.list_key}
+            )
+        if event.is_set():
+            break
+    logger.info(_("%(name)s stopped pushing tasks") % {"name": current_process().name})
 
-        logger.debug(_("All tasks were processed and workers where stopped"))
 
-        self.monitor.add_task("STOP")
-        while not self.monitor.is_done:
-            # in the case the monitor was behind, let's run through all
-            self.monitor.run_item()
-            sleep(0.5)
+def monitor(result_queue: Queue, broker: Broker = None):
+    """
+    Gets finished tasks from the result queue and saves them to Django
+    :type broker: brokers.Broker
+    :type result_queue: multiprocessing.Queue
+    """
+    if not broker:
+        broker = get_broker()
+    proc_name = current_process().name
+    if setproctitle:
+        setproctitle.setproctitle(f"qcluster {proc_name} monitor")
+    logger.info(
+        _("%(name)s monitoring at %(id)s") % {"name": proc_name, "id": current_process().pid}
+    )
+    for task in iter(result_queue.get, "STOP"):
+        # save the result
+        if task.get("cached", False):
+            save_cached(task, broker)
+        else:
+            save_task(task, broker)
+        # acknowledge result
+        ack_id = task.pop("ack_id", False)
+        if ack_id and (task["success"] or task.get("ack_failure", False)):
+            broker.acknowledge(ack_id)
+        # signal execution done
+        post_execute.send(sender="django_q", task=task)
+        # log the result
+        info_name = get_func_repr(task["func"])
+        if task["success"]:
+            # log success
+            logger.info(
+                _("Processed '%(info_name)s' (%(task_name)s)")
+                % {"info_name": info_name, "task_name": task["name"]}
+            )
+        else:
+            # log failure
+            logger.error(
+                _("Failed '%(info_name)s' (%(task_name)s) - %(task_result)s")
+                % {
+                    "info_name": info_name,
+                    "task_name": task["name"],
+                    "task_result": task["result"],
+                }
+            )
+    logger.info(_("%(name)s stopped monitoring results") % {"name": proc_name})
 
-        logger.debug(_("All tasks were saved"))
 
-        self.puller.stop_puller()
+def worker(
+    task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
+):
+    """
+    Takes a task from the task queue, tries to execute it and puts the result back in
+    the result queue
+    :param timeout: number of seconds wait for a worker to finish.
+    :type task_queue: multiprocessing.Queue
+    :type result_queue: multiprocessing.Queue
+    :type timer: multiprocessing.Value
+    """
+    proc_name = current_process().name
+    logger.info(
+        _("%(proc_name)s ready for work at %(id)s")
+        % {"proc_name": proc_name, "id": current_process().pid}
+    )
+    post_spawn.send(sender="django_q", proc_name=proc_name)
+    if setproctitle:
+        setproctitle.setproctitle(f"qcluster {proc_name} idle")
+    task_count = 0
+    if timeout is None:
+        timeout = -1
+    # Start reading the task queue
+    for task in iter(task_queue.get, "STOP"):
+        result = None
+        timer.value = -1  # Idle
+        task_count += 1
+        f = task["func"]
 
-        # make sure all processes are terminated
-        for worker in self.pool.workers:
-            worker.process.terminate()
+        # Log task creation and set process name
+        # Get the function from the task
+        func_name = get_func_repr(f)
+        task_name = task["name"]
+        task_desc = (
+            _("%(proc_name)s processing %(task_name)s '%(func_name)s'")
+            % {
+                "proc_name": proc_name,
+                "func_name": func_name,
+                "task_name": task_name,
+            }
+        )
+        if "group" in task:
+            task_desc += f" [{task['group']}]"
+        logger.info(task_desc)
 
-        self.monitor.process.terminate()
-        self.puller.process.terminate()
-        self.scheduler.process.terminate()
+        if setproctitle:
+            proc_title = f"qcluster {proc_name} processing {task_name} '{func_name}'"
+            if "group" in task:
+                proc_title += f" [{task['group']}]"
+            setproctitle.setproctitle(proc_title)
 
-        logger.debug(_("All processes were terminated"))
+        # if it's not an instance try to get it from the string
+        if not callable(f):
+            # locate() returns None if f cannot be loaded
+            f = pydoc.locate(f)
+        close_old_django_connections()
+        timer_value = task.pop("timeout", timeout)
+        # signal execution
+        pre_execute.send(sender="django_q", func=f, task=task)
+        # execute the payload
+        timer.value = timer_value  # Busy
+
+        try:
+            if f is None:
+                # raise a meaningfull error if task["func"] is not a valid function
+                raise ValueError(f"Function {task['func']} is not defined")
+            res = f(*task["args"], **task["kwargs"])
+            result = (res, True)
+        except Exception as e:
+            result = (f"{e} : {traceback.format_exc()}", False)
+            if error_reporter:
+                error_reporter.report()
+            if task.get("sync", False):
+                raise
+        with timer.get_lock():
+            # Process result
+            task["result"] = result[0]
+            task["success"] = result[1]
+            task["stopped"] = timezone.now()
+            result_queue.put(task)
+            timer.value = -1  # Idle
+            if setproctitle:
+                setproctitle.setproctitle(f"qcluster {proc_name} idle")
+            # Recycle
+            if task_count == Conf.RECYCLE or rss_check():
+                timer.value = -2  # Recycled
+                break
+    logger.info(_("%(proc_name)s stopped doing work") % {"proc_name": proc_name})
+
+
+def save_task(task, broker: Broker):
+    """
+    Saves the task package to Django or the cache
+    :param task: the task package
+    :type broker: brokers.Broker
+    """
+    # SAVE LIMIT < 0 : Don't save success
+    if not task.get("save", Conf.SAVE_LIMIT >= 0) and task["success"]:
+        return
+    # enqueues next in a chain
+    if task.get("chain", None):
+        django_q.tasks.async_chain(
+            task["chain"],
+            group=task["group"],
+            cached=task["cached"],
+            sync=task["sync"],
+            broker=broker,
+        )
+    # SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
+    close_old_django_connections()
+
+    try:
+        filters = {}
+        if (
+            Conf.SAVE_LIMIT_PER
+            and Conf.SAVE_LIMIT_PER in {"group", "name", "func"}
+            and Conf.SAVE_LIMIT_PER in task
+        ):
+            value = task[Conf.SAVE_LIMIT_PER]
+            if Conf.SAVE_LIMIT_PER == "func":
+                value = get_func_repr(value)
+            filters[Conf.SAVE_LIMIT_PER] = value
+
+        with db.transaction.atomic(using=db.router.db_for_write(Success)):
+            last = Success.objects.filter(**filters).select_for_update().last()
+            if (
+                task["success"]
+                and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
+            ):
+                last.delete()
+
+        # check if this task has previous results
+        try:
+            existing_task = Task.objects.get(id=task["id"], name=task["name"])
+            # only update the result if it hasn't succeeded yet
+            if not existing_task.success:
+                existing_task.stopped = task["stopped"]
+                existing_task.result = task["result"]
+                existing_task.success = task["success"]
+                existing_task.attempt_count = existing_task.attempt_count + 1
+                existing_task.save()
+
+            if (
+                Conf.MAX_ATTEMPTS > 0
+                and existing_task.attempt_count >= Conf.MAX_ATTEMPTS
+            ):
+                broker.acknowledge(task["ack_id"])
+
+        except Task.DoesNotExist:
+            # convert func to string
+            func = get_func_repr(task["func"])
+            Task.objects.create(
+                id=task["id"],
+                name=task["name"],
+                func=func,
+                hook=task.get("hook"),
+                args=task["args"],
+                kwargs=task["kwargs"],
+                cluster=task.get("cluster"),
+                started=task["started"],
+                stopped=task["stopped"],
+                result=task["result"],
+                group=task.get("group"),
+                success=task["success"],
+                attempt_count=1,
+            )
+    except Exception:
+        logger.exception("Could not save task result")
+
+
+def save_cached(task, broker: Broker):
+    task_key = f'{broker.list_key}:{task["id"]}'
+    timeout = task["cached"]
+    if timeout is True:
+        timeout = None
+    try:
+        group = task.get("group", None)
+        iter_count = task.get("iter_count", 0)
+        # if it's a group append to the group list
+        if group:
+            group_key = f"{broker.list_key}:{group}:keys"
+            group_list = broker.cache.get(group_key) or []
+            # if it's an iter group, check if we are ready
+            if iter_count and len(group_list) == iter_count - 1:
+                group_args = f"{broker.list_key}:{group}:args"
+                # collate the results into a Task result
+                results = [
+                    SignedPackage.loads(broker.cache.get(k))["result"]
+                    for k in group_list
+                ]
+                results.append(task["result"])
+                task["result"] = results
+                task["id"] = group
+                task["args"] = SignedPackage.loads(broker.cache.get(group_args))
+                task.pop("iter_count", None)
+                task.pop("group", None)
+                if task.get("iter_cached", None):
+                    task["cached"] = task.pop("iter_cached", None)
+                    save_cached(task, broker=broker)
+                else:
+                    save_task(task, broker)
+                broker.cache.delete_many(group_list)
+                broker.cache.delete_many([group_key, group_args])
+                return
+            # save the group list
+            group_list.append(task_key)
+            broker.cache.set(group_key, group_list, timeout)
+            # async_task next in a chain
+            if task.get("chain", None):
+                django_q.tasks.async_chain(
+                    task["chain"],
+                    group=group,
+                    cached=task["cached"],
+                    sync=task["sync"],
+                    broker=broker,
+                )
+        # save the task
+        broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
+    except Exception:
+        logger.exception("Could not save task result")
+
+
+def scheduler(broker: Broker = None):
+    """
+    Creates a task from a schedule at the scheduled time and schedules next run
+    """
+    if not broker:
+        broker = get_broker()
+    close_old_django_connections()
+    try:
+        # Only default cluster will handler schedule with default(null) cluster
+        Q_default = db.models.Q(cluster__isnull=True) if Conf.CLUSTER_NAME == Conf.PREFIX else db.models.Q(pk__in=[])
+
+        with db.transaction.atomic(using=db.router.db_for_write(Schedule)):
+            for s in (
+                Schedule.objects.select_for_update()
+                .exclude(repeats=0)
+                .filter(next_run__lt=timezone.now())
+                .filter(
+                    Q_default | db.models.Q(cluster=Conf.CLUSTER_NAME)
+                )
+            ):
+                args = ()
+                kwargs = {}
+                # get args, kwargs and hook
+                if s.kwargs:
+                    try:
+                        # first try the dict syntax
+                        kwargs = ast.literal_eval(s.kwargs)
+                    except (SyntaxError, ValueError):
+                        # else use the kwargs syntax
+                        try:
+                            parsed_kwargs = (
+                                ast.parse(f"f({s.kwargs})").body[0].value.keywords
+                            )
+                            kwargs = {
+                                kwarg.arg: ast.literal_eval(kwarg.value)
+                                for kwarg in parsed_kwargs
+                            }
+                        except (SyntaxError, ValueError):
+                            kwargs = {}
+                if s.args:
+                    args = ast.literal_eval(s.args)
+                    # single value won't eval to tuple, so:
+                    if type(args) != tuple:
+                        args = (args,)
+                q_options = kwargs.get("q_options", {})
+                if s.intended_date_kwarg:
+                    kwargs[s.intended_date_kwarg] = s.next_run.isoformat()
+                if s.hook:
+                    q_options["hook"] = s.hook
+                # set up the next run time
+                if s.schedule_type != s.ONCE:
+                    next_run = s.next_run
+                    while True:
+                        next_run = s.calculate_next_run(next_run)
+                        if Conf.CATCH_UP or next_run > localtime():
+                            break
+
+                    s.next_run = next_run
+                    s.repeats += -1
+                # send it to the cluster; any cluster name is allowed in multi-queue scenarios
+                # because `broker_name` is confusing, using `cluster` name is recommended and takes precedence
+                q_options["cluster"] = s.cluster or q_options.get("cluster", q_options.pop("broker_name", None))
+                if q_options['cluster'] is None or q_options['cluster'] == Conf.CLUSTER_NAME:
+                    q_options["broker"] = broker
+                q_options["group"] = q_options.get("group", s.name or s.id)
+                kwargs["q_options"] = q_options
+                s.task = django_q.tasks.async_task(s.func, *args, **kwargs)
+                # log it
+                if not s.task:
+                    logger.error(
+                        _(
+                            "%(process_name)s failed to create a task from schedule "
+                            "[%(schedule)s]"
+                        )
+                        % {
+                            "process_name": current_process().name,
+                            "schedule": s.name or s.id,
+                        }
+                    )
+                else:
+                    logger.info(
+                        _(
+                            "%(process_name)s created task %(task_name)s from schedule "
+                            "[%(schedule)s]"
+                        )
+                        % {
+                            "process_name": current_process().name,
+                            "task_name": humanize(s.task),
+                            "schedule": s.name or s.id,
+                        }
+                    )
+                # default behavior is to delete a ONCE schedule
+                if s.schedule_type == s.ONCE:
+                    if s.repeats < 0:
+                        s.delete()
+                        continue
+                    # but not if it has a positive repeats
+                    s.repeats = 0
+                # save the schedule
+                s.save()
+    except Exception:
+        logger.exception("Could not create task from schedule")
+
+
+def close_old_django_connections():
+    """
+    Close django connections unless running with sync=True.
+    """
+    if Conf.SYNC:
+        logger.warning(
+            "Preserving django database connections because sync=True. Beware "
+            "that tasks are now injected in the calling context/transactions "
+            "which may result in unexpected behaviour."
+        )
+    else:
+        db.close_old_connections()
 
 
 def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING):
@@ -345,3 +845,12 @@ def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING)
                 _("%(pid)s will use cpu %(affinity)s")
                 % {"pid": pid, "affinity": affinity}
             )
+
+
+def rss_check():
+    if Conf.MAX_RSS:
+        if resource:
+            return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
+        elif psutil:
+            return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
+    return False

django_q/conf.py

@@ -9,7 +9,7 @@ import pkg_resources
 from django.conf import settings
 from django.utils.translation import gettext_lazy as _
 
-from queue import Queue
+from django_q.queues import Queue
 
 # optional
 try:
@@ -91,7 +91,7 @@ class Conf:
     CLUSTER_NAME = conf.get("cluster_name", PREFIX)
 
     # Log output level
-    LOG_LEVEL = conf.get("log_level", "DEBUG")
+    LOG_LEVEL = conf.get("log_level", "INFO")
 
     # Maximum number of successful tasks kept in the database. 0 saves everything.
     # -1 saves none
@@ -112,7 +112,7 @@ class Conf:
         )
 
     # Guard loop sleep in seconds. Should be between 0 and 60 seconds.
-    GUARD_CYCLE = conf.get("guard_cycle", 1)
+    GUARD_CYCLE = conf.get("guard_cycle", 0.5)
 
     # Disable the scheduler
     SCHEDULER = conf.get("scheduler", True)
@@ -240,7 +240,6 @@ class Conf:
 # logger
 logger = logging.getLogger("django-q")
 
-
 # Set up standard logging handler in case there is none
 if not logger.hasHandlers():
     logger.setLevel(level=getattr(logging, Conf.LOG_LEVEL))

django_q/core_signing.py

@@ -37,7 +37,7 @@ def loads(
     """
     # TimestampSigner.unsign() returns str but base64 and zlib compression
     # operate on bytes.
-    base64d = force_bytes(TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
+    base64d = force_bytes(TimestampSigner(key=key, salt=salt).unsign(s, max_age=max_age))
     decompress = False
     if base64d[:1] == b".":
         # It's compressed; uncompress it first

django_q/exceptions.py

@@ -1,26 +0,0 @@
-import signal
-from typing import Optional
-
-class TimeoutException(SystemExit):
-    """Exception for when a worker takes too long to complete a task"""
-    pass
-
-
-class TimeoutHandler:
-    def __init__(self, timeout: Optional[int] = None):
-        self._timeout = timeout
-
-    def raise_timeout_exception(self, signum, frame):
-        raise TimeoutException('Task exceeded maximum timeout value '
-                              '({0} seconds)'.format(self._timeout))
-
-    def __enter__(self):
-        if self._timeout is None:
-            return
-        signal.signal(signal.SIGALRM, self.raise_timeout_exception)
-        signal.alarm(self._timeout)
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        """When getting out of the timeout, reset the alarm, so it won't trigger"""
-        signal.alarm(0)
-        signal.signal(signal.SIGALRM, signal.SIG_DFL)

django_q/helpers.py

@@ -1,43 +0,0 @@
-from copy import Error
-from django_q.worker import WorkerProcess
-from django_q.monitor import Monitor
-from django_q.models import Task
-from typing import Optional, Sequence, Tuple
-from django_q.conf import logger
-from django_q.queue_task import QueueTask
-from django_q.puller import Puller
-from django_q.scheduler import Scheduler
-
-
-def run_scheduler_once(broker=None) -> None:
-    Scheduler.schedule_tasks(broker)
-
-def get_scheduled_tasks(broker=None) -> Sequence[QueueTask]:
-    try:
-        return Puller.get_tasks_from_broker(broker=broker)
-    except ValueError:
-        logger.exception("Couldn't get items from broker")
-        return []
-
-def run_task(task=None) -> QueueTask:
-    if task is None:
-
-        scheduled_tasks = get_scheduled_tasks()
-        if not len(scheduled_tasks):
-            raise ValueError("No tasks scheduled and no task given to run for worker")
-        task = scheduled_tasks[0]
-
-    return WorkerProcess.run_task(task)
-
-def save_task(task, broker=None) -> Tuple[QueueTask, Optional[Task]]:
-    return Monitor.save_task(task, broker)
-
-def run_cluster_once(workers, tasks=[], broker=None) -> None:
-    if not len(tasks):
-        run_scheduler_once(broker=broker)
-        tasks = get_scheduled_tasks(broker=broker)
-
-    for idx, worker in enumerate(range(workers)):
-        if len(tasks) >= idx + 1:
-            task = run_task(tasks[idx])
-            save_task(task, broker=broker)

django_q/management/commands/qinfo.py

@@ -1,15 +1,9 @@
-from datetime import timedelta
-from django_q.brokers import get_broker
-from django_q.models import Failure, Schedule, Success
-from django_q.status import Stat
-from django.db.models import F, Sum
-from django.db import connection
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
-from django.utils import timezone
 
 from django_q import VERSION
 from django_q.conf import Conf
+from django_q.monitor import get_ids, info
 
 
 class Command(BaseCommand):
@@ -34,13 +28,7 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if options.get("ids", True):
-            stat = Stat.get_all()
-            if not stat:
-                print(_("No clusters appear to be running."))
-
-            for s in stat:
-                print(s.cluster_id)
-
+            get_ids()
         elif options.get("config", False):
             hide = [
                 "conf",
@@ -50,7 +38,6 @@ class Command(BaseCommand):
                 "WORKING",
                 "SIGNAL_NAMES",
                 "STOPPED",
-                "SECRET_KEY",
             ]
             settings = [
                 a for a in dir(Conf) if not a.startswith("__") and a not in hide
@@ -61,69 +48,4 @@ class Command(BaseCommand):
                 if value is not None:
                     self.stdout.write(f"{setting}: {value}")
         else:
-            broker = get_broker()
-
-            broker.ping()
-
-            stats = Stat.get_all(broker=broker)
-            clusters = len(stats)
-            workers = 0
-            reincarnations = 0
-            for cluster in stats:
-                workers += len(cluster.workers)
-                reincarnations += cluster.reincarnations
-
-            # calculate tasks pm and avg exec time
-            tasks_per = 0
-            per = _("day")
-            exec_time = 0
-            last_tasks = Success.objects.filter(
-                stopped__gte=timezone.now() - timedelta(hours=24)
-            )
-            tasks_per_day = last_tasks.count()
-            if tasks_per_day > 0:
-                # average execution time over the last 24 hours
-                if connection.vendor != "sqlite":
-                    exec_time = last_tasks.aggregate(
-                        time_taken=Sum(F("stopped") - F("started"))
-                    )
-                    exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
-                else:
-                    # can't sum timedeltas on sqlite
-                    for t in last_tasks:
-                        exec_time += t.time_taken()
-                    exec_time = exec_time / tasks_per_day
-                # tasks per second/minute/hour/day in the last 24 hours
-                if tasks_per_day > 24 * 60 * 60:
-                    tasks_per = tasks_per_day / (24 * 60 * 60)
-                    per = _("second")
-                elif tasks_per_day > 24 * 60:
-                    tasks_per = tasks_per_day / (24 * 60)
-                    per = _("minute")
-                elif tasks_per_day > 24:
-                    tasks_per = tasks_per_day / 24
-                    per = _("hour")
-                else:
-                    tasks_per = tasks_per_day
-
-            print(
-                _("-- %(prefix)s %(version)s on %(info)s --")
-                % {
-                    "prefix": Conf.PREFIX.capitalize(),
-                    "version": ".".join(str(v) for v in VERSION),
-                    "info": broker.info(),
-                }
-            )
-            print(_("Clusters: %(clusters)s") % {"clusters": clusters})
-            print(_("Workers: %(workers)s") % {"workers": workers})
-            print(_("Restarts: %(restarts)s") % {"restarts": reincarnations})
-
-            print("")
-            print(_("Queued: %(queue_size)s") % {"queue_size": str(broker.queue_size())})
-            print(_("Successes: %(success_count)s") % {"success_count": str(Success.objects.count())})
-            print(_("Failures: %(failure_count)s") % {"failure_count": str(Failure.objects.count())})
-
-            print("")
-            print(_("Schedules: %(schedules_count)s") % {"schedules_count": str(Schedule.objects.count())})
-            print(_("Tasks/%(per)s: %(amount)s") % {"per": per, "amount": f"{tasks_per:.2f}"})
-            print(_("Avg time: %(time)s") % {"time": f"{exec_time:.4f}"})
+            info()

django_q/management/commands/qmemory.py

@@ -1,19 +1,7 @@
-import curses
-from django_q.conf import Conf
-import signal
-
-import time
-from django_q.status import Stat
-from django_q.brokers import get_broker
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
-from django.utils import timezone
-import curses
 
-try:
-    import psutil
-except ImportError:
-    psutil = None
+from django_q.monitor import memory
 
 
 class Command(BaseCommand):
@@ -37,103 +25,7 @@ class Command(BaseCommand):
         )
 
     def handle(self, *args, **options):
-        memory_stats = MemoryTerminalStats(
+        memory(
             run_once=options.get("run_once", False),
             workers=options.get("workers", False),
         )
-        curses.wrapper(memory_stats.start)
-
-
-def get_process_mb(pid):
-    try:
-        process = psutil.Process(pid)
-        mb_used = round(process.memory_info().rss / 1024**2, 2)
-    except psutil.NoSuchProcess:
-        mb_used = "NO_PROCESS_FOUND"
-    return mb_used
-
-
-class MemoryTerminalStats:
-    stop_writing = False
-
-    def __init__(self, run_once=False, workers=False):
-        self.run_once = run_once
-        self.workers = workers
-
-    def start(self, stdscr):
-        self.show_stats()
-
-    def on_exit(self, signum, frame):
-        # exit clean
-        self.stop_writing = True
-
-    def show_stats(self):
-        signal.signal(signal.SIGTERM, self.on_exit)
-        signal.signal(signal.SIGINT, self.on_exit)
-        scr = curses.initscr()
-
-        if not broker:
-            broker = get_broker()
-
-        broker.ping()
-        if not psutil:
-            scr.addstr(0, 0, 'Cannot start "qmemory" command. Missing "psutil" library.')
-            scr.refresh()
-            return
-
-        MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
-        MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
-
-        stats = Stat.get_all(broker=broker)
-
-        if not stats:
-            scr.addstr(1, 0, "Cluster is not running")
-            scr.refresh()
-        while not self.stop_writing:
-            data = []
-            for stat in stats:
-                # memory available (%)
-                memory_available_percentage = round(
-                    psutil.virtual_memory().available
-                    * 100
-                    / psutil.virtual_memory().total,
-                    2,
-                )
-                # memory available (MB)
-                memory_available = round(
-                    psutil.virtual_memory().available / 1024**2, 2
-                )
-                if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
-                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
-                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
-
-                data.append(f"Host: {str(stat.host)}")
-                data.append(f"ID: {str(stat.cluster_id)[-8:]}")
-                data.append(f"Available (%): {memory_available_percentage}")
-                data.append(f"Available (MB): {memory_available}")
-                data.append(f"Total (MB): {round(psutil.virtual_memory().total / 1024**2, 2)}")
-                data.append(f"Sentinel (MB): {get_process_mb(stat.sentinel)}")
-                data.append(f"Monitor (MB): {get_process_mb(getattr(stat, 'monitor', None))}")
-
-                if self.workers:
-                    data.append("")
-                    for worker_num in range(Conf.WORKERS):
-                        data.append(f"Worker #{worker_num+1} (MB): {get_process_mb(stat.workers[worker_num])}")
-
-                data.append("")
-                data.append(_("Available lowest: %(memory_percent)s (%(at)s)")
-                % {
-                    "memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
-                    "at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
-                        "%Y-%m-%d %H:%M:%S+00:00"
-                    ),
-                })
-
-
-            for idx, item in enumerate(data):
-                scr.addstr(idx, 0, item)
-
-            scr.refresh()
-            time.sleep(0.5)
-            if self.run_once:
-                return

django_q/management/commands/qmonitor.py

@@ -1,14 +1,7 @@
-import curses
-from django_q.brokers import get_broker
-from django_q.models import Failure, Success
-from django_q.conf import Conf
-from django_q.status import Stat
 from django.core.management.base import BaseCommand
 from django.utils.translation import gettext as _
-from django.utils import timezone
-import time
 
-import signal
+from django_q.monitor import monitor
 
 
 class Command(BaseCommand):
@@ -25,109 +18,4 @@ class Command(BaseCommand):
         )
 
     def handle(self, *args, **options):
-        memory_stats = MonitorTerminalStats(
-            run_once=options.get("run_once", False),
-        )
-        curses.wrapper(memory_stats.start)
-
-
-class MonitorTerminalStats:
-    stop_writing = False
-    table_cell_size = 20
-
-    def __init__(self, run_once=False):
-        self.run_once = run_once
-
-    def start(self, stdscr):
-        self.show_stats()
-
-    def on_exit(self, signum, frame):
-        # exit clean
-        self.stop_writing = True
-
-    def get_table_cell(self, data):
-        spaces = self.table_cell_size - len(str(data))
-        return data + " " * spaces + "//  "
-
-    def show_stats(self):
-        signal.signal(signal.SIGTERM, self.on_exit)
-        signal.signal(signal.SIGINT, self.on_exit)
-        scr = curses.initscr()
-
-        broker = get_broker()
-
-        broker.ping()
-
-        stats = Stat.get_all(broker=broker)
-
-        if not stats:
-            scr.addstr(1, 0, "Cluster is not running")
-            scr.refresh()
-
-        while not self.stop_writing:
-            data = []
-            table_headers = [
-                _("Host"),
-                _("Id"),
-                _("State"),
-                _("Pool"),
-                _("TQ"),
-                _("RQ"),
-                _("RC"),
-                _("Up"),
-            ]
-
-            data.append("".join([self.get_table_cell(header) for header in table_headers]))
-
-            for stat in stats:
-                tasks = str(stat.task_q_size)
-
-                if stat.task_q_size > 0:
-                    tasks = str(stat.task_q_size)
-                    if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
-                        tasks += " (at maximum size)"
-                results = stat.done_q_size
-                if results > 0:
-                    results = str(results)
-                # color workers
-                workers = len(stat.workers)
-                # format uptime
-                uptime = (timezone.now() - stat.tob).total_seconds()
-                hours, remainder = divmod(uptime, 3600)
-                minutes, seconds = divmod(remainder, 60)
-                uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
-                # print to the terminal
-                stat_values = [
-                    str(stat.host),
-                    str(stat.cluster_id)[-8:],
-                    str(stat.status),
-                    str(workers),
-                    str(tasks),
-                    str(results),
-                    str(stat.reincarnations),
-                    str(uptime),
-                ]
-
-                data.append("".join([self.get_table_cell(val) for val in stat_values]))
-
-            data.append("")
-            queue_size = broker.queue_size()
-            lock_size = broker.lock_size()
-            if lock_size:
-                queue_size = f"{queue_size}({lock_size})"
-
-
-            data.append("")
-            data.append(_("info: %(broker_info)s") % {"broker_info": broker.info()})
-            data.append("")
-            data.append(_("Queued: %(queue_size)s") % {"queue_size": str(broker.queue_size())})
-            data.append(_("Successes: %(success_count)s") % {"success_count": str(Success.objects.count())})
-            data.append(_("Failures: %(failure_count)s") % {"failure_count": str(Failure.objects.count())})
-
-            for idx, item in enumerate(data):
-                scr.addstr(idx, 0, item)
-
-            scr.refresh()
-            time.sleep(0.5)
-            if self.run_once:
-                return
+        monitor(run_once=options.get("run_once", False))

django_q/models.py

@@ -1,6 +1,5 @@
 from datetime import datetime, timedelta
 from keyword import iskeyword
-import ast
 
 # Django
 from django import get_version
@@ -23,6 +22,8 @@ from django_q.conf import croniter, Conf
 from django_q.signing import SignedPackage
 from django_q.utils import localtime, add_months, add_years
 
+from .utils import get_func_repr
+
 
 class Task(models.Model):
     id = models.CharField(max_length=32, primary_key=True, editable=False)
@@ -228,34 +229,6 @@ class Schedule(models.Model):
         help_text=_("Name of kwarg to pass intended schedule date"),
     )
 
-    def parse_kwargs(self):
-        if not self.kwargs:
-            return {}
-        try:
-            # first try the dict syntax
-            return ast.literal_eval(self.kwargs)
-        except (SyntaxError, ValueError):
-            # else use the kwargs syntax
-            try:
-                parsed_kwargs = (
-                    ast.parse(f"f({self.kwargs})").body[0].value.keywords
-                )
-                return {
-                    kwarg.arg: ast.literal_eval(kwarg.value)
-                    for kwarg in parsed_kwargs
-                }
-            except (SyntaxError, ValueError):
-                return {}
-
-    def parse_args(self):
-        if not self.args:
-            return tuple()
-        args = ast.literal_eval(self.args)
-        # single value won't eval to tuple, so:
-        if type(args) != tuple:
-            args = (args,)
-        return args
-
     def calculate_next_run(self, next_run=None):
         # next run is always in UTC
         next_run = next_run or self.next_run
@@ -338,7 +311,6 @@ class OrmQ(models.Model):
     payload = models.TextField()
     lock = models.DateTimeField(null=True, help_text=_("Prevent any cluster from pulling until"))
 
-
     @cached_property
     def task(self):
         try:
@@ -347,24 +319,16 @@ class OrmQ(models.Model):
             return {"id": "*" + e.__class__.__name__}
 
     def func(self):
-        if isinstance(self.task, dict):
-            return self.task.get("func_name", "")
-        return self.task.func_name
+        return get_func_repr(self.task.get("func"))
 
     def task_id(self):
-        if isinstance(self.task, dict):
-            return self.task.get("id", "")
-        return self.task.id
+        return self.task.get("id")
 
     def name(self):
-        if isinstance(self.task, dict):
-            return self.task["name"]
-        return self.task.name
+        return self.task.get("name")
 
     def group(self):
-        if isinstance(self.task, dict):
-            return self.task.get("group", "")
-        return self.task.group
+        return self.task.get("group")
 
     def args(self):
         return self.task.get("args")

django_q/monitor.py

@@ -1,102 +1,510 @@
-from django_q.worker import WorkerProcess
-from django_q.queue_task import QueueTask
-from django_q.models import Task
-from queue import Queue
-from queue import Empty
-from typing import Optional, Tuple
+from datetime import timedelta
+
+# django
+from django.db import connection
+from django.db.models import F, Sum
+from django.utils import timezone
+from django.utils.translation import gettext as _
+
+from django_q import VERSION, models
 from django_q.brokers import get_broker
-from django_q.process_manager import ProcessManager
-from django_q.signals import post_execute
-from django_q.conf import logger
-from django.utils.translation import gettext_lazy as _
-from multiprocessing import current_process
 
+# local
+from django_q.conf import Conf
+from django_q.status import Stat
+
+# optional
 try:
-    import setproctitle
-except ModuleNotFoundError:
-    setproctitle = None
+    import psutil
+except ImportError:
+    psutil = None
 
 
-class Monitor(ProcessManager):
-    def __init__(self):
-        super().__init__()
-        self.task_queue = Queue()
-
-    @staticmethod
-    def save_task(task, broker=None) -> Tuple[QueueTask, Optional[Task]]:
-        task_db_obj = None
-        if broker is None:
-            broker = get_broker()
-        if task.cached:
-            task.save_cached(broker)
-        else:
-            print("SAVE TO DB")
-            task_db_obj = task.save_to_db(broker)
-        # acknowledge result
-        if task.ack_id and (task.has_succeeded or not task.ack_failure):
-            broker.acknowledge(task.ack_id)
-        # signal execution done
-        post_execute.send(sender="django_q", task=task)
-        return task, task_db_obj
+def get_process_mb(pid):
+    try:
+        process = psutil.Process(pid)
+        mb_used = round(process.memory_info().rss / 1024**2, 2)
+    except psutil.NoSuchProcess:
+        mb_used = "NO_PROCESS_FOUND"
+    return mb_used
 
 
-    @property
-    def is_done(self):
-        return self.status.value == self.Status.IDLE.value and self.task_queue.empty()
+BLESSED_INSTALL_MESSAGE = (
+    "Blessed is not installed. Please install blessed to use this: "
+    "https://pypi.org/project/blessed/"
+)
 
-    def get_target(self):
-        return self.run_monitor
 
-    def run_item(self):
-        if self.is_idle:
-            try:
-                task = self.task_queue.get_nowait()
-            except Empty:
-                # if the queue is empty, then just stop
-                return
-            try:
-                self.manager_pipe.send(task)
-            except BrokenPipeError:
-                # recycle process if pipe is broken
-                self.status.value = ProcessManager.Status.RECYCLE.value
-
-    def add_task(self, task):
-        self.task_queue.put(task)
-
-    def run_monitor(self, status, pipe) -> None:
+def monitor(run_once=False, broker=None):
+    if not broker:
         broker = get_broker()
-        proc_name = current_process().name
-        if setproctitle:
-            setproctitle.setproctitle(f"qcluster {proc_name} monitor")
-        logger.info(
-                _("%(name)s monitoring at %(id)s") % {"name": proc_name, "id": current_process().pid}
-                )
-        status.value = self.Status.IDLE.value
+    try:
+        from blessed import Terminal
 
-        while True:
-            task = pipe.recv()
-            if task == "STOP":
-                logger.info(f"Monitor {proc_name} shut down")
-                break
-            status.value = self.Status.BUSY.value
-            # save the result
-            task, __ = Monitor.save_task(task, broker=broker)
-            # log the result
-            if task.has_succeeded:
-                # log success
-                logger.info(
-                        _("Processed '%(info_name)s' (%(task_name)s)")
-                        % {"info_name": task.func_name, "task_name": task.name}
+        term = Terminal()
+    except ImportError:
+        print(BLESSED_INSTALL_MESSAGE)
+        return
+
+    broker.ping()
+    with term.fullscreen(), term.hidden_cursor(), term.cbreak():
+        val = None
+        start_width = int(term.width / 8)
+        while val not in (
+            "q",
+            "Q",
+        ):
+            col_width = int(term.width / 8)
+            # In case of resize
+            if col_width != start_width:
+                print(term.clear())
+                start_width = col_width
+            print(
+                term.move(0, 0)
+                + term.black_on_green(term.center(_("Host"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 1 * col_width)
+                + term.black_on_green(term.center(_("Id"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 2 * col_width)
+                + term.black_on_green(term.center(_("State"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 3 * col_width)
+                + term.black_on_green(term.center(_("Pool"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 4 * col_width)
+                + term.black_on_green(term.center(_("TQ"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 5 * col_width)
+                + term.black_on_green(term.center(_("RQ"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 6 * col_width)
+                + term.black_on_green(term.center(_("RC"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 7 * col_width)
+                + term.black_on_green(term.center(_("Up"), width=col_width - 1))
+            )
+            i = 2
+            stats = Stat.get_all(broker=broker)
+            print(term.clear_eos())
+            for stat in stats:
+                status = stat.status
+                # color status
+                if stat.status == Conf.WORKING:
+                    status = term.green(str(Conf.WORKING))
+                elif stat.status == Conf.STOPPING:
+                    status = term.yellow(str(Conf.STOPPING))
+                elif stat.status == Conf.STOPPED:
+                    status = term.red(str(Conf.STOPPED))
+                elif stat.status == Conf.IDLE:
+                    status = str(Conf.IDLE)
+                # color q's
+                tasks = str(stat.task_q_size)
+                if stat.task_q_size > 0:
+                    tasks = term.cyan(str(stat.task_q_size))
+                    if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
+                        tasks = term.green(str(stat.task_q_size))
+                results = stat.done_q_size
+                if results > 0:
+                    results = term.cyan(str(results))
+                # color workers
+                workers = len(stat.workers)
+                if workers < Conf.WORKERS:
+                    workers = term.yellow(str(workers))
+                # format uptime
+                uptime = (timezone.now() - stat.tob).total_seconds()
+                hours, remainder = divmod(uptime, 3600)
+                minutes, seconds = divmod(remainder, 60)
+                uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
+                # print to the terminal
+                print(
+                    term.move(i, 0)
+                    + term.center(stat.host[: col_width - 1], width=col_width - 1)
+                )
+                print(
+                    term.move(i, 1 * col_width)
+                    + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
+                )
+                print(
+                    term.move(i, 2 * col_width)
+                    + term.center(status, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 3 * col_width)
+                    + term.center(workers, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 4 * col_width)
+                    + term.center(tasks, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 5 * col_width)
+                    + term.center(results, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 6 * col_width)
+                    + term.center(stat.reincarnations, width=col_width - 1)
+                )
+                print(
+                    term.move(i, 7 * col_width)
+                    + term.center(uptime, width=col_width - 1)
+                )
+                i += 1
+            # bottom bar
+            i += 1
+            queue_size = broker.queue_size()
+            lock_size = broker.lock_size()
+            if lock_size:
+                queue_size = f"{queue_size}({lock_size})"
+            print(
+                term.move(i, 0)
+                + term.white_on_cyan(term.center(broker.info(), width=col_width * 2))
+            )
+            print(
+                term.move(i, 2 * col_width)
+                + term.black_on_cyan(term.center(_("Queued"), width=col_width))
+            )
+            print(
+                term.move(i, 3 * col_width)
+                + term.white_on_cyan(term.center(queue_size, width=col_width))
+            )
+            print(
+                term.move(i, 4 * col_width)
+                + term.black_on_cyan(term.center(_("Success"), width=col_width))
+            )
+            print(
+                term.move(i, 5 * col_width)
+                + term.white_on_cyan(
+                    term.center(models.Success.objects.count(), width=col_width)
+                )
+            )
+            print(
+                term.move(i, 6 * col_width)
+                + term.black_on_cyan(term.center(_("Failures"), width=col_width))
+            )
+            print(
+                term.move(i, 7 * col_width)
+                + term.white_on_cyan(
+                    term.center(models.Failure.objects.count(), width=col_width)
+                )
+            )
+            # for testing
+            if run_once:
+                return Stat.get_all(broker=broker)
+            print(term.move(i + 2, 0) + term.center(_("[Press q to quit]")))
+            val = term.inkey(timeout=1)
+
+
+def info(broker=None):
+    if not broker:
+        broker = get_broker()
+    try:
+        from blessed import Terminal
+
+        term = Terminal()
+    except ImportError:
+        print(BLESSED_INSTALL_MESSAGE)
+        return
+
+    broker.ping()
+    stat = Stat.get_all(broker=broker)
+    # general stats
+    clusters = len(stat)
+    workers = 0
+    reincarnations = 0
+    for cluster in stat:
+        workers += len(cluster.workers)
+        reincarnations += cluster.reincarnations
+    # calculate tasks pm and avg exec time
+    tasks_per = 0
+    per = _("day")
+    exec_time = 0
+    last_tasks = models.Success.objects.filter(
+        stopped__gte=timezone.now() - timedelta(hours=24)
+    )
+    tasks_per_day = last_tasks.count()
+    if tasks_per_day > 0:
+        # average execution time over the last 24 hours
+        if connection.vendor != "sqlite":
+            exec_time = last_tasks.aggregate(
+                time_taken=Sum(F("stopped") - F("started"))
+            )
+            exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
+        else:
+            # can't sum timedeltas on sqlite
+            for t in last_tasks:
+                exec_time += t.time_taken()
+            exec_time = exec_time / tasks_per_day
+        # tasks per second/minute/hour/day in the last 24 hours
+        if tasks_per_day > 24 * 60 * 60:
+            tasks_per = tasks_per_day / (24 * 60 * 60)
+            per = _("second")
+        elif tasks_per_day > 24 * 60:
+            tasks_per = tasks_per_day / (24 * 60)
+            per = _("minute")
+        elif tasks_per_day > 24:
+            tasks_per = tasks_per_day / 24
+            per = _("hour")
+        else:
+            tasks_per = tasks_per_day
+    # print to terminal
+    print(term.clear_eos())
+    col_width = int(term.width / 6)
+    print(
+        term.black_on_green(
+            term.center(
+                _("-- %(prefix)s %(version)s on %(info)s --")
+                % {
+                    "prefix": Conf.PREFIX.capitalize(),
+                    "version": ".".join(str(v) for v in VERSION),
+                    "info": broker.info(),
+                }
+            )
+        )
+    )
+    print(
+        term.cyan(_("Clusters"))
+        + term.move_x(1 * col_width)
+        + term.white(str(clusters))
+        + term.move_x(2 * col_width)
+        + term.cyan(_("Workers"))
+        + term.move_x(3 * col_width)
+        + term.white(str(workers))
+        + term.move_x(4 * col_width)
+        + term.cyan(_("Restarts"))
+        + term.move_x(5 * col_width)
+        + term.white(str(reincarnations))
+    )
+    print(
+        term.cyan(_("Queued"))
+        + term.move_x(1 * col_width)
+        + term.white(str(broker.queue_size()))
+        + term.move_x(2 * col_width)
+        + term.cyan(_("Successes"))
+        + term.move_x(3 * col_width)
+        + term.white(str(models.Success.objects.count()))
+        + term.move_x(4 * col_width)
+        + term.cyan(_("Failures"))
+        + term.move_x(5 * col_width)
+        + term.white(str(models.Failure.objects.count()))
+    )
+    print(
+        term.cyan(_("Schedules"))
+        + term.move_x(1 * col_width)
+        + term.white(str(models.Schedule.objects.count()))
+        + term.move_x(2 * col_width)
+        + term.cyan(_("Tasks/%(per)s") % {"per": per})
+        + term.move_x(3 * col_width)
+        + term.white(f"{tasks_per:.2f}")
+        + term.move_x(4 * col_width)
+        + term.cyan(_("Avg time"))
+        + term.move_x(5 * col_width)
+        + term.white(f"{exec_time:.4f}")
+    )
+    return True
+
+
+def memory(run_once=False, workers=False, broker=None):
+    if not broker:
+        broker = get_broker()
+    try:
+        from blessed import Terminal
+
+        term = Terminal()
+    except ImportError:
+        print(BLESSED_INSTALL_MESSAGE)
+        return
+    broker.ping()
+    if not psutil:
+        print(term.clear_eos())
+        print(
+            term.white_on_red(
+                'Cannot start "qmemory" command. Missing "psutil" library.'
+            )
+        )
+        return
+    with term.fullscreen(), term.hidden_cursor(), term.cbreak():
+        MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
+        MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
+        cols = 8
+        val = None
+        start_width = int(term.width / cols)
+        while val not in ["q", "Q"]:
+            col_width = int(term.width / cols)
+            # In case of resize
+            if col_width != start_width:
+                print(term.clear())
+                start_width = col_width
+            # sentinel, monitor and workers memory usage
+            print(
+                term.move(0, 0 * col_width)
+                + term.black_on_green(term.center(_("Host"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 1 * col_width)
+                + term.black_on_green(term.center(_("Id"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 2 * col_width)
+                + term.black_on_green(
+                    term.center(_("Available (%)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 3 * col_width)
+                + term.black_on_green(
+                    term.center(_("Available (MB)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 4 * col_width)
+                + term.black_on_green(term.center(_("Total (MB)"), width=col_width - 1))
+            )
+            print(
+                term.move(0, 5 * col_width)
+                + term.black_on_green(
+                    term.center(_("Sentinel (MB)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 6 * col_width)
+                + term.black_on_green(
+                    term.center(_("Monitor (MB)"), width=col_width - 1)
+                )
+            )
+            print(
+                term.move(0, 7 * col_width)
+                + term.black_on_green(
+                    term.center(_("Workers (MB)"), width=col_width - 1)
+                )
+            )
+            row = 2
+            stats = Stat.get_all(broker=broker)
+            print(term.clear_eos())
+            for stat in stats:
+                # memory available (%)
+                memory_available_percentage = round(
+                    psutil.virtual_memory().available
+                    * 100
+                    / psutil.virtual_memory().total,
+                    2,
+                )
+                # memory available (MB)
+                memory_available = round(
+                    psutil.virtual_memory().available / 1024**2, 2
+                )
+                if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
+                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
+                    MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
+                print(
+                    term.move(row, 0 * col_width)
+                    + term.center(stat.host[: col_width - 1], width=col_width - 1)
+                )
+                print(
+                    term.move(row, 1 * col_width)
+                    + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
+                )
+                print(
+                    term.move(row, 2 * col_width)
+                    + term.center(memory_available_percentage, width=col_width - 1)
+                )
+                print(
+                    term.move(row, 3 * col_width)
+                    + term.center(memory_available, width=col_width - 1)
+                )
+                print(
+                    term.move(row, 4 * col_width)
+                    + term.center(
+                        round(psutil.virtual_memory().total / 1024**2, 2),
+                        width=col_width - 1,
+                    )
+                )
+                print(
+                    term.move(row, 5 * col_width)
+                    + term.center(get_process_mb(stat.sentinel), width=col_width - 1)
+                )
+                print(
+                    term.move(row, 6 * col_width)
+                    + term.center(
+                        get_process_mb(getattr(stat, "monitor", None)),
+                        width=col_width - 1,
+                    )
+                )
+                workers_mb = 0
+                for worker_pid in stat.workers:
+                    result = get_process_mb(worker_pid)
+                    if isinstance(result, str):
+                        result = 0
+                    workers_mb += result
+                print(
+                    term.move(row, 7 * col_width)
+                    + term.center(
+                        workers_mb or "NO_PROCESSES_FOUND", width=col_width - 1
+                    )
+                )
+                row += 1
+            # each worker's memory usage
+            if workers:
+                row += 2
+                col_width = int(term.width / (1 + Conf.WORKERS))
+                print(
+                    term.move(row, 0 * col_width)
+                    + term.black_on_cyan(term.center(_("Id"), width=col_width - 1))
+                )
+                for worker_num in range(Conf.WORKERS):
+                    print(
+                        term.move(row, (worker_num + 1) * col_width)
+                        + term.black_on_cyan(
+                            term.center(
+                                "Worker #{} (MB)".format(worker_num + 1),
+                                width=col_width - 1,
+                            )
                         )
-            else:
-                # log failure
-                logger.error(
-                        _("Failed '%(info_name)s' (%(task_name)s) - %(task_result)s")
-                        % {
-                            "info_name": task.func_name,
-                            "task_name": task.name,
-                            "task_result": task.result,
-                            }
+                    )
+                row += 2
+                for stat in stats:
+                    print(
+                        term.move(row, 0 * col_width)
+                        + term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
+                    )
+                    for idx, worker_pid in enumerate(stat.workers):
+                        mb_used = get_process_mb(worker_pid)
+                        print(
+                            term.move(row, (idx + 1) * col_width)
+                            + term.center(mb_used, width=col_width - 1)
                         )
-            status.value = self.Status.IDLE.value
-        logger.info(_("%(name)s stopped monitoring results") % {"name": proc_name})
+                    row += 1
+            row += 1
+            print(
+                term.move(row, 0)
+                + _("Available lowest (): %(memory_percent)s ((at)s)")
+                % {
+                    "memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
+                    "at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
+                        "%Y-%m-%d %H:%M:%S+00:00"
+                    ),
+                }
+            )
+            # for testing
+            if run_once:
+                return Stat.get_all(broker=broker)
+            print(term.move(row + 2, 0) + term.center(_("[Press q to quit]")))
+            val = term.inkey(timeout=1)
+
+
+def get_ids():
+    # prints id (PID) of running clusters
+    stat = Stat.get_all()
+    if stat:
+        for s in stat:
+            print(s.cluster_id)
+    else:
+        print(_("No clusters appear to be running."))
+    return True

django_q/process_manager.py

@@ -1,75 +0,0 @@
-from abc import ABC
-from django_q.conf import Conf, logger
-from django_q.humanhash import humanize
-import uuid
-from django_q.conf import Conf
-import enum
-from django import db
-from typing import Callable
-from django.utils.translation import gettext_lazy as _
-import multiprocessing
-from multiprocessing import Process, Value
-
-class ProcessManager(ABC):
-    class Status(enum.IntEnum):
-        IDLE = 1
-        BUSY = 2
-        DONE = 3
-        RECYCLE = 4
-
-    target = None
-
-    def get_target(self) -> Callable:
-        if self.target is None:
-            raise ValueError("Process must have target specified")
-        return self.target
-
-    def __init__(self):
-        self.status = Value("i", self.Status.IDLE.value)
-        self.process = self.spawn_process()
-        self.name = humanize(uuid.uuid4().hex)
-
-    def spawn_process(self) -> Process:
-        self.manager_pipe, process_pipe = multiprocessing.Pipe(duplex=True)
-        p = Process(target=self.get_target(), args=(self.status, process_pipe))
-        p.start()
-        return p
-
-    def reincarnate_process(self):
-        # kill connections before killing the process
-        logger.critical(_("reincarnated worker %(name)s after death") % {"name": self.process.name})
-        if not Conf.SYNC:
-            db.connections.close_all()
-        self.process.kill()
-        self.process = self.spawn_process()
-        self.mark_idle()
-
-    @property
-    def has_results(self):
-        # poll puller pipe for new tasks
-        return self.manager_pipe.poll()
-
-    def get_result(self):
-        # get the puller pipe task object back from the worker
-        return self.manager_pipe.recv()
-
-    @property
-    def is_alive(self):
-        # get the puller status
-        return self.process.is_alive()
-
-    @property
-    def is_done(self):
-        return self.status.value == self.Status.DONE.value
-
-    @property
-    def is_idle(self):
-        return self.status.value == self.Status.IDLE.value
-
-    @property
-    def is_recycle(self):
-        # worker needs to be recycled/reincarnated
-        return self.status.value == self.Status.RECYCLE.value
-
-    def mark_idle(self):
-        self.status.value = self.Status.IDLE.value

django_q/puller.py

@@ -1,85 +0,0 @@
-from django_q.worker import Worker
-from django_q.signing import BadSignature, SignedPackage
-from time import sleep
-from django_q.brokers import get_broker
-import multiprocessing
-from django_q.queue_task import QueueTask
-from django.utils import timezone
-import enum
-import traceback
-
-from multiprocessing import Event, Process, Value, current_process
-from django_q.utils import close_old_django_connections
-from django.utils.translation import gettext_lazy as _
-from django_q.conf import Conf, logger, setproctitle, error_reporter, resource, psutil
-from django_q.exceptions import TimeoutException, TimeoutHandler
-from django_q.process_manager import ProcessManager
-
-
-
-class Puller(ProcessManager):
-    """The Puller is responsible for pulling the tasks from the broker, then return them to be picked up by the
-    guard"""
-
-    @staticmethod
-    def get_tasks_from_broker(broker=None):
-        queued_tasks = []
-        if broker is None:
-            broker = get_broker()
-        try:
-            task_set = broker.dequeue()
-        except Exception:
-            # broker probably crashed. Let the sentinel handle it.
-            raise ValueError("Failed to pull task from broker")
-        if task_set:
-            logger.info(
-                _("Found %(amount_tasks)s tasks") % {"amount_tasks": len(task_set)}
-            )
-            for task in task_set:
-                print(task)
-                logger.info("ONE TASK")
-                ack_id = task[0]
-                # unpack the task
-                try:
-                    queue_task = SignedPackage.loads(task[1])
-                except (TypeError, BadSignature):
-                    logger.exception("Failed to pull task from broker - bad task")
-                    broker.fail(ack_id)
-                    continue
-                queue_task.cluster = Conf.CLUSTER_NAME # save actual cluster name to orm task table
-                queue_task.ack_id = ack_id
-                # send back to main process
-                queued_tasks.append(queue_task)
-            logger.debug(
-                _("queueing from %(list_key)s") % {"list_key": broker.list_key}
-            )
-        return queued_tasks
-
-    def get_target(self):
-        return self.run_puller
-
-    def stop_puller(self):
-        self.status.value = self.Status.DONE.value
-
-    def run_puller(self, status, pipe) -> None:
-        broker = get_broker()
-        proc_name = current_process().name
-        if setproctitle:
-            setproctitle.setproctitle(f"qcluster {proc_name} puller")
-        logger.info(
-            _("%(name)s pulling tasks from broker %(id)s")
-            % {"name": proc_name, "id": current_process().pid}
-        )
-        while True:
-            if status.value == Worker.Status.DONE.value:
-                logger.info("Stopping Puller")
-                break
-            try:
-                queued_tasks = Puller.get_tasks_from_broker(broker=broker)
-            except Exception:
-                logger.exception("Couldn't get items from broker")
-                sleep(10)
-                break
-            for queue_task in queued_tasks:
-                pipe.send(queue_task)
-        logger.info(_("%(name)s stopped pushing tasks") % {"name": current_process().name})

django_q/queue_task.py

@@ -1,211 +0,0 @@
-from __future__ import annotations
-from datetime import datetime
-from django_q.brokers import Broker
-from django.utils import timezone
-from django_q.signing import SignedPackage
-from django_q.models import Success, Task
-from django_q.utils import close_old_django_connections
-import enum
-from django_q import tasks
-import inspect
-import pydoc
-from django import db
-
-from dataclasses import dataclass, field
-from typing import Any, Callable, Optional, Union
-
-from django_q.conf import Conf, logger
-
-
-@dataclass
-class QueueTask:
-    class Status(enum.IntEnum):
-        QUEUED = 0
-        SUCCESS = 1
-        FAILED = 2
-        TIMEOUT = 3
-
-    func: Union[Callable, str]
-    name: str
-    group: Optional[str] = None
-    cluster: str = Conf.CLUSTER_NAME
-    queued_at: Optional[datetime] = timezone.now()
-    finished_at: Optional[datetime] = None
-    ack_id: Optional[str] = None
-    started_at: Optional[datetime] = None
-    id: str = "-1"
-    timeout: Optional[int] = Conf.TIMEOUT
-    status: Optional[Status] = None
-    result: Any = None
-    save: bool = Conf.SAVE_LIMIT >= 0
-    chain: Union[str, QueueTask] = ""
-    cached: bool = Conf.CACHED
-    sync: bool = Conf.SYNC
-    hook: Optional[str] = None
-    args: tuple = field(default_factory=tuple)
-    kwargs: dict = field(default_factory=dict)
-    ack_failure: bool = Conf.ACK_FAILURES
-    iter_count: Optional[int] = None
-    iter_cached: Optional[int] = None
-
-    def callable_func(self):
-        func = self.func
-        if not callable(func):
-            func = pydoc.locate(func)
-        return func
-
-    @property
-    def has_succeeded(self):
-        return self.status == self.Status.SUCCESS
-
-    @property
-    def has_timed_out(self):
-        return self.status == self.Status.TIMEOUT
-
-    @property
-    def is_callable(self):
-        return self.callable_func is not None
-
-    @property
-    def func_name(self):
-        if inspect.isfunction(self.func):
-            return f"{self.func.__module__}.{self.func.__name__}"
-        elif inspect.ismethod(self.func) and hasattr(self.func.__self__, "__name__"):
-            return (
-                f"{self.func.__self__.__module__}." f"{self.func.__self__.__name__}.{self.func.__name__}"
-            )
-        else:
-            return str(self.func)
-
-    def save_to_db(self, broker: Broker):
-        """
-        Saves the task package to Django or the cache
-        :param task: the task package
-        :type broker: brokers.Broker
-        """
-        # SAVE LIMIT < 0 : Don't save success
-        if not self.save and self.has_succeeded:
-            return
-        # enqueues next in a chain
-        if self.chain:
-            tasks.async_chain(
-                self.chain,
-                group=self.group,
-                cached=self.cached,
-                sync=self.sync,
-                broker=broker,
-            )
-        close_old_django_connections()
-
-        logger.debug(self.func_name)
-        try:
-            filters = {}
-            if (
-                Conf.SAVE_LIMIT_PER
-                and Conf.SAVE_LIMIT_PER in {"group", "name", "func"}
-                and Conf.SAVE_LIMIT_PER in self
-            ):
-                value = getattr(self, Conf.SAVE_LIMIT_PER)
-                if Conf.SAVE_LIMIT_PER == "func":
-                    value = self.func_name
-                filters[Conf.SAVE_LIMIT_PER] = value
-
-            with db.transaction.atomic(using=db.router.db_for_write(Success)):
-                last = Success.objects.filter(**filters).select_for_update().last()
-                if (
-                    self.has_succeeded
-                    and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
-                ):
-                    # delete the last entry if we are hitting the limit
-                    last.delete()
-
-            # check if this task has previous results
-            existing_task, created = Task.objects.get_or_create(
-                id=self.id,
-                name=self.name,
-                defaults={
-                    'func': self.func_name,
-                    'stopped': self.finished_at,
-                    'hook': self.hook,
-                    'args': self.args,
-                    'kwargs': self.kwargs,
-                    'cluster': self.cluster,
-                    'started': self.started_at,
-                    'result': self.result,
-                    'group': self.group,
-                    'success': self.has_succeeded,
-                    'attempt_count': 1
-                }
-            )
-
-            # only update the result if it hasn't succeeded yet
-            if not created and not existing_task.success:
-                existing_task.stopped = self.finished_at
-                existing_task.result = self.result
-                existing_task.success = self.has_succeeded
-                existing_task.attempt_count += 1
-                existing_task.save()
-
-            if (
-                Conf.MAX_ATTEMPTS > 0
-                and existing_task.attempt_count >= Conf.MAX_ATTEMPTS
-            ):
-                broker.acknowledge(self.ack_id)
-
-            return existing_task
-
-        except Exception:
-            logger.exception("Could not save task result")
-
-
-    def save_cached(self, broker: Broker):
-        task_key = f'{broker.list_key}:{self.id}'
-        timeout = self.cached
-        if timeout is True:
-            timeout = None
-        try:
-            group = self.group
-            iter_count = self.iter_count
-            # if it's a group append to the group list
-            if group:
-                group_key = f"{broker.list_key}:{group}:keys"
-                group_list = broker.cache.get(group_key) or []
-                # if it's an iter group, check if we are ready
-                if iter_count and len(group_list) == iter_count - 1:
-                    group_args = f"{broker.list_key}:{group}:args"
-                    # collate the results into a Task result
-                    results = [
-                        SignedPackage.loads(broker.cache.get(k)).result
-                        for k in group_list
-                    ]
-                    results.append(self.result)
-                    self.result = results
-                    self.id = group
-                    self.args = SignedPackage.loads(broker.cache.get(group_args))
-                    self.iter_count = None
-                    self.group = None
-                    if self.iter_cached:
-                        self.cached = self.iter_cached
-                        self.save_cached(broker=broker)
-                    else:
-                        self.save_to_db(broker)
-                    broker.cache.delete_many(group_list)
-                    broker.cache.delete_many([group_key, group_args])
-                    return
-                # save the group list
-                group_list.append(task_key)
-                broker.cache.set(group_key, group_list, timeout)
-                # async_task next in a chain
-                if self.chain:
-                    tasks.async_chain(
-                        self.chain,
-                        group=group,
-                        cached=self.cached,
-                        sync=self.sync,
-                        broker=broker,
-                    )
-            # save the task
-            broker.cache.set(task_key, SignedPackage.dumps(self), timeout)
-        except Exception:
-            logger.exception("Could not save task result")
-

django_q/queues.py

@@ -0,0 +1,82 @@
+"""
+The code is derived from
+https://github.com/althonos/pronto/commit/3384010dfb4fc7c66a219f59276adef3288a886b
+"""
+import multiprocessing
+import multiprocessing.queues
+import sys
+
+
+class SharedCounter:
+    """A synchronized shared counter.
+
+    The locking done by multiprocessing.Value ensures that only a single
+    process or thread may read or write the in-memory ctypes object. However,
+    in order to do n += 1, Python performs a read followed by a write, so a
+    second process may read the old value before the new one is written by
+    the first process. The solution is to use a multiprocessing.Lock to
+    guarantee the atomicity of the modifications to Value.
+
+    This class comes almost entirely from Eli Bendersky's blog:
+    http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing/
+    """
+
+    def __init__(self, n=0):
+        self.count = multiprocessing.Value("i", n)
+
+    def increment(self, n=1):
+        """Increment the counter by n (default = 1)"""
+        with self.count.get_lock():
+            self.count.value += n
+
+    @property
+    def value(self):
+        """Return the value of the counter"""
+        return self.count.value
+
+
+class Queue(multiprocessing.queues.Queue):
+    """A portable implementation of multiprocessing.Queue.
+
+    Because of multithreading / multiprocessing semantics, Queue.qsize() may
+    raise the NotImplementedError exception on Unix platforms like Mac OS X
+    where sem_getvalue() is not implemented. This subclass addresses this
+    problem by using a synchronized shared counter (initialized to zero) and
+    increasing / decreasing its value every time the put() and get() methods
+    are called, respectively. This not only prevents NotImplementedError from
+    being raised, but also allows us to implement a reliable version of both
+    qsize() and empty().
+    """
+
+    def __init__(self, *args, **kwargs):
+        if sys.version_info < (3, 0):
+            super(Queue, self).__init__(*args, **kwargs)
+        else:
+            super(Queue, self).__init__(
+                *args, ctx=multiprocessing.get_context(), **kwargs
+            )
+        self.size = SharedCounter(0)
+
+    def __getstate__(self):
+        return super(Queue, self).__getstate__() + (self.size,)
+
+    def __setstate__(self, state):
+        super(Queue, self).__setstate__(state[:-1])
+        self.size = state[-1]
+
+    def put(self, *args, **kwargs):
+        super(Queue, self).put(*args, **kwargs)
+        self.size.increment(1)
+
+    def get(self, *args, **kwargs):
+        x = super(Queue, self).get(*args, **kwargs)
+        self.size.increment(-1)
+        return x
+
+    def qsize(self) -> int:
+        """Reliable implementation of multiprocessing.Queue.qsize()"""
+        return self.size.value
+
+    def empty(self) -> bool:
+        """Reliable implementation of multiprocessing.Queue.empty()"""
+        return not self.qsize() > 0

django_q/scheduler.py

@@ -1,125 +0,0 @@
-from django_q.utils import localtime
-import uuid
-from django_q.models import Schedule
-from django_q import tasks
-import ast
-from django_q.humanhash import humanize
-from django import db
-from time import sleep
-from django_q.brokers import get_broker
-from django.utils import timezone
-
-from multiprocessing import Value, current_process
-from django_q.utils import close_old_django_connections
-from django.utils.translation import gettext_lazy as _
-from django_q.conf import Conf, logger
-from django_q.process_manager import ProcessManager
-
-
-
-class Scheduler(ProcessManager):
-    """The Scheduler is responsible for scheduling new tasks"""
-
-    @staticmethod
-    def schedule_tasks(broker=None):
-        logger.debug("Start sheduling")
-        if broker is None:
-            broker = get_broker()
-        q_default = db.models.Q(cluster__isnull=True) if Conf.CLUSTER_NAME == Conf.PREFIX else db.models.Q(pk__in=[])
-        with db.transaction.atomic(using=db.router.db_for_write(Schedule)):
-            for s in (
-                Schedule.objects.select_for_update()
-                .exclude(repeats=0)
-                .filter(db.models.Q(next_run__lt=timezone.now()), q_default | db.models.Q(cluster=Conf.CLUSTER_NAME))
-            ):
-                args = s.parse_args()
-                kwargs = s.parse_kwargs()
-                q_options = kwargs.get("q_options", {})
-                if s.intended_date_kwarg:
-                    kwargs[s.intended_date_kwarg] = s.next_run.isoformat()
-                if s.hook:
-                    q_options["hook"] = s.hook
-                # set up the next run time
-                if s.schedule_type != s.ONCE:
-                    next_run = s.calculate_next_run(s.next_run)
-                    if not Conf.CATCH_UP:
-                        while next_run <= localtime():
-                            next_run = s.calculate_next_run(next_run)
-
-                    s.next_run = next_run
-                    s.repeats += -1
-                # send it to the cluster; any cluster name is allowed in multi-queue scenarios
-                # because `broker_name` is confusing, using `cluster` name is recommended and take
-                q_options["cluster"] = s.cluster or q_options.get("cluster", q_options.pop("broker_name", None))
-                if q_options['cluster'] is None or q_options['cluster'] == Conf.CLUSTER_NAME:
-                    q_options["broker"] = broker
-
-                q_options["group"] = q_options.get("group", s.name or s.id)
-                kwargs["q_options"] = q_options
-
-                s.task = tasks.async_task(s.func, *args, **kwargs)
-                # log it
-                if not s.task:
-                    logger.error(
-                        _(
-                            "%(process_name)s failed to create a task from schedule "
-                            "[%(schedule)s]"
-                        )
-                        % {
-                            "process_name": current_process().name,
-                            "schedule": s.name or s.id,
-                        }
-                    )
-                else:
-                    logger.info(
-                        _(
-                            "%(process_name)s created task %(task_name)s from schedule "
-                            "[%(schedule)s]"
-                        )
-                        % {
-                            "process_name": current_process().name,
-                            "task_name": humanize(s.task),
-                            "schedule": s.name or s.id,
-                        }
-                    )
-                # default behavior is to delete a ONCE schedule
-                if s.schedule_type == s.ONCE:
-                    if s.repeats < 0:
-                        s.delete()
-                        continue
-                    # but not if it has a positive repeats
-                    s.repeats = 0
-                # save the schedule
-                s.save()
-
-    def get_target(self):
-        return self.run_scheduler
-
-    def stop_scheduler(self) -> None:
-        # send stop signal to worker
-        try:
-            self.manager_pipe.send("STOP")
-        except BrokenPipeError:
-            # recycle process if pipe is broken
-            self.status.value = ProcessManager.Status.DONE.value
-
-    def run_scheduler(self, status, pipe) -> None:
-        self.process_name = current_process().name
-        self.process_id = current_process().pid
-        status.value = self.Status.BUSY.value
-        logger.info(
-            _("%(proc_name)s scheduling at %(id)s")
-            % {"proc_name": self.process_name, "id": self.process_id}
-        )
-        while True:
-            if pipe.poll() and pipe.recv() == "STOP":
-                status.value = self.Status.DONE.value
-                break
-            broker = get_broker()
-            close_old_django_connections()
-            try:
-                Scheduler.schedule_tasks(broker=broker)
-            except Exception:
-                logger.exception("Could not create task from schedule")
-            # sleep 60 seconds for next schedule
-            sleep(60)

django_q/signals.py

@@ -31,6 +31,8 @@ def call_hook(sender, instance, **kwargs):
                 % {"hook": instance.hook, "name": instance.name, "error": str(e)}
             )
 
+# args: proc_name
+post_spawn = Signal()
 
 # args: task
 pre_enqueue = Signal()

django_q/status.py

@@ -41,7 +41,14 @@ class Stat(Status):
         self.status = sentinel.status()
         self.done_q_size = 0
         self.task_q_size = 0
-        self.workers = [w.process.pid for w in sentinel.pool.workers]
+        if Conf.QSIZE:
+            self.done_q_size = sentinel.result_queue.qsize()
+            self.task_q_size = sentinel.task_queue.qsize()
+        if sentinel.monitor:
+            self.monitor = sentinel.monitor.pid
+        if sentinel.pusher:
+            self.pusher = sentinel.pusher.pid
+        self.workers = [w.pid for w in sentinel.pool]
 
     def uptime(self) -> float:
         return (timezone.now() - self.tob).total_seconds()

django_q/tasks.py

@@ -1,7 +1,5 @@
 """Provides task functionality."""
 # Standard
-from django_q.helpers import run_cluster_once
-from django_q.queue_task import QueueTask
 from multiprocessing import Value
 from time import sleep, time
 
@@ -14,13 +12,14 @@ from django_q.brokers import get_broker
 from django_q.conf import Conf, logger
 from django_q.humanhash import uuid
 from django_q.models import Schedule, Task
+from django_q.queues import Queue
 from django_q.signals import pre_enqueue
 from django_q.signing import SignedPackage
 
 
 def async_task(func, *args, **kwargs):
     """Queue a task for the cluster."""
-    given_kwargs = kwargs.copy()
+    keywords = kwargs.copy()
     opt_keys = (
         "hook",
         "group",
@@ -35,40 +34,47 @@ def async_task(func, *args, **kwargs):
         "cluster",
         "timeout",
     )
-    q_options = given_kwargs.pop("q_options", {})
+    q_options = keywords.pop("q_options", {})
     # get an id
     tag = uuid()
     # build the task package
-    task = QueueTask(
-        id=tag[1],
-        name=given_kwargs.pop("task_name", None) or q_options.pop("task_name", None) or tag[0],
-        func=func,
-        args=args
-    )
-
-    # don't serialize the broker
-    broker = given_kwargs.pop("broker", None) or q_options.pop("broker", None) or get_broker(task.cluster) or get_broker()
-
-    print(broker.list_key)
-
+    task = {
+        "id": tag[1],
+        "name": keywords.pop("task_name", None)
+        or q_options.pop("task_name", None)
+        or tag[0],
+        "func": func,
+        "args": args,
+    }
     # push optionals
     for key in opt_keys:
-        if key in q_options or key in given_kwargs:
-            setattr(task, key, q_options.pop(key, None) or given_kwargs.pop(key, None))
-
+        if q_options and key in q_options:
+            task[key] = q_options[key]
+        elif key in keywords:
+            task[key] = keywords.pop(key)
+    # don't serialize the broker
+    broker = task.pop("broker", None) or get_broker(task.get("cluster"))
+    # overrides
+    if "cached" not in task and Conf.CACHED:
+        task["cached"] = Conf.CACHED
+    if "sync" not in task and Conf.SYNC:
+        task["sync"] = Conf.SYNC
+    if "ack_failure" not in task and Conf.ACK_FAILURES:
+        task["ack_failure"] = Conf.ACK_FAILURES
     # finalize
-    task.kwargs = given_kwargs
+    task["kwargs"] = keywords
+    task["started"] = timezone.now()
     # signal it
     pre_enqueue.send(sender="django_q", task=task)
     # sign it
     pack = SignedPackage.dumps(task)
-    if task.sync:
+    if task.get("sync", False):
         return _sync(pack)
     # push it
     enqueue_id = broker.enqueue(pack)
     logger.info(f"Enqueued [{broker.list_key}] {enqueue_id}")
     logger.debug(f"Pushed {tag}")
-    return task.id
+    return task["id"]
 
 
 def schedule(func, *args, **kwargs):
@@ -105,7 +111,7 @@ def schedule(func, *args, **kwargs):
         raise IntegrityError("A schedule with the same name already exists.")
 
     # create and return the schedule
-    schedule = Schedule(
+    s = Schedule(
         name=name,
         func=func,
         hook=hook,
@@ -119,9 +125,11 @@ def schedule(func, *args, **kwargs):
         cluster=cluster,
         intended_date_kwarg=intended_date_kwarg,
     )
-    schedule.full_clean()
-    schedule.save()
-    return schedule
+    # make sure we trigger validation
+    s.full_clean()
+    s.save()
+    return s
+
 
 def result(task_id, wait=0, cached=Conf.CACHED):
     """
@@ -157,7 +165,7 @@ def result_cached(task_id, wait=0, broker=None):
     while True:
         r = broker.cache.get(f"{broker.list_key}:{task_id}")
         if r:
-            return SignedPackage.loads(r).result
+            return SignedPackage.loads(r)["result"]
         if (time() - start) * 1000 >= wait >= 0:
             break
         sleep(0.01)
@@ -216,8 +224,8 @@ def result_group_cached(group_id, failures=False, wait=0, count=None, broker=Non
             result_list = []
             for task_key in group_list:
                 task = SignedPackage.loads(broker.cache.get(task_key))
-                if task.has_succeeded or failures:
-                    result_list.append(task.result)
+                if task["success"] or failures:
+                    result_list.append(task["result"])
             return result_list
         if (time() - start) * 1000 >= wait >= 0:
             break
@@ -260,17 +268,17 @@ def fetch_cached(task_id, wait=0, broker=None):
         if r:
             task = SignedPackage.loads(r)
             return Task(
-                id=task.id,
-                name=task.name,
-                func=task.func,
-                hook=task.hook,
-                args=task.args,
-                kwargs=task.kwargs,
-                cluster=task.cluster,
-                started=task.started_at,
-                stopped=task.finished_at,
-                result=task.result,
-                success=task.has_succeeded,
+                id=task["id"],
+                name=task["name"],
+                func=task["func"],
+                hook=task.get("hook"),
+                args=task["args"],
+                kwargs=task["kwargs"],
+                cluster=task.get("cluster"),
+                started=task["started"],
+                stopped=task["stopped"],
+                result=task["result"],
+                success=task["success"],
             )
         if (time() - start) * 1000 >= wait >= 0:
             break
@@ -329,20 +337,20 @@ def fetch_group_cached(group_id, failures=True, wait=0, count=None, broker=None)
             task_list = []
             for task_key in group_list:
                 task = SignedPackage.loads(broker.cache.get(task_key))
-                if task.has_succeeded or failures:
+                if task["success"] or failures:
                     t = Task(
-                        id=task.id,
-                        name=task.name,
-                        func=task.func,
-                        hook=task.hook,
-                        args=task.args,
-                        kwargs=task.kwargs,
-                        cluster=task.cluster,
-                        started=task.started_at,
-                        stopped=task.finished_at,
-                        result=task.result,
-                        group=task.group,
-                        success=task.has_succeeded,
+                        id=task["id"],
+                        name=task["name"],
+                        func=task["func"],
+                        hook=task.get("hook"),
+                        args=task["args"],
+                        kwargs=task["kwargs"],
+                        cluster=task.get("cluster"),
+                        started=task["started"],
+                        stopped=task["stopped"],
+                        result=task["result"],
+                        group=task.get("group"),
+                        success=task["success"],
                     )
                     task_list.append(t)
             return task_list
@@ -379,7 +387,7 @@ def count_group_cached(group_id, failures=False, broker=None):
         failure_count = 0
         for task_key in group_list:
             task = SignedPackage.loads(broker.cache.get(task_key))
-            if not task.has_succeeded:
+            if not task["success"]:
                 failure_count += 1
         return failure_count
 
@@ -755,7 +763,18 @@ class AsyncTask:
 
 def _sync(pack):
     """Simulate a package travelling through the cluster."""
-    task = SignedPackage.loads(pack)
-    run_cluster_once(workers=1, tasks=[task])
+    from django_q.cluster import monitor, worker
 
-    return task.id
+    task_queue = Queue()
+    result_queue = Queue()
+    task = SignedPackage.loads(pack)
+    task_queue.put(task)
+    task_queue.put("STOP")
+    worker(task_queue, result_queue, Value("f", -1))
+    result_queue.put("STOP")
+    monitor(result_queue)
+    task_queue.close()
+    task_queue.join_thread()
+    result_queue.close()
+    result_queue.join_thread()
+    return task["id"]

django_q/tests/test_cached.py

@@ -1,11 +1,11 @@
-from django_q.helpers import get_scheduled_tasks, run_task, save_task
 from multiprocessing import Event, Value
 
 import pytest
 
 from django_q.brokers import get_broker
+from django_q.cluster import monitor, pusher, worker
 from django_q.conf import Conf
-from queue import Queue
+from django_q.queues import Queue
 from django_q.tasks import (
     AsyncTask,
     Chain,
@@ -54,14 +54,20 @@ def test_cached(broker):
     # run a single inline cluster
     task_count = 17
     assert broker.queue_size() == task_count
-    tasks = []
-    for task in range(17):
-        tasks += get_scheduled_tasks(broker=broker)
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
+    for i in range(task_count):
+        pusher(task_queue, stop_event, broker=broker)
     assert broker.queue_size() == 0
-    assert len(tasks) == task_count
-    for task in tasks:
-        run_task(task=task)
-        save_task(task=task, broker=broker)
+    assert task_queue.qsize() == task_count
+    task_queue.put("STOP")
+    result_queue = Queue()
+    worker(task_queue, result_queue, Value("f", -1))
+    assert result_queue.qsize() == task_count
+    result_queue.put("STOP")
+    monitor(result_queue)
+    assert result_queue.qsize() == 0
     # assert results
     assert result(task_id, wait=500, cached=True) == -1
     assert fetch(task_id, wait=500, cached=True).result == -1
@@ -155,7 +161,6 @@ def test_chain(broker):
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_asynctask_class(broker, monkeypatch):
     broker.purge_queue()
     broker.cache.clear()

django_q/tests/test_cluster.py

@@ -1,8 +1,4 @@
-from django_q.worker import Worker
-from django_q.queue_task import QueueTask
-from django_q.helpers import get_scheduled_tasks, run_cluster_once, run_task, save_task
 import os
-import copy
 import sys
 import threading
 import uuid as uuidlib
@@ -16,11 +12,11 @@ import pytest
 from django.utils import timezone
 
 from django_q.brokers import Broker, get_broker
-from django_q.cluster import Cluster, Sentinel
+from django_q.cluster import Cluster, Sentinel, monitor, pusher, save_task, worker
 from django_q.conf import Conf
 from django_q.humanhash import DEFAULT_WORDLIST, uuid
 from django_q.models import Success, Task
-from queue import Queue
+from django_q.queues import Queue
 from django_q.signals import post_execute, pre_enqueue, pre_execute
 from django_q.status import Stat
 from django_q.tasks import (
@@ -72,21 +68,42 @@ def test_sync_raise_exception(broker):
         async_task("django_q.tests.tasks.raise_exception", broker=broker, sync=True)
 
 
-# @pytest.mark.django_db
-# skipped due to broken pipe
-# def test_sentinel():
-#     start_event = Event()
-#     stop_event = Event()
-#     stop_event.set()
-#     cluster_id = uuidlib.uuid4()
-#     s = Sentinel(
-#         stop_event,
-#         start_event,
-#         cluster_id=cluster_id,
-#         broker=get_broker("sentinel_test:q"),
-#     )
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPING
+@pytest.mark.django_db
+def test_cluster_initial(broker):
+    broker.list_key = "initial_test:q"
+    broker.delete_queue()
+    c = Cluster(broker=broker)
+    assert c.sentinel is None
+    assert c.stat.status == Conf.STOPPED
+    assert c.start() > 0
+    assert c.sentinel.is_alive() is True
+    assert c.is_running
+    assert c.is_stopping is False
+    assert c.is_starting is False
+    sleep(0.5)
+    stat = c.stat
+    assert stat.status == Conf.IDLE
+    assert c.stop() is True
+    assert c.sentinel.is_alive() is False
+    assert c.has_stopped
+    assert c.stop() is False
+    broker.delete_queue()
+
+
+@pytest.mark.django_db
+def test_sentinel():
+    start_event = Event()
+    stop_event = Event()
+    stop_event.set()
+    cluster_id = uuidlib.uuid4()
+    s = Sentinel(
+        stop_event,
+        start_event,
+        cluster_id=cluster_id,
+        broker=get_broker("sentinel_test:q"),
+    )
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
 
 
 @pytest.mark.django_db
@@ -97,16 +114,27 @@ def test_cluster(broker):
         "django_q.tests.tasks.count_letters", DEFAULT_WORDLIST, broker=broker
     )
     assert broker.queue_size() == 1
+    task_queue = Queue()
+    assert task_queue.qsize() == 0
+    result_queue = Queue()
+    assert result_queue.qsize() == 0
+    event = Event()
+    event.set()
     # Test push
-    tasks = get_scheduled_tasks(broker=broker)
-    assert len(tasks) == 1
+    pusher(task_queue, event, broker=broker)
+    assert task_queue.qsize() == 1
     assert queue_size(broker=broker) == 0
     # Test work
-    task = run_task(tasks[0])
+    task_queue.put("STOP")
+    worker(task_queue, result_queue, Value("f", -1))
+    assert task_queue.qsize() == 0
+    assert result_queue.qsize() == 1
     # Test monitor
-    save_task(task=task)
+    result_queue.put("STOP")
+    monitor(result_queue)
+    assert result_queue.qsize() == 0
     # check result
-    assert result(task.id) == 1506
+    assert result(task) == 1506
     broker.delete_queue()
 
 
@@ -183,14 +211,15 @@ def test_enqueue(broker, admin_user):
     # run the cluster to execute the tasks
     task_count = 10
     assert broker.queue_size() == task_count
+    task_queue = Queue()
     stop_event = Event()
     stop_event.set()
     # push the tasks
-    tasks = []
     for _ in range(task_count):
-        tasks += get_scheduled_tasks(broker=broker)
+        pusher(task_queue, stop_event, broker=broker)
     assert broker.queue_size() == 0
-    assert len(tasks) == task_count
+    assert task_queue.qsize() == task_count
+    task_queue.put("STOP")
     # test wait timeout
     assert result(j, wait=10) is None
     assert fetch(j, wait=10) is None
@@ -199,10 +228,13 @@ def test_enqueue(broker, admin_user):
     assert fetch_group("test_j", wait=10) is None
     assert fetch_group("test_j", count=2, wait=10) is None
     # let a worker handle them
-    for task in tasks:
-        run_task(task=task)
-        save_task(task=task)
-
+    result_queue = Queue()
+    worker(task_queue, result_queue, Value("f", -1))
+    assert result_queue.qsize() == task_count
+    result_queue.put("STOP")
+    # store the results
+    monitor(result_queue)
+    assert result_queue.qsize() == 0
     # Check the results
     # task a
     result_a = fetch(a)
@@ -228,11 +260,10 @@ def test_enqueue(broker, admin_user):
     assert result_e.success is True
     assert result(e) is None
     # task f
-    # @TODO: fix this
-    # result_f = fetch(f)
-    # assert result_f is not None
-    # assert result_f.success is True
-    # assert result(f) == 1506
+    result_f = fetch(f)
+    assert result_f is not None
+    assert result_f.success is True
+    assert result(f) == 1506
     # task g
     result_g = fetch(g)
     assert result_g is not None
@@ -275,153 +306,152 @@ def test_enqueue(broker, admin_user):
     broker.delete_queue()
 
 
-# @pytest.mark.django_db
-# @pytest.mark.parametrize(
-#     "cluster_config_timeout, async_task_kwargs",
-#     (
-#         (1, {}),
-#         (10, {"timeout": 1}),
-#         (None, {"timeout": 1}),
-#     ),
-# )
-# def test_timeout(broker, cluster_config_timeout, async_task_kwargs):
-#     # set up the Sentinel
-#     broker.list_key = "timeout_test:q"
-#     broker.purge_queue()
-#     async_task("time.sleep", 5, broker=broker, **async_task_kwargs)
-#     start_event = Event()
-#     stop_event = Event()
-#     cluster_id = uuidlib.uuid4()
-#     # Set a timer to stop the Sentinel
-#     threading.Timer(3, stop_event.set).start()
-#     s = Sentinel(
-#         stop_event,
-#         start_event,
-#         cluster_id=cluster_id,
-#         broker=broker,
-#         timeout=cluster_config_timeout,
-#     )
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPED
-#     assert s.reincarnations == 1
-#     broker.delete_queue()
-
-
-# @pytest.mark.django_db
-# @pytest.mark.parametrize(
-#     "cluster_config_timeout, async_task_kwargs",
-#     (
-#         (5, {}),
-#         (10, {"timeout": 5}),
-#         (1, {"timeout": 5}),
-#         (None, {"timeout": 5}),
-#     ),
-# )
-# def test_timeout_task_finishes(broker, cluster_config_timeout, async_task_kwargs):
-#     # set up the Sentinel
-#     broker.list_key = "timeout_test:q"
-#     broker.purge_queue()
-#     async_task("time.sleep", 3, broker=broker, **async_task_kwargs)
-#     start_event = Event()
-#     stop_event = Event()
-#     cluster_id = uuidlib.uuid4()
-#     # Set a timer to stop the Sentinel
-#     threading.Timer(6, stop_event.set).start()
-#     s = Sentinel(
-#         stop_event,
-#         start_event,
-#         cluster_id=cluster_id,
-#         broker=broker,
-#         timeout=cluster_config_timeout,
-#     )
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPED
-#     assert s.reincarnations == 0
-#     broker.delete_queue()
-
-
-# @pytest.mark.django_db
-# def test_recycle(broker, monkeypatch):
-#     # set up the Sentinel
-#     broker.list_key = "test_recycle_test:q"
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     start_event = Event()
-#     stop_event = Event()
-#     cluster_id = uuidlib.uuid4()
-#     # override settings
-#     monkeypatch.setattr(Conf, "RECYCLE", 2)
-#     monkeypatch.setattr(Conf, "WORKERS", 1)
-#     # set a timer to stop the Sentinel
-#     threading.Timer(3, stop_event.set).start()
-#     s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPED
-#     assert s.reincarnations == 1
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     task_queue = Queue()
-#     result_queue = Queue()
-#     # push two tasks
-#     # pusher(task_queue, stop_event, broker=broker)
-#     # pusher(task_queue, stop_event, broker=broker)
-#     # worker should exit on recycle
-#     # worker(task_queue, result_queue, Value("f", -1))
-#     # check if the work has been done
-#     assert result_queue.qsize() == 2
-#     # save_limit test
-#     monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
-#     result_queue.put("STOP")
-#     # run monitor
-#     # monitor(result_queue)
-#     assert Success.objects.count() == Conf.SAVE_LIMIT
-#     broker.delete_queue()
-
-
-# @pytest.mark.django_db
-# def test_save_limit_per_func(broker, monkeypatch):
-#     # set up the Sentinel
-#     broker.list_key = "test_recycle_test:q"
-#     async_task("django_q.tests.tasks.hello", broker=broker)
-#     async_task("django_q.tests.tasks.countdown", 2, broker=broker)
-#     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-#     start_event = Event()
-#     stop_event = Event()
-#     cluster_id = uuidlib.uuid4()
-#     task_queue = Queue()
-#     result_queue = Queue()
-#     # override settings
-#     monkeypatch.setattr(Conf, "RECYCLE", 3)
-#     monkeypatch.setattr(Conf, "WORKERS", 1)
-#     # set a timer to stop the Sentinel
-#     threading.Timer(3, stop_event.set).start()
-#     # for i in range(3):
-#     #     pusher(task_queue, stop_event, broker=broker)
-#     # worker(task_queue, result_queue, Value("f", -1))
-#     s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
-#     assert start_event.is_set()
-#     assert s.status() == Conf.STOPPED
-#     # worker should exit on recycle
-#     # check if the work has been done
-#     assert result_queue.qsize() == 3
-#     # save_limit test
-#     monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
-#     monkeypatch.setattr(Conf, "SAVE_LIMIT_PER", "func")
-#     result_queue.put("STOP")
-#     # run monitor
-#     # monitor(result_queue)
-#     assert Success.objects.count() == 3
-#     assert set(Success.objects.filter().values_list("func", flat=True)) == {
-#         "django_q.tests.tasks.countdown",
-#         "django_q.tests.tasks.hello",
-#         "django_q.tests.tasks.multiply",
-#     }
-#     broker.delete_queue()
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "cluster_config_timeout, async_task_kwargs",
+    (
+        (1, {}),
+        (10, {"timeout": 1}),
+        (None, {"timeout": 1}),
+    ),
+)
+def test_timeout(broker, cluster_config_timeout, async_task_kwargs):
+    # set up the Sentinel
+    broker.list_key = "timeout_test:q"
+    broker.purge_queue()
+    async_task("time.sleep", 5, broker=broker, **async_task_kwargs)
+    start_event = Event()
+    stop_event = Event()
+    cluster_id = uuidlib.uuid4()
+    # Set a timer to stop the Sentinel
+    threading.Timer(3, stop_event.set).start()
+    s = Sentinel(
+        stop_event,
+        start_event,
+        cluster_id=cluster_id,
+        broker=broker,
+        timeout=cluster_config_timeout,
+    )
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
+    assert s.reincarnations == 1
+    broker.delete_queue()
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "cluster_config_timeout, async_task_kwargs",
+    (
+        (5, {}),
+        (10, {"timeout": 5}),
+        (1, {"timeout": 5}),
+        (None, {"timeout": 5}),
+    ),
+)
+def test_timeout_task_finishes(broker, cluster_config_timeout, async_task_kwargs):
+    # set up the Sentinel
+    broker.list_key = "timeout_test:q"
+    broker.purge_queue()
+    async_task("time.sleep", 3, broker=broker, **async_task_kwargs)
+    start_event = Event()
+    stop_event = Event()
+    cluster_id = uuidlib.uuid4()
+    # Set a timer to stop the Sentinel
+    threading.Timer(6, stop_event.set).start()
+    s = Sentinel(
+        stop_event,
+        start_event,
+        cluster_id=cluster_id,
+        broker=broker,
+        timeout=cluster_config_timeout,
+    )
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
+    assert s.reincarnations == 0
+    broker.delete_queue()
+
+
+@pytest.mark.django_db
+def test_recycle(broker, monkeypatch):
+    # set up the Sentinel
+    broker.list_key = "test_recycle_test:q"
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    start_event = Event()
+    stop_event = Event()
+    cluster_id = uuidlib.uuid4()
+    # override settings
+    monkeypatch.setattr(Conf, "RECYCLE", 2)
+    monkeypatch.setattr(Conf, "WORKERS", 1)
+    # set a timer to stop the Sentinel
+    threading.Timer(3, stop_event.set).start()
+    s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
+    assert s.reincarnations == 1
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    task_queue = Queue()
+    result_queue = Queue()
+    # push two tasks
+    pusher(task_queue, stop_event, broker=broker)
+    pusher(task_queue, stop_event, broker=broker)
+    # worker should exit on recycle
+    worker(task_queue, result_queue, Value("f", -1))
+    # check if the work has been done
+    assert result_queue.qsize() == 2
+    # save_limit test
+    monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
+    result_queue.put("STOP")
+    # run monitor
+    monitor(result_queue)
+    assert Success.objects.count() == Conf.SAVE_LIMIT
+    broker.delete_queue()
+
+
+@pytest.mark.django_db
+def test_save_limit_per_func(broker, monkeypatch):
+    # set up the Sentinel
+    broker.list_key = "test_recycle_test:q"
+    async_task("django_q.tests.tasks.hello", broker=broker)
+    async_task("django_q.tests.tasks.countdown", 2, broker=broker)
+    async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
+    start_event = Event()
+    stop_event = Event()
+    cluster_id = uuidlib.uuid4()
+    task_queue = Queue()
+    result_queue = Queue()
+    # override settings
+    monkeypatch.setattr(Conf, "RECYCLE", 3)
+    monkeypatch.setattr(Conf, "WORKERS", 1)
+    # set a timer to stop the Sentinel
+    threading.Timer(3, stop_event.set).start()
+    for i in range(3):
+        pusher(task_queue, stop_event, broker=broker)
+    worker(task_queue, result_queue, Value("f", -1))
+    s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
+    assert start_event.is_set()
+    assert s.status() == Conf.STOPPED
+    # worker should exit on recycle
+    # check if the work has been done
+    assert result_queue.qsize() == 3
+    # save_limit test
+    monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
+    monkeypatch.setattr(Conf, "SAVE_LIMIT_PER", "func")
+    result_queue.put("STOP")
+    # run monitor
+    monitor(result_queue)
+    assert Success.objects.count() == 3
+    assert set(Success.objects.filter().values_list("func", flat=True)) == {
+        "django_q.tests.tasks.countdown",
+        "django_q.tests.tasks.hello",
+        "django_q.tests.tasks.multiply",
+    }
+    broker.delete_queue()
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_max_rss(broker, monkeypatch):
     # set up the Sentinel
     broker.list_key = "test_max_rss_test:q"
@@ -436,18 +466,27 @@ def test_max_rss(broker, monkeypatch):
     threading.Timer(3, stop_event.set).start()
     s = Sentinel(stop_event, start_event, cluster_id=cluster_id, broker=broker)
     assert start_event.is_set()
-    assert s.status() == Conf.STOPPING
+    assert s.status() == Conf.STOPPED
     assert s.reincarnations == 1
     async_task("django_q.tests.tasks.multiply", 2, 2, broker=broker)
-    for _ in range(2):
-        get_scheduled_tasks(broker=broker)
-    worker = s.pool.workers[0]
-    s.pool.delegate_tasks()
-    assert worker.status == Worker.Status.Idle
+    task_queue = Queue()
+    result_queue = Queue()
+    # push the task
+    pusher(task_queue, stop_event, broker=broker)
+    # worker should exit on recycle
+    worker(task_queue, result_queue, Value("f", -1))
+    # check if the work has been done
+    assert result_queue.qsize() == 1
+    # save_limit test
+    monkeypatch.setattr(Conf, "SAVE_LIMIT", 1)
+    result_queue.put("STOP")
+    # run monitor
+    monitor(result_queue)
+    assert Success.objects.count() == Conf.SAVE_LIMIT
+    broker.delete_queue()
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_bad_secret(broker, monkeypatch):
     broker.list_key = "test_bad_secret:q"
     async_task("math.copysign", 1, -1, broker=broker)
@@ -464,8 +503,16 @@ def test_bad_secret(broker, monkeypatch):
     stat = Stat.get_all()
     assert len(stat) == 0
     assert Stat.get(pid=s.parent_pid, cluster_id=cluster_id) is None
-    task = get_scheduled_tasks(broker=broker)
-    assert task == []
+    task_queue = Queue()
+    pusher(task_queue, stop_event, broker=broker)
+    result_queue = Queue()
+    task_queue.put("STOP")
+    worker(
+        task_queue,
+        result_queue,
+        Value("f", -1),
+    )
+    assert result_queue.qsize() == 0
     broker.delete_queue()
 
 
@@ -473,32 +520,32 @@ def test_bad_secret(broker, monkeypatch):
 def test_attempt_count(broker, monkeypatch):
     monkeypatch.setattr(Conf, "MAX_ATTEMPTS", 3)
     tag = uuid()
-    task = QueueTask(
-        id=tag[1],
-        name=tag[0],
-        func="math.copysign",
-        args=(1, -1),
-        kwargs={},
-        started_at=timezone.now(),
-        finished_at=timezone.now(),
-        status=QueueTask.Status.FAILED,
-        result=None,
-    )
+    task = {
+        "id": tag[1],
+        "name": tag[0],
+        "func": "math.copysign",
+        "args": (1, -1),
+        "kwargs": {},
+        "started": timezone.now(),
+        "stopped": timezone.now(),
+        "success": False,
+        "result": None,
+    }
     # initial save - no success
     save_task(task, broker)
-    assert Task.objects.filter(id=task.id).exists()
-    saved_task = Task.objects.get(id=task.id)
+    assert Task.objects.filter(id=task["id"]).exists()
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.attempt_count == 1
     sleep(0.5)
     # second save
-    task.finished_at = timezone.now()
+    task["stopped"] = timezone.now()
     save_task(task, broker)
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.attempt_count == 2
     # third save -
-    task.finished_at = timezone.now()
+    task["stopped"] = timezone.now()
     save_task(task, broker)
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.attempt_count == 3
     # task should be removed from queue
     assert broker.queue_size() == 0
@@ -507,43 +554,43 @@ def test_attempt_count(broker, monkeypatch):
 @pytest.mark.django_db
 def test_update_failed(broker):
     tag = uuid()
-    task = QueueTask(
-        id=tag[1],
-        name=tag[0],
-        func="math.copysign",
-        args=(1, -1),
-        kwargs={},
-        started_at=timezone.now(),
-        finished_at=timezone.now(),
-        status=QueueTask.Status.FAILED,
-        result=None,
-    )
+    task = {
+        "id": tag[1],
+        "name": tag[0],
+        "func": "math.copysign",
+        "args": (1, -1),
+        "kwargs": {},
+        "started": timezone.now(),
+        "stopped": timezone.now(),
+        "success": False,
+        "result": None,
+    }
     # initial save - no success
     save_task(task, broker)
-    assert Task.objects.filter(id=task.id).exists()
-    saved_task = Task.objects.get(id=task.id)
+    assert Task.objects.filter(id=task["id"]).exists()
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.success is False
     sleep(0.5)
     # second save - no success
-    old_stopped = task.finished_at
-    task.finished_at = timezone.now()
+    old_stopped = task["stopped"]
+    task["stopped"] = timezone.now()
     save_task(task, broker)
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.stopped > old_stopped
     # third save - success
-    task.finished_at = timezone.now()
-    task.result = "result"
-    task.status = QueueTask.Status.SUCCESS
+    task["stopped"] = timezone.now()
+    task["result"] = "result"
+    task["success"] = True
     save_task(task, broker)
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.success is True
     # fourth save - no success
-    task.result = None
-    task.status = QueueTask.Status.FAILED
-    task.finished_at = old_stopped
+    task["result"] = None
+    task["success"] = False
+    task["stopped"] = old_stopped
     save_task(task, broker)
     # should not overwrite success
-    saved_task = Task.objects.get(id=task.id)
+    saved_task = Task.objects.get(id=task["id"])
     assert saved_task.success is True
     assert saved_task.result == "result"
 
@@ -560,40 +607,47 @@ def test_acknowledge_failure_override():
             self.acknowledgements[task_id] = count + 1
 
     tag = uuid()
-    task_fail_ack = QueueTask(
-        id=tag[1],
-        name=tag[0],
-        ack_id="test_fail_ack_id",
-        ack_failure=True,
-        func="math.copysign",
-        args=(1, -1),
-        kwargs={},
-        started_at=timezone.now(),
-        finished_at=timezone.now(),
-        status=QueueTask.Status.SUCCESS,
-        result=None,
+    task_fail_ack = {
+        "id": tag[1],
+        "name": tag[0],
+        "ack_id": "test_fail_ack_id",
+        "ack_failure": True,
+        "func": "math.copysign",
+        "args": (1, -1),
+        "kwargs": {},
+        "started": timezone.now(),
+        "stopped": timezone.now(),
+        "success": False,
+        "result": None,
+    }
+
+    tag = uuid()
+    task_fail_no_ack = task_fail_ack.copy()
+    task_fail_no_ack.update(
+        {"id": tag[1], "name": tag[0], "ack_id": "test_fail_no_ack_id"}
     )
+    del task_fail_no_ack["ack_failure"]
 
     tag = uuid()
-    task_fail_no_ack = copy.deepcopy(task_fail_ack)
-    task_fail_no_ack.id = tag[1]
-    task_fail_no_ack.name = tag[0]
-    task_fail_no_ack.ack_id = None
-    task_fail_no_ack.ack_failure = False
-
-    tag = uuid()
-    task_success_ack = copy.deepcopy(task_fail_ack)
-    task_success_ack.id = tag[1]
-    task_success_ack.name = tag[0]
-    task_success_ack.ack_id = "test_success_ack_id"
-    task_success_ack.status = QueueTask.Status.SUCCESS
-    task_success_ack.ack_failure = False
+    task_success_ack = task_fail_ack.copy()
+    task_success_ack.update(
+        {
+            "id": tag[1],
+            "name": tag[0],
+            "ack_id": "test_success_ack_id",
+            "success": True,
+        }
+    )
+    del task_success_ack["ack_failure"]
 
+    result_queue = Queue()
+    result_queue.put(task_fail_ack)
+    result_queue.put(task_fail_no_ack)
+    result_queue.put(task_success_ack)
+    result_queue.put("STOP")
     broker = VerifyAckMockBroker(list_key="key")
 
-    save_task(task_fail_ack, broker=broker)
-    save_task(task_fail_no_ack, broker=broker)
-    save_task(task_success_ack, broker=broker)
+    monitor(result_queue, broker)
 
     assert broker.acknowledgements.get("test_fail_ack_id") == 1
     assert broker.acknowledgements.get("test_fail_no_ack_id") is None
@@ -606,7 +660,7 @@ class TestSignals:
         broker.list_key = "pre_enqueue_test:q"
         broker.delete_queue()
         self.signal_was_called: bool = False
-        self.task = None
+        self.task: Optional[dict] = None
 
         def handler(sender, task, **kwargs):
             self.signal_was_called = True
@@ -615,7 +669,7 @@ class TestSignals:
         pre_enqueue.connect(handler)
         task_id = async_task("math.copysign", 1, -1, broker=broker)
         assert self.signal_was_called is True
-        assert self.task.id == task_id
+        assert self.task.get("id") == task_id
         pre_enqueue.disconnect(handler)
         broker.delete_queue()
 
@@ -624,7 +678,7 @@ class TestSignals:
         broker.list_key = "pre_execute_test:q"
         broker.delete_queue()
         self.signal_was_called: bool = False
-        self.task = None
+        self.task: Optional[dict] = None
         self.func = None
 
         def handler(sender, task, func, **kwargs):
@@ -634,19 +688,27 @@ class TestSignals:
 
         pre_execute.connect(handler)
         task_id = async_task("math.copysign", 1, -1, broker=broker)
-        run_cluster_once(workers=1, broker=broker)
+        task_queue = Queue()
+        result_queue = Queue()
+        event = Event()
+        event.set()
+        pusher(task_queue, event, broker=broker)
+        task_queue.put("STOP")
+        worker(task_queue, result_queue, Value("f", -1))
+        result_queue.put("STOP")
+        monitor(result_queue, broker)
         broker.delete_queue()
-        assert self.task.id == task_id
         assert self.signal_was_called is True
-        assert self.func == 'math.copysign'
+        assert self.task.get("id") == task_id
+        assert self.func == copysign
         pre_execute.disconnect(handler)
 
     @pytest.mark.django_db
     def test_post_execute_signal(self, broker):
         broker.list_key = "post_execute_test:q"
         broker.delete_queue()
-        self.signal_was_called = False
-        self.task = None
+        self.signal_was_called: bool = False
+        self.task: Optional[dict] = None
         self.func = None
 
         def handler(sender, task, **kwargs):
@@ -655,11 +717,19 @@ class TestSignals:
 
         post_execute.connect(handler)
         task_id = async_task("math.copysign", 1, -1, broker=broker)
-        run_cluster_once(workers=1, broker=broker)
+        task_queue = Queue()
+        result_queue = Queue()
+        event = Event()
+        event.set()
+        pusher(task_queue, event, broker=broker)
+        task_queue.put("STOP")
+        worker(task_queue, result_queue, Value("f", -1))
+        result_queue.put("STOP")
+        monitor(result_queue, broker)
         broker.delete_queue()
         assert self.signal_was_called is True
-        assert self.task.id == task_id
-        assert self.task.result == -1
+        assert self.task.get("id") == task_id
+        assert self.task.get("result") == -1
         post_execute.disconnect(handler)
 
 

django_q/tests/test_commands.py

@@ -8,7 +8,6 @@ def test_qcluster():
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_qmonitor():
     call_command("qmonitor", run_once=True)
 
@@ -21,7 +20,6 @@ def test_qinfo():
 
 
 @pytest.mark.django_db
-@pytest.mark.skip("broken")
 def test_qmemory():
     call_command("qmemory", run_once=True)
     call_command("qmemory", workers=True, run_once=True)

django_q/tests/test_monitor.py

@@ -0,0 +1,52 @@
+import uuid
+
+import pytest
+
+from django_q.brokers import get_broker
+from django_q.cluster import Cluster
+from django_q.conf import Conf
+from django_q.monitor import get_ids, info, monitor
+from django_q.status import Stat
+from django_q.tasks import async_task
+
+
+@pytest.mark.django_db
+def test_monitor(monkeypatch):
+    cluster_id = uuid.uuid4()
+    assert Stat.get(pid=0, cluster_id=cluster_id).sentinel == 0
+    c = Cluster()
+    c.start()
+    stats = monitor(run_once=True)
+    assert get_ids() is True
+    c.stop()
+    assert len(stats) > 0
+    found_c = False
+    for stat in stats:
+        if stat.cluster_id == c.cluster_id:
+            found_c = True
+            assert stat.uptime() > 0
+            assert stat.empty_queues() is True
+            break
+    assert found_c
+    # test lock size
+    monkeypatch.setattr(Conf, "ORM", "default")
+    b = get_broker("monitor_test")
+    b.enqueue("test")
+    b.dequeue()
+    assert b.lock_size() == 1
+    monitor(run_once=True, broker=b)
+    b.delete_queue()
+
+
+@pytest.mark.django_db
+def test_info():
+    info()
+    do_sync()
+    info()
+    for _ in range(24):
+        do_sync()
+    info()
+
+
+def do_sync():
+    async_task("django_q.tests.tasks.countdown", 1, sync=True, save=True)

django_q/tests/test_scheduler.py

@@ -2,7 +2,6 @@ import os
 from datetime import datetime, timedelta
 from multiprocessing import Event, Value
 from unittest import mock
-from django_q.utils import localtime
 
 import pytest
 import django
@@ -13,9 +12,9 @@ from django.utils import timezone
 from django.utils.timezone import is_naive
 
 from django_q.brokers import Broker, get_broker
-from django_q.helpers import run_scheduler_once, get_scheduled_tasks, save_task, run_task
+from django_q.cluster import localtime, monitor, pusher, scheduler, worker
 from django_q.conf import Conf
-from queue import Queue
+from django_q.queues import Queue
 from django_q.tasks import Schedule, fetch
 from django_q.tasks import schedule as create_schedule
 from django_q.tests.settings import BASE_DIR
@@ -104,7 +103,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     )
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     # It's now the day after exactly at midnight UTC
@@ -116,7 +115,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     assert str(next_run) == "2021-03-28 01:00:00+01:00"
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -127,7 +126,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     assert str(next_run) == "2021-03-29 01:00:00+02:00"
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -148,7 +147,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     )
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -159,7 +158,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     assert str(next_run) == "2021-10-30 01:00:00+02:00"
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -170,7 +169,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
     assert str(next_run) == "2021-10-31 01:00:00+02:00"
 
     # Run scheduler so we get the next run date
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule.refresh_from_db()
 
     next_run = schedule.next_run
@@ -209,15 +208,24 @@ def test_scheduler(broker, monkeypatch):
             repeats=1,
         )
     # run scheduler
-    run_scheduler_once(broker=broker)
-    # get tasks
-    tasks = get_scheduled_tasks(broker=broker)
-    for task in tasks:
-        # let a worker handle them
-        ran_task = run_task(task)
-        # store the results
-        save_task(task=ran_task, broker=broker)
-
+    scheduler(broker=broker)
+    # set up the workflow
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
+    # push it
+    pusher(task_queue, stop_event, broker=broker)
+    assert task_queue.qsize() == 1
+    assert broker.queue_size() == 0
+    task_queue.put("STOP")
+    # let a worker handle them
+    result_queue = Queue()
+    worker(task_queue, result_queue, Value("b", -1))
+    assert result_queue.qsize() == 1
+    result_queue.put("STOP")
+    # store the results
+    monitor(result_queue)
+    assert result_queue.qsize() == 0
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.repeats == 0
     assert schedule.last_run() is not None
@@ -289,7 +297,7 @@ def test_scheduler(broker, monkeypatch):
         )
         assert schedule is not None
         assert schedule.last_run() is None
-        run_scheduler_once(broker=broker)
+        scheduler(broker=broker)
     # via model
     Schedule.objects.create(
         func="django_q.tests.tasks.word_multiply",
@@ -298,7 +306,7 @@ def test_scheduler(broker, monkeypatch):
         schedule_type=Schedule.DAILY,
     )
     # scheduler
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     # ONCE schedule should be deleted
     assert Schedule.objects.filter(pk=once_schedule.pk).exists() is False
     # Catch up On
@@ -312,12 +320,12 @@ def test_scheduler(broker, monkeypatch):
         next_run=timezone.now() - timedelta(hours=12),
         repeats=-1,
     )
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.next_run < now
     # Catch up off
     monkeypatch.setattr(Conf, "CATCH_UP", False)
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.next_run > now
     # Done
@@ -330,7 +338,7 @@ def test_scheduler(broker, monkeypatch):
         word="catch_up",
         schedule_type=Schedule.BIMONTHLY,
     )
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.next_run.date() == add_months(timezone.now(), 2).date()
 
@@ -341,7 +349,7 @@ def test_scheduler(broker, monkeypatch):
         word="catch_up",
         schedule_type=Schedule.BIWEEKLY,
     )
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     schedule = Schedule.objects.get(pk=schedule.pk)
     assert schedule.next_run.date() == (timezone.now() + timedelta(weeks=2)).date()
     broker.delete_queue()
@@ -359,12 +367,16 @@ def test_scheduler(broker, monkeypatch):
         repeats=1,
     )
     # run scheduler
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
+    # set up the workflow
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
     # push it
-    tasks = get_scheduled_tasks(broker=broker)
+    pusher(task_queue, stop_event, broker=broker)
 
     # queue must be empty
-    assert len(tasks) == 0
+    assert task_queue.qsize() == 0
 
     monkeypatch.setattr(Conf, "CLUSTER_NAME", "default")
     # create a schedule on the same cluster
@@ -379,12 +391,16 @@ def test_scheduler(broker, monkeypatch):
         repeats=1,
     )
     # run scheduler
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
+    # set up the workflow
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
     # push it
-    tasks = get_scheduled_tasks(broker=broker)
+    pusher(task_queue, stop_event, broker=broker)
 
     # queue must contain a task
-    assert len(tasks) == 1
+    assert task_queue.qsize() == 1
 
 
 @pytest.mark.django_db
@@ -406,31 +422,35 @@ def test_intended_schedule_kwarg(broker, monkeypatch):
     assert schedule.last_run() is None
     assert schedule.intended_date_kwarg == 'intended_date'
     # run scheduler
-    run_scheduler_once(broker=broker)
+    scheduler(broker=broker)
     # set up the workflow
-    scheduled_tasks = get_scheduled_tasks(broker=broker)
-    assert len(scheduled_tasks) == 1
-    task = scheduled_tasks[0]
-    assert 'intended_date' in task.kwargs
-    assert task.kwargs['intended_date'] == run_date.isoformat()
+    task_queue = Queue()
+    stop_event = Event()
+    stop_event.set()
+    # push it
+    pusher(task_queue, stop_event, broker=broker)
+    assert task_queue.qsize() == 1
+    task = task_queue.get()
+    assert 'intended_date' in task['kwargs']
+    assert task['kwargs']['intended_date'] == run_date.isoformat()
 
 
-# @override_settings(
-#     DATABASE_ROUTERS=REPLICA_DATABASE_ROUTERS, DATABASES=REPLICA_DATABASES
-# )
-# @pytest.mark.django_db
-# def test_scheduler_atomic_must_specify_the_write_db(
-#     orm_broker: Broker,
-# ):
-#     """
-#     GIVEN a environment with a read/write configured replica database
-#     WHEN the scheduler is called
-#     THEN the transaction must be called with the write database.
-#     """
-#     broker = get_broker(list_key="scheduler_test:q")
-#     with mock.patch("django_q.scheduler.db.transaction") as mocked_db:
-#         run_scheduler_once(broker=broker)
-#         mocked_db.atomic.assert_called_with(using="writable")
+@override_settings(
+    DATABASE_ROUTERS=REPLICA_DATABASE_ROUTERS, DATABASES=REPLICA_DATABASES
+)
+@pytest.mark.django_db
+def test_scheduler_atomic_must_specify_the_write_db(
+    orm_broker: Broker,
+):
+    """
+    GIVEN a environment with a read/write configured replica database
+    WHEN the scheduler is called
+    THEN the transaction must be called with the write database.
+    """
+    broker = get_broker(list_key="scheduler_test:q")
+    with mock.patch("django_q.cluster.db.transaction") as mocked_db:
+        scheduler(broker=broker)
+        mocked_db.atomic.assert_called_with(using="writable")
 
 
 @override_settings(
@@ -447,7 +467,7 @@ def test_scheduler_atomic_must_specify_the_database_based_on_router_redirection(
     """
     broker = get_broker(list_key="scheduler_test:q")
     with mock.patch("django_q.cluster.db.transaction") as mocked_db:
-        run_scheduler_once(broker=broker)
+        scheduler(broker=broker)
         mocked_db.atomic.assert_called_with(using="default")
 
 

django_q/utils.py

@@ -1,13 +1,13 @@
 from datetime import datetime
-from django import db
 import calendar
+import inspect
 from datetime import date
 
 import django
 from django.utils import timezone
 from django.conf import settings
 
-from django_q.conf import Conf, logger
+from django_q.conf import Conf
 
 if django.VERSION < (4, 0):
     # pytz is the default in django 3.2. Remove when no support for 3.2
@@ -45,6 +45,18 @@ def add_years(d, years):
         return d.replace(year=new_date.year, month=new_date.month, day=new_date.day)
 
 
+def get_func_repr(func):
+    # convert func to string
+    if inspect.isfunction(func):
+        return f"{func.__module__}.{func.__name__}"
+    elif inspect.ismethod(func) and hasattr(func.__self__, "__name__"):
+        return (
+            f"{func.__self__.__module__}." f"{func.__self__.__name__}.{func.__name__}"
+        )
+    else:
+        return str(func) if func else None
+
+
 def localtime(value=None) -> datetime:
     """Override for timezone.localtime to deal with naive times and local times"""
     if settings.USE_TZ:
@@ -60,18 +72,3 @@ def localtime(value=None) -> datetime:
         return datetime.now()
     else:
         return value
-
-
-def close_old_django_connections():
-    """
-    Close django connections unless running with sync=True.
-    """
-    if Conf.SYNC:
-        logger.warning(
-            "Preserving django database connections because sync=True. Beware "
-            "that tasks are now injected in the calling context/transactions "
-            "which may result in unexpected behaviour."
-        )
-    else:
-        db.close_old_connections()
-

django_q/worker.py

@@ -1,221 +0,0 @@
-import multiprocessing
-from queue import Queue
-from queue import Empty
-from typing import Optional, Tuple, Union
-from django_q.queue_task import QueueTask
-from django.utils import timezone
-import traceback
-
-from multiprocessing import Process, Value, current_process
-from django_q.utils import close_old_django_connections
-from django.utils.translation import gettext_lazy as _
-from django_q.conf import Conf, logger, setproctitle, error_reporter, resource, psutil
-from django_q.signals import pre_execute
-from django_q.exceptions import TimeoutException, TimeoutHandler
-from django_q.process_manager import ProcessManager
-
-
-
-class Worker(ProcessManager):
-
-    def spawn_process(self) -> Process:
-        """
-        :type target: function or class
-        """
-        self.status = Value("i", Worker.Status.IDLE.value)
-        self.manager_pipe, worker_process_pipe = multiprocessing.Pipe(duplex=True)
-        p = WorkerProcess(args=(self.status, worker_process_pipe))
-        p.daemon = Conf.DAEMONIZE_WORKERS
-        p.start()
-        return p
-
-    def start_task(self, task) -> None:
-        # send task to worker
-        try:
-            self.manager_pipe.send(task)
-        except BrokenPipeError:
-            # recycle process if pipe is broken
-            self.status.value = ProcessManager.Status.RECYCLE.value
-
-
-class Pool:
-    """This will manager the individual workers"""
-    def __init__(self, workers=Conf.WORKERS):
-        self.amount_workers = workers
-        self.workers = []
-        self.task_queue = Queue()
-        self.start_workers()
-
-    def start_workers(self):
-        for __ in range(self.amount_workers):
-            self.workers.append(Worker())
-
-    def get_worker(self, id) -> Optional[Worker]:
-        worker = next((worker for worker in self.workers if worker.id == id), None)
-        if worker is None:
-            logger.error("Couldn't find worker")
-            return
-        return worker
-
-    @property
-    def is_healthy(self):
-        """Checks if all workers are still operating"""
-        return all(worker.is_alive for worker in self.workers)
-
-    @property
-    def is_idle(self):
-        """Checks if all workers are idle"""
-        return all(worker.is_idle for worker in self.workers)
-
-    @property
-    def is_done(self):
-        """Checks if all workers are idle and task queue is empty"""
-        return self.is_idle and self.task_queue.empty()
-
-    def reincarnate_stopped_workers(self):
-        """Reincarnates workers that are not alive anymore"""
-        stopped_workers = [worker for worker in self.workers if not worker.is_alive]
-        for worker in stopped_workers:
-            worker.reincarnate_process()
-
-    def add_task(self, task):
-        self.task_queue.put(task)
-
-    def get_done_workers(self):
-        """Worker tasks that have been completed, but need to be saved to cache/db - to be processed by monitor worker"""
-        return [worker for worker in self.workers if worker.is_done]
-
-    def mark_workers_idle(self, worker_ids):
-        """Mark workers idle when they are ready to be used again"""
-        for worker_id in worker_ids:
-            # We are going to process the result, mark them idle, so they can be used for a different task
-            worker = self.get_worker(id=worker_id)
-            if worker is not None:
-                worker.mark_idle()
-
-    def delegate_tasks(self):
-        available_workers = [worker for worker in self.workers if worker.is_idle]
-        for worker in available_workers:
-            try:
-                task = self.task_queue.get_nowait()
-            except Empty:
-                # if the queue is empty, then just stop
-                break
-            worker.start_task(task)
-
-
-class WorkerProcess(Process):
-
-    @staticmethod
-    def run_task(task) -> Tuple[QueueTask, bool]:
-        # signal execution
-        pre_execute.send(sender="django_q", func=task.func, task=task)
-        task.started_at = timezone.now()
-        try:
-            with TimeoutHandler(timeout=task.timeout):
-                func = task.callable_func()
-                res = func(*task.args, **task.kwargs)
-                result = res
-        except (TimeoutException, Exception) as e:
-            if isinstance(e, TimeoutException):
-                task.status = QueueTask.Status.TIMEOUT
-            else:
-                task.status = QueueTask.Status.FAILED
-            result = f"{e} : {traceback.format_exc()}"
-
-            if error_reporter:
-                error_reporter.report()
-            if task.sync:
-                raise
-            return task
-        else:
-            # succeeded
-            task.status = QueueTask.Status.SUCCESS
-        finally:
-            task.result = result
-            task.finished_at = timezone.now()
-        return task
-
-    def __init__(self, group=None, name=None, args=(), kwargs={}, daemon=None):
-        target = self.processing_tasks
-        super().__init__(group=group, target=target, name=name, args=args, kwargs=kwargs, daemon=daemon)
-
-    def mark_ready(self):
-        self.process_name = current_process().name
-        self.process_id = current_process().pid
-        self.task_count = 0
-        logger.info(
-            _("%(proc_name)s ready for work at %(id)s")
-            % {"proc_name": self.process_name, "id": self.process_id}
-        )
-
-
-    def mark_start_task(self, task):
-        # Log task creation and set process name
-        task_desc = (
-            _("%(proc_name)s processing %(task_name)s '%(func_name)s'")
-            % {
-                "proc_name": self.process_name,
-                "func_name": task.func_name,
-                "task_name": task.name,
-            }
-        )
-        if task.group is not None:
-            task_desc += f" [{task.group}]"
-        logger.info(task_desc)
-
-        if setproctitle:
-            proc_title = f"qcluster {self.process_name} processing {task.name} '{task.func_name}'"
-            if task.group is not None:
-                proc_title += f" [{task.group}]"
-            setproctitle.setproctitle(proc_title)
-
-    def processing_tasks(self, status: Value, pipe):
-        self.mark_ready()
-        while True:
-            task = pipe.recv()
-            if task == "STOP":
-                logger.info(f"Worker {self.process_name} stopped processing")
-                break
-            # got a new task, let's mark it starting
-            self.mark_start_task(task)
-
-            # make sure the function actually exists, before we try to run it
-            try:
-                if not task.is_callable:
-                    raise ValueError(f"Function {task.func_name} is not defined")
-            except Exception as e:
-                result = (f"{e} : {traceback.format_exc()}", False)
-                if error_reporter:
-                    error_reporter.report()
-                if task.sync:
-                    raise
-                # stop here, move on to the next one
-                continue
-
-            close_old_django_connections()
-
-            status.value = ProcessManager.Status.BUSY.value
-            task = WorkerProcess.run_task(task)
-            # Add task towards total
-            self.task_count += 1
-
-            # Set to DONE so main process can pick it up
-            status.value = ProcessManager.Status.DONE.value
-            if setproctitle:
-                setproctitle.setproctitle(f"qcluster {self.process_name} completed with task")
-
-            # Recreate a new process if this task has had the max amount of runs or exceeded resources
-            if self.task_count == Conf.RECYCLE or self.rss_check():
-                status.value = ProcessManager.Status.RECYCLE
-                break
-
-            pipe.send(task)
-
-    def rss_check(self):
-        if Conf.MAX_RSS:
-            if resource:
-                return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
-            elif psutil:
-                return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
-        return False

docs/conf.py

@@ -75,7 +75,7 @@ author = "Ilan Steemers, Stan Triepels"
 # The short X.Y version.
 version = "1.5"
 # The full version, including alpha/beta/rc tags.
-release = "1.5.1"
+release = "1.5.2"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/index.rst

@@ -27,7 +27,7 @@ Features
 -  Rollbar and Sentry support
 
 
-Django Q2 is tested with: Python 3.8, 3.9, 3.10, and 3.11. Works with Django 3.2.x and 4.1.x.
+Django Q2 is tested with: Python 3.8, 3.9, 3.10 and 3.11. Works with Django 3.2.x, 4.1.x and 4.2.x
 
 Currently available in English, German and French.
 

docs/install.rst

@@ -32,7 +32,7 @@ Django Q2 is tested for Python 3.8, 3.9, 3.10 and 3.11
 -  `Django <https://www.djangoproject.com>`__
 
     Django Q2 aims to use as much of Django's standard offerings as possible.
-    The code is tested against Django versions `3.2.x` and `4.1.x`.
+    The code is tested against Django versions `3.2.x`, `4.1.x` and`4.2.x`.
 
 -  `Django-picklefield <https://github.com/gintas/django-picklefield>`__
 
@@ -41,6 +41,10 @@ Django Q2 is tested for Python 3.8, 3.9, 3.10 and 3.11
 
 Optional
 ~~~~~~~~
+-  `Blessed <https://github.com/jquast/blessed>`__ is used to display the statistics in the terminal::
+
+    $ pip install blessed
+
 -  `Redis-py <https://github.com/andymccurdy/redis-py>`__ client by Andy McCurdy is used  to interface with both the Redis::
 
     $ pip install redis
@@ -136,7 +140,7 @@ You can reference the `requirements <https://github.com/GDay/django-q2/blob/mast
 Django
 ~~~~~~
 We strive to be compatible with last two major version of Django.
-At the moment this means we support the 3.2.x and 4.1.x releases.
+At the moment this means we support the 3.2.x, 4.1.x and 4.2.x releases.
 
 Since we are now no longer supporting Python 2, we can also not support older versions of Django that do not support Python >= 3.6
 For this you can always use older releases, but they are no longer maintained.

docs/monitor.rst

@@ -3,6 +3,10 @@ Monitor
 .. py:currentmodule::django_q.monitor
 
 
+.. warning::
+    Blessed needs to be installed to get this to work! See: https://pypi.org/project/blessed/
+
+
 The cluster monitor shows live information about all the Q clusters connected to your project.
 
 Start the monitor with Django's `manage.py` command::

docs/signals.rst

@@ -13,6 +13,12 @@ Before enqueuing a task
 The ``django_q.signals.pre_enqueue`` signal is emitted before a task is
 enqueued. The task dictionary is given as the ``task`` argument.
 
+After spawning a worker process
+"""""""""""""""""""""""""""""""
+
+The ``django_q.signals.post_spawn`` signal is emitted after a worker process has
+spawned. The process name is given as the ``proc_name`` argument (string).
+
 Before executing a task
 """""""""""""""""""""""
 
@@ -37,7 +43,7 @@ Connecting to a Django Q2 signal is done the same as any other Django
 signal::
 
     from django.dispatch import receiver
-    from django_q.signals import pre_enqueue, pre_execute, post_execute
+    from django_q.signals import pre_enqueue, pre_execute, post_execute, post_spawn
 
     @receiver(pre_enqueue)
     def my_pre_enqueue_callback(sender, task, **kwargs):
@@ -51,4 +57,8 @@ signal::
     def my_post_execute_callback(sender, task, **kwargs):
         print(f"Task {task['name']} was executed with result {task['result']}")
 
+    @receiver(post_spawn)
+    def my_post_spawn_callback(sender, proc_name, **kwargs):
+        print(f"Process {proc_name} has spawned")
+
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "django-q2"
-version = "1.5.1"
+version = "1.5.2"
 packages = [
     { include = "django_q" },
 ]