From 28887decabc6cbe1a640c14bafdc2c5905afe6c9 Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Fri, 3 Jul 2015 19:57:45 +0200 Subject: [PATCH] Getting docs ready fro 0.2.0 --- docs/admin.rst | 4 ++++ docs/cluster.rst | 24 +++++++++++++++++-- docs/index.rst | 29 ++++++++++++++++------ docs/monitor.rst | 60 ++++++++++++++++++++++++++++++++++++++++++++++ docs/schedules.rst | 37 ++++++++++++++++++++++++++++ docs/tasks.rst | 4 ++-- 6 files changed, 147 insertions(+), 11 deletions(-) diff --git a/docs/admin.rst b/docs/admin.rst index 1a5c821..afee212 100644 --- a/docs/admin.rst +++ b/docs/admin.rst @@ -1,2 +1,6 @@ +.. _admin_page: + Admin pages =========== + +TODO \ No newline at end of file diff --git a/docs/cluster.rst b/docs/cluster.rst index 463298c..81b07c0 100644 --- a/docs/cluster.rst +++ b/docs/cluster.rst @@ -24,7 +24,27 @@ You should see something like this:: 10:57:40 [Q] INFO Q Cluster-31781 running. -Stopping the cluster with ctrl-c or either the `SIGTERM` and `SIGKILL` signals, will initiate the :ref:`stop_procedure`. +Stopping the cluster with ctrl-c or either the `SIGTERM` and `SIGKILL` signals, will initiate the :ref:`stop_procedure`:: + + 16:44:12 [Q] INFO Q Cluster-31781 stopping. + 16:44:12 [Q] INFO Process-1 stopping cluster processes + 16:44:13 [Q] INFO Process-1:10 stopped pushing tasks + 16:44:13 [Q] INFO Process-1:6 stopped doing work + 16:44:13 [Q] INFO Process-1:4 stopped doing work + 16:44:13 [Q] INFO Process-1:1 stopped doing work + 16:44:13 [Q] INFO Process-1:5 stopped doing work + 16:44:13 [Q] INFO Process-1:7 stopped doing work + 16:44:13 [Q] INFO Process-1:3 stopped doing work + 16:44:13 [Q] INFO Process-1:8 stopped doing work + 16:44:13 [Q] INFO Process-1:2 stopped doing work + 16:44:14 [Q] INFO Process-1:9 stopped monitoring results + 16:44:15 [Q] INFO Q Cluster-31781 has stopped. + +Using a Procfile +---------------- +If you host on `Heroku `__ or you are using `Honcho `__ you can start the cluster from a :file:`Procfile` like this:: + + worker: python manage.py qcluster Architecture ------------ @@ -99,5 +119,5 @@ Afterwards the sentinel waits for the monitor to empty the result queue before t - Put a poison pill on the Result Queue - Wait for monitor to stop -.. :warning:: +.. warning:: If you force the cluster to terminate before the stop procedure has completed, you can lose tasks and their results. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index b9b4fd8..2c74631 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -5,19 +5,34 @@ Welcome to Django Q's documentation! ==================================== +Django Q is a native Django task queue and worker application using Python multiprocessing. + + +Features +-------- + +- Multiprocessing worker pool +- Asynchronous tasks +- Scheduled and repeated tasks +- Encrypted and compressed packages +- Failure and success database +- Result hooks +- Django Admin integration +- PaaS compatible with multiple instances +- Multi cluster monitor Contents: .. toctree:: :maxdepth: 1 - Installation - Tasks - Schedules - Cluster - Monitor - Admin + Installation + Tasks + Schedules + Cluster + Monitor + Admin - * :ref:`genindex` +* :ref:`genindex` * :ref:`search` diff --git a/docs/monitor.rst b/docs/monitor.rst index f03736d..7858132 100644 --- a/docs/monitor.rst +++ b/docs/monitor.rst @@ -7,3 +7,63 @@ Start the monitor with Django's `manage.py` command:: $ python manage.py qmonitor + +.. image:: monitor.png + +Legend +------ + +Host +~~~~ + +Shows the hostname of the server this cluster is running on. + +Id +~~ + +The cluster Id. Same as the cluster process ID or pid. + +State +~~~~~ + +Current state of the cluster: + +- **Starting** The cluster is spawning workers and getting ready. +- **Idle** Everything is ok, but there are no tasks to process. +- **Working** Processing tasks like a good cluster should. +- **Stopping** The cluster does not take on any new tasks and is finishing. +- **Stopped** All tasks have been processed and the cluster is shutting down. + +Pool +~~~~ + +The current number of workers in the cluster pool. + +TQ +~~ + +**Task Queue** counts the number of tasks in the queue + +If this keeps rising it means you are taking on more tasks than your cluster can handle. + +RQ +~~ + +**Result Queue** shows the number of results in the queue. + +Since results are only saved by a single process which has to access the database. +It's normal for the result queue to take slightly longer to clear than the task queue. + +RC +~~ + +**Reincarnations** shows the amount of processes that have been reincarnated after a sudden death or timeout. +If this number is unusually high, you are either suffering from repeated task errors or severe timeouts and you should check your logs for details. + +Up +~~ + +**Uptime** the amount of time that has passed since the cluster was started. + + +.. centered:: Press `q` to quit the monitor and return to your terminal. \ No newline at end of file diff --git a/docs/schedules.rst b/docs/schedules.rst index 867fa68..8c54727 100644 --- a/docs/schedules.rst +++ b/docs/schedules.rst @@ -1,2 +1,39 @@ Schedules ========= + +Schedules are regular Django models. You can manage them through the :ref:`admin_page` or directly from your code with the :func:`schedule` function or the :class:`Schedule` model: + +.. code:: python + + from django_q import Schedule, schedule + + # Use the schedule wrapper + + schedule('math.copysign', + 2, -2, + hook='hooks.print_result', + schedule_type=Schedule.DAILY) + + # Or create the object directly + + Schedule.objects.create(func='math.copysign', + hook='hooks.print_result', + args='2,-2', + schedule_type=Schedule.DAILY + ) + + +.. py:function:: schedule(func, *args, hook=None, schedule_type='O', repeats=-1, next_run=now() , **kwargs) + + Creates a schedule + + :param str func: the function to schedule. Dotted strings only. + :param args: arguments for the scheduled function. + :param str hook: optional result hook function. Dotted strings only. + :param str schedule_type: (O)nce, (H)ourly, (D)aily, (W)eekly, (M)onthly, (Q)uarterly, (Y)early + :param int repeats: Number of times to repeat schedule. `-1`=Always, `0`=Never, `n` =n. + :param datetime next_run: Next or first scheduled execution datetime. + :param kwargs: optional keyword arguments for the scheduled function. + +.. py:class:: Schedule + diff --git a/docs/tasks.rst b/docs/tasks.rst index 1a96ce4..79e61dd 100644 --- a/docs/tasks.rst +++ b/docs/tasks.rst @@ -27,7 +27,7 @@ Use :py:func:`async` from your code to quickly offload tasks to the py:module:` def print_result(task): print(task.result) -.. py:function:: async(func, *args, [hook=None,] **kwargs) +.. py:function:: async(func, *args, hook=None, **kwargs) Puts a task in the cluster queue @@ -47,7 +47,7 @@ Use :py:func:`async` from your code to quickly offload tasks to the py:module:` :param str name: the name of the task :returns: The result of the executed task -.. py:function:: get_task(name) +.. py:function:: fetch(name) Returns a previously executed task