38 Commits

Author SHA1 Message Date
Stan
2c456cd0aa Release v1.7.2 2024-09-10 00:16:18 +02:00
Stan Triepels
8f54e5d5ae Add twine check and fix readme (#216)
* Add twine check

* add poetry to install

* move up remove black/flake formatting

* fix path

* fix?

* fix rst?

* add rstcheck

* test all files?

* all

* remove complaining image

* only check readme

* remove not needed dependencies
2024-09-10 00:10:14 +02:00
Stan
689c881178 Release v1.7.1 2024-09-08 04:20:13 +02:00
Stan
923c5daa73 Release v1.7.0 2024-09-08 03:52:37 +02:00
Stan Triepels
69e794f1af Add index on succeeded tasks (#164)
* Add index on task table

* also add models file

* add make commands and limit index

* format
2024-09-08 03:37:28 +02:00
Stan Triepels
2191897825 Add example project (#215)
* add example project

* format

* fix format
2024-09-08 03:13:49 +02:00
Andrii Yurchuk
af505e8d70 Call mark_process_dead on worker pid if prometheus_client is installed (#212) 2024-09-03 22:29:00 +02:00
kimihito
48123af09d chore: Add Support Django 5.1 (#207) 2024-08-13 14:14:40 +02:00
kimihito
faaa57be93 chore: Update CI for Django 4.2 Python 3.12 support (#208) 2024-08-13 14:14:11 +02:00
jv3ga
5015d8ee4d fix: Oracle ORM backend compatibility #180 (#186) 2024-08-13 04:40:54 +02:00
Stan Triepels
cb253357fb Fix repeating task after timeout (#184) 2024-06-23 22:36:29 +02:00
GDay
efa0d6d9e4 ignore fuff formating 2024-05-11 03:06:58 +02:00
Stan Triepels
ad4d24e17c Replace black/isort with ruff (#188)
* Replace black/isort with ruff

* fix typo
2024-05-11 03:04:32 +02:00
Stan Triepels
1d1d0a9aa5 Fix max attempts for value 1 (#185) 2024-05-06 17:28:27 +02:00
Stan Triepels
5e4aa65f62 Remove support for Django 3.2 and 4.1 (#183) 2024-05-05 00:42:09 +02:00
GDay
7add68f770 Release v1.6.2 2024-03-05 16:46:08 +01:00
Stan Triepels
782b1de60d Allow different broker on chain (#156) 2024-03-02 22:45:40 +01:00
Josh Thomas
88430c9014 Fix formatting issues in README.rst (#159) 2024-02-20 18:04:46 +01:00
Stan Triepels
6265a54008 Update docs to add cluster option to the async_task (#157) 2024-02-18 04:00:07 +01:00
Stan Triepels
1b638e5802 Update release/test dependencies (#147)
* Update release dependencies

* Update test.yml
2023-12-14 16:06:34 +01:00
Tobias Menzel
13a382cca2 Fix for Negative Repeat Count in Scheduler (#101) (#146)
* Comment out decrement of s.repeats in scheduler to fix negative repeat count issue

* Fix repeat count handling in scheduler

- Reset `repeats` to -1 for values less than -1
- Decrement `repeats` by 1 if it's greater than 0
2023-12-14 14:44:07 +01:00
Stan Triepels
098aa66a2d Update django.po (#138) 2023-11-04 19:52:43 +01:00
Stan Triepels
b3f494571f Use importerror for b62_decode and avoid deprecation notification (#134) 2023-10-26 14:29:13 +02:00
Patrycja
503127233f Specify build system in pyproject.toml (#131)
Otherwise, trying to build the project with a tool like gpep517 or build
results in using the setuptools legacy backend instead
2023-10-19 22:05:16 +02:00
GDay
e3be22ac63 Release v1.6.1 2023-10-13 14:15:58 +02:00
Stan Triepels
0aaeb8d35d Fix strict versions for python/django - poetry install error (#130) 2023-10-13 14:13:22 +02:00
GDay
750c654940 Release v1.6.0 2023-10-13 00:59:42 +02:00
Stan Triepels
1d2a6059db Add support for Django 5 (#120) 2023-10-12 22:44:19 +02:00
Stan Triepels
54ab399758 Fix for "apps not ready" in Windows and Mac (#116) 2023-10-09 22:04:32 +02:00
Grayknife
bc50219141 Update broken MongoClient link in Docs (#127) 2023-10-07 22:15:49 +02:00
Grayknife
939d0ebef1 Fix German Translation Typo (#124) 2023-09-30 13:28:42 +02:00
Jrog
68af2ae13e Update Add-ons install command in install.rst (#115) 2023-09-14 14:37:12 +02:00
Dustin Blanchard
960f72a1d0 DOCS: Correct health check import in examples.rst (#110)
* Correct health check import in examples.rst

* Update monitor.rst

---------

Co-authored-by: Stan Triepels <1939656+GDay@users.noreply.github.com>
2023-09-05 17:05:47 +02:00
GDay
aa79a6e6b3 Release v1.5.5 2023-09-01 16:36:00 +02:00
Stan Triepels
59d761b574 Add documentation to migrate from django-q to django-q2 (#108) 2023-09-01 14:33:26 +02:00
Stan Triepels
d8cc0cc4f4 Fix not picking up result from falsy result (#107)
* Fix not picking up result from falsy result

* add test and docs

* typo
2023-09-01 02:36:15 +02:00
Rémy Pecqueur
a08ef61ba5 Remove deprecated usage pkg_resources (#103)
List entry points using importlib.metadata (Python >=3.10) or
importlib-metadata>=3.6 (Python < 3.10)
2023-08-01 02:33:31 +02:00
Stan Triepels
7fb7fca176 Move worker, scheduler, pusher and monitor to separate files (#100) 2023-07-02 02:47:05 +02:00
79 changed files with 2517 additions and 2026 deletions

View File

@@ -1,2 +1,4 @@
# flake8, black, isort
b1d000d007f3f77069719523268a0c6256dc0860
# move to ruff formatting/linting
ad4d24e17c9424b17cd8ae65c2def7ecc74e63c1

View File

@@ -11,15 +11,14 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: 3.8
python-version: 3.11
- name: Install dependencies
run: |
sudo apt-get update

View File

@@ -7,12 +7,38 @@ on:
branches:
- master
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Lint with ruff
run: |
pipx install ruff==0.4.10
ruff format . --check && ruff check .
- name: Check twine
run: |
python -m pip install twine poetry rstcheck
poetry build
rstcheck README.rst
twine check dist/*
test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.8", "3.9", "3.10", "3.11" ]
django: [ "3.2", "4.1", "4.2" ]
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
django: [ "4.2", "5.0", "5.1"]
exclude:
# django 5.1 does not support 3.8 and 3.9
- python-version: "3.8"
django: "5.1"
- python-version: "3.9"
django: "5.1"
# django 5.0 does not support 3.8 and 3.9
- python-version: "3.8"
django: "5.0"
- python-version: "3.9"
django: "5.0"
services:
disque:
@@ -39,16 +65,16 @@ jobs:
- 6379:6379
options: --entrypoint redis-server
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies with Django ${{ matrix.django }}
run: |
python -m pip install --upgrade pip
pip install poetry
poetry add "django==${{ matrix.django }}"
poetry add "django==${{ matrix.django }}" --python=${{ matrix.python-version }}
poetry install -E testing
- name: Run Tests
run: |
@@ -68,16 +94,12 @@ jobs:
finish:
needs: test
runs-on: ubuntu-latest
container: python:3-slim
container: python:3.11-bookworm
steps:
- name: Upload to coveralls
run: |
python -m pip install --upgrade pip
python -m pip install coveralls flake8 black
python -m pip install coveralls
coveralls --service=github --finish
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Check flake8/black
run: |
flake8 .
black --check .

View File

@@ -1,6 +1,67 @@
# Changelog
## [Unreleased](https://github.com/GDay/django-q2/tree/HEAD)
## [v1.7.2](https://github.com/django-q2/django-q2/tree/v1.7.2) (2024-09-09)
- Fix twine check
## [v1.7.1](https://github.com/django-q2/django-q2/tree/v1.7.1) (2024-09-08)
- Fixed date of v1.7.0
- Fixed README.rst formatting which is blocking release of latest version
## [v1.7.0](https://github.com/django-q2/django-q2/tree/v1.7.0) (2024-09-08)
**Merged pull requests:**
- Remove support for Django 3.2 and 4.1 https://github.com/django-q2/django-q2/pull/183
- Fix max attempts for value 1 https://github.com/django-q2/django-q2/pull/185
- Replace black/isort with ruff https://github.com/django-q2/django-q2/pull/188
- Fix repeating task after timeout https://github.com/django-q2/django-q2/pull/184
- fix: Oracle ORM backend compatibility #180 https://github.com/django-q2/django-q2/pull/186
- chore: Update CI for Django 4.2 Python 3.12 support https://github.com/django-q2/django-q2/pull/208
- chore: Add Support Django 5.1 https://github.com/django-q2/django-q2/pull/207
- Call mark_process_dead on worker pid if prometheus_client is installed https://github.com/django-q2/django-q2/pull/212
- Add example project https://github.com/django-q2/django-q2/pull/215
- Add index on succeeded tasks https://github.com/django-q2/django-q2/pull/164
## [v1.6.2](https://github.com/django-q2/django-q2/tree/v1.6.2) (2024-03-05)
**Merged pull requests:**
- Allow different broker on chain https://github.com/django-q2/django-q2/pull/156
- Fix formatting issues in README.rst https://github.com/django-q2/django-q2/pull/159
- Update docs to add cluster option to the async_task https://github.com/django-q2/django-q2/pull/157
- Update release/test dependencies https://github.com/django-q2/django-q2/pull/147
- Fix for Negative Repeat Count in Scheduler https://github.com/django-q2/django-q2/pull/146
- Update django.po https://github.com/django-q2/django-q2/pull/138
- Use importerror for b62_decode and avoid deprecation notification https://github.com/django-q2/django-q2/pull/134
- Specify build system in pyproject.toml https://github.com/django-q2/django-q2/pull/131
## [v1.6.1](https://github.com/django-q2/django-q2/tree/v1.6.1) (2023-10-13)
**Merged pull requests:**
- Fix strict versions for python/django https://github.com/django-q2/django-q2/pull/130
## [v1.6.0](https://github.com/django-q2/django-q2/tree/v1.6.0) (2023-10-12)
**Merged pull requests:**
- Add support for Django 5 and python 12 https://github.com/django-q2/django-q2/pull/120
- Fix for "apps not ready" in Windows and Mac https://github.com/django-q2/django-q2/pull/116
- Update broken MongoClient link in Docs https://github.com/django-q2/django-q2/pull/127
- Fix German Translation Typo https://github.com/django-q2/django-q2/pull/124
- Update Add-ons install command in install.rst https://github.com/django-q2/django-q2/pull/115
- DOCS: Correct health check import in examples.rst https://github.com/django-q2/django-q2/pull/110
## [v1.5.5](https://github.com/django-q2/django-q2/tree/v1.5.5) (2023-09-01)
**Merged pull requests:**
- Add documentation to migrate from django-q to django-q2 https://github.com/django-q2/django-q2/pull/108
- Fix not picking up result from falsy result https://github.com/django-q2/django-q2/pull/107
- Remove deprecated usage pkg_resources https://github.com/django-q2/django-q2/pull/103
- Move worker, scheduler, pusher and monitor to separate files https://github.com/django-q2/django-q2/pull/100
## [v1.5.4](https://github.com/GDay/django-q2/tree/v1.5.4) (2023-06-29)

View File

@@ -1,4 +1,4 @@
FROM python:3.9
FROM python:3.12
ENV PYTHONUNBUFFERED 1
RUN mkdir -p /app

View File

@@ -8,10 +8,10 @@ ENV PYTHONUNBUFFERED 1
ENV SHELL /bin/bash
# Upgrades pip
RUN pip install --upgrade pip
RUN pip install -U pip setuptools
# Install poetry
RUN pip install poetry
RUN pip install poetry==1.8.2
WORKDIR /app

21
Makefile Normal file
View File

@@ -0,0 +1,21 @@
dev:
docker compose -f web-docker-compose.yaml up
test:
docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run pytest
shell:
docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py shell
makemigrations:
docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py makemigrations
migrate:
docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run python manage.py migrate
createsuperuser:
docker compose -f web-docker-compose.yaml run --rm web python manage.py createsuperuser
format:
docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run ruff format .
docker compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run ruff check . --fix

View File

@@ -1,11 +1,9 @@
A multiprocessing distributed task queue for Django
---------------------------------------------------
|image0| |image1| |docs| |downloads|
|image0| |image1| |downloads|
::
Django Q2 is a fork of Django Q. Big thanks to Ilan Steemers for starting this project. Unfortunately, development has stalled since June 2021. Django Q2 is the new updated version of Django Q, with dependencies updates, docs updates and several bug fixes. Original repository: https://github.com/Koed00/django-q
Django Q2 is a fork of Django Q. Big thanks to Ilan Steemers for starting this project. Unfortunately, development has stalled since June 2021. Django Q2 is the new updated version of Django Q, with dependencies updates, docs updates and several bug fixes. Original repository: https://github.com/Koed00/django-q
Features
~~~~~~~~
@@ -27,7 +25,7 @@ Changes compared to the original Django-Q:
- Dropped support for Disque (hasn't been updated in a long time)
- Dropped Redis, Arrow and Blessed dependencies
- Updated all current dependencies
- Added tests for Django 4.x
- Added tests for Django 4.x and 5.x
- Added Turkish language
- Improved admin area
- Fixed a lot of issues
@@ -37,10 +35,10 @@ See the `changelog <https://github.com/GDay/django-q2/blob/master/CHANGELOG.md>`
Requirements
~~~~~~~~~~~~
- `Django <https://www.djangoproject.com>`__ > = 3.2
- `Django <https://www.djangoproject.com>`__ > = 4.2
- `Django-picklefield <https://github.com/gintas/django-picklefield>`__
Tested with: Python 3.8, 3.9, 3.10 and 3.11. Works with Django 3.2.X, 4.1.X and 4.2.X.
Tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 4.2.X and 5.0.X
Brokers
~~~~~~~
@@ -73,7 +71,6 @@ Installation
Read the full documentation at `https://django-q2.readthedocs.org <https://django-q2.readthedocs.org>`__
Configuration
~~~~~~~~~~~~~
@@ -104,10 +101,7 @@ For full configuration options, see the `configuration documentation <https://dj
Management Commands
~~~~~~~~~~~~~~~~~~~
::
For the management commands to work, you will need to install Blessed: <https://github.com/jquast/blessed>
For the management commands to work, you will need to install Blessed: <https://github.com/jquast/blessed>
Start a cluster with::
@@ -159,7 +153,6 @@ Use `async_task` from your code to quickly offload tasks:
For more info see `Tasks <https://django-q2.readthedocs.org/en/latest/tasks.html>`__
Schedule
~~~~~~~~
@@ -204,6 +197,22 @@ Admin page or directly from your code:
For more info check the `Schedules <https://django-q2.readthedocs.org/en/latest/schedules.html>`__ documentation.
Development
~~~~~~~~~~~
There is an example project that you can use to develop with. Docker (compose) is being used to set everything up.
Please note that you will have to restart the django-q container when changes have been made to tasks or django-q.
You can start the example project with:
.. code:: bash
make dev
Create a superuser with:
.. code:: bash
make createsuperuser
Testing
~~~~~~~
@@ -212,7 +221,7 @@ Running tests is easy with docker compose, it will also start the necessary data
.. code:: bash
docker-compose -f test-services-docker-compose.yaml run --rm django-q2 poetry run pytest
make test
Locale
~~~~~~
@@ -220,12 +229,6 @@ Locale
Currently available in English, German, Turkish, and French.
Translation pull requests are always welcome.
Todo
~~~~
- Better tests and coverage
- Less dependencies?
Acknowledgements
~~~~~~~~~~~~~~~~
@@ -242,9 +245,5 @@ Acknowledgements
:target: https://github.com/GDay/django-q2/actions?query=workflow%3Atests
.. |image1| image:: https://coveralls.io/repos/github/GDay/django-q2/badge.svg?branch=master
:target: https://coveralls.io/github/GDay/django-q2?branch=master
.. |docs| image:: https://readthedocs.org/projects/docs/badge/?version=latest
:alt: Documentation Status
:scale: 100
:target: https://django-q2.readthedocs.org/
.. |downloads| image:: https://img.shields.io/pypi/dm/django-q2
:target: https://img.shields.io/pypi/dm/django-q2

View File

@@ -1,6 +1,6 @@
import django
VERSION = (1, 5, 4)
VERSION = (1, 7, 2)
if django.VERSION < (3, 2):
default_app_config = "django_q.apps.DjangoQConfig"

View File

@@ -1,4 +1,5 @@
"""Admin module for Django."""
from django.contrib import admin
from django.db.models.expressions import OuterRef, Subquery
from django.urls import reverse
@@ -31,7 +32,15 @@ resubmit_task.short_description = _("Resubmit selected tasks to queue")
class TaskAdmin(admin.ModelAdmin):
"""model admin for success tasks."""
list_display = ("name", "group", "func", "cluster", "started", "stopped", "time_taken")
list_display = (
"name",
"group",
"func",
"cluster",
"started",
"stopped",
"time_taken",
)
actions = [resubmit_task]
def has_add_permission(self, request):
@@ -55,7 +64,15 @@ class TaskAdmin(admin.ModelAdmin):
class FailAdmin(admin.ModelAdmin):
"""model admin for failed tasks."""
list_display = ("name", "group", "func", "cluster", "started", "stopped", "short_result")
list_display = (
"name",
"group",
"func",
"cluster",
"started",
"stopped",
"short_result",
)
def has_add_permission(self, request):
"""Don't allow adds."""
@@ -132,7 +149,17 @@ class QueueAdmin(admin.ModelAdmin):
"""queue admin for ORM broker"""
list_display = ("id", "key", "name", "group", "func", "lock", "task_id")
fields = ("key", "lock", "task_id", "name", "group", "func", "args", "kwargs", "q_options")
fields = (
"key",
"lock",
"task_id",
"name",
"group",
"func",
"args",
"kwargs",
"q_options",
)
readonly_fields = fields[2:]
def save_model(self, request, obj, form, change):

View File

@@ -182,7 +182,7 @@ def get_broker(list_key: str = None) -> Broker:
return ironmq.IronMQBroker(list_key=list_key)
# SQS
elif type(Conf.SQS) == dict:
elif isinstance(Conf.SQS, dict):
from django_q.brokers import aws_sqs
return aws_sqs.Sqs(list_key=list_key)

View File

@@ -37,7 +37,9 @@ class ORM(Broker):
def lock_size(self) -> int:
return (
self.get_connection().filter(key=self.list_key, lock__gt=timezone.now()).count()
self.get_connection()
.filter(key=self.list_key, lock__gt=timezone.now())
.count()
)
def purge_queue(self):
@@ -62,7 +64,9 @@ class ORM(Broker):
return package.pk
def dequeue(self):
tasks = self.get_connection().filter(key=self.list_key, lock__lt=timezone.now())[
tasks = self.get_connection().filter(
key=self.list_key, lock__lt=timezone.now()
)[
0 : Conf.BULK # noqa: E203
]
if tasks:

View File

@@ -1,11 +1,7 @@
# Standard
import ast
import pydoc
import signal
import socket
import traceback
import uuid
from datetime import datetime, timedelta
from multiprocessing import Event, Process, Value, current_process
from time import sleep
@@ -24,26 +20,22 @@ from django.utils import timezone
from django.utils.translation import gettext_lazy as _
# Local
import django_q.tasks
from django_q.brokers import Broker, get_broker
from django_q.conf import (
Conf,
croniter,
error_reporter,
get_ppid,
logger,
prometheus_multiprocess,
psutil,
setproctitle,
resource,
)
from django_q.humanhash import humanize
from django_q.models import Schedule, Success, Task
from django_q.monitor import monitor
from django_q.pusher import pusher
from django_q.queues import Queue
from django_q.signals import post_execute, post_spawn, pre_execute
from django_q.signing import BadSignature, SignedPackage
from django_q.scheduler import scheduler
from django_q.status import Stat, Status
from .utils import get_func_repr, localtime
from django_q.worker import worker
class Cluster:
@@ -174,7 +166,7 @@ class Sentinel:
def queue_name(self):
# multi-queue: cluster name is (broker's) queue_name
return self.broker.list_key if self.broker else '--'
return self.broker.list_key if self.broker else "--"
def start(self):
self.broker.ping()
@@ -238,6 +230,8 @@ class Sentinel:
% {"name": process.name}
)
else:
if prometheus_multiprocess:
prometheus_multiprocess.mark_process_dead(process.pid)
self.pool.remove(process)
self.spawn_worker()
if process.timer.value == 0:
@@ -248,20 +242,22 @@ class Sentinel:
try:
process_name = psutil.Process(process.pid).name()
name_splits = process_name.split(" ")
task_name = name_splits[3] if len(name_splits) >= 4 and name_splits[2] == "processing" else ""
task_name = (
name_splits[3]
if len(name_splits) >= 4 and name_splits[2] == "processing"
else ""
)
except psutil.NoSuchProcess:
pass
process.terminate()
if task_name:
msg = (
_("reincarnated worker %(name)s after timeout while processing task %(task_name)s")
% {"name": process.name, "task_name": task_name}
)
msg = _(
"reincarnated worker %(name)s after timeout while processing task %(task_name)s"
) % {"name": process.name, "task_name": task_name}
else:
msg = (
_("reincarnated worker %(name)s after timeout")
% {"name": process.name}
)
msg = _("reincarnated worker %(name)s after timeout") % {
"name": process.name
}
logger.critical(msg)
elif int(process.timer.value) == -2:
logger.info(_("recycled worker %(name)s") % {"name": process.name})
@@ -294,14 +290,18 @@ class Sentinel:
_("%(name)s guarding cluster %(cluster_name)s")
% {
"name": current_process().name,
"cluster_name": humanize(self.cluster_id.hex) + f" [{self.queue_name()}]",
"cluster_name": humanize(self.cluster_id.hex)
+ f" [{self.queue_name()}]",
}
)
self.start_event.set()
Stat(self).save()
logger.info(
_("Q Cluster %(cluster_name)s running.")
% {"cluster_name": humanize(self.cluster_id.hex) + f" [{self.queue_name()}]"}
% {
"cluster_name": humanize(self.cluster_id.hex)
+ f" [{self.queue_name()}]"
}
)
counter = 0
cycle = Conf.GUARD_CYCLE # guard loop sleep in seconds
@@ -374,438 +374,6 @@ class Sentinel:
Stat(self).save()
def pusher(task_queue: Queue, event: Event, broker: Broker = None):
"""
Pulls tasks of the broker and puts them in the task queue
:type broker:
:type task_queue: multiprocessing.Queue
:type event: multiprocessing.Event
"""
if not broker:
broker = get_broker()
proc_name = current_process().name
if setproctitle:
setproctitle.setproctitle(f"qcluster {proc_name} pusher")
logger.info(
_("%(name)s pushing tasks at %(id)s")
% {"name": proc_name, "id": current_process().pid}
)
while True:
try:
task_set = broker.dequeue()
except Exception:
logger.exception("Failed to pull task from broker")
# broker probably crashed. Let the sentinel handle it.
sleep(10)
break
if task_set:
for task in task_set:
ack_id = task[0]
# unpack the task
try:
task = SignedPackage.loads(task[1])
except (TypeError, BadSignature):
logger.exception("Failed to push task to queue")
broker.fail(ack_id)
continue
task["cluster"] = Conf.CLUSTER_NAME # save actual cluster name to orm task table
task["ack_id"] = ack_id
task_queue.put(task)
logger.debug(
_("queueing from %(list_key)s") % {"list_key": broker.list_key}
)
if event.is_set():
break
logger.info(_("%(name)s stopped pushing tasks") % {"name": current_process().name})
def monitor(result_queue: Queue, broker: Broker = None):
"""
Gets finished tasks from the result queue and saves them to Django
:type broker: brokers.Broker
:type result_queue: multiprocessing.Queue
"""
if not broker:
broker = get_broker()
proc_name = current_process().name
if setproctitle:
setproctitle.setproctitle(f"qcluster {proc_name} monitor")
logger.info(
_("%(name)s monitoring at %(id)s") % {"name": proc_name, "id": current_process().pid}
)
for task in iter(result_queue.get, "STOP"):
# save the result
if task.get("cached", False):
save_cached(task, broker)
else:
save_task(task, broker)
# acknowledge result
ack_id = task.pop("ack_id", False)
if ack_id and (task["success"] or task.get("ack_failure", False)):
broker.acknowledge(ack_id)
# signal execution done
post_execute.send(sender="django_q", task=task)
# log the result
info_name = get_func_repr(task["func"])
if task["success"]:
# log success
logger.info(
_("Processed '%(info_name)s' (%(task_name)s)")
% {"info_name": info_name, "task_name": task["name"]}
)
else:
# log failure
logger.error(
_("Failed '%(info_name)s' (%(task_name)s) - %(task_result)s")
% {
"info_name": info_name,
"task_name": task["name"],
"task_result": task["result"],
}
)
logger.info(_("%(name)s stopped monitoring results") % {"name": proc_name})
def worker(
task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
):
"""
Takes a task from the task queue, tries to execute it and puts the result back in
the result queue
:param timeout: number of seconds wait for a worker to finish.
:type task_queue: multiprocessing.Queue
:type result_queue: multiprocessing.Queue
:type timer: multiprocessing.Value
"""
proc_name = current_process().name
logger.info(
_("%(proc_name)s ready for work at %(id)s")
% {"proc_name": proc_name, "id": current_process().pid}
)
post_spawn.send(sender="django_q", proc_name=proc_name)
if setproctitle:
setproctitle.setproctitle(f"qcluster {proc_name} idle")
task_count = 0
if timeout is None:
timeout = -1
# Start reading the task queue
for task in iter(task_queue.get, "STOP"):
result = None
timer.value = -1 # Idle
task_count += 1
f = task["func"]
# Log task creation and set process name
# Get the function from the task
func_name = get_func_repr(f)
task_name = task["name"]
task_desc = (
_("%(proc_name)s processing %(task_name)s '%(func_name)s'")
% {
"proc_name": proc_name,
"func_name": func_name,
"task_name": task_name,
}
)
if "group" in task:
task_desc += f" [{task['group']}]"
logger.info(task_desc)
if setproctitle:
proc_title = f"qcluster {proc_name} processing {task_name} '{func_name}'"
if "group" in task:
proc_title += f" [{task['group']}]"
setproctitle.setproctitle(proc_title)
# if it's not an instance try to get it from the string
if not callable(f):
# locate() returns None if f cannot be loaded
f = pydoc.locate(f)
close_old_django_connections()
timer_value = task.pop("timeout", timeout)
# signal execution
pre_execute.send(sender="django_q", func=f, task=task)
# execute the payload
timer.value = timer_value # Busy
try:
if f is None:
# raise a meaningfull error if task["func"] is not a valid function
raise ValueError(f"Function {task['func']} is not defined")
res = f(*task["args"], **task["kwargs"])
result = (res, True)
except Exception as e:
result = (f"{e} : {traceback.format_exc()}", False)
if error_reporter:
error_reporter.report()
if task.get("sync", False):
raise
with timer.get_lock():
# Process result
task["result"] = result[0]
task["success"] = result[1]
task["stopped"] = timezone.now()
result_queue.put(task)
timer.value = -1 # Idle
if setproctitle:
setproctitle.setproctitle(f"qcluster {proc_name} idle")
# Recycle
if task_count == Conf.RECYCLE or rss_check():
timer.value = -2 # Recycled
break
logger.info(_("%(proc_name)s stopped doing work") % {"proc_name": proc_name})
def save_task(task, broker: Broker):
"""
Saves the task package to Django or the cache
:param task: the task package
:type broker: brokers.Broker
"""
# SAVE LIMIT < 0 : Don't save success
if not task.get("save", Conf.SAVE_LIMIT >= 0) and task["success"]:
return
# enqueues next in a chain
if task.get("chain", None):
django_q.tasks.async_chain(
task["chain"],
group=task["group"],
cached=task["cached"],
sync=task["sync"],
broker=broker,
)
# SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
close_old_django_connections()
try:
filters = {}
if (
Conf.SAVE_LIMIT_PER
and Conf.SAVE_LIMIT_PER in {"group", "name", "func"}
and Conf.SAVE_LIMIT_PER in task
):
value = task[Conf.SAVE_LIMIT_PER]
if Conf.SAVE_LIMIT_PER == "func":
value = get_func_repr(value)
filters[Conf.SAVE_LIMIT_PER] = value
with db.transaction.atomic(using=db.router.db_for_write(Success)):
last = Success.objects.filter(**filters).select_for_update().last()
if (
task["success"]
and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
):
last.delete()
# check if this task has previous results
try:
existing_task = Task.objects.get(id=task["id"], name=task["name"])
# only update the result if it hasn't succeeded yet
if not existing_task.success:
existing_task.stopped = task["stopped"]
existing_task.result = task["result"]
existing_task.success = task["success"]
existing_task.attempt_count = existing_task.attempt_count + 1
existing_task.save()
if (
Conf.MAX_ATTEMPTS > 0
and existing_task.attempt_count >= Conf.MAX_ATTEMPTS
):
broker.acknowledge(task["ack_id"])
except Task.DoesNotExist:
# convert func to string
func = get_func_repr(task["func"])
Task.objects.create(
id=task["id"],
name=task["name"],
func=func,
hook=task.get("hook"),
args=task["args"],
kwargs=task["kwargs"],
cluster=task.get("cluster"),
started=task["started"],
stopped=task["stopped"],
result=task["result"],
group=task.get("group"),
success=task["success"],
attempt_count=1,
)
except Exception:
logger.exception("Could not save task result")
def save_cached(task, broker: Broker):
task_key = f'{broker.list_key}:{task["id"]}'
timeout = task["cached"]
if timeout is True:
timeout = None
try:
group = task.get("group", None)
iter_count = task.get("iter_count", 0)
# if it's a group append to the group list
if group:
group_key = f"{broker.list_key}:{group}:keys"
group_list = broker.cache.get(group_key) or []
# if it's an iter group, check if we are ready
if iter_count and len(group_list) == iter_count - 1:
group_args = f"{broker.list_key}:{group}:args"
# collate the results into a Task result
results = [
SignedPackage.loads(broker.cache.get(k))["result"]
for k in group_list
]
results.append(task["result"])
task["result"] = results
task["id"] = group
task["args"] = SignedPackage.loads(broker.cache.get(group_args))
task.pop("iter_count", None)
task.pop("group", None)
if task.get("iter_cached", None):
task["cached"] = task.pop("iter_cached", None)
save_cached(task, broker=broker)
else:
save_task(task, broker)
broker.cache.delete_many(group_list)
broker.cache.delete_many([group_key, group_args])
return
# save the group list
group_list.append(task_key)
broker.cache.set(group_key, group_list, timeout)
# async_task next in a chain
if task.get("chain", None):
django_q.tasks.async_chain(
task["chain"],
group=group,
cached=task["cached"],
sync=task["sync"],
broker=broker,
)
# save the task
broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
except Exception:
logger.exception("Could not save task result")
def scheduler(broker: Broker = None):
"""
Creates a task from a schedule at the scheduled time and schedules next run
"""
if not broker:
broker = get_broker()
close_old_django_connections()
try:
# Only default cluster will handler schedule with default(null) cluster
Q_default = db.models.Q(cluster__isnull=True) if Conf.CLUSTER_NAME == Conf.PREFIX else db.models.Q(pk__in=[])
with db.transaction.atomic(using=db.router.db_for_write(Schedule)):
for s in (
Schedule.objects.select_for_update()
.exclude(repeats=0)
.filter(next_run__lt=timezone.now())
.filter(
Q_default | db.models.Q(cluster=Conf.CLUSTER_NAME)
)
):
args = ()
kwargs = {}
# get args, kwargs and hook
if s.kwargs:
try:
# first try the dict syntax
kwargs = ast.literal_eval(s.kwargs)
except (SyntaxError, ValueError):
# else use the kwargs syntax
try:
parsed_kwargs = (
ast.parse(f"f({s.kwargs})").body[0].value.keywords
)
kwargs = {
kwarg.arg: ast.literal_eval(kwarg.value)
for kwarg in parsed_kwargs
}
except (SyntaxError, ValueError):
kwargs = {}
if s.args:
args = ast.literal_eval(s.args)
# single value won't eval to tuple, so:
if type(args) != tuple:
args = (args,)
q_options = kwargs.get("q_options", {})
if s.intended_date_kwarg:
kwargs[s.intended_date_kwarg] = s.next_run.isoformat()
if s.hook:
q_options["hook"] = s.hook
# set up the next run time
if s.schedule_type != s.ONCE:
next_run = s.next_run
while True:
next_run = s.calculate_next_run(next_run)
if Conf.CATCH_UP or next_run > localtime():
break
s.next_run = next_run
s.repeats += -1
# send it to the cluster; any cluster name is allowed in multi-queue scenarios
# because `broker_name` is confusing, using `cluster` name is recommended and takes precedence
q_options["cluster"] = s.cluster or q_options.get("cluster", q_options.pop("broker_name", None))
if q_options['cluster'] is None or q_options['cluster'] == Conf.CLUSTER_NAME:
q_options["broker"] = broker
q_options["group"] = q_options.get("group", s.name or s.id)
kwargs["q_options"] = q_options
s.task = django_q.tasks.async_task(s.func, *args, **kwargs)
# log it
if not s.task:
logger.error(
_(
"%(process_name)s failed to create a task from schedule "
"[%(schedule)s]"
)
% {
"process_name": current_process().name,
"schedule": s.name or s.id,
}
)
else:
logger.info(
_(
"%(process_name)s created task %(task_name)s from schedule "
"[%(schedule)s]"
)
% {
"process_name": current_process().name,
"task_name": humanize(s.task),
"schedule": s.name or s.id,
}
)
# default behavior is to delete a ONCE schedule
if s.schedule_type == s.ONCE:
if s.repeats < 0:
s.delete()
continue
# but not if it has a positive repeats
s.repeats = 0
# save the schedule
s.save()
except Exception:
logger.exception("Could not create task from schedule")
def close_old_django_connections():
"""
Close django connections unless running with sync=True.
"""
if Conf.SYNC:
logger.warning(
"Preserving django database connections because sync=True. Beware "
"that tasks are now injected in the calling context/transactions "
"which may result in unexpected behaviour."
)
else:
db.close_old_connections()
def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING):
"""
Sets the cpu affinity for the supplied processes.
@@ -846,12 +414,3 @@ def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING)
_("%(pid)s will use cpu %(affinity)s")
% {"pid": pid, "affinity": affinity}
)
def rss_check():
if Conf.MAX_RSS:
if resource:
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
elif psutil:
return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
return False

View File

@@ -1,16 +1,22 @@
import logging
import os
import sys
from copy import deepcopy
from multiprocessing import cpu_count
from signal import signal
from warnings import warn
import pkg_resources
from django.conf import settings
from django.utils.translation import gettext_lazy as _
from django_q.queues import Queue
# The "selectable" entry points were introduced in importlib_metadata 3.6 and Python 3.10.
if sys.version_info < (3, 10):
from importlib_metadata import entry_points
else:
from importlib.metadata import entry_points
# optional
try:
import psutil
@@ -32,6 +38,11 @@ try:
except ModuleNotFoundError:
setproctitle = None
try:
from prometheus_client import multiprocess as prometheus_multiprocess
except ModuleNotFoundError:
prometheus_multiprocess = None
class Conf:
"""
@@ -44,15 +55,18 @@ class Conf:
conf = {}
_Q_CLUSTER_NAME = os.getenv("Q_CLUSTER_NAME")
if _Q_CLUSTER_NAME and _Q_CLUSTER_NAME != conf.get("name") and \
_Q_CLUSTER_NAME != conf.get("cluster_name"):
if (
_Q_CLUSTER_NAME
and _Q_CLUSTER_NAME != conf.get("name")
and _Q_CLUSTER_NAME != conf.get("cluster_name")
):
conf["cluster_name"] = _Q_CLUSTER_NAME
alt_conf = conf.pop("ALT_CLUSTERS")
if isinstance(alt_conf, dict):
alt_conf = alt_conf.get(_Q_CLUSTER_NAME)
if isinstance(alt_conf, dict):
alt_conf.pop('name', None)
alt_conf.pop('cluster_name', None)
alt_conf.pop("name", None)
alt_conf.pop("cluster_name", None)
conf.update(alt_conf)
# Redis server configuration . Follows standard redis keywords
@@ -254,7 +268,6 @@ if not logger.hasHandlers():
# Error Reporting Interface
class ErrorReporter:
# initialize with iterator of reporters (better name, targets?)
def __init__(self, reporters):
self.targets = [target for target in reporters]
@@ -273,9 +286,7 @@ if Conf.ERROR_REPORTER:
# iterate through the configured error reporters,
# and instantiate an ErrorReporter using the provided config
for name, conf in error_conf.items():
for entry in pkg_resources.iter_entry_points(
"djangoq.errorreporters", name
):
for entry in entry_points(group="djangoq.errorreporters", name=name):
Reporter = entry.load()
reporters.append(Reporter(**conf))
error_reporter = ErrorReporter(reporters)

View File

@@ -2,16 +2,24 @@ import datetime
import time
import zlib
from django.core.signing import BadSignature, JSONSerializer, SignatureExpired
from django.core.signing import (
BadSignature,
JSONSerializer,
SignatureExpired,
b64_decode,
dumps,
)
from django.core.signing import Signer as Sgnr
from django.core.signing import TimestampSigner as TsS
from django.core.signing import b64_decode, dumps
try:
from django.core.signing import base62
from django.core.signing import b62_decode
except ImportError:
# For django < 4.0
# fallback for django 3.x
from django.utils.baseconv import base62
b62_decode = base62.decode
from django.utils.crypto import constant_time_compare
from django.utils.encoding import force_bytes, force_str
@@ -37,7 +45,9 @@ def loads(
"""
# TimestampSigner.unsign() returns str but base64 and zlib compression
# operate on bytes.
base64d = force_bytes(TimestampSigner(key=key, salt=salt).unsign(s, max_age=max_age))
base64d = force_bytes(
TimestampSigner(key=key, salt=salt).unsign(s, max_age=max_age)
)
decompress = False
if base64d[:1] == b".":
# It's compressed; uncompress it first
@@ -74,7 +84,7 @@ class TimestampSigner(Signer, TsS):
"""
result = super(TimestampSigner, self).unsign(value)
value, timestamp = result.rsplit(self.sep, 1)
timestamp = base62.decode(timestamp)
timestamp = b62_decode(timestamp)
if max_age is not None:
if isinstance(max_age, datetime.timedelta):
max_age = max_age.total_seconds()

7
django_q/exceptions.py Normal file
View File

@@ -0,0 +1,7 @@
class TimeoutException(SystemExit):
"""
Exception for when a worker takes too long to complete a task
Raising SystemExit will make sure the function terminates gracefully.
"""
pass

View File

@@ -4,6 +4,7 @@ humanhash: Human-readable representations of digests.
The simplest ways to use this module are the :func:`humanize` and :func:`uuid`
functions. For tighter control over the output, see :class:`HumanHasher`.
"""
import operator
import uuid as uuidlib
from argparse import ArgumentError
@@ -270,7 +271,6 @@ DEFAULT_WORDLIST = (
class HumanHasher:
"""
Transforms hex digests to human-readable strings.
@@ -290,7 +290,6 @@ class HumanHasher:
self.wordlist = wordlist
def humanize(self, hexdigest, words=4, separator="-"):
"""
Humanize a given hexadecimal digest.
@@ -314,7 +313,6 @@ class HumanHasher:
@staticmethod
def compress(bytes, target):
"""
Compress a list of byte values to a fixed target length.
@@ -353,7 +351,6 @@ class HumanHasher:
return checksums
def uuid(self, **params):
"""
Generate a UUID with a human-readable representation.

View File

@@ -318,7 +318,7 @@ msgstr "Anzahl Minuten für den Typ 'Minuten'"
#: models.py:205
msgid "Repeats"
msgstr "Wiederhohlungen"
msgstr "Wiederholungen"
#: models.py:205
msgid "n = n times, -1 = forever"

View File

@@ -303,7 +303,7 @@ msgstr "Bimestriel"
#: models.py:194
msgid "Quarterly"
msgstr "Tous les quart-dheure"
msgstr "Trimestriel"
#: models.py:195
msgid "Yearly"

View File

@@ -1,8 +1,9 @@
import os
from django.core.management.base import BaseCommand
from django.utils.translation import gettext as _
from django_q.cluster import Cluster
import os
class Command(BaseCommand):
@@ -23,7 +24,7 @@ class Command(BaseCommand):
dest="cluster_name",
default=None,
help="Set alternative cluster name instead of the name in Q_CLUSTER settings (for multi-queue setup). "
"On Linux you should set name through `Q_CLUSTER_NAME=cluster_name python manage.py qcluster` instead."
"On Linux you should set name through `Q_CLUSTER_NAME=cluster_name python manage.py qcluster` instead.",
)
def handle(self, *args, **options):

View File

@@ -3,7 +3,7 @@ from django.utils.translation import gettext as _
from django_q import VERSION
from django_q.conf import Conf
from django_q.monitor import get_ids, info
from django_q.monitor_terminal import get_ids, info
class Command(BaseCommand):

View File

@@ -1,7 +1,7 @@
from django.core.management.base import BaseCommand
from django.utils.translation import gettext as _
from django_q.monitor import memory
from django_q.monitor_terminal import memory
class Command(BaseCommand):

View File

@@ -1,7 +1,7 @@
from django.core.management.base import BaseCommand
from django.utils.translation import gettext as _
from django_q.monitor import monitor
from django_q.monitor_terminal import monitor
class Command(BaseCommand):

View File

@@ -4,7 +4,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = []
operations = [

View File

@@ -2,7 +2,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0001_initial"),
]

View File

@@ -2,7 +2,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0002_auto_20150630_1624"),
]

View File

@@ -2,7 +2,6 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("django_q", "0003_auto_20150708_1326"),
]

View File

@@ -2,7 +2,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0004_auto_20150710_1043"),
]

View File

@@ -2,7 +2,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0005_auto_20150718_1506"),
]

View File

@@ -2,7 +2,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0006_auto_20150805_1817"),
]

View File

@@ -2,7 +2,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0007_ormq"),
]

View File

@@ -2,7 +2,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0008_auto_20160224_1026"),
]

View File

@@ -3,7 +3,6 @@ from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("django_q", "0009_auto_20171009_0915"),
]

View File

@@ -4,7 +4,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0010_auto_20200610_0856"),
]

View File

@@ -6,7 +6,6 @@ import django_q.models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0011_auto_20200628_1055"),
]

View File

@@ -4,7 +4,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0012_auto_20200702_1608"),
]

View File

@@ -4,7 +4,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0013_task_attempt_count"),
]

View File

@@ -4,7 +4,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0014_schedule_cluster"),
]

View File

@@ -1,11 +1,11 @@
# Generated by Django 4.1.2 on 2023-01-15 22:34
from django.db import migrations, models
import django_q.models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0015_alter_schedule_schedule_type"),
]

View File

@@ -4,7 +4,6 @@ from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0016_schedule_intended_date_kwarg"),
]

View File

@@ -0,0 +1,20 @@
# Generated by Django 4.2.7 on 2024-03-05 17:03
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("django_q", "0017_task_cluster_alter"),
]
operations = [
migrations.AddIndex(
model_name="task",
index=models.Index(
condition=models.Q(("success", True)),
fields=["group", "name", "func"],
name="success_index",
),
),
]

View File

@@ -5,22 +5,22 @@ from keyword import iskeyword
from django import get_version
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models import Q
from django.template.defaultfilters import truncatechars
from django.urls import reverse
from django.utils import timezone
from django.utils.timezone import is_aware
from django.utils.functional import cached_property
from django.utils.html import format_html
from django.utils.translation import gettext_lazy as _
from django.utils.functional import cached_property
# External
from picklefield import PickledObjectField
from picklefield.fields import dbsafe_decode
# Local
from django_q.conf import croniter, Conf
from django_q.conf import croniter
from django_q.signing import SignedPackage
from django_q.utils import localtime, add_months, add_years
from django_q.utils import add_months, add_years, localtime
from .utils import get_func_repr
@@ -112,6 +112,13 @@ class Task(models.Model):
class Meta:
app_label = "django_q"
ordering = ["-stopped"]
indexes = [
models.Index(
name="success_index",
fields=["group", "name", "func"],
condition=Q(success=True),
),
]
class SuccessManager(models.Manager):
@@ -218,8 +225,11 @@ class Schedule(models.Model):
)
task = models.CharField(max_length=100, null=True, editable=False)
cluster = models.CharField(
max_length=100, default=None, null=True, blank=True,
help_text=_("Name of the target cluster")
max_length=100,
default=None,
null=True,
blank=True,
help_text=_("Name of the target cluster"),
)
intended_date_kwarg = models.CharField(
max_length=100,
@@ -309,7 +319,9 @@ class Schedule(models.Model):
class OrmQ(models.Model):
key = models.CharField(max_length=100, help_text=_("Name of the target cluster"))
payload = models.TextField()
lock = models.DateTimeField(null=True, help_text=_("Prevent any cluster from pulling until"))
lock = models.DateTimeField(
null=True, help_text=_("Prevent any cluster from pulling until")
)
@cached_property
def task(self):

View File

@@ -1,510 +1,204 @@
from datetime import timedelta
from multiprocessing.process import current_process
from multiprocessing.queues import Queue
# django
from django.db import connection
from django.db.models import F, Sum
from django.utils import timezone
from django.utils.translation import gettext as _
from django import core, db
from django.apps.registry import apps
from django.utils.translation import gettext_lazy as _
from django_q import VERSION, models
from django_q.brokers import get_broker
# local
from django_q.conf import Conf
from django_q.status import Stat
# optional
try:
import psutil
except ImportError:
psutil = None
apps.check_apps_ready()
except core.exceptions.AppRegistryNotReady:
import django
django.setup()
from django_q.brokers import Broker, get_broker
from django_q.conf import Conf, logger, setproctitle
from django_q.models import Success, Task
from django_q.signals import post_execute
from django_q.signing import SignedPackage
from django_q.tasks import async_chain
from django_q.utils import close_old_django_connections, get_func_repr
try:
import setproctitle
except ModuleNotFoundError:
setproctitle = None
def get_process_mb(pid):
try:
process = psutil.Process(pid)
mb_used = round(process.memory_info().rss / 1024**2, 2)
except psutil.NoSuchProcess:
mb_used = "NO_PROCESS_FOUND"
return mb_used
BLESSED_INSTALL_MESSAGE = (
"Blessed is not installed. Please install blessed to use this: "
"https://pypi.org/project/blessed/"
)
def monitor(run_once=False, broker=None):
def monitor(result_queue: Queue, broker: Broker = None):
"""
Gets finished tasks from the result queue and saves them to Django
:type broker: brokers.Broker
:type result_queue: multiprocessing.Queue
"""
if not broker:
broker = get_broker()
try:
from blessed import Terminal
proc_name = current_process().name
if setproctitle:
setproctitle.setproctitle(f"qcluster {proc_name} monitor")
logger.info(
_("%(name)s monitoring at %(id)s")
% {"name": proc_name, "id": current_process().pid}
)
for task in iter(result_queue.get, "STOP"):
# save the result
if task.get("cached", False):
save_cached(task, broker)
else:
save_task(task, broker)
# acknowledge result
ack_id = task.pop("ack_id", False)
if ack_id and (task["success"] or task.get("ack_failure", False)):
broker.acknowledge(ack_id)
# signal execution done
post_execute.send(sender="django_q", task=task)
# log the result
info_name = get_func_repr(task["func"])
if task["success"]:
# log success
logger.info(
_("Processed '%(info_name)s' (%(task_name)s)")
% {"info_name": info_name, "task_name": task["name"]}
)
else:
# log failure
logger.error(
_("Failed '%(info_name)s' (%(task_name)s) - %(task_result)s")
% {
"info_name": info_name,
"task_name": task["name"],
"task_result": task["result"],
}
)
logger.info(_("%(name)s stopped monitoring results") % {"name": proc_name})
term = Terminal()
except ImportError:
print(BLESSED_INSTALL_MESSAGE)
def save_task(task, broker: Broker):
"""
Saves the task package to Django or the cache
:param task: the task package
:type broker: brokers.Broker
"""
# SAVE LIMIT < 0 : Don't save success
if not task.get("save", Conf.SAVE_LIMIT >= 0) and task["success"]:
return
# enqueues next in a chain
if task.get("chain", None):
async_chain(
task["chain"],
group=task["group"],
cached=task["cached"],
sync=task["sync"],
broker=broker,
)
# SAVE LIMIT > 0: Prune database, SAVE_LIMIT 0: No pruning
close_old_django_connections()
broker.ping()
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
val = None
start_width = int(term.width / 8)
while val not in (
"q",
"Q",
try:
filters = {}
if (
Conf.SAVE_LIMIT_PER
and Conf.SAVE_LIMIT_PER in {"group", "name", "func"}
and Conf.SAVE_LIMIT_PER in task
):
col_width = int(term.width / 8)
# In case of resize
if col_width != start_width:
print(term.clear())
start_width = col_width
print(
term.move(0, 0)
+ term.black_on_green(term.center(_("Host"), width=col_width - 1))
value = task[Conf.SAVE_LIMIT_PER]
if Conf.SAVE_LIMIT_PER == "func":
value = get_func_repr(value)
filters[Conf.SAVE_LIMIT_PER] = value
with db.transaction.atomic(using=db.router.db_for_write(Success)):
list(Success.objects.filter(**filters).select_for_update())
if (
task["success"]
and 0 < Conf.SAVE_LIMIT <= Success.objects.filter(**filters).count()
):
Success.objects.filter(**filters).last().delete()
# check if this task has previous results
try:
task_obj = Task.objects.get(id=task["id"], name=task["name"])
# only update the result if it hasn't succeeded yet
if not task_obj.success:
task_obj.stopped = task["stopped"]
task_obj.result = task["result"]
task_obj.success = task["success"]
task_obj.attempt_count = task_obj.attempt_count + 1
task_obj.save()
except Task.DoesNotExist:
# convert func to string
func = get_func_repr(task["func"])
task_obj = Task.objects.create(
id=task["id"],
name=task["name"],
func=func,
hook=task.get("hook"),
args=task["args"],
kwargs=task["kwargs"],
cluster=task.get("cluster"),
started=task["started"],
stopped=task["stopped"],
result=task["result"],
group=task.get("group"),
success=task["success"],
attempt_count=1,
)
print(
term.move(0, 1 * col_width)
+ term.black_on_green(term.center(_("Id"), width=col_width - 1))
)
print(
term.move(0, 2 * col_width)
+ term.black_on_green(term.center(_("State"), width=col_width - 1))
)
print(
term.move(0, 3 * col_width)
+ term.black_on_green(term.center(_("Pool"), width=col_width - 1))
)
print(
term.move(0, 4 * col_width)
+ term.black_on_green(term.center(_("TQ"), width=col_width - 1))
)
print(
term.move(0, 5 * col_width)
+ term.black_on_green(term.center(_("RQ"), width=col_width - 1))
)
print(
term.move(0, 6 * col_width)
+ term.black_on_green(term.center(_("RC"), width=col_width - 1))
)
print(
term.move(0, 7 * col_width)
+ term.black_on_green(term.center(_("Up"), width=col_width - 1))
)
i = 2
stats = Stat.get_all(broker=broker)
print(term.clear_eos())
for stat in stats:
status = stat.status
# color status
if stat.status == Conf.WORKING:
status = term.green(str(Conf.WORKING))
elif stat.status == Conf.STOPPING:
status = term.yellow(str(Conf.STOPPING))
elif stat.status == Conf.STOPPED:
status = term.red(str(Conf.STOPPED))
elif stat.status == Conf.IDLE:
status = str(Conf.IDLE)
# color q's
tasks = str(stat.task_q_size)
if stat.task_q_size > 0:
tasks = term.cyan(str(stat.task_q_size))
if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
tasks = term.green(str(stat.task_q_size))
results = stat.done_q_size
if results > 0:
results = term.cyan(str(results))
# color workers
workers = len(stat.workers)
if workers < Conf.WORKERS:
workers = term.yellow(str(workers))
# format uptime
uptime = (timezone.now() - stat.tob).total_seconds()
hours, remainder = divmod(uptime, 3600)
minutes, seconds = divmod(remainder, 60)
uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
# print to the terminal
print(
term.move(i, 0)
+ term.center(stat.host[: col_width - 1], width=col_width - 1)
)
print(
term.move(i, 1 * col_width)
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
)
print(
term.move(i, 2 * col_width)
+ term.center(status, width=col_width - 1)
)
print(
term.move(i, 3 * col_width)
+ term.center(workers, width=col_width - 1)
)
print(
term.move(i, 4 * col_width)
+ term.center(tasks, width=col_width - 1)
)
print(
term.move(i, 5 * col_width)
+ term.center(results, width=col_width - 1)
)
print(
term.move(i, 6 * col_width)
+ term.center(stat.reincarnations, width=col_width - 1)
)
print(
term.move(i, 7 * col_width)
+ term.center(uptime, width=col_width - 1)
)
i += 1
# bottom bar
i += 1
queue_size = broker.queue_size()
lock_size = broker.lock_size()
if lock_size:
queue_size = f"{queue_size}({lock_size})"
print(
term.move(i, 0)
+ term.white_on_cyan(term.center(broker.info(), width=col_width * 2))
)
print(
term.move(i, 2 * col_width)
+ term.black_on_cyan(term.center(_("Queued"), width=col_width))
)
print(
term.move(i, 3 * col_width)
+ term.white_on_cyan(term.center(queue_size, width=col_width))
)
print(
term.move(i, 4 * col_width)
+ term.black_on_cyan(term.center(_("Success"), width=col_width))
)
print(
term.move(i, 5 * col_width)
+ term.white_on_cyan(
term.center(models.Success.objects.count(), width=col_width)
)
)
print(
term.move(i, 6 * col_width)
+ term.black_on_cyan(term.center(_("Failures"), width=col_width))
)
print(
term.move(i, 7 * col_width)
+ term.white_on_cyan(
term.center(models.Failure.objects.count(), width=col_width)
)
)
# for testing
if run_once:
return Stat.get_all(broker=broker)
print(term.move(i + 2, 0) + term.center(_("[Press q to quit]")))
val = term.inkey(timeout=1)
if Conf.MAX_ATTEMPTS > 0 and task_obj.attempt_count >= Conf.MAX_ATTEMPTS:
broker.acknowledge(task["ack_id"])
except Exception:
logger.exception("Could not save task result")
def info(broker=None):
if not broker:
broker = get_broker()
def save_cached(task, broker: Broker):
task_key = f'{broker.list_key}:{task["id"]}'
timeout = task["cached"]
if timeout is True:
timeout = None
try:
from blessed import Terminal
term = Terminal()
except ImportError:
print(BLESSED_INSTALL_MESSAGE)
return
broker.ping()
stat = Stat.get_all(broker=broker)
# general stats
clusters = len(stat)
workers = 0
reincarnations = 0
for cluster in stat:
workers += len(cluster.workers)
reincarnations += cluster.reincarnations
# calculate tasks pm and avg exec time
tasks_per = 0
per = _("day")
exec_time = 0
last_tasks = models.Success.objects.filter(
stopped__gte=timezone.now() - timedelta(hours=24)
)
tasks_per_day = last_tasks.count()
if tasks_per_day > 0:
# average execution time over the last 24 hours
if connection.vendor != "sqlite":
exec_time = last_tasks.aggregate(
time_taken=Sum(F("stopped") - F("started"))
)
exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
else:
# can't sum timedeltas on sqlite
for t in last_tasks:
exec_time += t.time_taken()
exec_time = exec_time / tasks_per_day
# tasks per second/minute/hour/day in the last 24 hours
if tasks_per_day > 24 * 60 * 60:
tasks_per = tasks_per_day / (24 * 60 * 60)
per = _("second")
elif tasks_per_day > 24 * 60:
tasks_per = tasks_per_day / (24 * 60)
per = _("minute")
elif tasks_per_day > 24:
tasks_per = tasks_per_day / 24
per = _("hour")
else:
tasks_per = tasks_per_day
# print to terminal
print(term.clear_eos())
col_width = int(term.width / 6)
print(
term.black_on_green(
term.center(
_("-- %(prefix)s %(version)s on %(info)s --")
% {
"prefix": Conf.PREFIX.capitalize(),
"version": ".".join(str(v) for v in VERSION),
"info": broker.info(),
}
)
)
)
print(
term.cyan(_("Clusters"))
+ term.move_x(1 * col_width)
+ term.white(str(clusters))
+ term.move_x(2 * col_width)
+ term.cyan(_("Workers"))
+ term.move_x(3 * col_width)
+ term.white(str(workers))
+ term.move_x(4 * col_width)
+ term.cyan(_("Restarts"))
+ term.move_x(5 * col_width)
+ term.white(str(reincarnations))
)
print(
term.cyan(_("Queued"))
+ term.move_x(1 * col_width)
+ term.white(str(broker.queue_size()))
+ term.move_x(2 * col_width)
+ term.cyan(_("Successes"))
+ term.move_x(3 * col_width)
+ term.white(str(models.Success.objects.count()))
+ term.move_x(4 * col_width)
+ term.cyan(_("Failures"))
+ term.move_x(5 * col_width)
+ term.white(str(models.Failure.objects.count()))
)
print(
term.cyan(_("Schedules"))
+ term.move_x(1 * col_width)
+ term.white(str(models.Schedule.objects.count()))
+ term.move_x(2 * col_width)
+ term.cyan(_("Tasks/%(per)s") % {"per": per})
+ term.move_x(3 * col_width)
+ term.white(f"{tasks_per:.2f}")
+ term.move_x(4 * col_width)
+ term.cyan(_("Avg time"))
+ term.move_x(5 * col_width)
+ term.white(f"{exec_time:.4f}")
)
return True
def memory(run_once=False, workers=False, broker=None):
if not broker:
broker = get_broker()
try:
from blessed import Terminal
term = Terminal()
except ImportError:
print(BLESSED_INSTALL_MESSAGE)
return
broker.ping()
if not psutil:
print(term.clear_eos())
print(
term.white_on_red(
'Cannot start "qmemory" command. Missing "psutil" library.'
)
)
return
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
cols = 8
val = None
start_width = int(term.width / cols)
while val not in ["q", "Q"]:
col_width = int(term.width / cols)
# In case of resize
if col_width != start_width:
print(term.clear())
start_width = col_width
# sentinel, monitor and workers memory usage
print(
term.move(0, 0 * col_width)
+ term.black_on_green(term.center(_("Host"), width=col_width - 1))
)
print(
term.move(0, 1 * col_width)
+ term.black_on_green(term.center(_("Id"), width=col_width - 1))
)
print(
term.move(0, 2 * col_width)
+ term.black_on_green(
term.center(_("Available (%)"), width=col_width - 1)
group = task.get("group", None)
iter_count = task.get("iter_count", 0)
# if it's a group append to the group list
if group:
group_key = f"{broker.list_key}:{group}:keys"
group_list = broker.cache.get(group_key) or []
# if it's an iter group, check if we are ready
if iter_count and len(group_list) == iter_count - 1:
group_args = f"{broker.list_key}:{group}:args"
# collate the results into a Task result
results = [
SignedPackage.loads(broker.cache.get(k))["result"]
for k in group_list
]
results.append(task["result"])
task["result"] = results
task["id"] = group
task["args"] = SignedPackage.loads(broker.cache.get(group_args))
task.pop("iter_count", None)
task.pop("group", None)
if task.get("iter_cached", None):
task["cached"] = task.pop("iter_cached", None)
save_cached(task, broker=broker)
else:
save_task(task, broker)
broker.cache.delete_many(group_list)
broker.cache.delete_many([group_key, group_args])
return
# save the group list
group_list.append(task_key)
broker.cache.set(group_key, group_list, timeout)
# async_task next in a chain
if task.get("chain", None):
async_chain(
task["chain"],
group=group,
cached=task["cached"],
sync=task["sync"],
broker=broker,
)
)
print(
term.move(0, 3 * col_width)
+ term.black_on_green(
term.center(_("Available (MB)"), width=col_width - 1)
)
)
print(
term.move(0, 4 * col_width)
+ term.black_on_green(term.center(_("Total (MB)"), width=col_width - 1))
)
print(
term.move(0, 5 * col_width)
+ term.black_on_green(
term.center(_("Sentinel (MB)"), width=col_width - 1)
)
)
print(
term.move(0, 6 * col_width)
+ term.black_on_green(
term.center(_("Monitor (MB)"), width=col_width - 1)
)
)
print(
term.move(0, 7 * col_width)
+ term.black_on_green(
term.center(_("Workers (MB)"), width=col_width - 1)
)
)
row = 2
stats = Stat.get_all(broker=broker)
print(term.clear_eos())
for stat in stats:
# memory available (%)
memory_available_percentage = round(
psutil.virtual_memory().available
* 100
/ psutil.virtual_memory().total,
2,
)
# memory available (MB)
memory_available = round(
psutil.virtual_memory().available / 1024**2, 2
)
if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
print(
term.move(row, 0 * col_width)
+ term.center(stat.host[: col_width - 1], width=col_width - 1)
)
print(
term.move(row, 1 * col_width)
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
)
print(
term.move(row, 2 * col_width)
+ term.center(memory_available_percentage, width=col_width - 1)
)
print(
term.move(row, 3 * col_width)
+ term.center(memory_available, width=col_width - 1)
)
print(
term.move(row, 4 * col_width)
+ term.center(
round(psutil.virtual_memory().total / 1024**2, 2),
width=col_width - 1,
)
)
print(
term.move(row, 5 * col_width)
+ term.center(get_process_mb(stat.sentinel), width=col_width - 1)
)
print(
term.move(row, 6 * col_width)
+ term.center(
get_process_mb(getattr(stat, "monitor", None)),
width=col_width - 1,
)
)
workers_mb = 0
for worker_pid in stat.workers:
result = get_process_mb(worker_pid)
if isinstance(result, str):
result = 0
workers_mb += result
print(
term.move(row, 7 * col_width)
+ term.center(
workers_mb or "NO_PROCESSES_FOUND", width=col_width - 1
)
)
row += 1
# each worker's memory usage
if workers:
row += 2
col_width = int(term.width / (1 + Conf.WORKERS))
print(
term.move(row, 0 * col_width)
+ term.black_on_cyan(term.center(_("Id"), width=col_width - 1))
)
for worker_num in range(Conf.WORKERS):
print(
term.move(row, (worker_num + 1) * col_width)
+ term.black_on_cyan(
term.center(
"Worker #{} (MB)".format(worker_num + 1),
width=col_width - 1,
)
)
)
row += 2
for stat in stats:
print(
term.move(row, 0 * col_width)
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
)
for idx, worker_pid in enumerate(stat.workers):
mb_used = get_process_mb(worker_pid)
print(
term.move(row, (idx + 1) * col_width)
+ term.center(mb_used, width=col_width - 1)
)
row += 1
row += 1
print(
term.move(row, 0)
+ _("Available lowest (): %(memory_percent)s ((at)s)")
% {
"memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
"at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
"%Y-%m-%d %H:%M:%S+00:00"
),
}
)
# for testing
if run_once:
return Stat.get_all(broker=broker)
print(term.move(row + 2, 0) + term.center(_("[Press q to quit]")))
val = term.inkey(timeout=1)
def get_ids():
# prints id (PID) of running clusters
stat = Stat.get_all()
if stat:
for s in stat:
print(s.cluster_id)
else:
print(_("No clusters appear to be running."))
return True
# save the task
broker.cache.set(task_key, SignedPackage.dumps(task), timeout)
except Exception:
logger.exception("Could not save task result")

View File

@@ -0,0 +1,508 @@
from datetime import timedelta
# django
from django.db import connection
from django.db.models import F, Sum
from django.utils import timezone
from django.utils.translation import gettext as _
from django_q import VERSION, models
from django_q.brokers import get_broker
# local
from django_q.conf import Conf
from django_q.status import Stat
# optional
try:
import psutil
except ImportError:
psutil = None
def get_process_mb(pid):
try:
process = psutil.Process(pid)
mb_used = round(process.memory_info().rss / 1024**2, 2)
except psutil.NoSuchProcess:
mb_used = "NO_PROCESS_FOUND"
return mb_used
BLESSED_INSTALL_MESSAGE = (
"Blessed is not installed. Please install blessed to use this: "
"https://pypi.org/project/blessed/"
)
def monitor(run_once=False, broker=None):
if not broker:
broker = get_broker()
try:
from blessed import Terminal
term = Terminal()
except ImportError:
print(BLESSED_INSTALL_MESSAGE)
return
broker.ping()
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
val = None
start_width = int(term.width / 8)
while val not in (
"q",
"Q",
):
col_width = int(term.width / 8)
# In case of resize
if col_width != start_width:
print(term.clear())
start_width = col_width
print(
term.move(0, 0)
+ term.black_on_green(term.center(_("Host"), width=col_width - 1))
)
print(
term.move(0, 1 * col_width)
+ term.black_on_green(term.center(_("Id"), width=col_width - 1))
)
print(
term.move(0, 2 * col_width)
+ term.black_on_green(term.center(_("State"), width=col_width - 1))
)
print(
term.move(0, 3 * col_width)
+ term.black_on_green(term.center(_("Pool"), width=col_width - 1))
)
print(
term.move(0, 4 * col_width)
+ term.black_on_green(term.center(_("TQ"), width=col_width - 1))
)
print(
term.move(0, 5 * col_width)
+ term.black_on_green(term.center(_("RQ"), width=col_width - 1))
)
print(
term.move(0, 6 * col_width)
+ term.black_on_green(term.center(_("RC"), width=col_width - 1))
)
print(
term.move(0, 7 * col_width)
+ term.black_on_green(term.center(_("Up"), width=col_width - 1))
)
i = 2
stats = Stat.get_all(broker=broker)
print(term.clear_eos())
for stat in stats:
status = stat.status
# color status
if stat.status == Conf.WORKING:
status = term.green(str(Conf.WORKING))
elif stat.status == Conf.STOPPING:
status = term.yellow(str(Conf.STOPPING))
elif stat.status == Conf.STOPPED:
status = term.red(str(Conf.STOPPED))
elif stat.status == Conf.IDLE:
status = str(Conf.IDLE)
# color q's
tasks = str(stat.task_q_size)
if stat.task_q_size > 0:
tasks = term.cyan(str(stat.task_q_size))
if Conf.QUEUE_LIMIT and stat.task_q_size == Conf.QUEUE_LIMIT:
tasks = term.green(str(stat.task_q_size))
results = stat.done_q_size
if results > 0:
results = term.cyan(str(results))
# color workers
workers = len(stat.workers)
if workers < Conf.WORKERS:
workers = term.yellow(str(workers))
# format uptime
uptime = (timezone.now() - stat.tob).total_seconds()
hours, remainder = divmod(uptime, 3600)
minutes, seconds = divmod(remainder, 60)
uptime = "%d:%02d:%02d" % (hours, minutes, seconds)
# print to the terminal
print(
term.move(i, 0)
+ term.center(stat.host[: col_width - 1], width=col_width - 1)
)
print(
term.move(i, 1 * col_width)
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
)
print(
term.move(i, 2 * col_width)
+ term.center(status, width=col_width - 1)
)
print(
term.move(i, 3 * col_width)
+ term.center(workers, width=col_width - 1)
)
print(
term.move(i, 4 * col_width)
+ term.center(tasks, width=col_width - 1)
)
print(
term.move(i, 5 * col_width)
+ term.center(results, width=col_width - 1)
)
print(
term.move(i, 6 * col_width)
+ term.center(stat.reincarnations, width=col_width - 1)
)
print(
term.move(i, 7 * col_width)
+ term.center(uptime, width=col_width - 1)
)
i += 1
# bottom bar
i += 1
queue_size = broker.queue_size()
lock_size = broker.lock_size()
if lock_size:
queue_size = f"{queue_size}({lock_size})"
print(
term.move(i, 0)
+ term.white_on_cyan(term.center(broker.info(), width=col_width * 2))
)
print(
term.move(i, 2 * col_width)
+ term.black_on_cyan(term.center(_("Queued"), width=col_width))
)
print(
term.move(i, 3 * col_width)
+ term.white_on_cyan(term.center(queue_size, width=col_width))
)
print(
term.move(i, 4 * col_width)
+ term.black_on_cyan(term.center(_("Success"), width=col_width))
)
print(
term.move(i, 5 * col_width)
+ term.white_on_cyan(
term.center(models.Success.objects.count(), width=col_width)
)
)
print(
term.move(i, 6 * col_width)
+ term.black_on_cyan(term.center(_("Failures"), width=col_width))
)
print(
term.move(i, 7 * col_width)
+ term.white_on_cyan(
term.center(models.Failure.objects.count(), width=col_width)
)
)
# for testing
if run_once:
return Stat.get_all(broker=broker)
print(term.move(i + 2, 0) + term.center(_("[Press q to quit]")))
val = term.inkey(timeout=1)
def info(broker=None):
if not broker:
broker = get_broker()
try:
from blessed import Terminal
term = Terminal()
except ImportError:
print(BLESSED_INSTALL_MESSAGE)
return
broker.ping()
stat = Stat.get_all(broker=broker)
# general stats
clusters = len(stat)
workers = 0
reincarnations = 0
for cluster in stat:
workers += len(cluster.workers)
reincarnations += cluster.reincarnations
# calculate tasks pm and avg exec time
tasks_per = 0
per = _("day")
exec_time = 0
last_tasks = models.Success.objects.filter(
stopped__gte=timezone.now() - timedelta(hours=24)
)
tasks_per_day = last_tasks.count()
if tasks_per_day > 0:
# average execution time over the last 24 hours
if connection.vendor != "sqlite":
exec_time = last_tasks.aggregate(
time_taken=Sum(F("stopped") - F("started"))
)
exec_time = exec_time["time_taken"].total_seconds() / tasks_per_day
else:
# can't sum timedeltas on sqlite
for t in last_tasks:
exec_time += t.time_taken()
exec_time = exec_time / tasks_per_day
# tasks per second/minute/hour/day in the last 24 hours
if tasks_per_day > 24 * 60 * 60:
tasks_per = tasks_per_day / (24 * 60 * 60)
per = _("second")
elif tasks_per_day > 24 * 60:
tasks_per = tasks_per_day / (24 * 60)
per = _("minute")
elif tasks_per_day > 24:
tasks_per = tasks_per_day / 24
per = _("hour")
else:
tasks_per = tasks_per_day
# print to terminal
print(term.clear_eos())
col_width = int(term.width / 6)
print(
term.black_on_green(
term.center(
_("-- %(prefix)s %(version)s on %(info)s --")
% {
"prefix": Conf.PREFIX.capitalize(),
"version": ".".join(str(v) for v in VERSION),
"info": broker.info(),
}
)
)
)
print(
term.cyan(_("Clusters"))
+ term.move_x(1 * col_width)
+ term.white(str(clusters))
+ term.move_x(2 * col_width)
+ term.cyan(_("Workers"))
+ term.move_x(3 * col_width)
+ term.white(str(workers))
+ term.move_x(4 * col_width)
+ term.cyan(_("Restarts"))
+ term.move_x(5 * col_width)
+ term.white(str(reincarnations))
)
print(
term.cyan(_("Queued"))
+ term.move_x(1 * col_width)
+ term.white(str(broker.queue_size()))
+ term.move_x(2 * col_width)
+ term.cyan(_("Successes"))
+ term.move_x(3 * col_width)
+ term.white(str(models.Success.objects.count()))
+ term.move_x(4 * col_width)
+ term.cyan(_("Failures"))
+ term.move_x(5 * col_width)
+ term.white(str(models.Failure.objects.count()))
)
print(
term.cyan(_("Schedules"))
+ term.move_x(1 * col_width)
+ term.white(str(models.Schedule.objects.count()))
+ term.move_x(2 * col_width)
+ term.cyan(_("Tasks/%(per)s") % {"per": per})
+ term.move_x(3 * col_width)
+ term.white(f"{tasks_per:.2f}")
+ term.move_x(4 * col_width)
+ term.cyan(_("Avg time"))
+ term.move_x(5 * col_width)
+ term.white(f"{exec_time:.4f}")
)
return True
def memory(run_once=False, workers=False, broker=None):
if not broker:
broker = get_broker()
try:
from blessed import Terminal
term = Terminal()
except ImportError:
print(BLESSED_INSTALL_MESSAGE)
return
broker.ping()
if not psutil:
print(term.clear_eos())
print(
term.white_on_red(
'Cannot start "qmemory" command. Missing "psutil" library.'
)
)
return
with term.fullscreen(), term.hidden_cursor(), term.cbreak():
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = 100.0
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
cols = 8
val = None
start_width = int(term.width / cols)
while val not in ["q", "Q"]:
col_width = int(term.width / cols)
# In case of resize
if col_width != start_width:
print(term.clear())
start_width = col_width
# sentinel, monitor and workers memory usage
print(
term.move(0, 0 * col_width)
+ term.black_on_green(term.center(_("Host"), width=col_width - 1))
)
print(
term.move(0, 1 * col_width)
+ term.black_on_green(term.center(_("Id"), width=col_width - 1))
)
print(
term.move(0, 2 * col_width)
+ term.black_on_green(
term.center(_("Available (%)"), width=col_width - 1)
)
)
print(
term.move(0, 3 * col_width)
+ term.black_on_green(
term.center(_("Available (MB)"), width=col_width - 1)
)
)
print(
term.move(0, 4 * col_width)
+ term.black_on_green(term.center(_("Total (MB)"), width=col_width - 1))
)
print(
term.move(0, 5 * col_width)
+ term.black_on_green(
term.center(_("Sentinel (MB)"), width=col_width - 1)
)
)
print(
term.move(0, 6 * col_width)
+ term.black_on_green(
term.center(_("Monitor (MB)"), width=col_width - 1)
)
)
print(
term.move(0, 7 * col_width)
+ term.black_on_green(
term.center(_("Workers (MB)"), width=col_width - 1)
)
)
row = 2
stats = Stat.get_all(broker=broker)
print(term.clear_eos())
for stat in stats:
# memory available (%)
memory_available_percentage = round(
psutil.virtual_memory().available
* 100
/ psutil.virtual_memory().total,
2,
)
# memory available (MB)
memory_available = round(psutil.virtual_memory().available / 1024**2, 2)
if memory_available_percentage < MEMORY_AVAILABLE_LOWEST_PERCENTAGE:
MEMORY_AVAILABLE_LOWEST_PERCENTAGE = memory_available_percentage
MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT = timezone.now()
print(
term.move(row, 0 * col_width)
+ term.center(stat.host[: col_width - 1], width=col_width - 1)
)
print(
term.move(row, 1 * col_width)
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
)
print(
term.move(row, 2 * col_width)
+ term.center(memory_available_percentage, width=col_width - 1)
)
print(
term.move(row, 3 * col_width)
+ term.center(memory_available, width=col_width - 1)
)
print(
term.move(row, 4 * col_width)
+ term.center(
round(psutil.virtual_memory().total / 1024**2, 2),
width=col_width - 1,
)
)
print(
term.move(row, 5 * col_width)
+ term.center(get_process_mb(stat.sentinel), width=col_width - 1)
)
print(
term.move(row, 6 * col_width)
+ term.center(
get_process_mb(getattr(stat, "monitor", None)),
width=col_width - 1,
)
)
workers_mb = 0
for worker_pid in stat.workers:
result = get_process_mb(worker_pid)
if isinstance(result, str):
result = 0
workers_mb += result
print(
term.move(row, 7 * col_width)
+ term.center(
workers_mb or "NO_PROCESSES_FOUND", width=col_width - 1
)
)
row += 1
# each worker's memory usage
if workers:
row += 2
col_width = int(term.width / (1 + Conf.WORKERS))
print(
term.move(row, 0 * col_width)
+ term.black_on_cyan(term.center(_("Id"), width=col_width - 1))
)
for worker_num in range(Conf.WORKERS):
print(
term.move(row, (worker_num + 1) * col_width)
+ term.black_on_cyan(
term.center(
"Worker #{} (MB)".format(worker_num + 1),
width=col_width - 1,
)
)
)
row += 2
for stat in stats:
print(
term.move(row, 0 * col_width)
+ term.center(str(stat.cluster_id)[-8:], width=col_width - 1)
)
for idx, worker_pid in enumerate(stat.workers):
mb_used = get_process_mb(worker_pid)
print(
term.move(row, (idx + 1) * col_width)
+ term.center(mb_used, width=col_width - 1)
)
row += 1
row += 1
print(
term.move(row, 0)
+ _("Available lowest (): %(memory_percent)s ((at)s)")
% {
"memory_percent": str(MEMORY_AVAILABLE_LOWEST_PERCENTAGE),
"at": MEMORY_AVAILABLE_LOWEST_PERCENTAGE_AT.strftime(
"%Y-%m-%d %H:%M:%S+00:00"
),
}
)
# for testing
if run_once:
return Stat.get_all(broker=broker)
print(term.move(row + 2, 0) + term.center(_("[Press q to quit]")))
val = term.inkey(timeout=1)
def get_ids():
# prints id (PID) of running clusters
stat = Stat.get_all()
if stat:
for s in stat:
print(s.cluster_id)
else:
print(_("No clusters appear to be running."))
return True

71
django_q/pusher.py Normal file
View File

@@ -0,0 +1,71 @@
from multiprocessing import Event
from multiprocessing.process import current_process
from multiprocessing.queues import Queue
from time import sleep
from django import core
from django.apps.registry import apps
from django.utils.translation import gettext_lazy as _
try:
apps.check_apps_ready()
except core.exceptions.AppRegistryNotReady:
import django
django.setup()
from django_q.brokers import Broker, get_broker
from django_q.conf import Conf, logger
from django_q.signing import BadSignature, SignedPackage
try:
import setproctitle
except ModuleNotFoundError:
setproctitle = None
def pusher(task_queue: Queue, event: Event, broker: Broker = None):
"""
Pulls tasks of the broker and puts them in the task queue
:type broker:
:type task_queue: multiprocessing.Queue
:type event: multiprocessing.Event
"""
if not broker:
broker = get_broker()
proc_name = current_process().name
if setproctitle:
setproctitle.setproctitle(f"qcluster {proc_name} pusher")
logger.info(
_("%(name)s pushing tasks at %(id)s")
% {"name": proc_name, "id": current_process().pid}
)
while True:
try:
task_set = broker.dequeue()
except Exception:
logger.exception("Failed to pull task from broker")
# broker probably crashed. Let the sentinel handle it.
sleep(10)
break
if task_set:
for task in task_set:
ack_id = task[0]
# unpack the task
try:
task = SignedPackage.loads(task[1])
except (TypeError, BadSignature):
logger.exception("Failed to push task to queue")
broker.fail(ack_id)
continue
task["cluster"] = (
Conf.CLUSTER_NAME
) # save actual cluster name to orm task table
task["ack_id"] = ack_id
task_queue.put(task)
logger.debug(
_("queueing from %(list_key)s") % {"list_key": broker.list_key}
)
if event.is_set():
break
logger.info(_("%(name)s stopped pushing tasks") % {"name": current_process().name})

View File

@@ -2,6 +2,7 @@
The code is derived from
https://github.com/althonos/pronto/commit/3384010dfb4fc7c66a219f59276adef3288a886b
"""
import multiprocessing
import multiprocessing.queues
import sys

140
django_q/scheduler.py Normal file
View File

@@ -0,0 +1,140 @@
import ast
from multiprocessing.process import current_process
from django import core, db
from django.apps.registry import apps
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
try:
apps.check_apps_ready()
except core.exceptions.AppRegistryNotReady:
import django
django.setup()
from django_q.brokers import Broker, get_broker
from django_q.conf import Conf, logger
from django_q.humanhash import humanize
from django_q.models import Schedule
from django_q.tasks import async_task
from django_q.utils import close_old_django_connections, localtime
def scheduler(broker: Broker = None):
"""
Creates a task from a schedule at the scheduled time and schedules next run
"""
if not broker:
broker = get_broker()
close_old_django_connections()
try:
# Only default cluster will handler schedule with default(null) cluster
Q_default = (
db.models.Q(cluster__isnull=True)
if Conf.CLUSTER_NAME == Conf.PREFIX
else db.models.Q(pk__in=[])
)
with db.transaction.atomic(using=db.router.db_for_write(Schedule)):
for s in (
Schedule.objects.select_for_update()
.exclude(repeats=0)
.filter(next_run__lt=timezone.now())
.filter(Q_default | db.models.Q(cluster=Conf.CLUSTER_NAME))
):
args = ()
kwargs = {}
# get args, kwargs and hook
if s.kwargs:
try:
# first try the dict syntax
kwargs = ast.literal_eval(s.kwargs)
except (SyntaxError, ValueError):
# else use the kwargs syntax
try:
parsed_kwargs = (
ast.parse(f"f({s.kwargs})").body[0].value.keywords
)
kwargs = {
kwarg.arg: ast.literal_eval(kwarg.value)
for kwarg in parsed_kwargs
}
except (SyntaxError, ValueError):
kwargs = {}
if s.args:
args = ast.literal_eval(s.args)
# single value won't eval to tuple, so:
if type(args) != tuple:
args = (args,)
q_options = kwargs.get("q_options", {})
if s.intended_date_kwarg:
kwargs[s.intended_date_kwarg] = s.next_run.isoformat()
if s.hook:
q_options["hook"] = s.hook
# set up the next run time
if s.schedule_type != s.ONCE:
next_run = s.next_run
while True:
next_run = s.calculate_next_run(next_run)
if Conf.CATCH_UP or next_run > localtime():
break
s.next_run = next_run
# Little Fix for already broken numbers
if s.repeats < -1:
s.repeats = -1
# Check if the value is not zero
if s.repeats > 0:
s.repeats -= 1
# send it to the cluster; any cluster name is allowed in multi-queue scenarios
# because `broker_name` is confusing, using `cluster` name is recommended and takes precedence
q_options["cluster"] = s.cluster or q_options.get(
"cluster", q_options.pop("broker_name", None)
)
if (
q_options["cluster"] is None
or q_options["cluster"] == Conf.CLUSTER_NAME
):
q_options["broker"] = broker
q_options["group"] = q_options.get("group", s.name or s.id)
kwargs["q_options"] = q_options
s.task = async_task(s.func, *args, **kwargs)
# log it
if not s.task:
logger.error(
_(
"%(process_name)s failed to create a task from schedule "
"[%(schedule)s]"
)
% {
"process_name": current_process().name,
"schedule": s.name or s.id,
}
)
else:
logger.info(
_(
"%(process_name)s created task %(task_name)s from schedule "
"[%(schedule)s]"
)
% {
"process_name": current_process().name,
"task_name": humanize(s.task),
"schedule": s.name or s.id,
}
)
# default behavior is to delete a ONCE schedule
if s.schedule_type == s.ONCE:
if s.repeats < 0:
s.delete()
continue
# but not if it has a positive repeats
s.repeats = 0
# save the schedule
s.save()
except Exception:
logger.exception("Could not create task from schedule")

View File

@@ -31,6 +31,7 @@ def call_hook(sender, instance, **kwargs):
% {"hook": instance.hook, "name": instance.name, "error": str(e)}
)
# args: proc_name
post_spawn = Signal()

View File

@@ -1,4 +1,5 @@
"""Package signing."""
import pickle
from django_q import core_signing as signing

View File

@@ -1,4 +1,5 @@
"""Provides task functionality."""
# Standard
from multiprocessing import Value
from time import sleep, time
@@ -148,7 +149,7 @@ def result(task_id, wait=0, cached=Conf.CACHED):
start = time()
while True:
r = Task.get_result(task_id)
if r:
if r is not None:
return r
if (time() - start) * 1000 >= wait >= 0:
break
@@ -569,10 +570,12 @@ class Chain:
A sequential chain of tasks
"""
def __init__(self, chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC):
def __init__(
self, chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC, broker=None
):
self.chain = chain or []
self.group = group or ""
self.broker = get_broker()
self.broker = broker or get_broker()
self.cached = cached
self.sync = sync
self.started = False
@@ -729,17 +732,14 @@ class AsyncTask:
return self.id
def result(self, wait=0):
if self.started:
return result(self.id, wait=wait, cached=self.cached)
def fetch(self, wait=0):
if self.started:
return fetch(self.id, wait=wait, cached=self.cached)
def result_group(self, failures=False, wait=0, count=None):
if self.started and self.group:
return result_group(
self.group,
@@ -750,7 +750,6 @@ class AsyncTask:
)
def fetch_group(self, failures=True, wait=0, count=None):
if self.started and self.group:
return fetch_group(
self.group,
@@ -763,7 +762,8 @@ class AsyncTask:
def _sync(pack):
"""Simulate a package travelling through the cluster."""
from django_q.cluster import monitor, worker
from django_q.monitor import monitor
from django_q.worker import worker
task_queue = Queue()
result_queue = Queue()

View File

@@ -46,6 +46,10 @@ def hello():
return "hello"
def return_falsy_value():
return []
def result(obj):
print(f"RESULT HOOK {obj.name} : {obj.result()}")

View File

@@ -3,8 +3,9 @@ from multiprocessing import Event, Value
import pytest
from django_q.brokers import get_broker
from django_q.cluster import monitor, pusher, worker
from django_q.conf import Conf
from django_q.monitor import monitor
from django_q.pusher import pusher
from django_q.queues import Queue
from django_q.tasks import (
AsyncTask,
@@ -21,6 +22,7 @@ from django_q.tasks import (
result,
result_group,
)
from django_q.worker import worker
@pytest.fixture

View File

@@ -12,10 +12,12 @@ import pytest
from django.utils import timezone
from django_q.brokers import Broker, get_broker
from django_q.cluster import Cluster, Sentinel, monitor, pusher, save_task, worker
from django_q.cluster import Cluster, Sentinel
from django_q.conf import Conf
from django_q.humanhash import DEFAULT_WORDLIST, uuid
from django_q.models import Success, Task
from django_q.monitor import monitor, save_task
from django_q.pusher import pusher
from django_q.queues import Queue
from django_q.signals import post_execute, pre_enqueue, pre_execute
from django_q.status import Stat
@@ -29,8 +31,9 @@ from django_q.tasks import (
result,
result_group,
)
from django_q.tests.tasks import multiply, TaskError
from django_q.tests.tasks import TaskError, multiply
from django_q.utils import add_months, add_years
from django_q.worker import worker
myPath = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, myPath + "/../")
@@ -138,6 +141,30 @@ def test_cluster(broker):
broker.delete_queue()
@pytest.mark.django_db
def test_results(broker):
broker.list_key = "cluster_test:q"
broker.delete_queue()
a = async_task(
"django_q.tests.tasks.return_falsy_value",
broker=broker,
)
task_queue = Queue()
stop_event = Event()
stop_event.set()
pusher(task_queue, stop_event, broker=broker)
task_queue.put("STOP")
result_queue = Queue()
worker(task_queue, result_queue, Value("f", -1))
result_queue.put("STOP")
monitor(result_queue)
# should not loop indefinitely when a real value is returned
value = result(a, wait=-1)
assert value == []
@pytest.mark.django_db
def test_enqueue(broker, admin_user):
broker.list_key = "cluster_test:q"

View File

@@ -5,7 +5,7 @@ import pytest
from django_q.brokers import get_broker
from django_q.cluster import Cluster
from django_q.conf import Conf
from django_q.monitor import get_ids, info, monitor
from django_q.monitor_terminal import get_ids, info, monitor
from django_q.status import Stat
from django_q.tasks import async_task

View File

@@ -3,8 +3,8 @@ from datetime import datetime, timedelta
from multiprocessing import Event, Value
from unittest import mock
import pytest
import django
import pytest
from django.core.exceptions import ValidationError
from django.db import IntegrityError
from django.test import override_settings
@@ -12,9 +12,11 @@ from django.utils import timezone
from django.utils.timezone import is_naive
from django_q.brokers import Broker, get_broker
from django_q.cluster import localtime, monitor, pusher, scheduler, worker
from django_q.conf import Conf
from django_q.monitor import monitor
from django_q.pusher import pusher
from django_q.queues import Queue
from django_q.scheduler import scheduler
from django_q.tasks import Schedule, fetch
from django_q.tasks import schedule as create_schedule
from django_q.tests.settings import BASE_DIR
@@ -22,7 +24,8 @@ from django_q.tests.testing_utilities.multiple_database_routers import (
TestingMultipleAppsDatabaseRouter,
TestingReplicaDatabaseRouter,
)
from django_q.utils import add_months
from django_q.utils import add_months, localtime
from django_q.worker import worker
if django.VERSION < (4, 0):
# pytz is the default in django 3.2. Remove when no support for 3.2
@@ -85,7 +88,7 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
# 28th of March 2021 is the day when sunlight saving starts (at 2 am)
monkeypatch.setattr(Conf, "TIME_ZONE", "Europe/Amsterdam")
tz = ZoneInfo('Europe/Amsterdam')
tz = ZoneInfo("Europe/Amsterdam")
broker.list_key = "scheduler_test:q"
# Let's start a schedule at 1 am on the 27th of March. This is in AMS timezone.
# So, 2021-03-27 00:00:00 when saved (due to TZ being Amsterdam and saved in UTC)
@@ -181,7 +184,6 @@ def test_scheduler_daylight_saving_time_daily(broker, monkeypatch):
assert str(next_run) == "2021-11-01 01:00:00+01:00"
@pytest.mark.django_db
def test_scheduler(broker, monkeypatch):
broker.list_key = "scheduler_test:q"
@@ -407,7 +409,7 @@ def test_scheduler(broker, monkeypatch):
def test_intended_schedule_kwarg(broker, monkeypatch):
broker.list_key = "scheduler_test:q"
broker.delete_queue()
run_date = timezone.now()-timedelta(hours=1)
run_date = timezone.now() - timedelta(hours=1)
schedule = create_schedule(
"math.copysign",
1,
@@ -417,10 +419,10 @@ def test_intended_schedule_kwarg(broker, monkeypatch):
schedule_type=Schedule.HOURLY,
repeats=1,
next_run=run_date,
intended_date_kwarg='intended_date',
intended_date_kwarg="intended_date",
)
assert schedule.last_run() is None
assert schedule.intended_date_kwarg == 'intended_date'
assert schedule.intended_date_kwarg == "intended_date"
# run scheduler
scheduler(broker=broker)
# set up the workflow
@@ -431,8 +433,8 @@ def test_intended_schedule_kwarg(broker, monkeypatch):
pusher(task_queue, stop_event, broker=broker)
assert task_queue.qsize() == 1
task = task_queue.get()
assert 'intended_date' in task['kwargs']
assert task['kwargs']['intended_date'] == run_date.isoformat()
assert "intended_date" in task["kwargs"]
assert task["kwargs"]["intended_date"] == run_date.isoformat()
@override_settings(

38
django_q/timeout.py Normal file
View File

@@ -0,0 +1,38 @@
import signal
from django.utils.translation import gettext_lazy as _
from django_q.conf import logger
from .exceptions import TimeoutException
class TimeoutHandler:
def __init__(self, timeout: int):
self._timeout = timeout
def raise_timeout_exception(self, signum, frame):
raise TimeoutException(
f"Task exceeded maximum timeout value ({self._timeout} seconds)"
)
def __enter__(self):
# if the timeout is -1, then there is no timeout and the task will always keep running until it's done or manually killed
if self._timeout == -1:
return
try:
signal.signal(signal.SIGALRM, self.raise_timeout_exception)
except ValueError: # ValueError is raised for Windows users
logger.debug(_("SIGALARM is not available on your platform"))
signal.alarm(self._timeout)
def __exit__(self, exc_type, exc_value, traceback):
if self._timeout == -1:
return
"""When getting out of the timeout, reset the alarm, so it won't trigger"""
try:
signal.alarm(0)
signal.signal(signal.SIGALRM, signal.SIG_DFL)
except ValueError: # ValueError is raised for Windows users
logger.debug(_("SIGALARM is not available on your platform"))

View File

@@ -1,13 +1,13 @@
from datetime import datetime
import calendar
import inspect
from datetime import date
from datetime import date, datetime
import django
from django.utils import timezone
from django import db
from django.conf import settings
from django.utils import timezone
from django_q.conf import Conf
from django_q.conf import Conf, logger
if django.VERSION < (4, 0):
# pytz is the default in django 3.2. Remove when no support for 3.2
@@ -60,7 +60,7 @@ def get_func_repr(func):
def localtime(value=None) -> datetime:
"""Override for timezone.localtime to deal with naive times and local times"""
if settings.USE_TZ:
if django.VERSION >= (4, 0) and settings.USE_DEPRECATED_PYTZ:
if django.VERSION >= (4, 0) and getattr(settings, "USE_DEPRECATED_PYTZ", False):
import pytz
convert_to_tz = pytz.timezone(Conf.TIME_ZONE)
@@ -72,3 +72,17 @@ def localtime(value=None) -> datetime:
return datetime.now()
else:
return value
def close_old_django_connections():
"""
Close django connections unless running with sync=True.
"""
if Conf.SYNC:
logger.warning(
"Preserving django database connections because sync=True. Beware "
"that tasks are now injected in the calling context/transactions "
"which may result in unexpected behaviour."
)
else:
db.close_old_connections()

141
django_q/worker.py Normal file
View File

@@ -0,0 +1,141 @@
import pydoc
import traceback
from multiprocessing import Value
from multiprocessing.process import current_process
from multiprocessing.queues import Queue
from django import core
from django.apps.registry import apps
from django.utils import timezone
from django.utils.translation import gettext_lazy as _
try:
apps.check_apps_ready()
except core.exceptions.AppRegistryNotReady:
import django
django.setup()
from django_q.conf import Conf, error_reporter, logger, resource, setproctitle
from django_q.exceptions import TimeoutException
from django_q.signals import post_spawn, pre_execute
from django_q.timeout import TimeoutHandler
from django_q.utils import close_old_django_connections, get_func_repr
try:
import psutil
except ImportError:
psutil = None
try:
import setproctitle
except ModuleNotFoundError:
setproctitle = None
def worker(
task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
):
"""
Takes a task from the task queue, tries to execute it and puts the result back in
the result queue
:param timeout: number of seconds wait for a worker to finish.
:type task_queue: multiprocessing.Queue
:type result_queue: multiprocessing.Queue
:type timer: multiprocessing.Value
"""
proc_name = current_process().name
logger.info(
_("%(proc_name)s ready for work at %(id)s")
% {"proc_name": proc_name, "id": current_process().pid}
)
post_spawn.send(sender="django_q", proc_name=proc_name)
if setproctitle:
setproctitle.setproctitle(f"qcluster {proc_name} idle")
task_count = 0
if timeout is None:
timeout = -1
# Start reading the task queue
for task in iter(task_queue.get, "STOP"):
result = None
timer.value = -1 # Idle
task_count += 1
f = task["func"]
# Log task creation and set process name
# Get the function from the task
func_name = get_func_repr(f)
task_name = task["name"]
task_desc = _("%(proc_name)s processing %(task_name)s '%(func_name)s'") % {
"proc_name": proc_name,
"func_name": func_name,
"task_name": task_name,
}
if "group" in task:
task_desc += f" [{task['group']}]"
logger.info(task_desc)
if setproctitle:
proc_title = f"qcluster {proc_name} processing {task_name} '{func_name}'"
if "group" in task:
proc_title += f" [{task['group']}]"
setproctitle.setproctitle(proc_title)
# if it's not an instance try to get it from the string
if not callable(f):
# locate() returns None if f cannot be loaded
f = pydoc.locate(f)
close_old_django_connections()
timer_value = task.pop("timeout", timeout)
# signal execution
pre_execute.send(sender="django_q", func=f, task=task)
# execute the payload
timer.value = timer_value # Busy
if timer.value != -1:
timer.value += 3 # Add buffer so that guard doesn't kill the process on timeout before it gets processed
timeout_error = False
try:
if f is None:
# raise a meaningfull error if task["func"] is not a valid function
raise ValueError(f"Function {task['func']} is not defined")
with TimeoutHandler(timer_value):
res = f(*task["args"], **task["kwargs"])
result = (res, True)
except (Exception, TimeoutException) as e:
if isinstance(e, TimeoutException):
timeout_error = True
result = (f"{e} : {traceback.format_exc()}", False)
if error_reporter:
error_reporter.report()
if task.get("sync", False):
raise
with timer.get_lock():
# Process result
task["result"] = result[0]
task["success"] = result[1]
task["stopped"] = timezone.now()
result_queue.put(task)
if timeout_error:
# force destroy process due to timeout
timer.value = 0
break
timer.value = -1 # Idle
if setproctitle:
setproctitle.setproctitle(f"qcluster {proc_name} idle")
# Recycle
if task_count == Conf.RECYCLE or rss_check():
timer.value = -2 # Recycled
break
logger.info(_("%(proc_name)s stopped doing work") % {"proc_name": proc_name})
def rss_check():
if Conf.MAX_RSS:
if resource:
return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
elif psutil:
return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
return False

View File

@@ -49,8 +49,9 @@ Reference
:param str group: an optional group name.
:param bool cached: run this against the cache backend
:param bool sync: execute this inline instead of asynchronous
:param broker: an optional broker instance
.. py:class:: Chain(chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC)
.. py:class:: Chain(chain=None, group=None, cached=Conf.CACHED, sync=Conf.SYNC, broker=None)
A sequential chain of tasks. Acts as a convenient wrapper for :func:`async_chain`
You can pass the task chain at construction or you can append individual tasks before running them.
@@ -59,6 +60,8 @@ Reference
:param str group: an optional group name.
:param bool cached: run this against the cache backend
:param bool sync: execute this inline instead of asynchronous
:param bool sync: execute this inline instead of asynchronous
:param broker: an optional broker instance
.. py:method:: append(func, *args, **kwargs)

View File

@@ -73,9 +73,9 @@ author = "Ilan Steemers, Stan Triepels"
# built documents.
#
# The short X.Y version.
version = "1.5"
version = "1.7"
# The full version, including alpha/beta/rc tags.
release = "1.5.4"
release = "1.7.2"
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.

View File

@@ -79,6 +79,8 @@ timeout
The number of seconds a worker is allowed to spend on a task before it's terminated. Defaults to ``None``, meaning it will never time out.
Set this to something that makes sense for your project. Can be overridden for individual tasks.
Note: for systems that don't have `SIGALRM` available (e.g. Windows), it will not raise an error properly. It will kill the task, but it will keep retrying until it finishes within the given time.
See :ref:`retry` for details how to set values for timeout and retry.
.. _time_zone:
@@ -348,7 +350,7 @@ To use MongoDB as a message broker you simply provide the connection information
}
}
The ``mongo`` dictionary can contain any of the parameters exposed by pymongo's `MongoClient <https://api.mongodb.org/python/current/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`__
The ``mongo`` dictionary can contain any of the parameters exposed by pymongo's `MongoClient <https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`__
If you want to use a mongodb uri, you can supply it as the ``host`` parameter.
mongo_db

View File

@@ -1,4 +1,3 @@
version: "3.0"
services:
docs:
container_name: djangoq2-docs

View File

@@ -328,7 +328,7 @@ Requires cache to be enabled. Save file in your Django project's root directory
# All django stuff has to come after the setup:
django.setup()
from django_q.monitor import Stat
from django_q.status import Stat
from django_q.conf import Conf
# Set host and port settings

View File

@@ -27,7 +27,7 @@ Features
- Rollbar and Sentry support
Django Q2 is tested with: Python 3.8, 3.9, 3.10 and 3.11. Works with Django 3.2.x, 4.1.x and 4.2.x
Django Q2 is tested with: Python 3.8, 3.9, 3.10, 3.11 and 3.12. Works with Django 4.2.x and 5.0.x
Currently available in English, German and French.

View File

@@ -24,15 +24,31 @@ Installation
$ python manage.py qcluster
Migrate from Django-Q to Django-Q2
----------------------------------
If you have an application with django-q running right now, you can simply swap the libraries and you should be good to go.::
$ pip uninstall django-q # you might have to uninstall django-q add-ons as well
$ pip install django-q2
Then migrate the database to get the latest tables/fields::
$ python manage.py migrate
Requirements
------------
Django Q2 is tested for Python 3.8, 3.9, 3.10 and 3.11
Django Q2 is tested for Python 3.8, 3.9, 3.10, 3.11 and 3.12
- `Django <https://www.djangoproject.com>`__
Django Q2 aims to use as much of Django's standard offerings as possible.
The code is tested against Django versions `3.2.x`, `4.1.x` and`4.2.x`.
The code is tested against Django versions `3.2.x`, `4.1.x`, `4.2.x` and `5.0.x`.
- `Django-picklefield <https://github.com/gintas/django-picklefield>`__
@@ -99,11 +115,11 @@ Add-ons
-------
- `django-q-rollbar <https://github.com/danielwelch/django-q-rollbar>`__ is a Rollbar error reporter::
$ pip install django-q[rollbar]
$ pip install django-q2[rollbar]
- `django-q-sentry <https://github.com/danielwelch/django-q-sentry>`__ is a Sentry error reporter::
$ pip install django-q[sentry]
$ pip install django-q2[sentry]
- `django-q-email <https://github.com/joeyespo/django-q-email>`__ is a compatible Django email backend that will automatically async queue your emails.
@@ -129,7 +145,7 @@ Other known issues are:
Python
~~~~~~
Current tests are performed with 3.8, 3.9, 3.10 and 3.11
Current tests are performed with 3.8, 3.9, 3.10, 3.11 and 3.12
If you do encounter any regressions with earlier versions, please submit an issue on `github <https://github.com/GDay/django-q2>`__
Open-source packages
@@ -139,8 +155,8 @@ You can reference the `requirements <https://github.com/GDay/django-q2/blob/mast
Django
~~~~~~
We strive to be compatible with last two major version of Django.
At the moment this means we support the 3.2.x, 4.1.x and 4.2.x releases.
We strive to be compatible with the last two major version of Django.
At the moment this means we support the 3.2.x, 4.1.x, 4.2.x and 5.0.x releases.
Since we are now no longer supporting Python 2, we can also not support older versions of Django that do not support Python >= 3.6
Since we are now no longer supporting Python 2, we can also not support older versions of Django that do not support Python >= 3.8
For this you can always use older releases, but they are no longer maintained.

View File

@@ -111,7 +111,7 @@ You can check the status of your clusters straight from your code with the :clas
.. code:: python
from django_q.monitor import Stat
from django_q.status import Stat
for stat in Stat.get_all():
print(stat.cluster_id, stat.status)

View File

@@ -75,6 +75,10 @@ broker
""""""
A broker instance, in case you want to control your own connections.
cluster
""""""
The name of the cluster. Only useful if you are using [alternative queues](https://django-q2.readthedocs.io/en/master/cluster.html#multiple-queues).
task_name
"""""""""
@@ -121,6 +125,7 @@ Optionally you can use the :class:`AsyncTask` class to instantiate a task and ke
a.run()
# wait indefinitely for the result and print it
# don't let the task return `None` or it will wait indefinitely
print(a.result(wait=-1))
# change the args
@@ -240,7 +245,7 @@ Reference
---------
.. py:function:: async_task(func, *args, hook=None, group=None, timeout=None,\
save=None, sync=False, cached=False, broker=None, q_options=None, **kwargs)
save=None, sync=False, cached=False, broker=None, cluster=None, q_options=None, **kwargs)
Puts a task in the cluster queue
@@ -254,6 +259,7 @@ Reference
:param bool sync: If set to True, async_task will simulate a task execution
:param cached: Output the result to the cache backend. Bool or timeout in seconds
:param broker: Optional broker connection from :func:`brokers.get_broker`
:param cluster: Optional cluster name if using alternative queues
:param dict q_options: Options dict, overrides option keywords
:param dict kwargs: Keyword arguments for the task function
:returns: The uuid of the task
@@ -264,7 +270,7 @@ Reference
Gets the result of a previously executed task
:param str task_id: the uuid or name of the task
:param int wait: optional milliseconds to wait for a result. -1 for indefinite
:param int wait: optional milliseconds to wait for a result. -1 for indefinite, but be sure the result will not be `None` otherwise it will wait indefinitely!
:param bool cached: run this against the cache backend.
:returns: The result of the executed task

View File

16
exampleproject/asgi.py Normal file
View File

@@ -0,0 +1,16 @@
"""
ASGI config for exampleproject project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/5.1/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "exampleproject.settings")
application = get_asgi_application()

134
exampleproject/settings.py Normal file
View File

@@ -0,0 +1,134 @@
"""
Django settings for exampleproject project.
Generated by 'django-admin startproject' using Django 5.1.1.
For more information on this file, see
https://docs.djangoproject.com/en/5.1/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/5.1/ref/settings/
"""
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/5.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "django-insecure-)oouh93bjg+c=b!l5-*w)7et1l+!3nmp223rr^1r4v#jn&ow3f"
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = ["*"]
# Application definition
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"django_q",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = "exampleproject.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
WSGI_APPLICATION = "exampleproject.wsgi.application"
# Database
# https://docs.djangoproject.com/en/5.1/ref/settings/#databases
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": BASE_DIR / "db.sqlite3",
}
}
# Password validation
# https://docs.djangoproject.com/en/5.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
# Internationalization
# https://docs.djangoproject.com/en/5.1/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/5.1/howto/static-files/
STATIC_URL = "static/"
# Default primary key field type
# https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
Q_CLUSTER = {
"name": "DjangORM",
"workers": 4,
"timeout": 90,
"retry": 120,
"queue_limit": 50,
"bulk": 10,
"orm": "default",
}

27
exampleproject/urls.py Normal file
View File

@@ -0,0 +1,27 @@
"""
URL configuration for exampleproject project.
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/5.1/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
from exampleproject import views
urlpatterns = [
path("admin/", admin.site.urls),
path("new_task/", views.add_task, name="add_task"),
path("result/<slug:task_id>/", views.get_result, name="get_result"),
]

31
exampleproject/views.py Normal file
View File

@@ -0,0 +1,31 @@
import time
from django.http import HttpResponse
from django.urls import reverse
from django_q import tasks
# internal function to be called with django_q
def new_task(run_for_minutes):
print("Task started")
time.sleep(run_for_minutes)
print("Task done")
return True
def add_task(request):
task_id = tasks.async_task(new_task, 5)
result_url = reverse("get_result", args=[task_id])
return HttpResponse(
f"Added async task with <a href='{result_url}'>Go to results</a>"
)
def get_result(request, task_id):
task = tasks.fetch(task_id)
if not task:
msg = "Task running... please refresh after some time"
else:
msg = f"Async task result: {task.result}"
return HttpResponse(msg)

16
exampleproject/wsgi.py Normal file
View File

@@ -0,0 +1,16 @@
"""
WSGI config for exampleproject project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/5.1/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "exampleproject.settings")
application = get_wsgi_application()

1591
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,10 @@
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.poetry]
name = "django-q2"
version = "1.5.4"
version = "1.7.2"
packages = [
{ include = "django_q" },
]
@@ -30,6 +34,7 @@ classifiers = [
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Programming Language :: Python :: 3.12',
'Topic :: Internet :: WWW/HTTP',
'Topic :: System :: Distributed Computing',
'Topic :: Software Development :: Libraries :: Python Modules',
@@ -44,8 +49,8 @@ include = ['CHANGELOG.md']
[tool.poetry.dependencies]
python = ">=3.8, <4"
django = ">=3.2"
python = ">=3.8,<4"
django = ">=4.2, <6"
django-picklefield = "^3.1"
blessed = { version = "^1.19.1", optional = true }
@@ -55,11 +60,12 @@ django-redis = { version = "^5.2.0", optional = true }
iron-mq = { version = "^0.9", optional = true }
boto3 = { version = "^1.24.92", optional = true }
pymongo = { version = "^4.2.0", optional = true }
croniter = { version = "^1.3.7", optional = true }
croniter = { version = "^2.0.1", optional = true }
django-q-rollbar = {version = ">=0.1", optional = true}
django-q-sentry = {version = ">=0.1", optional = true}
redis = {version = "^4.3.4", optional = true}
setproctitle = {version = "^1.3.2", optional = true}
importlib-metadata = {version = ">=3.6", python = "<3.10"}
[tool.poetry.dev-dependencies]
@@ -67,8 +73,7 @@ pytest = "^7.1.3"
pytest-django = "^4.5.2"
Sphinx = "^4.0.2"
pytest-cov = "^4.0.0"
black = "^22.10.0"
isort = {extras = ["requirements_deprecated_finder"], version = "^5.10.1"}
ruff = "^0.4.4"
[tool.poetry.extras]
requires = ["poetry_core"]

View File

@@ -1,40 +1,41 @@
ansicon==1.89.0; platform_system == "Windows" and python_version >= "2.7" \
--hash=sha256:f1def52d17f65c2c9682cf8370c03f541f410c1752d6a14029f97318e4b9dfec \
--hash=sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1
arrow==1.1.1; python_version >= "3.6" \
--hash=sha256:77a60a4db5766d900a2085ce9074c5c7b8e2c99afeaa98ad627637ff6f292510 \
--hash=sha256:dee7602f6c60e3ec510095b5e301441bc56288cb8f51def14dcb3079f623823a
asgiref==3.3.4; python_version >= "3.6" \
--hash=sha256:92906c611ce6c967347bbfea733f13d6313901d54dcca88195eaeb52b2a8e8ee \
--hash=sha256:d1216dfbdfb63826470995d31caed36225dcaf34f182e0fa257a4dd9e86f1b78
blessed==1.18.1; python_version >= "2.7" \
--hash=sha256:dd7c0d33db9a2e7f597b446996484d0ed46e1586239db064fb5025008937dcae \
--hash=sha256:8b09936def6bc06583db99b65636b980075733e13550cb6af262ce724a55da23
django-picklefield==3.0.1; python_version >= "3" \
--hash=sha256:15ccba592ca953b9edf9532e64640329cd47b136b7f8f10f2939caa5f9ce4287 \
--hash=sha256:3c702a54fde2d322fe5b2f39b8f78d9f655b8f77944ab26f703be6c0ed335a35
django==3.2.4; python_version >= "3.6" \
--hash=sha256:ea735cbbbb3b2fba6d4da4784a0043d84c67c92f1fdf15ad6db69900e792c10f \
--hash=sha256:66c9d8db8cc6fe938a28b7887c1596e42d522e27618562517cc8929eb7e7f296
jinxed==1.1.0; platform_system == "Windows" and python_version >= "2.7" \
--hash=sha256:6a61ccf963c16aa885304f27e6e5693783676897cea0c7f223270c8b8e78baf8 \
--hash=sha256:d8f1731f134e9e6b04d95095845ae6c10eb15cb223a5f0cabdea87d4a279c305
python-dateutil==2.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
--hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
--hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a
pytz==2021.1; python_version >= "3.6" \
--hash=sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798 \
--hash=sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da
six==1.16.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" \
--hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
--hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926
sqlparse==0.4.1; python_version >= "3.6" \
--hash=sha256:017cde379adbd6a1f15a61873f43e8274179378e95ef3fede90b5aa64d304ed0 \
--hash=sha256:0f91fd2e829c44362cbcfab3e9ae12e22badaa8a29ad5ff599f9ec109f0454e8
typing-extensions==3.10.0.0; python_version < "3.8" and python_version >= "3.6" \
--hash=sha256:0ac0f89795dd19de6b97debb0c6af1c70987fd80a2d62d1958f7e56fcc31b497 \
--hash=sha256:779383f6086d90c99ae41cf0ff39aac8a7937a9283ce0a414e5dd782f4c94a84 \
--hash=sha256:50b6f157849174217d0656f99dc82fe932884fb250826c18350e159ec6cdf342
wcwidth==0.2.5; python_version >= "2.7" \
--hash=sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784 \
--hash=sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83
asgiref==3.8.1 ; python_version >= "3.8" and python_version < "4" \
--hash=sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47 \
--hash=sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590
backports-zoneinfo==0.2.1 ; python_version >= "3.8" and python_version < "3.9" \
--hash=sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf \
--hash=sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328 \
--hash=sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546 \
--hash=sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6 \
--hash=sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570 \
--hash=sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9 \
--hash=sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7 \
--hash=sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987 \
--hash=sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722 \
--hash=sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582 \
--hash=sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc \
--hash=sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b \
--hash=sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1 \
--hash=sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08 \
--hash=sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac \
--hash=sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2
django-picklefield==3.2 ; python_version >= "3.8" and python_version < "4" \
--hash=sha256:aa463f5d79d497dbe789f14b45180f00a51d0d670067d0729f352a3941cdfa4d \
--hash=sha256:e9a73539d110f69825d9320db18bcb82e5189ff48dbed41821c026a20497764c
django==4.2.11 ; python_version >= "3.8" and python_version < "4" \
--hash=sha256:6e6ff3db2d8dd0c986b4eec8554c8e4f919b5c1ff62a5b4390c17aff2ed6e5c4 \
--hash=sha256:ddc24a0a8280a0430baa37aff11f28574720af05888c62b7cfe71d219f4599d3
importlib-metadata==7.1.0 ; python_version >= "3.8" and python_version < "3.10" \
--hash=sha256:30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570 \
--hash=sha256:b78938b926ee8d5f020fc4772d487045805a55ddbad2ecf21c6d60938dc7fcd2
sqlparse==0.5.0 ; python_version >= "3.8" and python_version < "4" \
--hash=sha256:714d0a4932c059d16189f58ef5411ec2287a4360f17cdd0edd2d09d4c5087c93 \
--hash=sha256:c204494cd97479d0e39f28c93d46c0b2d5959c7b9ab904762ea6c7af211c8663
typing-extensions==4.11.0 ; python_version >= "3.8" and python_version < "3.11" \
--hash=sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0 \
--hash=sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a
tzdata==2024.1 ; python_version >= "3.8" and python_version < "4" and sys_platform == "win32" \
--hash=sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd \
--hash=sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252
zipp==3.18.1 ; python_version >= "3.8" and python_version < "3.10" \
--hash=sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b \
--hash=sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715

1
ruff.toml Normal file
View File

@@ -0,0 +1 @@
lint.extend-select = ["I"]

View File

@@ -1,5 +1,3 @@
version: '3'
services:
redis:
image: redis:latest

View File

@@ -1,11 +1,16 @@
version: '3'
services:
web:
restart: always
command: python manage.py runserver 0.0.0.0:8000
command: bash -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000"
ports:
- "127.0.0.1:8000:8000"
build: .
volumes:
- .:/app
django-q:
restart: always
command: bash -c "python manage.py migrate && python manage.py qcluster"
build: .
volumes:
- .:/app