fork/django-q2
1
0
Fork 0
mirror of https://github.com/django-q2/django-q2.git synced 2026-09-15 13:37:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #452 from Koed00/cron

Browse Source 
Support for Cron expressions
This commit is contained in:
Ilan Steemers
2020-07-01 14:38:30 +02:00
committed by GitHub
parent 29b7e27573 f223cfcb30
commit bcd119ca2f
24 changed files with 355 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -66,4 +66,5 @@ db.sqlite3
.venv
.idea
djq
node_modules
node_modules
/c.cache/

6
.travis.yml
View File

@@ -1,7 +1,7 @@
language: python
services:
- redis-server
- redis
- mongodb
python:
@@ -9,8 +9,8 @@ python:
- "3.8"
env:
- DJANGO=3.0.7
- DJANGO=2.2.13
- DJANGO=3.0.8
- DJANGO=2.2.14
sudo: true
dist: xenial

8
README.rst
View File

@@ -13,7 +13,7 @@ Features
- Multiprocessing worker pool
- Asynchronous tasks
- Scheduled and repeated tasks
- Scheduled, cron and repeated tasks
- Signed and compressed packages
- Failure and success database or cache
- Result hooks, groups and chains
@@ -180,6 +180,12 @@ Admin page or directly from your code:
repeats=24,
next_run=arrow.utcnow().replace(hour=18, minute=0))
# Use a cron expression
schedule('math.hypot',
3, 4,
schedule_type=Schedule.CRON,
cron = '0 22 * * 1-5')
For more info check the `Schedules <https://django-q.readthedocs.org/en/latest/schedules.html>`__ documentation.

10
django_q/__init__.py
View File

@@ -1,6 +1,12 @@
VERSION = (1, 2, 4)
default_app_config = 'django_q.apps.DjangoQConfig'
default_app_config = "django_q.apps.DjangoQConfig"
__all__ = ['conf', 'cluster', 'models', 'tasks']
__all__ = ["conf", "cluster", "models", "tasks", "croniter"]
# Optional Imports
try:
from croniter import croniter
except ImportError:
croniter = None

62
django_q/admin.py
View File

@@ -5,19 +5,13 @@ from django.utils.translation import gettext_lazy as _
from django_q.conf import Conf
from django_q.models import Success, Failure, Schedule, OrmQ
from django_q.tasks import async_task
from django_q import croniter
class TaskAdmin(admin.ModelAdmin):
"""model admin for success tasks."""
list_display = (
'name',
'func',
'started',
'stopped',
'time_taken',
'group'
)
list_display = ("name", "func", "started", "stopped", "time_taken", "group")
def has_add_permission(self, request):
"""Don't allow adds."""
@@ -28,9 +22,9 @@ class TaskAdmin(admin.ModelAdmin):
qs = super(TaskAdmin, self).get_queryset(request)
return qs.filter(success=True)
search_fields = ('name', 'func', 'group')
search_fields = ("name", "func", "group")
readonly_fields = []
list_filter = ('group',)
list_filter = ("group",)
def get_readonly_fields(self, request, obj=None):
"""Set all fields readonly."""
@@ -50,21 +44,15 @@ retry_failed.short_description = _("Resubmit selected tasks to queue")
class FailAdmin(admin.ModelAdmin):
"""model admin for failed tasks."""
list_display = (
'name',
'func',
'started',
'stopped',
'short_result'
)
list_display = ("name", "func", "started", "stopped", "short_result")
def has_add_permission(self, request):
"""Don't allow adds."""
return False
actions = [retry_failed]
search_fields = ('name', 'func')
list_filter = ('group',)
search_fields = ("name", "func")
list_filter = ("group",)
readonly_fields = []
def get_readonly_fields(self, request, obj=None):
@@ -76,31 +64,29 @@ class ScheduleAdmin(admin.ModelAdmin):
""" model admin for schedules """
list_display = (
'id',
'name',
'func',
'schedule_type',
'repeats',
'next_run',
'last_run',
'success'
"id",
"name",
"func",
"schedule_type",
"repeats",
"next_run",
"last_run",
"success",
)
list_filter = ('next_run', 'schedule_type')
search_fields = ('func',)
list_display_links = ('id', 'name')
# optional cron strings
if not croniter:
readonly_fields = ("cron",)
list_filter = ("next_run", "schedule_type")
search_fields = ("func",)
list_display_links = ("id", "name")
class QueueAdmin(admin.ModelAdmin):
""" queue admin for ORM broker """
list_display = (
'id',
'key',
'task_id',
'name',
'func',
'lock'
)
list_display = ("id", "key", "task_id", "name", "func", "lock")
def save_model(self, request, obj, form, change):
obj.save(using=Conf.ORM)

2
django_q/apps.py
View File

@@ -4,7 +4,7 @@ from django_q.conf import Conf
class DjangoQConfig(AppConfig):
name = 'django_q'
name = "django_q"
verbose_name = Conf.LABEL
def ready(self):

9
django_q/brokers/disque.py
View File

@@ -1,8 +1,11 @@
import random
# External
import redis
from redis import Redis
# Django
from django.utils.translation import gettext_lazy as _
from django_q.brokers import Broker
from django_q.conf import Conf
@@ -52,6 +55,8 @@ class Disque(Broker):
@staticmethod
def get_connection(list_key: str = Conf.PREFIX) -> Redis:
if not Conf.DISQUE_NODES:
raise redis.exceptions.ConnectionError(_("No Disque nodes configured"))
# randomize nodes
random.shuffle(Conf.DISQUE_NODES)
# find one that works
@@ -67,4 +72,6 @@ class Disque(Broker):
return redis_client
except redis.exceptions.ConnectionError:
continue
raise redis.exceptions.ConnectionError("Could not connect to any Disque nodes")
raise redis.exceptions.ConnectionError(
_("Could not connect to any Disque nodes")
)

49
django_q/cluster.py
View File

@@ -1,6 +1,5 @@
import ast
# Standard
import ast
import importlib
import signal
import socket
@@ -9,9 +8,8 @@ import uuid
from multiprocessing import Event, Process, Value, current_process
from time import sleep
# external
# External
import arrow
# Django
from django import db
from django.conf import settings
@@ -28,6 +26,7 @@ from django_q.queues import Queue
from django_q.signals import pre_execute
from django_q.signing import SignedPackage, BadSignature
from django_q.status import Stat, Status
from django_q import croniter
class Cluster:
@@ -103,10 +102,10 @@ class Cluster:
@property
def is_stopping(self) -> bool:
return (
self.stop_event
and self.start_event
and self.start_event.is_set()
and self.stop_event.is_set()
self.stop_event
and self.start_event
and self.start_event.is_set()
and self.stop_event.is_set()
)
@property
@@ -116,13 +115,13 @@ class Cluster:
class Sentinel:
def __init__(
self,
stop_event,
start_event,
cluster_id,
broker=None,
timeout=Conf.TIMEOUT,
start=True,
self,
stop_event,
start_event,
cluster_id,
broker=None,
timeout=Conf.TIMEOUT,
start=True,
):
# Make sure we catch signals for the pool
signal.signal(signal.SIGINT, signal.SIG_IGN)
@@ -208,7 +207,7 @@ class Sentinel:
if process.timer.value == 0:
# only need to terminate on timeout, otherwise we risk destabilizing the queues
process.terminate()
logger.warn(_(f"reincarnated worker {process.name} after timeout"))
logger.warning(_(f"reincarnated worker {process.name} after timeout"))
elif int(process.timer.value) == -2:
logger.info(_(f"recycled worker {process.name}"))
else:
@@ -377,7 +376,7 @@ def monitor(result_queue: Queue, broker: Broker = None):
def worker(
task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
):
"""
Takes a task from the task queue, tries to execute it and puts the result back in the result queue
@@ -552,9 +551,9 @@ def scheduler(broker: Broker = None):
try:
with db.transaction.atomic(using=Schedule.objects.db):
for s in (
Schedule.objects.select_for_update()
.exclude(repeats=0)
.filter(next_run__lt=timezone.now())
Schedule.objects.select_for_update()
.exclude(repeats=0)
.filter(next_run__lt=timezone.now())
):
args = ()
kwargs = {}
@@ -591,6 +590,16 @@ def scheduler(broker: Broker = None):
next_run = next_run.shift(months=+3)
elif s.schedule_type == s.YEARLY:
next_run = next_run.shift(years=+1)
elif s.schedule_type == s.CRON:
if not croniter:
raise ImportError(
_(
"Please install croniter to enable cron expressions"
)
)
next_run = arrow.get(
croniter(s.cron, timezone.now()).get_next()
)
if Conf.CATCH_UP or next_run > arrow.utcnow():
break
# arrow always returns a tz aware datetime, and we don't want

10
django_q/core_signing.py
View File

@@ -24,11 +24,11 @@ The difference is that `this` loads function calls `TimestampSigner` and `Signer
def loads(
s,
key=None,
salt: str = "django.core.signing",
serializer=JSONSerializer,
max_age=None,
s,
key=None,
salt: str = "django.core.signing",
serializer=JSONSerializer,
max_age=None,
):
"""
Reverse of dumps(), raise BadSignature if signature fails.

10
django_q/management/commands/qmonitor.py
View File

@@ -10,12 +10,12 @@ class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
'--run-once',
action='store_true',
dest='run_once',
"--run-once",
action="store_true",
dest="run_once",
default=False,
help='Run once and then stop.',
help="Run once and then stop.",
)
def handle(self, *args, **options):
monitor(run_once=options.get('run_once', False))
monitor(run_once=options.get("run_once", False))

23
django_q/migrations/0011_auto_20200628_1055.py Normal file
View File

@@ -0,0 +1,23 @@
# Generated by Django 3.0.7 on 2020-06-28 10:55
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('django_q', '0010_auto_20200610_0856'),
]
operations = [
migrations.AddField(
model_name='schedule',
name='cron',
field=models.CharField(blank=True, help_text='Cron expression', max_length=100, null=True),
),
migrations.AlterField(
model_name='schedule',
name='schedule_type',
field=models.CharField(choices=[('O', 'Once'), ('I', 'Minutes'), ('H', 'Hourly'), ('D', 'Daily'), ('W', 'Weekly'), ('M', 'Monthly'), ('Q', 'Quarterly'), ('Y', 'Yearly'), ('C', 'Cron')], default='O', max_length=1, verbose_name='Schedule Type'),
),
]

30
django_q/models.py
View File

@@ -1,15 +1,20 @@
# Django
from django import get_version
from django.core.exceptions import ValidationError
from django.db import models
from django.template.defaultfilters import truncatechars
from django.urls import reverse
from django.utils import timezone
from django.utils.html import format_html
from django.utils.translation import gettext_lazy as _
from django.db import models
from django.utils import timezone
# External
from picklefield import PickledObjectField
from picklefield.fields import dbsafe_decode
# Local
from django_q.signing import SignedPackage
from django_q import croniter
class Task(models.Model):
@@ -131,6 +136,16 @@ class Failure(Task):
proxy = True
# Optional Cron validator
def validate_cron(value):
if not croniter:
raise ImportError(_("Please install croniter to enable cron expressions"))
try:
croniter.expand(value)
except ValueError as e:
raise ValidationError(e)
class Schedule(models.Model):
name = models.CharField(max_length=100, null=True, blank=True)
func = models.CharField(max_length=256, help_text="e.g. module.tasks.function")
@@ -152,6 +167,7 @@ class Schedule(models.Model):
MONTHLY = "M"
QUARTERLY = "Q"
YEARLY = "Y"
CRON = "C"
TYPE = (
(ONCE, _("Once")),
(MINUTES, _("Minutes")),
@@ -161,6 +177,7 @@ class Schedule(models.Model):
(MONTHLY, _("Monthly")),
(QUARTERLY, _("Quarterly")),
(YEARLY, _("Yearly")),
(CRON, _("Cron")),
)
schedule_type = models.CharField(
max_length=1, choices=TYPE, default=TYPE[0][0], verbose_name=_("Schedule Type")
@@ -174,6 +191,13 @@ class Schedule(models.Model):
next_run = models.DateTimeField(
verbose_name=_("Next Run"), default=timezone.now, null=True
)
cron = models.CharField(
max_length=100,
null=True,
blank=True,
validators=[validate_cron],
help_text=_("Cron expression"),
)
task = models.CharField(max_length=100, null=True, editable=False)
def success(self):

9
django_q/tasks.py
View File

@@ -88,6 +88,7 @@ def schedule(func, *args, **kwargs):
:param repeats: how many times to repeat. 0=never, -1=always.
:param next_run: Next scheduled run.
:type next_run: datetime.datetime
:param cron: optional cron expression
:param kwargs: function keyword arguments.
:return: the schedule object.
:rtype: Schedule
@@ -98,13 +99,14 @@ def schedule(func, *args, **kwargs):
minutes = kwargs.pop("minutes", None)
repeats = kwargs.pop("repeats", -1)
next_run = kwargs.pop("next_run", timezone.now())
cron = kwargs.pop("cron", None)
# check for name duplicates instead of am unique constraint
if name and Schedule.objects.filter(name=name).exists():
raise IntegrityError("A schedule with the same name already exists.")
# create and return the schedule
return Schedule.objects.create(
s = Schedule(
name=name,
func=func,
hook=hook,
@@ -114,7 +116,12 @@ def schedule(func, *args, **kwargs):
minutes=minutes,
repeats=repeats,
next_run=next_run,
cron=cron,
)
# make sure we trigger validation
s.full_clean()
s.save()
return s
def result(task_id, wait=0, cached=Conf.CACHED):

186
django_q/tests/test_brokers.py
View File

@@ -11,72 +11,72 @@ from django_q.humanhash import uuid
def test_broker(monkeypatch):
broker = Broker()
broker.enqueue('test')
broker.enqueue("test")
broker.dequeue()
broker.queue_size()
broker.lock_size()
broker.purge_queue()
broker.delete('id')
broker.delete("id")
broker.delete_queue()
broker.acknowledge('test')
broker.acknowledge("test")
broker.ping()
broker.info()
# stats
assert broker.get_stat('test_1') is None
broker.set_stat('test_1', 'test', 3)
assert broker.get_stat('test_1') == 'test'
assert broker.get_stats('test:*')[0] == 'test'
assert broker.get_stat("test_1") is None
broker.set_stat("test_1", "test", 3)
assert broker.get_stat("test_1") == "test"
assert broker.get_stats("test:*")[0] == "test"
# stats with no cache
monkeypatch.setattr(Conf, 'CACHE', 'not_configured')
monkeypatch.setattr(Conf, "CACHE", "not_configured")
broker.cache = broker.get_cache()
assert broker.get_stat('test_1') is None
broker.set_stat('test_1', 'test', 3)
assert broker.get_stat('test_1') is None
assert broker.get_stats('test:*') is None
assert broker.get_stat("test_1") is None
broker.set_stat("test_1", "test", 3)
assert broker.get_stat("test_1") is None
assert broker.get_stats("test:*") is None
def test_redis(monkeypatch):
monkeypatch.setattr(Conf, 'DJANGO_REDIS', None)
monkeypatch.setattr(Conf, "DJANGO_REDIS", None)
broker = get_broker()
assert broker.ping() is True
assert broker.info() is not None
monkeypatch.setattr(Conf, 'REDIS', {'host': '127.0.0.1', 'port': 7799})
monkeypatch.setattr(Conf, "REDIS", {"host": "127.0.0.1", "port": 7799})
broker = get_broker()
with pytest.raises(Exception):
broker.ping()
monkeypatch.setattr(Conf, 'REDIS', 'redis://127.0.0.1:7799')
monkeypatch.setattr(Conf, "REDIS", "redis://127.0.0.1:7799")
broker = get_broker()
with pytest.raises(Exception):
broker.ping()
def test_custom(monkeypatch):
monkeypatch.setattr(Conf, 'BROKER_CLASS', 'brokers.redis_broker.Redis')
monkeypatch.setattr(Conf, "BROKER_CLASS", "brokers.redis_broker.Redis")
broker = get_broker()
assert broker.ping() is True
assert broker.info() is not None
assert broker.__class__.__name__ == 'Redis'
assert broker.__class__.__name__ == "Redis"
def test_disque(monkeypatch):
monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7711'])
monkeypatch.setattr(Conf, "DISQUE_NODES", ["127.0.0.1:7711"])
# check broker
broker = get_broker(list_key='disque_test')
broker = get_broker(list_key="disque_test")
assert broker.ping() is True
assert broker.info() is not None
# clear before we start
broker.delete_queue()
# async_task
broker.enqueue('test')
broker.enqueue("test")
assert broker.queue_size() == 1
# dequeue
task = broker.dequeue()[0]
assert task[1] == 'test'
assert task[1] == "test"
broker.acknowledge(task[0])
assert broker.queue_size() == 0
# Retry test
monkeypatch.setattr(Conf, 'RETRY', 1)
broker.enqueue('test')
monkeypatch.setattr(Conf, "RETRY", 1)
broker.enqueue("test")
assert broker.queue_size() == 1
broker.dequeue()
assert broker.queue_size() == 0
@@ -88,17 +88,17 @@ def test_disque(monkeypatch):
sleep(1.5)
assert broker.queue_size() == 0
# delete job
task_id = broker.enqueue('test')
task_id = broker.enqueue("test")
broker.delete(task_id)
assert broker.dequeue() is None
# fail
task_id = broker.enqueue('test')
task_id = broker.enqueue("test")
broker.fail(task_id)
# bulk test
for i in range(5):
broker.enqueue('test')
monkeypatch.setattr(Conf, 'BULK', 5)
monkeypatch.setattr(Conf, 'DISQUE_FASTACK', True)
broker.enqueue("test")
monkeypatch.setattr(Conf, "BULK", 5)
monkeypatch.setattr(Conf, "DISQUE_FASTACK", True)
tasks = broker.dequeue()
for task in tasks:
assert task is not None
@@ -106,35 +106,46 @@ def test_disque(monkeypatch):
# test duplicate acknowledge
broker.acknowledge(task[0])
# delete queue
broker.enqueue('test')
broker.enqueue('test')
broker.enqueue("test")
broker.enqueue("test")
broker.delete_queue()
assert broker.queue_size() == 0
# connection test
monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7798', '127.0.0.1:7799'])
monkeypatch.setattr(Conf, "DISQUE_NODES", ["127.0.0.1:7798", "127.0.0.1:7799"])
with pytest.raises(redis.exceptions.ConnectionError):
broker.get_connection()
# connection test with no nodes
monkeypatch.setattr(Conf, "DISQUE_NODES", None)
with pytest.raises(redis.exceptions.ConnectionError):
broker.get_connection()
@pytest.mark.skipif(not os.getenv('IRON_MQ_TOKEN'),
reason="requires IronMQ credentials")
@pytest.mark.skipif(
not os.getenv("IRON_MQ_TOKEN"), reason="requires IronMQ credentials"
)
def test_ironmq(monkeypatch):
monkeypatch.setattr(Conf, 'IRON_MQ', {'token': os.getenv('IRON_MQ_TOKEN'),
'project_id': os.getenv('IRON_MQ_PROJECT_ID')})
monkeypatch.setattr(
Conf,
"IRON_MQ",
{
"token": os.getenv("IRON_MQ_TOKEN"),
"project_id": os.getenv("IRON_MQ_PROJECT_ID"),
},
)
# check broker
broker = get_broker(list_key=uuid()[0])
assert broker.ping() is True
assert broker.info() is not None
# initialize the queue
broker.enqueue('test')
broker.enqueue("test")
# clear before we start
broker.purge_queue()
assert broker.queue_size() == 0
# async_task
broker.enqueue('test')
broker.enqueue("test")
# dequeue
task = broker.dequeue()[0]
assert task[1] == 'test'
assert task[1] == "test"
broker.acknowledge(task[0])
assert broker.dequeue() is None
# Retry test
@@ -148,16 +159,16 @@ def test_ironmq(monkeypatch):
# broker.acknowledge(task[0])
# sleep(3)
# delete job
task_id = broker.enqueue('test')
task_id = broker.enqueue("test")
broker.delete(task_id)
assert broker.dequeue() is None
# fail
task_id = broker.enqueue('test')
task_id = broker.enqueue("test")
broker.fail(task_id)
# bulk test
for i in range(5):
broker.enqueue('test')
monkeypatch.setattr(Conf, 'BULK', 5)
broker.enqueue("test")
monkeypatch.setattr(Conf, "BULK", 5)
tasks = broker.dequeue()
for task in tasks:
assert task is not None
@@ -165,62 +176,69 @@ def test_ironmq(monkeypatch):
# duplicate acknowledge
broker.acknowledge(task[0])
# delete queue
broker.enqueue('test')
broker.enqueue('test')
broker.enqueue("test")
broker.enqueue("test")
broker.purge_queue()
assert broker.dequeue() is None
broker.delete_queue()
@pytest.mark.skipif(not os.getenv('AWS_ACCESS_KEY_ID'),
reason="requires AWS credentials")
@pytest.mark.skipif(
not os.getenv("AWS_ACCESS_KEY_ID"), reason="requires AWS credentials"
)
def canceled_sqs(monkeypatch):
monkeypatch.setattr(Conf, 'SQS', {'aws_region': os.getenv('AWS_REGION'),
'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY')})
monkeypatch.setattr(
Conf,
"SQS",
{
"aws_region": os.getenv("AWS_REGION"),
"aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"),
"aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"),
},
)
# check broker
broker = get_broker(list_key=uuid()[0])
assert broker.ping() is True
assert broker.info() is not None
assert broker.queue_size() == 0
# async_task
broker.enqueue('test')
broker.enqueue("test")
# dequeue
task = broker.dequeue()[0]
assert task[1] == 'test'
assert task[1] == "test"
broker.acknowledge(task[0])
assert broker.dequeue() is None
# Retry test
monkeypatch.setattr(Conf, 'RETRY', 1)
broker.enqueue('test')
monkeypatch.setattr(Conf, "RETRY", 1)
broker.enqueue("test")
sleep(2)
# Sometimes SQS is not linear
task = broker.dequeue()
if not task:
pytest.skip('SQS being weird')
pytest.skip("SQS being weird")
task = task[0]
assert len(task) > 0
broker.acknowledge(task[0])
sleep(2)
# delete job
monkeypatch.setattr(Conf, 'RETRY', 60)
broker.enqueue('test')
monkeypatch.setattr(Conf, "RETRY", 60)
broker.enqueue("test")
sleep(1)
task = broker.dequeue()
if not task:
pytest.skip('SQS being weird')
pytest.skip("SQS being weird")
task_id = task[0][0]
broker.delete(task_id)
assert broker.dequeue() is None
# fail
broker.enqueue('test')
broker.enqueue("test")
while task is None:
task = broker.dequeue()[0]
broker.fail(task[0])
# bulk test
for i in range(10):
broker.enqueue('test')
monkeypatch.setattr(Conf, 'BULK', 12)
broker.enqueue("test")
monkeypatch.setattr(Conf, "BULK", 12)
tasks = broker.dequeue()
for task in tasks:
assert task is not None
@@ -229,31 +247,31 @@ def canceled_sqs(monkeypatch):
broker.acknowledge(task[0])
assert broker.lock_size() == 0
# delete queue
broker.enqueue('test')
broker.enqueue("test")
broker.purge_queue()
broker.delete_queue()
@pytest.mark.django_db
def test_orm(monkeypatch):
monkeypatch.setattr(Conf, 'ORM', 'default')
monkeypatch.setattr(Conf, "ORM", "default")
# check broker
broker = get_broker(list_key='orm_test')
broker = get_broker(list_key="orm_test")
assert broker.ping() is True
assert broker.info() is not None
# clear before we start
broker.delete_queue()
# async_task
broker.enqueue('test')
broker.enqueue("test")
assert broker.queue_size() == 1
# dequeue
task = broker.dequeue()[0]
assert task[1] == 'test'
assert task[1] == "test"
broker.acknowledge(task[0])
assert broker.queue_size() == 0
# Retry test
monkeypatch.setattr(Conf, 'RETRY', 1)
broker.enqueue('test')
monkeypatch.setattr(Conf, "RETRY", 1)
broker.enqueue("test")
assert broker.queue_size() == 1
broker.dequeue()
assert broker.queue_size() == 0
@@ -265,16 +283,16 @@ def test_orm(monkeypatch):
sleep(1.5)
assert broker.queue_size() == 0
# delete job
task_id = broker.enqueue('test')
task_id = broker.enqueue("test")
broker.delete(task_id)
assert broker.dequeue() is None
# fail
task_id = broker.enqueue('test')
task_id = broker.enqueue("test")
broker.fail(task_id)
# bulk test
for i in range(5):
broker.enqueue('test')
monkeypatch.setattr(Conf, 'BULK', 5)
broker.enqueue("test")
monkeypatch.setattr(Conf, "BULK", 5)
tasks = broker.dequeue()
assert broker.lock_size() == Conf.BULK
for task in tasks:
@@ -285,32 +303,32 @@ def test_orm(monkeypatch):
# test duplicate acknowledge
broker.acknowledge(task[0])
# delete queue
broker.enqueue('test')
broker.enqueue('test')
broker.enqueue("test")
broker.enqueue("test")
broker.delete_queue()
assert broker.queue_size() == 0
@pytest.mark.django_db
def test_mongo(monkeypatch):
monkeypatch.setattr(Conf, 'MONGO', {'host': '127.0.0.1', 'port': 27017})
monkeypatch.setattr(Conf, "MONGO", {"host": "127.0.0.1", "port": 27017})
# check broker
broker = get_broker(list_key='mongo_test')
broker = get_broker(list_key="mongo_test")
assert broker.ping() is True
assert broker.info() is not None
# clear before we start
broker.delete_queue()
# async_task
broker.enqueue('test')
broker.enqueue("test")
assert broker.queue_size() == 1
# dequeue
task = broker.dequeue()[0]
assert task[1] == 'test'
assert task[1] == "test"
broker.acknowledge(task[0])
assert broker.queue_size() == 0
# Retry test
monkeypatch.setattr(Conf, 'RETRY', 1)
broker.enqueue('test')
monkeypatch.setattr(Conf, "RETRY", 1)
broker.enqueue("test")
assert broker.queue_size() == 1
broker.dequeue()
assert broker.queue_size() == 0
@@ -322,15 +340,15 @@ def test_mongo(monkeypatch):
sleep(1.5)
assert broker.queue_size() == 0
# delete job
task_id = broker.enqueue('test')
task_id = broker.enqueue("test")
broker.delete(task_id)
assert broker.dequeue() is None
# fail
task_id = broker.enqueue('test')
task_id = broker.enqueue("test")
broker.fail(task_id)
# bulk test
for i in range(5):
broker.enqueue('test')
broker.enqueue("test")
tasks = []
for i in range(5):
tasks.append(broker.dequeue()[0])
@@ -343,8 +361,8 @@ def test_mongo(monkeypatch):
# test duplicate acknowledge
broker.acknowledge(task[0])
# delete queue
broker.enqueue('test')
broker.enqueue('test')
broker.enqueue("test")
broker.enqueue("test")
broker.purge_queue()
broker.delete_queue()
assert broker.queue_size() == 0

2
django_q/tests/test_cluster.py
View File

@@ -142,6 +142,8 @@ def test_enqueue(broker, admin_user):
# q_options and save opt_out test
k = async_task('django_q.tests.tasks.get_user_id', admin_user,
q_options={'broker': broker, 'group': 'test_k', 'save': False, 'timeout': 90})
# test unicode
assert Task(name='Amalia').__unicode__()=='Amalia'
# check if everything has a task id
assert isinstance(a, str)
assert isinstance(b, str)

1
django_q/tests/test_commands.py
View File

@@ -16,3 +16,4 @@ def test_qmonitor():
def test_qinfo():
call_command('qinfo')
call_command('qinfo', config=True)
call_command('qinfo', ids=True)

3
django_q/tests/test_monitor.py
View File

@@ -4,7 +4,7 @@ import uuid
from django_q.tasks import async_task
from django_q.brokers import get_broker
from django_q.cluster import Cluster
from django_q.monitor import monitor, info
from django_q.monitor import monitor, info, get_ids
from django_q.status import Stat
from django_q.conf import Conf
@@ -16,6 +16,7 @@ def test_monitor(monkeypatch):
c = Cluster()
c.start()
stats = monitor(run_once=True)
assert get_ids() is True
c.stop()
assert len(stats) > 0
found_c = False

20
django_q/tests/test_scheduler.py
View File

@@ -3,14 +3,15 @@ from multiprocessing import Event, Value
import arrow
import pytest
from django.core.exceptions import ValidationError
from django.db import IntegrityError
from django.utils import timezone
from django_q.brokers import get_broker
from django_q.cluster import pusher, worker, monitor, scheduler
from django_q.conf import Conf
from django_q.tasks import Schedule, fetch, schedule as create_schedule
from django_q.queues import Queue
from django_q.tasks import Schedule, fetch, schedule as create_schedule
@pytest.fixture
@@ -91,8 +92,25 @@ def test_scheduler(broker, monkeypatch):
schedule_type=Schedule.MINUTES,
minutes=10)
assert hasattr(minute_schedule, 'pk') is True
# Cron schedule
cron_schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.CRON,
cron="0 22 * * 1-5")
assert hasattr(cron_schedule, 'pk') is True
assert cron_schedule.full_clean() is None
assert cron_schedule.__unicode__() == 'django_q.tests.tasks.word_multiply'
with pytest.raises(ValidationError):
create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=Schedule.CRON,
cron="0 22 * * 1-12")
# All other types
for t in Schedule.TYPE:
if t[0] == Schedule.CRON:
continue
schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',

7
docs/install.rst
View File

@@ -88,6 +88,13 @@ Optional
- `Croniter <https://github.com/kiorky/croniter>`__ is an optional package that is used to parse cron expressions for the scheduler::
$ pip install croniter
Add-ons
-------
- `django-q-rollbar <https://github.com/danielwelch/django-q-rollbar>`__ is a Rollbar error reporter::

22
docs/schedules.rst
View File

@@ -45,6 +45,13 @@ You can manage them through the :ref:`admin_page` or directly from your code wit
repeats=24,
next_run=arrow.utcnow().replace(hour=18, minute=0))
# Use a cron expression
schedule('math.hypot',
3, 4,
schedule_type=Schedule.CRON,
cron = '0 22 * * 1-5')
Missed schedules
----------------
@@ -99,8 +106,9 @@ Reference
:param args: arguments for the scheduled function.
:param str name: An optional name for your schedule.
:param str hook: optional result hook function. Dotted strings only.
:param str schedule_type: (O)nce, M(I)nutes, (H)ourly, (D)aily, (W)eekly, (M)onthly, (Q)uarterly, (Y)early or :attr:`Schedule.TYPE`
:param str schedule_type: (O)nce, M(I)nutes, (H)ourly, (D)aily, (W)eekly, (M)onthly, (Q)uarterly, (Y)early or (C)ron :attr:`Schedule.TYPE`
:param int minutes: Number of minutes for the Minutes type.
:param str cron: Cron expression for the Cron type.
:param int repeats: Number of times to repeat schedule. -1=Always, 0=Never, n =n.
:param datetime next_run: Next or first scheduled execution datetime.
:param dict q_options: options passed to async_task for this schedule
@@ -140,7 +148,7 @@ Reference
.. py:attribute:: TYPE
:attr:`ONCE`, :attr:`MINUTES`, :attr:`HOURLY`, :attr:`DAILY`, :attr:`WEEKLY`, :attr:`MONTHLY`, :attr:`QUARTERLY`, :attr:`YEARLY`
:attr:`ONCE`, :attr:`MINUTES`, :attr:`HOURLY`, :attr:`DAILY`, :attr:`WEEKLY`, :attr:`MONTHLY`, :attr:`QUARTERLY`, :attr:`YEARLY`, :attr:`CRON`
.. py:attribute:: minutes
@@ -148,6 +156,10 @@ Reference
The number of minutes the :attr:`MINUTES` schedule should use.
Is ignored for other schedule types.
.. py:attribute:: cron
A cron string describing the schedule. You need the optional `croniter` package installed for this.
.. py:attribute:: repeats
Number of times to repeat the schedule. -1=Always, 0=Never, n =n.
@@ -208,3 +220,9 @@ Reference
`'Y'` only runs once a year. The same caution as with months apply;
If you set this to february 29th, it will run on february 28th in the following years.
.. py:attribute:: CRON
`'C'` uses the optional `croniter` package to determine a schedule based a cron expression.

54
poetry.lock generated
View File

@@ -71,10 +71,10 @@ description = "The AWS SDK for Python"
name = "boto3"
optional = true
python-versions = "*"
version = "1.14.12"
version = "1.14.14"
[package.dependencies]
botocore = ">=1.17.12,<1.18.0"
botocore = ">=1.17.14,<1.18.0"
jmespath = ">=0.7.1,<1.0.0"
s3transfer = ">=0.3.0,<0.4.0"
@@ -84,7 +84,7 @@ description = "Low-level, data-driven core of boto 3."
name = "botocore"
optional = true
python-versions = "*"
version = "1.17.12"
version = "1.17.14"
[package.dependencies]
docutils = ">=0.10,<0.16"
@@ -120,13 +120,25 @@ optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "0.4.3"
[[package]]
category = "main"
description = "croniter provides iteration for datetime object with cron like format"
name = "croniter"
optional = true
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "0.3.34"
[package.dependencies]
natsort = "*"
python-dateutil = "*"
[[package]]
category = "main"
description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design."
name = "django"
optional = false
python-versions = ">=3.6"
version = "3.0.7"
version = "3.0.8"
[package.dependencies]
asgiref = ">=3.2,<4.0"
@@ -270,6 +282,18 @@ optional = false
python-versions = ">=3.5"
version = "8.4.0"
[[package]]
category = "main"
description = "Simple yet flexible natural sorting in Python."
name = "natsort"
optional = true
python-versions = ">=3.4"
version = "7.0.1"
[package.extras]
fast = ["fastnumbers (>=2.0.0)"]
icu = ["PyICU (>=1.0.0)"]
[[package]]
category = "dev"
description = "Core utilities for Python packages"
@@ -497,7 +521,7 @@ rollbar = ["django-q-rollbar"]
sentry = []
[metadata]
content-hash = "d89c929e8b6951968228b1aab1aa5c0a7c014acceb3b986bbaa3e41df57d8b39"
content-hash = "6d89bff8bd465aa4a5facd812172a970bf5edebdd8099141054f72aa63260c2a"
python-versions = ">=3.6"
[metadata.files]
@@ -526,12 +550,12 @@ blessed = [
{file = "blessed-1.17.8.tar.gz", hash = "sha256:7671d057b2df6ddbefd809009fb08feb2f8d2d163d240b5e765088a90519b2f1"},
]
boto3 = [
{file = "boto3-1.14.12-py2.py3-none-any.whl", hash = "sha256:7daad26a008c91dd7b82fde17d246d1fe6e4b3813426689ef8bac9017a277cfb"},
{file = "boto3-1.14.12.tar.gz", hash = "sha256:2616351c98eec18d20a1d64b33355c86cd855ac96219d1b8428c9bfc590bde53"},
{file = "boto3-1.14.14-py2.py3-none-any.whl", hash = "sha256:4c2f5f9f28930e236845e2cddbe01cb093ca96dc1f5c6e2b2b254722018a2268"},
{file = "boto3-1.14.14.tar.gz", hash = "sha256:87beffba2360b8077413f2d473cb828d0a5bda513bd1d6fb7b137c57b686aeb6"},
]
botocore = [
{file = "botocore-1.17.12-py2.py3-none-any.whl", hash = "sha256:45934d880378777cefeca727f369d1f5aebf6b254e9be58e7c77dd0b059338bb"},
{file = "botocore-1.17.12.tar.gz", hash = "sha256:a94e0e2307f1b9fe3a84660842909cd2680b57a9fc9fb0c3a03b0afb2eadbe21"},
{file = "botocore-1.17.14-py2.py3-none-any.whl", hash = "sha256:6a2e9768dad8ae9771302d5922b977dca6bb9693f9b6a5f6ed0e7ac375e2ca40"},
{file = "botocore-1.17.14.tar.gz", hash = "sha256:96d668ae5246d236ea83e4586349552d6584e8b1551ae2fccc0bd4ed528a746f"},
]
certifi = [
{file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"},
@@ -545,9 +569,13 @@ colorama = [
{file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"},
{file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"},
]
croniter = [
{file = "croniter-0.3.34-py2.py3-none-any.whl", hash = "sha256:15597ef0639f8fbab09cbf8c277fa8c65c8b9dbe818c4b2212f95dbc09c6f287"},
{file = "croniter-0.3.34.tar.gz", hash = "sha256:7186b9b464f45cf3d3c83a18bc2344cc101d7b9fd35a05f2878437b14967e964"},
]
django = [
{file = "Django-3.0.7-py3-none-any.whl", hash = "sha256:e1630333248c9b3d4e38f02093a26f1e07b271ca896d73097457996e0fae12e8"},
{file = "Django-3.0.7.tar.gz", hash = "sha256:5052b34b34b3425233c682e0e11d658fd6efd587d11335a0203d827224ada8f2"},
{file = "Django-3.0.8-py3-none-any.whl", hash = "sha256:5457fc953ec560c5521b41fad9e6734a4668b7ba205832191bbdff40ec61073c"},
{file = "Django-3.0.8.tar.gz", hash = "sha256:31a5fbbea5fc71c99e288ec0b2f00302a0a92c44b13ede80b73a6a4d6d205582"},
]
django-picklefield = [
{file = "django-picklefield-3.0.1.tar.gz", hash = "sha256:15ccba592ca953b9edf9532e64640329cd47b136b7f8f10f2939caa5f9ce4287"},
@@ -638,6 +666,10 @@ more-itertools = [
{file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"},
{file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"},
]
natsort = [
{file = "natsort-7.0.1-py3-none-any.whl", hash = "sha256:d3fd728a3ceb7c78a59aa8539692a75e37cbfd9b261d4d702e8016639820f90a"},
{file = "natsort-7.0.1.tar.gz", hash = "sha256:a633464dc3a22b305df0f27abcb3e83515898aa1fd0ed2f9726c3571a27258cf"},
]
packaging = [
{file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"},
{file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"},

3
pyproject.toml
View File

@@ -1,6 +1,6 @@
[tool.poetry]
name = "django-q"
version = "1.2.4"
version = "1.3.0"
description = "A multiprocessing distributed task queue for Django"
authors = ["Ilan Steemers <koed00@gmail.com>"]
license = "MIT"
@@ -52,6 +52,7 @@ django-redis = {version = "^4.12.1", optional = true}
iron-mq = {version = "^0.9", optional = true}
boto3 = {version = "^1.14.12", optional = true}
pymongo = {version = "^3.10.1", optional = true}
croniter = {version = "^0.3.34", optional = true}
[tool.poetry.dev-dependencies]
pytest = "^5.4.2"

1
requirements.in
View File

@@ -8,3 +8,4 @@ django-redis
iron-mq
boto3
pymongo
croniter

10
requirements.txt
View File

@@ -7,22 +7,24 @@
arrow==0.15.7 # via -r requirements.in
asgiref==3.2.10 # via django
blessed==1.17.8 # via -r requirements.in
boto3==1.14.12 # via -r requirements.in
botocore==1.17.12 # via boto3, s3transfer
boto3==1.14.14 # via -r requirements.in
botocore==1.17.14 # via boto3, s3transfer
certifi==2020.6.20 # via requests
chardet==3.0.4 # via requests
croniter==0.3.34 # via -r requirements.in
django-picklefield==3.0.1 # via -r requirements.in
django-redis==4.12.1 # via -r requirements.in
django==3.0.7 # via django-picklefield, django-redis
django==3.0.8 # via django-picklefield, django-redis
docutils==0.15.2 # via botocore
hiredis==1.0.1 # via -r requirements.in
idna==2.10 # via requests
iron-core==1.2.0 # via iron-mq
iron-mq==0.9 # via -r requirements.in
jmespath==0.10.0 # via boto3, botocore
natsort==7.0.1 # via croniter
psutil==5.7.0 # via -r requirements.in
pymongo==3.10.1 # via -r requirements.in
python-dateutil==2.8.1 # via arrow, botocore, iron-core
python-dateutil==2.8.1 # via arrow, botocore, croniter, iron-core
pytz==2020.1 # via django
redis==3.5.3 # via -r requirements.in, django-redis
requests==2.24.0 # via iron-core