Merge pull request #452 from Koed00/cron

Support for Cron expressions

.gitignore

@@ -66,4 +66,5 @@ db.sqlite3
 .venv
 .idea
 djq
-node_modules
+node_modules
+/c.cache/

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 
 services:
-  - redis-server
+  - redis
   - mongodb
 
 python:
@@ -9,8 +9,8 @@ python:
   - "3.8"
 
 env:
-  - DJANGO=3.0.7
+  - DJANGO=3.0.8
-  - DJANGO=2.2.13
+  - DJANGO=2.2.14
 
 sudo: true
 dist: xenial

README.rst

@@ -13,7 +13,7 @@ Features
 
 -  Multiprocessing worker pool
 -  Asynchronous tasks
--  Scheduled and repeated tasks
+-  Scheduled, cron and repeated tasks
 -  Signed and compressed packages
 -  Failure and success database or cache
 -  Result hooks, groups and chains
@@ -180,6 +180,12 @@ Admin page or directly from your code:
              repeats=24,
              next_run=arrow.utcnow().replace(hour=18, minute=0))
 
+    # Use a cron expression
+    schedule('math.hypot',
+             3, 4,
+             schedule_type=Schedule.CRON,
+             cron = '0 22 * * 1-5')
+
 For more info check the `Schedules <https://django-q.readthedocs.org/en/latest/schedules.html>`__ documentation.
 
 

django_q/__init__.py

@@ -1,6 +1,12 @@
 VERSION = (1, 2, 4)
 
-default_app_config = 'django_q.apps.DjangoQConfig'
+default_app_config = "django_q.apps.DjangoQConfig"
 
 
-__all__ = ['conf', 'cluster', 'models', 'tasks']
+__all__ = ["conf", "cluster", "models", "tasks", "croniter"]
+
+# Optional Imports
+try:
+    from croniter import croniter
+except ImportError:
+    croniter = None

django_q/admin.py

@@ -5,19 +5,13 @@ from django.utils.translation import gettext_lazy as _
 from django_q.conf import Conf
 from django_q.models import Success, Failure, Schedule, OrmQ
 from django_q.tasks import async_task
+from django_q import croniter
 
 
 class TaskAdmin(admin.ModelAdmin):
     """model admin for success tasks."""
 
-    list_display = (
+    list_display = ("name", "func", "started", "stopped", "time_taken", "group")
-        'name',
-        'func',
-        'started',
-        'stopped',
-        'time_taken',
-        'group'
-    )
 
     def has_add_permission(self, request):
         """Don't allow adds."""
@@ -28,9 +22,9 @@ class TaskAdmin(admin.ModelAdmin):
         qs = super(TaskAdmin, self).get_queryset(request)
         return qs.filter(success=True)
 
-    search_fields = ('name', 'func', 'group')
+    search_fields = ("name", "func", "group")
     readonly_fields = []
-    list_filter = ('group',)
+    list_filter = ("group",)
 
     def get_readonly_fields(self, request, obj=None):
         """Set all fields readonly."""
@@ -50,21 +44,15 @@ retry_failed.short_description = _("Resubmit selected tasks to queue")
 class FailAdmin(admin.ModelAdmin):
     """model admin for failed tasks."""
 
-    list_display = (
+    list_display = ("name", "func", "started", "stopped", "short_result")
-        'name',
-        'func',
-        'started',
-        'stopped',
-        'short_result'
-    )
 
     def has_add_permission(self, request):
         """Don't allow adds."""
         return False
 
     actions = [retry_failed]
-    search_fields = ('name', 'func')
+    search_fields = ("name", "func")
-    list_filter = ('group',)
+    list_filter = ("group",)
     readonly_fields = []
 
     def get_readonly_fields(self, request, obj=None):
@@ -76,31 +64,29 @@ class ScheduleAdmin(admin.ModelAdmin):
     """ model admin for schedules """
 
     list_display = (
-        'id',
+        "id",
-        'name',
+        "name",
-        'func',
+        "func",
-        'schedule_type',
+        "schedule_type",
-        'repeats',
+        "repeats",
-        'next_run',
+        "next_run",
-        'last_run',
+        "last_run",
-        'success'
+        "success",
     )
 
-    list_filter = ('next_run', 'schedule_type')
+    # optional cron strings
-    search_fields = ('func',)
+    if not croniter:
-    list_display_links = ('id', 'name')
+        readonly_fields = ("cron",)
+
+    list_filter = ("next_run", "schedule_type")
+    search_fields = ("func",)
+    list_display_links = ("id", "name")
 
 
 class QueueAdmin(admin.ModelAdmin):
     """  queue admin for ORM broker """
-    list_display = (
+
-        'id',
+    list_display = ("id", "key", "task_id", "name", "func", "lock")
-        'key',
-        'task_id',
-        'name',
-        'func',
-        'lock'
-    )
 
     def save_model(self, request, obj, form, change):
         obj.save(using=Conf.ORM)

django_q/apps.py

@@ -4,7 +4,7 @@ from django_q.conf import Conf
 
 
 class DjangoQConfig(AppConfig):
-    name = 'django_q'
+    name = "django_q"
     verbose_name = Conf.LABEL
 
     def ready(self):

django_q/brokers/disque.py

@@ -1,8 +1,11 @@
 import random
 
+# External
 import redis
 from redis import Redis
 
+# Django
+from django.utils.translation import gettext_lazy as _
 from django_q.brokers import Broker
 from django_q.conf import Conf
 
@@ -52,6 +55,8 @@ class Disque(Broker):
 
     @staticmethod
     def get_connection(list_key: str = Conf.PREFIX) -> Redis:
+        if not Conf.DISQUE_NODES:
+            raise redis.exceptions.ConnectionError(_("No Disque nodes configured"))
         # randomize nodes
         random.shuffle(Conf.DISQUE_NODES)
         # find one that works
@@ -67,4 +72,6 @@ class Disque(Broker):
                 return redis_client
             except redis.exceptions.ConnectionError:
                 continue
-        raise redis.exceptions.ConnectionError("Could not connect to any Disque nodes")
+        raise redis.exceptions.ConnectionError(
+            _("Could not connect to any Disque nodes")
+        )

django_q/cluster.py

@@ -1,6 +1,5 @@
-import ast
-
 # Standard
+import ast
 import importlib
 import signal
 import socket
@@ -9,9 +8,8 @@ import uuid
 from multiprocessing import Event, Process, Value, current_process
 from time import sleep
 
-# external
+# External
 import arrow
-
 # Django
 from django import db
 from django.conf import settings
@@ -28,6 +26,7 @@ from django_q.queues import Queue
 from django_q.signals import pre_execute
 from django_q.signing import SignedPackage, BadSignature
 from django_q.status import Stat, Status
+from django_q import croniter
 
 
 class Cluster:
@@ -103,10 +102,10 @@ class Cluster:
     @property
     def is_stopping(self) -> bool:
         return (
-            self.stop_event
+                self.stop_event
-            and self.start_event
+                and self.start_event
-            and self.start_event.is_set()
+                and self.start_event.is_set()
-            and self.stop_event.is_set()
+                and self.stop_event.is_set()
         )
 
     @property
@@ -116,13 +115,13 @@ class Cluster:
 
 class Sentinel:
     def __init__(
-        self,
+            self,
-        stop_event,
+            stop_event,
-        start_event,
+            start_event,
-        cluster_id,
+            cluster_id,
-        broker=None,
+            broker=None,
-        timeout=Conf.TIMEOUT,
+            timeout=Conf.TIMEOUT,
-        start=True,
+            start=True,
     ):
         # Make sure we catch signals for the pool
         signal.signal(signal.SIGINT, signal.SIG_IGN)
@@ -208,7 +207,7 @@ class Sentinel:
             if process.timer.value == 0:
                 # only need to terminate on timeout, otherwise we risk destabilizing the queues
                 process.terminate()
-                logger.warn(_(f"reincarnated worker {process.name} after timeout"))
+                logger.warning(_(f"reincarnated worker {process.name} after timeout"))
             elif int(process.timer.value) == -2:
                 logger.info(_(f"recycled worker {process.name}"))
             else:
@@ -377,7 +376,7 @@ def monitor(result_queue: Queue, broker: Broker = None):
 
 
 def worker(
-    task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
+        task_queue: Queue, result_queue: Queue, timer: Value, timeout: int = Conf.TIMEOUT
 ):
     """
     Takes a task from the task queue, tries to execute it and puts the result back in the result queue
@@ -552,9 +551,9 @@ def scheduler(broker: Broker = None):
     try:
         with db.transaction.atomic(using=Schedule.objects.db):
             for s in (
-                Schedule.objects.select_for_update()
+                    Schedule.objects.select_for_update()
-                .exclude(repeats=0)
+                            .exclude(repeats=0)
-                .filter(next_run__lt=timezone.now())
+                            .filter(next_run__lt=timezone.now())
             ):
                 args = ()
                 kwargs = {}
@@ -591,6 +590,16 @@ def scheduler(broker: Broker = None):
                             next_run = next_run.shift(months=+3)
                         elif s.schedule_type == s.YEARLY:
                             next_run = next_run.shift(years=+1)
+                        elif s.schedule_type == s.CRON:
+                            if not croniter:
+                                raise ImportError(
+                                    _(
+                                        "Please install croniter to enable cron expressions"
+                                    )
+                                )
+                            next_run = arrow.get(
+                                croniter(s.cron, timezone.now()).get_next()
+                            )
                         if Conf.CATCH_UP or next_run > arrow.utcnow():
                             break
                     # arrow always returns a tz aware datetime, and we don't want

django_q/core_signing.py

@@ -24,11 +24,11 @@ The difference is that `this` loads function calls `TimestampSigner` and `Signer
 
 
 def loads(
-        s,
+    s,
-        key=None,
+    key=None,
-        salt: str = "django.core.signing",
+    salt: str = "django.core.signing",
-        serializer=JSONSerializer,
+    serializer=JSONSerializer,
-        max_age=None,
+    max_age=None,
 ):
     """
     Reverse of dumps(), raise BadSignature if signature fails.

django_q/management/commands/qmonitor.py

@@ -10,12 +10,12 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.add_argument(
-            '--run-once',
+            "--run-once",
-            action='store_true',
+            action="store_true",
-            dest='run_once',
+            dest="run_once",
             default=False,
-            help='Run once and then stop.',
+            help="Run once and then stop.",
         )
 
     def handle(self, *args, **options):
-        monitor(run_once=options.get('run_once', False))
+        monitor(run_once=options.get("run_once", False))

django_q/migrations/0011_auto_20200628_1055.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.7 on 2020-06-28 10:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_q', '0010_auto_20200610_0856'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='schedule',
+            name='cron',
+            field=models.CharField(blank=True, help_text='Cron expression', max_length=100, null=True),
+        ),
+        migrations.AlterField(
+            model_name='schedule',
+            name='schedule_type',
+            field=models.CharField(choices=[('O', 'Once'), ('I', 'Minutes'), ('H', 'Hourly'), ('D', 'Daily'), ('W', 'Weekly'), ('M', 'Monthly'), ('Q', 'Quarterly'), ('Y', 'Yearly'), ('C', 'Cron')], default='O', max_length=1, verbose_name='Schedule Type'),
+        ),
+    ]

django_q/models.py

@@ -1,15 +1,20 @@
+# Django
 from django import get_version
+from django.core.exceptions import ValidationError
+from django.db import models
 from django.template.defaultfilters import truncatechars
-
 from django.urls import reverse
+from django.utils import timezone
 from django.utils.html import format_html
 from django.utils.translation import gettext_lazy as _
-from django.db import models
+
-from django.utils import timezone
+# External
 from picklefield import PickledObjectField
 from picklefield.fields import dbsafe_decode
 
+# Local
 from django_q.signing import SignedPackage
+from django_q import croniter
 
 
 class Task(models.Model):
@@ -131,6 +136,16 @@ class Failure(Task):
         proxy = True
 
 
+# Optional Cron validator
+def validate_cron(value):
+    if not croniter:
+        raise ImportError(_("Please install croniter to enable cron expressions"))
+    try:
+        croniter.expand(value)
+    except ValueError as e:
+        raise ValidationError(e)
+
+
 class Schedule(models.Model):
     name = models.CharField(max_length=100, null=True, blank=True)
     func = models.CharField(max_length=256, help_text="e.g. module.tasks.function")
@@ -152,6 +167,7 @@ class Schedule(models.Model):
     MONTHLY = "M"
     QUARTERLY = "Q"
     YEARLY = "Y"
+    CRON = "C"
     TYPE = (
         (ONCE, _("Once")),
         (MINUTES, _("Minutes")),
@@ -161,6 +177,7 @@ class Schedule(models.Model):
         (MONTHLY, _("Monthly")),
         (QUARTERLY, _("Quarterly")),
         (YEARLY, _("Yearly")),
+        (CRON, _("Cron")),
     )
     schedule_type = models.CharField(
         max_length=1, choices=TYPE, default=TYPE[0][0], verbose_name=_("Schedule Type")
@@ -174,6 +191,13 @@ class Schedule(models.Model):
     next_run = models.DateTimeField(
         verbose_name=_("Next Run"), default=timezone.now, null=True
     )
+    cron = models.CharField(
+        max_length=100,
+        null=True,
+        blank=True,
+        validators=[validate_cron],
+        help_text=_("Cron expression"),
+    )
     task = models.CharField(max_length=100, null=True, editable=False)
 
     def success(self):

django_q/tasks.py

@@ -88,6 +88,7 @@ def schedule(func, *args, **kwargs):
     :param repeats: how many times to repeat. 0=never, -1=always.
     :param next_run: Next scheduled run.
     :type next_run: datetime.datetime
+    :param cron: optional cron expression
     :param kwargs: function keyword arguments.
     :return: the schedule object.
     :rtype: Schedule
@@ -98,13 +99,14 @@ def schedule(func, *args, **kwargs):
     minutes = kwargs.pop("minutes", None)
     repeats = kwargs.pop("repeats", -1)
     next_run = kwargs.pop("next_run", timezone.now())
+    cron = kwargs.pop("cron", None)
 
     # check for name duplicates instead of am unique constraint
     if name and Schedule.objects.filter(name=name).exists():
         raise IntegrityError("A schedule with the same name already exists.")
 
     # create and return the schedule
-    return Schedule.objects.create(
+    s = Schedule(
         name=name,
         func=func,
         hook=hook,
@@ -114,7 +116,12 @@ def schedule(func, *args, **kwargs):
         minutes=minutes,
         repeats=repeats,
         next_run=next_run,
+        cron=cron,
     )
+    # make sure we trigger validation
+    s.full_clean()
+    s.save()
+    return s
 
 
 def result(task_id, wait=0, cached=Conf.CACHED):

django_q/tests/test_brokers.py

@@ -11,72 +11,72 @@ from django_q.humanhash import uuid
 
 def test_broker(monkeypatch):
     broker = Broker()
-    broker.enqueue('test')
+    broker.enqueue("test")
     broker.dequeue()
     broker.queue_size()
     broker.lock_size()
     broker.purge_queue()
-    broker.delete('id')
+    broker.delete("id")
     broker.delete_queue()
-    broker.acknowledge('test')
+    broker.acknowledge("test")
     broker.ping()
     broker.info()
     # stats
-    assert broker.get_stat('test_1') is None
+    assert broker.get_stat("test_1") is None
-    broker.set_stat('test_1', 'test', 3)
+    broker.set_stat("test_1", "test", 3)
-    assert broker.get_stat('test_1') == 'test'
+    assert broker.get_stat("test_1") == "test"
-    assert broker.get_stats('test:*')[0] == 'test'
+    assert broker.get_stats("test:*")[0] == "test"
     # stats with no cache
-    monkeypatch.setattr(Conf, 'CACHE', 'not_configured')
+    monkeypatch.setattr(Conf, "CACHE", "not_configured")
     broker.cache = broker.get_cache()
-    assert broker.get_stat('test_1') is None
+    assert broker.get_stat("test_1") is None
-    broker.set_stat('test_1', 'test', 3)
+    broker.set_stat("test_1", "test", 3)
-    assert broker.get_stat('test_1') is None
+    assert broker.get_stat("test_1") is None
-    assert broker.get_stats('test:*') is None
+    assert broker.get_stats("test:*") is None
 
 
 def test_redis(monkeypatch):
-    monkeypatch.setattr(Conf, 'DJANGO_REDIS', None)
+    monkeypatch.setattr(Conf, "DJANGO_REDIS", None)
     broker = get_broker()
     assert broker.ping() is True
     assert broker.info() is not None
-    monkeypatch.setattr(Conf, 'REDIS', {'host': '127.0.0.1', 'port': 7799})
+    monkeypatch.setattr(Conf, "REDIS", {"host": "127.0.0.1", "port": 7799})
     broker = get_broker()
     with pytest.raises(Exception):
         broker.ping()
-    monkeypatch.setattr(Conf, 'REDIS', 'redis://127.0.0.1:7799')
+    monkeypatch.setattr(Conf, "REDIS", "redis://127.0.0.1:7799")
     broker = get_broker()
     with pytest.raises(Exception):
         broker.ping()
 
 
 def test_custom(monkeypatch):
-    monkeypatch.setattr(Conf, 'BROKER_CLASS', 'brokers.redis_broker.Redis')
+    monkeypatch.setattr(Conf, "BROKER_CLASS", "brokers.redis_broker.Redis")
     broker = get_broker()
     assert broker.ping() is True
     assert broker.info() is not None
-    assert broker.__class__.__name__ == 'Redis'
+    assert broker.__class__.__name__ == "Redis"
 
 
 def test_disque(monkeypatch):
-    monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7711'])
+    monkeypatch.setattr(Conf, "DISQUE_NODES", ["127.0.0.1:7711"])
     # check broker
-    broker = get_broker(list_key='disque_test')
+    broker = get_broker(list_key="disque_test")
     assert broker.ping() is True
     assert broker.info() is not None
     # clear before we start
     broker.delete_queue()
     # async_task
-    broker.enqueue('test')
+    broker.enqueue("test")
     assert broker.queue_size() == 1
     # dequeue
     task = broker.dequeue()[0]
-    assert task[1] == 'test'
+    assert task[1] == "test"
     broker.acknowledge(task[0])
     assert broker.queue_size() == 0
     # Retry test
-    monkeypatch.setattr(Conf, 'RETRY', 1)
+    monkeypatch.setattr(Conf, "RETRY", 1)
-    broker.enqueue('test')
+    broker.enqueue("test")
     assert broker.queue_size() == 1
     broker.dequeue()
     assert broker.queue_size() == 0
@@ -88,17 +88,17 @@ def test_disque(monkeypatch):
     sleep(1.5)
     assert broker.queue_size() == 0
     # delete job
-    task_id = broker.enqueue('test')
+    task_id = broker.enqueue("test")
     broker.delete(task_id)
     assert broker.dequeue() is None
     # fail
-    task_id = broker.enqueue('test')
+    task_id = broker.enqueue("test")
     broker.fail(task_id)
     # bulk test
     for i in range(5):
-        broker.enqueue('test')
+        broker.enqueue("test")
-    monkeypatch.setattr(Conf, 'BULK', 5)
+    monkeypatch.setattr(Conf, "BULK", 5)
-    monkeypatch.setattr(Conf, 'DISQUE_FASTACK', True)
+    monkeypatch.setattr(Conf, "DISQUE_FASTACK", True)
     tasks = broker.dequeue()
     for task in tasks:
         assert task is not None
@@ -106,35 +106,46 @@ def test_disque(monkeypatch):
     # test duplicate acknowledge
     broker.acknowledge(task[0])
     # delete queue
-    broker.enqueue('test')
+    broker.enqueue("test")
-    broker.enqueue('test')
+    broker.enqueue("test")
     broker.delete_queue()
     assert broker.queue_size() == 0
     # connection test
-    monkeypatch.setattr(Conf, 'DISQUE_NODES', ['127.0.0.1:7798', '127.0.0.1:7799'])
+    monkeypatch.setattr(Conf, "DISQUE_NODES", ["127.0.0.1:7798", "127.0.0.1:7799"])
+    with pytest.raises(redis.exceptions.ConnectionError):
+        broker.get_connection()
+    # connection test with no nodes
+    monkeypatch.setattr(Conf, "DISQUE_NODES", None)
     with pytest.raises(redis.exceptions.ConnectionError):
         broker.get_connection()
 
 
-@pytest.mark.skipif(not os.getenv('IRON_MQ_TOKEN'),
+@pytest.mark.skipif(
-                    reason="requires IronMQ credentials")
+    not os.getenv("IRON_MQ_TOKEN"), reason="requires IronMQ credentials"
+)
 def test_ironmq(monkeypatch):
-    monkeypatch.setattr(Conf, 'IRON_MQ', {'token': os.getenv('IRON_MQ_TOKEN'),
+    monkeypatch.setattr(
-                                          'project_id': os.getenv('IRON_MQ_PROJECT_ID')})
+        Conf,
+        "IRON_MQ",
+        {
+            "token": os.getenv("IRON_MQ_TOKEN"),
+            "project_id": os.getenv("IRON_MQ_PROJECT_ID"),
+        },
+    )
     # check broker
     broker = get_broker(list_key=uuid()[0])
     assert broker.ping() is True
     assert broker.info() is not None
     # initialize the queue
-    broker.enqueue('test')
+    broker.enqueue("test")
     # clear before we start
     broker.purge_queue()
     assert broker.queue_size() == 0
     # async_task
-    broker.enqueue('test')
+    broker.enqueue("test")
     # dequeue
     task = broker.dequeue()[0]
-    assert task[1] == 'test'
+    assert task[1] == "test"
     broker.acknowledge(task[0])
     assert broker.dequeue() is None
     # Retry test
@@ -148,16 +159,16 @@ def test_ironmq(monkeypatch):
     # broker.acknowledge(task[0])
     # sleep(3)
     # delete job
-    task_id = broker.enqueue('test')
+    task_id = broker.enqueue("test")
     broker.delete(task_id)
     assert broker.dequeue() is None
     # fail
-    task_id = broker.enqueue('test')
+    task_id = broker.enqueue("test")
     broker.fail(task_id)
     # bulk test
     for i in range(5):
-        broker.enqueue('test')
+        broker.enqueue("test")
-    monkeypatch.setattr(Conf, 'BULK', 5)
+    monkeypatch.setattr(Conf, "BULK", 5)
     tasks = broker.dequeue()
     for task in tasks:
         assert task is not None
@@ -165,62 +176,69 @@ def test_ironmq(monkeypatch):
     # duplicate acknowledge
     broker.acknowledge(task[0])
     # delete queue
-    broker.enqueue('test')
+    broker.enqueue("test")
-    broker.enqueue('test')
+    broker.enqueue("test")
     broker.purge_queue()
     assert broker.dequeue() is None
     broker.delete_queue()
 
 
-@pytest.mark.skipif(not os.getenv('AWS_ACCESS_KEY_ID'),
+@pytest.mark.skipif(
-                    reason="requires AWS credentials")
+    not os.getenv("AWS_ACCESS_KEY_ID"), reason="requires AWS credentials"
+)
 def canceled_sqs(monkeypatch):
-    monkeypatch.setattr(Conf, 'SQS', {'aws_region': os.getenv('AWS_REGION'),
+    monkeypatch.setattr(
-                                      'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'),
+        Conf,
-                                      'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY')})
+        "SQS",
+        {
+            "aws_region": os.getenv("AWS_REGION"),
+            "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"),
+            "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"),
+        },
+    )
     # check broker
     broker = get_broker(list_key=uuid()[0])
     assert broker.ping() is True
     assert broker.info() is not None
     assert broker.queue_size() == 0
     # async_task
-    broker.enqueue('test')
+    broker.enqueue("test")
     # dequeue
     task = broker.dequeue()[0]
-    assert task[1] == 'test'
+    assert task[1] == "test"
     broker.acknowledge(task[0])
     assert broker.dequeue() is None
     # Retry test
-    monkeypatch.setattr(Conf, 'RETRY', 1)
+    monkeypatch.setattr(Conf, "RETRY", 1)
-    broker.enqueue('test')
+    broker.enqueue("test")
     sleep(2)
     # Sometimes SQS is not linear
     task = broker.dequeue()
     if not task:
-        pytest.skip('SQS being weird')
+        pytest.skip("SQS being weird")
     task = task[0]
     assert len(task) > 0
     broker.acknowledge(task[0])
     sleep(2)
     # delete job
-    monkeypatch.setattr(Conf, 'RETRY', 60)
+    monkeypatch.setattr(Conf, "RETRY", 60)
-    broker.enqueue('test')
+    broker.enqueue("test")
     sleep(1)
     task = broker.dequeue()
     if not task:
-        pytest.skip('SQS being weird')
+        pytest.skip("SQS being weird")
     task_id = task[0][0]
     broker.delete(task_id)
     assert broker.dequeue() is None
     # fail
-    broker.enqueue('test')
+    broker.enqueue("test")
     while task is None:
         task = broker.dequeue()[0]
     broker.fail(task[0])
     # bulk test
     for i in range(10):
-        broker.enqueue('test')
+        broker.enqueue("test")
-    monkeypatch.setattr(Conf, 'BULK', 12)
+    monkeypatch.setattr(Conf, "BULK", 12)
     tasks = broker.dequeue()
     for task in tasks:
         assert task is not None
@@ -229,31 +247,31 @@ def canceled_sqs(monkeypatch):
     broker.acknowledge(task[0])
     assert broker.lock_size() == 0
     # delete queue
-    broker.enqueue('test')
+    broker.enqueue("test")
     broker.purge_queue()
     broker.delete_queue()
 
 
 @pytest.mark.django_db
 def test_orm(monkeypatch):
-    monkeypatch.setattr(Conf, 'ORM', 'default')
+    monkeypatch.setattr(Conf, "ORM", "default")
     # check broker
-    broker = get_broker(list_key='orm_test')
+    broker = get_broker(list_key="orm_test")
     assert broker.ping() is True
     assert broker.info() is not None
     # clear before we start
     broker.delete_queue()
     # async_task
-    broker.enqueue('test')
+    broker.enqueue("test")
     assert broker.queue_size() == 1
     # dequeue
     task = broker.dequeue()[0]
-    assert task[1] == 'test'
+    assert task[1] == "test"
     broker.acknowledge(task[0])
     assert broker.queue_size() == 0
     # Retry test
-    monkeypatch.setattr(Conf, 'RETRY', 1)
+    monkeypatch.setattr(Conf, "RETRY", 1)
-    broker.enqueue('test')
+    broker.enqueue("test")
     assert broker.queue_size() == 1
     broker.dequeue()
     assert broker.queue_size() == 0
@@ -265,16 +283,16 @@ def test_orm(monkeypatch):
     sleep(1.5)
     assert broker.queue_size() == 0
     # delete job
-    task_id = broker.enqueue('test')
+    task_id = broker.enqueue("test")
     broker.delete(task_id)
     assert broker.dequeue() is None
     # fail
-    task_id = broker.enqueue('test')
+    task_id = broker.enqueue("test")
     broker.fail(task_id)
     # bulk test
     for i in range(5):
-        broker.enqueue('test')
+        broker.enqueue("test")
-    monkeypatch.setattr(Conf, 'BULK', 5)
+    monkeypatch.setattr(Conf, "BULK", 5)
     tasks = broker.dequeue()
     assert broker.lock_size() == Conf.BULK
     for task in tasks:
@@ -285,32 +303,32 @@ def test_orm(monkeypatch):
     # test duplicate acknowledge
     broker.acknowledge(task[0])
     # delete queue
-    broker.enqueue('test')
+    broker.enqueue("test")
-    broker.enqueue('test')
+    broker.enqueue("test")
     broker.delete_queue()
     assert broker.queue_size() == 0
 
 
 @pytest.mark.django_db
 def test_mongo(monkeypatch):
-    monkeypatch.setattr(Conf, 'MONGO', {'host': '127.0.0.1', 'port': 27017})
+    monkeypatch.setattr(Conf, "MONGO", {"host": "127.0.0.1", "port": 27017})
     # check broker
-    broker = get_broker(list_key='mongo_test')
+    broker = get_broker(list_key="mongo_test")
     assert broker.ping() is True
     assert broker.info() is not None
     # clear before we start
     broker.delete_queue()
     # async_task
-    broker.enqueue('test')
+    broker.enqueue("test")
     assert broker.queue_size() == 1
     # dequeue
     task = broker.dequeue()[0]
-    assert task[1] == 'test'
+    assert task[1] == "test"
     broker.acknowledge(task[0])
     assert broker.queue_size() == 0
     # Retry test
-    monkeypatch.setattr(Conf, 'RETRY', 1)
+    monkeypatch.setattr(Conf, "RETRY", 1)
-    broker.enqueue('test')
+    broker.enqueue("test")
     assert broker.queue_size() == 1
     broker.dequeue()
     assert broker.queue_size() == 0
@@ -322,15 +340,15 @@ def test_mongo(monkeypatch):
     sleep(1.5)
     assert broker.queue_size() == 0
     # delete job
-    task_id = broker.enqueue('test')
+    task_id = broker.enqueue("test")
     broker.delete(task_id)
     assert broker.dequeue() is None
     # fail
-    task_id = broker.enqueue('test')
+    task_id = broker.enqueue("test")
     broker.fail(task_id)
     # bulk test
     for i in range(5):
-        broker.enqueue('test')
+        broker.enqueue("test")
     tasks = []
     for i in range(5):
         tasks.append(broker.dequeue()[0])
@@ -343,8 +361,8 @@ def test_mongo(monkeypatch):
     # test duplicate acknowledge
     broker.acknowledge(task[0])
     # delete queue
-    broker.enqueue('test')
+    broker.enqueue("test")
-    broker.enqueue('test')
+    broker.enqueue("test")
     broker.purge_queue()
     broker.delete_queue()
     assert broker.queue_size() == 0

django_q/tests/test_cluster.py

@@ -142,6 +142,8 @@ def test_enqueue(broker, admin_user):
     # q_options and save opt_out test
     k = async_task('django_q.tests.tasks.get_user_id', admin_user,
                    q_options={'broker': broker, 'group': 'test_k', 'save': False, 'timeout': 90})
+    # test unicode
+    assert Task(name='Amalia').__unicode__()=='Amalia'
     # check if everything has a task id
     assert isinstance(a, str)
     assert isinstance(b, str)

django_q/tests/test_commands.py

@@ -16,3 +16,4 @@ def test_qmonitor():
 def test_qinfo():
     call_command('qinfo')
     call_command('qinfo', config=True)
+    call_command('qinfo', ids=True)

django_q/tests/test_monitor.py

@@ -4,7 +4,7 @@ import uuid
 from django_q.tasks import async_task
 from django_q.brokers import get_broker
 from django_q.cluster import Cluster
-from django_q.monitor import monitor, info
+from django_q.monitor import monitor, info, get_ids
 from django_q.status import Stat
 from django_q.conf import Conf
 
@@ -16,6 +16,7 @@ def test_monitor(monkeypatch):
     c = Cluster()
     c.start()
     stats = monitor(run_once=True)
+    assert get_ids() is True
     c.stop()
     assert len(stats) > 0
     found_c = False

django_q/tests/test_scheduler.py

@@ -3,14 +3,15 @@ from multiprocessing import Event, Value
 
 import arrow
 import pytest
+from django.core.exceptions import ValidationError
 from django.db import IntegrityError
 from django.utils import timezone
 
 from django_q.brokers import get_broker
 from django_q.cluster import pusher, worker, monitor, scheduler
 from django_q.conf import Conf
-from django_q.tasks import Schedule, fetch, schedule as create_schedule
 from django_q.queues import Queue
+from django_q.tasks import Schedule, fetch, schedule as create_schedule
 
 
 @pytest.fixture
@@ -91,8 +92,25 @@ def test_scheduler(broker, monkeypatch):
                                       schedule_type=Schedule.MINUTES,
                                       minutes=10)
     assert hasattr(minute_schedule, 'pk') is True
+    # Cron schedule
+    cron_schedule = create_schedule('django_q.tests.tasks.word_multiply',
+                                    2,
+                                    word='django',
+                                    schedule_type=Schedule.CRON,
+                                    cron="0 22 * * 1-5")
+    assert hasattr(cron_schedule, 'pk') is True
+    assert cron_schedule.full_clean() is None
+    assert cron_schedule.__unicode__() == 'django_q.tests.tasks.word_multiply'
+    with pytest.raises(ValidationError):
+        create_schedule('django_q.tests.tasks.word_multiply',
+                        2,
+                        word='django',
+                        schedule_type=Schedule.CRON,
+                        cron="0 22 * * 1-12")
     # All other types
     for t in Schedule.TYPE:
+        if t[0] == Schedule.CRON:
+            continue
         schedule = create_schedule('django_q.tests.tasks.word_multiply',
                                    2,
                                    word='django',

docs/install.rst

@@ -88,6 +88,13 @@ Optional
 
 
 
+- `Croniter <https://github.com/kiorky/croniter>`__ is an optional package that is used to parse cron expressions for the scheduler::
+
+    $ pip install croniter
+
+
+
+
 Add-ons
 -------
 - `django-q-rollbar <https://github.com/danielwelch/django-q-rollbar>`__ is a Rollbar error reporter::

docs/schedules.rst

@@ -45,6 +45,13 @@ You can manage them through the :ref:`admin_page` or directly from your code wit
              repeats=24,
              next_run=arrow.utcnow().replace(hour=18, minute=0))
 
+    # Use a cron expression
+    schedule('math.hypot',
+             3, 4,
+             schedule_type=Schedule.CRON,
+             cron = '0 22 * * 1-5')
+
+
 
 Missed schedules
 ----------------
@@ -99,8 +106,9 @@ Reference
     :param args: arguments for the scheduled function.
     :param str name: An optional name for your schedule.
     :param str hook: optional result hook function. Dotted strings only.
-    :param str schedule_type: (O)nce, M(I)nutes, (H)ourly, (D)aily, (W)eekly, (M)onthly, (Q)uarterly, (Y)early or :attr:`Schedule.TYPE`
+    :param str schedule_type: (O)nce, M(I)nutes, (H)ourly, (D)aily, (W)eekly, (M)onthly, (Q)uarterly, (Y)early or (C)ron :attr:`Schedule.TYPE`
     :param int minutes: Number of minutes for the Minutes type.
+    :param str cron: Cron expression for the Cron type.
     :param int repeats: Number of times to repeat schedule. -1=Always, 0=Never, n =n.
     :param datetime next_run: Next or first scheduled execution datetime.
     :param dict q_options: options passed to async_task for this schedule
@@ -140,7 +148,7 @@ Reference
 
     .. py:attribute:: TYPE
 
-    :attr:`ONCE`, :attr:`MINUTES`, :attr:`HOURLY`, :attr:`DAILY`, :attr:`WEEKLY`, :attr:`MONTHLY`, :attr:`QUARTERLY`, :attr:`YEARLY`
+    :attr:`ONCE`, :attr:`MINUTES`, :attr:`HOURLY`, :attr:`DAILY`, :attr:`WEEKLY`, :attr:`MONTHLY`, :attr:`QUARTERLY`, :attr:`YEARLY`, :attr:`CRON`
 
 
     .. py:attribute:: minutes
@@ -148,6 +156,10 @@ Reference
     The number of minutes the :attr:`MINUTES` schedule should use.
     Is ignored for other schedule types.
 
+    .. py:attribute:: cron
+
+    A cron string describing the schedule. You need the optional `croniter` package installed for this.
+
     .. py:attribute:: repeats
 
     Number of times to repeat the schedule. -1=Always, 0=Never, n =n.
@@ -208,3 +220,9 @@ Reference
 
     `'Y'` only runs once a year. The same caution as with months apply;
     If you set this to february 29th, it will run on february 28th in the following years.
+
+    .. py:attribute:: CRON
+
+    `'C'` uses the optional `croniter` package to determine a schedule based a cron expression.
+
+

poetry.lock

@@ -71,10 +71,10 @@ description = "The AWS SDK for Python"
 name = "boto3"
 optional = true
 python-versions = "*"
-version = "1.14.12"
+version = "1.14.14"
 
 [package.dependencies]
-botocore = ">=1.17.12,<1.18.0"
+botocore = ">=1.17.14,<1.18.0"
 jmespath = ">=0.7.1,<1.0.0"
 s3transfer = ">=0.3.0,<0.4.0"
 
@@ -84,7 +84,7 @@ description = "Low-level, data-driven core of boto 3."
 name = "botocore"
 optional = true
 python-versions = "*"
-version = "1.17.12"
+version = "1.17.14"
 
 [package.dependencies]
 docutils = ">=0.10,<0.16"
@@ -120,13 +120,25 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
 version = "0.4.3"
 
+[[package]]
+category = "main"
+description = "croniter provides iteration for datetime object with cron like format"
+name = "croniter"
+optional = true
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "0.3.34"
+
+[package.dependencies]
+natsort = "*"
+python-dateutil = "*"
+
 [[package]]
 category = "main"
 description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design."
 name = "django"
 optional = false
 python-versions = ">=3.6"
-version = "3.0.7"
+version = "3.0.8"
 
 [package.dependencies]
 asgiref = ">=3.2,<4.0"
@@ -270,6 +282,18 @@ optional = false
 python-versions = ">=3.5"
 version = "8.4.0"
 
+[[package]]
+category = "main"
+description = "Simple yet flexible natural sorting in Python."
+name = "natsort"
+optional = true
+python-versions = ">=3.4"
+version = "7.0.1"
+
+[package.extras]
+fast = ["fastnumbers (>=2.0.0)"]
+icu = ["PyICU (>=1.0.0)"]
+
 [[package]]
 category = "dev"
 description = "Core utilities for Python packages"
@@ -497,7 +521,7 @@ rollbar = ["django-q-rollbar"]
 sentry = []
 
 [metadata]
-content-hash = "d89c929e8b6951968228b1aab1aa5c0a7c014acceb3b986bbaa3e41df57d8b39"
+content-hash = "6d89bff8bd465aa4a5facd812172a970bf5edebdd8099141054f72aa63260c2a"
 python-versions = ">=3.6"
 
 [metadata.files]
@@ -526,12 +550,12 @@ blessed = [
     {file = "blessed-1.17.8.tar.gz", hash = "sha256:7671d057b2df6ddbefd809009fb08feb2f8d2d163d240b5e765088a90519b2f1"},
 ]
 boto3 = [
-    {file = "boto3-1.14.12-py2.py3-none-any.whl", hash = "sha256:7daad26a008c91dd7b82fde17d246d1fe6e4b3813426689ef8bac9017a277cfb"},
+    {file = "boto3-1.14.14-py2.py3-none-any.whl", hash = "sha256:4c2f5f9f28930e236845e2cddbe01cb093ca96dc1f5c6e2b2b254722018a2268"},
-    {file = "boto3-1.14.12.tar.gz", hash = "sha256:2616351c98eec18d20a1d64b33355c86cd855ac96219d1b8428c9bfc590bde53"},
+    {file = "boto3-1.14.14.tar.gz", hash = "sha256:87beffba2360b8077413f2d473cb828d0a5bda513bd1d6fb7b137c57b686aeb6"},
 ]
 botocore = [
-    {file = "botocore-1.17.12-py2.py3-none-any.whl", hash = "sha256:45934d880378777cefeca727f369d1f5aebf6b254e9be58e7c77dd0b059338bb"},
+    {file = "botocore-1.17.14-py2.py3-none-any.whl", hash = "sha256:6a2e9768dad8ae9771302d5922b977dca6bb9693f9b6a5f6ed0e7ac375e2ca40"},
-    {file = "botocore-1.17.12.tar.gz", hash = "sha256:a94e0e2307f1b9fe3a84660842909cd2680b57a9fc9fb0c3a03b0afb2eadbe21"},
+    {file = "botocore-1.17.14.tar.gz", hash = "sha256:96d668ae5246d236ea83e4586349552d6584e8b1551ae2fccc0bd4ed528a746f"},
 ]
 certifi = [
     {file = "certifi-2020.6.20-py2.py3-none-any.whl", hash = "sha256:8fc0819f1f30ba15bdb34cceffb9ef04d99f420f68eb75d901e9560b8749fc41"},
@@ -545,9 +569,13 @@ colorama = [
     {file = "colorama-0.4.3-py2.py3-none-any.whl", hash = "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff"},
     {file = "colorama-0.4.3.tar.gz", hash = "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1"},
 ]
+croniter = [
+    {file = "croniter-0.3.34-py2.py3-none-any.whl", hash = "sha256:15597ef0639f8fbab09cbf8c277fa8c65c8b9dbe818c4b2212f95dbc09c6f287"},
+    {file = "croniter-0.3.34.tar.gz", hash = "sha256:7186b9b464f45cf3d3c83a18bc2344cc101d7b9fd35a05f2878437b14967e964"},
+]
 django = [
-    {file = "Django-3.0.7-py3-none-any.whl", hash = "sha256:e1630333248c9b3d4e38f02093a26f1e07b271ca896d73097457996e0fae12e8"},
+    {file = "Django-3.0.8-py3-none-any.whl", hash = "sha256:5457fc953ec560c5521b41fad9e6734a4668b7ba205832191bbdff40ec61073c"},
-    {file = "Django-3.0.7.tar.gz", hash = "sha256:5052b34b34b3425233c682e0e11d658fd6efd587d11335a0203d827224ada8f2"},
+    {file = "Django-3.0.8.tar.gz", hash = "sha256:31a5fbbea5fc71c99e288ec0b2f00302a0a92c44b13ede80b73a6a4d6d205582"},
 ]
 django-picklefield = [
     {file = "django-picklefield-3.0.1.tar.gz", hash = "sha256:15ccba592ca953b9edf9532e64640329cd47b136b7f8f10f2939caa5f9ce4287"},
@@ -638,6 +666,10 @@ more-itertools = [
     {file = "more-itertools-8.4.0.tar.gz", hash = "sha256:68c70cc7167bdf5c7c9d8f6954a7837089c6a36bf565383919bb595efb8a17e5"},
     {file = "more_itertools-8.4.0-py3-none-any.whl", hash = "sha256:b78134b2063dd214000685165d81c154522c3ee0a1c0d4d113c80361c234c5a2"},
 ]
+natsort = [
+    {file = "natsort-7.0.1-py3-none-any.whl", hash = "sha256:d3fd728a3ceb7c78a59aa8539692a75e37cbfd9b261d4d702e8016639820f90a"},
+    {file = "natsort-7.0.1.tar.gz", hash = "sha256:a633464dc3a22b305df0f27abcb3e83515898aa1fd0ed2f9726c3571a27258cf"},
+]
 packaging = [
     {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"},
     {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"},

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "django-q"
-version = "1.2.4"
+version = "1.3.0"
 description = "A multiprocessing distributed task queue for Django"
 authors = ["Ilan Steemers <koed00@gmail.com>"]
 license = "MIT"
@@ -52,6 +52,7 @@ django-redis = {version = "^4.12.1", optional = true}
 iron-mq = {version = "^0.9", optional = true}
 boto3 = {version = "^1.14.12", optional = true}
 pymongo = {version = "^3.10.1", optional = true}
+croniter = {version = "^0.3.34", optional = true}
 
 [tool.poetry.dev-dependencies]
 pytest = "^5.4.2"

requirements.in

@@ -8,3 +8,4 @@ django-redis
 iron-mq
 boto3
 pymongo
+croniter

requirements.txt

@@ -7,22 +7,24 @@
 arrow==0.15.7             # via -r requirements.in
 asgiref==3.2.10           # via django
 blessed==1.17.8           # via -r requirements.in
-boto3==1.14.12            # via -r requirements.in
+boto3==1.14.14            # via -r requirements.in
-botocore==1.17.12         # via boto3, s3transfer
+botocore==1.17.14         # via boto3, s3transfer
 certifi==2020.6.20        # via requests
 chardet==3.0.4            # via requests
+croniter==0.3.34          # via -r requirements.in
 django-picklefield==3.0.1  # via -r requirements.in
 django-redis==4.12.1      # via -r requirements.in
-django==3.0.7             # via django-picklefield, django-redis
+django==3.0.8             # via django-picklefield, django-redis
 docutils==0.15.2          # via botocore
 hiredis==1.0.1            # via -r requirements.in
 idna==2.10                # via requests
 iron-core==1.2.0          # via iron-mq
 iron-mq==0.9              # via -r requirements.in
 jmespath==0.10.0          # via boto3, botocore
+natsort==7.0.1            # via croniter
 psutil==5.7.0             # via -r requirements.in
 pymongo==3.10.1           # via -r requirements.in
-python-dateutil==2.8.1    # via arrow, botocore, iron-core
+python-dateutil==2.8.1    # via arrow, botocore, croniter, iron-core
 pytz==2020.1              # via django
 redis==3.5.3              # via -r requirements.in, django-redis
 requests==2.24.0          # via iron-core