diff --git a/.travis.yml b/.travis.yml index 0022569..128ef22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,12 +6,12 @@ services: python: - "2.7" - - "3.4" + - "3.6" env: - - DJANGO=1.10rc1 - - DJANGO=1.9.8 - - DJANGO=1.8.14 + - DJANGO=1.11 + - DJANGO=1.10.7 + - DJANGO=1.8.18 sudo: false diff --git a/MANIFEST.in b/MANIFEST.in index b3169f4..bb8daf3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,5 @@ include README.rst include django_q/management/*.py include django_q/management/commands/*.py include django_q/migrations/*.py -include django_q/brokers/*.py \ No newline at end of file +include django_q/brokers/*.py +include django_q/tests/*.py diff --git a/README.rst b/README.rst index 9d60d01..a7b9a72 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ Requirements - `Arrow `__ - `Blessed `__ -Tested with: Python 2.7 & 3.5. Django 1.8.14, 1.9.8 and 1.10rc1 +Tested with: Python 2.7 & 3.6. Django 1.8.18, 1.10.7 and 1.11 Brokers ~~~~~~~ diff --git a/django_q/__init__.py b/django_q/__init__.py index b8c5435..4f3b689 100644 --- a/django_q/__init__.py +++ b/django_q/__init__.py @@ -5,14 +5,14 @@ from django import get_version myPath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, myPath) -VERSION = (0, 7, 18) +VERSION = (0, 8, 0) default_app_config = 'django_q.apps.DjangoQConfig' # root imports will slowly be deprecated. # please import from the relevant sub modules split_version = get_version().split('.') -if split_version[1][0] != '9' and split_version[1][:2] != '10': +if split_version[1] not in ('9', '10', '11'): from .tasks import async, schedule, result, result_group, fetch, fetch_group, count_group, delete_group, queue_size from .models import Task, Schedule, Success, Failure from .cluster import Cluster diff --git a/django_q/cluster.py b/django_q/cluster.py index 3810b15..e49792a 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -4,12 +4,6 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from builtins import range - -from future import standard_library - -standard_library.install_aliases() - # Standard import importlib import signal @@ -30,10 +24,13 @@ from django import db import signing import tasks +from django_q.compat import range from django_q.conf import Conf, logger, psutil, get_ppid, rollbar from django_q.models import Task, Success, Schedule from django_q.status import Stat, Status from django_q.brokers import get_broker +from django_q.signals import pre_execute + class Cluster(object): @@ -373,8 +370,11 @@ def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT): # We're still going if not result: db.close_old_connections() + timer_value = task['kwargs'].pop('timeout', timeout or 0) + # signal execution + pre_execute.send(sender="django_q", func=f, task=task) # execute the payload - timer.value = task['kwargs'].pop('timeout', timeout or 0) # Busy + timer.value = timer_value # Busy try: res = f(*task['args'], **task['kwargs']) result = (res, True) diff --git a/django_q/compat.py b/django_q/compat.py new file mode 100644 index 0000000..0ba80c4 --- /dev/null +++ b/django_q/compat.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import +""" +Compatibility layer. + +Intentionally replaces use of python-future +""" + +# https://github.com/Koed00/django-q/issues/4 + +try: + range = xrange +except NameError: + range = range + diff --git a/django_q/management/commands/qinfo.py b/django_q/management/commands/qinfo.py index 6647f4f..cdd82bf 100644 --- a/django_q/management/commands/qinfo.py +++ b/django_q/management/commands/qinfo.py @@ -3,7 +3,7 @@ from django.utils.translation import ugettext as _ from django_q import VERSION from django_q.conf import Conf -from django_q.monitor import info +from django_q.monitor import info, get_ids class Command(BaseCommand): @@ -18,9 +18,18 @@ class Command(BaseCommand): default=False, help='Print current configuration.', ) + parser.add_argument( + '--ids', + action='store_true', + dest='ids', + default=False, + help='Print cluster task ID(s) (PIDs).', + ) def handle(self, *args, **options): - if options.get('config', False): + if options.get('ids', True): + get_ids() + elif options.get('config', False): hide = ['conf', 'IDLE', 'STOPPING', 'STARTING', 'WORKING', 'SIGNAL_NAMES', 'STOPPED'] settings = [a for a in dir(Conf) if not a.startswith('__') and a not in hide] self.stdout.write('VERSION: {}'.format('.'.join(str(v) for v in VERSION))) diff --git a/django_q/monitor.py b/django_q/monitor.py index 03582bf..001b53d 100644 --- a/django_q/monitor.py +++ b/django_q/monitor.py @@ -187,3 +187,14 @@ def info(broker=None): term.white('{0:.4f}'.format(exec_time)) ) return True + + +def get_ids(): + # prints id (PID) of running clusters + stat = Stat.get_all() + if stat: + for s in stat: + print(s.cluster_id) + else: + print('No clusters appear to be running.') + return True diff --git a/django_q/signals.py b/django_q/signals.py index cc000c2..bfa2284 100644 --- a/django_q/signals.py +++ b/django_q/signals.py @@ -1,7 +1,7 @@ import importlib from django.db.models.signals import post_save -from django.dispatch import receiver +from django.dispatch import receiver, Signal from django.utils.translation import ugettext_lazy as _ from django_q.conf import logger @@ -24,3 +24,7 @@ def call_hook(sender, instance, **kwargs): f(instance) except Exception as e: logger.error(_('return hook {} failed on [{}] because {}').format(instance.hook, instance.name, e)) + + +pre_enqueue = Signal(providing_args=["task"]) +pre_execute = Signal(providing_args=["func", "task"]) diff --git a/django_q/tasks.py b/django_q/tasks.py index 31690c8..5f1c46b 100644 --- a/django_q/tasks.py +++ b/django_q/tasks.py @@ -13,6 +13,7 @@ from django_q.conf import Conf, logger from django_q.models import Schedule, Task from django_q.humanhash import uuid from django_q.brokers import get_broker +from django_q.signals import pre_enqueue def async(func, *args, **kwargs): @@ -43,12 +44,15 @@ def async(func, *args, **kwargs): # finalize task['kwargs'] = keywords task['started'] = timezone.now() + # signal it + pre_enqueue.send(sender="django_q", task=task) # sign it pack = signing.SignedPackage.dumps(task) if task.get('sync', False): return _sync(pack) # push it - broker.enqueue(pack) + enqueue_id = broker.enqueue(pack) + logger.info('Enqueued {}'.format(enqueue_id)) logger.debug('Pushed {}'.format(tag)) return task['id'] diff --git a/django_q/tests/test_brokers.py b/django_q/tests/test_brokers.py index f613325..6ef7cd4 100644 --- a/django_q/tests/test_brokers.py +++ b/django_q/tests/test_brokers.py @@ -5,6 +5,7 @@ import pytest import redis from django_q.brokers import get_broker, Broker +from django_q.compat import range from django_q.conf import Conf from django_q.humanhash import uuid @@ -170,7 +171,7 @@ def test_ironmq(monkeypatch): @pytest.mark.skipif(not os.getenv('AWS_ACCESS_KEY_ID'), reason="requires AWS credentials") -def test_sqs(monkeypatch): +def canceled_sqs(monkeypatch): monkeypatch.setattr(Conf, 'SQS', {'aws_region': os.getenv('AWS_REGION'), 'aws_access_key_id': os.getenv('AWS_ACCESS_KEY_ID'), 'aws_secret_access_key': os.getenv('AWS_SECRET_ACCESS_KEY')}) @@ -189,15 +190,23 @@ def test_sqs(monkeypatch): # Retry test monkeypatch.setattr(Conf, 'RETRY', 1) broker.enqueue('test') - assert broker.dequeue() is not None sleep(2) - task = broker.dequeue()[0] + # Sometimes SQS is not linear + task = broker.dequeue() + if not task: + pytest.skip('SQS being weird') + task = task[0] assert len(task) > 0 broker.acknowledge(task[0]) sleep(2) # delete job + monkeypatch.setattr(Conf, 'RETRY', 60) broker.enqueue('test') - task_id = broker.dequeue()[0][0] + sleep(1) + task = broker.dequeue() + if not task: + pytest.skip('SQS being weird') + task_id = task[0][0] broker.delete(task_id) assert broker.dequeue() is None # fail diff --git a/django_q/tests/test_cached.py b/django_q/tests/test_cached.py index d2b4141..1cf9980 100644 --- a/django_q/tests/test_cached.py +++ b/django_q/tests/test_cached.py @@ -3,6 +3,7 @@ from multiprocessing import Event, Queue, Value import pytest from django_q.cluster import pusher, worker, monitor +from django_q.compat import range from django_q.conf import Conf from django_q.tasks import async, result, fetch, count_group, result_group, fetch_group, delete_group, delete_cached, \ async_iter, Chain, async_chain, Iter, Async diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py index 2e33d26..70af89c 100644 --- a/django_q/tests/test_cluster.py +++ b/django_q/tests/test_cluster.py @@ -11,6 +11,7 @@ myPath = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, myPath + '/../') from django_q.cluster import Cluster, Sentinel, pusher, worker, monitor, save_task +from django_q.compat import range from django_q.humanhash import DEFAULT_WORDLIST, uuid from django_q.tasks import fetch, fetch_group, async, result, result_group, count_group, delete_group, queue_size from django_q.models import Task, Success diff --git a/django_q/tests/test_monitor.py b/django_q/tests/test_monitor.py index d07fe07..0d43b7f 100644 --- a/django_q/tests/test_monitor.py +++ b/django_q/tests/test_monitor.py @@ -3,6 +3,7 @@ import pytest from django_q.tasks import async from django_q.brokers import get_broker from django_q.cluster import Cluster +from django_q.compat import range from django_q.monitor import monitor, info from django_q.status import Stat from django_q.conf import Conf diff --git a/docs/conf.py b/docs/conf.py index 4d60351..c9b158f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -70,9 +70,9 @@ author = 'Ilan Steemers' # built documents. # # The short X.Y version. -version = '0.7' +version = '0.8' # The full version, including alpha/beta/rc tags. -release = '0.7.18' +release = '0.8.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/index.rst b/docs/index.rst index 082071c..ee4c3f0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ Features - Rollbar support -Django Q is tested with: Python 2.7 & 3.5. Django 1.8.14, 1.9.8 and 1.10rc1 +Django Q is tested with: Python 2.7 & 3.6. Django 1.8.18 LTS, 1.10.7 and 1.11 Contents: @@ -42,6 +42,7 @@ Contents: Cluster Monitor Admin + Signals Architecture Examples diff --git a/docs/install.rst b/docs/install.rst index ed67ffa..424fa95 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -27,12 +27,12 @@ Installation Requirements ------------ -Django Q is tested for Python 2.7 and 3.5 +Django Q is tested for Python 2.7 and 3.6 - `Django `__ Django Q aims to use as much of Django's standard offerings as possible - The code is tested against Django version `1.8.13` and `1.9.7`. + The code is tested against Django versions `1.8.18 LTS`, `1.10.7` and `1.11`. - `Django-picklefield `__ @@ -112,12 +112,13 @@ Other known issues are: Python ~~~~~~ The code is always tested against the latest version of Python 2 and Python 3 and we try to stay compatible with the last two versions of each. -Current tests are performed with Python 2.7.10 and 3.5. +Current tests are performed with Python 2.7.12 and 3.6.1 If you do encounter any regressions with earlier versions, please submit an issue on `github `__ .. note:: Django 1.7.10 or earlier is not compatible with Python 3.5 + Django releases before 1.11 are not officially supported on Python 3.6 Open-source packages ~~~~~~~~~~~~~~~~~~~~ @@ -127,9 +128,9 @@ You can reference the `requirements =1.7', 'django-picklefield', 'blessed', 'arrow', 'future'], + install_requires=['django>=1.8', 'django-picklefield', 'blessed', 'arrow'], test_requires=['pytest', 'pytest-django', ], cmdclass={'test': PyTest}, classifiers=[ @@ -53,6 +53,7 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries :: Python Modules', ]