From d554e0e9423eaa194675649cd75ee1ee3a88598d Mon Sep 17 00:00:00 2001
From: ilan <koed00@gmail.com>
Date: Mon, 4 Dec 2017 12:15:40 +0100
Subject: [PATCH 1/9] Updates Travis to test for Django 2 and the LTS versions

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index a0a5579..19ea9ed 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,8 +9,8 @@ python:
   - "3.6"
 
 env:
-    - DJANGO=1.11.6
-    - DJANGO=1.10.8
+    - DJANGO=2.0
+    - DJANGO=1.11.8
     - DJANGO=1.8.18
 
 sudo: false

From a318fc13c520ddaa18875c7d5b7f4ee19234ae87 Mon Sep 17 00:00:00 2001
From: ilan <koed00@gmail.com>
Date: Tue, 5 Dec 2017 10:19:40 +0100
Subject: [PATCH 2/9] Removing Python 2.7 tests

---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 19ea9ed..f7345b7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,6 @@ services:
   - mongodb
 
 python:
-  - "2.7"
   - "3.6"
 
 env:

From 805390fb9ed961dc03632a71b53f9fe71b47541a Mon Sep 17 00:00:00 2001
From: Ronald van Zon <ronald.vanzon@nedap.com>
Date: Tue, 12 Dec 2017 15:09:03 +0100
Subject: [PATCH 3/9] Remove include for admin site urls

---
 django_q/tests/urls.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/django_q/tests/urls.py b/django_q/tests/urls.py
index eb91d28..ede2ec9 100644
--- a/django_q/tests/urls.py
+++ b/django_q/tests/urls.py
@@ -1,6 +1,6 @@
-from django.conf.urls import include, url
+from django.conf.urls import url
 from django.contrib import admin
 
 urlpatterns = [
-    url(r'^admin/', include(admin.site.urls)),
+    url(r'^admin/', admin.site.urls),
 ]

From a4086bffd706cc03c7346002976a48192f068083 Mon Sep 17 00:00:00 2001
From: Ronald van Zon <ronald.vanzon@nedap.com>
Date: Tue, 12 Dec 2017 15:15:40 +0100
Subject: [PATCH 4/9] Update MIDDLEWARE_CLASSES to represent new style.

In Django 1.10 MIDDLEWARE_CLASSES is set to deprecated and is removed in Django 2.0.
The SessionAuthenticationMiddleware is also removed from Django 2.0 and was already
unconditionally enabled in Django 1.10.
---
 django_q/tests/settings.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/django_q/tests/settings.py b/django_q/tests/settings.py
index 8bad66c..468d663 100644
--- a/django_q/tests/settings.py
+++ b/django_q/tests/settings.py
@@ -1,4 +1,5 @@
 import os
+import django
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -28,15 +29,24 @@ INSTALLED_APPS = (
     'django_redis'
 )
 
-MIDDLEWARE_CLASSES = (
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.common.CommonMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
-)
+if django.VERSION[:2] < (1, 10):
+    MIDDLEWARE_CLASSES = (
+        'django.contrib.sessions.middleware.SessionMiddleware',
+        'django.middleware.common.CommonMiddleware',
+        'django.middleware.csrf.CsrfViewMiddleware',
+        'django.contrib.auth.middleware.AuthenticationMiddleware',
+        'django.contrib.messages.middleware.MessageMiddleware',
+        'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    )
+else:
+    MIDDLEWARE = (
+        'django.contrib.sessions.middleware.SessionMiddleware',
+        'django.middleware.common.CommonMiddleware',
+        'django.middleware.csrf.CsrfViewMiddleware',
+        'django.contrib.auth.middleware.AuthenticationMiddleware',
+        'django.contrib.messages.middleware.MessageMiddleware',
+        'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    )
 
 ROOT_URLCONF = 'tests.urls'
 

From dd4f02f1226cdac0f18b50b8d4ef6abfcc0d8be3 Mon Sep 17 00:00:00 2001
From: Ronald van Zon <ronald.vanzon@nedap.com>
Date: Wed, 13 Dec 2017 14:36:12 +0100
Subject: [PATCH 5/9] Add custom SharedCounter / Queue

This will not fix anything related to Django 2.0 but will allow MAC
users to run Django-Q allowing tests to be run locally.
---
 django_q/cluster.py              |  4 +-
 django_q/conf.py                 |  5 ++-
 django_q/queues.py               | 69 ++++++++++++++++++++++++++++++++
 django_q/tasks.py                |  3 +-
 django_q/tests/test_cached.py    |  3 +-
 django_q/tests/test_cluster.py   |  3 +-
 django_q/tests/test_scheduler.py |  3 +-
 7 files changed, 83 insertions(+), 7 deletions(-)
 create mode 100644 django_q/queues.py

diff --git a/django_q/cluster.py b/django_q/cluster.py
index 23db963..4dd61f6 100644
--- a/django_q/cluster.py
+++ b/django_q/cluster.py
@@ -10,7 +10,7 @@ import signal
 import socket
 import ast
 from time import sleep
-from multiprocessing import Queue, Event, Process, Value, current_process
+from multiprocessing import Event, Process, Value, current_process
 
 # external
 import arrow
@@ -30,7 +30,7 @@ from django_q.models import Task, Success, Schedule
 from django_q.status import Stat, Status
 from django_q.brokers import get_broker
 from django_q.signals import pre_execute
-
+from django_q.queues import Queue
 
 
 class Cluster(object):
diff --git a/django_q/conf.py b/django_q/conf.py
index 6e955a1..e8afac1 100644
--- a/django_q/conf.py
+++ b/django_q/conf.py
@@ -1,7 +1,7 @@
 import logging
 from copy import deepcopy
 from signal import signal
-from multiprocessing import cpu_count, Queue
+from multiprocessing import cpu_count
 
 # django
 from django.utils.translation import ugettext_lazy as _
@@ -11,6 +11,9 @@ from django.conf import settings
 import os
 import pkg_resources
 
+# local
+from django_q.queues import Queue
+
 # optional
 try:
     import psutil
diff --git a/django_q/queues.py b/django_q/queues.py
new file mode 100644
index 0000000..ad729a1
--- /dev/null
+++ b/django_q/queues.py
@@ -0,0 +1,69 @@
+"""
+The code is derived from https://github.com/althonos/pronto/commit/3384010dfb4fc7c66a219f59276adef3288a886b
+"""
+
+import multiprocessing
+import multiprocessing.queues
+
+
+class SharedCounter(object):
+    """ A synchronized shared counter.
+
+    The locking done by multiprocessing.Value ensures that only a single
+    process or thread may read or write the in-memory ctypes object. However,
+    in order to do n += 1, Python performs a read followed by a write, so a
+    second process may read the old value before the new one is written by
+    the first process. The solution is to use a multiprocessing.Lock to
+    guarantee the atomicity of the modifications to Value.
+
+    This class comes almost entirely from Eli Bendersky's blog:
+    http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing/
+    """
+
+    def __init__(self, n=0):
+        self.count = multiprocessing.Value('i', n)
+
+    def increment(self, n=1):
+        """ Increment the counter by n (default = 1) """
+        with self.count.get_lock():
+            self.count.value += n
+
+    @property
+    def value(self):
+        """ Return the value of the counter """
+        return self.count.value
+
+
+class Queue(multiprocessing.queues.Queue):
+    """ A portable implementation of multiprocessing.Queue.
+
+    Because of multithreading / multiprocessing semantics, Queue.qsize() may
+    raise the NotImplementedError exception on Unix platforms like Mac OS X
+    where sem_getvalue() is not implemented. This subclass addresses this
+    problem by using a synchronized shared counter (initialized to zero) and
+    increasing / decreasing its value every time the put() and get() methods
+    are called, respectively. This not only prevents NotImplementedError from
+    being raised, but also allows us to implement a reliable version of both
+    qsize() and empty().
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(Queue, self).__init__(*args, ctx=multiprocessing.get_context(), **kwargs)
+        self.size = SharedCounter(0)
+
+    def put(self, *args, **kwargs):
+        super(Queue, self).put(*args, **kwargs)
+        self.size.increment(1)
+
+    def get(self, *args, **kwargs):
+        x = super(Queue, self).get(*args, **kwargs)
+        self.size.increment(-1)
+        return x
+
+    def qsize(self):
+        """ Reliable implementation of multiprocessing.Queue.qsize() """
+        return self.size.value
+
+    def empty(self):
+        """ Reliable implementation of multiprocessing.Queue.empty() """
+        return not self.qsize() > 0
diff --git a/django_q/tasks.py b/django_q/tasks.py
index 5f1c46b..e64bfc9 100644
--- a/django_q/tasks.py
+++ b/django_q/tasks.py
@@ -1,5 +1,5 @@
 """Provides task functionality."""
-from multiprocessing import Queue, Value
+from multiprocessing import Value
 
 # django
 from django.db import IntegrityError
@@ -14,6 +14,7 @@ from django_q.models import Schedule, Task
 from django_q.humanhash import uuid
 from django_q.brokers import get_broker
 from django_q.signals import pre_enqueue
+from django_q.queues import Queue
 
 
 def async(func, *args, **kwargs):
diff --git a/django_q/tests/test_cached.py b/django_q/tests/test_cached.py
index 1cf9980..e32c60c 100644
--- a/django_q/tests/test_cached.py
+++ b/django_q/tests/test_cached.py
@@ -1,4 +1,4 @@
-from multiprocessing import Event, Queue, Value
+from multiprocessing import Event, Value
 
 import pytest
 
@@ -8,6 +8,7 @@ from django_q.conf import Conf
 from django_q.tasks import async, result, fetch, count_group, result_group, fetch_group, delete_group, delete_cached, \
     async_iter, Chain, async_chain, Iter, Async
 from django_q.brokers import get_broker
+from django_q.queues import Queue
 
 
 @pytest.fixture
diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py
index d16d0ac..15461d0 100644
--- a/django_q/tests/test_cluster.py
+++ b/django_q/tests/test_cluster.py
@@ -1,6 +1,6 @@
 import sys
 import threading
-from multiprocessing import Queue, Event, Value
+from multiprocessing import Event, Value
 from time import sleep
 from django.utils import timezone
 
@@ -19,6 +19,7 @@ from django_q.conf import Conf
 from django_q.status import Stat
 from django_q.brokers import get_broker
 from django_q.tests.tasks import multiply
+from django_q.queues import Queue
 
 
 class WordClass(object):
diff --git a/django_q/tests/test_scheduler.py b/django_q/tests/test_scheduler.py
index 47a0860..5160776 100644
--- a/django_q/tests/test_scheduler.py
+++ b/django_q/tests/test_scheduler.py
@@ -1,5 +1,5 @@
 from datetime import timedelta
-from multiprocessing import Queue, Event, Value
+from multiprocessing import Event, Value
 
 import arrow
 import pytest
@@ -10,6 +10,7 @@ from django_q.brokers import get_broker
 from django_q.cluster import pusher, worker, monitor, scheduler
 from django_q.conf import Conf
 from django_q.tasks import Schedule, fetch, schedule as create_schedule
+from django_q.queues import Queue
 
 
 @pytest.fixture

From 5eaa7008660d487e50d0ddf596108a58a672f944 Mon Sep 17 00:00:00 2001
From: Ronald van Zon <ronald.vanzon@nedap.com>
Date: Wed, 13 Dec 2017 14:49:07 +0100
Subject: [PATCH 6/9] Reverting core_signing to the old way.

By reverting django.core.signing to Django 1.11.8 version Django-Q works
This is not designed as a 'good' solution just to show where I think the
problem is.
---
 django_q/core_signing.py | 76 ++++++++++++++++++++++++++++++++++++++++
 django_q/signing.py      |  2 +-
 2 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 django_q/core_signing.py

diff --git a/django_q/core_signing.py b/django_q/core_signing.py
new file mode 100644
index 0000000..1790a0b
--- /dev/null
+++ b/django_q/core_signing.py
@@ -0,0 +1,76 @@
+from __future__ import unicode_literals
+
+import datetime
+import time
+import zlib
+
+from django.utils import baseconv
+from django.utils.crypto import constant_time_compare
+from django.utils.encoding import force_bytes, force_str, force_text
+from django.core.signing import BadSignature, SignatureExpired, b64_decode, JSONSerializer, \
+    Signer as Sgnr, TimestampSigner as TsS, dumps
+
+dumps = dumps
+
+
+"""
+The loads function is the same as the `django.core.signing.loads` function
+The difference is that `this` loads function calls `TimestampSigner` and `Signer`
+"""
+def loads(s, key=None, salt='django.core.signing', serializer=JSONSerializer, max_age=None):
+    """
+    Reverse of dumps(), raise BadSignature if signature fails.
+
+    The serializer is expected to accept a bytestring.
+    """
+    # TimestampSigner.unsign() returns str but base64 and zlib compression
+    # operate on bytes.
+    base64d = force_bytes(TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
+    decompress = False
+    if base64d[:1] == b'.':
+        # It's compressed; uncompress it first
+        base64d = base64d[1:]
+        decompress = True
+    data = b64_decode(base64d)
+    if decompress:
+        data = zlib.decompress(data)
+    return serializer().loads(data)
+
+
+class Signer(Sgnr):
+
+    def unsign(self, signed_value):
+        # force_str is removed in Django 2.0
+        signed_value = force_str(signed_value)
+        if self.sep not in signed_value:
+            raise BadSignature('No "%s" found in value' % self.sep)
+        value, sig = signed_value.rsplit(self.sep, 1)
+        if constant_time_compare(sig, self.signature(value)):
+            # force_text is removed in Django 2.0
+            return force_text(value)
+        raise BadSignature('Signature "%s" does not match' % sig)
+
+
+"""
+TimestampSigner is also the same as `django.core.signing.TimestampSigner` but is
+calling `this` Signer.
+"""
+class TimestampSigner(Signer, TsS):
+
+    def unsign(self, value, max_age=None):
+        """
+        Retrieve original value and check it wasn't signed more
+        than max_age seconds ago.
+        """
+        result = super().unsign(value)
+        value, timestamp = result.rsplit(self.sep, 1)
+        timestamp = baseconv.base62.decode(timestamp)
+        if max_age is not None:
+            if isinstance(max_age, datetime.timedelta):
+                max_age = max_age.total_seconds()
+            # Check timestamp is not older than max_age
+            age = time.time() - timestamp
+            if age > max_age:
+                raise SignatureExpired(
+                    'Signature age %s > %s seconds' % (age, max_age))
+        return value
diff --git a/django_q/signing.py b/django_q/signing.py
index 4ee8bf3..2460aaa 100644
--- a/django_q/signing.py
+++ b/django_q/signing.py
@@ -4,7 +4,7 @@ try:
 except ImportError:
     import pickle
 
-from django.core import signing
+from django_q import core_signing as signing
 
 from django_q.conf import Conf
 

From 98aa75faf1875862147af5c48c60d5c60d88c28b Mon Sep 17 00:00:00 2001
From: Daniel Welch <dwelch2102@gmail.com>
Date: Thu, 7 Dec 2017 18:30:54 -0500
Subject: [PATCH 7/9] fix usage of iter_entry_points

---
 django_q/conf.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/django_q/conf.py b/django_q/conf.py
index 2bb6c98..6e955a1 100644
--- a/django_q/conf.py
+++ b/django_q/conf.py
@@ -216,10 +216,10 @@ if Conf.ERROR_REPORTER:
         # iterate through the configured error reporters,
         # and instantiate an ErrorReporter using the provided config
         for name, conf in error_conf.items():
-            Reporter = pkg_resources.iter_entry_points(
-                    'djangoq.errorreporters', name).load()
-            e = Reporter(**conf)
-            reporters.append(e)
+            for entry in pkg_resources.iter_entry_points(
+                    'djangoq.errorreporters', name):
+                Reporter = entry.load()
+                reporters.append(Reporter(**conf))
         error_reporter = ErrorReporter(reporters)
     except ImportError:
         error_reporter = None

From 81d4b36b1b7c9f05aa5ec4fd1241875f27e2ff04 Mon Sep 17 00:00:00 2001
From: ilan <koed00@gmail.com>
Date: Mon, 8 Jan 2018 09:57:20 +0100
Subject: [PATCH 8/9] Cherry picking compatibility commits

---
 .travis.yml                |  2 +-
 django_q/tests/settings.py |  3 ++-
 django_q/tests/urls.py     |  4 ++--
 requirements.txt           | 16 ++++++++--------
 4 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index f7345b7..3461373 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ python:
   - "3.6"
 
 env:
-    - DJANGO=2.0
+    - DJANGO=2.0.1
     - DJANGO=1.11.8
     - DJANGO=1.8.18
 
diff --git a/django_q/tests/settings.py b/django_q/tests/settings.py
index 8bad66c..276aca0 100644
--- a/django_q/tests/settings.py
+++ b/django_q/tests/settings.py
@@ -33,11 +33,12 @@ MIDDLEWARE_CLASSES = (
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 )
 
+MIDDLEWARE = MIDDLEWARE_CLASSES
+
 ROOT_URLCONF = 'tests.urls'
 
 TEMPLATES = [
diff --git a/django_q/tests/urls.py b/django_q/tests/urls.py
index eb91d28..ede2ec9 100644
--- a/django_q/tests/urls.py
+++ b/django_q/tests/urls.py
@@ -1,6 +1,6 @@
-from django.conf.urls import include, url
+from django.conf.urls import url
 from django.contrib import admin
 
 urlpatterns = [
-    url(r'^admin/', include(admin.site.urls)),
+    url(r'^admin/', admin.site.urls),
 ]
diff --git a/requirements.txt b/requirements.txt
index 6bd4a89..d88c43d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,11 +4,11 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in
 #
-arrow==0.10.0
+arrow==0.12.0
 blessed==1.14.2
-boto3==1.4.7
-botocore==1.7.28          # via boto3, s3transfer
-certifi==2017.7.27.1      # via requests
+boto3==1.5.9
+botocore==1.8.23          # via boto3, s3transfer
+certifi==2017.11.5        # via requests
 chardet==3.0.4            # via requests
 django-picklefield==1.0.0
 django-redis==4.8.0
@@ -18,13 +18,13 @@ idna==2.6                 # via requests
 iron-core==1.2.0          # via iron-mq
 iron-mq==0.9
 jmespath==0.9.3           # via boto3, botocore
-psutil==5.4.0
-pymongo==3.5.1
+psutil==5.4.3
+pymongo==3.6.0
 python-dateutil==2.6.1    # via arrow, botocore, iron-core
 redis==2.10.6
 requests==2.18.4          # via iron-core, rollbar
-rollbar==0.13.13
-s3transfer==0.1.11        # via boto3
+rollbar==0.13.17
+s3transfer==0.1.12        # via boto3
 six==1.11.0               # via blessed, python-dateutil, rollbar
 urllib3==1.22             # via requests
 wcwidth==0.1.7            # via blessed

From 72b52b026f2536fecff599b6059e2b0fa5f0c6f8 Mon Sep 17 00:00:00 2001
From: ilan <koed00@gmail.com>
Date: Mon, 8 Jan 2018 10:21:08 +0100
Subject: [PATCH 9/9] Downgraded to Django 2.0

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 3461373..269d24f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,8 +8,8 @@ python:
   - "3.6"
 
 env:
-    - DJANGO=2.0.1
-    - DJANGO=1.11.8
+    - DJANGO=2.0
+    - DJANGO=1.11.9
     - DJANGO=1.8.18
 
 sudo: false