Migrate to Github Action CI (#507)

* Some small refactors

* Initial test script

* Fixes django matrix

* Trying to set up Disque

* Disque as a docker service

* lowercase action name

* Remove Travis

* Replaces Travis badge with Github actions badge

* Show django version in step

* Typo

.github/workflows/test.yml

@@ -0,0 +1,54 @@
+name: tests
+on: [ push, pull_request ]
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ 3.6, 3.7, 3.8 ]
+        django: [ "2.2", "3.1" ]
+    services:
+      disque:
+        image: efrecon/disque:1.0-rc1
+        ports:
+          - '7711:7711/tcp'
+      mongodb:
+        image: mongo
+        ports:
+          - 27017:27017
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        # needed because the postgres container does not provide a health check
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: --entrypoint redis-server
+    steps:
+        - uses: actions/checkout@v2
+        - uses: codecov/codecov-action@v1
+        - name: Set up Python ${{ matrix.python-version }}
+          uses: actions/setup-python@v2
+          with:
+            python-version: ${{ matrix.python-version }}
+        - name: Install dependencies with Django ${{ matrix.django }}
+          run: |
+            python -m pip install --upgrade pip
+            pip install -q django==${{ matrix.django }}
+            pip install -r requirements.txt
+            pip install pytest --upgrade
+            pip install pytest-django codecov sphinx poetry
+            poetry install
+        - name: Run Tests
+          run: |
+            coverage run --source=django_q -m py.test
+        - name: Build docs
+          run: |
+            sphinx-build -b html -d docs/_build/doctrees  -nW docs docs/_build/html

.gitignore

@@ -70,3 +70,4 @@ djq
 node_modules
 /c.cache/
 /dq
+/venv/

.travis.yml

@@ -1,49 +0,0 @@
-language: python
-
-services:
-  - redis
-  - mongodb
-
-python:
-  - "3.7"
-  - "3.8"
-
-env:
-  - DJANGO=3.1.2
-  - DJANGO=2.2.16
-
-sudo: true
-dist: xenial
-
-addons:
-  apt:
-    packages:
-      - tcl8.5
-
-before_script:
-  - git clone https://github.com/antirez/disque.git disque_server
-  - "cd disque_server/src && make && PREFIX=../ make install && cd -"
-  - "./disque_server/bin/disque-server &"
-  - ./disque_server/bin/disque PING
-
-install:
-  - pip install -q django==$DJANGO
-  - pip install -r requirements.txt
-  - pip install pytest --upgrade
-  - pip install pytest-django codecov sphinx poetry
-  - poetry install
-
-script:
-  - coverage run --source=django_q -m py.test
-  - sphinx-build -b html -d docs/_build/doctrees  -nW docs docs/_build/html
-
-after_success:
-  - codecov
-
-notifications:
-  webhooks:
-    urls:
-      - https://webhooks.gitter.im/e/cbcff78c4be241602332
-    on_success: change
-    on_failure: always
-    on_start: never

README.rst

@@ -247,8 +247,8 @@ Acknowledgements
    `HumanHash <https://github.com/zacharyvoase/humanhash>`__
 -  Redditors feedback at `r/django <https://www.reddit.com/r/django/>`__
 
-.. |image0| image:: https://travis-ci.org/Koed00/django-q.svg?branch=master
-   :target: https://travis-ci.org/Koed00/django-q
+.. |image0| image:: https://github.com/koed00/django-q/workflows/Tests/badge.svg?branche=master
+   :target: https://github.com/Koed00/django-q/actions?query=workflow%3Atests
 .. |image1| image:: http://codecov.io/github/Koed00/django-q/coverage.svg?branch=master
    :target: http://codecov.io/github/Koed00/django-q?branch=master
 .. |image2| image:: http://badges.gitter.im/Join%20Chat.svg

django_q/brokers/aws_sqs.py

@@ -4,7 +4,6 @@ from botocore.client import ClientError
 from django_q.brokers import Broker
 from django_q.conf import Conf
 
-
 QUEUE_DOES_NOT_EXIST = "AWS.SimpleQueueService.NonExistentQueue"
 
 
@@ -77,7 +76,7 @@ class Sqs(Broker):
             # exist try to create it.
             return self.sqs.get_queue_by_name(QueueName=self.list_key)
         except ClientError as exp:
-            if not exp.response["Error"]["Code"] == QUEUE_DOES_NOT_EXIST:
+            if exp.response["Error"]["Code"] != QUEUE_DOES_NOT_EXIST:
                 raise exp
 
         return self.sqs.create_queue(QueueName=self.list_key)

django_q/cluster.py

@@ -595,7 +595,7 @@ def scheduler(broker: Broker = None):
                 if s.hook:
                     q_options["hook"] = s.hook
                 # set up the next run time
-                if not s.schedule_type == s.ONCE:
+                if s.schedule_type != s.ONCE:
                     next_run = arrow.get(s.next_run)
                     while True:
                         if s.schedule_type == s.MINUTES:
@@ -722,8 +722,9 @@ def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING)
 
 
 def rss_check():
-    if Conf.MAX_RSS and resource:
-        return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
-    elif Conf.MAX_RSS and psutil:
-        return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
+    if Conf.MAX_RSS:
+        if resource:
+            return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS
+        elif psutil:
+            return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024
     return False

django_q/monitor.py

@@ -196,7 +196,7 @@ def info(broker=None):
     tasks_per_day = last_tasks.count()
     if tasks_per_day > 0:
         # average execution time over the last 24 hours
-        if not connection.vendor == "sqlite":
+        if connection.vendor != "sqlite":
             exec_time = last_tasks.aggregate(
                 time_taken=Sum(F("stopped") - F("started"))
             )

django_q/tasks.py

@@ -260,7 +260,7 @@ def fetch_cached(task_id, wait=0, broker=None):
         r = broker.cache.get(f"{broker.list_key}:{task_id}")
         if r:
             task = SignedPackage.loads(r)
-            t = Task(
+            return Task(
                 id=task["id"],
                 name=task["name"],
                 func=task["func"],
@@ -272,7 +272,6 @@ def fetch_cached(task_id, wait=0, broker=None):
                 result=task["result"],
                 success=task["success"],
             )
-            return t
         if (time() - start) * 1000 >= wait >= 0:
             break
         sleep(0.01)

django_q/tests/test_brokers.py

@@ -95,7 +95,7 @@ def test_disque(monkeypatch):
     task_id = broker.enqueue("test")
     broker.fail(task_id)
     # bulk test
-    for i in range(5):
+    for _ in range(5):
         broker.enqueue("test")
     monkeypatch.setattr(Conf, "BULK", 5)
     monkeypatch.setattr(Conf, "DISQUE_FASTACK", True)
@@ -166,7 +166,7 @@ def test_ironmq(monkeypatch):
     task_id = broker.enqueue("test")
     broker.fail(task_id)
     # bulk test
-    for i in range(5):
+    for _ in range(5):
         broker.enqueue("test")
     monkeypatch.setattr(Conf, "BULK", 5)
     tasks = broker.dequeue()
@@ -236,7 +236,7 @@ def canceled_sqs(monkeypatch):
         task = broker.dequeue()[0]
     broker.fail(task[0])
     # bulk test
-    for i in range(10):
+    for _ in range(10):
         broker.enqueue("test")
     monkeypatch.setattr(Conf, "BULK", 12)
     tasks = broker.dequeue()
@@ -290,7 +290,7 @@ def test_orm(monkeypatch):
     task_id = broker.enqueue("test")
     broker.fail(task_id)
     # bulk test
-    for i in range(5):
+    for _ in range(5):
         broker.enqueue("test")
     monkeypatch.setattr(Conf, "BULK", 5)
     tasks = broker.dequeue()
@@ -347,11 +347,9 @@ def test_mongo(monkeypatch):
     task_id = broker.enqueue("test")
     broker.fail(task_id)
     # bulk test
-    for i in range(5):
+    for _ in range(5):
         broker.enqueue("test")
-    tasks = []
-    for i in range(5):
-        tasks.append(broker.dequeue()[0])
+    tasks = [broker.dequeue()[0] for _ in range(5)]
     assert broker.lock_size() == 5
     for task in tasks:
         assert task is not None

django_q/tests/test_cached.py

@@ -152,9 +152,9 @@ def test_asynctask_class(broker, monkeypatch):
     a.args = (1, -1)
     assert a.started is False
     a.cached = True
-    assert a.cached is True
+    assert a.cached
     a.sync = True
-    assert a.sync is True
+    assert a.sync
     a.broker = broker
     assert a.broker == broker
     a.run()
@@ -166,7 +166,7 @@ def test_asynctask_class(broker, monkeypatch):
     assert a.result() == -1
     # with q_options
     a = AsyncTask('math.copysign', 1, -1, q_options={'cached': True, 'sync': False, 'broker': broker})
-    assert a.sync is False
+    assert not a.sync
     a.sync = True
     assert a.kwargs['q_options']['sync'] is True
     a.run()
@@ -174,7 +174,7 @@ def test_asynctask_class(broker, monkeypatch):
     a.group = 'async_class_test'
     assert a.group == 'async_class_test'
     a.save = False
-    assert a.save is False
+    assert not a.save
     a.hook = 'djq.tests.tasks.hello'
     assert a.hook == 'djq.tests.tasks.hello'
     assert a.started is False

django_q/tests/test_cluster.py

@@ -162,7 +162,7 @@ def test_enqueue(broker, admin_user):
     stop_event = Event()
     stop_event.set()
     # push the tasks
-    for i in range(task_count):
+    for _ in range(task_count):
         pusher(task_queue, stop_event, broker=broker)
     assert broker.queue_size() == 0
     assert task_queue.qsize() == task_count

django_q/tests/test_monitor.py

@@ -26,7 +26,7 @@ def test_monitor(monkeypatch):
             assert stat.uptime() > 0
             assert stat.empty_queues() is True
             break
-    assert found_c is True
+    assert found_c
     # test lock size
     monkeypatch.setattr(Conf, 'ORM', 'default')
     b = get_broker('monitor_test')