From 7d03536e891fba5a206821272752f289c57f905e Mon Sep 17 00:00:00 2001 From: Ilan Steemers Date: Sun, 14 Feb 2021 18:59:04 +0100 Subject: [PATCH] Migrate to Github Action CI (#507) * Some small refactors * Initial test script * Fixes django matrix * Trying to set up Disque * Disque as a docker service * lowercase action name * Remove Travis * Replaces Travis badge with Github actions badge * Show django version in step * Typo --- .github/workflows/test.yml | 54 ++++++++++++++++++++++++++++++++++ .gitignore | 1 + .travis.yml | 49 ------------------------------ README.rst | 4 +-- django_q/brokers/aws_sqs.py | 3 +- django_q/cluster.py | 11 +++---- django_q/monitor.py | 2 +- django_q/tasks.py | 3 +- django_q/tests/test_brokers.py | 14 ++++----- django_q/tests/test_cached.py | 8 ++--- django_q/tests/test_cluster.py | 2 +- django_q/tests/test_monitor.py | 2 +- 12 files changed, 78 insertions(+), 75 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..582059d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,54 @@ +name: tests +on: [ push, pull_request ] +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [ 3.6, 3.7, 3.8 ] + django: [ "2.2", "3.1" ] + services: + disque: + image: efrecon/disque:1.0-rc1 + ports: + - '7711:7711/tcp' + mongodb: + image: mongo + ports: + - 27017:27017 + postgres: + image: postgres + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + # needed because the postgres container does not provide a health check + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + redis: + image: redis + ports: + - 6379:6379 + options: --entrypoint redis-server + steps: + - uses: actions/checkout@v2 + - uses: codecov/codecov-action@v1 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies with Django ${{ matrix.django }} + run: | + python -m pip install --upgrade pip + pip install -q django==${{ matrix.django }} + pip install -r requirements.txt + pip install pytest --upgrade + pip install pytest-django codecov sphinx poetry + poetry install + - name: Run Tests + run: | + coverage run --source=django_q -m py.test + - name: Build docs + run: | + sphinx-build -b html -d docs/_build/doctrees -nW docs docs/_build/html diff --git a/.gitignore b/.gitignore index d540666..6c151bc 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,4 @@ djq node_modules /c.cache/ /dq +/venv/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1f8988b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,49 +0,0 @@ -language: python - -services: - - redis - - mongodb - -python: - - "3.7" - - "3.8" - -env: - - DJANGO=3.1.2 - - DJANGO=2.2.16 - -sudo: true -dist: xenial - -addons: - apt: - packages: - - tcl8.5 - -before_script: - - git clone https://github.com/antirez/disque.git disque_server - - "cd disque_server/src && make && PREFIX=../ make install && cd -" - - "./disque_server/bin/disque-server &" - - ./disque_server/bin/disque PING - -install: - - pip install -q django==$DJANGO - - pip install -r requirements.txt - - pip install pytest --upgrade - - pip install pytest-django codecov sphinx poetry - - poetry install - -script: - - coverage run --source=django_q -m py.test - - sphinx-build -b html -d docs/_build/doctrees -nW docs docs/_build/html - -after_success: - - codecov - -notifications: - webhooks: - urls: - - https://webhooks.gitter.im/e/cbcff78c4be241602332 - on_success: change - on_failure: always - on_start: never diff --git a/README.rst b/README.rst index 18c9c00..51e80d4 100644 --- a/README.rst +++ b/README.rst @@ -247,8 +247,8 @@ Acknowledgements `HumanHash `__ - Redditors feedback at `r/django `__ -.. |image0| image:: https://travis-ci.org/Koed00/django-q.svg?branch=master - :target: https://travis-ci.org/Koed00/django-q +.. |image0| image:: https://github.com/koed00/django-q/workflows/Tests/badge.svg?branche=master + :target: https://github.com/Koed00/django-q/actions?query=workflow%3Atests .. |image1| image:: http://codecov.io/github/Koed00/django-q/coverage.svg?branch=master :target: http://codecov.io/github/Koed00/django-q?branch=master .. |image2| image:: http://badges.gitter.im/Join%20Chat.svg diff --git a/django_q/brokers/aws_sqs.py b/django_q/brokers/aws_sqs.py index 4004af4..2bc58c8 100644 --- a/django_q/brokers/aws_sqs.py +++ b/django_q/brokers/aws_sqs.py @@ -4,7 +4,6 @@ from botocore.client import ClientError from django_q.brokers import Broker from django_q.conf import Conf - QUEUE_DOES_NOT_EXIST = "AWS.SimpleQueueService.NonExistentQueue" @@ -77,7 +76,7 @@ class Sqs(Broker): # exist try to create it. return self.sqs.get_queue_by_name(QueueName=self.list_key) except ClientError as exp: - if not exp.response["Error"]["Code"] == QUEUE_DOES_NOT_EXIST: + if exp.response["Error"]["Code"] != QUEUE_DOES_NOT_EXIST: raise exp return self.sqs.create_queue(QueueName=self.list_key) diff --git a/django_q/cluster.py b/django_q/cluster.py index d5532fa..cc5cbff 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -595,7 +595,7 @@ def scheduler(broker: Broker = None): if s.hook: q_options["hook"] = s.hook # set up the next run time - if not s.schedule_type == s.ONCE: + if s.schedule_type != s.ONCE: next_run = arrow.get(s.next_run) while True: if s.schedule_type == s.MINUTES: @@ -722,8 +722,9 @@ def set_cpu_affinity(n: int, process_ids: list, actual: bool = not Conf.TESTING) def rss_check(): - if Conf.MAX_RSS and resource: - return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS - elif Conf.MAX_RSS and psutil: - return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024 + if Conf.MAX_RSS: + if resource: + return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss >= Conf.MAX_RSS + elif psutil: + return psutil.Process().memory_info().rss >= Conf.MAX_RSS * 1024 return False diff --git a/django_q/monitor.py b/django_q/monitor.py index 488eda8..7c99255 100644 --- a/django_q/monitor.py +++ b/django_q/monitor.py @@ -196,7 +196,7 @@ def info(broker=None): tasks_per_day = last_tasks.count() if tasks_per_day > 0: # average execution time over the last 24 hours - if not connection.vendor == "sqlite": + if connection.vendor != "sqlite": exec_time = last_tasks.aggregate( time_taken=Sum(F("stopped") - F("started")) ) diff --git a/django_q/tasks.py b/django_q/tasks.py index 1e6ca3c..77aa55e 100644 --- a/django_q/tasks.py +++ b/django_q/tasks.py @@ -260,7 +260,7 @@ def fetch_cached(task_id, wait=0, broker=None): r = broker.cache.get(f"{broker.list_key}:{task_id}") if r: task = SignedPackage.loads(r) - t = Task( + return Task( id=task["id"], name=task["name"], func=task["func"], @@ -272,7 +272,6 @@ def fetch_cached(task_id, wait=0, broker=None): result=task["result"], success=task["success"], ) - return t if (time() - start) * 1000 >= wait >= 0: break sleep(0.01) diff --git a/django_q/tests/test_brokers.py b/django_q/tests/test_brokers.py index 064ecf1..0068150 100644 --- a/django_q/tests/test_brokers.py +++ b/django_q/tests/test_brokers.py @@ -95,7 +95,7 @@ def test_disque(monkeypatch): task_id = broker.enqueue("test") broker.fail(task_id) # bulk test - for i in range(5): + for _ in range(5): broker.enqueue("test") monkeypatch.setattr(Conf, "BULK", 5) monkeypatch.setattr(Conf, "DISQUE_FASTACK", True) @@ -166,7 +166,7 @@ def test_ironmq(monkeypatch): task_id = broker.enqueue("test") broker.fail(task_id) # bulk test - for i in range(5): + for _ in range(5): broker.enqueue("test") monkeypatch.setattr(Conf, "BULK", 5) tasks = broker.dequeue() @@ -236,7 +236,7 @@ def canceled_sqs(monkeypatch): task = broker.dequeue()[0] broker.fail(task[0]) # bulk test - for i in range(10): + for _ in range(10): broker.enqueue("test") monkeypatch.setattr(Conf, "BULK", 12) tasks = broker.dequeue() @@ -290,7 +290,7 @@ def test_orm(monkeypatch): task_id = broker.enqueue("test") broker.fail(task_id) # bulk test - for i in range(5): + for _ in range(5): broker.enqueue("test") monkeypatch.setattr(Conf, "BULK", 5) tasks = broker.dequeue() @@ -347,11 +347,9 @@ def test_mongo(monkeypatch): task_id = broker.enqueue("test") broker.fail(task_id) # bulk test - for i in range(5): + for _ in range(5): broker.enqueue("test") - tasks = [] - for i in range(5): - tasks.append(broker.dequeue()[0]) + tasks = [broker.dequeue()[0] for _ in range(5)] assert broker.lock_size() == 5 for task in tasks: assert task is not None diff --git a/django_q/tests/test_cached.py b/django_q/tests/test_cached.py index 8c618c9..a6f16a6 100644 --- a/django_q/tests/test_cached.py +++ b/django_q/tests/test_cached.py @@ -152,9 +152,9 @@ def test_asynctask_class(broker, monkeypatch): a.args = (1, -1) assert a.started is False a.cached = True - assert a.cached is True + assert a.cached a.sync = True - assert a.sync is True + assert a.sync a.broker = broker assert a.broker == broker a.run() @@ -166,7 +166,7 @@ def test_asynctask_class(broker, monkeypatch): assert a.result() == -1 # with q_options a = AsyncTask('math.copysign', 1, -1, q_options={'cached': True, 'sync': False, 'broker': broker}) - assert a.sync is False + assert not a.sync a.sync = True assert a.kwargs['q_options']['sync'] is True a.run() @@ -174,7 +174,7 @@ def test_asynctask_class(broker, monkeypatch): a.group = 'async_class_test' assert a.group == 'async_class_test' a.save = False - assert a.save is False + assert not a.save a.hook = 'djq.tests.tasks.hello' assert a.hook == 'djq.tests.tasks.hello' assert a.started is False diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py index be04cdf..179c9ba 100644 --- a/django_q/tests/test_cluster.py +++ b/django_q/tests/test_cluster.py @@ -162,7 +162,7 @@ def test_enqueue(broker, admin_user): stop_event = Event() stop_event.set() # push the tasks - for i in range(task_count): + for _ in range(task_count): pusher(task_queue, stop_event, broker=broker) assert broker.queue_size() == 0 assert task_queue.qsize() == task_count diff --git a/django_q/tests/test_monitor.py b/django_q/tests/test_monitor.py index 25a5dd0..8a9b76f 100644 --- a/django_q/tests/test_monitor.py +++ b/django_q/tests/test_monitor.py @@ -26,7 +26,7 @@ def test_monitor(monkeypatch): assert stat.uptime() > 0 assert stat.empty_queues() is True break - assert found_c is True + assert found_c # test lock size monkeypatch.setattr(Conf, 'ORM', 'default') b = get_broker('monitor_test')