diff --git a/Dockerfile.dev b/Dockerfile.dev index 9c6ecee..44e5023 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -7,6 +7,10 @@ ENV PYTHONUNBUFFERED 1 # Sets the default shell to bash ENV SHELL /bin/bash +RUN set -ex \ + && apt update \ + && apt-get install gcc python3-dev --yes + # Upgrades pip RUN pip install -U pip setuptools diff --git a/containers/localstack/init.sh b/containers/localstack/init.sh new file mode 100755 index 0000000..332fd90 --- /dev/null +++ b/containers/localstack/init.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Note that this file needs to have the executable bit set for it to work with later localstack implementations. + +export DEFAULT_REGION=us-west-2 + +create_sqs() { + QUEUE_NAME="$1" + TIMEOUT=${2:-60} + DL_QUEUE_URL=$(awslocal sqs create-queue --queue-name "dl-$QUEUE_NAME" --query QueueUrl --output text) + echo ">>> Created $DL_QUEUE_URL queue!" + DL_QUEUE_ARN=$(awslocal sqs get-queue-attributes --queue-url "$DL_QUEUE_URL" --attribute-names QueueArn --query Attributes.QueueArn --output text) + awslocal sqs create-queue --queue-name "$QUEUE_NAME" --attributes '{ + "RedrivePolicy": "{\"deadLetterTargetArn\": \"'"$DL_QUEUE_ARN"'\",\"maxReceiveCount\":\"3\"}", + "VisibilityTimeout": "'"$TIMEOUT"'" + }' +} + +# Create SQS queues +create_sqs testing \ No newline at end of file diff --git a/django_q/brokers/aws_sqs.py b/django_q/brokers/aws_sqs.py index 6c0ba50..cd4ac71 100644 --- a/django_q/brokers/aws_sqs.py +++ b/django_q/brokers/aws_sqs.py @@ -1,3 +1,5 @@ +import copy + from boto3 import Session from botocore.client import ClientError @@ -78,15 +80,15 @@ class Sqs(Broker): @staticmethod def get_connection(list_key: str = None) -> Session: - config = Conf.SQS - if "aws_region" in config: - config["region_name"] = config["aws_region"] - del config["aws_region"] + config_cloned = copy.deepcopy(Conf.SQS) + if "aws_region" in config_cloned: + config_cloned["region_name"] = config_cloned["aws_region"] + del config_cloned["aws_region"] - if "receive_message_wait_time_seconds" in config: - del config["receive_message_wait_time_seconds"] + if "receive_message_wait_time_seconds" in config_cloned: + del config_cloned["receive_message_wait_time_seconds"] - return Session(**config) + return Session(**config_cloned) def get_queue(self): self.sqs = self.connection.resource("sqs") diff --git a/django_q/tests/test_brokers.py b/django_q/tests/test_brokers.py index d253f9b..661d0f5 100644 --- a/django_q/tests/test_brokers.py +++ b/django_q/tests/test_brokers.py @@ -124,7 +124,7 @@ def test_ironmq(monkeypatch): @pytest.mark.skipif( not os.getenv("AWS_ACCESS_KEY_ID"), reason="requires AWS credentials" ) -def canceled_sqs(monkeypatch): +def test_sqs(monkeypatch): monkeypatch.setattr( Conf, "SQS", @@ -132,11 +132,13 @@ def canceled_sqs(monkeypatch): "aws_region": os.getenv("AWS_REGION"), "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"), "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"), - "receive_message_wait_time_seconds": 20, + "receive_message_wait_time_seconds": 5, }, ) # check broker - broker = get_broker(list_key=uuid()[0]) + broker = get_broker(list_key="testing") + assert "receive_message_wait_time_seconds" in Conf.SQS + assert "aws_region" in Conf.SQS assert broker.ping() is True assert broker.info() is not None assert broker.queue_size() == 0 @@ -173,7 +175,7 @@ def canceled_sqs(monkeypatch): broker.enqueue("test") while task is None: task = broker.dequeue()[0] - broker.fail(task[0]) + broker.fail(task[0][0]) # bulk test for _ in range(10): broker.enqueue("test") diff --git a/test-services-docker-compose.yaml b/test-services-docker-compose.yaml index 5f8818f..c0b48d6 100644 --- a/test-services-docker-compose.yaml +++ b/test-services-docker-compose.yaml @@ -13,15 +13,41 @@ services: networks: - main + aws: + container_name: aws + image: localstack/localstack:3.4.0 + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # External services port range + environment: + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-west-2} + DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-west-2} + SQS_ENDPOINT_STRATEGY: path + SERVICES: sqs + LOCALSTACK_HOST: aws + DEBUG: 1 + LS_LOG: trace + volumes: + - ./containers/localstack:/etc/localstack/init/ready.d + networks: + - main + django-q2: build: dockerfile: ./Dockerfile.dev context: . + environment: + AWS_ENDPOINT_URL: http://aws:4566 + AWS_REGION: ${AWS_REGION:-us-west-2} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-test} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-test} + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-west-2} volumes: - .:/app depends_on: - redis - mongo + - aws networks: - main