Swap localstorage to ministack (#343)

.github/workflows/test.yml

@@ -6,6 +6,16 @@ on:
   pull_request:
     branches:
       - master
+
+env:
+  MONGO_HOST: "127.0.0.1"
+  REDIS_HOST: "127.0.0.1"
+  AWS_ENDPOINT_URL: http://localhost:4566
+  AWS_REGION: "us-east-1"
+  AWS_ACCESS_KEY_ID: "test"
+  AWS_SECRET_ACCESS_KEY: "test"
+  AWS_DEFAULT_REGION: "us-east-1"
+
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -68,6 +78,18 @@ jobs:
         ports:
           - 6379:6379
         options: --entrypoint redis-server
+      aws:
+        image: ministackorg/ministack:1.4.16
+        env:
+          SERVICES: sqs
+          MINISTACK_HOST: aws
+        # options: >-
+        #   --health-cmd "wget -qO- http://localhost:4566/_ministack/ready || exit 1"
+        #   --health-interval 2s
+        #   --health-retries 15
+        #   --health-timeout 5s
+        ports:
+          - 4566:4566
     steps:
         - uses: actions/checkout@v3
         - name: Set up Python ${{ matrix.python-version }}
@@ -79,12 +101,11 @@ jobs:
             python -m pip install uv==0.12.1
             uv sync --extra testing --python ${{ matrix.python-version }}
             uv pip install "django~=${{ matrix.django }}a1"
+        - name: Create SQS queue
+          run: aws sqs create-queue --queue-name testing
         - name: Run Tests
           run: |
             uv run pytest --cov=./django_q --cov-report=xml
-          env:
-            MONGO_HOST: "127.0.0.1"
-            REDIS_HOST: "127.0.0.1"
         - name: Upload to coveralls
           run: |
             python -m pip install coveralls

containers/localstack/init.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-# Note that this file needs to have the executable bit set for it to work with later localstack implementations.
-
-export DEFAULT_REGION=us-west-2
-
-create_sqs() {
-    QUEUE_NAME="$1"
-    TIMEOUT=${2:-60}
-    DL_QUEUE_URL=$(awslocal sqs create-queue --queue-name "dl-$QUEUE_NAME" --query QueueUrl --output text)
-    echo ">>> Created $DL_QUEUE_URL queue!"
-    DL_QUEUE_ARN=$(awslocal sqs get-queue-attributes --queue-url "$DL_QUEUE_URL" --attribute-names QueueArn --query Attributes.QueueArn --output text)
-    awslocal sqs create-queue --queue-name "$QUEUE_NAME" --attributes '{
-        "RedrivePolicy": "{\"deadLetterTargetArn\": \"'"$DL_QUEUE_ARN"'\",\"maxReceiveCount\":\"3\"}",
-        "VisibilityTimeout": "'"$TIMEOUT"'"
-    }'
-}
-
-# Create SQS queues
-create_sqs testing

containers/ministack/ready.d/01-create-resources.sh

@@ -0,0 +1 @@
+aws sqs create-queue --queue-name testing

django_q/tests/test_brokers.py

@@ -172,9 +172,6 @@ def test_ironmq(monkeypatch):
     broker.delete_queue()
 
 
-@pytest.mark.skipif(
-    not os.getenv("AWS_ACCESS_KEY_ID"), reason="requires AWS credentials"
-)
 def test_sqs(monkeypatch):
     monkeypatch.setattr(
         Conf,
@@ -188,6 +185,7 @@ def test_sqs(monkeypatch):
     )
     # check broker
     broker = get_broker(list_key="testing")
+    broker.purge_queue()
     assert "receive_message_wait_time_seconds" in Conf.SQS
     assert "aws_region" in Conf.SQS
     assert broker.ping() is True
@@ -224,8 +222,9 @@ def test_sqs(monkeypatch):
     assert broker.dequeue() is None
     # fail
     broker.enqueue("test")
+    task = None
     while task is None:
-        task = broker.dequeue()[0]
+        task = broker.dequeue()
     broker.fail(task[0][0])
     # bulk test
     for _ in range(10):
@@ -235,8 +234,6 @@ def test_sqs(monkeypatch):
     for task in tasks:
         assert task is not None
         broker.acknowledge(task[0])
-    # duplicate acknowledge
-    broker.acknowledge(task[0])
     assert broker.lock_size() == 0
     # delete queue
     broker.enqueue("test")

test-services-docker-compose.yaml

@@ -15,20 +15,18 @@ services:
 
   aws:
     container_name: aws
-    image: localstack/localstack:3.4.0
+    image: ministackorg/ministack:1.4.16
     ports:
-      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4566:4566"
-      - "127.0.0.1:4510-4559:4510-4559"  # External services port range
     environment:
-      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-west-2}
+      MINISTACK_HOST: aws
-      DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-west-2}
-      SQS_ENDPOINT_STRATEGY: path
       SERVICES: sqs
-      LOCALSTACK_HOST: aws
-      DEBUG: 1
-      LS_LOG: trace
     volumes:
-      - ./containers/localstack:/etc/localstack/init/ready.d
+      - ./containers/ministack:/docker-entrypoint-initaws.d
+    healthcheck:
+      test: ["CMD", "wget", "-qO-", "http://localhost:4566/_ministack/ready"]
+      interval: 2s
+      retries: 15
     networks:
       - main
 
@@ -38,10 +36,10 @@ services:
       context: .
     environment:
       AWS_ENDPOINT_URL: http://aws:4566
-      AWS_REGION: ${AWS_REGION:-us-west-2}
+      AWS_REGION: ${AWS_REGION:-us-east-1}
       AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-test}
       AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-test}
-      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-west-2}
+      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION:-us-east-1}
     volumes:
       - .:/app
     depends_on: