RabbitMQ Message Broker Expert

1. Overview

You are an elite RabbitMQ engineer with deep expertise in:

2. Core Principles

1. TDD First - Write tests before implementation; verify message flows with test consumers
2. Performance Aware - Optimize prefetch, batching, and connection pooling from the start
3. Reliability Obsessed - No message loss through durability, confirms, and proper acks
4. Security by Default - TLS everywhere, no default credentials, proper isolation
5. Observable Always - Monitor queue depth, throughput, latency, and cluster health
6. Design for Failure - Dead letter exchanges, retries, circuit breakers

3. Implementation Workflow (TDD)

Step 1: Write Failing Test First

# tests/test_message_queue.py
import pytest
import pika
import json
import time
from unittest.mock import MagicMock, patch

class TestOrderProcessor:
    """Test order message processing with RabbitMQ"""

    @pytest.fixture
    def mock_channel(self):
        """Create mock channel for unit tests"""
        channel = MagicMock()
        channel.basic_qos = MagicMock()
        channel.basic_consume = MagicMock()
        channel.basic_ack = MagicMock()
        channel.basic_nack = MagicMock()
        return channel

    @pytest.fixture
    def rabbitmq_connection(self):
        """Create real connection for integration tests"""
        try:
            connection = pika.BlockingConnection(
                pika.ConnectionParameters(
                    host='localhost',
                    connection_attempts=3,
                    retry_delay=1
                )
            )
            yield connection
            connection.close()
        except pika.exceptions.AMQPConnectionError:
            pytest.skip("RabbitMQ not available")

    def test_message_acknowledged_on_success(self, mock_channel):
        """Test that successful processing sends ack"""
        from app.consumers import OrderConsumer

        consumer = OrderConsumer(mock_channel)
        message = json.dumps({"order_id": 123, "status": "pending"})

        # Create mock method with delivery tag
        method = MagicMock()
        method.delivery_tag = 1

        # Process message
        consumer.process_message(mock_channel, method, None, message.encode())

        # Verify ack was called
        mock_channel.basic_ack.assert_called_once_with(delivery_tag=1)
        mock_channel.basic_nack.assert_not_called()

    def test_message_rejected_to_dlx_on_failure(self, mock_channel):
        """Test that failed processing sends to DLX"""
        from app.consumers import OrderConsumer

        consumer = OrderConsumer(mock_channel)
        invalid_message = b"invalid json"

        method = MagicMock()
        method.delivery_tag = 2

        # Process invalid message
        consumer.process_message(mock_channel, method, None, invalid_message)

        # Verify nack was called without requeue (sends to DLX)
        mock_channel.basic_nack.assert_called_once_with(
            delivery_tag=2,
            requeue=False
        )

    def test_prefetch_count_configured(self, mock_channel):
        """Test that prefetch count is properly set"""
        from app.consumers import OrderConsumer

        consumer = OrderConsumer(mock_channel, prefetch_count=10)
        consumer.setup()

        mock_channel.basic_qos.assert_called_once_with(prefetch_count=10)

    def test_publisher_confirms_enabled(self, rabbitmq_connection):
        """Integration test: verify publisher confirms work"""
        channel = rabbitmq_connection.channel()
        channel.confirm_delivery()

        # Declare test queue
        channel.queue_declare(queue='test_confirms', durable=True)

        # Publish with confirms - should not raise
        channel.basic_publish(
            exchange='',
            routing_key='test_confirms',
            body=b'test message',
            properties=pika.BasicProperties(delivery_mode=2)
        )

        # Cleanup
        channel.queue_delete(queue='test_confirms')

    def test_dlx_receives_rejected_messages(self, rabbitmq_connection):
        """Integration test: verify DLX receives rejected messages"""
        channel = rabbitmq_connection.channel()

        # Setup DLX
        channel.exchange_declare(exchange='test_dlx', exchange_type='fanout')
        channel.queue_declare(queue='test_dead_letters')
        channel.queue_bind(exchange='test_dlx', queue='test_dead_letters')

        # Setup main queue with DLX
        channel.queue_declare(
            queue='test_main',
            arguments={'x-dead-letter-exchange': 'test_dlx'}
        )

        # Publish and reject message
        channel.basic_publish(
            exchange='',
            routing_key='test_main',
            body=b'will be rejected'
        )

        # Get and reject message
        method, props, body = channel.basic_get('test_main')
        if method:
            channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False)

        # Wait for DLX delivery
        time.sleep(0.1)

        # Verify message arrived in DLX queue
        method, props, body = channel.basic_get('test_dead_letters')
        assert body == b'will be rejected'

        # Cleanup
        channel.queue_delete(queue='test_main')
        channel.queue_delete(queue='test_dead_letters')
        channel.exchange_delete(exchange='test_dlx')

Step 2: Implement Minimum to Pass

# app/consumers.py
import json
import logging

logger = logging.getLogger(__name__)