"""Tests for Phase 5: Outbox DLQ, Monitoring, Consumer-Registry. Covers: - DLQ: failed events have error_message and failed_at set - Replay: single and bulk replay resets failed events to pending - Stats: get_outbox_stats returns correct counts per status - Failed events listing: get_failed_events returns error details - Consumer registry: get_consumer_registry returns event→handlers mapping - outbox_deliveries: entries are written during event processing """ from __future__ import annotations import uuid import pytest from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from app.core.db import set_tenant_context from app.core.event_bus import get_event_bus from app.core.outbox import ( enqueue_outbox_event, get_consumer_registry, get_failed_events, get_outbox_stats, process_outbox_batch, replay_all_failed_events, replay_failed_event, ) # ── DLQ: error_message and failed_at ────────────────────────────────────────── @pytest.mark.asyncio async def test_failed_event_has_error_message( db_session: AsyncSession, session_factory: async_sessionmaker[AsyncSession], ): """When a handler fails permanently, error_message and failed_at are set.""" tenant_id = uuid.uuid4() async def _always_fails(payload: dict) -> None: raise RuntimeError("Permanent failure for DLQ test") bus = get_event_bus() bus.subscribe("test.dlq.fail", _always_fails) try: # Insert event with attempts = max_attempts - 1 so next failure marks it failed await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts) " "VALUES (:tid, 'test.dlq.fail', CAST(:payload AS JSONB), 'pending', 4, 5)" ), {"tid": str(tenant_id), "payload": '{"k": "v"}'}, ) await db_session.flush() await db_session.commit() async with session_factory() as worker_session: count = await process_outbox_batch( worker_session, batch_size=10, tenant_ids=[tenant_id] ) assert count == 0 # nothing was successfully published await set_tenant_context(db_session, tenant_id) rows = ( await db_session.execute( text( "SELECT status, error_message, failed_at " "FROM event_outbox WHERE tenant_id = :tid" ), {"tid": str(tenant_id)}, ) ).fetchall() assert rows[0][0] == "failed" assert rows[0][1] is not None assert "Permanent failure" in rows[0][1] assert rows[0][2] is not None # failed_at finally: bus.unsubscribe("test.dlq.fail", _always_fails) # ── Replay single failed event ──────────────────────────────────────────────── @pytest.mark.asyncio async def test_replay_failed_event(db_session: AsyncSession): """replay_failed_event resets a failed event to pending.""" tenant_id = uuid.uuid4() await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) " "VALUES (:tid, 'test.replay', CAST(:payload AS JSONB), 'failed', 5, 5, 'some error', now())" ), {"tid": str(tenant_id), "payload": '{"k": "v"}'}, ) await db_session.flush() await db_session.commit() row = ( await db_session.execute( text("SELECT id FROM event_outbox WHERE tenant_id = :tid"), {"tid": str(tenant_id)}, ) ).first() event_id = row[0] await set_tenant_context(db_session, tenant_id) replayed = await replay_failed_event(db_session, event_id) await db_session.commit() assert replayed is True rows = ( await db_session.execute( text( "SELECT status, attempts, error_message, next_retry_at " "FROM event_outbox WHERE id = :eid" ), {"eid": str(event_id)}, ) ).fetchall() assert rows[0][0] == "pending" assert rows[0][1] == 0 assert rows[0][2] is None assert rows[0][3] is None @pytest.mark.asyncio async def test_replay_failed_event_not_found(db_session: AsyncSession): """replay_failed_event returns False for non-existent or non-failed events.""" tenant_id = uuid.uuid4() fake_id = uuid.uuid4() await set_tenant_context(db_session, tenant_id) replayed = await replay_failed_event(db_session, fake_id) await db_session.commit() assert replayed is False # ── Replay all failed events for a tenant ───────────────────────────────────── @pytest.mark.asyncio async def test_replay_all_failed_events(db_session: AsyncSession): """replay_all_failed_events resets all failed events for a tenant.""" tenant_id = uuid.uuid4() for i in range(3): await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) " "VALUES (:tid, :event_name, CAST(:payload AS JSONB), 'failed', 5, 5, :error, now())" ), { "tid": str(tenant_id), "event_name": f"test.replay_all.{i}", "payload": '{"k": "v"}', "error": f"error {i}", }, ) await db_session.flush() await db_session.commit() await set_tenant_context(db_session, tenant_id) count = await replay_all_failed_events(db_session, tenant_id) await db_session.commit() assert count == 3 rows = ( await db_session.execute( text( "SELECT status, attempts, error_message " "FROM event_outbox WHERE tenant_id = :tid" ), {"tid": str(tenant_id)}, ) ).fetchall() for row in rows: assert row[0] == "pending" assert row[1] == 0 assert row[2] is None # ── Outbox stats ─────────────────────────────────────────────────────────────── @pytest.mark.asyncio async def test_get_outbox_stats(db_session: AsyncSession): """get_outbox_stats returns correct counts per status.""" tenant_id = uuid.uuid4() for status in ["pending", "pending", "published", "failed"]: await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status) " "VALUES (:tid, 'test.stats', CAST(:payload AS JSONB), :status)" ), {"tid": str(tenant_id), "payload": '{"k": "v"}', "status": status}, ) await db_session.flush() await db_session.commit() await set_tenant_context(db_session, tenant_id) stats = await get_outbox_stats(db_session) assert stats["counts"]["pending"] == 2 assert stats["counts"]["published"] == 1 assert stats["counts"]["failed"] == 1 assert stats["total"] == 4 assert stats["oldest_pending_age_seconds"] is not None @pytest.mark.asyncio async def test_get_outbox_stats_empty(db_session: AsyncSession): """get_outbox_stats returns zeros when no events exist.""" tenant_id = uuid.uuid4() await set_tenant_context(db_session, tenant_id) stats = await get_outbox_stats(db_session) assert stats["total"] == 0 assert stats["oldest_pending_age_seconds"] is None # ── Failed events listing ───────────────────────────────────────────────────── @pytest.mark.asyncio async def test_get_failed_events(db_session: AsyncSession): """get_failed_events returns failed events with error details.""" tenant_id = uuid.uuid4() await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) " "VALUES (:tid, 'test.failed_list', CAST(:payload AS JSONB), 'failed', 5, 5, 'test error', now())" ), {"tid": str(tenant_id), "payload": '{"k": "v"}'}, ) await db_session.flush() await db_session.commit() await set_tenant_context(db_session, tenant_id) events = await get_failed_events(db_session, limit=10, offset=0) assert len(events) == 1 assert events[0]["event_name"] == "test.failed_list" assert events[0]["error_message"] == "test error" assert events[0]["status"] == "failed" assert events[0]["attempts"] == 5 @pytest.mark.asyncio async def test_get_failed_events_pagination(db_session: AsyncSession): """get_failed_events respects limit and offset.""" tenant_id = uuid.uuid4() for i in range(5): await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) " "VALUES (:tid, :name, CAST(:payload AS JSONB), 'failed', 5, 5, :error, now())" ), { "tid": str(tenant_id), "name": f"test.pag.{i}", "payload": '{"k": "v"}', "error": f"err {i}", }, ) await db_session.flush() await db_session.commit() await set_tenant_context(db_session, tenant_id) page1 = await get_failed_events(db_session, limit=2, offset=0) page2 = await get_failed_events(db_session, limit=2, offset=2) assert len(page1) == 2 assert len(page2) == 2 # ── Consumer registry ────────────────────────────────────────────────────────── @pytest.mark.asyncio async def test_get_consumer_registry(): """get_consumer_registry returns event_name → consumer_names mapping.""" bus = get_event_bus() async def handler_alpha(payload: dict) -> None: pass async def handler_beta(payload: dict) -> None: pass bus.subscribe("test.registry.event", handler_alpha) bus.subscribe("test.registry.event", handler_beta) try: registry = get_consumer_registry() assert "test.registry.event" in registry assert len(registry["test.registry.event"]) == 2 assert "handler_alpha" in registry["test.registry.event"] assert "handler_beta" in registry["test.registry.event"] finally: bus.unsubscribe("test.registry.event", handler_alpha) bus.unsubscribe("test.registry.event", handler_beta) # ── outbox_deliveries written during processing ─────────────────────────────── @pytest.mark.asyncio async def test_outbox_deliveries_written_on_success( db_session: AsyncSession, session_factory: async_sessionmaker[AsyncSession], ): """outbox_deliveries entries are written with status='delivered' on success.""" tenant_id = uuid.uuid4() async def _success_handler(payload: dict) -> None: pass bus = get_event_bus() bus.subscribe("test.deliveries.ok", _success_handler) try: await enqueue_outbox_event( db_session, tenant_id, "test.deliveries.ok", {"key": "value"} ) await db_session.flush() await db_session.commit() async with session_factory() as worker_session: count = await process_outbox_batch( worker_session, batch_size=10, tenant_ids=[tenant_id] ) assert count == 1 await set_tenant_context(db_session, tenant_id) rows = ( await db_session.execute( text( "SELECT d.consumer_name, d.status, d.last_error " "FROM outbox_deliveries d " "JOIN event_outbox e ON d.event_id = e.id " "WHERE e.tenant_id = :tid" ), {"tid": str(tenant_id)}, ) ).fetchall() assert len(rows) == 1 assert rows[0][0] == "_success_handler" assert rows[0][1] == "delivered" assert rows[0][2] is None # no error finally: bus.unsubscribe("test.deliveries.ok", _success_handler) @pytest.mark.asyncio async def test_outbox_deliveries_written_on_failure( db_session: AsyncSession, session_factory: async_sessionmaker[AsyncSession], ): """outbox_deliveries entries are written with status='failed' on handler error.""" tenant_id = uuid.uuid4() async def _failing_handler(payload: dict) -> None: raise RuntimeError("Delivery failure") bus = get_event_bus() bus.subscribe("test.deliveries.fail", _failing_handler) try: await enqueue_outbox_event( db_session, tenant_id, "test.deliveries.fail", {"key": "value"} ) await db_session.flush() await db_session.commit() async with session_factory() as worker_session: count = await process_outbox_batch( worker_session, batch_size=10, tenant_ids=[tenant_id] ) assert count == 0 # not published await set_tenant_context(db_session, tenant_id) rows = ( await db_session.execute( text( "SELECT d.consumer_name, d.status, d.last_error " "FROM outbox_deliveries d " "JOIN event_outbox e ON d.event_id = e.id " "WHERE e.tenant_id = :tid" ), {"tid": str(tenant_id)}, ) ).fetchall() assert len(rows) == 1 assert rows[0][0] == "_failing_handler" assert rows[0][1] == "failed" assert rows[0][2] is not None assert "Delivery failure" in rows[0][2] finally: bus.unsubscribe("test.deliveries.fail", _failing_handler) # ── Route import smoke test ──────────────────────────────────────────────────── def test_route_import(): """Verify that the outbox route module imports cleanly.""" from app.routes.outbox import router assert router is not None assert router.prefix == "/api/v1/outbox" # Check all 5 endpoints are registered paths = {route.path for route in router.routes} assert "/api/v1/outbox/stats" in paths assert "/api/v1/outbox/failed" in paths assert "/api/v1/outbox/replay/{event_id}" in paths assert "/api/v1/outbox/replay-all" in paths assert "/api/v1/outbox/consumer-registry" in paths # ── Processing recovery (stuck events) ───────────────────────────────────────── @pytest.mark.asyncio async def test_recover_stuck_events(db_session: AsyncSession): """recover_stuck_events resets events stuck in 'processing' back to 'pending'.""" from app.core.outbox import recover_stuck_events tenant_id = uuid.uuid4() # Insert a processing event with old updated_at (simulating crash) await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, updated_at) " "VALUES (:tid, 'test.stuck', CAST(:payload AS JSONB), 'processing', now() - interval '5 minutes')" ), {"tid": str(tenant_id), "payload": '{"k": "v"}'}, ) await db_session.flush() await db_session.commit() await set_tenant_context(db_session, tenant_id) count = await recover_stuck_events(db_session, timeout_seconds=60) await db_session.commit() assert count == 1 rows = ( await db_session.execute( text("SELECT status FROM event_outbox WHERE tenant_id = :tid"), {"tid": str(tenant_id)}, ) ).fetchall() assert rows[0][0] == "pending" @pytest.mark.asyncio async def test_recover_stuck_events_skips_recent(db_session: AsyncSession): """recover_stuck_events does not reset recently-claimed processing events.""" from app.core.outbox import recover_stuck_events tenant_id = uuid.uuid4() # Insert a processing event with recent updated_at (should not be recovered) await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, updated_at) " "VALUES (:tid, 'test.stuck.recent', CAST(:payload AS JSONB), 'processing', now())" ), {"tid": str(tenant_id), "payload": '{"k": "v"}'}, ) await db_session.flush() await db_session.commit() await set_tenant_context(db_session, tenant_id) count = await recover_stuck_events(db_session, timeout_seconds=60) await db_session.commit() assert count == 0 # should not recover recent events # ── Retention cleanup ────────────────────────────────────────────────────────── @pytest.mark.asyncio async def test_cleanup_published_events(db_session: AsyncSession): """cleanup_published_events deletes old published events.""" from app.core.outbox import cleanup_published_events tenant_id = uuid.uuid4() # Insert an old published event (31 days ago) await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, published_at) " "VALUES (:tid, 'test.old', CAST(:payload AS JSONB), 'published', now() - interval '31 days')" ), {"tid": str(tenant_id), "payload": '{"k": "v"}'}, ) # Insert a recent published event (1 day ago) await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, published_at) " "VALUES (:tid, 'test.recent', CAST(:payload AS JSONB), 'published', now() - interval '1 day')" ), {"tid": str(tenant_id), "payload": '{"k": "v"}'}, ) await db_session.flush() await db_session.commit() await set_tenant_context(db_session, tenant_id) deleted = await cleanup_published_events(db_session, retention_days=30) await db_session.commit() assert deleted == 1 # only the old one rows = ( await db_session.execute( text("SELECT event_name FROM event_outbox WHERE tenant_id = :tid"), {"tid": str(tenant_id)}, ) ).fetchall() assert len(rows) == 1 assert rows[0][0] == "test.recent" # ── Replay resets outbox_deliveries ───────────────────────────────────────────── @pytest.mark.asyncio async def test_replay_resets_deliveries(db_session: AsyncSession): """replay_failed_event also deletes old outbox_deliveries for the event.""" tenant_id = uuid.uuid4() # Insert a failed event await db_session.execute( text( "INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) " "VALUES (:tid, 'test.replay.deliveries', CAST(:payload AS JSONB), 'failed', 5, 5, 'err', now())" ), {"tid": str(tenant_id), "payload": '{"k": "v"}'}, ) await db_session.flush() await db_session.commit() row = ( await db_session.execute( text("SELECT id FROM event_outbox WHERE tenant_id = :tid"), {"tid": str(tenant_id)}, ) ).first() event_id = row[0] # Insert a delivery record for this event await db_session.execute( text( "INSERT INTO outbox_deliveries (event_id, consumer_name, status, attempt_count, last_error) " "VALUES (:eid, 'test_consumer', 'failed', 5, 'err')" ), {"eid": str(event_id)}, ) await db_session.flush() await db_session.commit() await set_tenant_context(db_session, tenant_id) replayed = await replay_failed_event(db_session, event_id) await db_session.commit() assert replayed is True # Check deliveries were deleted delivery_rows = ( await db_session.execute( text("SELECT count(*) FROM outbox_deliveries WHERE event_id = :eid"), {"eid": str(event_id)}, ) ).first() assert delivery_rows[0] == 0 # ── New endpoints registered ──────────────────────────────────────────────────── def test_new_endpoints_registered(): """Verify that recover-stuck and cleanup-published endpoints are registered.""" from app.routes.outbox import router paths = {route.path for route in router.routes} assert "/api/v1/outbox/recover-stuck" in paths assert "/api/v1/outbox/cleanup-published" in paths