Agent Zero
07a99975ec
- Migration 0092: DLQ columns (error_message, failed_at) + consumer_inbox RLS fix - outbox.py: DLQ logic, replay functions, stats, consumer registry - app/routes/outbox.py: 5 API endpoints (stats, failed, replay, replay-all, consumer-registry) - outbox_deliveries tracking per consumer handler - 18/18 tests passing
423 lines
15 KiB
Python
423 lines
15 KiB
Python
"""Tests for Phase 5: Outbox DLQ, Monitoring, Consumer-Registry.
|
|
|
|
Covers:
|
|
- DLQ: failed events have error_message and failed_at set
|
|
- Replay: single and bulk replay resets failed events to pending
|
|
- Stats: get_outbox_stats returns correct counts per status
|
|
- Failed events listing: get_failed_events returns error details
|
|
- Consumer registry: get_consumer_registry returns event→handlers mapping
|
|
- outbox_deliveries: entries are written during event processing
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
|
|
import pytest
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
|
|
|
from app.core.db import set_tenant_context
|
|
from app.core.event_bus import get_event_bus
|
|
from app.core.outbox import (
|
|
enqueue_outbox_event,
|
|
get_consumer_registry,
|
|
get_failed_events,
|
|
get_outbox_stats,
|
|
process_outbox_batch,
|
|
replay_all_failed_events,
|
|
replay_failed_event,
|
|
)
|
|
|
|
|
|
# ── DLQ: error_message and failed_at ──────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_failed_event_has_error_message(
|
|
db_session: AsyncSession,
|
|
session_factory: async_sessionmaker[AsyncSession],
|
|
):
|
|
"""When a handler fails permanently, error_message and failed_at are set."""
|
|
tenant_id = uuid.uuid4()
|
|
|
|
async def _always_fails(payload: dict) -> None:
|
|
raise RuntimeError("Permanent failure for DLQ test")
|
|
|
|
bus = get_event_bus()
|
|
bus.subscribe("test.dlq.fail", _always_fails)
|
|
try:
|
|
# Insert event with attempts = max_attempts - 1 so next failure marks it failed
|
|
await db_session.execute(
|
|
text(
|
|
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts) "
|
|
"VALUES (:tid, 'test.dlq.fail', CAST(:payload AS JSONB), 'pending', 4, 5)"
|
|
),
|
|
{"tid": str(tenant_id), "payload": '{"k": "v"}'},
|
|
)
|
|
await db_session.flush()
|
|
await db_session.commit()
|
|
|
|
async with session_factory() as worker_session:
|
|
count = await process_outbox_batch(
|
|
worker_session, batch_size=10, tenant_ids=[tenant_id]
|
|
)
|
|
|
|
assert count == 0 # nothing was successfully published
|
|
|
|
await set_tenant_context(db_session, tenant_id)
|
|
rows = (
|
|
await db_session.execute(
|
|
text(
|
|
"SELECT status, error_message, failed_at "
|
|
"FROM event_outbox WHERE tenant_id = :tid"
|
|
),
|
|
{"tid": str(tenant_id)},
|
|
)
|
|
).fetchall()
|
|
assert rows[0][0] == "failed"
|
|
assert rows[0][1] is not None
|
|
assert "Permanent failure" in rows[0][1]
|
|
assert rows[0][2] is not None # failed_at
|
|
finally:
|
|
bus.unsubscribe("test.dlq.fail", _always_fails)
|
|
|
|
|
|
# ── Replay single failed event ────────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_replay_failed_event(db_session: AsyncSession):
|
|
"""replay_failed_event resets a failed event to pending."""
|
|
tenant_id = uuid.uuid4()
|
|
|
|
await db_session.execute(
|
|
text(
|
|
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) "
|
|
"VALUES (:tid, 'test.replay', CAST(:payload AS JSONB), 'failed', 5, 5, 'some error', now())"
|
|
),
|
|
{"tid": str(tenant_id), "payload": '{"k": "v"}'},
|
|
)
|
|
await db_session.flush()
|
|
await db_session.commit()
|
|
|
|
row = (
|
|
await db_session.execute(
|
|
text("SELECT id FROM event_outbox WHERE tenant_id = :tid"),
|
|
{"tid": str(tenant_id)},
|
|
)
|
|
).first()
|
|
event_id = row[0]
|
|
|
|
await set_tenant_context(db_session, tenant_id)
|
|
replayed = await replay_failed_event(db_session, event_id)
|
|
await db_session.commit()
|
|
|
|
assert replayed is True
|
|
|
|
rows = (
|
|
await db_session.execute(
|
|
text(
|
|
"SELECT status, attempts, error_message, next_retry_at "
|
|
"FROM event_outbox WHERE id = :eid"
|
|
),
|
|
{"eid": str(event_id)},
|
|
)
|
|
).fetchall()
|
|
assert rows[0][0] == "pending"
|
|
assert rows[0][1] == 0
|
|
assert rows[0][2] is None
|
|
assert rows[0][3] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_replay_failed_event_not_found(db_session: AsyncSession):
|
|
"""replay_failed_event returns False for non-existent or non-failed events."""
|
|
tenant_id = uuid.uuid4()
|
|
fake_id = uuid.uuid4()
|
|
|
|
await set_tenant_context(db_session, tenant_id)
|
|
replayed = await replay_failed_event(db_session, fake_id)
|
|
await db_session.commit()
|
|
|
|
assert replayed is False
|
|
|
|
|
|
# ── Replay all failed events for a tenant ─────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_replay_all_failed_events(db_session: AsyncSession):
|
|
"""replay_all_failed_events resets all failed events for a tenant."""
|
|
tenant_id = uuid.uuid4()
|
|
|
|
for i in range(3):
|
|
await db_session.execute(
|
|
text(
|
|
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) "
|
|
"VALUES (:tid, :event_name, CAST(:payload AS JSONB), 'failed', 5, 5, :error, now())"
|
|
),
|
|
{
|
|
"tid": str(tenant_id),
|
|
"event_name": f"test.replay_all.{i}",
|
|
"payload": '{"k": "v"}',
|
|
"error": f"error {i}",
|
|
},
|
|
)
|
|
await db_session.flush()
|
|
await db_session.commit()
|
|
|
|
await set_tenant_context(db_session, tenant_id)
|
|
count = await replay_all_failed_events(db_session, tenant_id)
|
|
await db_session.commit()
|
|
|
|
assert count == 3
|
|
|
|
rows = (
|
|
await db_session.execute(
|
|
text(
|
|
"SELECT status, attempts, error_message "
|
|
"FROM event_outbox WHERE tenant_id = :tid"
|
|
),
|
|
{"tid": str(tenant_id)},
|
|
)
|
|
).fetchall()
|
|
for row in rows:
|
|
assert row[0] == "pending"
|
|
assert row[1] == 0
|
|
assert row[2] is None
|
|
|
|
|
|
# ── Outbox stats ───────────────────────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_outbox_stats(db_session: AsyncSession):
|
|
"""get_outbox_stats returns correct counts per status."""
|
|
tenant_id = uuid.uuid4()
|
|
|
|
for status in ["pending", "pending", "published", "failed"]:
|
|
await db_session.execute(
|
|
text(
|
|
"INSERT INTO event_outbox (tenant_id, event_name, payload, status) "
|
|
"VALUES (:tid, 'test.stats', CAST(:payload AS JSONB), :status)"
|
|
),
|
|
{"tid": str(tenant_id), "payload": '{"k": "v"}', "status": status},
|
|
)
|
|
await db_session.flush()
|
|
await db_session.commit()
|
|
|
|
await set_tenant_context(db_session, tenant_id)
|
|
stats = await get_outbox_stats(db_session)
|
|
|
|
assert stats["counts"]["pending"] == 2
|
|
assert stats["counts"]["published"] == 1
|
|
assert stats["counts"]["failed"] == 1
|
|
assert stats["total"] == 4
|
|
assert stats["oldest_pending_age_seconds"] is not None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_outbox_stats_empty(db_session: AsyncSession):
|
|
"""get_outbox_stats returns zeros when no events exist."""
|
|
tenant_id = uuid.uuid4()
|
|
await set_tenant_context(db_session, tenant_id)
|
|
|
|
stats = await get_outbox_stats(db_session)
|
|
|
|
assert stats["total"] == 0
|
|
assert stats["oldest_pending_age_seconds"] is None
|
|
|
|
|
|
# ── Failed events listing ─────────────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_failed_events(db_session: AsyncSession):
|
|
"""get_failed_events returns failed events with error details."""
|
|
tenant_id = uuid.uuid4()
|
|
|
|
await db_session.execute(
|
|
text(
|
|
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) "
|
|
"VALUES (:tid, 'test.failed_list', CAST(:payload AS JSONB), 'failed', 5, 5, 'test error', now())"
|
|
),
|
|
{"tid": str(tenant_id), "payload": '{"k": "v"}'},
|
|
)
|
|
await db_session.flush()
|
|
await db_session.commit()
|
|
|
|
await set_tenant_context(db_session, tenant_id)
|
|
events = await get_failed_events(db_session, limit=10, offset=0)
|
|
|
|
assert len(events) == 1
|
|
assert events[0]["event_name"] == "test.failed_list"
|
|
assert events[0]["error_message"] == "test error"
|
|
assert events[0]["status"] == "failed"
|
|
assert events[0]["attempts"] == 5
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_failed_events_pagination(db_session: AsyncSession):
|
|
"""get_failed_events respects limit and offset."""
|
|
tenant_id = uuid.uuid4()
|
|
|
|
for i in range(5):
|
|
await db_session.execute(
|
|
text(
|
|
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) "
|
|
"VALUES (:tid, :name, CAST(:payload AS JSONB), 'failed', 5, 5, :error, now())"
|
|
),
|
|
{
|
|
"tid": str(tenant_id),
|
|
"name": f"test.pag.{i}",
|
|
"payload": '{"k": "v"}',
|
|
"error": f"err {i}",
|
|
},
|
|
)
|
|
await db_session.flush()
|
|
await db_session.commit()
|
|
|
|
await set_tenant_context(db_session, tenant_id)
|
|
|
|
page1 = await get_failed_events(db_session, limit=2, offset=0)
|
|
page2 = await get_failed_events(db_session, limit=2, offset=2)
|
|
|
|
assert len(page1) == 2
|
|
assert len(page2) == 2
|
|
|
|
|
|
# ── Consumer registry ──────────────────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_consumer_registry():
|
|
"""get_consumer_registry returns event_name → consumer_names mapping."""
|
|
bus = get_event_bus()
|
|
|
|
async def handler_alpha(payload: dict) -> None:
|
|
pass
|
|
|
|
async def handler_beta(payload: dict) -> None:
|
|
pass
|
|
|
|
bus.subscribe("test.registry.event", handler_alpha)
|
|
bus.subscribe("test.registry.event", handler_beta)
|
|
try:
|
|
registry = get_consumer_registry()
|
|
assert "test.registry.event" in registry
|
|
assert len(registry["test.registry.event"]) == 2
|
|
assert "handler_alpha" in registry["test.registry.event"]
|
|
assert "handler_beta" in registry["test.registry.event"]
|
|
finally:
|
|
bus.unsubscribe("test.registry.event", handler_alpha)
|
|
bus.unsubscribe("test.registry.event", handler_beta)
|
|
|
|
|
|
# ── outbox_deliveries written during processing ───────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_outbox_deliveries_written_on_success(
|
|
db_session: AsyncSession,
|
|
session_factory: async_sessionmaker[AsyncSession],
|
|
):
|
|
"""outbox_deliveries entries are written with status='delivered' on success."""
|
|
tenant_id = uuid.uuid4()
|
|
|
|
async def _success_handler(payload: dict) -> None:
|
|
pass
|
|
|
|
bus = get_event_bus()
|
|
bus.subscribe("test.deliveries.ok", _success_handler)
|
|
try:
|
|
await enqueue_outbox_event(
|
|
db_session, tenant_id, "test.deliveries.ok", {"key": "value"}
|
|
)
|
|
await db_session.flush()
|
|
await db_session.commit()
|
|
|
|
async with session_factory() as worker_session:
|
|
count = await process_outbox_batch(
|
|
worker_session, batch_size=10, tenant_ids=[tenant_id]
|
|
)
|
|
|
|
assert count == 1
|
|
|
|
await set_tenant_context(db_session, tenant_id)
|
|
rows = (
|
|
await db_session.execute(
|
|
text(
|
|
"SELECT d.consumer_name, d.status, d.last_error "
|
|
"FROM outbox_deliveries d "
|
|
"JOIN event_outbox e ON d.event_id = e.id "
|
|
"WHERE e.tenant_id = :tid"
|
|
),
|
|
{"tid": str(tenant_id)},
|
|
)
|
|
).fetchall()
|
|
assert len(rows) == 1
|
|
assert rows[0][0] == "_success_handler"
|
|
assert rows[0][1] == "delivered"
|
|
assert rows[0][2] is None # no error
|
|
finally:
|
|
bus.unsubscribe("test.deliveries.ok", _success_handler)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_outbox_deliveries_written_on_failure(
|
|
db_session: AsyncSession,
|
|
session_factory: async_sessionmaker[AsyncSession],
|
|
):
|
|
"""outbox_deliveries entries are written with status='failed' on handler error."""
|
|
tenant_id = uuid.uuid4()
|
|
|
|
async def _failing_handler(payload: dict) -> None:
|
|
raise RuntimeError("Delivery failure")
|
|
|
|
bus = get_event_bus()
|
|
bus.subscribe("test.deliveries.fail", _failing_handler)
|
|
try:
|
|
await enqueue_outbox_event(
|
|
db_session, tenant_id, "test.deliveries.fail", {"key": "value"}
|
|
)
|
|
await db_session.flush()
|
|
await db_session.commit()
|
|
|
|
async with session_factory() as worker_session:
|
|
count = await process_outbox_batch(
|
|
worker_session, batch_size=10, tenant_ids=[tenant_id]
|
|
)
|
|
|
|
assert count == 0 # not published
|
|
|
|
await set_tenant_context(db_session, tenant_id)
|
|
rows = (
|
|
await db_session.execute(
|
|
text(
|
|
"SELECT d.consumer_name, d.status, d.last_error "
|
|
"FROM outbox_deliveries d "
|
|
"JOIN event_outbox e ON d.event_id = e.id "
|
|
"WHERE e.tenant_id = :tid"
|
|
),
|
|
{"tid": str(tenant_id)},
|
|
)
|
|
).fetchall()
|
|
assert len(rows) == 1
|
|
assert rows[0][0] == "_failing_handler"
|
|
assert rows[0][1] == "failed"
|
|
assert rows[0][2] is not None
|
|
assert "Delivery failure" in rows[0][2]
|
|
finally:
|
|
bus.unsubscribe("test.deliveries.fail", _failing_handler)
|
|
|
|
|
|
# ── Route import smoke test ────────────────────────────────────────────────────
|
|
|
|
def test_route_import():
|
|
"""Verify that the outbox route module imports cleanly."""
|
|
from app.routes.outbox import router
|
|
|
|
assert router is not None
|
|
assert router.prefix == "/api/v1/outbox"
|
|
# Check all 5 endpoints are registered
|
|
paths = {route.path for route in router.routes}
|
|
assert "/api/v1/outbox/stats" in paths
|
|
assert "/api/v1/outbox/failed" in paths
|
|
assert "/api/v1/outbox/replay/{event_id}" in paths
|
|
assert "/api/v1/outbox/replay-all" in paths
|
|
assert "/api/v1/outbox/consumer-registry" in paths
|