Phase 5 (v2): Processing-Recovery, Retention-Cleanup, Replay-Delivery-Reset
- recover_stuck_events: Reset processing events stuck >120s back to pending - cleanup_published_events: Delete published events older than 30 days - Replay now resets outbox_deliveries for clean retry - Worker: hourly retention cleanup cron job - API: /recover-stuck and /cleanup-published endpoints - process_outbox_batch: auto-recovery at start of each tenant iteration - 23/23 tests passing (5 new tests)
This commit is contained in:
Agent Zero
@@ -420,3 +420,171 @@ def test_route_import():
|
||||
assert "/api/v1/outbox/replay/{event_id}" in paths
|
||||
assert "/api/v1/outbox/replay-all" in paths
|
||||
assert "/api/v1/outbox/consumer-registry" in paths
|
||||
|
||||
|
||||
# ── Processing recovery (stuck events) ─────────────────────────────────────────
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_stuck_events(db_session: AsyncSession):
|
||||
"""recover_stuck_events resets events stuck in 'processing' back to 'pending'."""
|
||||
from app.core.outbox import recover_stuck_events
|
||||
|
||||
tenant_id = uuid.uuid4()
|
||||
|
||||
# Insert a processing event with old updated_at (simulating crash)
|
||||
await db_session.execute(
|
||||
text(
|
||||
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, updated_at) "
|
||||
"VALUES (:tid, 'test.stuck', CAST(:payload AS JSONB), 'processing', now() - interval '5 minutes')"
|
||||
),
|
||||
{"tid": str(tenant_id), "payload": '{"k": "v"}'},
|
||||
)
|
||||
await db_session.flush()
|
||||
await db_session.commit()
|
||||
|
||||
await set_tenant_context(db_session, tenant_id)
|
||||
count = await recover_stuck_events(db_session, timeout_seconds=60)
|
||||
await db_session.commit()
|
||||
|
||||
assert count == 1
|
||||
|
||||
rows = (
|
||||
await db_session.execute(
|
||||
text("SELECT status FROM event_outbox WHERE tenant_id = :tid"),
|
||||
{"tid": str(tenant_id)},
|
||||
)
|
||||
).fetchall()
|
||||
assert rows[0][0] == "pending"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_recover_stuck_events_skips_recent(db_session: AsyncSession):
|
||||
"""recover_stuck_events does not reset recently-claimed processing events."""
|
||||
from app.core.outbox import recover_stuck_events
|
||||
|
||||
tenant_id = uuid.uuid4()
|
||||
|
||||
# Insert a processing event with recent updated_at (should not be recovered)
|
||||
await db_session.execute(
|
||||
text(
|
||||
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, updated_at) "
|
||||
"VALUES (:tid, 'test.stuck.recent', CAST(:payload AS JSONB), 'processing', now())"
|
||||
),
|
||||
{"tid": str(tenant_id), "payload": '{"k": "v"}'},
|
||||
)
|
||||
await db_session.flush()
|
||||
await db_session.commit()
|
||||
|
||||
await set_tenant_context(db_session, tenant_id)
|
||||
count = await recover_stuck_events(db_session, timeout_seconds=60)
|
||||
await db_session.commit()
|
||||
|
||||
assert count == 0 # should not recover recent events
|
||||
|
||||
|
||||
# ── Retention cleanup ──────────────────────────────────────────────────────────
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cleanup_published_events(db_session: AsyncSession):
|
||||
"""cleanup_published_events deletes old published events."""
|
||||
from app.core.outbox import cleanup_published_events
|
||||
|
||||
tenant_id = uuid.uuid4()
|
||||
|
||||
# Insert an old published event (31 days ago)
|
||||
await db_session.execute(
|
||||
text(
|
||||
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, published_at) "
|
||||
"VALUES (:tid, 'test.old', CAST(:payload AS JSONB), 'published', now() - interval '31 days')"
|
||||
),
|
||||
{"tid": str(tenant_id), "payload": '{"k": "v"}'},
|
||||
)
|
||||
# Insert a recent published event (1 day ago)
|
||||
await db_session.execute(
|
||||
text(
|
||||
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, published_at) "
|
||||
"VALUES (:tid, 'test.recent', CAST(:payload AS JSONB), 'published', now() - interval '1 day')"
|
||||
),
|
||||
{"tid": str(tenant_id), "payload": '{"k": "v"}'},
|
||||
)
|
||||
await db_session.flush()
|
||||
await db_session.commit()
|
||||
|
||||
await set_tenant_context(db_session, tenant_id)
|
||||
deleted = await cleanup_published_events(db_session, retention_days=30)
|
||||
await db_session.commit()
|
||||
|
||||
assert deleted == 1 # only the old one
|
||||
|
||||
rows = (
|
||||
await db_session.execute(
|
||||
text("SELECT event_name FROM event_outbox WHERE tenant_id = :tid"),
|
||||
{"tid": str(tenant_id)},
|
||||
)
|
||||
).fetchall()
|
||||
assert len(rows) == 1
|
||||
assert rows[0][0] == "test.recent"
|
||||
|
||||
|
||||
# ── Replay resets outbox_deliveries ─────────────────────────────────────────────
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replay_resets_deliveries(db_session: AsyncSession):
|
||||
"""replay_failed_event also deletes old outbox_deliveries for the event."""
|
||||
tenant_id = uuid.uuid4()
|
||||
|
||||
# Insert a failed event
|
||||
await db_session.execute(
|
||||
text(
|
||||
"INSERT INTO event_outbox (tenant_id, event_name, payload, status, attempts, max_attempts, error_message, failed_at) "
|
||||
"VALUES (:tid, 'test.replay.deliveries', CAST(:payload AS JSONB), 'failed', 5, 5, 'err', now())"
|
||||
),
|
||||
{"tid": str(tenant_id), "payload": '{"k": "v"}'},
|
||||
)
|
||||
await db_session.flush()
|
||||
await db_session.commit()
|
||||
|
||||
row = (
|
||||
await db_session.execute(
|
||||
text("SELECT id FROM event_outbox WHERE tenant_id = :tid"),
|
||||
{"tid": str(tenant_id)},
|
||||
)
|
||||
).first()
|
||||
event_id = row[0]
|
||||
|
||||
# Insert a delivery record for this event
|
||||
await db_session.execute(
|
||||
text(
|
||||
"INSERT INTO outbox_deliveries (event_id, consumer_name, status, attempt_count, last_error) "
|
||||
"VALUES (:eid, 'test_consumer', 'failed', 5, 'err')"
|
||||
),
|
||||
{"eid": str(event_id)},
|
||||
)
|
||||
await db_session.flush()
|
||||
await db_session.commit()
|
||||
|
||||
await set_tenant_context(db_session, tenant_id)
|
||||
replayed = await replay_failed_event(db_session, event_id)
|
||||
await db_session.commit()
|
||||
|
||||
assert replayed is True
|
||||
|
||||
# Check deliveries were deleted
|
||||
delivery_rows = (
|
||||
await db_session.execute(
|
||||
text("SELECT count(*) FROM outbox_deliveries WHERE event_id = :eid"),
|
||||
{"eid": str(event_id)},
|
||||
)
|
||||
).first()
|
||||
assert delivery_rows[0] == 0
|
||||
|
||||
|
||||
# ── New endpoints registered ────────────────────────────────────────────────────
|
||||
|
||||
def test_new_endpoints_registered():
|
||||
"""Verify that recover-stuck and cleanup-published endpoints are registered."""
|
||||
from app.routes.outbox import router
|
||||
|
||||
paths = {route.path for route in router.routes}
|
||||
assert "/api/v1/outbox/recover-stuck" in paths
|
||||
assert "/api/v1/outbox/cleanup-published" in paths
|
||||
|
||||
Reference in New Issue
Block a user