Phase 5 (v2): Processing-Recovery, Retention-Cleanup, Replay-Delivery-Reset

- recover_stuck_events: Reset processing events stuck >120s back to pending
- cleanup_published_events: Delete published events older than 30 days
- Replay now resets outbox_deliveries for clean retry
- Worker: hourly retention cleanup cron job
- API: /recover-stuck and /cleanup-published endpoints
- process_outbox_batch: auto-recovery at start of each tenant iteration
- 23/23 tests passing (5 new tests)
This commit is contained in:
Agent Zero
2026-08-02 23:47:29 +02:00
parent 4b0d32f8f0
commit 74936b3972
4 changed files with 356 additions and 2 deletions
+32
View File
@@ -18,9 +18,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.db import get_db
from app.core.outbox import (
cleanup_published_events,
get_consumer_registry,
get_failed_events,
get_outbox_stats,
recover_stuck_events,
replay_all_failed_events,
replay_failed_event,
)
@@ -128,3 +130,33 @@ async def outbox_consumer_registry(
This is read from ``event_bus._handlers`` at request time.
"""
return {"registry": get_consumer_registry()}
@router.post("/recover-stuck")
async def outbox_recover_stuck(
timeout_seconds: int = Query(120, ge=10, le=3600),
db: AsyncSession = Depends(get_db),
current_user: dict[str, Any] = Depends(require_admin),
):
"""Reset events stuck in 'processing' status back to 'pending'.
If a worker crashes mid-processing, events remain in 'processing' forever.
This endpoint resets events that have been in 'processing' longer than
*timeout_seconds* back to 'pending' so they can be retried.
"""
count = await recover_stuck_events(db, timeout_seconds=timeout_seconds)
return {"recovered_count": count, "timeout_seconds": timeout_seconds}
@router.post("/cleanup-published")
async def outbox_cleanup_published(
retention_days: int = Query(30, ge=1, le=365),
db: AsyncSession = Depends(get_db),
current_user: dict[str, Any] = Depends(require_admin),
):
"""Delete published events older than *retention_days*.
Prevents the outbox table from growing indefinitely.
"""
count = await cleanup_published_events(db, retention_days=retention_days)
return {"deleted_count": count, "retention_days": retention_days}