Phase 5 (v2): Processing-Recovery, Retention-Cleanup, Replay-Delivery-Reset

- recover_stuck_events: Reset processing events stuck >120s back to pending
- cleanup_published_events: Delete published events older than 30 days
- Replay now resets outbox_deliveries for clean retry
- Worker: hourly retention cleanup cron job
- API: /recover-stuck and /cleanup-published endpoints
- process_outbox_batch: auto-recovery at start of each tenant iteration
- 23/23 tests passing (5 new tests)
This commit is contained in:
Agent Zero
2026-08-02 23:47:29 +02:00
parent 4b0d32f8f0
commit 74936b3972
4 changed files with 356 additions and 2 deletions
+43
View File
@@ -256,6 +256,44 @@ async def process_outbox_job(ctx: dict[str, Any]) -> None:
register_job("process_outbox", process_outbox_job)
# ── Outbox retention cleanup job ─────────────────────────────────────────────
async def cleanup_outbox_job(ctx: dict[str, Any]) -> None:
"""Delete published outbox events older than 30 days.
Runs hourly to prevent the outbox table from growing indefinitely.
Iterates per-tenant for RLS compliance.
"""
from app.core.db import get_worker_session_factory
from app.core.outbox import cleanup_published_events
from sqlalchemy import text as sa_text
factory = get_worker_session_factory()
async with factory() as db:
try:
tenant_result = await db.execute(sa_text("SELECT id FROM tenants"))
tenant_ids = [row[0] for row in tenant_result]
total_deleted = 0
for tenant_id in tenant_ids:
await db.execute(
sa_text("SELECT set_config('app.current_tenant_id', :tid, true)"),
{"tid": str(tenant_id)},
)
deleted = await cleanup_published_events(db, retention_days=30)
total_deleted += deleted
await db.commit()
if total_deleted:
logger.info("Outbox retention: cleaned up %d published events", total_deleted)
except Exception:
logger.error("Outbox retention cleanup failed", exc_info=True)
await db.rollback()
register_job("cleanup_outbox", cleanup_outbox_job)
class WorkerSettings:
"""ARQ worker settings."""
functions = get_all_jobs()
@@ -280,4 +318,9 @@ class WorkerSettings:
_wrap_cron_with_lock("process_outbox", process_outbox_job, ttl_seconds=30),
second={0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55},
),
# Outbox retention cleanup — hourly
cron(
_wrap_cron_with_lock("cleanup_outbox", cleanup_outbox_job, ttl_seconds=300),
minute=0,
),
]