Phase 5 (v2): Processing-Recovery, Retention-Cleanup, Replay-Delivery-Reset
- recover_stuck_events: Reset processing events stuck >120s back to pending - cleanup_published_events: Delete published events older than 30 days - Replay now resets outbox_deliveries for clean retry - Worker: hourly retention cleanup cron job - API: /recover-stuck and /cleanup-published endpoints - process_outbox_batch: auto-recovery at start of each tenant iteration - 23/23 tests passing (5 new tests)
This commit is contained in:
@@ -256,6 +256,44 @@ async def process_outbox_job(ctx: dict[str, Any]) -> None:
|
||||
register_job("process_outbox", process_outbox_job)
|
||||
|
||||
|
||||
# ── Outbox retention cleanup job ─────────────────────────────────────────────
|
||||
|
||||
async def cleanup_outbox_job(ctx: dict[str, Any]) -> None:
|
||||
"""Delete published outbox events older than 30 days.
|
||||
|
||||
Runs hourly to prevent the outbox table from growing indefinitely.
|
||||
Iterates per-tenant for RLS compliance.
|
||||
"""
|
||||
from app.core.db import get_worker_session_factory
|
||||
from app.core.outbox import cleanup_published_events
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
factory = get_worker_session_factory()
|
||||
async with factory() as db:
|
||||
try:
|
||||
tenant_result = await db.execute(sa_text("SELECT id FROM tenants"))
|
||||
tenant_ids = [row[0] for row in tenant_result]
|
||||
|
||||
total_deleted = 0
|
||||
for tenant_id in tenant_ids:
|
||||
await db.execute(
|
||||
sa_text("SELECT set_config('app.current_tenant_id', :tid, true)"),
|
||||
{"tid": str(tenant_id)},
|
||||
)
|
||||
deleted = await cleanup_published_events(db, retention_days=30)
|
||||
total_deleted += deleted
|
||||
await db.commit()
|
||||
|
||||
if total_deleted:
|
||||
logger.info("Outbox retention: cleaned up %d published events", total_deleted)
|
||||
except Exception:
|
||||
logger.error("Outbox retention cleanup failed", exc_info=True)
|
||||
await db.rollback()
|
||||
|
||||
|
||||
register_job("cleanup_outbox", cleanup_outbox_job)
|
||||
|
||||
|
||||
class WorkerSettings:
|
||||
"""ARQ worker settings."""
|
||||
functions = get_all_jobs()
|
||||
@@ -280,4 +318,9 @@ class WorkerSettings:
|
||||
_wrap_cron_with_lock("process_outbox", process_outbox_job, ttl_seconds=30),
|
||||
second={0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55},
|
||||
),
|
||||
# Outbox retention cleanup — hourly
|
||||
cron(
|
||||
_wrap_cron_with_lock("cleanup_outbox", cleanup_outbox_job, ttl_seconds=300),
|
||||
minute=0,
|
||||
),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user