Phase 5 (v2): Processing-Recovery, Retention-Cleanup, Replay-Delivery-Reset
- recover_stuck_events: Reset processing events stuck >120s back to pending - cleanup_published_events: Delete published events older than 30 days - Replay now resets outbox_deliveries for clean retry - Worker: hourly retention cleanup cron job - API: /recover-stuck and /cleanup-published endpoints - process_outbox_batch: auto-recovery at start of each tenant iteration - 23/23 tests passing (5 new tests)
This commit is contained in:
Agent Zero
+113
-2
@@ -96,6 +96,19 @@ _RETRY_SQL = text(
|
||||
"""
|
||||
)
|
||||
|
||||
# ── Phase 5: Processing recovery (stuck events) ──────────────────────────────
|
||||
|
||||
_RECOVER_STUCK_SQL = text(
|
||||
"""
|
||||
UPDATE event_outbox
|
||||
SET status = 'pending',
|
||||
updated_at = now()
|
||||
WHERE status = 'processing'
|
||||
AND updated_at < now() - make_interval(secs => :timeout_seconds)
|
||||
RETURNING id
|
||||
"""
|
||||
)
|
||||
|
||||
# ── Phase 5: outbox_deliveries SQL ──────────────────────────────────────────
|
||||
|
||||
_INSERT_DELIVERY_SQL = text(
|
||||
@@ -139,6 +152,29 @@ _REPLAY_ALL_SQL = text(
|
||||
"""
|
||||
)
|
||||
|
||||
# ── Phase 5: Reset deliveries on replay ─────────────────────────────────────
|
||||
|
||||
_RESET_DELIVERIES_FOR_EVENT_SQL = text(
|
||||
"DELETE FROM outbox_deliveries WHERE event_id = :event_id"
|
||||
)
|
||||
|
||||
_RESET_DELIVERIES_FOR_TENANT_SQL = text(
|
||||
"""
|
||||
DELETE FROM outbox_deliveries
|
||||
WHERE event_id IN (SELECT id FROM event_outbox WHERE tenant_id = :tenant_id AND status = 'pending')
|
||||
"""
|
||||
)
|
||||
|
||||
# ── Phase 5: Retention SQL ──────────────────────────────────────────────────
|
||||
|
||||
_RETENTION_PUBLISHED_SQL = text(
|
||||
"""
|
||||
DELETE FROM event_outbox
|
||||
WHERE status = 'published'
|
||||
AND published_at < now() - make_interval(secs => :retention_seconds)
|
||||
"""
|
||||
)
|
||||
|
||||
# ── Phase 5: Stats SQL ──────────────────────────────────────────────────────
|
||||
|
||||
_STATS_COUNT_SQL = text(
|
||||
@@ -405,6 +441,9 @@ async def process_outbox_batch(
|
||||
{"tid": str(tenant_id)},
|
||||
)
|
||||
|
||||
# Phase 5: Recover stuck processing events (worker crash recovery)
|
||||
await recover_stuck_events(db, timeout_seconds=120)
|
||||
|
||||
# Claim a batch of pending events for this tenant
|
||||
rows = (
|
||||
await db.execute(_CLAIM_SQL, {"batch_size": batch_size})
|
||||
@@ -444,7 +483,15 @@ async def replay_failed_event(db: AsyncSession, event_id: uuid.UUID) -> bool:
|
||||
{"event_id": str(event_id)},
|
||||
)
|
||||
row = result.first()
|
||||
return row is not None
|
||||
if row is not None:
|
||||
# Reset delivery records so consumers get a clean slate
|
||||
await db.execute(
|
||||
_RESET_DELIVERIES_FOR_EVENT_SQL,
|
||||
{"event_id": str(event_id)},
|
||||
)
|
||||
await db.commit()
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
async def replay_all_failed_events(db: AsyncSession, tenant_id: uuid.UUID) -> int:
|
||||
@@ -464,7 +511,15 @@ async def replay_all_failed_events(db: AsyncSession, tenant_id: uuid.UUID) -> in
|
||||
_REPLAY_ALL_SQL,
|
||||
{"tenant_id": str(tenant_id)},
|
||||
)
|
||||
return len(result.fetchall())
|
||||
replayed_ids = result.fetchall()
|
||||
if replayed_ids:
|
||||
# Reset delivery records for all replayed events
|
||||
await db.execute(
|
||||
_RESET_DELIVERIES_FOR_TENANT_SQL,
|
||||
{"tenant_id": str(tenant_id)},
|
||||
)
|
||||
await db.commit()
|
||||
return len(replayed_ids)
|
||||
|
||||
|
||||
# ── Phase 5: Monitoring functions ───────────────────────────────────────────
|
||||
@@ -553,3 +608,59 @@ def get_consumer_registry() -> dict[str, list[str]]:
|
||||
if handlers:
|
||||
registry[event_name] = [_get_handler_name(h) for h in handlers]
|
||||
return registry
|
||||
|
||||
|
||||
# ── Phase 5: Processing recovery ──────────────────────────────────────────────
|
||||
|
||||
async def recover_stuck_events(db: AsyncSession, timeout_seconds: int = 120) -> int:
|
||||
"""Reset events stuck in 'processing' status back to 'pending'.
|
||||
|
||||
If a worker crashes mid-processing, events remain in 'processing' forever.
|
||||
This function resets events that have been in 'processing' longer than
|
||||
*timeout_seconds* back to 'pending' so they can be retried.
|
||||
|
||||
Must be called with tenant context set (RLS filters automatically).
|
||||
|
||||
Args:
|
||||
db: Active async SQLAlchemy session.
|
||||
timeout_seconds: How long an event can stay in 'processing' before reset.
|
||||
|
||||
Returns:
|
||||
Number of events reset to 'pending'.
|
||||
"""
|
||||
result = await db.execute(
|
||||
_RECOVER_STUCK_SQL,
|
||||
{"timeout_seconds": timeout_seconds},
|
||||
)
|
||||
count = len(result.fetchall()) if result.returns_rows else 0
|
||||
if count:
|
||||
logger.info("Recovered %d stuck processing events (timeout=%ds)", count, timeout_seconds)
|
||||
return count
|
||||
|
||||
|
||||
# ── Phase 5: Retention cleanup ────────────────────────────────────────────────
|
||||
|
||||
async def cleanup_published_events(db: AsyncSession, retention_days: int = 30) -> int:
|
||||
"""Delete published events older than *retention_days*.
|
||||
|
||||
Prevents the outbox table from growing indefinitely. Published events
|
||||
are no longer needed after retention period.
|
||||
|
||||
Must be called with tenant context set (RLS filters automatically).
|
||||
|
||||
Args:
|
||||
db: Active async SQLAlchemy session.
|
||||
retention_days: Delete published events older than this many days.
|
||||
|
||||
Returns:
|
||||
Number of events deleted.
|
||||
"""
|
||||
retention_seconds = retention_days * 86400
|
||||
result = await db.execute(
|
||||
_RETENTION_PUBLISHED_SQL,
|
||||
{"retention_seconds": retention_seconds},
|
||||
)
|
||||
count = result.rowcount if hasattr(result, 'rowcount') else 0
|
||||
if count:
|
||||
logger.info("Cleaned up %d published events older than %d days", count, retention_days)
|
||||
return count
|
||||
|
||||
@@ -256,6 +256,44 @@ async def process_outbox_job(ctx: dict[str, Any]) -> None:
|
||||
register_job("process_outbox", process_outbox_job)
|
||||
|
||||
|
||||
# ── Outbox retention cleanup job ─────────────────────────────────────────────
|
||||
|
||||
async def cleanup_outbox_job(ctx: dict[str, Any]) -> None:
|
||||
"""Delete published outbox events older than 30 days.
|
||||
|
||||
Runs hourly to prevent the outbox table from growing indefinitely.
|
||||
Iterates per-tenant for RLS compliance.
|
||||
"""
|
||||
from app.core.db import get_worker_session_factory
|
||||
from app.core.outbox import cleanup_published_events
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
factory = get_worker_session_factory()
|
||||
async with factory() as db:
|
||||
try:
|
||||
tenant_result = await db.execute(sa_text("SELECT id FROM tenants"))
|
||||
tenant_ids = [row[0] for row in tenant_result]
|
||||
|
||||
total_deleted = 0
|
||||
for tenant_id in tenant_ids:
|
||||
await db.execute(
|
||||
sa_text("SELECT set_config('app.current_tenant_id', :tid, true)"),
|
||||
{"tid": str(tenant_id)},
|
||||
)
|
||||
deleted = await cleanup_published_events(db, retention_days=30)
|
||||
total_deleted += deleted
|
||||
await db.commit()
|
||||
|
||||
if total_deleted:
|
||||
logger.info("Outbox retention: cleaned up %d published events", total_deleted)
|
||||
except Exception:
|
||||
logger.error("Outbox retention cleanup failed", exc_info=True)
|
||||
await db.rollback()
|
||||
|
||||
|
||||
register_job("cleanup_outbox", cleanup_outbox_job)
|
||||
|
||||
|
||||
class WorkerSettings:
|
||||
"""ARQ worker settings."""
|
||||
functions = get_all_jobs()
|
||||
@@ -280,4 +318,9 @@ class WorkerSettings:
|
||||
_wrap_cron_with_lock("process_outbox", process_outbox_job, ttl_seconds=30),
|
||||
second={0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55},
|
||||
),
|
||||
# Outbox retention cleanup — hourly
|
||||
cron(
|
||||
_wrap_cron_with_lock("cleanup_outbox", cleanup_outbox_job, ttl_seconds=300),
|
||||
minute=0,
|
||||
),
|
||||
]
|
||||
|
||||
@@ -18,9 +18,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.db import get_db
|
||||
from app.core.outbox import (
|
||||
cleanup_published_events,
|
||||
get_consumer_registry,
|
||||
get_failed_events,
|
||||
get_outbox_stats,
|
||||
recover_stuck_events,
|
||||
replay_all_failed_events,
|
||||
replay_failed_event,
|
||||
)
|
||||
@@ -128,3 +130,33 @@ async def outbox_consumer_registry(
|
||||
This is read from ``event_bus._handlers`` at request time.
|
||||
"""
|
||||
return {"registry": get_consumer_registry()}
|
||||
|
||||
|
||||
@router.post("/recover-stuck")
|
||||
async def outbox_recover_stuck(
|
||||
timeout_seconds: int = Query(120, ge=10, le=3600),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict[str, Any] = Depends(require_admin),
|
||||
):
|
||||
"""Reset events stuck in 'processing' status back to 'pending'.
|
||||
|
||||
If a worker crashes mid-processing, events remain in 'processing' forever.
|
||||
This endpoint resets events that have been in 'processing' longer than
|
||||
*timeout_seconds* back to 'pending' so they can be retried.
|
||||
"""
|
||||
count = await recover_stuck_events(db, timeout_seconds=timeout_seconds)
|
||||
return {"recovered_count": count, "timeout_seconds": timeout_seconds}
|
||||
|
||||
|
||||
@router.post("/cleanup-published")
|
||||
async def outbox_cleanup_published(
|
||||
retention_days: int = Query(30, ge=1, le=365),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict[str, Any] = Depends(require_admin),
|
||||
):
|
||||
"""Delete published events older than *retention_days*.
|
||||
|
||||
Prevents the outbox table from growing indefinitely.
|
||||
"""
|
||||
count = await cleanup_published_events(db, retention_days=retention_days)
|
||||
return {"deleted_count": count, "retention_days": retention_days}
|
||||
|
||||
Reference in New Issue
Block a user