Phase 5 (v2): Processing-Recovery, Retention-Cleanup, Replay-Delivery-Reset

- recover_stuck_events: Reset processing events stuck >120s back to pending
- cleanup_published_events: Delete published events older than 30 days
- Replay now resets outbox_deliveries for clean retry
- Worker: hourly retention cleanup cron job
- API: /recover-stuck and /cleanup-published endpoints
- process_outbox_batch: auto-recovery at start of each tenant iteration
- 23/23 tests passing (5 new tests)

app/core/outbox.py

@@ -96,6 +96,19 @@ _RETRY_SQL = text(
     """
 )
 
+# ── Phase 5: Processing recovery (stuck events) ──────────────────────────────
+
+_RECOVER_STUCK_SQL = text(
+    """
+    UPDATE event_outbox
+       SET status = 'pending',
+           updated_at = now()
+     WHERE status = 'processing'
+       AND updated_at < now() - make_interval(secs => :timeout_seconds)
+    RETURNING id
+    """
+)
+
 # ── Phase 5: outbox_deliveries SQL ──────────────────────────────────────────
 
 _INSERT_DELIVERY_SQL = text(
@@ -139,6 +152,29 @@ _REPLAY_ALL_SQL = text(
     """
 )
 
+# ── Phase 5: Reset deliveries on replay ─────────────────────────────────────
+
+_RESET_DELIVERIES_FOR_EVENT_SQL = text(
+    "DELETE FROM outbox_deliveries WHERE event_id = :event_id"
+)
+
+_RESET_DELIVERIES_FOR_TENANT_SQL = text(
+    """
+    DELETE FROM outbox_deliveries
+     WHERE event_id IN (SELECT id FROM event_outbox WHERE tenant_id = :tenant_id AND status = 'pending')
+    """
+)
+
+# ── Phase 5: Retention SQL ──────────────────────────────────────────────────
+
+_RETENTION_PUBLISHED_SQL = text(
+    """
+    DELETE FROM event_outbox
+     WHERE status = 'published'
+       AND published_at < now() - make_interval(secs => :retention_seconds)
+    """
+)
+
 # ── Phase 5: Stats SQL ──────────────────────────────────────────────────────
 
 _STATS_COUNT_SQL = text(
@@ -405,6 +441,9 @@ async def process_outbox_batch(
             {"tid": str(tenant_id)},
         )
 
+        # Phase 5: Recover stuck processing events (worker crash recovery)
+        await recover_stuck_events(db, timeout_seconds=120)
+
         # Claim a batch of pending events for this tenant
         rows = (
             await db.execute(_CLAIM_SQL, {"batch_size": batch_size})
@@ -444,7 +483,15 @@ async def replay_failed_event(db: AsyncSession, event_id: uuid.UUID) -> bool:
         {"event_id": str(event_id)},
     )
     row = result.first()
-    return row is not None
+    if row is not None:
+        # Reset delivery records so consumers get a clean slate
+        await db.execute(
+            _RESET_DELIVERIES_FOR_EVENT_SQL,
+            {"event_id": str(event_id)},
+        )
+        await db.commit()
+        return True
+    return False
 
 
 async def replay_all_failed_events(db: AsyncSession, tenant_id: uuid.UUID) -> int:
@@ -464,7 +511,15 @@ async def replay_all_failed_events(db: AsyncSession, tenant_id: uuid.UUID) -> in
         _REPLAY_ALL_SQL,
         {"tenant_id": str(tenant_id)},
     )
-    return len(result.fetchall())
+    replayed_ids = result.fetchall()
+    if replayed_ids:
+        # Reset delivery records for all replayed events
+        await db.execute(
+            _RESET_DELIVERIES_FOR_TENANT_SQL,
+            {"tenant_id": str(tenant_id)},
+        )
+        await db.commit()
+    return len(replayed_ids)
 
 
 # ── Phase 5: Monitoring functions ───────────────────────────────────────────
@@ -553,3 +608,59 @@ def get_consumer_registry() -> dict[str, list[str]]:
         if handlers:
             registry[event_name] = [_get_handler_name(h) for h in handlers]
     return registry
+
+
+# ── Phase 5: Processing recovery ──────────────────────────────────────────────
+
+async def recover_stuck_events(db: AsyncSession, timeout_seconds: int = 120) -> int:
+    """Reset events stuck in 'processing' status back to 'pending'.
+
+    If a worker crashes mid-processing, events remain in 'processing' forever.
+    This function resets events that have been in 'processing' longer than
+    *timeout_seconds* back to 'pending' so they can be retried.
+
+    Must be called with tenant context set (RLS filters automatically).
+
+    Args:
+        db: Active async SQLAlchemy session.
+        timeout_seconds: How long an event can stay in 'processing' before reset.
+
+    Returns:
+        Number of events reset to 'pending'.
+    """
+    result = await db.execute(
+        _RECOVER_STUCK_SQL,
+        {"timeout_seconds": timeout_seconds},
+    )
+    count = len(result.fetchall()) if result.returns_rows else 0
+    if count:
+        logger.info("Recovered %d stuck processing events (timeout=%ds)", count, timeout_seconds)
+    return count
+
+
+# ── Phase 5: Retention cleanup ────────────────────────────────────────────────
+
+async def cleanup_published_events(db: AsyncSession, retention_days: int = 30) -> int:
+    """Delete published events older than *retention_days*.
+
+    Prevents the outbox table from growing indefinitely.  Published events
+    are no longer needed after retention period.
+
+    Must be called with tenant context set (RLS filters automatically).
+
+    Args:
+        db: Active async SQLAlchemy session.
+        retention_days: Delete published events older than this many days.
+
+    Returns:
+        Number of events deleted.
+    """
+    retention_seconds = retention_days * 86400
+    result = await db.execute(
+        _RETENTION_PUBLISHED_SQL,
+        {"retention_seconds": retention_seconds},
+    )
+    count = result.rowcount if hasattr(result, 'rowcount') else 0
+    if count:
+        logger.info("Cleaned up %d published events older than %d days", count, retention_days)
+    return count