Phase 5: Outbox DLQ, Monitoring, Consumer-Registry

- Migration 0092: DLQ columns (error_message, failed_at) + consumer_inbox RLS fix - outbox.py: DLQ logic, replay functions, stats, consumer registry - app/routes/outbox.py: 5 API endpoints (stats, failed, replay, replay-all, consumer-registry) - outbox_deliveries tracking per consumer handler - 18/18 tests passing
2026-08-02 23:25:54 +02:00
parent 24cb10a7a2
commit 07a99975ec
12 changed files with 1017 additions and 79 deletions
@@ -15,6 +15,12 @@ Usage in services::
        "tenant_id": str(tenant_id),
    })
    # ... later, the transaction commits and the event is durable.
+
+Phase 5 additions:
+- DLQ: ``error_message`` and ``failed_at`` columns on ``event_outbox``
+- Replay: ``replay_failed_event`` and ``replay_all_failed_events``
+- Monitoring: ``get_outbox_stats`` and ``get_failed_events``
+- Consumer registry: ``get_consumer_registry`` and ``outbox_deliveries``
 """

 from __future__ import annotations
@@ -72,6 +78,8 @@ _FAIL_SQL = text(
    """
    UPDATE event_outbox
       SET status = 'failed',
+           error_message = :error_message,
+           failed_at = now(),
           updated_at = now()
     WHERE id = :id
    """
@@ -88,6 +96,77 @@ _RETRY_SQL = text(
    """
 )

+# ── Phase 5: outbox_deliveries SQL ──────────────────────────────────────────
+
+_INSERT_DELIVERY_SQL = text(
+    """
+    INSERT INTO outbox_deliveries (event_id, consumer_name, status, attempt_count, last_error, processed_at)
+    VALUES (:event_id, :consumer_name, :status, :attempt_count, :last_error, :processed_at)
+    ON CONFLICT (event_id, consumer_name) DO UPDATE SET
+        status = EXCLUDED.status,
+        attempt_count = EXCLUDED.attempt_count,
+        last_error = EXCLUDED.last_error,
+        processed_at = EXCLUDED.processed_at,
+        updated_at = now()
+    """
+)
+
+# ── Phase 5: Replay SQL ─────────────────────────────────────────────────────
+
+_REPLAY_ONE_SQL = text(
+    """
+    UPDATE event_outbox
+       SET status = 'pending',
+           attempts = 0,
+           error_message = NULL,
+           next_retry_at = NULL,
+           updated_at = now()
+     WHERE id = :event_id AND status = 'failed'
+    RETURNING id
+    """
+)
+
+_REPLAY_ALL_SQL = text(
+    """
+    UPDATE event_outbox
+       SET status = 'pending',
+           attempts = 0,
+           error_message = NULL,
+           next_retry_at = NULL,
+           updated_at = now()
+     WHERE status = 'failed' AND tenant_id = :tenant_id
+    RETURNING id
+    """
+)
+
+# ── Phase 5: Stats SQL ──────────────────────────────────────────────────────
+
+_STATS_COUNT_SQL = text(
+    "SELECT status, COUNT(*) as count FROM event_outbox GROUP BY status"
+)
+
+_STATS_OLDEST_PENDING_SQL = text(
+    """
+    SELECT EXTRACT(EPOCH FROM (now() - created_at)) as age_seconds
+      FROM event_outbox
+     WHERE status = 'pending'
+     ORDER BY created_at ASC
+     LIMIT 1
+    """
+)
+
+# ── Phase 5: Failed events SQL ──────────────────────────────────────────────
+
+_FAILED_EVENTS_SQL = text(
+    """
+    SELECT id, tenant_id, event_name, error_message, failed_at, attempts, created_at
+      FROM event_outbox
+     WHERE status = 'failed'
+     ORDER BY failed_at DESC
+     LIMIT :limit OFFSET :offset
+    """
+)
+

 def _json_payload(payload: dict[str, Any]) -> str:
    """Serialise payload to a JSON string suitable for JSONB cast."""
@@ -96,6 +175,17 @@ def _json_payload(payload: dict[str, Any]) -> str:
    return json.dumps(payload, default=str)


+def _get_handler_name(handler: Any) -> str:
+    """Extract a human-readable name from a handler callable."""
+    name = getattr(handler, "__name__", None)
+    if name:
+        return name
+    name = getattr(handler, "__qualname__", None)
+    if name:
+        return name
+    return str(handler)
+
+
 async def enqueue_outbox_event(
    db: AsyncSession,
    tenant_id: uuid.UUID,
@@ -187,8 +277,43 @@ async def _process_single_outbox_event(
            logger.debug("Outbox event %s already processed, marking as published", event_id)
            return True

+        # Phase 5: Get handler names for consumer registry before publishing
+        handlers = list(event_bus._handlers.get(event_name, [])) + list(event_bus._handlers.get("*", []))
+        handler_names = [_get_handler_name(h) for h in handlers]
+
        results = await event_bus.publish_with_results(event_name, payload_dict)

+        # Phase 5: Write outbox_deliveries for each handler
+        current_attempt = attempts + 1
+        for i, result in enumerate(results):
+            consumer_name = handler_names[i] if i < len(handler_names) else f"handler_{i}"
+            if result is None:
+                # Success
+                await db.execute(
+                    _INSERT_DELIVERY_SQL,
+                    {
+                        "event_id": str(event_id),
+                        "consumer_name": consumer_name,
+                        "status": "delivered",
+                        "attempt_count": current_attempt,
+                        "last_error": None,
+                        "processed_at": datetime.now(timezone.utc),
+                    },
+                )
+            else:
+                # Failure
+                await db.execute(
+                    _INSERT_DELIVERY_SQL,
+                    {
+                        "event_id": str(event_id),
+                        "consumer_name": consumer_name,
+                        "status": "failed",
+                        "attempt_count": current_attempt,
+                        "last_error": str(result),
+                        "processed_at": None,
+                    },
+                )
+
        # Check if any handlers were registered at all
        handler_count = len(results)
        # If any handler raised, treat as failure
@@ -219,10 +344,13 @@ async def _process_single_outbox_event(
        )
        new_attempts = attempts + 1
        if new_attempts >= max_attempts:
-            await db.execute(_FAIL_SQL, {"id": str(event_id)})
+            await db.execute(
+                _FAIL_SQL,
+                {"id": str(event_id), "error_message": str(exc)},
+            )
            logger.warning(
-                "Outbox event %s marked as failed after %d attempts",
-                event_id, new_attempts,
+                "Outbox event %s marked as failed after %d attempts: %s",
+                event_id, new_attempts, exc,
            )
        else:
            backoff = timedelta(seconds=(2 ** new_attempts) * 10)
@@ -294,3 +422,134 @@ async def process_outbox_batch(
        await db.commit()

    return published_count
+
+
+# ── Phase 5: Replay functions ────────────────────────────────────────────────
+
+async def replay_failed_event(db: AsyncSession, event_id: uuid.UUID) -> bool:
+    """Replay a single failed outbox event.
+
+    Resets the event to ``pending`` status with attempts=0,
+    error_message=NULL, next_retry_at=NULL.
+
+    Args:
+        db: Active async SQLAlchemy session with tenant context set.
+        event_id: UUID of the event to replay.
+
+    Returns:
+        True if the event was replayed, False if not found or not in 'failed' status.
+    """
+    result = await db.execute(
+        _REPLAY_ONE_SQL,
+        {"event_id": str(event_id)},
+    )
+    row = result.first()
+    return row is not None
+
+
+async def replay_all_failed_events(db: AsyncSession, tenant_id: uuid.UUID) -> int:
+    """Replay all failed outbox events for a tenant.
+
+    Resets all failed events to ``pending`` status with attempts=0,
+    error_message=NULL, next_retry_at=NULL.
+
+    Args:
+        db: Active async SQLAlchemy session with tenant context set.
+        tenant_id: Tenant scope for replay.
+
+    Returns:
+        Number of events replayed.
+    """
+    result = await db.execute(
+        _REPLAY_ALL_SQL,
+        {"tenant_id": str(tenant_id)},
+    )
+    return len(result.fetchall())
+
+
+# ── Phase 5: Monitoring functions ───────────────────────────────────────────
+
+async def get_outbox_stats(db: AsyncSession) -> dict[str, Any]:
+    """Get outbox statistics for the current tenant.
+
+    Requires tenant context to be set (RLS filters automatically).
+
+    Returns:
+        Dict with:
+        - ``counts``: dict mapping each status to its count.
+        - ``total``: total number of events.
+        - ``oldest_pending_age_seconds``: age of oldest pending event in seconds,
+          or None if no pending events.
+    """
+    result = await db.execute(_STATS_COUNT_SQL)
+    counts = {row[0]: row[1] for row in result.fetchall()}
+
+    total = sum(counts.values())
+
+    oldest_result = await db.execute(_STATS_OLDEST_PENDING_SQL)
+    oldest_row = oldest_result.first()
+    oldest_pending_age_seconds = float(oldest_row[0]) if oldest_row else None
+
+    return {
+        "counts": counts,
+        "total": total,
+        "oldest_pending_age_seconds": oldest_pending_age_seconds,
+    }
+
+
+async def get_failed_events(
+    db: AsyncSession,
+    limit: int = 50,
+    offset: int = 0,
+) -> list[dict[str, Any]]:
+    """Get failed outbox events for the current tenant.
+
+    Requires tenant context to be set (RLS filters automatically).
+
+    Args:
+        db: Active async SQLAlchemy session with tenant context set.
+        limit: Maximum number of events to return.
+        offset: Number of events to skip.
+
+    Returns:
+        List of dicts with: id, tenant_id, event_name, error_message,
+        failed_at, attempts, created_at, status.
+    """
+    result = await db.execute(
+        _FAILED_EVENTS_SQL,
+        {"limit": limit, "offset": offset},
+    )
+    return [
+        {
+            "id": str(row[0]),
+            "tenant_id": str(row[1]),
+            "event_name": row[2],
+            "error_message": row[3],
+            "failed_at": row[4].isoformat() if row[4] else None,
+            "attempts": row[5],
+            "created_at": row[6].isoformat() if row[6] else None,
+            "status": "failed",
+        }
+        for row in result.fetchall()
+    ]
+
+
+# ── Phase 5: Consumer registry ──────────────────────────────────────────────
+
+def get_consumer_registry() -> dict[str, list[str]]:
+    """Get the registered event handler registry from the in-process event bus.
+
+    Returns a mapping of ``event_name`` to a list of consumer (handler) names.
+    This is read from ``event_bus._handlers`` at call time.
+
+    Returns:
+        Dict mapping event names to lists of handler names.
+    """
+    from app.core.event_bus import get_event_bus
+
+    event_bus = get_event_bus()
+    registry: dict[str, list[str]] = {}
+    for event_name, handlers in event_bus._handlers.items():
+        if handlers:
+            registry[event_name] = [_get_handler_name(h) for h in handlers]
+    return registry