Phase 5: Outbox DLQ, Monitoring, Consumer-Registry

- Migration 0092: DLQ columns (error_message, failed_at) + consumer_inbox RLS fix
- outbox.py: DLQ logic, replay functions, stats, consumer registry
- app/routes/outbox.py: 5 API endpoints (stats, failed, replay, replay-all, consumer-registry)
- outbox_deliveries tracking per consumer handler
- 18/18 tests passing

app/core/outbox.py

@@ -15,6 +15,12 @@ Usage in services::
         "tenant_id": str(tenant_id),
     })
     # ... later, the transaction commits and the event is durable.
+
+Phase 5 additions:
+- DLQ: ``error_message`` and ``failed_at`` columns on ``event_outbox``
+- Replay: ``replay_failed_event`` and ``replay_all_failed_events``
+- Monitoring: ``get_outbox_stats`` and ``get_failed_events``
+- Consumer registry: ``get_consumer_registry`` and ``outbox_deliveries``
 """
 
 from __future__ import annotations
@@ -72,6 +78,8 @@ _FAIL_SQL = text(
     """
     UPDATE event_outbox
        SET status = 'failed',
+           error_message = :error_message,
+           failed_at = now(),
            updated_at = now()
      WHERE id = :id
     """
@@ -88,6 +96,77 @@ _RETRY_SQL = text(
     """
 )
 
+# ── Phase 5: outbox_deliveries SQL ──────────────────────────────────────────
+
+_INSERT_DELIVERY_SQL = text(
+    """
+    INSERT INTO outbox_deliveries (event_id, consumer_name, status, attempt_count, last_error, processed_at)
+    VALUES (:event_id, :consumer_name, :status, :attempt_count, :last_error, :processed_at)
+    ON CONFLICT (event_id, consumer_name) DO UPDATE SET
+        status = EXCLUDED.status,
+        attempt_count = EXCLUDED.attempt_count,
+        last_error = EXCLUDED.last_error,
+        processed_at = EXCLUDED.processed_at,
+        updated_at = now()
+    """
+)
+
+# ── Phase 5: Replay SQL ─────────────────────────────────────────────────────
+
+_REPLAY_ONE_SQL = text(
+    """
+    UPDATE event_outbox
+       SET status = 'pending',
+           attempts = 0,
+           error_message = NULL,
+           next_retry_at = NULL,
+           updated_at = now()
+     WHERE id = :event_id AND status = 'failed'
+    RETURNING id
+    """
+)
+
+_REPLAY_ALL_SQL = text(
+    """
+    UPDATE event_outbox
+       SET status = 'pending',
+           attempts = 0,
+           error_message = NULL,
+           next_retry_at = NULL,
+           updated_at = now()
+     WHERE status = 'failed' AND tenant_id = :tenant_id
+    RETURNING id
+    """
+)
+
+# ── Phase 5: Stats SQL ──────────────────────────────────────────────────────
+
+_STATS_COUNT_SQL = text(
+    "SELECT status, COUNT(*) as count FROM event_outbox GROUP BY status"
+)
+
+_STATS_OLDEST_PENDING_SQL = text(
+    """
+    SELECT EXTRACT(EPOCH FROM (now() - created_at)) as age_seconds
+      FROM event_outbox
+     WHERE status = 'pending'
+     ORDER BY created_at ASC
+     LIMIT 1
+    """
+)
+
+# ── Phase 5: Failed events SQL ──────────────────────────────────────────────
+
+_FAILED_EVENTS_SQL = text(
+    """
+    SELECT id, tenant_id, event_name, error_message, failed_at, attempts, created_at
+      FROM event_outbox
+     WHERE status = 'failed'
+     ORDER BY failed_at DESC
+     LIMIT :limit OFFSET :offset
+    """
+)
+
 
 def _json_payload(payload: dict[str, Any]) -> str:
     """Serialise payload to a JSON string suitable for JSONB cast."""
@@ -96,6 +175,17 @@ def _json_payload(payload: dict[str, Any]) -> str:
     return json.dumps(payload, default=str)
 
 
+def _get_handler_name(handler: Any) -> str:
+    """Extract a human-readable name from a handler callable."""
+    name = getattr(handler, "__name__", None)
+    if name:
+        return name
+    name = getattr(handler, "__qualname__", None)
+    if name:
+        return name
+    return str(handler)
+
+
 async def enqueue_outbox_event(
     db: AsyncSession,
     tenant_id: uuid.UUID,
@@ -187,8 +277,43 @@ async def _process_single_outbox_event(
             logger.debug("Outbox event %s already processed, marking as published", event_id)
             return True
 
+        # Phase 5: Get handler names for consumer registry before publishing
+        handlers = list(event_bus._handlers.get(event_name, [])) + list(event_bus._handlers.get("*", []))
+        handler_names = [_get_handler_name(h) for h in handlers]
+
         results = await event_bus.publish_with_results(event_name, payload_dict)
 
+        # Phase 5: Write outbox_deliveries for each handler
+        current_attempt = attempts + 1
+        for i, result in enumerate(results):
+            consumer_name = handler_names[i] if i < len(handler_names) else f"handler_{i}"
+            if result is None:
+                # Success
+                await db.execute(
+                    _INSERT_DELIVERY_SQL,
+                    {
+                        "event_id": str(event_id),
+                        "consumer_name": consumer_name,
+                        "status": "delivered",
+                        "attempt_count": current_attempt,
+                        "last_error": None,
+                        "processed_at": datetime.now(timezone.utc),
+                    },
+                )
+            else:
+                # Failure
+                await db.execute(
+                    _INSERT_DELIVERY_SQL,
+                    {
+                        "event_id": str(event_id),
+                        "consumer_name": consumer_name,
+                        "status": "failed",
+                        "attempt_count": current_attempt,
+                        "last_error": str(result),
+                        "processed_at": None,
+                    },
+                )
+
         # Check if any handlers were registered at all
         handler_count = len(results)
         # If any handler raised, treat as failure
@@ -219,10 +344,13 @@ async def _process_single_outbox_event(
         )
         new_attempts = attempts + 1
         if new_attempts >= max_attempts:
-            await db.execute(_FAIL_SQL, {"id": str(event_id)})
+            await db.execute(
+                _FAIL_SQL,
+                {"id": str(event_id), "error_message": str(exc)},
+            )
             logger.warning(
-                "Outbox event %s marked as failed after %d attempts",
-                event_id, new_attempts,
+                "Outbox event %s marked as failed after %d attempts: %s",
+                event_id, new_attempts, exc,
             )
         else:
             backoff = timedelta(seconds=(2 ** new_attempts) * 10)
@@ -294,3 +422,134 @@ async def process_outbox_batch(
         await db.commit()
 
     return published_count
+
+
+# ── Phase 5: Replay functions ────────────────────────────────────────────────
+
+async def replay_failed_event(db: AsyncSession, event_id: uuid.UUID) -> bool:
+    """Replay a single failed outbox event.
+
+    Resets the event to ``pending`` status with attempts=0,
+    error_message=NULL, next_retry_at=NULL.
+
+    Args:
+        db: Active async SQLAlchemy session with tenant context set.
+        event_id: UUID of the event to replay.
+
+    Returns:
+        True if the event was replayed, False if not found or not in 'failed' status.
+    """
+    result = await db.execute(
+        _REPLAY_ONE_SQL,
+        {"event_id": str(event_id)},
+    )
+    row = result.first()
+    return row is not None
+
+
+async def replay_all_failed_events(db: AsyncSession, tenant_id: uuid.UUID) -> int:
+    """Replay all failed outbox events for a tenant.
+
+    Resets all failed events to ``pending`` status with attempts=0,
+    error_message=NULL, next_retry_at=NULL.
+
+    Args:
+        db: Active async SQLAlchemy session with tenant context set.
+        tenant_id: Tenant scope for replay.
+
+    Returns:
+        Number of events replayed.
+    """
+    result = await db.execute(
+        _REPLAY_ALL_SQL,
+        {"tenant_id": str(tenant_id)},
+    )
+    return len(result.fetchall())
+
+
+# ── Phase 5: Monitoring functions ───────────────────────────────────────────
+
+async def get_outbox_stats(db: AsyncSession) -> dict[str, Any]:
+    """Get outbox statistics for the current tenant.
+
+    Requires tenant context to be set (RLS filters automatically).
+
+    Returns:
+        Dict with:
+        - ``counts``: dict mapping each status to its count.
+        - ``total``: total number of events.
+        - ``oldest_pending_age_seconds``: age of oldest pending event in seconds,
+          or None if no pending events.
+    """
+    result = await db.execute(_STATS_COUNT_SQL)
+    counts = {row[0]: row[1] for row in result.fetchall()}
+
+    total = sum(counts.values())
+
+    oldest_result = await db.execute(_STATS_OLDEST_PENDING_SQL)
+    oldest_row = oldest_result.first()
+    oldest_pending_age_seconds = float(oldest_row[0]) if oldest_row else None
+
+    return {
+        "counts": counts,
+        "total": total,
+        "oldest_pending_age_seconds": oldest_pending_age_seconds,
+    }
+
+
+async def get_failed_events(
+    db: AsyncSession,
+    limit: int = 50,
+    offset: int = 0,
+) -> list[dict[str, Any]]:
+    """Get failed outbox events for the current tenant.
+
+    Requires tenant context to be set (RLS filters automatically).
+
+    Args:
+        db: Active async SQLAlchemy session with tenant context set.
+        limit: Maximum number of events to return.
+        offset: Number of events to skip.
+
+    Returns:
+        List of dicts with: id, tenant_id, event_name, error_message,
+        failed_at, attempts, created_at, status.
+    """
+    result = await db.execute(
+        _FAILED_EVENTS_SQL,
+        {"limit": limit, "offset": offset},
+    )
+    return [
+        {
+            "id": str(row[0]),
+            "tenant_id": str(row[1]),
+            "event_name": row[2],
+            "error_message": row[3],
+            "failed_at": row[4].isoformat() if row[4] else None,
+            "attempts": row[5],
+            "created_at": row[6].isoformat() if row[6] else None,
+            "status": "failed",
+        }
+        for row in result.fetchall()
+    ]
+
+
+# ── Phase 5: Consumer registry ──────────────────────────────────────────────
+
+def get_consumer_registry() -> dict[str, list[str]]:
+    """Get the registered event handler registry from the in-process event bus.
+
+    Returns a mapping of ``event_name`` to a list of consumer (handler) names.
+    This is read from ``event_bus._handlers`` at call time.
+
+    Returns:
+        Dict mapping event names to lists of handler names.
+    """
+    from app.core.event_bus import get_event_bus
+
+    event_bus = get_event_bus()
+    registry: dict[str, list[str]] = {}
+    for event_name, handlers in event_bus._handlers.items():
+        if handlers:
+            registry[event_name] = [_get_handler_name(h) for h in handlers]
+    return registry

app/main.py

@@ -69,6 +69,7 @@ from app.routes import (
     policies,
     guest_auth,
     guests,
+    outbox,
 )
 
 
@@ -443,6 +444,7 @@ def create_app() -> FastAPI:
     app.include_router(guest_auth.router)
     app.include_router(guests.router)
     app.include_router(workspaces.router)
+    app.include_router(outbox.router)
 
     # ── Register plugin routes for all built-in plugins ──
     # Routes are registered at app creation time so OpenAPI docs are complete.

app/models/__init__.py

@@ -13,6 +13,7 @@ from app.models.contact_merge import ContactMergeHistory
 from app.models.entity_permission import EntityPermission
 from app.models.guest_user import GuestUser
 from app.models.consumer_inbox import ConsumerInbox
+from app.models.outbox_delivery import OutboxDelivery
 from app.models.guest_invitation import GuestInvitation
 from app.models.entity_policy import EntityPolicy
 from app.models.permission_template import PermissionTemplate

app/models/consumer_inbox.py

@@ -25,7 +25,7 @@ class ConsumerInbox(Base):
     )
 
     id: Mapped[uuid.UUID] = mapped_column(
-        PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+        PGUUID(as_uuid=True), primary_key=True, server_default=func.gen_random_uuid(),
     )
     event_id: Mapped[uuid.UUID] = mapped_column(
         PGUUID(as_uuid=True),

app/models/outbox.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import uuid
 from datetime import datetime
 
-from sqlalchemy import DateTime, Integer, String, func
+from sqlalchemy import DateTime, Integer, String, Text, func
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.dialects.postgresql import UUID as PGUUID
 from sqlalchemy.orm import Mapped, mapped_column
@@ -54,3 +54,26 @@ class EventOutbox(Base):
     published_at: Mapped[datetime | None] = mapped_column(
         DateTime(timezone=True), nullable=True,
     )
+    # Envelope columns (migration 0075)
+    aggregate_type: Mapped[str | None] = mapped_column(
+        String(100), nullable=True,
+    )
+    aggregate_id: Mapped[uuid.UUID | None] = mapped_column(
+        PGUUID(as_uuid=True), nullable=True,
+    )
+    occurred_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+    correlation_id: Mapped[uuid.UUID | None] = mapped_column(
+        PGUUID(as_uuid=True), nullable=True,
+    )
+    schema_version: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default="1",
+    )
+    # Phase 5: DLQ columns
+    error_message: Mapped[str | None] = mapped_column(
+        Text, nullable=True,
+    )
+    failed_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )

app/models/outbox_delivery.py

@@ -0,0 +1,57 @@
+"""Outbox delivery model for per-consumer delivery tracking."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
+from sqlalchemy.dialects.postgresql import UUID as PGUUID
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.core.db import Base
+
+
+class OutboxDelivery(Base):
+    """Tracks per-consumer delivery status for outbox events.
+
+    Each row represents one consumer (event handler) processing one outbox
+    event.  An event is only fully 'published' when all mandatory deliveries
+    succeed.
+    """
+
+    __tablename__ = "outbox_deliveries"
+    __table_args__ = (
+        UniqueConstraint("event_id", "consumer_name", name="uq_outbox_deliveries_event_consumer"),
+    )
+
+    id: Mapped[uuid.UUID] = mapped_column(
+        PGUUID(as_uuid=True),
+        primary_key=True,
+        server_default=func.gen_random_uuid(),
+    )
+    event_id: Mapped[uuid.UUID] = mapped_column(
+        PGUUID(as_uuid=True),
+        ForeignKey("event_outbox.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+    consumer_name: Mapped[str] = mapped_column(String(150), nullable=False)
+    status: Mapped[str] = mapped_column(
+        String(30), nullable=False, server_default="pending",
+    )
+    attempt_count: Mapped[int] = mapped_column(
+        Integer, nullable=False, server_default="0",
+    )
+    next_attempt_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    processed_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True,
+    )
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime(timezone=True), nullable=False, server_default=func.now(),
+    )

app/routes/outbox.py

@@ -0,0 +1,130 @@
+"""Outbox monitoring and management endpoints — DLQ, stats, replay, consumer registry.
+
+All endpoints require authentication and admin role.
+Tenant context is set automatically via the ``get_current_user`` dependency
+(used internally by ``require_admin``), which calls ``set_tenant_context``
+on the shared database session.  RLS policies then filter all outbox queries
+automatically.
+"""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.db import get_db
+from app.core.outbox import (
+    get_consumer_registry,
+    get_failed_events,
+    get_outbox_stats,
+    replay_all_failed_events,
+    replay_failed_event,
+)
+from app.deps import require_admin
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v1/outbox", tags=["outbox"])
+
+
+@router.get("/stats")
+async def outbox_stats(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict[str, Any] = Depends(require_admin),
+):
+    """Outbox statistics: counts per status, oldest pending age, total events.
+
+    Returns a JSON object with:
+    - ``counts``: dict mapping each status to its count (pending, processing,
+      published, failed, no_handlers).
+    - ``total``: total number of events for the current tenant.
+    - ``oldest_pending_age_seconds``: age in seconds of the oldest pending
+      event, or ``null`` if none pending.
+    """
+    return await get_outbox_stats(db)
+
+
+@router.get("/failed")
+async def outbox_failed(
+    limit: int = Query(50, ge=1, le=500),
+    offset: int = Query(0, ge=0),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict[str, Any] = Depends(require_admin),
+):
+    """List failed outbox events with error details (paginated).
+
+    Each event includes: id, tenant_id, event_name, error_message,
+    failed_at, attempts, created_at, status.
+    """
+    events = await get_failed_events(db, limit=limit, offset=offset)
+    return {
+        "events": events,
+        "limit": limit,
+        "offset": offset,
+        "count": len(events),
+    }
+
+
+@router.post("/replay/{event_id}")
+async def outbox_replay_single(
+    event_id: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict[str, Any] = Depends(require_admin),
+):
+    """Replay a single failed outbox event.
+
+    Resets the event to ``pending`` status with attempts=0,
+    error_message=NULL, next_retry_at=NULL.
+    """
+    try:
+        eid = uuid.UUID(event_id)
+    except ValueError:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "detail": "Invalid event ID format",
+                "code": "invalid_uuid",
+            },
+        )
+
+    replayed = await replay_failed_event(db, eid)
+    if not replayed:
+        raise HTTPException(
+            status_code=404,
+            detail={
+                "detail": "Failed event not found or not in 'failed' status",
+                "code": "not_found",
+            },
+        )
+    return {"replayed": True, "event_id": event_id}
+
+
+@router.post("/replay-all")
+async def outbox_replay_all(
+    db: AsyncSession = Depends(get_db),
+    current_user: dict[str, Any] = Depends(require_admin),
+):
+    """Replay all failed outbox events for the current tenant.
+
+    Resets all failed events to ``pending`` status.
+    Returns the number of events replayed.
+    """
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    count = await replay_all_failed_events(db, tenant_id)
+    return {"replayed_count": count, "tenant_id": str(tenant_id)}
+
+
+@router.get("/consumer-registry")
+async def outbox_consumer_registry(
+    current_user: dict[str, Any] = Depends(require_admin),
+):
+    """List registered event handlers from the in-process event bus.
+
+    Returns a mapping of ``event_name`` to a list of consumer (handler) names.
+    This is read from ``event_bus._handlers`` at request time.
+    """
+    return {"registry": get_consumer_registry()}