"""Transactional outbox for reliable domain event delivery. Instead of publishing events directly to an in-process bus (which is lost on crash/restart), domain events are written to the ``event_outbox`` table **within the same database transaction** as the business operation. A background worker then polls the outbox and publishes events to the in-process event bus. Usage in services:: from app.core.outbox import enqueue_outbox_event await enqueue_outbox_event(db, tenant_id, "contact.created", { "contact_id": str(contact.id), "tenant_id": str(tenant_id), }) # ... later, the transaction commits and the event is durable. """ from __future__ import annotations import logging import uuid from datetime import datetime, timedelta, timezone from typing import Any import redis.asyncio as aioredis from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession logger = logging.getLogger(__name__) # ── SQL statements (raw text for FOR UPDATE SKIP LOCKED) ──────────────────── _INSERT_SQL = text( """ INSERT INTO event_outbox (tenant_id, event_name, payload, aggregate_type, aggregate_id, occurred_at, correlation_id, schema_version) VALUES (:tenant_id, :event_name, CAST(:payload AS JSONB), :aggregate_type, :aggregate_id, COALESCE(:occurred_at, now()), :correlation_id, COALESCE(:schema_version, 1)) RETURNING id """ ) _CLAIM_SQL = text( """ UPDATE event_outbox SET status = 'processing', updated_at = now() WHERE id IN ( SELECT id FROM event_outbox WHERE status = 'pending' AND (next_retry_at IS NULL OR next_retry_at <= now()) ORDER BY created_at LIMIT :batch_size FOR UPDATE SKIP LOCKED ) RETURNING id, tenant_id, event_name, payload, attempts, max_attempts, aggregate_type, aggregate_id, occurred_at, correlation_id, schema_version """ ) _MARK_PUBLISHED_SQL = text( """ UPDATE event_outbox SET status = 'published', published_at = now(), updated_at = now() WHERE id = :id """ ) _FAIL_SQL = text( """ UPDATE event_outbox SET status = 'failed', updated_at = now() WHERE id = :id """ ) _RETRY_SQL = text( """ UPDATE event_outbox SET status = 'pending', attempts = :attempts, next_retry_at = :next_retry_at, updated_at = now() WHERE id = :id """ ) def _json_payload(payload: dict[str, Any]) -> str: """Serialise payload to a JSON string suitable for JSONB cast.""" import json return json.dumps(payload, default=str) async def enqueue_outbox_event( db: AsyncSession, tenant_id: uuid.UUID, event_name: str, payload: dict[str, Any], *, aggregate_type: str | None = None, aggregate_id: uuid.UUID | None = None, correlation_id: uuid.UUID | None = None, schema_version: int = 1, ) -> None: """Insert an event into the outbox table within the current transaction. The event is only persisted when the surrounding transaction commits. This guarantees at-least-once delivery — no event is lost even if the process crashes after the business operation but before the event is published. Args: db: Active async SQLAlchemy session (part of the business transaction). tenant_id: Tenant scope for the event. event_name: Logical event name (e.g. ``"crm.contact.created.v1"``). payload: Event payload dict (will be stored as JSONB). aggregate_type: Type of the aggregate (e.g. 'contact', 'task'). aggregate_id: UUID of the aggregate entity. correlation_id: Optional correlation UUID for tracing across services. schema_version: Event schema version (default 1). """ await db.execute( _INSERT_SQL, { "tenant_id": str(tenant_id), "event_name": event_name, "payload": _json_payload(payload), "aggregate_type": aggregate_type, "aggregate_id": str(aggregate_id) if aggregate_id else None, "occurred_at": None, # DB defaults to NOW() "correlation_id": str(correlation_id) if correlation_id else None, "schema_version": schema_version, }, ) async def _process_single_outbox_event( db: AsyncSession, event_bus, row: tuple, ) -> bool: """Process a single outbox event. Returns True if published successfully.""" event_id = row[0] tenant_id = row[1] event_name = row[2] payload = row[3] attempts = row[4] max_attempts = row[5] aggregate_type = row[6] if len(row) > 6 else None aggregate_id = row[7] if len(row) > 7 else None occurred_at = row[8] if len(row) > 8 else None correlation_id = row[9] if len(row) > 9 else None schema_version = row[10] if len(row) > 10 else 1 # payload comes back as a dict from JSONB if isinstance(payload, str): import json payload_dict = json.loads(payload) else: payload_dict = payload try: # Enrich payload with standardized event envelope metadata payload_dict.setdefault("_event_id", str(event_id)) payload_dict.setdefault("_event_name", event_name) payload_dict.setdefault("_event_timestamp", datetime.now(timezone.utc).isoformat()) payload_dict.setdefault("_tenant_id", str(tenant_id)) payload_dict.setdefault("_aggregate_type", aggregate_type) payload_dict.setdefault("_aggregate_id", str(aggregate_id) if aggregate_id else None) payload_dict.setdefault("_occurred_at", occurred_at.isoformat() if occurred_at else None) payload_dict.setdefault("_correlation_id", str(correlation_id) if correlation_id else None) payload_dict.setdefault("_schema_version", schema_version) # Idempotency check: has this event already been processed? (P1.5 fix) already_processed = await db.execute( text("SELECT 1 FROM consumer_inbox WHERE event_id = :eid AND status = 'processed' LIMIT 1"), {"eid": str(event_id)}, ) if already_processed.first(): # Event was already processed by all consumers — mark as published await db.execute(_MARK_PUBLISHED_SQL, {"id": str(event_id)}) logger.debug("Outbox event %s already processed, marking as published", event_id) return True results = await event_bus.publish_with_results(event_name, payload_dict) # Check if any handlers were registered at all handler_count = len(results) # If any handler raised, treat as failure handler_errors = [r for r in results if r is not None] if handler_errors: raise handler_errors[0] if handler_count == 0: # No handlers registered — mark as 'no_handlers' not 'published' await db.execute( text("UPDATE event_outbox SET status = 'no_handlers', published_at = now() WHERE id = :id"), {"id": str(event_id)}, ) logger.warning("Outbox event %s (%s) had no handlers registered", event_id, event_name) else: # Record in consumer_inbox for idempotency (P1.5 fix) await db.execute( text("INSERT INTO consumer_inbox (event_id, consumer_name, status, processed_at) VALUES (:eid, :name, 'processed', now()) ON CONFLICT DO NOTHING"), {"eid": str(event_id), "name": event_name}, ) await db.execute(_MARK_PUBLISHED_SQL, {"id": str(event_id)}) return True except Exception as exc: logger.error( "Failed to publish outbox event %s (%s): %s", event_id, event_name, exc, exc_info=True, ) new_attempts = attempts + 1 if new_attempts >= max_attempts: await db.execute(_FAIL_SQL, {"id": str(event_id)}) logger.warning( "Outbox event %s marked as failed after %d attempts", event_id, new_attempts, ) else: backoff = timedelta(seconds=(2 ** new_attempts) * 10) next_retry = datetime.now(timezone.utc) + backoff await db.execute( _RETRY_SQL, { "id": str(event_id), "attempts": new_attempts, "next_retry_at": next_retry, }, ) return False async def process_outbox_batch( db: AsyncSession, redis: aioredis.Redis | None = None, batch_size: int = 50, tenant_ids: list[uuid.UUID] | None = None, ) -> int: """Process one batch of pending outbox events. Iterates over all tenants, setting tenant context for RLS before claiming and processing events for each tenant. Args: db: Async SQLAlchemy session for this batch (crm_worker role). redis: Optional Redis client (unused for now, reserved for future cross-process pub/sub). batch_size: Maximum events to process per tenant in one batch. tenant_ids: Optional list of tenant IDs to process. If None, all tenants are loaded from the database. Returns: Number of events successfully published. """ from app.core.event_bus import get_event_bus event_bus = get_event_bus() published_count = 0 # Load tenant IDs if not provided if tenant_ids is None: result = await db.execute(text("SELECT id FROM tenants")) tenant_ids = [row[0] for row in result] for tenant_id in tenant_ids: # Set tenant context for RLS — required for event_outbox and consumer_inbox await db.execute( text("SELECT set_config('app.current_tenant_id', :tid, true)"), {"tid": str(tenant_id)}, ) # Claim a batch of pending events for this tenant rows = ( await db.execute(_CLAIM_SQL, {"batch_size": batch_size}) ).fetchall() if not rows: continue for row in rows: published = await _process_single_outbox_event(db, event_bus, row) if published: published_count += 1 # Commit after each tenant to release locks await db.commit() return published_count