feat(B.13-B.17+B.16): Error-Handling-Infra, Observability, Graceful Shutdown, Cost-Cap, API Versioning

B.13 Error-Handling-Infrastruktur: - ErrorCategory Enum (TRANSIENT/PERMANENT/PARTIAL), ApiError erweitert - Einheitliches Error-Response-Format: {code, detail, field, trace_id, retryable, category} - 3 FastAPI Exception-Handler (ApiError, HTTPException, unhandled) - classify_exception() Helper, 6 neue Error-Codes - 28 Tests in test_error_handling.py B.14 Observability & trace_id-Korrelation: - trace_id pro Request (UUID4 short) in structlog contextvars - X-Trace-Id Response-Header - Sensitive Fields structlog processor - llm_complete()/llm_embed() akzeptieren trace_id kwarg - 12 Tests in test_observability.py B.15 Graceful Shutdown & Connection Draining: - _shutdown_event + _inflight_requests Tracking in main.py - drain_all_connections() in ws_helpers.py - Worker on_shutdown pausiert WorkflowInstances (status=paused) - 8 Tests in test_graceful_shutdown.py B.16 API Versioning Strategie: - Plugin-Dev-Guide Kapitel 30: URL-basiertes Versioning, Breaking Change Prozess B.17 Cost Overrun Protection: - llm_monthly_budget_usd + llm_hard_cutoff Settings - _check_tenant_budget() vor jedem LLM-Call - _track_tenant_cost() in Redis (INCRBYFLOAT) - _check_cost_alerts() bei 50%/80%/100% -> post_system_message() - 20 Tests in test_cost_protection.py Total: 68 neue Tests, alle grün. Keine Regressionen.
2026-08-13 21:33:14 +02:00
parent 1baa9481a2
commit b5546ea7bd
13 changed files with 1461 additions and 31 deletions
@@ -1,19 +1,177 @@
-"""Standardized error codes for consistent frontend handling."""
+"""Standardized error codes, categories, and unified error-response format.

-ERROR_CODES = {
-    'not_found': {'status': 404, 'message': 'Resource not found'},
-    'permission_denied': {'status': 403, 'message': 'Permission denied'},
-    'validation_error': {'status': 422, 'message': 'Validation failed'},
-    'rate_limited': {'status': 429, 'message': 'Too many requests'},
-    'internal_error': {'status': 500, 'message': 'Internal server error'},
-    'service_unavailable': {'status': 503, 'message': 'Service temporarily unavailable'},
+Every API error response follows the schema:
+
+    {
+        "code": "not_found",
+        "detail": "Resource not found",
+        "field": null,
+        "trace_id": "a1b2c3d4",
+        "retryable": false,
+        "category": "permanent"
+    }
+
+``ErrorCategory`` classifies errors so callers can decide retry strategy.
+"""
+
+from __future__ import annotations
+
+import enum
+from typing import Any
+
+
+class ErrorCategory(str, enum.Enum):
+    """Error classification for retry decisions."""
+
+    TRANSIENT = "transient"   # retryable: timeout, rate-limit, connection
+    PERMANENT = "permanent"    # non-retryable: validation, permission, not_found
+    PARTIAL = "partial"        # partly successful: batch, bulk operations
+
+
+ERROR_CODES: dict[str, dict[str, Any]] = {
+    # ── Original 6 codes ──
+    "not_found":          {"status": 404, "message": "Resource not found",          "category": ErrorCategory.PERMANENT,  "retryable": False},
+    "permission_denied":  {"status": 403, "message": "Permission denied",           "category": ErrorCategory.PERMANENT,  "retryable": False},
+    "validation_error":   {"status": 422, "message": "Validation failed",            "category": ErrorCategory.PERMANENT,  "retryable": False},
+    "rate_limited":       {"status": 429, "message": "Too many requests",            "category": ErrorCategory.TRANSIENT,  "retryable": True},
+    "internal_error":     {"status": 500, "message": "Internal server error",        "category": ErrorCategory.TRANSIENT,  "retryable": True},
+    "service_unavailable": {"status": 503, "message": "Service temporarily unavailable", "category": ErrorCategory.TRANSIENT, "retryable": True},
+    # ── New codes (B-ERR-FMT) ──
+    "forbidden":          {"status": 403, "message": "Forbidden",                    "category": ErrorCategory.PERMANENT,  "retryable": False},
+    "conflict":           {"status": 409, "message": "Conflict with current state",  "category": ErrorCategory.PERMANENT,  "retryable": False},
+    "unprocessable":      {"status": 422, "message": "Unprocessable entity",         "category": ErrorCategory.PERMANENT,  "retryable": False},
+    "not_implemented":    {"status": 501, "message": "Not implemented",               "category": ErrorCategory.PERMANENT,  "retryable": False},
+    "service_timeout":    {"status": 504, "message": "Service timed out",             "category": ErrorCategory.TRANSIENT,  "retryable": True},
+    "bad_gateway":        {"status": 502, "message": "Bad gateway",                   "category": ErrorCategory.TRANSIENT,  "retryable": True},
+    # ── Partial success ──
+    "partial_success":    {"status": 207, "message": "Partial success",               "category": ErrorCategory.PARTIAL,    "retryable": False},
 }


 class ApiError(Exception):
-    def __init__(self, code: str, detail: str = None, field: str = None, status: int = None):
+    """Application-level error with code, category, and retryable flag.
+
+    Attributes:
+        code: Error code key from ``ERROR_CODES``.
+        detail: Human-readable detail message.
+        field: Optional field name that caused the error.
+        status: HTTP status code.
+        category: ``ErrorCategory`` for retry decisions.
+        retryable: Whether the caller may retry the request.
+    """
+
+    def __init__(
+        self,
+        code: str,
+        detail: str | None = None,
+        field: str | None = None,
+        status: int | None = None,
+        category: ErrorCategory | None = None,
+        retryable: bool | None = None,
+    ):
+        meta = ERROR_CODES.get(code, {})
        self.code = code
-        self.detail = detail or ERROR_CODES.get(code, {}).get('message', 'Unknown error')
+        self.detail = detail or meta.get("message", "Unknown error")
        self.field = field
-        self.status = status or ERROR_CODES.get(code, {}).get('status', 500)
+        self.status = status or meta.get("status", 500)
+        self.category = category or meta.get("category", ErrorCategory.TRANSIENT)
+        self.retryable = retryable if retryable is not None else meta.get("retryable", True)
        super().__init__(self.detail)
+
+    def to_response(self, trace_id: str | None = None) -> dict[str, Any]:
+        """Build the unified error-response dict."""
+        resp: dict[str, Any] = {
+            "code": self.code,
+            "detail": self.detail,
+            "field": self.field,
+            "trace_id": trace_id,
+            "retryable": self.retryable,
+            "category": self.category.value if isinstance(self.category, ErrorCategory) else str(self.category),
+        }
+        return resp
+
+
+# ── Exception classification helper ──────────────────────────────────────────
+
+# Transient error indicators
+_TRANSIENT_KEYWORDS = frozenset({
+    "timeout", "timed out", "rate limit", "rate_limit", "429", "503", "502", "504",
+    "service unavailable", "overloaded", "connection reset", "connection aborted",
+    "temporary", "transient", "retry",
+})
+
+# Permanent error indicators
+_PERMANENT_KEYWORDS = frozenset({
+    "authentication", "auth", "401", "403", "unauthorized", "forbidden",
+    "invalid api key", "invalid_api_key", "validation", "invalid_request",
+    "400", "bad request", "model_not_found", "not found", "404", "409",
+    "conflict", "not implemented", "501", "permission",
+})
+
+# Partial error indicators
+_PARTIAL_KEYWORDS = frozenset({
+    "partial", "batch", "bulk", "some failed", "multi-status", "207",
+})
+
+
+def classify_exception(exc: Exception) -> ErrorCategory:
+    """Classify an exception into an ``ErrorCategory``.
+
+    Uses string matching on the exception message and type name.
+    Falls back to ``ErrorCategory.TRANSIENT`` for unknown errors (safer to retry).
+
+    Args:
+        exc: The exception to classify.
+
+    Returns:
+        ``ErrorCategory.TRANSIENT``, ``ErrorCategory.PERMANENT``, or
+        ``ErrorCategory.PARTIAL``.
+    """
+    # If it's already an ApiError, use its category
+    if isinstance(exc, ApiError):
+        return exc.category if isinstance(exc.category, ErrorCategory) else ErrorCategory(exc.category)
+
+    import asyncio as _asyncio
+
+    msg = str(exc).lower()
+    exc_type_name = type(exc).__name__.lower()
+
+    # Check partial first — batch/bulk errors
+    if any(kw in msg or kw in exc_type_name for kw in _PARTIAL_KEYWORDS):
+        return ErrorCategory.PARTIAL
+
+    # Check permanent — auth/validation/permission errors should never be retried
+    if any(kw in msg or kw in exc_type_name for kw in _PERMANENT_KEYWORDS):
+        return ErrorCategory.PERMANENT
+
+    # Check transient
+    if any(kw in msg or kw in exc_type_name for kw in _TRANSIENT_KEYWORDS):
+        return ErrorCategory.TRANSIENT
+
+    # asyncio.TimeoutError is always transient
+    if isinstance(exc, (_asyncio.TimeoutError, TimeoutError, ConnectionError)):
+        return ErrorCategory.TRANSIENT
+
+    # Default: treat as transient (safe to retry)
+    return ErrorCategory.TRANSIENT
+
+
+def build_error_response(
+    code: str,
+    detail: str | None = None,
+    field: str | None = None,
+    status: int | None = None,
+    trace_id: str | None = None,
+) -> dict[str, Any]:
+    """Build a unified error-response dict without raising an exception."""
+    meta = ERROR_CODES.get(code, {})
+    return {
+        "code": code,
+        "detail": detail or meta.get("message", "Unknown error"),
+        "field": field,
+        "trace_id": trace_id,
+        "retryable": meta.get("retryable", True),
+        "category": meta.get("category", ErrorCategory.TRANSIENT).value
+        if isinstance(meta.get("category"), ErrorCategory)
+        else str(meta.get("category", ErrorCategory.TRANSIENT)),
+    }
@@ -51,10 +51,50 @@ arq_jobs_total = Counter(

 # ─── Structured Logging (structlog) ───

+# Sensitive field names that should be redacted from log output
+_SENSITIVE_FIELDS = frozenset({
+    "password", "passwd", "secret", "api_key", "apikey", "token",
+    "authorization", "auth", "cookie", "session_id", "session",
+    "private_key", "privatekey", "credentials", "smtp_password",
+    "mail_password", "encryption_key", "secret_key",
+})
+
+
+def _sanitize_sensitive_fields(logger, method_name, event_dict):
+    """structlog processor that redacts sensitive field values."""
+    for key in list(event_dict.keys()):
+        if key.lower() in _SENSITIVE_FIELDS:
+            event_dict[key] = "***REDACTED***"
+    return event_dict
+
+
+def sanitize_dict(data: dict[str, Any], extra_sensitive: set[str] | None = None) -> dict[str, Any]:
+    """Sanitize a dictionary by redacting sensitive field values.
+
+    Args:
+        data: The dictionary to sanitize.
+        extra_sensitive: Additional field names to treat as sensitive.
+
+    Returns:
+        A copy of *data* with sensitive values replaced by ``"***REDACTED***"``.
+    """
+    sensitive = _SENSITIVE_FIELDS | (extra_sensitive or set())
+    result = {}
+    for key, value in data.items():
+        if key.lower() in sensitive:
+            result[key] = "***REDACTED***"
+        elif isinstance(value, dict):
+            result[key] = sanitize_dict(value, extra_sensitive)
+        else:
+            result[key] = value
+    return result
+
+
 structlog.configure(
    processors=[
        structlog.contextvars.merge_contextvars,
        structlog.processors.add_log_level,
+        _sanitize_sensitive_fields,
        structlog.processors.TimeStamper(fmt="iso"),
        structlog.processors.JSONRenderer(),
    ],
@@ -176,8 +176,35 @@ async def on_startup(ctx: dict[str, Any]) -> None:


 async def on_shutdown(ctx: dict[str, Any]) -> None:
-    """Called when worker shuts down."""
+    """Called when worker shuts down.
+
+    Pauses any running WorkflowRun instances so they can be resumed after
+    restart, then closes Redis.
+    """
    logger.info("ARQ worker shutting down...")
+
+    # Pause running workflow instances so they can be resumed after restart
+    try:
+        from app.core.db import get_worker_session_factory
+        from sqlalchemy import select as sa_select
+        from app.models.workflow import WorkflowInstance
+
+        session_factory = get_worker_session_factory()
+        async with session_factory() as db:
+            result = await db.execute(
+                sa_select(WorkflowInstance).where(
+                    WorkflowInstance.status == "running"
+                )
+            )
+            running = result.scalars().all()
+            if running:
+                for wf in running:
+                    wf.status = "paused"
+                await db.commit()
+                logger.info(f"Paused {len(running)} running workflow(s) for graceful shutdown")
+    except Exception as exc:
+        logger.warning(f"Failed to pause running workflows during shutdown: {exc}")
+
    from app.core.auth import close_redis
    await close_redis()

@@ -146,6 +146,57 @@ async def send_ws_error(
        logger.debug("Failed to send WS error to client")


+# ── Global connection registry for drain_all_connections ────────────────────
+# Plugin WS endpoints register their connection registries here so that
+# drain_all_connections() can close them all during graceful shutdown.
+_global_ws_registries: list[dict[str, list[WebSocket]]] = []
+
+
+def register_ws_registry(registry: dict[str, list[WebSocket]]) -> None:
+    """Register a WebSocket connection registry for graceful shutdown."""
+    if registry not in _global_ws_registries:
+        _global_ws_registries.append(registry)
+
+
+async def drain_all_connections(grace_period_seconds: float = 5.0) -> None:
+    """Notify all connected WS clients about reconnect and close connections.
+
+    Sends a ``reconnect`` hint message to every connected client, waits
+    for ``grace_period_seconds``, then forcefully closes all sockets.
+    Called during application graceful shutdown.
+    """
+    total_connections = 0
+    for registry in _global_ws_registries:
+        for user_id, conns in list(registry.items()):
+            for ws in list(conns):
+                try:
+                    await ws.send_text(json.dumps({
+                        "type": "reconnect",
+                        "reason": "server_shutdown",
+                        "message": "Server is shutting down. Please reconnect shortly.",
+                    }))
+                    total_connections += 1
+                except Exception:
+                    logger.debug("Failed to send reconnect hint to WS client")
+
+    logger.info(f"WS drain: notified {total_connections} connections, waiting {grace_period_seconds}s")
+
+    if grace_period_seconds > 0:
+        await asyncio.sleep(grace_period_seconds)
+
+    # Close all connections
+    for registry in _global_ws_registries:
+        for user_id, conns in list(registry.items()):
+            for ws in list(conns):
+                try:
+                    await ws.close(code=1001, reason="Server shutting down")
+                except Exception:
+                    logger.debug("WS already closed during drain")
+            registry.clear()
+    _global_ws_registries.clear()
+    logger.info("WS drain: all connections closed")
+
+
 async def handle_ws_message(
    websocket: WebSocket,
    message: str,