Files
leocrm/app/core/ws_helpers.py
T
Agent Zero b5546ea7bd feat(B.13-B.17+B.16): Error-Handling-Infra, Observability, Graceful Shutdown, Cost-Cap, API Versioning
B.13 Error-Handling-Infrastruktur:
- ErrorCategory Enum (TRANSIENT/PERMANENT/PARTIAL), ApiError erweitert
- Einheitliches Error-Response-Format: {code, detail, field, trace_id, retryable, category}
- 3 FastAPI Exception-Handler (ApiError, HTTPException, unhandled)
- classify_exception() Helper, 6 neue Error-Codes
- 28 Tests in test_error_handling.py

B.14 Observability & trace_id-Korrelation:
- trace_id pro Request (UUID4 short) in structlog contextvars
- X-Trace-Id Response-Header
- Sensitive Fields structlog processor
- llm_complete()/llm_embed() akzeptieren trace_id kwarg
- 12 Tests in test_observability.py

B.15 Graceful Shutdown & Connection Draining:
- _shutdown_event + _inflight_requests Tracking in main.py
- drain_all_connections() in ws_helpers.py
- Worker on_shutdown pausiert WorkflowInstances (status=paused)
- 8 Tests in test_graceful_shutdown.py

B.16 API Versioning Strategie:
- Plugin-Dev-Guide Kapitel 30: URL-basiertes Versioning, Breaking Change Prozess

B.17 Cost Overrun Protection:
- llm_monthly_budget_usd + llm_hard_cutoff Settings
- _check_tenant_budget() vor jedem LLM-Call
- _track_tenant_cost() in Redis (INCRBYFLOAT)
- _check_cost_alerts() bei 50%/80%/100% -> post_system_message()
- 20 Tests in test_cost_protection.py

Total: 68 neue Tests, alle grün. Keine Regressionen.
2026-08-13 21:33:14 +02:00

236 lines
7.8 KiB
Python

"""Shared WebSocket helpers: auth, origin check, tenant check, cleanup, heartbeat, error handling, message dispatch."""
from __future__ import annotations
import asyncio
import json
import logging
import uuid
from typing import Any, Callable
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from starlette.websockets import WebSocket
from app.core.auth import get_redis, get_session_data, verify_ws_origin
from app.config import get_settings
logger = logging.getLogger(__name__)
async def authenticate_ws(websocket: WebSocket, db: AsyncSession) -> dict[str, Any] | None:
"""Authenticate a WebSocket connection via session cookie.
Extracts the session cookie, validates the session in Redis,
and returns user/tenant info. On failure, closes the WebSocket
with code 4401 and returns ``None``.
"""
settings = get_settings()
session_id = websocket.cookies.get(settings.session_cookie_name)
if not session_id:
await websocket.close(code=4401, reason="Unauthorized")
return None
redis = get_redis()
session_data = await get_session_data(redis, session_id)
if session_data is None:
await websocket.close(code=4401, reason="Unauthorized")
return None
if not session_data.get("is_active", False):
await websocket.close(code=4401, reason="Unauthorized")
return None
return {
"user_id": session_data["user_id"],
"tenant_id": session_data["tenant_id"],
"session_id": session_id,
"role": session_data.get("role"),
"email": session_data.get("email"),
"name": session_data.get("name"),
"is_system_admin": session_data.get("is_system_admin", False),
}
async def check_ws_origin(websocket: WebSocket) -> bool:
"""Verify the WebSocket origin and CSRF token.
Delegates to :func:`verify_ws_origin`. On failure, closes the
WebSocket with code 4403 and returns ``False``.
"""
result = await verify_ws_origin(websocket)
if not result:
await websocket.close(code=4403, reason="Forbidden origin")
return False
return True
async def check_ws_tenant(
websocket: WebSocket,
tenant_id: uuid.UUID,
user_id: uuid.UUID,
db: AsyncSession,
) -> bool:
"""Verify that *user_id* belongs to *tenant_id*.
On failure, closes the WebSocket with code 4403 and returns ``False``.
"""
from app.models.user import UserTenant
result = await db.execute(
select(UserTenant).where(
UserTenant.user_id == user_id,
UserTenant.tenant_id == tenant_id,
)
)
if result.scalar_one_or_none() is None:
await websocket.close(code=4403, reason="Forbidden tenant")
return False
return True
async def cleanup_ws_connection(
websocket: WebSocket,
user_id: str,
connection_registry: dict[str, list[WebSocket]],
) -> None:
"""Remove a WebSocket from the connection registry and close it cleanly."""
conns = connection_registry.get(user_id, [])
if websocket in conns:
conns.remove(websocket)
if not conns:
connection_registry.pop(user_id, None)
try:
await websocket.close()
except Exception:
logger.debug("WebSocket already closed during cleanup for user %s", user_id)
async def start_heartbeat(websocket: WebSocket, interval: int = 30) -> asyncio.Task:
"""Start a background heartbeat task that sends periodic pings.
Returns the :class:`asyncio.Task` so the caller can cancel it on disconnect.
"""
async def _heartbeat() -> None:
while True:
try:
await asyncio.sleep(interval)
await websocket.send_text(json.dumps({"type": "ping"}))
except asyncio.CancelledError:
break
except Exception:
logger.debug("Heartbeat stopped — WebSocket likely closed")
break
return asyncio.create_task(_heartbeat())
async def send_ws_error(
websocket: WebSocket,
code: str,
detail: str,
trace_id: str | None = None,
) -> None:
"""Send a structured error message to the WebSocket client."""
payload: dict[str, Any] = {
"type": "error",
"code": code,
"detail": detail,
}
if trace_id is not None:
payload["trace_id"] = trace_id
try:
await websocket.send_text(json.dumps(payload, default=str))
except Exception:
logger.debug("Failed to send WS error to client")
# ── Global connection registry for drain_all_connections ────────────────────
# Plugin WS endpoints register their connection registries here so that
# drain_all_connections() can close them all during graceful shutdown.
_global_ws_registries: list[dict[str, list[WebSocket]]] = []
def register_ws_registry(registry: dict[str, list[WebSocket]]) -> None:
"""Register a WebSocket connection registry for graceful shutdown."""
if registry not in _global_ws_registries:
_global_ws_registries.append(registry)
async def drain_all_connections(grace_period_seconds: float = 5.0) -> None:
"""Notify all connected WS clients about reconnect and close connections.
Sends a ``reconnect`` hint message to every connected client, waits
for ``grace_period_seconds``, then forcefully closes all sockets.
Called during application graceful shutdown.
"""
total_connections = 0
for registry in _global_ws_registries:
for user_id, conns in list(registry.items()):
for ws in list(conns):
try:
await ws.send_text(json.dumps({
"type": "reconnect",
"reason": "server_shutdown",
"message": "Server is shutting down. Please reconnect shortly.",
}))
total_connections += 1
except Exception:
logger.debug("Failed to send reconnect hint to WS client")
logger.info(f"WS drain: notified {total_connections} connections, waiting {grace_period_seconds}s")
if grace_period_seconds > 0:
await asyncio.sleep(grace_period_seconds)
# Close all connections
for registry in _global_ws_registries:
for user_id, conns in list(registry.items()):
for ws in list(conns):
try:
await ws.close(code=1001, reason="Server shutting down")
except Exception:
logger.debug("WS already closed during drain")
registry.clear()
_global_ws_registries.clear()
logger.info("WS drain: all connections closed")
async def handle_ws_message(
websocket: WebSocket,
message: str,
handlers: dict[str, Callable[[WebSocket, dict[str, Any]], Any]],
) -> None:
"""Dispatch a WebSocket text message to the appropriate handler.
*handlers* maps message ``type`` strings to async callables that accept
``(websocket, msg)``. Unknown types and handler exceptions are
reported back to the client via :func:`send_ws_error`.
"""
try:
msg = json.loads(message)
except (json.JSONDecodeError, TypeError):
await send_ws_error(websocket, "invalid_json", "Message is not valid JSON")
return
if not isinstance(msg, dict):
await send_ws_error(websocket, "invalid_message", "Message must be a JSON object")
return
msg_type = msg.get("type")
if not msg_type:
await send_ws_error(websocket, "missing_type", "Message missing 'type' field")
return
handler = handlers.get(msg_type)
if handler is None:
await send_ws_error(websocket, "unknown_type", f"Unknown message type: {msg_type}")
return
try:
await handler(websocket, msg)
except Exception as exc:
logger.exception("Handler error for message type '%s'", msg_type)
await send_ws_error(websocket, "handler_error", str(exc))