Leopoldadmin/leocrm
1
0
Fork 0
Code Issues 363 Pull Requests Actions 5 Packages Projects Releases 1 Wiki Activity

feat(B.13-B.17+B.16): Error-Handling-Infra, Observability, Graceful Shutdown, Cost-Cap, API Versioning

Browse Source 
B.13 Error-Handling-Infrastruktur:
- ErrorCategory Enum (TRANSIENT/PERMANENT/PARTIAL), ApiError erweitert
- Einheitliches Error-Response-Format: {code, detail, field, trace_id, retryable, category}
- 3 FastAPI Exception-Handler (ApiError, HTTPException, unhandled)
- classify_exception() Helper, 6 neue Error-Codes
- 28 Tests in test_error_handling.py

B.14 Observability & trace_id-Korrelation:
- trace_id pro Request (UUID4 short) in structlog contextvars
- X-Trace-Id Response-Header
- Sensitive Fields structlog processor
- llm_complete()/llm_embed() akzeptieren trace_id kwarg
- 12 Tests in test_observability.py

B.15 Graceful Shutdown & Connection Draining:
- _shutdown_event + _inflight_requests Tracking in main.py
- drain_all_connections() in ws_helpers.py
- Worker on_shutdown pausiert WorkflowInstances (status=paused)
- 8 Tests in test_graceful_shutdown.py

B.16 API Versioning Strategie:
- Plugin-Dev-Guide Kapitel 30: URL-basiertes Versioning, Breaking Change Prozess

B.17 Cost Overrun Protection:
- llm_monthly_budget_usd + llm_hard_cutoff Settings
- _check_tenant_budget() vor jedem LLM-Call
- _track_tenant_cost() in Redis (INCRBYFLOAT)
- _check_cost_alerts() bei 50%/80%/100% -> post_system_message()
- 20 Tests in test_cost_protection.py

Total: 68 neue Tests, alle grün. Keine Regressionen.
This commit is contained in:
Agent Zero
2026-08-13 21:33:14 +02:00
parent 1baa9481a2
commit b5546ea7bd
13 changed files with 1461 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/main.py
+120 -14
View File
@@ -2,10 +2,13 @@
from __future__ import annotations
import asyncio
import time
import traceback
import uuid as _uuid
from contextlib import asynccontextmanager
import structlog
from fastapi import FastAPI, HTTPException, Request, Depends, APIRouter
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import FileResponse, JSONResponse
@@ -20,7 +23,7 @@ logger = logging.getLogger(__name__)
from app.config import get_settings
from app.core.db import close_engine, get_engine
from app.core.error_codes import ApiError
from app.core.error_codes import ApiError, ErrorCategory, classify_exception, build_error_response
from app.core.middleware import CSRFMiddleware, SecurityHeadersMiddleware
from app.core.rate_limit import GeneralRateLimitMiddleware
from app.core.resilience import CircuitBreakerMiddleware
@@ -74,14 +77,58 @@ from app.routes import (
)
# ── Graceful shutdown signal ─────────────────────────────────────────────────
# Set during lifespan shutdown so middleware and handlers can stop accepting work.
_shutdown_event = asyncio.Event()
# Track in-flight requests for graceful draining
_inflight_requests: set[asyncio.Task] = set()
async def _drain_inflight(timeout_per_request: float = 25.0) -> None:
"""Wait for all in-flight request tasks to complete."""
if not _inflight_requests:
return
logger.info(f"Waiting for {len(_inflight_requests)} in-flight requests to complete")
# Give tasks a chance to finish; cancel remaining after timeout
done, pending = await asyncio.wait(
_inflight_requests,
timeout=timeout_per_request,
)
if pending:
logger.warning(f"Cancelling {len(pending)} in-flight requests that exceeded grace period")
for task in pending:
task.cancel()
await asyncio.gather(*pending, return_exceptions=True)
def _generate_trace_id() -> str:
"""Generate a short trace ID (first 8 chars of UUID4)."""
return _uuid.uuid4().hex[:8]
def _get_trace_id() -> str | None:
"""Get the current trace_id from structlog contextvars (best-effort)."""
try:
ctx = structlog.contextvars.get_contextvars()
return ctx.get("trace_id")
except Exception:
return None
class RequestLoggingMiddleware(BaseHTTPMiddleware):
"""Structured logging + Prometheus metrics for every HTTP request."""
"""Structured logging + Prometheus metrics + trace_id for every HTTP request."""
async def dispatch(self, request: Request, call_next):
start_time = time.perf_counter()
method = request.method
path = request.url.path
# Generate trace_id and bind to structlog contextvars for this request
trace_id = _generate_trace_id()
structlog.contextvars.clear_contextvars()
structlog.contextvars.bind_contextvars(trace_id=trace_id)
# Extract tenant_id from session cookie if available (best-effort)
tenant_id = None
@@ -106,7 +153,7 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
"message": f"[Backend] {method} {path}: {exc}",
"stack": tb_str,
"url": str(request.url),
"context": {"method": method, "path": path, "source": "backend_middleware"},
"context": {"method": method, "path": path, "source": "backend_middleware", "trace_id": trace_id},
})
except Exception:
pass # Never let error reporting break the request
@@ -115,6 +162,9 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
duration_ms = (time.perf_counter() - start_time) * 1000
status_code = response.status_code
# Add trace_id to response header
response.headers["X-Trace-Id"] = trace_id
# Report 4xx and 5xx errors to Forgejo (except 401/403 which are expected)
if status_code >= 400 and status_code not in (401, 403):
try:
@@ -122,7 +172,7 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
await report_error_to_forgejo({
"message": f"[Backend] {method} {path} → {status_code}",
"url": str(request.url),
"context": {"method": method, "path": path, "status": status_code, "source": "backend_response"},
"context": {"method": method, "path": path, "status": status_code, "source": "backend_response", "trace_id": trace_id},
})
except Exception:
pass # Never let error reporting break the response
@@ -137,6 +187,9 @@ class RequestLoggingMiddleware(BaseHTTPMiddleware):
tenant_id=tenant_id,
)
# Clear contextvars after request completes
structlog.contextvars.clear_contextvars()
return response
@@ -308,10 +361,29 @@ async def lifespan(app: FastAPI):
yield
# Shutdown: close global Redis and ARQ pool
# ── Graceful shutdown ────────────────────────────────────────────────────
# Signal that we're shutting down — no new requests should be accepted.
_shutdown_event.set()
logger.info("Graceful shutdown initiated — draining in-flight requests")
# Drain WebSocket connections (notify clients to reconnect)
try:
from app.core.ws_helpers import drain_all_connections
await drain_all_connections(grace_period_seconds=5)
except Exception as exc:
logger.warning(f"WS drain failed during shutdown: {exc}")
# Give in-flight requests time to complete (max 30s)
try:
await asyncio.wait_for(_drain_inflight(), timeout=30.0)
except asyncio.TimeoutError:
logger.warning("Graceful shutdown: 30s timeout reached, forcing shutdown")
# Close global Redis and ARQ pool
await close_job_pool()
await close_redis()
await close_engine()
logger.info("Graceful shutdown complete")
def create_app() -> FastAPI:
@@ -393,7 +465,8 @@ def create_app() -> FastAPI:
# ── Global exception handler — catch ALL unhandled exceptions ──
@app.exception_handler(Exception)
async def global_exception_handler(request: Request, exc: Exception):
logger.error(f"Unhandled exception: {exc}", exc_info=True)
trace_id = _get_trace_id()
logger.error(f"Unhandled exception: {exc}", exc_info=True, extra={"trace_id": trace_id})
record_error(
event="unhandled_exception",
method=request.method,
@@ -401,18 +474,51 @@ def create_app() -> FastAPI:
status_code=500,
error=str(exc),
)
return JSONResponse(
status_code=500,
content={"detail": "Internal server error", "code": "internal_error"},
body = build_error_response(
code="internal_error",
detail="Internal server error",
trace_id=trace_id,
)
resp = JSONResponse(status_code=500, content=body)
if trace_id:
resp.headers["X-Trace-Id"] = trace_id
return resp
# ── ApiError handler — structured error responses ──
# ── HTTPException handler — unified format ──
@app.exception_handler(HTTPException)
async def http_exception_handler(request: Request, exc: HTTPException):
trace_id = _get_trace_id()
# Map common HTTP status codes to error codes
status_to_code = {
404: "not_found",
403: "forbidden",
422: "unprocessable",
429: "rate_limited",
501: "not_implemented",
502: "bad_gateway",
503: "service_unavailable",
504: "service_timeout",
}
code = status_to_code.get(exc.status_code, "internal_error" if exc.status_code >= 500 else "validation_error")
body = build_error_response(
code=code,
detail=str(exc.detail) if exc.detail else None,
trace_id=trace_id,
)
resp = JSONResponse(status_code=exc.status_code, content=body)
if trace_id:
resp.headers["X-Trace-Id"] = trace_id
return resp
# ── ApiError handler — structured error responses with category/retryable ──
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
return JSONResponse(
status_code=exc.status,
content={'code': exc.code, 'detail': exc.detail, 'field': exc.field}
)
trace_id = _get_trace_id()
body = exc.to_response(trace_id=trace_id)
resp = JSONResponse(status_code=exc.status, content=body)
if trace_id:
resp.headers["X-Trace-Id"] = trace_id
return resp
app.include_router(health.router)
app.include_router(metrics.router)