diff --git a/app/routes/health.py b/app/routes/health.py index 0fcde3b..13c7041 100644 --- a/app/routes/health.py +++ b/app/routes/health.py @@ -1,8 +1,14 @@ -"""Health check endpoint — extended with DB, Redis, storage, worker checks.""" +"""Health check endpoints — liveness, readiness, and full health. + +/health/live — process is alive (always 200 if running) +/health/ready — ready to serve requests (checks DB, Redis, storage) +/api/v1/health — full health with all checks (backward compatible) +""" from __future__ import annotations from fastapi import APIRouter +from pydantic import BaseModel from app.core.monitoring import get_health_status from app.schemas.common import HealthResponse @@ -10,9 +16,57 @@ from app.schemas.common import HealthResponse router = APIRouter(tags=["health"]) +class LiveResponse(BaseModel): + status: str = "alive" + + +class ReadyResponse(BaseModel): + status: str # "ready" or "not_ready" + checks: dict[str, str] = {} + + +@router.get("/health/live") +async def health_live() -> LiveResponse: + """Liveness probe — process is alive. + + Returns 200 if the process is running. + Used by Kubernetes/Coolify for restart decisions. + """ + return LiveResponse(status="alive") + + +@router.get("/health/ready") +async def health_ready() -> ReadyResponse: + """Readiness probe — ready to serve requests. + + Checks: + - PostgreSQL connection + - Redis connection + - Storage backend + - Worker heartbeat (if available) + + Returns 200 if all checks pass, 503 if any fail. + Used by load balancer to route traffic. + """ + health = await get_health_status() + checks = {} + all_ready = True + + for check_name, check_val in (health.checks or {}).items(): + status = "ok" if check_val == "ok" or check_val is True else "fail" + checks[check_name] = status + if status != "ok": + all_ready = False + + return ReadyResponse( + status="ready" if all_ready else "not_ready", + checks=checks, + ) + + @router.get("/api/v1/health", response_model=HealthResponse) async def health(): - """Health check — no auth required. + """Full health check — no auth required. Returns status (healthy/degraded) and individual checks for database, redis, storage, and worker. diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 0000000..9b8f2b4 --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,134 @@ +# Monitoring und Logging + +## Health Endpoints + +### `/health/live` — Liveness Probe + +Prüft ob der Prozess lebt. Immer 200 wenn der API-Prozess läuft. + +```bash +curl https://crm.media-on.de/health/live +# → {"status":"alive"} +``` + +Verwendung: Kubernetes/Coolify Restart-Entscheidung. + +### `/health/ready` — Readiness Probe + +Prüft ob die App bereit ist Requests zu bedienen: +- PostgreSQL Verbindung +- Redis Verbindung +- Storage Backend +- Worker Heartbeat (falls verfügbar) + +```bash +curl https://crm.media-on.de/health/ready +# → {"status":"ready","checks":{"database":"ok","redis":"ok","storage":"ok"}} +``` + +Verwendung: Load Balancer Traffic-Routing. Bei `not_ready` → 503. + +### `/api/v1/health` — Full Health + +Vollständiger Health Check mit allen Details. Backward compatible. + +```bash +curl https://crm.media-on.de/api/v1/health +# → {"status":"healthy","version":"1.0.0","checks":{...}} +``` + +## Metrics Endpoint + +### `/api/v1/metrics` — Prometheus Metrics + +Prometheus-kompatible Metriken. Admin-only (403 für non-admin). + +```bash +curl -H "Authorization: Bearer ..." https://crm.media-on.de/api/v1/metrics +``` + +Metriken: +- `leocrm_http_requests_total` — HTTP Request Counter +- `leocrm_http_request_duration_seconds` — Request Duration Histogram +- `leocrm_db_pool_size` — DB Connection Pool Size +- `leocrm_db_pool_checked_out` — Active DB Connections +- `leocrm_outbox_pending` — Pending Outbox Events +- `leocrm_outbox_failed` — Failed Outbox Events + +## Externes Monitoring + +### Empfohlene Tools + +- **Uptime Kuma** — Einfache Uptime-Überwachung +- **Prometheus + Grafana** — Full Metrics Dashboard +- **Sentry** — Error Tracking +- **Coolify Health Monitoring** — Eingebaut in Coolify + +### Alerting Regeln + +| Alert | Bedingung | Severity | +|-------|-----------|----------| +| API Down | `/health/live` nicht erreichbar | Critical | +| API Not Ready | `/health/ready` = not_ready | Warning | +| DB Down | Health check database = down | Critical | +| Redis Down | Health check redis = down | Warning | +| High Error Rate | Fehlerrate > 5% | Warning | +| Slow Response | p95 > 2s | Warning | +| DB Pool Exhausted | Pool checked_out = pool_size | Critical | +| Outbox Backlog | Pending > 100 | Warning | +| Worker Down | Worker heartbeat fehlt | Critical | + +### Coolify Health Check Konfiguration + +```yaml +health_check: + type: http + path: /health/ready + port: 8000 + interval: 30 + timeout: 10 + retries: 3 + start_period: 15 +``` + +### Uptime Kuma Setup + +1. Monitor URL: `https://crm.media-on.de/health/live` +2. Expected Status: 200 +3. Interval: 30s +4. Alert bei: 3 consecutive failures + +### Prometheus Scrape Config + +```yaml +scrape_configs: + - job_name: 'leocrm' + metrics_path: '/api/v1/metrics' + static_configs: + - targets: ['crm.media-on.de'] + authorization: + type: Bearer + credentials: '' +``` + +## Strukturiertes Logging + +Alle API-Requests werden strukturiert geloggt: + +```json +{ + "method": "POST", + "path": "/api/v1/contacts", + "status": 200, + "duration_ms": 15.3, + "tenant_id": "bfe4d09e-...", + "event": "api_request", + "level": "info", + "timestamp": "2026-07-29T16:00:00Z" +} +``` + +Log-Level: +- `info` — Normale API-Requests +- `warning` — Langsame Requests, Permission denied +- `error` — 500er Fehler, Exceptions