diff --git a/app/main.py b/app/main.py index a3d0bfc..7beb176 100644 --- a/app/main.py +++ b/app/main.py @@ -65,6 +65,7 @@ from app.routes import ( # noqa: E402 saved_filters, saved_views, sequences, + system_dashboard, system_settings, taxes, tenants, @@ -559,6 +560,7 @@ def create_app() -> FastAPI: app.include_router(currencies.router) app.include_router(taxes.router) app.include_router(sequences.router) + app.include_router(system_dashboard.router) app.include_router(system_settings.router) app.include_router(attachments.router) app.include_router(addresses.router) diff --git a/app/routes/__init__.py b/app/routes/__init__.py index 85b9b51..a44e025 100644 --- a/app/routes/__init__.py +++ b/app/routes/__init__.py @@ -19,6 +19,7 @@ from app.routes import ( plugins, # noqa: F401 roles, # noqa: F401 sequences, # noqa: F401 + system_dashboard, # noqa: F401 system_settings, # noqa: F401 taxes, # noqa: F401 tenants, # noqa: F401 diff --git a/app/routes/system_dashboard.py b/app/routes/system_dashboard.py new file mode 100644 index 0000000..299d934 --- /dev/null +++ b/app/routes/system_dashboard.py @@ -0,0 +1,369 @@ +"""System Dashboard routes — admin-only system monitoring. + +Endpoints: + GET /api/v1/system/dashboard — comprehensive system metrics + GET /api/v1/system/alerts — active alerts with notification dispatch +""" + +from __future__ import annotations + +import os +import shutil +import uuid +from datetime import UTC, datetime +from typing import Any + +from fastapi import APIRouter, Depends +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.db import get_db, get_engine +from app.core.monitoring import ( + check_database, + check_redis, + check_worker, + get_health_status, +) +from app.core.notifications import post_system_message +from app.deps import require_admin +from app.plugins.registry import get_registry + +router = APIRouter(prefix="/api/v1/system", tags=["system"]) + + +# ─── Helpers ────────────────────────────────────────────────────────────────── + + +def _format_bytes(size: int | float) -> str: + """Format bytes to human-readable string.""" + for unit in ["B", "KB", "MB", "GB", "TB"]: + if size < 1024: + return f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} PB" + + +async def _get_db_stats() -> dict[str, Any]: + """Get database statistics.""" + try: + engine = get_engine() + async with engine.connect() as conn: + # Active connections + result = await conn.execute(text("SELECT count(*) FROM pg_stat_activity")) + connections = result.scalar() + + # Table count (public schema) + result = await conn.execute( + text( + "SELECT count(*) FROM information_schema.tables " + "WHERE table_schema = 'public'" + ) + ) + table_count = result.scalar() + + # Database size + result = await conn.execute( + text("SELECT pg_database_size(current_database())") + ) + db_size = result.scalar() + + return { + "status": "up", + "connections": connections, + "table_count": table_count, + "db_size_bytes": db_size, + "db_size_human": _format_bytes(db_size), + } + except Exception as e: + return {"status": "down", "error": str(e)} + + +async def _get_redis_stats() -> dict[str, Any]: + """Get Redis statistics.""" + try: + from app.core.auth import get_redis + + r = get_redis() + info = await r.info() + return { + "status": "up", + "connections": info.get("connected_clients", 0), + "used_memory": info.get("used_memory", 0), + "used_memory_human": info.get("used_memory_human", "N/A"), + "peak_memory_human": info.get("used_memory_peak_human", "N/A"), + "uptime_seconds": info.get("uptime_in_seconds", 0), + } + except Exception as e: + return {"status": "down", "error": str(e)} + + +async def _get_worker_stats() -> dict[str, Any]: + """Get ARQ worker statistics.""" + try: + from app.core.auth import get_redis + + r = get_redis() + queue_length = await r.zcard("arq:queue") + # Check for worker heartbeat keys + heartbeat_keys = await r.keys("arq:heartbeat:*") + active_workers = len(heartbeat_keys) if heartbeat_keys else 0 + + return { + "status": "up" if active_workers > 0 or queue_length == 0 else "degraded", + "queue_length": queue_length, + "active_workers": active_workers, + } + except Exception as e: + return {"status": "down", "error": str(e)} + + +def _get_api_stats() -> dict[str, Any]: + """Get API statistics from Prometheus metrics registry.""" + try: + from app.core.monitoring import REGISTRY + + total_requests = 0 + error_count = 0 + duration_sum = 0.0 + duration_count = 0 + + for metric in REGISTRY.collect(): + for sample in metric.samples: + if sample.name == "leocrm_http_requests_total": + total_requests += int(sample.value) + status_label = sample.labels.get("status", "") + if status_label.startswith("5"): + error_count += int(sample.value) + elif sample.name == "leocrm_http_request_duration_seconds_sum": + duration_sum += sample.value + elif sample.name == "leocrm_http_request_duration_seconds_count": + duration_count += int(sample.value) + + avg_duration = ( + (duration_sum / duration_count * 1000) if duration_count > 0 else 0 + ) + error_rate = (error_count / total_requests * 100) if total_requests > 0 else 0 + + return { + "total_requests": total_requests, + "error_count": error_count, + "error_rate": round(error_rate, 2), + "avg_response_time_ms": round(avg_duration, 2), + } + except Exception as e: + return {"error": str(e)} + + +def _get_plugin_stats() -> dict[str, Any]: + """Get plugin statistics from registry.""" + try: + registry = get_registry() + discovered = registry.list_discovered() + active_plugins: list[dict[str, Any]] = [] + + for name in discovered: + plugin = registry.get_plugin(name) + if plugin: + active_plugins.append( + { + "name": name, + "version": plugin.manifest.version, + "is_core": plugin.manifest.is_core, + } + ) + + return { + "total_discovered": len(discovered), + "active_plugins": active_plugins, + "error_count": 0, + "errors": [], + } + except Exception as e: + return {"error": str(e)} + + +def _get_storage_stats() -> dict[str, Any]: + """Get storage statistics.""" + try: + from app.config import get_settings + + settings = get_settings() + storage_path = getattr(settings, "storage_path", "/tmp") + + if not os.path.exists(storage_path): + return {"status": "down", "error": f"Storage path {storage_path} not found"} + + disk = shutil.disk_usage(storage_path) + file_count = sum(len(files) for _, _, files in os.walk(storage_path)) + + return { + "status": "up", + "path": storage_path, + "disk_total_bytes": disk.total, + "disk_used_bytes": disk.used, + "disk_free_bytes": disk.free, + "disk_usage_percent": ( + round(disk.used / disk.total * 100, 1) if disk.total > 0 else 0 + ), + "file_count": file_count, + } + except Exception as e: + return {"status": "down", "error": str(e)} + + +def _get_llm_usage() -> dict[str, Any]: + """Get LLM usage statistics (best effort — returns zeros if no tracking).""" + return { + "total_tokens": 0, + "estimated_cost": 0.0, + "last_24h_tokens": 0, + "last_24h_cost": 0.0, + } + + +# ─── Alert Notification ────────────────────────────────────────────────────── + + +async def _send_alert_notifications( + db: AsyncSession, + admin: dict[str, Any], + alerts: list[dict[str, Any]], +) -> None: + """Send notifications for new alerts (throttled via Redis 5-min TTL).""" + try: + from app.core.auth import get_redis + + r = get_redis() + tenant_id = uuid.UUID(admin["tenant_id"]) + user_id = uuid.UUID(admin["user_id"]) + + for alert in alerts: + alert_key = f"system_alert:{alert['type']}" + # Only send if not already sent in the last 5 minutes + already_sent = await r.exists(alert_key) + if not already_sent: + await r.setex(alert_key, 300, "1") # 5-minute TTL + await post_system_message( + db=db, + tenant_id=tenant_id, + user_id=user_id, + message_type="system_alert", + title=f"System Alert: {alert['type']}", + body=alert["message"], + severity=alert["severity"], + ) + except Exception: + pass # Never let notification failures break the alerts endpoint + + +# ─── Endpoints ─────────────────────────────────────────────────────────────── + + +@router.get("/dashboard") +async def get_system_dashboard( + admin: dict[str, Any] = Depends(require_admin), +): + """Get comprehensive system metrics. Admin-only. + + Returns DB, Redis, Worker, API, Plugin, Storage, and LLM usage stats. + """ + health = await get_health_status() + db_stats = await _get_db_stats() + redis_stats = await _get_redis_stats() + worker_stats = await _get_worker_stats() + api_stats = _get_api_stats() + plugin_stats = _get_plugin_stats() + storage_stats = _get_storage_stats() + llm_usage = _get_llm_usage() + + return { + "overall_status": health.get("status", "unknown"), + "timestamp": datetime.now(UTC).isoformat(), + "database": db_stats, + "redis": redis_stats, + "worker": worker_stats, + "api": api_stats, + "plugins": plugin_stats, + "storage": storage_stats, + "llm": llm_usage, + } + + +@router.get("/alerts") +async def get_system_alerts( + admin: dict[str, Any] = Depends(require_admin), + db: AsyncSession = Depends(get_db), +): + """Get active system alerts. Admin-only. + + Checks DB, Redis, Worker, Storage, and API error rate. + Sends notifications for new alerts (throttled via Redis). + """ + alerts: list[dict[str, Any]] = [] + + # Check DB + db_check = await check_database() + if db_check["status"] == "down": + alerts.append( + { + "severity": "critical", + "type": "db_down", + "message": f"Database is down: {db_check.get('error', 'Unknown error')}", + } + ) + + # Check Redis + redis_check = await check_redis() + if redis_check["status"] == "down": + alerts.append( + { + "severity": "critical", + "type": "redis_down", + "message": f"Redis is down: {redis_check.get('error', 'Unknown error')}", + } + ) + + # Check Worker + worker_check = await check_worker() + if worker_check["status"] == "down": + alerts.append( + { + "severity": "warning", + "type": "worker_down", + "message": f"Worker is down: {worker_check.get('error', 'Unknown error')}", + } + ) + + # Check Storage disk usage + storage_stats = _get_storage_stats() + disk_percent = storage_stats.get("disk_usage_percent", 0) + if disk_percent > 90: + alerts.append( + { + "severity": "critical", + "type": "disk_full", + "message": f"Disk usage at {disk_percent}%", + } + ) + + # Check high API error rate + api_stats = _get_api_stats() + error_rate = api_stats.get("error_rate", 0) + if error_rate > 5: + alerts.append( + { + "severity": "warning", + "type": "high_error_rate", + "message": f"High error rate: {error_rate:.1f}%", + } + ) + + # Send notifications for new alerts (throttled via Redis) + if alerts: + await _send_alert_notifications(db, admin, alerts) + + return { + "alerts": alerts, + "alert_count": len(alerts), + "timestamp": datetime.now(UTC).isoformat(), + } diff --git a/frontend/src/api/systemDashboard.ts b/frontend/src/api/systemDashboard.ts new file mode 100644 index 0000000..953d094 --- /dev/null +++ b/frontend/src/api/systemDashboard.ts @@ -0,0 +1,104 @@ +/** + * System Dashboard API hooks — admin-only system monitoring. + */ + +import { useQuery } from '@tanstack/react-query'; +import { apiGet } from './client'; + +// ── Types matching backend responses ── + +export interface SystemDashboardData { + overall_status: string; + timestamp: string; + database: { + status: string; + connections?: number; + table_count?: number; + db_size_bytes?: number; + db_size_human?: string; + error?: string; + }; + redis: { + status: string; + connections?: number; + used_memory?: number; + used_memory_human?: string; + peak_memory_human?: string; + uptime_seconds?: number; + error?: string; + }; + worker: { + status: string; + queue_length?: number; + active_workers?: number; + error?: string; + }; + api: { + total_requests?: number; + error_count?: number; + error_rate?: number; + avg_response_time_ms?: number; + error?: string; + }; + plugins: { + total_discovered?: number; + active_plugins?: Array<{ + name: string; + version: string; + is_core: boolean; + }>; + error_count?: number; + errors?: string[]; + error?: string; + }; + storage: { + status: string; + path?: string; + disk_total_bytes?: number; + disk_used_bytes?: number; + disk_free_bytes?: number; + disk_usage_percent?: number; + file_count?: number; + error?: string; + }; + llm: { + total_tokens: number; + estimated_cost: number; + last_24h_tokens: number; + last_24h_cost: number; + }; +} + +export interface SystemAlert { + severity: 'critical' | 'warning'; + type: string; + message: string; +} + +export interface SystemAlertsData { + alerts: SystemAlert[]; + alert_count: number; + timestamp: string; +} + +// ── Hooks ── + +const REFRESH_INTERVAL = 30 * 1000; // 30 seconds + +export function useSystemDashboard() { + return useQuery({ + queryKey: ['system-dashboard'], + queryFn: () => apiGet('/system/dashboard'), + refetchInterval: REFRESH_INTERVAL, + staleTime: REFRESH_INTERVAL, + }); +} + +export function useSystemAlerts() { + return useQuery({ + queryKey: ['system-alerts'], + queryFn: () => apiGet('/system/alerts'), + refetchInterval: REFRESH_INTERVAL, + staleTime: REFRESH_INTERVAL, + }); +} diff --git a/frontend/src/components/layout/Sidebar.tsx b/frontend/src/components/layout/Sidebar.tsx index 58f4152..1cbc8bf 100644 --- a/frontend/src/components/layout/Sidebar.tsx +++ b/frontend/src/components/layout/Sidebar.tsx @@ -55,6 +55,7 @@ const singleItems: NavSingleItem[] = [ { to: '/dashboard', labelKey: 'nav.dashboard', icon: