leocrm-bot
69e91fd5d0
- Extended health endpoint: DB+Redis+Storage+Worker checks with degraded status - Prometheus metrics endpoint: admin-only, text/plain format - Metrics: http_requests_total, db_pool_connections, arq_jobs_total - Structured JSON logging (structlog): timestamp, level, method, path, status, duration_ms, tenant_id - Performance: page_size max 100 enforced (422), streaming CSV export (StreamingResponse) - Scripts: seed_perf_data.py, check_indexes.py - Docs: admin-guide.md (Deploy, Backup, Restore, Env-Vars, Troubleshooting), api-overview.md - README updated: prod setup, API section, env profiles, admin-guide link - .env.example: added SECRET_KEY, STORAGE_PATH, SMTP_* vars - 38 new tests, full regression 564/564 pass (0 failures) - Ruff: all checks passed
235 lines
9.7 KiB
Python
235 lines
9.7 KiB
Python
"""Monitoring tests — AC1-6: Health checks, Prometheus metrics, structured logging."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
|
|
from app.core.monitoring import (
|
|
check_database,
|
|
check_redis,
|
|
check_storage,
|
|
check_worker,
|
|
generate_metrics,
|
|
record_arq_job,
|
|
record_error,
|
|
record_request,
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestHealthChecks:
|
|
"""AC1-2: Health endpoint with DB, Redis, storage, worker checks."""
|
|
|
|
async def test_health_returns_200_with_checks_structure(self, client: AsyncClient):
|
|
"""AC1: GET /api/v1/health → 200 + JSON with status, checks.database, checks.redis,
|
|
checks.storage, checks.worker."""
|
|
resp = await client.get("/api/v1/health")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert "status" in data
|
|
assert "checks" in data
|
|
assert "database" in data["checks"]
|
|
assert "redis" in data["checks"]
|
|
assert "storage" in data["checks"]
|
|
assert "worker" in data["checks"]
|
|
# Each check should have a status field
|
|
for check_name in ["database", "redis", "storage", "worker"]:
|
|
assert "status" in data["checks"][check_name]
|
|
|
|
async def test_health_db_down_returns_degraded(self, client: AsyncClient):
|
|
"""AC2: GET /api/v1/health with DB down → 200 + status=degraded,
|
|
checks.database.status=down."""
|
|
with patch("app.routes.health.get_health_status") as mock_health:
|
|
mock_health.return_value = {
|
|
"status": "degraded",
|
|
"version": "1.0.0",
|
|
"checks": {
|
|
"database": {"status": "down", "error": "Connection refused"},
|
|
"redis": {"status": "up"},
|
|
"storage": {"status": "up"},
|
|
"worker": {"status": "up"},
|
|
},
|
|
}
|
|
resp = await client.get("/api/v1/health")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["status"] == "degraded"
|
|
assert data["checks"]["database"]["status"] == "down"
|
|
|
|
async def test_health_no_auth_required(self, client: AsyncClient):
|
|
"""Health endpoint should work without authentication."""
|
|
resp = await client.get("/api/v1/health")
|
|
assert resp.status_code == 200
|
|
|
|
async def test_check_database_returns_dict(self, client: AsyncClient):
|
|
"""check_database returns a dict with status field."""
|
|
result = await check_database()
|
|
assert isinstance(result, dict)
|
|
assert "status" in result
|
|
|
|
async def test_check_redis_returns_dict(self, client: AsyncClient):
|
|
"""check_redis returns a dict with status field."""
|
|
result = await check_redis()
|
|
assert isinstance(result, dict)
|
|
assert "status" in result
|
|
|
|
async def test_check_storage_returns_dict(self, client: AsyncClient):
|
|
"""check_storage returns a dict with status field."""
|
|
result = await check_storage()
|
|
assert isinstance(result, dict)
|
|
assert "status" in result
|
|
|
|
async def test_check_worker_returns_dict(self, client: AsyncClient):
|
|
"""check_worker returns a dict with status field."""
|
|
result = await check_worker()
|
|
assert isinstance(result, dict)
|
|
assert "status" in result
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
class TestMetricsEndpoint:
|
|
"""AC3-4: Prometheus metrics endpoint (admin-only)."""
|
|
|
|
async def test_metrics_admin_returns_200_text_plain(self, client: AsyncClient, db_session):
|
|
"""AC3: GET /api/v1/metrics → 200 + text/plain Prometheus format (admin only)."""
|
|
from tests.conftest import login_client, seed_tenant_and_users
|
|
|
|
await seed_tenant_and_users(db_session)
|
|
await login_client(client, "admin@tenanta.com")
|
|
resp = await client.get("/api/v1/metrics")
|
|
assert resp.status_code == 200
|
|
assert "text/plain" in resp.headers.get("content-type", "")
|
|
|
|
async def test_metrics_non_admin_returns_403(self, client: AsyncClient, db_session):
|
|
"""AC3: GET /api/v1/metrics → 403 for non-admin users."""
|
|
from tests.conftest import login_client, seed_tenant_and_users
|
|
|
|
await seed_tenant_and_users(db_session)
|
|
await login_client(client, "viewer@tenanta.com")
|
|
resp = await client.get("/api/v1/metrics")
|
|
assert resp.status_code == 403
|
|
|
|
async def test_metrics_unauthenticated_returns_401(self, client: AsyncClient):
|
|
"""Metrics endpoint requires authentication — 401 without auth."""
|
|
resp = await client.get("/api/v1/metrics")
|
|
assert resp.status_code == 401
|
|
|
|
async def test_metrics_include_required_metric_names(self, client: AsyncClient, db_session):
|
|
"""AC4: Prometheus metrics include leocrm_http_requests_total, leocrm_db_pool_connections,
|
|
leocrm_arq_jobs_total."""
|
|
from tests.conftest import login_client, seed_tenant_and_users
|
|
|
|
await seed_tenant_and_users(db_session)
|
|
await login_client(client, "admin@tenanta.com")
|
|
# Make a request first to generate some metrics
|
|
await client.get("/api/v1/health")
|
|
resp = await client.get("/api/v1/metrics")
|
|
assert resp.status_code == 200
|
|
text = resp.text
|
|
assert "leocrm_http_requests_total" in text
|
|
assert "leocrm_db_pool_connections" in text
|
|
assert "leocrm_arq_jobs_total" in text
|
|
|
|
def test_generate_metrics_returns_bytes(self):
|
|
"""generate_metrics() returns bytes in Prometheus format."""
|
|
data = generate_metrics()
|
|
assert isinstance(data, bytes)
|
|
text = data.decode("utf-8")
|
|
assert "leocrm_" in text
|
|
|
|
|
|
class TestStructuredLogging:
|
|
"""AC5-6: Structured JSON logging and error logging."""
|
|
|
|
def test_record_request_increments_counter(self):
|
|
"""AC5: record_request increments http_requests_total counter and logs JSON entry."""
|
|
with patch("app.core.monitoring.logger") as mock_logger:
|
|
record_request(
|
|
method="GET",
|
|
path="/api/v1/health",
|
|
status_code=200,
|
|
duration_ms=12.5,
|
|
tenant_id="test-tenant-id",
|
|
)
|
|
mock_logger.info.assert_called_once()
|
|
call_args = mock_logger.info.call_args
|
|
assert call_args[1]["method"] == "GET"
|
|
assert call_args[1]["path"] == "/api/v1/health"
|
|
assert call_args[1]["status"] == 200
|
|
assert call_args[1]["duration_ms"] == 12.5
|
|
assert call_args[1]["tenant_id"] == "test-tenant-id"
|
|
|
|
def test_record_request_log_has_required_fields(self):
|
|
"""AC5: Structured JSON log entry has: timestamp, level, event, method, path, status,
|
|
duration_ms, tenant_id."""
|
|
with patch("app.core.monitoring.logger") as mock_logger:
|
|
record_request(
|
|
method="POST",
|
|
path="/api/v1/contacts",
|
|
status_code=201,
|
|
duration_ms=45.3,
|
|
tenant_id="tenant-123",
|
|
)
|
|
call_args = mock_logger.info.call_args
|
|
logged_data = call_args[1]
|
|
# event is passed as positional arg by structlog
|
|
assert call_args[0][0] == "api_request"
|
|
# Check all required fields are present in kwargs
|
|
logged_data = call_args[1]
|
|
assert "method" in logged_data
|
|
assert "path" in logged_data
|
|
assert "status" in logged_data
|
|
assert "duration_ms" in logged_data
|
|
assert "tenant_id" in logged_data
|
|
|
|
def test_record_error_logs_stacktrace_and_context(self):
|
|
"""AC6: Error log includes stacktrace and request context."""
|
|
with patch("app.core.monitoring.logger") as mock_logger:
|
|
tb = "Traceback (most recent call last):\n File 'test.py', line 1\n raise ValueError('test')\n"
|
|
record_error(
|
|
event="unhandled_exception",
|
|
method="GET",
|
|
path="/api/v1/contacts/123",
|
|
status_code=500,
|
|
error="ValueError: test error",
|
|
traceback_str=tb,
|
|
tenant_id="tenant-456",
|
|
)
|
|
mock_logger.error.assert_called_once()
|
|
call_args = mock_logger.error.call_args
|
|
logged_data = call_args[1]
|
|
# event is passed as keyword arg in structlog
|
|
assert call_args[1]["event"] == "unhandled_exception"
|
|
assert logged_data["error"] == "ValueError: test error"
|
|
assert logged_data["traceback"] == tb
|
|
assert logged_data["method"] == "GET"
|
|
assert logged_data["path"] == "/api/v1/contacts/123"
|
|
assert logged_data["status"] == 500
|
|
assert logged_data["tenant_id"] == "tenant-456"
|
|
|
|
def test_record_error_without_traceback(self):
|
|
"""Error logging works even without traceback."""
|
|
with patch("app.core.monitoring.logger") as mock_logger:
|
|
record_error(
|
|
event="db_error",
|
|
method="GET",
|
|
path="/api/v1/health",
|
|
status_code=503,
|
|
error="Connection refused",
|
|
)
|
|
mock_logger.error.assert_called_once()
|
|
call_args = mock_logger.error.call_args
|
|
assert call_args[1]["error"] == "Connection refused"
|
|
assert call_args[1]["traceback"] is None
|
|
|
|
def test_record_arq_job_increments_counter(self):
|
|
"""record_arq_job increments the arq_jobs_total counter."""
|
|
with patch("app.core.monitoring.arq_jobs_total") as mock_counter:
|
|
record_arq_job("export_contacts", "completed")
|
|
mock_counter.labels.assert_called_once_with(
|
|
function="export_contacts", status="completed"
|
|
)
|