T10: Monitoring, Performance, Documentation & Environment Config — 38 tests, ruff clean, docs OK
- Extended health endpoint: DB+Redis+Storage+Worker checks with degraded status - Prometheus metrics endpoint: admin-only, text/plain format - Metrics: http_requests_total, db_pool_connections, arq_jobs_total - Structured JSON logging (structlog): timestamp, level, method, path, status, duration_ms, tenant_id - Performance: page_size max 100 enforced (422), streaming CSV export (StreamingResponse) - Scripts: seed_perf_data.py, check_indexes.py - Docs: admin-guide.md (Deploy, Backup, Restore, Env-Vars, Troubleshooting), api-overview.md - README updated: prod setup, API section, env profiles, admin-guide link - .env.example: added SECRET_KEY, STORAGE_PATH, SMTP_* vars - 38 new tests, full regression 564/564 pass (0 failures) - Ruff: all checks passed
This commit is contained in:
leocrm-bot
@@ -1,6 +1,7 @@
|
||||
"""Routes package."""
|
||||
|
||||
from app.routes import (
|
||||
metrics, # noqa: F401
|
||||
ai_copilot, # noqa: F401
|
||||
auth, # noqa: F401
|
||||
companies, # noqa: F401
|
||||
|
||||
+57
-9
@@ -71,18 +71,66 @@ async def export_companies(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""Export companies as CSV or XLSX."""
|
||||
"""Export companies as CSV (streaming) or XLSX (buffered)."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
|
||||
if format == "csv":
|
||||
csv_data = await company_service.export_companies_csv(
|
||||
db,
|
||||
tenant_id,
|
||||
industry=industry,
|
||||
search=search,
|
||||
)
|
||||
return Response(
|
||||
content=csv_data,
|
||||
from app.models.company import Company
|
||||
from sqlalchemy import select as sa_select
|
||||
from app.core.db import get_session_factory
|
||||
import csv
|
||||
import io
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
async def generate_csv():
|
||||
"""Async generator yielding CSV rows (streaming, not buffered).
|
||||
Creates its own DB session to avoid request-scoped session issues.
|
||||
"""
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
header = [
|
||||
"id", "name", "account_number", "industry", "phone",
|
||||
"email", "website", "description", "created_at", "updated_at",
|
||||
]
|
||||
writer.writerow(header)
|
||||
yield output.getvalue()
|
||||
output.seek(0)
|
||||
output.truncate(0)
|
||||
|
||||
batch_size = 500
|
||||
offset = 0
|
||||
factory = get_session_factory()
|
||||
async with factory() as session:
|
||||
while True:
|
||||
q = sa_select(Company).where(
|
||||
Company.tenant_id == tenant_id,
|
||||
Company.deleted_at.is_(None),
|
||||
)
|
||||
if industry:
|
||||
q = q.where(Company.industry == industry)
|
||||
if search:
|
||||
pattern = f"%{search}%"
|
||||
q = q.where(Company.name.ilike(pattern))
|
||||
q = q.order_by(Company.name.asc()).offset(offset).limit(batch_size)
|
||||
result = await session.execute(q)
|
||||
companies = result.scalars().all()
|
||||
if not companies:
|
||||
break
|
||||
for c in companies:
|
||||
writer.writerow([
|
||||
str(c.id), c.name, c.account_number or "", c.industry or "",
|
||||
c.phone or "", c.email or "", c.website or "",
|
||||
c.description or "",
|
||||
c.created_at.isoformat() if c.created_at else "",
|
||||
c.updated_at.isoformat() if c.updated_at else "",
|
||||
])
|
||||
yield output.getvalue()
|
||||
output.seek(0)
|
||||
output.truncate(0)
|
||||
offset += batch_size
|
||||
|
||||
return StreamingResponse(
|
||||
generate_csv(),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": "attachment; filename=companies.csv"},
|
||||
)
|
||||
|
||||
+107
-2
@@ -1,15 +1,21 @@
|
||||
"""Contact routes — CRUD, N:M, soft-delete, GDPR hard-delete."""
|
||||
"""Contact routes — CRUD, N:M, soft-delete, GDPR hard-delete, streaming CSV export."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import csv
|
||||
import io
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.auth import check_permission
|
||||
from app.core.db import get_db
|
||||
from app.deps import get_current_user
|
||||
from app.models.contact import Contact
|
||||
from app.schemas.contact import ContactCreate, ContactUpdate
|
||||
from app.services import contact_service
|
||||
|
||||
@@ -26,7 +32,10 @@ async def list_contacts(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""List contacts with pagination and optional search."""
|
||||
"""List contacts with pagination and optional search.
|
||||
|
||||
page_size is capped at 100 — values >100 return 422.
|
||||
"""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
result = await contact_service.list_contacts(
|
||||
db,
|
||||
@@ -40,6 +49,102 @@ async def list_contacts(
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/export")
|
||||
async def export_contacts(
|
||||
format: str = Query("csv", pattern="^(csv)$"),
|
||||
search: str | None = Query(None),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
):
|
||||
"""Stream contacts as CSV (not buffered — uses StreamingResponse).
|
||||
|
||||
For large exports (>1000 records), an ARQ background job should be started
|
||||
with a notification on completion. This endpoint streams directly for
|
||||
immediate download.
|
||||
"""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
from app.core.db import get_session_factory
|
||||
|
||||
async def generate_csv():
|
||||
"""Async generator yielding CSV rows one at a time (streaming).
|
||||
Creates its own DB session to avoid using the request-scoped session
|
||||
which gets closed after the endpoint function returns.
|
||||
"""
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
|
||||
# Write header row
|
||||
header = [
|
||||
"id",
|
||||
"first_name",
|
||||
"last_name",
|
||||
"email",
|
||||
"phone",
|
||||
"mobile",
|
||||
"position",
|
||||
"department",
|
||||
"linkedin_url",
|
||||
"notes",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
]
|
||||
writer.writerow(header)
|
||||
yield output.getvalue()
|
||||
output.seek(0)
|
||||
output.truncate(0)
|
||||
|
||||
# Stream rows in batches to avoid loading all into memory
|
||||
batch_size = 500
|
||||
offset = 0
|
||||
factory = get_session_factory()
|
||||
async with factory() as session:
|
||||
while True:
|
||||
base = select(Contact).where(
|
||||
Contact.tenant_id == tenant_id,
|
||||
Contact.deleted_at.is_(None),
|
||||
)
|
||||
if search:
|
||||
pattern = f"%{search}%"
|
||||
base = base.where(
|
||||
(Contact.first_name.ilike(pattern))
|
||||
| (Contact.last_name.ilike(pattern))
|
||||
| (Contact.email.ilike(pattern))
|
||||
)
|
||||
base = base.order_by(Contact.last_name.asc()).offset(offset).limit(batch_size)
|
||||
result = await session.execute(base)
|
||||
contacts = result.scalars().all()
|
||||
if not contacts:
|
||||
break
|
||||
|
||||
for c in contacts:
|
||||
writer.writerow(
|
||||
[
|
||||
str(c.id),
|
||||
c.first_name,
|
||||
c.last_name,
|
||||
c.email or "",
|
||||
c.phone or "",
|
||||
c.mobile or "",
|
||||
c.position or "",
|
||||
c.department or "",
|
||||
c.linkedin_url or "",
|
||||
c.notes or "",
|
||||
c.created_at.isoformat() if c.created_at else "",
|
||||
c.updated_at.isoformat() if c.updated_at else "",
|
||||
]
|
||||
)
|
||||
yield output.getvalue()
|
||||
output.seek(0)
|
||||
output.truncate(0)
|
||||
offset += batch_size
|
||||
|
||||
return StreamingResponse(
|
||||
generate_csv(),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": "attachment; filename=contacts.csv"},
|
||||
)
|
||||
|
||||
|
||||
@router.post("", status_code=status.HTTP_201_CREATED)
|
||||
async def create_contact(
|
||||
body: ContactCreate,
|
||||
|
||||
@@ -1,13 +1,19 @@
|
||||
"""Health check endpoint."""
|
||||
"""Health check endpoint — extended with DB, Redis, storage, worker checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from app.core.monitoring import get_health_status
|
||||
|
||||
router = APIRouter(tags=["health"])
|
||||
|
||||
|
||||
@router.get("/api/v1/health")
|
||||
async def health():
|
||||
"""Health check — no auth required."""
|
||||
return {"status": "ok", "version": "1.0.0"}
|
||||
"""Health check — no auth required.
|
||||
|
||||
Returns status (healthy/degraded) and individual checks for
|
||||
database, redis, storage, and worker.
|
||||
"""
|
||||
return await get_health_status()
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Prometheus metrics endpoint — admin-only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from fastapi.responses import PlainTextResponse
|
||||
|
||||
from app.core.monitoring import generate_metrics
|
||||
from app.deps import require_admin
|
||||
|
||||
router = APIRouter(tags=["metrics"])
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/metrics",
|
||||
response_class=PlainTextResponse,
|
||||
dependencies=[Depends(require_admin)],
|
||||
)
|
||||
async def metrics():
|
||||
"""Prometheus metrics endpoint.
|
||||
|
||||
Returns Prometheus-formatted text/plain metrics.
|
||||
Requires admin role — 403 for non-admin users.
|
||||
"""
|
||||
data = generate_metrics()
|
||||
return PlainTextResponse(
|
||||
content=data.decode("utf-8"),
|
||||
media_type="text/plain; version=0.0.4; charset=utf-8",
|
||||
)
|
||||
Reference in New Issue
Block a user