Phase 4: Webhooks, Backup/Restore UI, Onboarding/Tutorial
- Webhooks Backend: model, schema, service (HMAC-SHA256, httpx), routes, event bus dispatcher, migration 0042 - Webhooks Frontend: SettingsWebhooksPage (CRUD, test button, event multi-select), API client - Backup/Restore Backend: model, schema, service (pg_dump/pg_restore), routes (admin-only), migration 0043 - Backup/Restore Frontend: SettingsBackupPage (create, list, restore dialog with RESTORE confirmation, auto-refresh) - Onboarding: OnboardingTour (8 steps, custom CSS overlay), WelcomeDialog, onboardingStore (zustand + localStorage) - Onboarding integrated into AppShell - Routes: /settings/webhooks, /settings/backup registered - Settings nav: Webhooks, Backup & Restore entries added - Migration conflict fixed: 0042_webhooks → 0043_backups chain
This commit is contained in:
Agent Zero
@@ -0,0 +1,268 @@
|
||||
"""Service for database backup and restore operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.backup import Backup
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BACKUP_DIR = Path("/tmp/leocrm-backups")
|
||||
|
||||
|
||||
def _ensure_backup_dir() -> None:
|
||||
"""Ensure the backup directory exists."""
|
||||
BACKUP_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _get_database_url() -> str:
|
||||
"""Get DATABASE_URL from environment."""
|
||||
url = os.environ.get("DATABASE_URL")
|
||||
if not url:
|
||||
raise RuntimeError("DATABASE_URL environment variable is not set")
|
||||
return url
|
||||
|
||||
|
||||
def _parse_pg_url(url: str) -> dict[str, str]:
|
||||
"""Parse a PostgreSQL connection URL into components for pg_dump/pg_restore.
|
||||
|
||||
Handles formats:
|
||||
postgresql://user:pass@host:port/dbname
|
||||
postgresql+asyncpg://user:pass@host:port/dbname
|
||||
"""
|
||||
# Remove async driver prefix if present
|
||||
if "+asyncpg" in url:
|
||||
url = url.replace("+asyncpg", "")
|
||||
if "+psycopg2" in url:
|
||||
url = url.replace("+psycopg2", "")
|
||||
|
||||
# Parse the URL manually to avoid dependency on urllib parsing quirks
|
||||
# Format: postgresql://user:pass@host:port/dbname
|
||||
rest = url.split("://", 1)[1] if "://" in url else url
|
||||
user_info, rest = rest.split("@", 1) if "@" in rest else ("", rest)
|
||||
user = ""
|
||||
password = ""
|
||||
if ":" in user_info:
|
||||
user, password = user_info.split(":", 1)
|
||||
else:
|
||||
user = user_info
|
||||
|
||||
host_port, dbname = rest.split("/", 1) if "/" in rest else (rest, "")
|
||||
host = host_port
|
||||
port = "5432"
|
||||
if ":" in host_port:
|
||||
host, port = host_port.split(":", 1)
|
||||
|
||||
return {
|
||||
"host": host,
|
||||
"port": port,
|
||||
"user": user,
|
||||
"password": password,
|
||||
"dbname": dbname,
|
||||
}
|
||||
|
||||
|
||||
async def list_backups(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
) -> list[Backup]:
|
||||
"""List all backups for a tenant, ordered by creation date descending."""
|
||||
stmt = (
|
||||
select(Backup)
|
||||
.where(Backup.tenant_id == tenant_id)
|
||||
.order_by(Backup.created_at.desc())
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def create_backup(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
user_id: uuid.UUID | None = None,
|
||||
) -> Backup:
|
||||
"""Create a database backup using pg_dump.
|
||||
|
||||
Creates a backup record, runs pg_dump to a file, then updates the record
|
||||
with the file size and status.
|
||||
"""
|
||||
_ensure_backup_dir()
|
||||
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
|
||||
filename = f"leocrm_backup_{tenant_id}_{timestamp}.dump"
|
||||
filepath = BACKUP_DIR / filename
|
||||
|
||||
# Create initial pending record
|
||||
backup = Backup(
|
||||
tenant_id=tenant_id,
|
||||
filename=filename,
|
||||
status="pending",
|
||||
created_by=user_id,
|
||||
)
|
||||
db.add(backup)
|
||||
await db.flush()
|
||||
await db.refresh(backup)
|
||||
|
||||
try:
|
||||
database_url = _get_database_url()
|
||||
pg = _parse_pg_url(database_url)
|
||||
|
||||
# Build pg_dump command
|
||||
env = os.environ.copy()
|
||||
if pg["password"]:
|
||||
env["PGPASSWORD"] = pg["password"]
|
||||
|
||||
cmd = [
|
||||
"pg_dump",
|
||||
"--host", pg["host"],
|
||||
"--port", pg["port"],
|
||||
"--username", pg["user"],
|
||||
"--format", "custom",
|
||||
"--file", str(filepath),
|
||||
pg["dbname"],
|
||||
]
|
||||
|
||||
logger.info("Running pg_dump: %s to %s", cmd, filepath)
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
env=env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
error_msg = stderr.decode() if stderr else "pg_dump failed with unknown error"
|
||||
logger.error("pg_dump failed: %s", error_msg)
|
||||
backup.status = "failed"
|
||||
backup.error_message = error_msg
|
||||
await db.flush()
|
||||
await db.refresh(backup)
|
||||
return backup
|
||||
|
||||
# Get file size
|
||||
size_bytes = filepath.stat().st_size if filepath.exists() else 0
|
||||
|
||||
# Update backup record
|
||||
backup.status = "completed"
|
||||
backup.size_bytes = size_bytes
|
||||
backup.completed_at = datetime.utcnow()
|
||||
await db.flush()
|
||||
await db.refresh(backup)
|
||||
|
||||
logger.info("Backup completed: %s (%d bytes)", filename, size_bytes)
|
||||
return backup
|
||||
|
||||
except Exception as exc:
|
||||
logger.exception("Backup creation failed")
|
||||
backup.status = "failed"
|
||||
backup.error_message = str(exc)
|
||||
await db.flush()
|
||||
await db.refresh(backup)
|
||||
return backup
|
||||
|
||||
|
||||
async def restore_backup(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
backup_id: uuid.UUID,
|
||||
) -> Backup:
|
||||
"""Restore a database backup using pg_restore.
|
||||
|
||||
WARNING: This is a destructive operation. It drops and recreates the database.
|
||||
"""
|
||||
stmt = select(Backup).where(
|
||||
Backup.id == backup_id,
|
||||
Backup.tenant_id == tenant_id,
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
backup = result.scalar_one_or_none()
|
||||
|
||||
if backup is None:
|
||||
raise ValueError("Backup not found")
|
||||
|
||||
if backup.status != "completed":
|
||||
raise ValueError(f"Backup status is '{backup.status}', cannot restore")
|
||||
|
||||
filepath = BACKUP_DIR / backup.filename
|
||||
if not filepath.exists():
|
||||
raise FileNotFoundError(f"Backup file not found: {filepath}")
|
||||
|
||||
try:
|
||||
database_url = _get_database_url()
|
||||
pg = _parse_pg_url(database_url)
|
||||
|
||||
env = os.environ.copy()
|
||||
if pg["password"]:
|
||||
env["PGPASSWORD"] = pg["password"]
|
||||
|
||||
cmd = [
|
||||
"pg_restore",
|
||||
"--host", pg["host"],
|
||||
"--port", pg["port"],
|
||||
"--username", pg["user"],
|
||||
"--dbname", pg["dbname"],
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
str(filepath),
|
||||
]
|
||||
|
||||
logger.info("Running pg_restore: %s", cmd)
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
env=env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
error_msg = stderr.decode() if stderr else "pg_restore failed with unknown error"
|
||||
logger.error("pg_restore failed: %s", error_msg)
|
||||
raise RuntimeError(f"Restore failed: {error_msg}")
|
||||
|
||||
logger.info("Restore completed from backup: %s", backup.filename)
|
||||
return backup
|
||||
|
||||
except Exception as exc:
|
||||
logger.exception("Restore failed")
|
||||
raise
|
||||
|
||||
|
||||
async def delete_backup(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
backup_id: uuid.UUID,
|
||||
) -> bool:
|
||||
"""Delete a backup record and its file."""
|
||||
stmt = select(Backup).where(
|
||||
Backup.id == backup_id,
|
||||
Backup.tenant_id == tenant_id,
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
backup = result.scalar_one_or_none()
|
||||
|
||||
if backup is None:
|
||||
return False
|
||||
|
||||
# Delete the file if it exists
|
||||
filepath = BACKUP_DIR / backup.filename
|
||||
if filepath.exists():
|
||||
filepath.unlink()
|
||||
|
||||
await db.delete(backup)
|
||||
await db.flush()
|
||||
return True
|
||||
@@ -0,0 +1,183 @@
|
||||
"""CRUD service for Webhook subscriptions and delivery."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select, delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.webhook import Webhook
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def list_webhooks(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
event: str | None = None,
|
||||
) -> list[Webhook]:
|
||||
"""List webhooks for a tenant, optionally filtered by event."""
|
||||
stmt = select(Webhook).where(
|
||||
Webhook.tenant_id == tenant_id,
|
||||
)
|
||||
if event:
|
||||
stmt = stmt.where(Webhook.events.any(event))
|
||||
stmt = stmt.order_by(Webhook.created_at.desc())
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def get_webhook(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
webhook_id: uuid.UUID,
|
||||
) -> Webhook | None:
|
||||
"""Get a single webhook by ID."""
|
||||
stmt = select(Webhook).where(
|
||||
Webhook.id == webhook_id,
|
||||
Webhook.tenant_id == tenant_id,
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
|
||||
async def create_webhook(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
user_id: uuid.UUID,
|
||||
data: dict[str, Any],
|
||||
) -> Webhook:
|
||||
"""Create a new webhook subscription."""
|
||||
webhook = Webhook(
|
||||
tenant_id=tenant_id,
|
||||
url=data["url"],
|
||||
events=data["events"],
|
||||
secret=data.get("secret"),
|
||||
is_active=data.get("is_active", True),
|
||||
retry_count=data.get("retry_count", 3),
|
||||
timeout_seconds=data.get("timeout_seconds", 30),
|
||||
created_by=user_id,
|
||||
updated_by=user_id,
|
||||
)
|
||||
db.add(webhook)
|
||||
await db.flush()
|
||||
await db.refresh(webhook)
|
||||
return webhook
|
||||
|
||||
|
||||
async def update_webhook(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
webhook_id: uuid.UUID,
|
||||
data: dict[str, Any],
|
||||
user_id: uuid.UUID | None = None,
|
||||
) -> Webhook | None:
|
||||
"""Update an existing webhook subscription."""
|
||||
webhook = await get_webhook(db, tenant_id, webhook_id)
|
||||
if webhook is None:
|
||||
return None
|
||||
|
||||
update_fields = ["url", "events", "secret", "is_active", "retry_count", "timeout_seconds"]
|
||||
for field in update_fields:
|
||||
if field in data:
|
||||
setattr(webhook, field, data[field])
|
||||
|
||||
if user_id is not None:
|
||||
webhook.updated_by = user_id
|
||||
|
||||
await db.flush()
|
||||
await db.refresh(webhook)
|
||||
return webhook
|
||||
|
||||
|
||||
async def delete_webhook(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
webhook_id: uuid.UUID,
|
||||
) -> bool:
|
||||
"""Delete a webhook subscription."""
|
||||
stmt = select(Webhook).where(
|
||||
Webhook.id == webhook_id,
|
||||
Webhook.tenant_id == tenant_id,
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
webhook = result.scalar_one_or_none()
|
||||
if webhook is None:
|
||||
return False
|
||||
await db.delete(webhook)
|
||||
await db.flush()
|
||||
return True
|
||||
|
||||
|
||||
def _sign_payload(payload: bytes, secret: str) -> str:
|
||||
"""Create HMAC-SHA256 signature for a payload."""
|
||||
return hmac.new(
|
||||
secret.encode("utf-8"),
|
||||
payload,
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
|
||||
|
||||
async def send_webhook(
|
||||
webhook: Webhook,
|
||||
event_name: str,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Send a webhook HTTP POST with HMAC signature.
|
||||
|
||||
Returns a dict with 'success' (bool), 'status_code' (int | None),
|
||||
and 'error' (str | None).
|
||||
"""
|
||||
body = {
|
||||
"event": event_name,
|
||||
"payload": payload,
|
||||
"timestamp": __import__("datetime").datetime.utcnow().isoformat() + "Z",
|
||||
}
|
||||
body_bytes = json.dumps(body, default=str).encode("utf-8")
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "LeoCRM-Webhook/1.0",
|
||||
}
|
||||
|
||||
if webhook.secret:
|
||||
signature = _sign_payload(body_bytes, webhook.secret)
|
||||
headers["X-Webhook-Signature"] = f"sha256={signature}"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=webhook.timeout_seconds) as client:
|
||||
response = await client.post(
|
||||
webhook.url,
|
||||
content=body_bytes,
|
||||
headers=headers,
|
||||
)
|
||||
return {
|
||||
"success": response.is_success,
|
||||
"status_code": response.status_code,
|
||||
"error": None,
|
||||
}
|
||||
except httpx.TimeoutException:
|
||||
return {"success": False, "status_code": None, "error": "timeout"}
|
||||
except httpx.RequestError as exc:
|
||||
return {"success": False, "status_code": None, "error": str(exc)}
|
||||
|
||||
|
||||
async def retry_webhook(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
webhook_id: uuid.UUID,
|
||||
event_name: str,
|
||||
payload: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Retry sending a webhook with the given event and payload."""
|
||||
webhook = await get_webhook(db, tenant_id, webhook_id)
|
||||
if webhook is None:
|
||||
return {"success": False, "status_code": None, "error": "webhook_not_found"}
|
||||
return await send_webhook(webhook, event_name, payload)
|
||||
Reference in New Issue
Block a user