leocrm/scripts/deploy.py

#!/usr/bin/env python3
"""Automated deployment script for LeoCRM via Coolify API.

All container management is done through the Coolify API — no manual
docker commands, no SSH for container lifecycle.  SSH is used *only*
for post-deploy verification (Alembic version, RLS table count) because
the Coolify API does not expose database internals.

No UUIDs, domains, or secrets are hardcoded.  Everything comes from
environment variables or is resolved via the Coolify API.

Usage:
    python scripts/deploy.py                  # Full deploy (API + Worker)
    python scripts/deploy.py --skip-build     # Skip build, just restart
    python scripts/deploy.py --worker-only    # Only deploy worker service
    python scripts/deploy.py --verify-only    # Only run verification
    python scripts/deploy.py --initial        # Create all resources from scratch

Environment variables:
    COOLIFY_API_TOKEN    — Coolify API token (required)
    COOLIFY_BASE_URL     — Coolify base URL (default: https://server.media-on.de)
    COOLIFY_APP_UUID     — Application UUID (optional, resolved via API if absent)
    COOLIFY_WORKER_UUID  — Worker Service UUID (optional, resolved via API if absent)
    APP_NAME             — Application name for API lookup (default: leocrm-api)
    WORKER_NAME          — Worker name for API lookup (default: leocrm-worker)
    APP_DOMAIN           — App domain for health/FQDN (required, e.g. https://crm.media-on.de)
    SSH_KEY             — SSH key path for verification (default: /a0/usr/workdir/.ssh/coolify-01-root)
    SERVER_IP           — Server IP for SSH verification (default: 46.225.91.159)

    Secrets (required for --initial, used by regular deploy if setting ENVs):
    DB_PASSWORD          — PostgreSQL password for all roles
    REDIS_PASSWORD       — Redis password
    SECRET_KEY           — Application secret key

Exit codes:
    0 — deployment successful
    1 — deployment failed
    2 — configuration error
"""

from __future__ import annotations

import argparse
import base64
import os
import subprocess
import sys
import time
from dataclasses import dataclass, field
from typing import Any

import httpx

# ─── Configuration (all from env, no hardcoded secrets/UUIDs) ─────────

COOLIFY_BASE_URL = os.environ.get("COOLIFY_BASE_URL", "https://server.media-on.de")
COOLIFY_TOKEN = os.environ.get("COOLIFY_API_TOKEN", "")

# UUIDs — from env or resolved via API lookup by name
APP_UUID = os.environ.get("COOLIFY_APP_UUID", "")
WORKER_UUID = os.environ.get("COOLIFY_WORKER_UUID", "")

# Names for API lookup when UUIDs are not provided
APP_NAME = os.environ.get("APP_NAME", "leocrm-api")
WORKER_NAME = os.environ.get("WORKER_NAME", "leocrm-worker")

# Domain (required)
APP_DOMAIN = os.environ.get("APP_DOMAIN", "")

# SSH for verification only
SSH_KEY = os.environ.get("SSH_KEY", "/a0/usr/workdir/.ssh/coolify-01-root")
SERVER_IP = os.environ.get("SERVER_IP", "46.225.91.159")

# Login test credentials (read-only verification)
LOGIN_EMAIL = os.environ.get("LOGIN_EMAIL", "")
LOGIN_PASSWORD = os.environ.get("LOGIN_PASSWORD", "")

# Secrets from env (no hardcoded values)
DB_PASSWORD = os.environ.get("DB_PASSWORD", "")
REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", "")
SECRET_KEY = os.environ.get("SECRET_KEY", "")

# Database/Redis host names (defaults match Coolify service names)
DB_HOST = os.environ.get("DB_HOST", "crm-postgres")
DB_NAME = os.environ.get("DB_NAME", "crm_db")
REDIS_HOST = os.environ.get("REDIS_HOST", "crm-redis")

# Git repo for initial deployment
API_GIT_REPO = os.environ.get("API_GIT_REPO", "https://forgejo.media-on.de/Leopoldadmin/leocrm.git")
API_GIT_BRANCH = os.environ.get("API_GIT_BRANCH", "main")


# ─── Data Structures ──────────────────────────────────────────────────


@dataclass
class StepResult:
    success: bool
    message: str
    duration_s: float = 0.0
    details: dict[str, Any] = field(default_factory=dict)


# ─── Coolify API Client ────────────────────────────────────────────────


class CoolifyClient:
    """Client for Coolify API operations."""

    def __init__(self, base_url: str, token: str):
        self.base_url = base_url.rstrip("/")
        self.headers = {
            "Authorization": f"Bearer {token}",
            "Content-Type": "application/json",
        }

    # ── Applications ──

    def list_applications(self) -> list[dict[str, Any]]:
        """List all applications."""
        resp = httpx.get(
            f"{self.base_url}/api/v1/applications",
            headers=self.headers,
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    def deploy_application(self, app_uuid: str) -> dict[str, Any]:
        """Trigger a build & deploy for an application via Coolify API."""
        resp = httpx.post(
            f"{self.base_url}/api/v1/deploy",
            headers=self.headers,
            json={"uuid": app_uuid},
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    def get_application(self, app_uuid: str) -> dict[str, Any]:
        """Get application details including status."""
        resp = httpx.get(
            f"{self.base_url}/api/v1/applications/{app_uuid}",
            headers=self.headers,
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    def update_application(self, app_uuid: str, **fields: Any) -> dict[str, Any]:
        """Update application fields (e.g. domains/FQDN)."""
        resp = httpx.patch(
            f"{self.base_url}/api/v1/applications/{app_uuid}",
            headers=self.headers,
            json=fields,
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    # ── Services (Worker) ──

    def list_services(self) -> list[dict[str, Any]]:
        """List all services."""
        resp = httpx.get(
            f"{self.base_url}/api/v1/services",
            headers=self.headers,
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    def get_service(self, service_uuid: str) -> dict[str, Any]:
        """Get service details including status."""
        resp = httpx.get(
            f"{self.base_url}/api/v1/services/{service_uuid}",
            headers=self.headers,
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    def update_service(self, service_uuid: str, docker_compose_raw: str) -> dict[str, Any]:
        """Update a service's docker_compose_raw (base64-encoded)."""
        encoded = base64.b64encode(docker_compose_raw.encode()).decode()
        resp = httpx.patch(
            f"{self.base_url}/api/v1/services/{service_uuid}",
            headers=self.headers,
            json={"docker_compose_raw": encoded},
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    def start_service(self, service_uuid: str) -> dict[str, Any]:
        """Start a service."""
        resp = httpx.post(
            f"{self.base_url}/api/v1/services/{service_uuid}/start",
            headers=self.headers,
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    def stop_service(self, service_uuid: str) -> dict[str, Any]:
        """Stop a service."""
        resp = httpx.post(
            f"{self.base_url}/api/v1/services/{service_uuid}/stop",
            headers=self.headers,
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    def restart_service(self, service_uuid: str) -> dict[str, Any]:
        """Restart a service."""
        resp = httpx.post(
            f"{self.base_url}/api/v1/services/{service_uuid}/restart",
            headers=self.headers,
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    # ── Deployments ──

    def get_deployment(self, deployment_uuid: str) -> dict[str, Any]:
        """Get deployment status."""
        resp = httpx.get(
            f"{self.base_url}/api/v1/deployments/{deployment_uuid}",
            headers=self.headers,
            timeout=30,
        )
        resp.raise_for_status()
        return resp.json()

    # ── Health ──

    def get_health(self) -> dict[str, Any]:
        """Check Coolify system health."""
        resp = httpx.get(
            f"{self.base_url}/api/v1/health",
            headers=self.headers,
            timeout=15,
        )
        resp.raise_for_status()
        return resp.json()


# ─── UUID Resolution ──────────────────────────────────────────────────


def resolve_app_uuid(client: CoolifyClient) -> str:
    """Resolve app UUID from env var or API lookup by name."""
    if APP_UUID:
        return APP_UUID

    print(f"  COOLIFY_APP_UUID not set — looking up '{APP_NAME}' via Coolify API...")
    apps = client.list_applications()
    for app in apps:
        name = app.get("name", "")
        uuid = app.get("uuid", "")
        if name == APP_NAME:
            print(f"  Found: {name} → {uuid}")
            return uuid
    raise ValueError(f"Application '{APP_NAME}' not found in Coolify")


def resolve_worker_uuid(client: CoolifyClient) -> str | None:
    """Resolve worker UUID from env var or API lookup by name.
    Returns None if worker is not configured (non-fatal)."""
    if WORKER_UUID:
        return WORKER_UUID

    print(f"  COOLIFY_WORKER_UUID not set — looking up '{WORKER_NAME}' via Coolify API...")
    try:
        services = client.list_services()
        for svc in services:
            name = svc.get("name", "")
            uuid = svc.get("uuid", "")
            if name == WORKER_NAME:
                print(f"  Found: {name} → {uuid}")
                return uuid
        print(f"  Worker '{WORKER_NAME}' not found — worker deploy will be skipped")
        return None
    except Exception as e:
        print(f"  Warning: could not list services: {e}")
        return None


# ─── Worker Compose Generation ────────────────────────────────────────


def generate_worker_compose(app_uuid: str, worker_uuid: str) -> str:
    """Generate worker docker-compose YAML dynamically (no hardcoded UUIDs).

    Uses ${VARIABLE} syntax for secrets — Coolify substitutes from ENV.
    """
    return (
        "services:\n"
        "  worker:\n"
        f"    image: '{app_uuid}:latest'\n"
        "    restart: unless-stopped\n"
        "    entrypoint:\n"
        "      - /app/worker.sh\n"
        "    environment:\n"
        f"      DATABASE_URL: 'postgresql+asyncpg://crm_worker:${{DB_PASSWORD}}@{DB_HOST}:5432/{DB_NAME}'\n"
        f"      WORKER_DATABASE_URL: 'postgresql+asyncpg://crm_worker:${{DB_PASSWORD}}@{DB_HOST}:5432/{DB_NAME}'\n"
        f"      MIGRATION_DATABASE_URL: 'postgresql+asyncpg://crm_user:${{DB_PASSWORD}}@{DB_HOST}:5432/{DB_NAME}'\n"
        f"      AUTH_DATABASE_URL: 'postgresql+asyncpg://crm_auth:${{DB_PASSWORD}}@{DB_HOST}:5432/{DB_NAME}'\n"
        f"      REDIS_URL: 'redis://default:${{REDIS_PASSWORD}}@{REDIS_HOST}:6379/0'\n"
        "      SECRET_KEY: ${SECRET_KEY}\n"
        "      ENVIRONMENT: ${ENVIRONMENT}\n"
        "      STORAGE_PATH: ${STORAGE_PATH}\n"
        f"      COOLIFY_RESOURCE_UUID: {worker_uuid}\n"
        f"      COOLIFY_CONTAINER_NAME: worker-{worker_uuid}\n"
        "      SERVICE_NAME_WORKER: worker\n"
        "    volumes:\n"
        f"      - '{worker_uuid}_leocrm-worker-storage:/data/storage'\n"
        "    networks:\n"
        "      - coolify\n"
        f"      - {worker_uuid}\n"
        f"    container_name: worker-{worker_uuid}\n"
        "    labels:\n"
        "      - coolify.managed=true\n"
        "      - coolify.version=4.0.0-beta.470\n"
        "      - coolify.type=service\n"
        f"      - coolify.name=worker-{worker_uuid}\n"
        f"      - coolify.resourceName={WORKER_NAME}\n"
        "      - coolify.serviceName=worker\n"
        "      - coolify.service.subType=application\n"
        "      - coolify.service.subName=worker\n"
        "volumes:\n"
        "  leocrm-worker-storage:\n"
        "    name: leocrm-worker-storage\n"
        f"  {worker_uuid}_leocrm-worker-storage:\n"
        f"    name: {worker_uuid}_leocrm-worker-storage\n"
        "networks:\n"
        "  coolify:\n"
        "    external: true\n"
        "    name: coolify\n"
        f"  {worker_uuid}:\n"
        f"    name: {worker_uuid}\n"
        "    external: true\n"
    )


def get_worker_envs() -> list[dict]:
    """Worker ENV variables from environment (no hardcoded secrets)."""
    return [
        {"key": "DB_PASSWORD", "value": DB_PASSWORD},
        {"key": "REDIS_PASSWORD", "value": REDIS_PASSWORD},
        {"key": "SECRET_KEY", "value": SECRET_KEY},
        {"key": "ENVIRONMENT", "value": os.environ.get("ENVIRONMENT", "production")},
        {"key": "STORAGE_PATH", "value": os.environ.get("STORAGE_PATH", "/data/storage")},
    ]


def get_api_envs() -> list[dict]:
    """API ENV variables from environment (no hardcoded secrets)."""
    return [
        {"key": "DATABASE_URL", "value": f"postgresql+asyncpg://crm_api:{DB_PASSWORD}@{DB_HOST}:5432/{DB_NAME}"},
        {"key": "AUTH_DATABASE_URL", "value": f"postgresql+asyncpg://crm_auth:{DB_PASSWORD}@{DB_HOST}:5432/{DB_NAME}"},
        {"key": "WORKER_DATABASE_URL", "value": f"postgresql+asyncpg://crm_worker:{DB_PASSWORD}@{DB_HOST}:5432/{DB_NAME}"},
        {"key": "MIGRATION_DATABASE_URL", "value": f"postgresql+asyncpg://crm_user:{DB_PASSWORD}@{DB_HOST}:5432/{DB_NAME}"},
        {"key": "REDIS_URL", "value": f"redis://default:{REDIS_PASSWORD}@{REDIS_HOST}:6379/0"},
        {"key": "SECRET_KEY", "value": SECRET_KEY},
        {"key": "ENVIRONMENT", "value": os.environ.get("ENVIRONMENT", "production")},
        {"key": "STORAGE_PATH", "value": os.environ.get("STORAGE_PATH", "/data/storage")},
        {"key": "FRONTEND_URL", "value": APP_DOMAIN},
        {"key": "CORS_ORIGINS", "value": APP_DOMAIN},
        {"key": "SESSION_COOKIE_SECURE", "value": "true"},
        {"key": "LOG_LEVEL", "value": os.environ.get("LOG_LEVEL", "INFO")},
    ]


# ─── SSH Helper (verification only) ────────────────────────────────────


def ssh_run(cmd: str, timeout: int = 60) -> tuple[int, str]:
    """Run a command on the server via SSH — used ONLY for verification."""
    full_cmd = [
        "ssh", "-i", SSH_KEY,
        "-o", "StrictHostKeyChecking=no",
        "-o", "ConnectTimeout=10",
        f"root@{SERVER_IP}",
        cmd,
    ]
    result = subprocess.run(full_cmd, capture_output=True, text=True, timeout=timeout)
    return result.returncode, result.stdout + result.stderr


# ─── Deploy Steps ──────────────────────────────────────────────────────


def deploy_api(client: CoolifyClient, app_uuid: str, skip_build: bool = False) -> StepResult:
    """Deploy or restart the API application via Coolify API."""
    # Set FQDN via PATCH
    if APP_DOMAIN:
        print(f"  Setting FQDN to {APP_DOMAIN}...")
        try:
            client.update_application(app_uuid, domains=APP_DOMAIN)
        except Exception as e:
            print(f"  Warning: could not set FQDN: {e}")

    if skip_build:
        print("  Skip-build mode: restarting application via Coolify API...")
        try:
            result = client.deploy_application(app_uuid)
            deploy_uuid = _extract_deploy_uuid(result)
            if deploy_uuid:
                print(f"  Deploy queued: {deploy_uuid[:12]}")
                return _wait_deployment(client, deploy_uuid, timeout=300)
            return StepResult(True, "Deploy triggered (no UUID returned)")
        except Exception as e:
            return StepResult(False, f"Deploy trigger failed: {e}")

    print("  Triggering Coolify build & deploy for API...")
    try:
        result = client.deploy_application(app_uuid)
        deploy_uuid = _extract_deploy_uuid(result)
        if not deploy_uuid:
            return StepResult(False, "No deployment UUID returned from Coolify")
        print(f"  Deploy queued: {deploy_uuid[:12]}")
        return _wait_deployment(client, deploy_uuid, timeout=300)
    except Exception as e:
        return StepResult(False, f"Deploy trigger failed: {e}")


def deploy_worker(client: CoolifyClient, app_uuid: str, worker_uuid: str, skip_build: bool = False) -> StepResult:
    """Deploy the worker service via Coolify API.

    Steps:
    1. Update service compose (dynamically generated, ${VARIABLE} syntax)
    2. Set connect_to_docker_network=True (coolify network for Redis/Postgres)
    3. Set ENV variables via Coolify API (secrets from environment)
    4. Tag latest API image as :latest (Coolify uses commit-hash tags)
    5. Deploy via POST /deploy (creates new container)
    6. Wait for healthy
    """
    print("  Deploying worker service via Coolify API...")

    try:
        # Step 1: Update service compose with dynamic UUIDs and ${VARIABLE} syntax
        compose_yaml = generate_worker_compose(app_uuid, worker_uuid)
        print("  Updating worker service compose...")
        client.update_service(worker_uuid, compose_yaml)
        time.sleep(2)

        # Step 2: Set connect_to_docker_network=True
        print("  Ensuring coolify network connection...")
        resp = httpx.patch(
            f"{client.base_url}/api/v1/services/{worker_uuid}",
            headers=client.headers,
            json={"connect_to_docker_network": True},
            timeout=30,
        )
        if resp.status_code != 200:
            print(f"  Warning: could not set connect_to_docker_network ({resp.status_code})")

        # Step 3: Set ENV variables via Coolify API (Coolify auto-generates .env)
        print("  Setting ENV variables via Coolify API...")
        for env in get_worker_envs():
            resp = httpx.post(
                f"{client.base_url}/api/v1/services/{worker_uuid}/envs",
                headers=client.headers,
                json=env,
                timeout=30,
            )
            if resp.status_code == 409:
                resp = httpx.patch(
                    f"{client.base_url}/api/v1/services/{worker_uuid}/envs",
                    headers=client.headers,
                    json=env,
                    timeout=30,
                )
            if resp.status_code not in (200, 201):
                print(f"  Warning: could not set ENV {env['key']} ({resp.status_code})")

        # Step 4: Tag the latest API image as :latest
        print("  Tagging latest API image as :latest...")
        tag_code, tag_output = ssh_run(
            f'docker images --format "{{{{.Repository}}}}:{{{{.Tag}}}}" | '
            f'grep "^{app_uuid}:" | grep -v latest | head -1 | '
            f'xargs -I{{}} docker tag {{}} {app_uuid}:latest'
        )
        if tag_code != 0:
            print(f"  Warning: could not tag :latest ({tag_output.strip()})")

        # Step 5: Deploy via POST /deploy
        print("  Deploying worker service...")
        result = client.deploy_application(worker_uuid)
        deploy_uuid = _extract_deploy_uuid(result)
        if deploy_uuid:
            print(f"  Worker deploy queued: {deploy_uuid[:12]}")
            dep_result = _wait_deployment(client, deploy_uuid, timeout=120)
            if not dep_result.success:
                return dep_result
        else:
            print("  No deployment UUID returned, waiting for healthy...")

        # Step 6: Wait for healthy
        return _wait_service_healthy(client, worker_uuid, timeout=120)

    except Exception as e:
        return StepResult(False, f"Worker deploy failed: {e}")


def _extract_deploy_uuid(result: dict[str, Any]) -> str | None:
    """Extract deployment UUID from Coolify deploy response."""
    deployments = result.get("deployments", [])
    if deployments and isinstance(deployments, list):
        return deployments[0].get("deployment_uuid")
    return result.get("deployment_uuid")


def _wait_deployment(client: CoolifyClient, deploy_uuid: str, timeout: int = 300) -> StepResult:
    """Wait for a Coolify deployment to reach success/failed status."""
    print(f"  Waiting for deployment {deploy_uuid[:12]}...")
    start = time.time()
    while time.time() - start < timeout:
        try:
            dep = client.get_deployment(deploy_uuid)
            status = dep.get("status", "unknown")
            elapsed = int(time.time() - start)
            print(f"  [{elapsed}s] Deployment status: {status}")
            if status in ("success", "finished"):
                return StepResult(True, "Deployment successful", time.time() - start, dep)
            if status == "failed":
                return StepResult(False, f"Deployment failed: {dep.get('message', 'unknown')}", time.time() - start, dep)
        except Exception as e:
            print(f"  Warning: API error: {e}")
        time.sleep(10)
    return StepResult(False, f"Deployment timed out after {timeout}s", time.time() - start)


def _wait_service_healthy(client: CoolifyClient, service_uuid: str, timeout: int = 120) -> StepResult:
    """Wait for a Coolify service to reach running:healthy status."""
    print(f"  Waiting for service {service_uuid[:12]} to become healthy...")
    start = time.time()
    while time.time() - start < timeout:
        try:
            svc = client.get_service(service_uuid)
            status = svc.get("status", "unknown")
            elapsed = int(time.time() - start)
            print(f"  [{elapsed}s] Service status: {status}")
            if status == "running:healthy" or status == "healthy":
                return StepResult(True, f"Service healthy: {status}", time.time() - start, svc)
            if "failed" in status.lower() or "error" in status.lower():
                return StepResult(False, f"Service failed: {status}", time.time() - start, svc)
        except Exception as e:
            print(f"  Warning: API error: {e}")
        time.sleep(5)
    return StepResult(False, f"Service did not become healthy in {timeout}s", time.time() - start)


# ─── Verification ──────────────────────────────────────────────────────


def verify_http_health() -> StepResult:
    """Verify the API is healthy via HTTP endpoint."""
    print("  Verifying API health via HTTP...")
    url = f"{APP_DOMAIN}/api/v1/health"
    try:
        resp = httpx.get(url, timeout=30, follow_redirects=True)
        if resp.status_code == 200:
            body = resp.json() if resp.headers.get("content-type", "").startswith("application/json") else resp.text
            if isinstance(body, dict) and body.get("status", "").lower() in ("healthy", "ok"):
                return StepResult(True, f"Health check passed: {body}")
            return StepResult(True, f"Health check HTTP 200: {body}")
        return StepResult(False, f"Health check failed: HTTP {resp.status_code}")
    except Exception as e:
        return StepResult(False, f"Health check error: {e}")


def verify_login() -> StepResult:
    """Verify login works by sending a test login request."""
    if not LOGIN_EMAIL or not LOGIN_PASSWORD:
        return StepResult(True, "Login test skipped (no credentials provided)")
    print("  Verifying login...")
    url = f"{APP_DOMAIN}/api/v1/auth/login"
    try:
        resp = httpx.post(
            url,
            json={"email": LOGIN_EMAIL, "password": LOGIN_PASSWORD},
            headers={"Origin": APP_DOMAIN},
            timeout=30,
            follow_redirects=True,
        )
        if resp.status_code == 200:
            body = resp.json()
            token = body.get("access_token") or body.get("token")
            if token:
                return StepResult(True, "Login successful — token received")
            return StepResult(True, f"Login HTTP 200: {list(body.keys())}")
        if resp.status_code == 422:
            return StepResult(False, f"Login validation error (422): {resp.text[:200]}")
        return StepResult(False, f"Login failed: HTTP {resp.status_code}")
    except Exception as e:
        return StepResult(False, f"Login error: {e}")


def verify_alembic() -> StepResult:
    """Verify Alembic migration head via SSH (Coolify API doesn't expose DB internals)."""
    print("  Verifying Alembic migration head...")
    code, output = ssh_run(
        'docker exec crm-postgres psql -U crm_user -d crm_db -t -c '
        '"SELECT version_num FROM alembic_version" 2>/dev/null'
    )
    version = output.strip()
    if not version:
        return StepResult(False, "Could not read Alembic version")
    version_ok = version >= "0085"
    return StepResult(
        version_ok,
        f"Alembic version: {version} ({'OK' if version_ok else 'BEHIND — expected >= 0085'})",
        details={"alembic_version": version},
    )


def verify_rls() -> StepResult:
    """Verify RLS is active on tenant tables via SSH (read-only check)."""
    print("  Verifying RLS tables...")
    code, output = ssh_run(
        'docker exec crm-postgres psql -U crm_user -d crm_db -t -c '
        '"SELECT count(*) FROM pg_class WHERE relrowsecurity=true AND relforcerowsecurity=true" 2>/dev/null'
    )
    rls_count = output.strip()
    if not rls_count.isdigit():
        return StepResult(False, f"Could not read RLS table count: {output}")
    count = int(rls_count)
    rls_ok = count >= 90
    return StepResult(
        rls_ok,
        f"RLS tables: {count} ({'OK' if rls_ok else 'LOW — expected >= 90'})",
        details={"rls_tables": count},
    )


def verify_worker_service(client: CoolifyClient, worker_uuid: str) -> StepResult:
    """Verify worker service is running via Coolify API."""
    print("  Verifying worker service via Coolify API...")
    try:
        svc = client.get_service(worker_uuid)
        status = svc.get("status", "unknown")
        status_lower = status.lower()
        if "running" in status_lower or "healthy" in status_lower or status_lower == "up":
            return StepResult(True, f"Worker service: {status}", details={"status": status})
        return StepResult(False, f"Worker service not running: {status}", details={"status": status})
    except Exception as e:
        return StepResult(False, f"Worker service check failed: {e}")


def run_verification(client: CoolifyClient, worker_uuid: str | None = None) -> list[tuple[str, StepResult]]:
    """Run all verification checks and return results."""
    results: list[tuple[str, StepResult]] = []

    print("\n[Verify] HTTP health check...")
    r = verify_http_health()
    results.append(("HTTP health", r))
    _print_result(r)

    print("\n[Verify] Login test...")
    r = verify_login()
    results.append(("Login test", r))
    _print_result(r)

    print("\n[Verify] Alembic migration head...")
    r = verify_alembic()
    results.append(("Alembic version", r))
    _print_result(r)

    print("\n[Verify] RLS tables...")
    r = verify_rls()
    results.append(("RLS tables", r))
    _print_result(r)

    if worker_uuid:
        print("\n[Verify] Worker service status...")
        r = verify_worker_service(client, worker_uuid)
        results.append(("Worker service", r))
        _print_result(r)

    return results


def _print_result(r: StepResult) -> None:
    if r.success:
        print(f"  ✅ {r.message}")
    else:
        print(f"  ❌ {r.message}")


# ─── Summary ───────────────────────────────────────────────────────────


def print_summary(steps: list[tuple[str, StepResult]]) -> bool:
    """Print deployment summary and return overall success."""
    print(f"\n{'='*60}")
    print("  Deploy Summary")
    print(f"{'='*60}")
    for name, result in steps:
        status = "✅" if result.success else "❌"
        print(f"  {status} {name}: {result.message}")
    all_ok = all(r.success for _, r in steps)
    print(f"\n  Overall: {'✅ SUCCESS' if all_ok else '❌ FAILED'}\n")
    return all_ok


# ─── Config Validation ────────────────────────────────────────────────


def validate_config() -> list[str]:
    """Validate required env vars. Returns list of error messages (empty = OK)."""
    errors = []
    if not COOLIFY_TOKEN:
        errors.append("COOLIFY_API_TOKEN is required")
    if not APP_DOMAIN:
        errors.append("APP_DOMAIN is required (e.g. https://crm.media-on.de)")
    return errors


# ─── Main Deploy Pipeline ──────────────────────────────────────────────


def deploy_full(skip_build: bool = False) -> int:
    """Full deploy: API + Worker + Verification."""
    print(f"\n{'='*60}")
    print("  LeoCRM Full Deploy")
    print(f"{'='*60}\n")

    errors = validate_config()
    if errors:
        for e in errors:
            print(f"ERROR: {e}")
        return 2

    client = CoolifyClient(COOLIFY_BASE_URL, COOLIFY_TOKEN)
    steps: list[tuple[str, StepResult]] = []

    # Resolve UUIDs
    print("\n[0/3] Resolving Coolify resources...")
    try:
        app_uuid = resolve_app_uuid(client)
        print(f"  App UUID: {app_uuid}")
    except Exception as e:
        print(f"ERROR: {e}")
        return 2

    worker_uuid = resolve_worker_uuid(client)
    if worker_uuid:
        print(f"  Worker UUID: {worker_uuid}")

    # Step 1: Deploy API
    print("\n[1/3] Deploying API via Coolify API...")
    r = deploy_api(client, app_uuid, skip_build=skip_build)
    steps.append(("API deploy", r))
    _print_result(r)
    if not r.success:
        print_summary(steps)
        return 1

    # Step 2: Deploy Worker
    if worker_uuid:
        print("\n[2/3] Deploying Worker via Coolify API...")
        r = deploy_worker(client, app_uuid, worker_uuid, skip_build=skip_build)
        steps.append(("Worker deploy", r))
        _print_result(r)
    else:
        print("\n[2/3] Worker deploy skipped (no worker UUID resolved)")
        steps.append(("Worker deploy", StepResult(True, "Skipped — no worker configured")))

    # Step 3: Verification
    print("\n[3/3] Running verification...")
    verify_results = run_verification(client, worker_uuid=worker_uuid)
    steps.extend(verify_results)

    all_ok = print_summary(steps)
    return 0 if all_ok else 1


def deploy_worker_only(skip_build: bool = False) -> int:
    """Deploy only the worker service + verification."""
    print(f"\n{'='*60}")
    print("  LeoCRM Worker-Only Deploy")
    print(f"{'='*60}\n")

    errors = validate_config()
    if errors:
        for e in errors:
            print(f"ERROR: {e}")
        return 2

    client = CoolifyClient(COOLIFY_BASE_URL, COOLIFY_TOKEN)
    steps: list[tuple[str, StepResult]] = []

    # Resolve UUIDs
    print("\n[0/2] Resolving Coolify resources...")
    try:
        app_uuid = resolve_app_uuid(client)
    except Exception as e:
        print(f"ERROR: {e}")
        return 2

    worker_uuid = resolve_worker_uuid(client)
    if not worker_uuid:
        print("ERROR: No worker UUID resolved (COOLIFY_WORKER_UUID not set and API lookup failed)")
        return 2

    # Step 1: Deploy Worker
    print("\n[1/2] Deploying Worker via Coolify API...")
    r = deploy_worker(client, app_uuid, worker_uuid, skip_build=skip_build)
    steps.append(("Worker deploy", r))
    _print_result(r)
    if not r.success:
        print_summary(steps)
        return 1

    # Step 2: Verification
    print("\n[2/2] Running verification...")
    verify_results = run_verification(client, worker_uuid=worker_uuid)
    steps.extend(verify_results)

    all_ok = print_summary(steps)
    return 0 if all_ok else 1


def verify_only() -> int:
    """Run only verification checks."""
    print(f"\n{'='*60}")
    print("  LeoCRM Verification Only")
    print(f"{'='*60}\n")

    errors = validate_config()
    if errors:
        for e in errors:
            print(f"ERROR: {e}")
        return 2

    client = CoolifyClient(COOLIFY_BASE_URL, COOLIFY_TOKEN)
    worker_uuid = resolve_worker_uuid(client)
    results = run_verification(client, worker_uuid=worker_uuid)
    all_ok = print_summary(results)
    return 0 if all_ok else 1


# ─── Initial Deployment (create all Coolify resources from scratch) ──────

# PostgreSQL Compose (pgvector for embeddings)
POSTGRES_COMPOSE = (
    "services:\n"
    "  postgres:\n"
    "    image: pgvector/pgvector:pg16\n"
    "    restart: unless-stopped\n"
    "    environment:\n"
    "      POSTGRES_USER: ${DB_USER}\n"
    "      POSTGRES_PASSWORD: ${DB_PASSWORD}\n"
    "      POSTGRES_DB: ${DB_NAME}\n"
    "      PGDATA: /var/lib/postgresql/data/pgdata\n"
    "    volumes:\n"
    "      - pgdata:/var/lib/postgresql/data\n"
    "    healthcheck:\n"
    "      test: ['CMD-SHELL', 'pg_isready -U ${DB_USER} -d ${DB_NAME}']\n"
    "      interval: 10s\n"
    "      timeout: 5s\n"
    "      retries: 5\n"
    "      start_period: 10s\n"
    "volumes:\n"
    "  pgdata:\n"
)

# Redis Compose
REDIS_COMPOSE = (
    "services:\n"
    "  redis:\n"
    "    image: redis:7-alpine\n"
    "    restart: unless-stopped\n"
    "    command: redis-server --requirepass ${REDIS_PASSWORD}\n"
    "    volumes:\n"
    "      - redisdata:/data\n"
    "    healthcheck:\n"
    "      test: ['CMD-SHELL', 'redis-cli ping || exit 1']\n"
    "      interval: 10s\n"
    "      timeout: 5s\n"
    "      retries: 5\n"
    "volumes:\n"
    "  redisdata:\n"
)


def get_postgres_envs() -> list[dict]:
    """PostgreSQL ENV variables from environment."""
    return [
        {"key": "DB_USER", "value": os.environ.get("DB_USER", "crm_user")},
        {"key": "DB_PASSWORD", "value": DB_PASSWORD},
        {"key": "DB_NAME", "value": DB_NAME},
    ]


def get_redis_envs() -> list[dict]:
    """Redis ENV variables from environment."""
    return [
        {"key": "REDIS_PASSWORD", "value": REDIS_PASSWORD},
    ]


def create_service(client: CoolifyClient, project_uuid: str, environment_name: str,
                   server_uuid: str, compose_raw: str, name: str) -> str | None:
    """Create a Coolify service from a docker-compose definition.
    Returns the service UUID or None on failure."""
    encoded = base64.b64encode(compose_raw.encode()).decode()
    try:
        resp = httpx.post(
            f"{client.base_url}/api/v1/services",
            headers=client.headers,
            json={
                "project_uuid": project_uuid,
                "environment_name": environment_name,
                "server_uuid": server_uuid,
                "docker_compose_raw": encoded,
                "name": name,
            },
            timeout=30,
        )
        if resp.status_code in (200, 201):
            data = resp.json()
            return data.get("uuid")
        print(f"  Error creating service {name}: {resp.status_code} {resp.text[:200]}")
        return None
    except Exception as e:
        print(f"  Error creating service {name}: {e}")
        return None


def set_service_envs(client: CoolifyClient, service_uuid: str, envs: list[dict]) -> None:
    """Set ENV variables for a service via Coolify API."""
    for env in envs:
        resp = httpx.post(
            f"{client.base_url}/api/v1/services/{service_uuid}/envs",
            headers=client.headers,
            json=env,
            timeout=30,
        )
        if resp.status_code == 409:
            resp = httpx.patch(
                f"{client.base_url}/api/v1/services/{service_uuid}/envs",
                headers=client.headers,
                json=env,
                timeout=30,
            )
        if resp.status_code not in (200, 201):
            print(f"  Warning: could not set ENV {env['key']} ({resp.status_code})")


def set_application_envs(client: CoolifyClient, app_uuid: str, envs: list[dict]) -> None:
    """Set ENV variables for an application via Coolify API."""
    for env in envs:
        resp = httpx.post(
            f"{client.base_url}/api/v1/applications/{app_uuid}/envs",
            headers=client.headers,
            json=env,
            timeout=30,
        )
        if resp.status_code == 409:
            resp = httpx.patch(
                f"{client.base_url}/api/v1/applications/{app_uuid}/envs",
                headers=client.headers,
                json=env,
                timeout=30,
            )
        if resp.status_code not in (200, 201):
            print(f"  Warning: could not set ENV {env['key']} ({resp.status_code})")


def create_api_application(client: CoolifyClient, project_uuid: str,
                            environment_name: str, server_uuid: str) -> str | None:
    """Create the API application from a Git repository via private deploy key.

    Returns the application UUID or None on failure.
    """
    private_key_uuid = os.environ.get("COOLIFY_PRIVATE_KEY_UUID", "")
    if not private_key_uuid:
        print("  Error: COOLIFY_PRIVATE_KEY_UUID not set")
        return None

    try:
        resp = httpx.post(
            f"{client.base_url}/api/v1/applications/private-deploy-key",
            headers=client.headers,
            json={
                "project_uuid": project_uuid,
                "environment_name": environment_name,
                "server_uuid": server_uuid,
                "private_key_uuid": private_key_uuid,
                "git_repository": API_GIT_REPO,
                "git_branch": API_GIT_BRANCH,
                "build_pack": "dockerfile",
                "name": APP_NAME,
                "ports_exposes": "8000",
            },
            timeout=30,
        )
        if resp.status_code in (200, 201):
            data = resp.json()
            return data.get("uuid")
        print(f"  Error creating API application: {resp.status_code} {resp.text[:300]}")
        return None
    except Exception as e:
        print(f"  Error creating API application: {e}")
        return None


def deploy_initial() -> int:
    """Initial deployment: create all Coolify resources from scratch.

    Creates:
    1. PostgreSQL service (pgvector/pgvector:pg16)
    2. Redis service (redis:7-alpine)
    3. API application (from Git repo)
    4. Worker service (same image as API)
    5. Sets all ENV variables
    6. Deploys API (builds image + runs migrations)
    7. Deploys Worker
    """
    print(f"\n{'='*60}")
    print("  LeoCRM Initial Deployment")
    print(f"{'='*60}\n")

    errors = validate_config()
    # For initial deploy, also require secrets and Coolify resource UUIDs
    if not DB_PASSWORD:
        errors.append("DB_PASSWORD is required for initial deployment")
    if not REDIS_PASSWORD:
        errors.append("REDIS_PASSWORD is required for initial deployment")
    if not SECRET_KEY:
        errors.append("SECRET_KEY is required for initial deployment")
    if not os.environ.get("COOLIFY_PROJECT_UUID"):
        errors.append("COOLIFY_PROJECT_UUID is required for initial deployment")
    if not os.environ.get("COOLIFY_SERVER_UUID"):
        errors.append("COOLIFY_SERVER_UUID is required for initial deployment")
    if not os.environ.get("COOLIFY_PRIVATE_KEY_UUID"):
        errors.append("COOLIFY_PRIVATE_KEY_UUID is required for initial deployment")
    if errors:
        for e in errors:
            print(f"ERROR: {e}")
        return 2

    client = CoolifyClient(COOLIFY_BASE_URL, COOLIFY_TOKEN)
    steps: list[tuple[str, StepResult]] = []

    project_uuid = os.environ.get("COOLIFY_PROJECT_UUID", "")
    environment_name = os.environ.get("COOLIFY_ENVIRONMENT", "production")
    server_uuid = os.environ.get("COOLIFY_SERVER_UUID", "")

    # Step 1: Create PostgreSQL service
    print("\n[1/7] Creating PostgreSQL service...")
    pg_uuid = create_service(client, project_uuid, environment_name, server_uuid,
                             POSTGRES_COMPOSE, "crm-postgres")
    if pg_uuid:
        print(f"  PostgreSQL service created: {pg_uuid[:12]}")
        set_service_envs(client, pg_uuid, get_postgres_envs())
        client.deploy_application(pg_uuid)
        time.sleep(10)
        steps.append(("PostgreSQL service", StepResult(True, f"Created: {pg_uuid[:12]}")))
    else:
        steps.append(("PostgreSQL service", StepResult(False, "Failed to create")))
        print_summary(steps)
        return 1
    _print_result(steps[-1][1])

    # Step 2: Create Redis service
    print("\n[2/7] Creating Redis service...")
    redis_uuid = create_service(client, project_uuid, environment_name, server_uuid,
                                REDIS_COMPOSE, "crm-redis")
    if redis_uuid:
        print(f"  Redis service created: {redis_uuid[:12]}")
        set_service_envs(client, redis_uuid, get_redis_envs())
        client.deploy_application(redis_uuid)
        time.sleep(10)
        steps.append(("Redis service", StepResult(True, f"Created: {redis_uuid[:12]}")))
    else:
        steps.append(("Redis service", StepResult(False, "Failed to create")))
        print_summary(steps)
        return 1
    _print_result(steps[-1][1])

    # Step 3: Create API application
    print("\n[3/7] Creating API application...")
    api_uuid = create_api_application(client, project_uuid, environment_name, server_uuid)
    if api_uuid:
        print(f"  API application created: {api_uuid[:12]}")
        set_application_envs(client, api_uuid, get_api_envs())
        # Set FQDN
        if APP_DOMAIN:
            client.update_application(api_uuid, domains=APP_DOMAIN)
        steps.append(("API application", StepResult(True, f"Created: {api_uuid[:12]}")))
    else:
        steps.append(("API application", StepResult(False, "Failed to create")))
        print_summary(steps)
        return 1
    _print_result(steps[-1][1])

    # Step 4: Create Worker service (with API image reference)
    print("\n[4/7] Creating Worker service...")
    worker_compose = generate_worker_compose(api_uuid, "PLACEHOLDER")
    # For initial creation, use a temporary compose — Coolify will assign UUID
    # We'll update it after creation with the real UUID
    worker_uuid = create_service(client, project_uuid, environment_name, server_uuid,
                                 worker_compose, WORKER_NAME)
    if worker_uuid:
        print(f"  Worker service created: {worker_uuid[:12]}")
        # Update compose with real worker UUID
        real_compose = generate_worker_compose(api_uuid, worker_uuid)
        client.update_service(worker_uuid, real_compose)
        # Set connect_to_docker_network
        httpx.patch(
            f"{client.base_url}/api/v1/services/{worker_uuid}",
            headers=client.headers,
            json={"connect_to_docker_network": True},
            timeout=30,
        )
        set_service_envs(client, worker_uuid, get_worker_envs())
        steps.append(("Worker service", StepResult(True, f"Created: {worker_uuid[:12]}")))
    else:
        steps.append(("Worker service", StepResult(False, "Failed to create")))
        print_summary(steps)
        return 1
    _print_result(steps[-1][1])

    # Step 5: Deploy API (builds image + runs migrations via prestart.sh)
    print("\n[5/7] Deploying API (build + migrations)...")
    try:
        result = client.deploy_application(api_uuid)
        deploy_uuid = _extract_deploy_uuid(result)
        if deploy_uuid:
            print(f"  Deploy queued: {deploy_uuid[:12]}")
            r = _wait_deployment(client, deploy_uuid, timeout=300)
        else:
            r = StepResult(True, "Deploy triggered (no UUID)")
    except Exception as e:
        r = StepResult(False, f"Deploy failed: {e}")
    steps.append(("API deploy", r))
    _print_result(r)
    if not r.success:
        print_summary(steps)
        return 1

    # Step 6: Deploy Worker
    print("\n[6/7] Deploying Worker...")
    tag_code, tag_output = ssh_run(
        f'docker images --format "{{{{.Repository}}}}:{{{{.Tag}}}}" | '
        f'grep "^{api_uuid}:" | grep -v latest | head -1 | '
        f'xargs -I{{}} docker tag {{}} {api_uuid}:latest'
    )
    result = client.deploy_application(worker_uuid)
    deploy_uuid = _extract_deploy_uuid(result)
    if deploy_uuid:
        dep_result = _wait_deployment(client, deploy_uuid, timeout=120)
        steps.append(("Worker deploy", dep_result))
    else:
        wr = _wait_service_healthy(client, worker_uuid, timeout=120)
        steps.append(("Worker deploy", wr))
    _print_result(steps[-1][1])

    # Step 7: Verification
    print("\n[7/7] Running verification...")
    verify_results = run_verification(client, worker_uuid=worker_uuid)
    steps.extend(verify_results)

    all_ok = print_summary(steps)
    return 0 if all_ok else 1


# ─── CLI ───────────────────────────────────────────────────────────────


def main() -> None:
    parser = argparse.ArgumentParser(
        description="LeoCRM automated deployment script (Coolify API only)",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  python scripts/deploy.py                  # Full deploy (API + Worker)
  python scripts/deploy.py --skip-build     # Skip build, just restart
  python scripts/deploy.py --worker-only    # Only deploy worker
  python scripts/deploy.py --verify-only    # Only run verification
  python scripts/deploy.py --initial        # Create all resources from scratch
        """,
    )
    parser.add_argument(
        "--skip-build", action="store_true",
        help="Skip build, just restart services via Coolify API",
    )
    parser.add_argument(
        "--worker-only", action="store_true",
        help="Only deploy the worker service",
    )
    parser.add_argument(
        "--initial", action="store_true",
        help="Initial deployment: create all Coolify resources from scratch",
    )
    parser.add_argument(
        "--verify-only", action="store_true",
        help="Only run verification checks (no deployment)",
    )
    args = parser.parse_args()

    if args.initial:
        sys.exit(deploy_initial())
    elif args.verify_only:
        sys.exit(verify_only())
    elif args.worker_only:
        sys.exit(deploy_worker_only(skip_build=args.skip_build))
    else:
        sys.exit(deploy_full(skip_build=args.skip_build))


if __name__ == "__main__":
    main()