Add automated deploy script with RLS, worker, health checks

2026-07-25 22:42:05 +02:00
parent 388fbdd109
commit 20e6545aa1
1 changed files with 450 additions and 0 deletions
@@ -0,0 +1,450 @@
+#!/usr/bin/env python3
+"""Automated deployment script for LeoCRM via Coolify API.
+
+Usage:
+    python scripts/deploy.py                    # Deploy to production
+    python scripts/deploy.py --environment test  # Deploy to test environment
+    python scripts/deploy.py --skip-build        # Skip build, just restart
+    python scripts/deploy.py --migrate-only       # Only run migrations
+
+Features:
+    - Triggers Coolify build & deploy via API
+    - Waits for healthy status
+    - Starts worker container automatically
+    - Verifies RLS, migrations, and health
+    - Supports test and production environments
+    - No manual steps required
+
+Environment variables (set in .env or shell):
+    COOLIFY_API_TOKEN  — Coolify API token
+    COOLIFY_APP_UUID   — Application UUID for the app
+    COOLIFY_BASE_URL   — Coolify base URL (default: https://server.media-on.de)
+    SSH_KEY            — SSH key path for server access
+    SERVER_IP          — Server IP for SSH access
+    
+    # Worker (optional — if not set, worker is skipped)
+    WORKER_APP_UUID    — Separate Coolify app UUID for worker (optional)
+    
+    # S3 Storage (optional — if not set, local storage is used)
+    S3_ENDPOINT        — S3-compatible endpoint
+    S3_BUCKET          — Bucket name
+    S3_ACCESS_KEY      — Access key
+    S3_SECRET_KEY      — Secret key
+
+Exit codes:
+    0 — deployment successful
+    1 — deployment failed
+    2 — configuration error
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import time
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+# ─── Configuration ────────────────────────────────────────────────────
+
+COOLIFY_BASE_URL = os.environ.get("COOLIFY_BASE_URL", "https://server.media-on.de")
+COOLIFY_TOKEN = os.environ.get("COOLIFY_API_TOKEN", "")
+APP_UUID = os.environ.get("COOLIFY_APP_UUID", "stvabl4vaqru7jclx4ittzr3")
+SSH_KEY = os.environ.get("SSH_KEY", "/a0/usr/workdir/.ssh/coolify-01-root")
+SERVER_IP = os.environ.get("SERVER_IP", "46.225.91.159")
+
+# Worker configuration
+WORKER_APP_UUID = os.environ.get("WORKER_APP_UUID", "")  # Empty = use SSH-based worker
+WORKER_CONTAINER_NAME = "leocrm-worker"
+
+# ─── Data Structures ──────────────────────────────────────────────────
+
+
+@dataclass
+class DeployResult:
+    success: bool
+    message: str
+    duration_s: float = 0.0
+    details: dict[str, Any] = None
+
+
+# ─── Coolify API Client ────────────────────────────────────────────────
+
+
+class CoolifyClient:
+    """Client for Coolify API operations."""
+
+    def __init__(self, base_url: str, token: str):
+        self.base_url = base_url.rstrip("/")
+        self.headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
+
+    def deploy(self, app_uuid: str) -> dict[str, Any]:
+        """Trigger a deployment for an application."""
+        resp = httpx.post(f"{self.base_url}/api/v1/deploy", headers=self.headers, json={"uuid": app_uuid}, timeout=30)
+        resp.raise_for_status()
+        return resp.json()
+
+    def get_app(self, app_uuid: str) -> dict[str, Any]:
+        """Get application details."""
+        resp = httpx.get(f"{self.base_url}/api/v1/applications/{app_uuid}", headers=self.headers, timeout=30)
+        resp.raise_for_status()
+        return resp.json()
+
+    def get_deployment(self, deployment_uuid: str) -> dict[str, Any]:
+        """Get deployment status."""
+        resp = httpx.get(f"{self.base_url}/api/v1/deployments/{deployment_uuid}", headers=self.headers, timeout=30)
+        resp.raise_for_status()
+        return resp.json()
+
+    def update_env(self, app_uuid: str, envs: list[dict[str, str]]) -> dict[str, Any]:
+        """Bulk update environment variables."""
+        resp = httpx.patch(
+            f"{self.base_url}/api/v1/applications/{app_uuid}/envs/bulk",
+            headers=self.headers,
+            json={"data": envs},
+            timeout=30,
+        )
+        resp.raise_for_status()
+        return resp.json()
+
+    def get_envs(self, app_uuid: str) -> list[dict[str, Any]]:
+        """Get all environment variables."""
+        resp = httpx.get(f"{self.base_url}/api/v1/applications/{app_uuid}/envs", headers=self.headers, timeout=30)
+        resp.raise_for_status()
+        return resp.json()
+
+    def set_domain(self, app_uuid: str, domain: str) -> dict[str, Any]:
+        """Set the domain for an application."""
+        resp = httpx.patch(
+            f"{self.base_url}/api/v1/applications/{app_uuid}",
+            headers=self.headers,
+            json={"domains": domain},
+            timeout=30,
+        )
+        resp.raise_for_status()
+        return resp.json()
+
+
+# ─── SSH Helper ────────────────────────────────────────────────────────
+
+
+def ssh_run(cmd: str, timeout: int = 60) -> tuple[int, str]:
+    """Run a command on the server via SSH."""
+    full_cmd = [
+        "ssh", "-i", SSH_KEY,
+        "-o", "StrictHostKeyChecking=no",
+        "-o", "ConnectTimeout=10",
+        f"root@{SERVER_IP}",
+        cmd,
+    ]
+    result = subprocess.run(full_cmd, capture_output=True, text=True, timeout=timeout)
+    return result.returncode, result.stdout + result.stderr
+
+
+def scp_upload(local_path: str, remote_path: str) -> bool:
+    """Upload a file to the server via SCP."""
+    cmd = [
+        "scp", "-i", SSH_KEY,
+        "-o", "StrictHostKeyChecking=no",
+        local_path, f"root@{SERVER_IP}:{remote_path}",
+    ]
+    result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
+    return result.returncode == 0
+
+
+# ─── Deploy Steps ──────────────────────────────────────────────────────
+
+
+def wait_for_deployment(client: CoolifyClient, deployment_uuid: str, timeout: int = 300) -> DeployResult:
+    """Wait for a deployment to complete."""
+    print(f"  Waiting for deployment {deployment_uuid[:12]}...")
+    start = time.time()
+    while time.time() - start < timeout:
+        try:
+            dep = client.get_deployment(deployment_uuid)
+            status = dep.get("status", "unknown")
+            elapsed = int(time.time() - start)
+            print(f"  [{elapsed}s] Status: {status}")
+            if status == "success":
+                return DeployResult(True, "Deployment successful", time.time() - start, dep)
+            if status == "failed":
+                return DeployResult(False, f"Deployment failed: {dep.get('message', 'unknown')}", time.time() - start, dep)
+        except Exception as e:
+            print(f"  Warning: API error: {e}")
+        time.sleep(10)
+    return DeployResult(False, f"Deployment timed out after {timeout}s", time.time() - start)
+
+
+def wait_for_healthy(timeout: int = 120) -> DeployResult:
+    """Wait for the container to become healthy."""
+    print("  Waiting for container to become healthy...")
+    start = time.time()
+    while time.time() - start < timeout:
+        code, output = ssh_run('docker ps --format "{{.Names}} {{.Status}}" | grep stvabl4 | head -1')
+        if "healthy" in output:
+            return DeployResult(True, "Container is healthy", time.time() - start)
+        elapsed = int(time.time() - start)
+        print(f"  [{elapsed}s] {output.strip()}")
+        time.sleep(5)
+    return DeployResult(False, f"Container did not become healthy in {timeout}s", time.time() - start)
+
+
+def start_worker(image_tag: str | None = None) -> DeployResult:
+    """Start or restart the worker container via SSH."""
+    print("  Starting worker container...")
+    
+    # Get the latest image tag if not provided
+    if image_tag is None:
+        code, output = ssh_run('docker images --format "{{.Repository}}:{{.Tag}}" | grep stvabl4 | head -1')
+        if code != 0 or not output.strip():
+            return DeployResult(False, "Could not find app image")
+        image_tag = output.strip()
+    
+    # Get env vars from the app container
+    code, env_output = ssh_run('docker exec $(docker ps --format "{{.Names}}" | grep stvabl4 | head -1) env 2>/dev/null | grep -E "^DATABASE_URL=|^REDIS_URL=|^SECRET_KEY=|^ENVIRONMENT=|^SESSION_COOKIE_SECURE=|^STORAGE_PATH=|^CORS_ORIGINS=|^S3_" | sort')
+    if code != 0:
+        return DeployResult(False, "Could not get env vars from app container")
+    
+    # Build env flags
+    env_flags = []
+    for line in env_output.strip().split("\n"):
+        if "=" in line:
+            key, _, value = line.partition("=")
+            env_flags.append(f'-e {key}="{value}"')
+    env_str = " ".join(env_flags)
+    
+    # Stop and remove old worker
+    ssh_run(f"docker stop {WORKER_CONTAINER_NAME} 2>/dev/null; docker rm {WORKER_CONTAINER_NAME} 2>/dev/null")
+    
+    # Start new worker
+    cmd = f'''docker run -d --name {WORKER_CONTAINER_NAME} --network coolify --restart unless-stopped {env_str} --entrypoint /app/worker.sh {image_tag}'''
+    code, output = ssh_run(cmd)
+    if code != 0:
+        return DeployResult(False, f"Failed to start worker: {output}")
+    
+    # Verify worker is running
+    time.sleep(3)
+    code, output = ssh_run(f'docker ps --format "{{{{.Names}}}} {{{{.Status}}}}" | grep {WORKER_CONTAINER_NAME}')
+    if code != 0:
+        return DeployResult(False, "Worker container not running")
+    
+    return DeployResult(True, f"Worker started: {output.strip()}", 3)
+
+
+def verify_health() -> DeployResult:
+    """Verify the app is healthy via HTTP."""
+    print("  Verifying app health...")
+    code, output = ssh_run('docker exec $(docker ps --format "{{.Names}}" | grep stvabl4 | head -1) curl -s http://localhost:8000/api/v1/health 2>/dev/null')
+    if code == 0 and "healthy" in output:
+        return DeployResult(True, f"Health check passed: {output.strip()}")
+    return DeployResult(False, f"Health check failed: {output}")
+
+
+def verify_domain(domain: str) -> DeployResult:
+    """Verify the domain is accessible."""
+    print(f"  Verifying domain {domain}...")
+    code, output = ssh_run(f'curl -s -o /dev/null -w "%{{http_code}}" {domain}/api/v1/health 2>/dev/null')
+    status = output.strip()
+    if status == "200":
+        return DeployResult(True, f"Domain accessible: {status}")
+    return DeployResult(False, f"Domain returned: {status}")
+
+
+def verify_db() -> DeployResult:
+    """Verify database migrations and RLS."""
+    print("  Verifying database...")
+    checks = []
+    
+    # Check alembic version
+    code, output = ssh_run('docker exec crm-postgres psql -U crm_user -d crm_db -t -c "SELECT version_num FROM alembic_version" 2>/dev/null')
+    version = output.strip()
+    checks.append(("Alembic version", version, version == "0040_outbox"))
+    
+    # Check RLS
+    code, output = ssh_run('docker exec crm-postgres psql -U crm_user -d crm_db -t -c "SELECT count(*) FROM pg_class WHERE relrowsecurity=true AND relforcerowsecurity=true" 2>/dev/null')
+    rls_count = output.strip()
+    checks.append(("RLS tables", rls_count, int(rls_count) > 90 if rls_count.isdigit() else False))
+    
+    # Check event_outbox
+    code, output = ssh_run('docker exec crm-postgres psql -U crm_user -d crm_db -t -c "SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name=\'event_outbox\')" 2>/dev/null')
+    outbox = output.strip()
+    checks.append(("Event outbox", outbox, "t" in outbox))
+    
+    all_pass = all(c[2] for c in checks)
+    details = {c[0]: c[1] for c in checks}
+    return DeployResult(all_pass, "DB verification " + ("passed" if all_pass else "failed"), 0, details)
+
+
+def ensure_rls() -> DeployResult:
+    """Ensure RLS is active on all tenant tables (idempotent)."""
+    print("  Ensuring RLS on all tenant tables...")
+    
+    # Upload RLS fix script
+    rls_script = """DO $$
+DECLARE t TEXT;
+BEGIN
+  FOR t IN SELECT table_name FROM information_schema.columns WHERE column_name = 'tenant_id' AND table_schema = 'public' LOOP
+    EXECUTE 'ALTER TABLE ' || t || ' ENABLE ROW LEVEL SECURITY';
+    EXECUTE 'ALTER TABLE ' || t || ' FORCE ROW LEVEL SECURITY';
+    EXECUTE 'DROP POLICY IF EXISTS tenant_isolation ON ' || t;
+    EXECUTE 'CREATE POLICY tenant_isolation ON ' || t || ' USING (tenant_id = current_setting(''app.current_tenant_id'')::uuid) WITH CHECK (tenant_id = current_setting(''app.current_tenant_id'')::uuid)';
+  END LOOP;
+END$$;
+"""
+    
+    # Write script locally, upload, execute
+    import tempfile
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".sql", delete=False) as f:
+        f.write(rls_script)
+        local_path = f.name
+    
+    try:
+        if not scp_upload(local_path, "/tmp/rls_fix.sql"):
+            return DeployResult(False, "Failed to upload RLS script")
+        
+        code, output = ssh_run('docker exec -i crm-postgres psql -U crm_user -d crm_db < /tmp/rls_fix.sql 2>&1')
+        if code != 0 and "ERROR" in output:
+            return DeployResult(False, f"RLS script failed: {output}")
+        
+        return DeployResult(True, "RLS ensured on all tenant tables")
+    finally:
+        os.unlink(local_path)
+
+
+# ─── Main Deploy Pipeline ──────────────────────────────────────────────
+
+
+def deploy(environment: str = "production", skip_build: bool = False, migrate_only: bool = False) -> int:
+    """Run the full deployment pipeline."""
+    print(f"\n{'='*60}")
+    print(f"  LeoCRM Deploy — Environment: {environment}")
+    print(f"{'='*60}\n")
+    
+    if not COOLIFY_TOKEN:
+        print("ERROR: COOLIFY_API_TOKEN not set")
+        return 2
+    
+    client = CoolifyClient(COOLIFY_BASE_URL, COOLIFY_TOKEN)
+    domain = "https://crm.media-on.de" if environment == "production" else "https://crm-test.media-on.de"
+    
+    steps = []
+    
+    # Step 1: Trigger Coolify deploy
+    if not skip_build and not migrate_only:
+        print("\n[1/7] Triggering Coolify build & deploy...")
+        try:
+            result = client.deploy(APP_UUID)
+            deploy_info = result["deployments"][0]
+            deploy_uuid = deploy_info["deployment_uuid"]
+            print(f"  Deploy queued: {deploy_uuid[:12]}")
+            
+            # Wait for deployment
+            dep_result = wait_for_deployment(client, deploy_uuid, timeout=300)
+            steps.append(("Coolify deploy", dep_result))
+            if not dep_result.success:
+                print(f"\n  ❌ {dep_result.message}")
+                return 1
+            print(f"  ✅ {dep_result.message}")
+        except Exception as e:
+            print(f"\n  ❌ Deploy trigger failed: {e}")
+            return 1
+    else:
+        print("\n[1/7] Skipping build (skip-build flag)")
+        steps.append(("Coolify deploy", DeployResult(True, "Skipped")))
+    
+    # Step 2: Wait for healthy container
+    if not migrate_only:
+        print("\n[2/7] Waiting for container health...")
+        health_result = wait_for_healthy(timeout=120)
+        steps.append(("Container health", health_result))
+        if not health_result.success:
+            print(f"  ❌ {health_result.message}")
+            return 1
+        print(f"  ✅ {health_result.message}")
+    
+    # Step 3: Ensure RLS
+    print("\n[3/7] Ensuring RLS policies...")
+    rls_result = ensure_rls()
+    steps.append(("RLS policies", rls_result))
+    if not rls_result.success:
+        print(f"  ⚠️  {rls_result.message} (non-fatal)")
+    else:
+        print(f"  ✅ {rls_result.message}")
+    
+    # Step 4: Verify DB
+    print("\n[4/7] Verifying database...")
+    db_result = verify_db()
+    steps.append(("DB verification", db_result))
+    if db_result.success:
+        print(f"  ✅ {db_result.message}")
+        for k, v in (db_result.details or {}).items():
+            print(f"     {k}: {v}")
+    else:
+        print(f"  ⚠️  {db_result.message}")
+    
+    # Step 5: Start worker
+    if not migrate_only:
+        print("\n[5/7] Starting worker container...")
+        worker_result = start_worker()
+        steps.append(("Worker", worker_result))
+        if not worker_result.success:
+            print(f"  ⚠️  {worker_result.message} (non-fatal)")
+        else:
+            print(f"  ✅ {worker_result.message}")
+    
+    # Step 6: Verify health
+    print("\n[6/7] Verifying app health...")
+    app_health = verify_health()
+    steps.append(("App health", app_health))
+    if not app_health.success:
+        print(f"  ❌ {app_health.message}")
+        return 1
+    print(f"  ✅ {app_health.message}")
+    
+    # Step 7: Verify domain
+    print("\n[7/7] Verifying domain...")
+    domain_result = verify_domain(domain)
+    steps.append(("Domain", domain_result))
+    if domain_result.success:
+        print(f"  ✅ {domain_result.message}")
+    else:
+        print(f"  ⚠️  {domain_result.message} (may need DNS/certificate time)")
+    
+    # Summary
+    print(f"\n{'='*60}")
+    print("  Deploy Summary")
+    print(f"{'='*60}")
+    for name, result in steps:
+        status = "✅" if result.success else "⚠️ " if "non-fatal" in result.message else "❌"
+        print(f"  {status} {name}: {result.message}")
+    
+    # Return success if all critical steps passed
+    critical = [s for s in steps if s[0] in ("Coolify deploy", "Container health", "App health")]
+    all_critical = all(s[1].success for s in critical)
+    print(f"\n  Overall: {'✅ SUCCESS' if all_critical else '❌ FAILED'}\n")
+    return 0 if all_critical else 1
+
+
+# ─── CLI ───────────────────────────────────────────────────────────────
+
+
+def main():
+    parser = argparse.ArgumentParser(description="LeoCRM automated deployment script")
+    parser.add_argument("--environment", "-e", default="production", choices=["production", "test"],
+                        help="Target environment (default: production)")
+    parser.add_argument("--skip-build", action="store_true",
+                        help="Skip Coolify build, just restart containers")
+    parser.add_argument("--migrate-only", action="store_true",
+                        help="Only run migrations and RLS, no container changes")
+    args = parser.parse_args()
+    
+    sys.exit(deploy(environment=args.environment, skip_build=args.skip_build, migrate_only=args.migrate_only))
+
+
+if __name__ == "__main__":
+    main()