fix: deploy.py rewrite — everything via Coolify API, no manual docker
This commit is contained in:
Agent Zero
+498
-374
@@ -1,35 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Automated deployment script for LeoCRM via Coolify API.
|
||||
|
||||
All container management is done through the Coolify API — no manual
|
||||
docker commands, no SSH for container lifecycle. SSH is used *only*
|
||||
for post-deploy verification (Alembic version, RLS table count) because
|
||||
the Coolify API does not expose database internals.
|
||||
|
||||
Usage:
|
||||
python scripts/deploy.py # Deploy to production
|
||||
python scripts/deploy.py --environment test # Deploy to test environment
|
||||
python scripts/deploy.py --skip-build # Skip build, just restart
|
||||
python scripts/deploy.py --migrate-only # Only run migrations
|
||||
python scripts/deploy.py # Full deploy (API + Worker)
|
||||
python scripts/deploy.py --skip-build # Skip build, just restart
|
||||
python scripts/deploy.py --worker-only # Only deploy worker service
|
||||
python scripts/deploy.py --verify-only # Only run verification
|
||||
|
||||
Features:
|
||||
- Triggers Coolify build & deploy via API
|
||||
- Waits for healthy status
|
||||
- Starts worker container automatically
|
||||
- Verifies RLS, migrations, and health
|
||||
- Supports test and production environments
|
||||
- No manual steps required
|
||||
|
||||
Environment variables (set in .env or shell):
|
||||
COOLIFY_API_TOKEN — Coolify API token
|
||||
COOLIFY_APP_UUID — Application UUID for the app
|
||||
COOLIFY_BASE_URL — Coolify base URL (default: https://server.media-on.de)
|
||||
SSH_KEY — SSH key path for server access
|
||||
SERVER_IP — Server IP for SSH access
|
||||
|
||||
# Worker (optional — if not set, worker is skipped)
|
||||
WORKER_APP_UUID — Separate Coolify app UUID for worker (optional)
|
||||
|
||||
# S3 Storage (optional — if not set, local storage is used)
|
||||
S3_ENDPOINT — S3-compatible endpoint
|
||||
S3_BUCKET — Bucket name
|
||||
S3_ACCESS_KEY — Access key
|
||||
S3_SECRET_KEY — Secret key
|
||||
Environment variables:
|
||||
COOLIFY_API_TOKEN — Coolify API token (required)
|
||||
COOLIFY_APP_UUID — Application UUID (default: stvabl4vaqru7jclx4ittzr3)
|
||||
COOLIFY_WORKER_UUID — Worker Service UUID (default: asxqaq3566to108xordck0ff)
|
||||
COOLIFY_BASE_URL — Coolify base URL (default: https://server.media-on.de)
|
||||
SSH_KEY — SSH key path for verification (default: /a0/usr/workdir/.ssh/coolify-01-root)
|
||||
SERVER_IP — Server IP for SSH verification (default: 46.225.91.159)
|
||||
|
||||
Exit codes:
|
||||
0 — deployment successful
|
||||
@@ -40,12 +29,12 @@ Exit codes:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import base64
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -55,22 +44,57 @@ import httpx
|
||||
COOLIFY_BASE_URL = os.environ.get("COOLIFY_BASE_URL", "https://server.media-on.de")
|
||||
COOLIFY_TOKEN = os.environ.get("COOLIFY_API_TOKEN", "")
|
||||
APP_UUID = os.environ.get("COOLIFY_APP_UUID", "stvabl4vaqru7jclx4ittzr3")
|
||||
WORKER_UUID = os.environ.get("COOLIFY_WORKER_UUID", "asxqaq3566to108xordck0ff")
|
||||
|
||||
SSH_KEY = os.environ.get("SSH_KEY", "/a0/usr/workdir/.ssh/coolify-01-root")
|
||||
SERVER_IP = os.environ.get("SERVER_IP", "46.225.91.159")
|
||||
|
||||
# Worker configuration
|
||||
WORKER_APP_UUID = os.environ.get("WORKER_APP_UUID", "") # Empty = use SSH-based worker
|
||||
WORKER_CONTAINER_NAME = "leocrm-worker"
|
||||
# Domain for HTTP health / login checks
|
||||
APP_DOMAIN = os.environ.get("APP_DOMAIN", "https://crm.media-on.de")
|
||||
|
||||
# Login test credentials (read-only verification)
|
||||
LOGIN_EMAIL = os.environ.get("LOGIN_EMAIL", "admin@media-on.de")
|
||||
LOGIN_PASSWORD = os.environ.get("LOGIN_PASSWORD", "Admin123!")
|
||||
|
||||
# Worker docker-compose definition (base64 will be computed at runtime)
|
||||
WORKER_COMPOSE_YAML = """\
|
||||
services:
|
||||
worker:
|
||||
image: 'stvabl4vaqru7jclx4ittzr3:latest'
|
||||
restart: unless-stopped
|
||||
entrypoint:
|
||||
- /app/worker.sh
|
||||
environment:
|
||||
DATABASE_URL: 'postgresql+asyncpg://crm_worker:PW@crm-postgres:5432/crm_db'
|
||||
WORKER_DATABASE_URL: 'postgresql+asyncpg://crm_worker:PW@crm-postgres:5432/crm_db'
|
||||
MIGRATION_DATABASE_URL: 'postgresql+asyncpg://crm_user:PW@crm-postgres:5432/crm_db'
|
||||
REDIS_URL: 'redis://default:PW@crm-redis:6379/0'
|
||||
SECRET_KEY: 'PW'
|
||||
ENVIRONMENT: production
|
||||
STORAGE_PATH: /data/storage
|
||||
volumes:
|
||||
- 'leocrm-worker-storage:/data/storage'
|
||||
networks:
|
||||
- coolify
|
||||
networks:
|
||||
coolify:
|
||||
external: true
|
||||
name: coolify
|
||||
volumes:
|
||||
leocrm-worker-storage:
|
||||
name: leocrm-worker-storage
|
||||
"""
|
||||
|
||||
|
||||
# ─── Data Structures ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeployResult:
|
||||
class StepResult:
|
||||
success: bool
|
||||
message: str
|
||||
duration_s: float = 0.0
|
||||
details: dict[str, Any] = None
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ─── Coolify API Client ────────────────────────────────────────────────
|
||||
@@ -81,60 +105,118 @@ class CoolifyClient:
|
||||
|
||||
def __init__(self, base_url: str, token: str):
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
||||
self.headers = {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def deploy(self, app_uuid: str) -> dict[str, Any]:
|
||||
"""Trigger a deployment for an application."""
|
||||
resp = httpx.post(f"{self.base_url}/api/v1/deploy", headers=self.headers, json={"uuid": app_uuid}, timeout=30)
|
||||
# ── Applications ──
|
||||
|
||||
def deploy_application(self, app_uuid: str) -> dict[str, Any]:
|
||||
"""Trigger a build & deploy for an application via Coolify API."""
|
||||
resp = httpx.post(
|
||||
f"{self.base_url}/api/v1/deploy",
|
||||
headers=self.headers,
|
||||
json={"uuid": app_uuid},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def get_app(self, app_uuid: str) -> dict[str, Any]:
|
||||
"""Get application details."""
|
||||
resp = httpx.get(f"{self.base_url}/api/v1/applications/{app_uuid}", headers=self.headers, timeout=30)
|
||||
def get_application(self, app_uuid: str) -> dict[str, Any]:
|
||||
"""Get application details including status."""
|
||||
resp = httpx.get(
|
||||
f"{self.base_url}/api/v1/applications/{app_uuid}",
|
||||
headers=self.headers,
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
# ── Services (Worker) ──
|
||||
|
||||
def get_service(self, service_uuid: str) -> dict[str, Any]:
|
||||
"""Get service details including status."""
|
||||
resp = httpx.get(
|
||||
f"{self.base_url}/api/v1/services/{service_uuid}",
|
||||
headers=self.headers,
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def update_service(self, service_uuid: str, docker_compose_raw: str) -> dict[str, Any]:
|
||||
"""Update a service's docker_compose_raw (base64-encoded)."""
|
||||
encoded = base64.b64encode(docker_compose_raw.encode()).decode()
|
||||
resp = httpx.patch(
|
||||
f"{self.base_url}/api/v1/services/{service_uuid}",
|
||||
headers=self.headers,
|
||||
json={"docker_compose_raw": encoded},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def start_service(self, service_uuid: str) -> dict[str, Any]:
|
||||
"""Start a service."""
|
||||
resp = httpx.post(
|
||||
f"{self.base_url}/api/v1/services/{service_uuid}/start",
|
||||
headers=self.headers,
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def stop_service(self, service_uuid: str) -> dict[str, Any]:
|
||||
"""Stop a service."""
|
||||
resp = httpx.post(
|
||||
f"{self.base_url}/api/v1/services/{service_uuid}/stop",
|
||||
headers=self.headers,
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def restart_service(self, service_uuid: str) -> dict[str, Any]:
|
||||
"""Restart a service."""
|
||||
resp = httpx.post(
|
||||
f"{self.base_url}/api/v1/services/{service_uuid}/restart",
|
||||
headers=self.headers,
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
# ── Deployments ──
|
||||
|
||||
def get_deployment(self, deployment_uuid: str) -> dict[str, Any]:
|
||||
"""Get deployment status."""
|
||||
resp = httpx.get(f"{self.base_url}/api/v1/deployments/{deployment_uuid}", headers=self.headers, timeout=30)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def update_env(self, app_uuid: str, envs: list[dict[str, str]]) -> dict[str, Any]:
|
||||
"""Bulk update environment variables."""
|
||||
resp = httpx.patch(
|
||||
f"{self.base_url}/api/v1/applications/{app_uuid}/envs/bulk",
|
||||
resp = httpx.get(
|
||||
f"{self.base_url}/api/v1/deployments/{deployment_uuid}",
|
||||
headers=self.headers,
|
||||
json={"data": envs},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def get_envs(self, app_uuid: str) -> list[dict[str, Any]]:
|
||||
"""Get all environment variables."""
|
||||
resp = httpx.get(f"{self.base_url}/api/v1/applications/{app_uuid}/envs", headers=self.headers, timeout=30)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
# ── Health ──
|
||||
|
||||
def set_domain(self, app_uuid: str, domain: str) -> dict[str, Any]:
|
||||
"""Set the domain for an application."""
|
||||
resp = httpx.patch(
|
||||
f"{self.base_url}/api/v1/applications/{app_uuid}",
|
||||
def get_health(self) -> dict[str, Any]:
|
||||
"""Check Coolify system health."""
|
||||
resp = httpx.get(
|
||||
f"{self.base_url}/api/v1/health",
|
||||
headers=self.headers,
|
||||
json={"domains": domain},
|
||||
timeout=30,
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
# ─── SSH Helper ────────────────────────────────────────────────────────
|
||||
# ─── SSH Helper (verification only) ────────────────────────────────────
|
||||
|
||||
|
||||
def ssh_run(cmd: str, timeout: int = 60) -> tuple[int, str]:
|
||||
"""Run a command on the server via SSH."""
|
||||
"""Run a command on the server via SSH — used ONLY for verification."""
|
||||
full_cmd = [
|
||||
"ssh", "-i", SSH_KEY,
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
@@ -146,360 +228,402 @@ def ssh_run(cmd: str, timeout: int = 60) -> tuple[int, str]:
|
||||
return result.returncode, result.stdout + result.stderr
|
||||
|
||||
|
||||
def scp_upload(local_path: str, remote_path: str) -> bool:
|
||||
"""Upload a file to the server via SCP."""
|
||||
cmd = [
|
||||
"scp", "-i", SSH_KEY,
|
||||
"-o", "StrictHostKeyChecking=no",
|
||||
local_path, f"root@{SERVER_IP}:{remote_path}",
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=30)
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
# ─── Deploy Steps ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def wait_for_deployment(client: CoolifyClient, deployment_uuid: str, timeout: int = 300) -> DeployResult:
|
||||
"""Wait for a deployment to complete."""
|
||||
print(f" Waiting for deployment {deployment_uuid[:12]}...")
|
||||
def deploy_api(client: CoolifyClient, skip_build: bool = False) -> StepResult:
|
||||
"""Deploy or restart the API application via Coolify API."""
|
||||
if skip_build:
|
||||
print(" Skip-build mode: restarting application via Coolify API...")
|
||||
# For skip-build we still trigger a deploy — Coolify will use cached image
|
||||
try:
|
||||
result = client.deploy_application(APP_UUID)
|
||||
deploy_uuid = _extract_deploy_uuid(result)
|
||||
if deploy_uuid:
|
||||
print(f" Deploy queued: {deploy_uuid[:12]}")
|
||||
return _wait_deployment(client, deploy_uuid, timeout=300)
|
||||
return StepResult(True, "Deploy triggered (no UUID returned)")
|
||||
except Exception as e:
|
||||
return StepResult(False, f"Deploy trigger failed: {e}")
|
||||
|
||||
print(" Triggering Coolify build & deploy for API...")
|
||||
try:
|
||||
result = client.deploy_application(APP_UUID)
|
||||
deploy_uuid = _extract_deploy_uuid(result)
|
||||
if not deploy_uuid:
|
||||
return StepResult(False, "No deployment UUID returned from Coolify")
|
||||
print(f" Deploy queued: {deploy_uuid[:12]}")
|
||||
return _wait_deployment(client, deploy_uuid, timeout=300)
|
||||
except Exception as e:
|
||||
return StepResult(False, f"Deploy trigger failed: {e}")
|
||||
|
||||
|
||||
def deploy_worker(client: CoolifyClient, skip_build: bool = False) -> StepResult:
|
||||
"""Deploy or restart the worker service via Coolify API."""
|
||||
print(" Deploying worker service via Coolify API...")
|
||||
|
||||
try:
|
||||
if skip_build:
|
||||
# Just restart the existing service
|
||||
print(" Skip-build mode: restarting worker service...")
|
||||
client.restart_service(WORKER_UUID)
|
||||
time.sleep(3)
|
||||
return _wait_service_healthy(client, WORKER_UUID, timeout=120)
|
||||
|
||||
# Update the service with the latest docker_compose_raw
|
||||
print(" Updating worker service compose definition...")
|
||||
client.update_service(WORKER_UUID, WORKER_COMPOSE_YAML)
|
||||
|
||||
# Trigger deploy via the deploy endpoint
|
||||
print(" Triggering worker deployment...")
|
||||
result = client.deploy_application(WORKER_UUID)
|
||||
deploy_uuid = _extract_deploy_uuid(result)
|
||||
|
||||
if deploy_uuid:
|
||||
print(f" Worker deploy queued: {deploy_uuid[:12]}")
|
||||
dep_result = _wait_deployment(client, deploy_uuid, timeout=300)
|
||||
if not dep_result.success:
|
||||
return dep_result
|
||||
else:
|
||||
print(" No deployment UUID returned, checking service status directly...")
|
||||
|
||||
# Wait for service to be healthy
|
||||
return _wait_service_healthy(client, WORKER_UUID, timeout=120)
|
||||
|
||||
except Exception as e:
|
||||
return StepResult(False, f"Worker deploy failed: {e}")
|
||||
|
||||
|
||||
def _extract_deploy_uuid(result: dict[str, Any]) -> str | None:
|
||||
"""Extract deployment UUID from Coolify deploy response."""
|
||||
# Coolify returns {"deployments": [{"deployment_uuid": "..."}]}
|
||||
deployments = result.get("deployments", [])
|
||||
if deployments and isinstance(deployments, list):
|
||||
return deployments[0].get("deployment_uuid")
|
||||
# Some versions return {"deployment_uuid": "..."} directly
|
||||
return result.get("deployment_uuid")
|
||||
|
||||
|
||||
def _wait_deployment(client: CoolifyClient, deploy_uuid: str, timeout: int = 300) -> StepResult:
|
||||
"""Wait for a Coolify deployment to reach success/failed status."""
|
||||
print(f" Waiting for deployment {deploy_uuid[:12]}...")
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
try:
|
||||
dep = client.get_deployment(deployment_uuid)
|
||||
dep = client.get_deployment(deploy_uuid)
|
||||
status = dep.get("status", "unknown")
|
||||
elapsed = int(time.time() - start)
|
||||
print(f" [{elapsed}s] Status: {status}")
|
||||
print(f" [{elapsed}s] Deployment status: {status}")
|
||||
if status in ("success", "finished"):
|
||||
return DeployResult(True, "Deployment successful", time.time() - start, dep)
|
||||
return StepResult(True, "Deployment successful", time.time() - start, dep)
|
||||
if status == "failed":
|
||||
return DeployResult(False, f"Deployment failed: {dep.get('message', 'unknown')}", time.time() - start, dep)
|
||||
return StepResult(False, f"Deployment failed: {dep.get('message', 'unknown')}", time.time() - start, dep)
|
||||
except Exception as e:
|
||||
print(f" Warning: API error: {e}")
|
||||
time.sleep(10)
|
||||
return DeployResult(False, f"Deployment timed out after {timeout}s", time.time() - start)
|
||||
return StepResult(False, f"Deployment timed out after {timeout}s", time.time() - start)
|
||||
|
||||
|
||||
def wait_for_healthy(timeout: int = 120) -> DeployResult:
|
||||
"""Wait for the container to become healthy."""
|
||||
print(" Waiting for container to become healthy...")
|
||||
def _wait_service_healthy(client: CoolifyClient, service_uuid: str, timeout: int = 120) -> StepResult:
|
||||
"""Wait for a Coolify service to reach running:healthy status."""
|
||||
print(f" Waiting for service {service_uuid[:12]} to become healthy...")
|
||||
start = time.time()
|
||||
while time.time() - start < timeout:
|
||||
code, output = ssh_run('docker ps --format "{{.Names}} {{.Status}}" | grep stvabl4 | head -1')
|
||||
if "healthy" in output:
|
||||
return DeployResult(True, "Container is healthy", time.time() - start)
|
||||
elapsed = int(time.time() - start)
|
||||
print(f" [{elapsed}s] {output.strip()}")
|
||||
try:
|
||||
svc = client.get_service(service_uuid)
|
||||
status = svc.get("status", "unknown")
|
||||
elapsed = int(time.time() - start)
|
||||
print(f" [{elapsed}s] Service status: {status}")
|
||||
if "healthy" in status.lower():
|
||||
return StepResult(True, f"Service healthy: {status}", time.time() - start, svc)
|
||||
if "failed" in status.lower() or "error" in status.lower():
|
||||
return StepResult(False, f"Service failed: {status}", time.time() - start, svc)
|
||||
except Exception as e:
|
||||
print(f" Warning: API error: {e}")
|
||||
time.sleep(5)
|
||||
return DeployResult(False, f"Container did not become healthy in {timeout}s", time.time() - start)
|
||||
return StepResult(False, f"Service did not become healthy in {timeout}s", time.time() - start)
|
||||
|
||||
|
||||
def start_worker(image_tag: str | None = None) -> DeployResult:
|
||||
"""Start or restart the worker container via SSH."""
|
||||
print(" Starting worker container...")
|
||||
|
||||
# Get the latest image tag if not provided
|
||||
if image_tag is None:
|
||||
code, output = ssh_run('docker images --format "{{.Repository}}:{{.Tag}}" | grep stvabl4 | head -1')
|
||||
if code != 0 or not output.strip():
|
||||
return DeployResult(False, "Could not find app image")
|
||||
image_tag = output.strip()
|
||||
|
||||
# Get env vars from the app container
|
||||
code, env_output = ssh_run('docker exec $(docker ps --format "{{.Names}}" | grep stvabl4 | head -1) env 2>/dev/null | grep -E "^DATABASE_URL=|^REDIS_URL=|^SECRET_KEY=|^ENVIRONMENT=|^SESSION_COOKIE_SECURE=|^STORAGE_PATH=|^CORS_ORIGINS=|^S3_" | sort')
|
||||
if code != 0:
|
||||
return DeployResult(False, "Could not get env vars from app container")
|
||||
|
||||
# Build env flags
|
||||
env_flags = []
|
||||
for line in env_output.strip().split("\n"):
|
||||
if "=" in line:
|
||||
key, _, value = line.partition("=")
|
||||
env_flags.append(f'-e {key}="{value}"')
|
||||
env_str = " ".join(env_flags)
|
||||
|
||||
# Stop and remove old worker
|
||||
ssh_run(f"docker stop {WORKER_CONTAINER_NAME} 2>/dev/null; docker rm {WORKER_CONTAINER_NAME} 2>/dev/null")
|
||||
|
||||
# Start new worker
|
||||
cmd = f'''docker run -d --name {WORKER_CONTAINER_NAME} --network coolify --restart unless-stopped {env_str} --entrypoint /app/worker.sh {image_tag}'''
|
||||
code, output = ssh_run(cmd)
|
||||
if code != 0:
|
||||
return DeployResult(False, f"Failed to start worker: {output}")
|
||||
|
||||
# Verify worker is running
|
||||
time.sleep(3)
|
||||
code, output = ssh_run(f'docker ps --format "{{{{.Names}}}} {{{{.Status}}}}" | grep {WORKER_CONTAINER_NAME}')
|
||||
if code != 0:
|
||||
return DeployResult(False, "Worker container not running")
|
||||
|
||||
return DeployResult(True, f"Worker started: {output.strip()}", 3)
|
||||
# ─── Verification ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def verify_health() -> DeployResult:
|
||||
"""Verify the app is healthy via HTTP."""
|
||||
print(" Verifying app health...")
|
||||
code, output = ssh_run('docker exec $(docker ps --format "{{.Names}}" | grep stvabl4 | head -1) curl -s http://localhost:8000/api/v1/health 2>/dev/null')
|
||||
if code == 0 and "healthy" in output:
|
||||
return DeployResult(True, f"Health check passed: {output.strip()}")
|
||||
return DeployResult(False, f"Health check failed: {output}")
|
||||
|
||||
|
||||
def verify_domain(domain: str) -> DeployResult:
|
||||
"""Verify the domain is accessible."""
|
||||
print(f" Verifying domain {domain}...")
|
||||
code, output = ssh_run(f'curl -s -o /dev/null -w "%{{http_code}}" {domain}/api/v1/health 2>/dev/null')
|
||||
status = output.strip()
|
||||
if status == "200":
|
||||
return DeployResult(True, f"Domain accessible: {status}")
|
||||
return DeployResult(False, f"Domain returned: {status}")
|
||||
|
||||
|
||||
def verify_db() -> DeployResult:
|
||||
"""Verify database migrations and RLS."""
|
||||
print(" Verifying database...")
|
||||
checks = []
|
||||
|
||||
# Check alembic version — must be at least 0045 (latest migration)
|
||||
code, output = ssh_run('docker exec crm-postgres psql -U crm_user -d crm_db -t -c "SELECT version_num FROM alembic_version" 2>/dev/null')
|
||||
version = output.strip()
|
||||
# Accept any version >= 0045 (covers 0045, 0044_repair_contact_migration, etc.)
|
||||
version_ok = version >= "0045" or version.startswith("0045")
|
||||
checks.append(("Alembic version", version, version_ok))
|
||||
|
||||
# Check RLS
|
||||
code, output = ssh_run('docker exec crm-postgres psql -U crm_user -d crm_db -t -c "SELECT count(*) FROM pg_class WHERE relrowsecurity=true AND relforcerowsecurity=true" 2>/dev/null')
|
||||
rls_count = output.strip()
|
||||
checks.append(("RLS tables", rls_count, int(rls_count) > 90 if rls_count.isdigit() else False))
|
||||
|
||||
# Check event_outbox
|
||||
code, output = ssh_run('docker exec crm-postgres psql -U crm_user -d crm_db -t -c "SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name=\'event_outbox\')" 2>/dev/null')
|
||||
outbox = output.strip()
|
||||
checks.append(("Event outbox", outbox, "t" in outbox))
|
||||
|
||||
all_pass = all(c[2] for c in checks)
|
||||
details = {c[0]: c[1] for c in checks}
|
||||
return DeployResult(all_pass, "DB verification " + ("passed" if all_pass else "failed"), 0, details)
|
||||
|
||||
|
||||
def ensure_volume() -> DeployResult:
|
||||
"""Ensure persistent volume is configured in Coolify DB.
|
||||
|
||||
Creates a persistent volume entry in Coolify's local_persistent_volumes table
|
||||
if it doesn't exist. Coolify will then automatically mount the volume in
|
||||
every generated docker-compose.yaml — no post-deploy patching needed.
|
||||
|
||||
This works on any Coolify instance and with multiple apps per server.
|
||||
"""
|
||||
print(" Ensuring persistent volume in Coolify DB...")
|
||||
|
||||
# Get app ID from Coolify DB
|
||||
code, output = ssh_run(f"docker exec coolify-db psql -U coolify -d coolify -t -c \"SELECT id FROM applications WHERE uuid = '{APP_UUID}'\" 2>/dev/null")
|
||||
if code != 0 or not output.strip():
|
||||
return DeployResult(False, "Could not find app in Coolify DB")
|
||||
app_id = output.strip()
|
||||
|
||||
# Check if volume entry already exists
|
||||
vol_name = f"{APP_UUID}_storage"
|
||||
code, output = ssh_run(f"docker exec coolify-db psql -U coolify -d coolify -t -c \"SELECT id FROM local_persistent_volumes WHERE resource_id = {app_id} AND name = '{vol_name}'\" 2>/dev/null")
|
||||
if code == 0 and output.strip():
|
||||
return DeployResult(True, "Volume already configured in Coolify DB")
|
||||
|
||||
# Create volume entry in Coolify DB
|
||||
code, output = ssh_run(f"docker exec coolify-db psql -U coolify -d coolify -c \"INSERT INTO local_persistent_volumes (name, mount_path, resource_type, resource_id, created_at, updated_at, is_preview_suffix_enabled, uuid) VALUES ('{vol_name}', '/data/storage', 'App\\\\\\\\Models\\\\\\\\Application', {app_id}, NOW(), NOW(), false, gen_random_uuid())\" 2>&1")
|
||||
if code != 0:
|
||||
return DeployResult(False, f"Failed to create volume entry: {output}")
|
||||
|
||||
return DeployResult(True, "Volume entry created in Coolify DB")
|
||||
|
||||
|
||||
def ensure_rls() -> DeployResult:
|
||||
"""Ensure RLS is active on all tenant tables (idempotent)."""
|
||||
print(" Ensuring RLS on all tenant tables...")
|
||||
|
||||
# Upload RLS fix script
|
||||
rls_script = """DO $$
|
||||
DECLARE t TEXT;
|
||||
BEGIN
|
||||
FOR t IN SELECT table_name FROM information_schema.columns WHERE column_name = 'tenant_id' AND table_schema = 'public' LOOP
|
||||
-- Skip system/auth/config tables that need access without tenant context
|
||||
IF t IN ('users', 'tenants', 'user_tenants', 'sessions',
|
||||
'audit_log', 'password_reset_tokens', 'api_tokens',
|
||||
'groups', 'roles', 'permissions', 'user_groups',
|
||||
'system_settings', 'currencies', 'tax_rates', 'sequences',
|
||||
'plugins', 'tenant_plugin_activation',
|
||||
'workspaces', 'workspace_modules', 'workspace_users', 'workspace_widgets',
|
||||
'automation_cron_jobs', 'automation_definitions', 'automation_runs',
|
||||
'automation_versions', 'automation_agent_definitions',
|
||||
'automation_agent_runs', 'automation_agent_versions',
|
||||
'user_preferences', 'saved_filters', 'saved_views', 'webhooks',
|
||||
'notification_preferences', 'notification_types') THEN
|
||||
CONTINUE;
|
||||
END IF;
|
||||
EXECUTE 'ALTER TABLE ' || t || ' ENABLE ROW LEVEL SECURITY';
|
||||
EXECUTE 'DROP POLICY IF EXISTS tenant_isolation ON ' || t;
|
||||
EXECUTE 'CREATE POLICY tenant_isolation ON ' || t || ' USING (tenant_id = current_setting(''app.current_tenant_id'')::uuid) WITH CHECK (tenant_id = current_setting(''app.current_tenant_id'')::uuid)';
|
||||
END LOOP;
|
||||
END$$;
|
||||
"""
|
||||
|
||||
# Write script locally, upload, execute
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".sql", delete=False) as f:
|
||||
f.write(rls_script)
|
||||
local_path = f.name
|
||||
|
||||
def verify_http_health() -> StepResult:
|
||||
"""Verify the API is healthy via HTTP endpoint."""
|
||||
print(" Verifying API health via HTTP...")
|
||||
url = f"{APP_DOMAIN}/api/v1/health"
|
||||
try:
|
||||
if not scp_upload(local_path, "/tmp/rls_fix.sql"):
|
||||
return DeployResult(False, "Failed to upload RLS script")
|
||||
|
||||
code, output = ssh_run('docker exec -i crm-postgres psql -U crm_user -d crm_db < /tmp/rls_fix.sql 2>&1')
|
||||
if code != 0 and "ERROR" in output:
|
||||
return DeployResult(False, f"RLS script failed: {output}")
|
||||
|
||||
return DeployResult(True, "RLS ensured on all tenant tables")
|
||||
finally:
|
||||
os.unlink(local_path)
|
||||
resp = httpx.get(url, timeout=30, follow_redirects=True)
|
||||
if resp.status_code == 200:
|
||||
body = resp.json() if resp.headers.get("content-type", "").startswith("application/json") else resp.text
|
||||
if isinstance(body, dict) and body.get("status", "").lower() in ("healthy", "ok"):
|
||||
return StepResult(True, f"Health check passed: {body}")
|
||||
return StepResult(True, f"Health check HTTP 200: {body}")
|
||||
return StepResult(False, f"Health check failed: HTTP {resp.status_code}")
|
||||
except Exception as e:
|
||||
return StepResult(False, f"Health check error: {e}")
|
||||
|
||||
|
||||
def verify_login() -> StepResult:
|
||||
"""Verify login works by sending a test login request."""
|
||||
print(" Verifying login...")
|
||||
url = f"{APP_DOMAIN}/api/v1/auth/login"
|
||||
try:
|
||||
resp = httpx.post(
|
||||
url,
|
||||
json={"email": LOGIN_EMAIL, "password": LOGIN_PASSWORD},
|
||||
timeout=30,
|
||||
follow_redirects=True,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
body = resp.json()
|
||||
token = body.get("access_token") or body.get("token")
|
||||
if token:
|
||||
return StepResult(True, "Login successful — token received")
|
||||
return StepResult(True, f"Login HTTP 200: {list(body.keys())}")
|
||||
if resp.status_code == 422:
|
||||
return StepResult(False, f"Login validation error (422): {resp.text[:200]}")
|
||||
return StepResult(False, f"Login failed: HTTP {resp.status_code}")
|
||||
except Exception as e:
|
||||
return StepResult(False, f"Login error: {e}")
|
||||
|
||||
|
||||
def verify_alembic() -> StepResult:
|
||||
"""Verify Alembic migration head via SSH (Coolify API doesn't expose DB internals)."""
|
||||
print(" Verifying Alembic migration head...")
|
||||
code, output = ssh_run(
|
||||
'docker exec crm-postgres psql -U crm_user -d crm_db -t -c '
|
||||
'"SELECT version_num FROM alembic_version" 2>/dev/null'
|
||||
)
|
||||
version = output.strip()
|
||||
if not version:
|
||||
return StepResult(False, "Could not read Alembic version")
|
||||
# Accept versions >= 0085 (migrations 0085-0090 handle RLS)
|
||||
version_ok = version >= "0085"
|
||||
return StepResult(
|
||||
version_ok,
|
||||
f"Alembic version: {version} ({'OK' if version_ok else 'BEHIND — expected >= 0085'})",
|
||||
details={"alembic_version": version},
|
||||
)
|
||||
|
||||
|
||||
def verify_rls() -> StepResult:
|
||||
"""Verify RLS is active on tenant tables via SSH (read-only check)."""
|
||||
print(" Verifying RLS tables...")
|
||||
code, output = ssh_run(
|
||||
'docker exec crm-postgres psql -U crm_user -d crm_db -t -c '
|
||||
'"SELECT count(*) FROM pg_class WHERE relrowsecurity=true AND relforcerowsecurity=true" 2>/dev/null'
|
||||
)
|
||||
rls_count = output.strip()
|
||||
if not rls_count.isdigit():
|
||||
return StepResult(False, f"Could not read RLS table count: {output}")
|
||||
count = int(rls_count)
|
||||
# RLS should be active on all tenant tables (typically 90+)
|
||||
rls_ok = count >= 90
|
||||
return StepResult(
|
||||
rls_ok,
|
||||
f"RLS tables: {count} ({'OK' if rls_ok else 'LOW — expected >= 90'})",
|
||||
details={"rls_tables": count},
|
||||
)
|
||||
|
||||
|
||||
def verify_worker_service(client: CoolifyClient) -> StepResult:
|
||||
"""Verify worker service is running via Coolify API."""
|
||||
print(" Verifying worker service via Coolify API...")
|
||||
try:
|
||||
svc = client.get_service(WORKER_UUID)
|
||||
status = svc.get("status", "unknown")
|
||||
if "running" in status.lower():
|
||||
return StepResult(True, f"Worker service: {status}", details={"status": status})
|
||||
return StepResult(False, f"Worker service not running: {status}", details={"status": status})
|
||||
except Exception as e:
|
||||
return StepResult(False, f"Worker service check failed: {e}")
|
||||
|
||||
|
||||
def run_verification(client: CoolifyClient, check_worker: bool = True) -> list[tuple[str, StepResult]]:
|
||||
"""Run all verification checks and return results."""
|
||||
results: list[tuple[str, StepResult]] = []
|
||||
|
||||
# 1. HTTP health
|
||||
print("\n[Verify] HTTP health check...")
|
||||
r = verify_http_health()
|
||||
results.append(("HTTP health", r))
|
||||
_print_result(r)
|
||||
|
||||
# 2. Login test
|
||||
print("\n[Verify] Login test...")
|
||||
r = verify_login()
|
||||
results.append(("Login test", r))
|
||||
_print_result(r)
|
||||
|
||||
# 3. Alembic version
|
||||
print("\n[Verify] Alembic migration head...")
|
||||
r = verify_alembic()
|
||||
results.append(("Alembic version", r))
|
||||
_print_result(r)
|
||||
|
||||
# 4. RLS tables
|
||||
print("\n[Verify] RLS tables...")
|
||||
r = verify_rls()
|
||||
results.append(("RLS tables", r))
|
||||
_print_result(r)
|
||||
|
||||
# 5. Worker service (optional)
|
||||
if check_worker:
|
||||
print("\n[Verify] Worker service status...")
|
||||
r = verify_worker_service(client)
|
||||
results.append(("Worker service", r))
|
||||
_print_result(r)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _print_result(r: StepResult) -> None:
|
||||
if r.success:
|
||||
print(f" ✅ {r.message}")
|
||||
else:
|
||||
print(f" ❌ {r.message}")
|
||||
|
||||
|
||||
# ─── Summary ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def print_summary(steps: list[tuple[str, StepResult]]) -> bool:
|
||||
"""Print deployment summary and return overall success."""
|
||||
print(f"\n{'='*60}")
|
||||
print(" Deploy Summary")
|
||||
print(f"{'='*60}")
|
||||
for name, result in steps:
|
||||
status = "✅" if result.success else "❌"
|
||||
print(f" {status} {name}: {result.message}")
|
||||
all_ok = all(r.success for _, r in steps)
|
||||
print(f"\n Overall: {'✅ SUCCESS' if all_ok else '❌ FAILED'}\n")
|
||||
return all_ok
|
||||
|
||||
|
||||
# ─── Main Deploy Pipeline ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def deploy(environment: str = "production", skip_build: bool = False, migrate_only: bool = False) -> int:
|
||||
"""Run the full deployment pipeline."""
|
||||
def deploy_full(skip_build: bool = False) -> int:
|
||||
"""Full deploy: API + Worker + Verification."""
|
||||
print(f"\n{'='*60}")
|
||||
print(f" LeoCRM Deploy — Environment: {environment}")
|
||||
print(" LeoCRM Full Deploy")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
|
||||
if not COOLIFY_TOKEN:
|
||||
print("ERROR: COOLIFY_API_TOKEN not set")
|
||||
return 2
|
||||
|
||||
|
||||
client = CoolifyClient(COOLIFY_BASE_URL, COOLIFY_TOKEN)
|
||||
domain = "https://crm.media-on.de" if environment == "production" else "https://crm-test.media-on.de"
|
||||
|
||||
steps = []
|
||||
|
||||
# Step 1: Trigger Coolify deploy
|
||||
if not skip_build and not migrate_only:
|
||||
print("\n[1/8] Triggering Coolify build & deploy...")
|
||||
try:
|
||||
result = client.deploy(APP_UUID)
|
||||
deploy_info = result["deployments"][0]
|
||||
deploy_uuid = deploy_info["deployment_uuid"]
|
||||
print(f" Deploy queued: {deploy_uuid[:12]}")
|
||||
|
||||
# Wait for deployment
|
||||
dep_result = wait_for_deployment(client, deploy_uuid, timeout=300)
|
||||
steps.append(("Coolify deploy", dep_result))
|
||||
if not dep_result.success:
|
||||
print(f"\n ❌ {dep_result.message}")
|
||||
return 1
|
||||
print(f" ✅ {dep_result.message}")
|
||||
except Exception as e:
|
||||
print(f"\n ❌ Deploy trigger failed: {e}")
|
||||
return 1
|
||||
else:
|
||||
print("\n[1/7] Skipping build (skip-build flag)")
|
||||
steps.append(("Coolify deploy", DeployResult(True, "Skipped")))
|
||||
|
||||
# Step 2: Ensure persistent volume
|
||||
if not migrate_only:
|
||||
print("\n[2/8] Ensuring persistent volume...")
|
||||
vol_result = ensure_volume()
|
||||
steps.append(("Persistent volume", vol_result))
|
||||
if not vol_result.success:
|
||||
print(f" ⚠️ {vol_result.message} (non-fatal)")
|
||||
else:
|
||||
print(f" ✅ {vol_result.message}")
|
||||
|
||||
# Step 3: Wait for healthy container
|
||||
if not migrate_only:
|
||||
print("\n[3/8] Waiting for container health...")
|
||||
health_result = wait_for_healthy(timeout=120)
|
||||
steps.append(("Container health", health_result))
|
||||
if not health_result.success:
|
||||
print(f" ❌ {health_result.message}")
|
||||
return 1
|
||||
print(f" ✅ {health_result.message}")
|
||||
|
||||
# Step 4: Ensure RLS
|
||||
print("\n[4/8] Ensuring RLS policies...")
|
||||
rls_result = ensure_rls()
|
||||
steps.append(("RLS policies", rls_result))
|
||||
if not rls_result.success:
|
||||
print(f" ⚠️ {rls_result.message} (non-fatal)")
|
||||
else:
|
||||
print(f" ✅ {rls_result.message}")
|
||||
|
||||
# Step 5: Verify DB
|
||||
print("\n[5/8] Verifying database...")
|
||||
db_result = verify_db()
|
||||
steps.append(("DB verification", db_result))
|
||||
if db_result.success:
|
||||
print(f" ✅ {db_result.message}")
|
||||
for k, v in (db_result.details or {}).items():
|
||||
print(f" {k}: {v}")
|
||||
else:
|
||||
print(f" ⚠️ {db_result.message}")
|
||||
|
||||
# Step 6: Start worker
|
||||
if not migrate_only:
|
||||
print("\n[6/8] Starting worker container...")
|
||||
worker_result = start_worker()
|
||||
steps.append(("Worker", worker_result))
|
||||
if not worker_result.success:
|
||||
print(f" ⚠️ {worker_result.message} (non-fatal)")
|
||||
else:
|
||||
print(f" ✅ {worker_result.message}")
|
||||
|
||||
# Step 7: Verify health
|
||||
print("\n[7/8] Verifying app health...")
|
||||
app_health = verify_health()
|
||||
steps.append(("App health", app_health))
|
||||
if not app_health.success:
|
||||
print(f" ❌ {app_health.message}")
|
||||
steps: list[tuple[str, StepResult]] = []
|
||||
|
||||
# Step 1: Deploy API
|
||||
print("\n[1/3] Deploying API via Coolify API...")
|
||||
r = deploy_api(client, skip_build=skip_build)
|
||||
steps.append(("API deploy", r))
|
||||
_print_result(r)
|
||||
if not r.success:
|
||||
print_summary(steps)
|
||||
return 1
|
||||
print(f" ✅ {app_health.message}")
|
||||
|
||||
# Step 8: Verify domain
|
||||
print("\n[8/8] Verifying domain...")
|
||||
domain_result = verify_domain(domain)
|
||||
steps.append(("Domain", domain_result))
|
||||
if domain_result.success:
|
||||
print(f" ✅ {domain_result.message}")
|
||||
else:
|
||||
print(f" ⚠️ {domain_result.message} (may need DNS/certificate time)")
|
||||
|
||||
|
||||
# Step 2: Deploy Worker
|
||||
print("\n[2/3] Deploying Worker via Coolify API...")
|
||||
r = deploy_worker(client, skip_build=skip_build)
|
||||
steps.append(("Worker deploy", r))
|
||||
_print_result(r)
|
||||
# Worker failure is non-fatal but reported
|
||||
|
||||
# Step 3: Verification
|
||||
print("\n[3/3] Running verification...")
|
||||
verify_results = run_verification(client, check_worker=True)
|
||||
steps.extend(verify_results)
|
||||
|
||||
# Summary
|
||||
all_ok = print_summary(steps)
|
||||
return 0 if all_ok else 1
|
||||
|
||||
|
||||
def deploy_worker_only(skip_build: bool = False) -> int:
|
||||
"""Deploy only the worker service + verification."""
|
||||
print(f"\n{'='*60}")
|
||||
print(" Deploy Summary")
|
||||
print(f"{'='*60}")
|
||||
for name, result in steps:
|
||||
status = "✅" if result.success else "⚠️ " if "non-fatal" in result.message else "❌"
|
||||
print(f" {status} {name}: {result.message}")
|
||||
|
||||
# Return success if all critical steps passed
|
||||
critical = [s for s in steps if s[0] in ("Coolify deploy", "Container health", "App health", "Persistent volume")]
|
||||
all_critical = all(s[1].success for s in critical)
|
||||
print(f"\n Overall: {'✅ SUCCESS' if all_critical else '❌ FAILED'}\n")
|
||||
return 0 if all_critical else 1
|
||||
print(" LeoCRM Worker-Only Deploy")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
if not COOLIFY_TOKEN:
|
||||
print("ERROR: COOLIFY_API_TOKEN not set")
|
||||
return 2
|
||||
|
||||
client = CoolifyClient(COOLIFY_BASE_URL, COOLIFY_TOKEN)
|
||||
steps: list[tuple[str, StepResult]] = []
|
||||
|
||||
# Step 1: Deploy Worker
|
||||
print("\n[1/2] Deploying Worker via Coolify API...")
|
||||
r = deploy_worker(client, skip_build=skip_build)
|
||||
steps.append(("Worker deploy", r))
|
||||
_print_result(r)
|
||||
if not r.success:
|
||||
print_summary(steps)
|
||||
return 1
|
||||
|
||||
# Step 2: Verification
|
||||
print("\n[2/2] Running verification...")
|
||||
verify_results = run_verification(client, check_worker=True)
|
||||
steps.extend(verify_results)
|
||||
|
||||
all_ok = print_summary(steps)
|
||||
return 0 if all_ok else 1
|
||||
|
||||
|
||||
def verify_only() -> int:
|
||||
"""Run only verification checks."""
|
||||
print(f"\n{'='*60}")
|
||||
print(" LeoCRM Verification Only")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
if not COOLIFY_TOKEN:
|
||||
print("ERROR: COOLIFY_API_TOKEN not set")
|
||||
return 2
|
||||
|
||||
client = CoolifyClient(COOLIFY_BASE_URL, COOLIFY_TOKEN)
|
||||
results = run_verification(client, check_worker=True)
|
||||
all_ok = print_summary(results)
|
||||
return 0 if all_ok else 1
|
||||
|
||||
|
||||
# ─── CLI ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="LeoCRM automated deployment script")
|
||||
parser.add_argument("--environment", "-e", default="production", choices=["production", "test"],
|
||||
help="Target environment (default: production)")
|
||||
parser.add_argument("--skip-build", action="store_true",
|
||||
help="Skip Coolify build, just restart containers")
|
||||
parser.add_argument("--migrate-only", action="store_true",
|
||||
help="Only run migrations and RLS, no container changes")
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="LeoCRM automated deployment script (Coolify API only)",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python scripts/deploy.py # Full deploy (API + Worker)
|
||||
python scripts/deploy.py --skip-build # Skip build, just restart
|
||||
python scripts/deploy.py --worker-only # Only deploy worker
|
||||
python scripts/deploy.py --verify-only # Only run verification
|
||||
""",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-build", action="store_true",
|
||||
help="Skip build, just restart services via Coolify API",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--worker-only", action="store_true",
|
||||
help="Only deploy the worker service",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verify-only", action="store_true",
|
||||
help="Only run verification checks (no deployment)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
sys.exit(deploy(environment=args.environment, skip_build=args.skip_build, migrate_only=args.migrate_only))
|
||||
|
||||
if args.verify_only:
|
||||
sys.exit(verify_only())
|
||||
elif args.worker_only:
|
||||
sys.exit(deploy_worker_only(skip_build=args.skip_build))
|
||||
else:
|
||||
sys.exit(deploy_full(skip_build=args.skip_build))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user