leocrm/scripts/test_suite/marathon.py

#!/usr/bin/env python3
"""Test Marathon — Runs all trace scripts and test scripts, combines results.

Usage: python3 scripts/test_suite/marathon.py
"""
from __future__ import annotations

import json
import subprocess
import sys
import time
from datetime import datetime
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent.parent
SUITE_DIR = Path(__file__).resolve().parent

# All trace scripts (static analysis — find verkabelungsfehler)
TRACE_SCRIPTS = [
    ("trace_api_contracts", "Frontend↔Backend API Contracts"),
    ("trace_hooks", "Hook Registrations vs Triggers"),
    ("trace_functions", "Dead Functions (defined but never called)"),
    ("trace_stores", "Unused Store Actions/State"),
    ("trace_contracts", "Contract Attribute Mismatches"),
    ("trace_plugins", "Plugin→Manifest→Frontend Verkabelung"),
    ("trace_imports", "Broken/Missing Imports"),
]

# All test scripts (runtime — find crashes and errors)
# These will be added as they are built
TEST_SCRIPTS = [
    # ("test_all_endpoints", "All 224 API Endpoints"),
    # ("test_all_routes", "All 62 Frontend Routes"),
    # ("test_schema_vs_models", "DB Schema vs Models"),
    # ("test_permission_matrix", "Permission Matrix"),
]


def run_script(name: str, description: str) -> dict:
    """Run a single script and capture results."""
    script_path = SUITE_DIR / f"{name}.py"
    if not script_path.exists():
        return {"name": name, "description": description, "status": "SKIP", "reason": "Script not found", "issues": 0}

    print(f"\n{'─' * 70}")
    print(f"RUNNING: {name} — {description}")
    print(f"{'─' * 70}")

    start = time.time()
    try:
        result = subprocess.run(
            [sys.executable, str(script_path)],
            capture_output=True,
            text=True,
            timeout=300,
            cwd=str(ROOT),
        )
        elapsed = time.time() - start

        # Print stdout
        if result.stdout:
            print(result.stdout)
        if result.stderr:
            print(f"STDERR: {result.stderr[:500]}")

        # Try to read results JSON
        results_file = SUITE_DIR / f"results_{name}.json"
        issues_count = 0
        if results_file.exists():
            try:
                with open(results_file) as f:
                    data = json.load(f)
                    if "summary" in data:
                        issues_count = data["summary"].get("total_issues", 0)
                    elif "issues" in data:
                        issues_count = len(data["issues"])
                    elif "mismatches" in data:
                        issues_count = len(data["mismatches"])
                    elif "broken" in data:
                        issues_count = len(data["broken"])
                    elif "critical_dead" in data:
                        issues_count = len(data["critical_dead"])
                    elif "orphan_registrations" in data:
                        issues_count = len(data.get("orphan_registrations", [])) + len(data.get("orphan_triggers", []))
                    elif "unused" in data:
                        issues_count = sum(len(v) for v in data["unused"].values()) if isinstance(data["unused"], dict) else 0
            except Exception:
                pass

        return {
            "name": name,
            "description": description,
            "status": "PASS" if result.returncode == 0 else "FAIL",
            "exit_code": result.returncode,
            "elapsed": round(elapsed, 2),
            "issues": issues_count,
        }
    except subprocess.TimeoutExpired:
        elapsed = time.time() - start
        return {"name": name, "description": description, "status": "TIMEOUT", "elapsed": round(elapsed, 2), "issues": 0}
    except Exception as e:
        elapsed = time.time() - start
        return {"name": name, "description": description, "status": "ERROR", "error": str(e), "elapsed": round(elapsed, 2), "issues": 0}


def main():
    print("╔" + "═" * 70 + "╗")
    print("║" + " LEOCRM TEST MARATHON — Vollständige System-Audit".center(70) + "║")
    print("║" + f" Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}".center(70) + "║")
    print("╚" + "═" * 70 + "╝")

    all_results = []
    total_start = time.time()

    # Phase 1: Trace Scripts (static analysis)
    print("\n" + "=" * 70)
    print("PHASE 1: TRACE SCRIPTS — Statische Verkabelungs-Analyse")
    print("=" * 70)

    for name, desc in TRACE_SCRIPTS:
        result = run_script(name, desc)
        all_results.append(result)

    # Phase 2: Test Scripts (runtime)
    if TEST_SCRIPTS:
        print("\n" + "=" * 70)
        print("PHASE 2: TEST SCRIPTS — Runtime Tests")
        print("=" * 70)

        for name, desc in TEST_SCRIPTS:
            result = run_script(name, desc)
            all_results.append(result)

    # Summary
    total_elapsed = time.time() - total_start
    print("\n" + "╔" + "═" * 70 + "╗")
    print("║" + " MARATHON SUMMARY".center(70) + "║")
    print("╠" + "═" * 70 + "╣")

    total_issues = 0
    passed = 0
    failed = 0
    skipped = 0

    for r in all_results:
        status_icon = {"PASS": "✅", "FAIL": "❌", "SKIP": "⏭️", "TIMEOUT": "⏰", "ERROR": "💥"}.get(r["status"], "?")
        issues = r.get("issues", 0)
        total_issues += issues
        if r["status"] == "PASS":
            passed += 1
        elif r["status"] in ("FAIL", "ERROR", "TIMEOUT"):
            failed += 1
        else:
            skipped += 1
        elapsed = r.get("elapsed", 0)
        line = f" {status_icon} {r['name']:<30} {issues:>5} issues  {elapsed:>6}s"
        print("║" + line.ljust(70) + "║")

    print("╠" + "═" * 70 + "╣")
    summary_line = f" Total: {len(all_results)} scripts | ✅ {passed} passed | ❌ {failed} failed | ⏭️ {skipped} skipped"
    print("║" + summary_line.ljust(70) + "║")
    issues_line = f" Total issues found: {total_issues}"
    print("║" + issues_line.ljust(70) + "║")
    time_line = f" Total time: {round(total_elapsed, 2)}s"
    print("║" + time_line.ljust(70) + "║")
    print("╚" + "═" * 70 + "╝")

    # Write combined results
    combined_file = SUITE_DIR / "marathon_results.json"
    with open(combined_file, "w") as f:
        json.dump({
            "timestamp": datetime.now().isoformat(),
            "total_time": round(total_elapsed, 2),
            "total_scripts": len(all_results),
            "total_issues": total_issues,
            "passed": passed,
            "failed": failed,
            "skipped": skipped,
            "results": all_results,
        }, f, indent=2)
    print(f"\nCombined results: {combined_file}")

    return total_issues


if __name__ == "__main__":
    sys.exit(main())