hms-mediaengine/apps/launcher/hms_launcher/supervisor.py

"""Supervisor/Launcher (PLAN.md §6.1A, §26.2, §26.4).

Aufgaben (§6.1A):
- Start und Überwachung der Teilprozesse (Control Core, Renderer)
- Wahl freier lokaler Ports (127.0.0.1)
- Setzen portabler Runtime-Pfade (AppPaths.portable_environment)
- Heartbeat-Überwachung auf Prozessebene (Lebenszyklus)
- kontrolliertes Beenden mit Timeout (§26.4)
- Wiederanlauf nach Control-Core-Absturz, Crashloop-Erkennung (§26.2)
- Recovery-Markierung bei Zwangsbeendigung (§26.4)

Der Supervisor verwaltet echte Prozesse; keine Mock-Implementierung.
"""

from __future__ import annotations

import os
import signal
import socket
import subprocess
import time
from dataclasses import dataclass, field
from pathlib import Path

from hms_launcher.paths import AppPaths

DEFAULT_STOP_TIMEOUT_S = 10.0
DEFAULT_MAX_RESTARTS = 5
DEFAULT_RESTART_WINDOW_S = 60.0


@dataclass(frozen=True)
class ProcessSpec:
    """Beschreibung eines zu überwachenden Teilprozesses."""

    name: str
    cmd: list[str]
    restartable: bool = True
    max_restarts: int = DEFAULT_MAX_RESTARTS
    restart_window_s: float = DEFAULT_RESTART_WINDOW_S
    stop_timeout_s: float = DEFAULT_STOP_TIMEOUT_S


@dataclass
class ProcessState:
    """Laufzeitinformation zu einem überwachten Prozess (§26.2)."""

    spec: ProcessSpec
    proc: subprocess.Popen | None = None
    restarts: list[float] = field(default_factory=list)  # Zeitstempel je Neustart
    last_start_ns: int = 0
    crashlooped: bool = False
    stopped_by_supervisor: bool = False

    @property
    def running(self) -> bool:
        return self.proc is not None and self.proc.poll() is None

    @property
    def returncode(self) -> int | None:
        return self.proc.poll() if self.proc is not None else None


def find_free_port(host: str = "127.0.0.1") -> int:
    """Wählt einen freien lokalen Port (§6.1A). Socket wird sofort wieder
    freigegeben; der Kindprozess bindet ihn anschließend selbst."""
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
        sock.bind((host, 0))
        return sock.getsockname()[1]


class Supervisor:
    """Überwacht Teilprozesse mit Restart-Policy und Crashloop-Schwelle."""

    def __init__(self, paths: AppPaths | None = None) -> None:
        self._paths = paths
        self._states: dict[str, ProcessState] = {}
        self._recovery_marker: Path | None = None
        if paths is not None:
            self._recovery_marker = paths.userdata / "recovery" / "unclean_shutdown"

    # ---------- Start/Stop ----------

    def start(self, spec: ProcessSpec) -> None:
        """Startet einen Teilprozess mit portabler Umgebung."""
        if spec.name in self._states and self._states[spec.name].running:
            raise RuntimeError(f"process {spec.name!r} already running")
        env = dict(os.environ)
        if self._paths is not None:
            env.update(self._paths.portable_environment())
        proc = subprocess.Popen(
            spec.cmd,
            env=env,
            cwd=str(self._paths.root) if self._paths is not None else None,
        )
        state = self._states.get(spec.name)
        if state is None:
            state = ProcessState(spec=spec)
            self._states[spec.name] = state
        else:
            state.spec = spec
        state.proc = proc
        state.last_start_ns = time.monotonic_ns()
        state.stopped_by_supervisor = False

    def stop(self, name: str, timeout: float | None = None) -> int | None:
        """Kontrolliertes Beenden: terminate → warten → kill (§26.4).

        Gibt den Rückgabecode zurück; None falls der Prozess nicht lief.
        """
        state = self._states.get(name)
        if state is None or state.proc is None:
            return None
        if not state.running:
            return state.returncode
        state.stopped_by_supervisor = True
        timeout = timeout if timeout is not None else state.spec.stop_timeout_s
        state.proc.terminate()  # SIGTERM: laufende Writes abschließen (§26.4)
        try:
            return state.proc.wait(timeout=timeout)
        except subprocess.TimeoutExpired:
            # Zwangsbeendigung: Recovery-Markierung setzen (§26.4)
            state.proc.kill()
            self._write_recovery_marker(name)
            return state.proc.wait(timeout=5)

    def shutdown(self) -> None:
        """Beendet alle Prozesse kontrolliert (Renderer zuletzt, um Output
        so lange wie möglich zu halten; §26.2)."""
        for name in reversed(list(self._states)):
            self.stop(name)

    # ---------- Überwachung (§26.2) ----------

    def check(self) -> dict[str, str]:
        """Prüft alle Prozesse; startet Abgestürzte gemäß Policy neu.

        Rückgabe: name → Zustand (running/restarted/crashloop/stopped).
        """
        result: dict[str, str] = {}
        now = time.monotonic()
        for name, state in list(self._states.items()):
            if state.running:
                result[name] = "running"
                continue
            if state.stopped_by_supervisor:
                result[name] = "stopped"
                continue
            if not state.spec.restartable or state.crashlooped:
                result[name] = "crashloop" if state.crashlooped else "stopped"
                continue
            # Neustarts innerhalb des Zeitfensters zählen (Crashloop, §26.2)
            state.restarts = [
                t for t in state.restarts if now - t < state.spec.restart_window_s
            ]
            if len(state.restarts) >= state.spec.max_restarts:
                state.crashlooped = True  # endlose Neustarts verhindern
                result[name] = "crashloop"
                continue
            state.restarts.append(now)
            self.start(state.spec)
            result[name] = "restarted"
        return result

    def state(self, name: str) -> ProcessState:
        return self._states[name]

    def names(self) -> list[str]:
        return list(self._states)

    # ---------- Recovery (§26.4) ----------

    def _write_recovery_marker(self, name: str) -> None:
        if self._recovery_marker is None:
            return
        self._recovery_marker.parent.mkdir(parents=True, exist_ok=True)
        with open(self._recovery_marker, "a", encoding="utf-8") as fh:
            fh.write(
                f"{time.strftime('%Y-%m-%dT%H:%M:%S%z')} forced-kill {name}\n"
            )

    def consume_recovery_marker(self) -> list[str]:
        """Liest und löscht die Recovery-Markierung (Crash-Recovery-Dialog,
        §24.3/§26.4). Gibt die Zeilen zurück."""
        if self._recovery_marker is None or not self._recovery_marker.is_file():
            return []
        lines = self._recovery_marker.read_text(encoding="utf-8").splitlines()
        self._recovery_marker.unlink()
        return lines


# Windows-kompatibles SIGTERM: terminate() nutzt auf Windows TerminateProcess,
# das kein SIGTERM ist. Für sauberes Shutdown nutzen Kindprozesse dort einen
# Steuerkanal (IPC-command) – der Supervisor sendet SIGTERM nur auf POSIX.


def request_graceful_stop(proc: subprocess.Popen, timeout: float) -> int | None:
    """POSIX: SIGTERM; Windows: proc.terminate(). Wartet dann kontrolliert."""
    if os.name == "posix":
        proc.send_signal(signal.SIGTERM)
    else:
        proc.terminate()
    try:
        return proc.wait(timeout=timeout)
    except subprocess.TimeoutExpired:
        return None