From de7d1a840f37bb8cc8b19e5b2b1bc45afd82bf19 Mon Sep 17 00:00:00 2001 From: HMS MediaEngine Agent Date: Fri, 11 Sep 2026 00:52:27 +0200 Subject: [PATCH] =?UTF-8?q?Phase=201:=20SQLite-Persistenz=20(WAL,=20Migrat?= =?UTF-8?q?ionen,=20Backup,=20Integrit=C3=A4tscheck)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hms_persistence: Database-Klasse nach §24 - WAL-Modus, Foreign Keys, kurze Transaktionen, Busy-Timeout - Migrationen mit Backup vor Schemaänderung (§24.4) - Integritätscheck beim Öffnen (§24.1) - Projekt-/Einstellungs-/Plugin-Status-CRUD - 18 Unit-Tests inkl. Altdaten-Migration und Reopen-Roundtrips --- .../persistence/hms_persistence/__init__.py | 263 ++++++++++++++++++ pyproject.toml | 1 + tests/conftest.py | 1 + tests/unit/test_persistence.py | 249 +++++++++++++++++ 4 files changed, 514 insertions(+) create mode 100644 packages/persistence/hms_persistence/__init__.py create mode 100644 tests/unit/test_persistence.py diff --git a/packages/persistence/hms_persistence/__init__.py b/packages/persistence/hms_persistence/__init__.py new file mode 100644 index 0000000..6ec33c9 --- /dev/null +++ b/packages/persistence/hms_persistence/__init__.py @@ -0,0 +1,263 @@ +"""SQLite-Persistenz mit Migrationen (PLAN.md §24). + +Regeln (§24.1, §24.4): +- WAL-Modus, Foreign Keys aktiv, kurze Transaktionen +- keine Datenbankoperation im Renderthread (nur Control Core nutzt sie) +- automatisches Backup vor Migration +- Integritätscheck beim Start nach unsauberem Shutdown +- jede Schemaänderung: Vorwärtsmigration, Test mit Altdaten, Backup, + dokumentierte Nicht-Rückwärtskompatibilität, neue schema_version + +Projektdateien liegen als JSON-Dateien (§24.2: große Medien als Dateien, +SQLite speichert Index, Einstellungen, Pluginstatus, Projektmetadaten). +""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +import time +from dataclasses import dataclass +from pathlib import Path + +SCHEMA_VERSION = 1 + +_MIGRATIONS: dict[int, str] = { + 1: """ + CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS projects ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + schema_version INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + data TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS plugin_status ( + plugin_id TEXT PRIMARY KEY, + version TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 0, + state TEXT NOT NULL DEFAULT 'discovered', + package_hash TEXT, + updated_at TEXT NOT NULL + ); + """, +} + + +@dataclass +class MigrationResult: + """Ergebnis einer Migration (für TEST_REPORT und Gate-Doku).""" + + from_version: int + to_version: int + backup_path: Path | None + integrity_ok: bool + + +class Database: + """SQLite-Wrapper für den Control Core (nicht im Renderthread!). + + - open(): öffnet mit WAL, FK, Integritätscheck + - migrate(): führt fehlende Migrationen aus, Backup vorher + - save_project/load_project/list_projects: Projektmetadaten + JSON-Daten + - set_setting/get_setting: App-Einstellungen + - upsert_plugin_status/get_plugin_status: Pluginverwaltung (§14.5) + """ + + def __init__(self, path: Path) -> None: + self._path = Path(path) + self._conn: sqlite3.Connection | None = None + + @property + def connection(self) -> sqlite3.Connection: + if self._conn is None: + raise RuntimeError("database not opened") + return self._conn + + @property + def schema_version(self) -> int: + row = self.connection.execute( + "SELECT value FROM meta WHERE key='schema_version'" + ).fetchone() + return int(row[0]) if row else 0 + + def open(self, integrity_check: bool = True) -> None: + """Öffnet die DB: WAL, Foreign Keys, Busy-Timeout, Integritätscheck.""" + self._path.parent.mkdir(parents=True, exist_ok=True) + self._conn = sqlite3.connect(self._path, timeout=5.0) + self._conn.execute("PRAGMA journal_mode=WAL") # §24.1 + self._conn.execute("PRAGMA foreign_keys=ON") # §24.1 + self._conn.execute("PRAGMA busy_timeout=5000") + if integrity_check: # nach unsauberem Shutdown (§24.1) + row = self._conn.execute("PRAGMA integrity_check").fetchone() + if row and row[0] != "ok": + raise sqlite3.DatabaseError(f"integrity check failed: {row[0]}") + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None + + def migrate(self, backup_dir: Path | None = None) -> MigrationResult: + """Führt Migrationen bis SCHEMA_VERSION aus; Backup vorher (§24.4). + + Migrationen sind reine Vorwärtsmigrationen; jede Änderung erhöht + schema_version. Altdaten werden beim Backup erhalten. + """ + conn = self.connection + current = 0 + try: + current = self.schema_version + except sqlite3.OperationalError: + pass # meta-Tabelle existiert noch nicht → Version 0 + if current >= SCHEMA_VERSION: + return MigrationResult(current, current, None, True) + + backup_path: Path | None = None + if self._path.exists() and backup_dir is not None: + backup_dir.mkdir(parents=True, exist_ok=True) + stamp = time.strftime("%Y%m%d-%H%M%S") + backup_path = backup_dir / f"{self._path.name}.pre-migration-{stamp}.bak" + shutil.copy2(self._path, backup_path) # §24.4: Backup vor Migration + + with conn: # kurze Transaktion je Version (§24.1) + for version in range(current + 1, SCHEMA_VERSION + 1): + sql = _MIGRATIONS.get(version) + if sql is None: + raise RuntimeError(f"missing migration for version {version}") + conn.executescript(sql) + conn.execute( + "INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)", + (str(SCHEMA_VERSION),), + ) + row = conn.execute("PRAGMA integrity_check").fetchone() + ok = bool(row and row[0] == "ok") + return MigrationResult(current, SCHEMA_VERSION, backup_path, ok) + + # ---------- Projekte (§24.2) ---------- + + def save_project(self, project: dict) -> None: + """Speichert Projektmetadaten + JSON-Daten transaktionell.""" + required = ("id", "name", "schema_version", "created_at", "updated_at") + for key in required: + if key not in project: + raise ValueError(f"project missing field {key!r}") + with self.connection: + self.connection.execute( + """INSERT OR REPLACE INTO projects + (id, name, schema_version, created_at, updated_at, data) + VALUES (?, ?, ?, ?, ?, ?)""", + ( + project["id"], + project["name"], + int(project["schema_version"]), + project["created_at"], + project["updated_at"], + json.dumps(project, ensure_ascii=False), + ), + ) + + def load_project(self, project_id: str) -> dict | None: + row = self.connection.execute( + "SELECT data FROM projects WHERE id=?", (project_id,) + ).fetchone() + if row is None: + return None + return json.loads(row[0]) + + def list_projects(self) -> list[dict]: + rows = self.connection.execute( + "SELECT id, name, schema_version, updated_at FROM projects" + " ORDER BY updated_at DESC" + ).fetchall() + return [ + { + "id": r[0], + "name": r[1], + "schema_version": r[2], + "updated_at": r[3], + } + for r in rows + ] + + def delete_project(self, project_id: str) -> None: + with self.connection: + self.connection.execute("DELETE FROM projects WHERE id=?", (project_id,)) + + # ---------- Einstellungen ---------- + + def set_setting(self, key: str, value: str) -> None: + with self.connection: + self.connection.execute( + "INSERT OR REPLACE INTO settings (key, value, updated_at) VALUES (?, ?, ?)", + (key, value, time.strftime("%Y-%m-%dT%H:%M:%S%z")), + ) + + def get_setting(self, key: str, default: str | None = None) -> str | None: + row = self.connection.execute( + "SELECT value FROM settings WHERE key=?", (key,) + ).fetchone() + return row[0] if row else default + + # ---------- Plugin-Status (§14.5) ---------- + + def upsert_plugin_status( + self, + plugin_id: str, + version: str, + enabled: bool, + state: str, + package_hash: str | None = None, + ) -> None: + if state not in { + "discovered", + "validated", + "installed", + "enabled", + "compiled", + "active", + "quarantined", + "incompatible", + "disabled", + }: + raise ValueError(f"invalid plugin state {state!r}") + with self.connection: + self.connection.execute( + """INSERT OR REPLACE INTO plugin_status + (plugin_id, version, enabled, state, package_hash, updated_at) + VALUES (?, ?, ?, ?, ?, ?)""", + ( + plugin_id, + version, + int(enabled), + state, + package_hash, + time.strftime("%Y-%m-%dT%H:%M:%S%z"), + ), + ) + + def get_plugin_status(self) -> dict[str, dict]: + rows = self.connection.execute( + "SELECT plugin_id, version, enabled, state, package_hash, updated_at" + " FROM plugin_status" + ).fetchall() + return { + r[0]: { + "version": r[1], + "enabled": bool(r[2]), + "state": r[3], + "package_hash": r[4], + "updated_at": r[5], + } + for r in rows + } diff --git a/pyproject.toml b/pyproject.toml index 5674893..f879e5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ packages = [ "packages/adaptive_quality/hms_adaptive", "packages/capabilities/hms_capabilities", "packages/plugin_sdk/hms_plugin_sdk", + "packages/persistence/hms_persistence", "apps/renderer/hms_renderer", "apps/control_server/hms_control_server", "apps/launcher/hms_launcher", diff --git a/tests/conftest.py b/tests/conftest.py index 3056f2e..ab4d985 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,7 @@ _PACKAGE_DIRS = [ "packages/adaptive_quality", "packages/capabilities", "packages/plugin_sdk", + "packages/persistence", "apps/renderer", "apps/control_server", "apps/launcher", diff --git a/tests/unit/test_persistence.py b/tests/unit/test_persistence.py new file mode 100644 index 0000000..04903ba --- /dev/null +++ b/tests/unit/test_persistence.py @@ -0,0 +1,249 @@ +"""Unit-Tests SQLite-Persistenz (PLAN.md §24). + +Deckung von §29.1 (Schema- und Projektmigrationen) und §24.1/§24.4: +WAL, Foreign Keys, kurze Transaktionen, Backup vor Migration, +Integritätscheck, Projekt-Roundtrip, Plugin-Status-Zyklus. +""" + +from __future__ import annotations + +import sqlite3 +import time +import uuid +from pathlib import Path + +import pytest +from hms_persistence import SCHEMA_VERSION, Database + + +@pytest.fixture() +def db(tmp_path: Path) -> Database: + database = Database(tmp_path / "userdata" / "database" / "hms.db") + database.open() + yield database + database.close() + + +def _project(name: str = "Test Show", version: int = 1) -> dict: + return { + "id": str(uuid.uuid4()), + "name": name, + "schema_version": version, + "created_at": "2026-09-11T00:00:00+00:00", + "updated_at": "2026-09-11T00:00:00+00:00", + "compositions": [], + } + + +# ---------- Öffnen & PRAGMA (§24.1) ---------- + + +def test_open_sets_wal_mode(db: Database) -> None: + mode = db.connection.execute("PRAGMA journal_mode").fetchone() + assert mode and mode[0].lower() == "wal" + + +def test_open_enables_foreign_keys(db: Database) -> None: + fk = db.connection.execute("PRAGMA foreign_keys").fetchone() + assert fk and fk[0] == 1 + + +def test_open_creates_parent_directories(tmp_path: Path) -> None: + deep = tmp_path / "userdata" / "database" + database = Database(deep / "hms.db") + database.open() + database.close() + assert (deep / "hms.db").is_file() + + +def test_integrity_check_failure_raises(tmp_path: Path) -> None: + """Beschädigte Datei darf nicht still geöffnet werden (§24.1).""" + bad = tmp_path / "corrupt.db" + bad.write_bytes(b"this is definitely not a sqlite database" * 10) + database = Database(bad) + with pytest.raises(sqlite3.DatabaseError): + database.open() + + +# ---------- Migrationen (§24.4) ---------- + + +def test_migrate_from_empty_sets_schema_version(db: Database) -> None: + result = db.migrate() + assert result.from_version == 0 + assert result.to_version == SCHEMA_VERSION + assert result.integrity_ok + assert db.schema_version == SCHEMA_VERSION + + +def test_migrate_is_idempotent(db: Database) -> None: + db.migrate() + second = db.migrate() + assert second.from_version == SCHEMA_VERSION + assert second.backup_path is None # kein erneutes Backup ohne Änderung + + +def test_migrate_creates_backup_of_existing_data(tmp_path: Path) -> None: + db_path = tmp_path / "hms.db" + database = Database(db_path) + database.open() + database.migrate() + proj = _project("Altdaten") + database.save_project(proj) + database.close() + + # zweites Öffnen: Migration bereits aktuell → kein Backup nötig + database2 = Database(db_path) + database2.open(integrity_check=False) # Backup-Szenario simulieren + result = database2.migrate(backup_dir=tmp_path / "backups") + assert result.backup_path is None # Version identisch + database2.close() + + # Backup-Pfad existsiert nur bei echter Migration; hier simulieren wir + # eine Vorwärtsmigration über eine Versionserhöhung nicht (SCHEMA_VERSION + # ist 1) – stattdessen prüfen wir, dass Altdaten nach Reopen lesbar bleiben. + database3 = Database(db_path) + database3.open() + loaded = database3.load_project(proj["id"]) + assert loaded is not None and loaded["name"] == "Altdaten" # §24.4: Test mit Altdaten + database3.close() + + +def test_migration_backup_before_change(monkeypatch, tmp_path: Path) -> None: + """Backup wird vor Schemaänderung angelegt, wenn Migration läuft.""" + import hms_persistence as persistence + + db_path = tmp_path / "hms.db" + database = Database(db_path) + database.open() + # Schema v1 anlegen und befüllen + database.migrate() + proj = _project("Vor-Backup") + database.save_project(proj) + # Simulation: eine neue Version existiert → Migration greift + monkeypatch.setattr(persistence, "SCHEMA_VERSION", 2) + monkeypatch.setitem( + persistence._MIGRATIONS, + 2, + "CREATE TABLE IF NOT EXISTS future_table (id TEXT PRIMARY KEY);", + ) + backups = tmp_path / "backups" + result = database.migrate(backup_dir=backups) + assert result.backup_path is not None and result.backup_path.is_file() + assert result.to_version == 2 + # Altdaten weiterhin vorhanden (Vorwärtsmigration verliert nichts) + loaded = database.load_project(proj["id"]) + assert loaded is not None and loaded["name"] == "Vor-Backup" + database.close() + + +# ---------- Projekte (§24.2) ---------- + + +def test_project_save_load_roundtrip(db: Database) -> None: + db.migrate() + proj = _project("Meine Show") + proj["compositions"] = [{"id": str(uuid.uuid4()), "layers": []}] + db.save_project(proj) + loaded = db.load_project(proj["id"]) + assert loaded == proj + + +def test_project_list_sorted_by_update(db: Database) -> None: + db.migrate() + older = _project("Alt") + newer = _project("Neu") + newer["updated_at"] = "2026-09-11T12:00:00+00:00" + db.save_project(older) + db.save_project(newer) + names = [p["name"] for p in db.list_projects()] + assert names[0] == "Neu" # jüngstes zuerst + + +def test_project_delete(db: Database) -> None: + db.migrate() + proj = _project() + db.save_project(proj) + db.delete_project(proj["id"]) + assert db.load_project(proj["id"]) is None + + +def test_project_save_validates_required_fields(db: Database) -> None: + db.migrate() + with pytest.raises(ValueError, match="missing field"): + db.save_project({"id": "x"}) + + +def test_project_save_overwrites_same_id(db: Database) -> None: + db.migrate() + proj = _project("Version A") + db.save_project(proj) + proj["name"] = "Version B" + proj["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%S%z") + db.save_project(proj) + loaded = db.load_project(proj["id"]) + assert loaded is not None and loaded["name"] == "Version B" + assert len(db.list_projects()) == 1 + + +# ---------- Einstellungen ---------- + + +def test_settings_roundtrip_and_default(db: Database) -> None: + db.migrate() + assert db.get_setting("artnet.bind") is None + assert db.get_setting("artnet.bind", "0.0.0.0") == "0.0.0.0" + db.set_setting("artnet.bind", "127.0.0.1") + db.set_setting("artnet.bind", "10.0.0.5") # Überschreiben + assert db.get_setting("artnet.bind") == "10.0.0.5" + + +# ---------- Plugin-Status (§14.5) ---------- + + +def test_plugin_status_lifecycle(db: Database) -> None: + db.migrate() + db.upsert_plugin_status( + "com.hms.fx.example_passthrough", "1.0.0", enabled=False, state="discovered" + ) + db.upsert_plugin_status( + "com.hms.fx.example_passthrough", "1.0.0", enabled=True, state="active" + ) + status = db.get_plugin_status() + entry = status["com.hms.fx.example_passthrough"] + assert entry["enabled"] is True + assert entry["state"] == "active" + + +def test_plugin_status_rejects_invalid_state(db: Database) -> None: + db.migrate() + with pytest.raises(ValueError, match="invalid plugin state"): + db.upsert_plugin_status("com.x.y", "1.0.0", enabled=True, state="exploded") + + +def test_plugin_status_persists_across_reopen(tmp_path: Path) -> None: + db_path = tmp_path / "hms.db" + database = Database(db_path) + database.open() + database.migrate() + database.upsert_plugin_status( + "com.hms.fx.gaussian_blur", "1.0.0", enabled=True, state="compiled", + package_hash="deadbeef", + ) + database.close() + + database2 = Database(db_path) + database2.open() + entry = database2.get_plugin_status()["com.hms.fx.gaussian_blur"] + assert entry["package_hash"] == "deadbeef" + assert entry["state"] == "compiled" + database2.close() + + +# ---------- Verbindungsschutz ---------- + + +def test_connection_property_requires_open(tmp_path: Path) -> None: + database = Database(tmp_path / "hms.db") + with pytest.raises(RuntimeError, match="not opened"): + _ = database.connection