Phase 1: SQLite-Persistenz (WAL, Migrationen, Backup, Integritätscheck)
- hms_persistence: Database-Klasse nach §24 - WAL-Modus, Foreign Keys, kurze Transaktionen, Busy-Timeout - Migrationen mit Backup vor Schemaänderung (§24.4) - Integritätscheck beim Öffnen (§24.1) - Projekt-/Einstellungs-/Plugin-Status-CRUD - 18 Unit-Tests inkl. Altdaten-Migration und Reopen-Roundtrips
This commit is contained in:
HMS MediaEngine Agent
@@ -0,0 +1,263 @@
|
||||
"""SQLite-Persistenz mit Migrationen (PLAN.md §24).
|
||||
|
||||
Regeln (§24.1, §24.4):
|
||||
- WAL-Modus, Foreign Keys aktiv, kurze Transaktionen
|
||||
- keine Datenbankoperation im Renderthread (nur Control Core nutzt sie)
|
||||
- automatisches Backup vor Migration
|
||||
- Integritätscheck beim Start nach unsauberem Shutdown
|
||||
- jede Schemaänderung: Vorwärtsmigration, Test mit Altdaten, Backup,
|
||||
dokumentierte Nicht-Rückwärtskompatibilität, neue schema_version
|
||||
|
||||
Projektdateien liegen als JSON-Dateien (§24.2: große Medien als Dateien,
|
||||
SQLite speichert Index, Einstellungen, Pluginstatus, Projektmetadaten).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import sqlite3
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
_MIGRATIONS: dict[int, str] = {
|
||||
1: """
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS projects (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
schema_version INTEGER NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
data TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS plugin_status (
|
||||
plugin_id TEXT PRIMARY KEY,
|
||||
version TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 0,
|
||||
state TEXT NOT NULL DEFAULT 'discovered',
|
||||
package_hash TEXT,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class MigrationResult:
|
||||
"""Ergebnis einer Migration (für TEST_REPORT und Gate-Doku)."""
|
||||
|
||||
from_version: int
|
||||
to_version: int
|
||||
backup_path: Path | None
|
||||
integrity_ok: bool
|
||||
|
||||
|
||||
class Database:
|
||||
"""SQLite-Wrapper für den Control Core (nicht im Renderthread!).
|
||||
|
||||
- open(): öffnet mit WAL, FK, Integritätscheck
|
||||
- migrate(): führt fehlende Migrationen aus, Backup vorher
|
||||
- save_project/load_project/list_projects: Projektmetadaten + JSON-Daten
|
||||
- set_setting/get_setting: App-Einstellungen
|
||||
- upsert_plugin_status/get_plugin_status: Pluginverwaltung (§14.5)
|
||||
"""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
self._path = Path(path)
|
||||
self._conn: sqlite3.Connection | None = None
|
||||
|
||||
@property
|
||||
def connection(self) -> sqlite3.Connection:
|
||||
if self._conn is None:
|
||||
raise RuntimeError("database not opened")
|
||||
return self._conn
|
||||
|
||||
@property
|
||||
def schema_version(self) -> int:
|
||||
row = self.connection.execute(
|
||||
"SELECT value FROM meta WHERE key='schema_version'"
|
||||
).fetchone()
|
||||
return int(row[0]) if row else 0
|
||||
|
||||
def open(self, integrity_check: bool = True) -> None:
|
||||
"""Öffnet die DB: WAL, Foreign Keys, Busy-Timeout, Integritätscheck."""
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._conn = sqlite3.connect(self._path, timeout=5.0)
|
||||
self._conn.execute("PRAGMA journal_mode=WAL") # §24.1
|
||||
self._conn.execute("PRAGMA foreign_keys=ON") # §24.1
|
||||
self._conn.execute("PRAGMA busy_timeout=5000")
|
||||
if integrity_check: # nach unsauberem Shutdown (§24.1)
|
||||
row = self._conn.execute("PRAGMA integrity_check").fetchone()
|
||||
if row and row[0] != "ok":
|
||||
raise sqlite3.DatabaseError(f"integrity check failed: {row[0]}")
|
||||
|
||||
def close(self) -> None:
|
||||
if self._conn is not None:
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
def migrate(self, backup_dir: Path | None = None) -> MigrationResult:
|
||||
"""Führt Migrationen bis SCHEMA_VERSION aus; Backup vorher (§24.4).
|
||||
|
||||
Migrationen sind reine Vorwärtsmigrationen; jede Änderung erhöht
|
||||
schema_version. Altdaten werden beim Backup erhalten.
|
||||
"""
|
||||
conn = self.connection
|
||||
current = 0
|
||||
try:
|
||||
current = self.schema_version
|
||||
except sqlite3.OperationalError:
|
||||
pass # meta-Tabelle existiert noch nicht → Version 0
|
||||
if current >= SCHEMA_VERSION:
|
||||
return MigrationResult(current, current, None, True)
|
||||
|
||||
backup_path: Path | None = None
|
||||
if self._path.exists() and backup_dir is not None:
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
stamp = time.strftime("%Y%m%d-%H%M%S")
|
||||
backup_path = backup_dir / f"{self._path.name}.pre-migration-{stamp}.bak"
|
||||
shutil.copy2(self._path, backup_path) # §24.4: Backup vor Migration
|
||||
|
||||
with conn: # kurze Transaktion je Version (§24.1)
|
||||
for version in range(current + 1, SCHEMA_VERSION + 1):
|
||||
sql = _MIGRATIONS.get(version)
|
||||
if sql is None:
|
||||
raise RuntimeError(f"missing migration for version {version}")
|
||||
conn.executescript(sql)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)",
|
||||
(str(SCHEMA_VERSION),),
|
||||
)
|
||||
row = conn.execute("PRAGMA integrity_check").fetchone()
|
||||
ok = bool(row and row[0] == "ok")
|
||||
return MigrationResult(current, SCHEMA_VERSION, backup_path, ok)
|
||||
|
||||
# ---------- Projekte (§24.2) ----------
|
||||
|
||||
def save_project(self, project: dict) -> None:
|
||||
"""Speichert Projektmetadaten + JSON-Daten transaktionell."""
|
||||
required = ("id", "name", "schema_version", "created_at", "updated_at")
|
||||
for key in required:
|
||||
if key not in project:
|
||||
raise ValueError(f"project missing field {key!r}")
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""INSERT OR REPLACE INTO projects
|
||||
(id, name, schema_version, created_at, updated_at, data)
|
||||
VALUES (?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
project["id"],
|
||||
project["name"],
|
||||
int(project["schema_version"]),
|
||||
project["created_at"],
|
||||
project["updated_at"],
|
||||
json.dumps(project, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
|
||||
def load_project(self, project_id: str) -> dict | None:
|
||||
row = self.connection.execute(
|
||||
"SELECT data FROM projects WHERE id=?", (project_id,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return json.loads(row[0])
|
||||
|
||||
def list_projects(self) -> list[dict]:
|
||||
rows = self.connection.execute(
|
||||
"SELECT id, name, schema_version, updated_at FROM projects"
|
||||
" ORDER BY updated_at DESC"
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"id": r[0],
|
||||
"name": r[1],
|
||||
"schema_version": r[2],
|
||||
"updated_at": r[3],
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
def delete_project(self, project_id: str) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute("DELETE FROM projects WHERE id=?", (project_id,))
|
||||
|
||||
# ---------- Einstellungen ----------
|
||||
|
||||
def set_setting(self, key: str, value: str) -> None:
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"INSERT OR REPLACE INTO settings (key, value, updated_at) VALUES (?, ?, ?)",
|
||||
(key, value, time.strftime("%Y-%m-%dT%H:%M:%S%z")),
|
||||
)
|
||||
|
||||
def get_setting(self, key: str, default: str | None = None) -> str | None:
|
||||
row = self.connection.execute(
|
||||
"SELECT value FROM settings WHERE key=?", (key,)
|
||||
).fetchone()
|
||||
return row[0] if row else default
|
||||
|
||||
# ---------- Plugin-Status (§14.5) ----------
|
||||
|
||||
def upsert_plugin_status(
|
||||
self,
|
||||
plugin_id: str,
|
||||
version: str,
|
||||
enabled: bool,
|
||||
state: str,
|
||||
package_hash: str | None = None,
|
||||
) -> None:
|
||||
if state not in {
|
||||
"discovered",
|
||||
"validated",
|
||||
"installed",
|
||||
"enabled",
|
||||
"compiled",
|
||||
"active",
|
||||
"quarantined",
|
||||
"incompatible",
|
||||
"disabled",
|
||||
}:
|
||||
raise ValueError(f"invalid plugin state {state!r}")
|
||||
with self.connection:
|
||||
self.connection.execute(
|
||||
"""INSERT OR REPLACE INTO plugin_status
|
||||
(plugin_id, version, enabled, state, package_hash, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
plugin_id,
|
||||
version,
|
||||
int(enabled),
|
||||
state,
|
||||
package_hash,
|
||||
time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
),
|
||||
)
|
||||
|
||||
def get_plugin_status(self) -> dict[str, dict]:
|
||||
rows = self.connection.execute(
|
||||
"SELECT plugin_id, version, enabled, state, package_hash, updated_at"
|
||||
" FROM plugin_status"
|
||||
).fetchall()
|
||||
return {
|
||||
r[0]: {
|
||||
"version": r[1],
|
||||
"enabled": bool(r[2]),
|
||||
"state": r[3],
|
||||
"package_hash": r[4],
|
||||
"updated_at": r[5],
|
||||
}
|
||||
for r in rows
|
||||
}
|
||||
@@ -30,6 +30,7 @@ packages = [
|
||||
"packages/adaptive_quality/hms_adaptive",
|
||||
"packages/capabilities/hms_capabilities",
|
||||
"packages/plugin_sdk/hms_plugin_sdk",
|
||||
"packages/persistence/hms_persistence",
|
||||
"apps/renderer/hms_renderer",
|
||||
"apps/control_server/hms_control_server",
|
||||
"apps/launcher/hms_launcher",
|
||||
|
||||
@@ -19,6 +19,7 @@ _PACKAGE_DIRS = [
|
||||
"packages/adaptive_quality",
|
||||
"packages/capabilities",
|
||||
"packages/plugin_sdk",
|
||||
"packages/persistence",
|
||||
"apps/renderer",
|
||||
"apps/control_server",
|
||||
"apps/launcher",
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Unit-Tests SQLite-Persistenz (PLAN.md §24).
|
||||
|
||||
Deckung von §29.1 (Schema- und Projektmigrationen) und §24.1/§24.4:
|
||||
WAL, Foreign Keys, kurze Transaktionen, Backup vor Migration,
|
||||
Integritätscheck, Projekt-Roundtrip, Plugin-Status-Zyklus.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from hms_persistence import SCHEMA_VERSION, Database
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(tmp_path: Path) -> Database:
|
||||
database = Database(tmp_path / "userdata" / "database" / "hms.db")
|
||||
database.open()
|
||||
yield database
|
||||
database.close()
|
||||
|
||||
|
||||
def _project(name: str = "Test Show", version: int = 1) -> dict:
|
||||
return {
|
||||
"id": str(uuid.uuid4()),
|
||||
"name": name,
|
||||
"schema_version": version,
|
||||
"created_at": "2026-09-11T00:00:00+00:00",
|
||||
"updated_at": "2026-09-11T00:00:00+00:00",
|
||||
"compositions": [],
|
||||
}
|
||||
|
||||
|
||||
# ---------- Öffnen & PRAGMA (§24.1) ----------
|
||||
|
||||
|
||||
def test_open_sets_wal_mode(db: Database) -> None:
|
||||
mode = db.connection.execute("PRAGMA journal_mode").fetchone()
|
||||
assert mode and mode[0].lower() == "wal"
|
||||
|
||||
|
||||
def test_open_enables_foreign_keys(db: Database) -> None:
|
||||
fk = db.connection.execute("PRAGMA foreign_keys").fetchone()
|
||||
assert fk and fk[0] == 1
|
||||
|
||||
|
||||
def test_open_creates_parent_directories(tmp_path: Path) -> None:
|
||||
deep = tmp_path / "userdata" / "database"
|
||||
database = Database(deep / "hms.db")
|
||||
database.open()
|
||||
database.close()
|
||||
assert (deep / "hms.db").is_file()
|
||||
|
||||
|
||||
def test_integrity_check_failure_raises(tmp_path: Path) -> None:
|
||||
"""Beschädigte Datei darf nicht still geöffnet werden (§24.1)."""
|
||||
bad = tmp_path / "corrupt.db"
|
||||
bad.write_bytes(b"this is definitely not a sqlite database" * 10)
|
||||
database = Database(bad)
|
||||
with pytest.raises(sqlite3.DatabaseError):
|
||||
database.open()
|
||||
|
||||
|
||||
# ---------- Migrationen (§24.4) ----------
|
||||
|
||||
|
||||
def test_migrate_from_empty_sets_schema_version(db: Database) -> None:
|
||||
result = db.migrate()
|
||||
assert result.from_version == 0
|
||||
assert result.to_version == SCHEMA_VERSION
|
||||
assert result.integrity_ok
|
||||
assert db.schema_version == SCHEMA_VERSION
|
||||
|
||||
|
||||
def test_migrate_is_idempotent(db: Database) -> None:
|
||||
db.migrate()
|
||||
second = db.migrate()
|
||||
assert second.from_version == SCHEMA_VERSION
|
||||
assert second.backup_path is None # kein erneutes Backup ohne Änderung
|
||||
|
||||
|
||||
def test_migrate_creates_backup_of_existing_data(tmp_path: Path) -> None:
|
||||
db_path = tmp_path / "hms.db"
|
||||
database = Database(db_path)
|
||||
database.open()
|
||||
database.migrate()
|
||||
proj = _project("Altdaten")
|
||||
database.save_project(proj)
|
||||
database.close()
|
||||
|
||||
# zweites Öffnen: Migration bereits aktuell → kein Backup nötig
|
||||
database2 = Database(db_path)
|
||||
database2.open(integrity_check=False) # Backup-Szenario simulieren
|
||||
result = database2.migrate(backup_dir=tmp_path / "backups")
|
||||
assert result.backup_path is None # Version identisch
|
||||
database2.close()
|
||||
|
||||
# Backup-Pfad existsiert nur bei echter Migration; hier simulieren wir
|
||||
# eine Vorwärtsmigration über eine Versionserhöhung nicht (SCHEMA_VERSION
|
||||
# ist 1) – stattdessen prüfen wir, dass Altdaten nach Reopen lesbar bleiben.
|
||||
database3 = Database(db_path)
|
||||
database3.open()
|
||||
loaded = database3.load_project(proj["id"])
|
||||
assert loaded is not None and loaded["name"] == "Altdaten" # §24.4: Test mit Altdaten
|
||||
database3.close()
|
||||
|
||||
|
||||
def test_migration_backup_before_change(monkeypatch, tmp_path: Path) -> None:
|
||||
"""Backup wird vor Schemaänderung angelegt, wenn Migration läuft."""
|
||||
import hms_persistence as persistence
|
||||
|
||||
db_path = tmp_path / "hms.db"
|
||||
database = Database(db_path)
|
||||
database.open()
|
||||
# Schema v1 anlegen und befüllen
|
||||
database.migrate()
|
||||
proj = _project("Vor-Backup")
|
||||
database.save_project(proj)
|
||||
# Simulation: eine neue Version existiert → Migration greift
|
||||
monkeypatch.setattr(persistence, "SCHEMA_VERSION", 2)
|
||||
monkeypatch.setitem(
|
||||
persistence._MIGRATIONS,
|
||||
2,
|
||||
"CREATE TABLE IF NOT EXISTS future_table (id TEXT PRIMARY KEY);",
|
||||
)
|
||||
backups = tmp_path / "backups"
|
||||
result = database.migrate(backup_dir=backups)
|
||||
assert result.backup_path is not None and result.backup_path.is_file()
|
||||
assert result.to_version == 2
|
||||
# Altdaten weiterhin vorhanden (Vorwärtsmigration verliert nichts)
|
||||
loaded = database.load_project(proj["id"])
|
||||
assert loaded is not None and loaded["name"] == "Vor-Backup"
|
||||
database.close()
|
||||
|
||||
|
||||
# ---------- Projekte (§24.2) ----------
|
||||
|
||||
|
||||
def test_project_save_load_roundtrip(db: Database) -> None:
|
||||
db.migrate()
|
||||
proj = _project("Meine Show")
|
||||
proj["compositions"] = [{"id": str(uuid.uuid4()), "layers": []}]
|
||||
db.save_project(proj)
|
||||
loaded = db.load_project(proj["id"])
|
||||
assert loaded == proj
|
||||
|
||||
|
||||
def test_project_list_sorted_by_update(db: Database) -> None:
|
||||
db.migrate()
|
||||
older = _project("Alt")
|
||||
newer = _project("Neu")
|
||||
newer["updated_at"] = "2026-09-11T12:00:00+00:00"
|
||||
db.save_project(older)
|
||||
db.save_project(newer)
|
||||
names = [p["name"] for p in db.list_projects()]
|
||||
assert names[0] == "Neu" # jüngstes zuerst
|
||||
|
||||
|
||||
def test_project_delete(db: Database) -> None:
|
||||
db.migrate()
|
||||
proj = _project()
|
||||
db.save_project(proj)
|
||||
db.delete_project(proj["id"])
|
||||
assert db.load_project(proj["id"]) is None
|
||||
|
||||
|
||||
def test_project_save_validates_required_fields(db: Database) -> None:
|
||||
db.migrate()
|
||||
with pytest.raises(ValueError, match="missing field"):
|
||||
db.save_project({"id": "x"})
|
||||
|
||||
|
||||
def test_project_save_overwrites_same_id(db: Database) -> None:
|
||||
db.migrate()
|
||||
proj = _project("Version A")
|
||||
db.save_project(proj)
|
||||
proj["name"] = "Version B"
|
||||
proj["updated_at"] = time.strftime("%Y-%m-%dT%H:%M:%S%z")
|
||||
db.save_project(proj)
|
||||
loaded = db.load_project(proj["id"])
|
||||
assert loaded is not None and loaded["name"] == "Version B"
|
||||
assert len(db.list_projects()) == 1
|
||||
|
||||
|
||||
# ---------- Einstellungen ----------
|
||||
|
||||
|
||||
def test_settings_roundtrip_and_default(db: Database) -> None:
|
||||
db.migrate()
|
||||
assert db.get_setting("artnet.bind") is None
|
||||
assert db.get_setting("artnet.bind", "0.0.0.0") == "0.0.0.0"
|
||||
db.set_setting("artnet.bind", "127.0.0.1")
|
||||
db.set_setting("artnet.bind", "10.0.0.5") # Überschreiben
|
||||
assert db.get_setting("artnet.bind") == "10.0.0.5"
|
||||
|
||||
|
||||
# ---------- Plugin-Status (§14.5) ----------
|
||||
|
||||
|
||||
def test_plugin_status_lifecycle(db: Database) -> None:
|
||||
db.migrate()
|
||||
db.upsert_plugin_status(
|
||||
"com.hms.fx.example_passthrough", "1.0.0", enabled=False, state="discovered"
|
||||
)
|
||||
db.upsert_plugin_status(
|
||||
"com.hms.fx.example_passthrough", "1.0.0", enabled=True, state="active"
|
||||
)
|
||||
status = db.get_plugin_status()
|
||||
entry = status["com.hms.fx.example_passthrough"]
|
||||
assert entry["enabled"] is True
|
||||
assert entry["state"] == "active"
|
||||
|
||||
|
||||
def test_plugin_status_rejects_invalid_state(db: Database) -> None:
|
||||
db.migrate()
|
||||
with pytest.raises(ValueError, match="invalid plugin state"):
|
||||
db.upsert_plugin_status("com.x.y", "1.0.0", enabled=True, state="exploded")
|
||||
|
||||
|
||||
def test_plugin_status_persists_across_reopen(tmp_path: Path) -> None:
|
||||
db_path = tmp_path / "hms.db"
|
||||
database = Database(db_path)
|
||||
database.open()
|
||||
database.migrate()
|
||||
database.upsert_plugin_status(
|
||||
"com.hms.fx.gaussian_blur", "1.0.0", enabled=True, state="compiled",
|
||||
package_hash="deadbeef",
|
||||
)
|
||||
database.close()
|
||||
|
||||
database2 = Database(db_path)
|
||||
database2.open()
|
||||
entry = database2.get_plugin_status()["com.hms.fx.gaussian_blur"]
|
||||
assert entry["package_hash"] == "deadbeef"
|
||||
assert entry["state"] == "compiled"
|
||||
database2.close()
|
||||
|
||||
|
||||
# ---------- Verbindungsschutz ----------
|
||||
|
||||
|
||||
def test_connection_property_requires_open(tmp_path: Path) -> None:
|
||||
database = Database(tmp_path / "hms.db")
|
||||
with pytest.raises(RuntimeError, match="not opened"):
|
||||
_ = database.connection
|
||||
Reference in New Issue
Block a user