210 lines
6.8 KiB
Python
210 lines
6.8 KiB
Python
|
"""ContentManifest (PLAN.md §10.1, §6.4).
|
|||
|
|
|
|||
|
|
Content Sync (§6.4): SHA-256-Manifest, resumierbare Chunks, Hashprüfung,
|
|||
|
|
Staging – Ziel sind byte-identische freigegebene Inhalte auf allen Nodes.
|
|||
|
|
|
|||
|
|
Manifest-Einträge decken Medien, Plugins und Projektdateien ab; die Art
|
|||
|
|
wird je Eintrag vermerkt, damit Preflight Plugin-API- und
|
|||
|
|
Backend-Anforderungen prüfen kann (§10.1).
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import hashlib
|
|||
|
|
import os
|
|||
|
|
import uuid
|
|||
|
|
from dataclasses import dataclass, field
|
|||
|
|
from pathlib import Path, PurePosixPath
|
|||
|
|
|
|||
|
|
CHUNK_SIZE = 4 * 1024 * 1024 # 4 MiB Standard-Chunk für Resume (§6.4)
|
|||
|
|
|
|||
|
|
KIND_MEDIA = "media"
|
|||
|
|
KIND_PLUGIN = "plugin"
|
|||
|
|
KIND_PROJECT = "project"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def hash_file(path: Path | str, chunk_size: int = CHUNK_SIZE) -> tuple[str, list[str]]:
|
|||
|
|
"""SHA-256 über die ganze Datei + je Chunk (resumierbar, §6.4).
|
|||
|
|
|
|||
|
|
Rückgabe: (file_hash_hex, [chunk_hash_hex, ...] in Dateireihenfolge).
|
|||
|
|
"""
|
|||
|
|
if chunk_size <= 0:
|
|||
|
|
raise ValueError("chunk_size muss positiv sein")
|
|||
|
|
file_hash = hashlib.sha256()
|
|||
|
|
chunk_hashes: list[str] = []
|
|||
|
|
with open(path, "rb") as fh:
|
|||
|
|
while True:
|
|||
|
|
data = fh.read(chunk_size)
|
|||
|
|
if not data:
|
|||
|
|
break
|
|||
|
|
chunk_hashes.append(hashlib.sha256(data).hexdigest())
|
|||
|
|
file_hash.update(data)
|
|||
|
|
if not chunk_hashes: # leere Datei: ein leerer Chunk-Hash
|
|||
|
|
chunk_hashes.append(hashlib.sha256(b"").hexdigest())
|
|||
|
|
return file_hash.hexdigest(), chunk_hashes
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass(frozen=True)
|
|||
|
|
class ManifestEntry:
|
|||
|
|
"""Eine Datei im Manifest (§10.1): Pfad, Größe, Hash, Art."""
|
|||
|
|
|
|||
|
|
rel_path: str # portabel, POSIX-relativ
|
|||
|
|
kind: str # media | plugin | project
|
|||
|
|
size_bytes: int
|
|||
|
|
sha256: str
|
|||
|
|
chunk_hashes: tuple[str, ...] = ()
|
|||
|
|
|
|||
|
|
def to_dict(self) -> dict:
|
|||
|
|
return {
|
|||
|
|
"rel_path": self.rel_path,
|
|||
|
|
"kind": self.kind,
|
|||
|
|
"size_bytes": self.size_bytes,
|
|||
|
|
"sha256": self.sha256,
|
|||
|
|
"chunk_hashes": list(self.chunk_hashes),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@classmethod
|
|||
|
|
def from_dict(cls, data: dict) -> ManifestEntry:
|
|||
|
|
return cls(
|
|||
|
|
rel_path=data["rel_path"],
|
|||
|
|
kind=data["kind"],
|
|||
|
|
size_bytes=int(data["size_bytes"]),
|
|||
|
|
sha256=data["sha256"],
|
|||
|
|
chunk_hashes=tuple(data.get("chunk_hashes", ())),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@dataclass(frozen=True)
|
|||
|
|
class ManifestDiff:
|
|||
|
|
"""Unterschied zweier Manifeste (Basis für Sync-Transfer, §6.4)."""
|
|||
|
|
|
|||
|
|
added: frozenset[str] = field(default_factory=frozenset)
|
|||
|
|
removed: frozenset[str] = field(default_factory=frozenset)
|
|||
|
|
changed: frozenset[str] = field(default_factory=frozenset)
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def empty(self) -> bool:
|
|||
|
|
return not (self.added or self.removed or self.changed)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ContentManifest:
|
|||
|
|
"""Versioniertes Inhalts-Manifest (§10.1 ContentManifest).
|
|||
|
|
|
|||
|
|
- manifest_id: Instanz-ID (UUID)
|
|||
|
|
- revision: monotone Manifest-Revision (neu bei jeder Änderung)
|
|||
|
|
- entries: rel_path → ManifestEntry
|
|||
|
|
|
|||
|
|
Der Manifest-Inhalt selbst ist deterministisch (sortierte Einträge);
|
|||
|
|
die manifest_id identifiziert den Ausgabezeitpunkt.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
manifest_id: str | None = None,
|
|||
|
|
revision: int = 0,
|
|||
|
|
entries: dict[str, ManifestEntry] | None = None,
|
|||
|
|
) -> None:
|
|||
|
|
self.manifest_id = manifest_id or str(uuid.uuid4())
|
|||
|
|
self.revision = revision
|
|||
|
|
self._entries: dict[str, ManifestEntry] = dict(entries or {})
|
|||
|
|
|
|||
|
|
@property
|
|||
|
|
def entries(self) -> dict[str, ManifestEntry]:
|
|||
|
|
return dict(self._entries)
|
|||
|
|
|
|||
|
|
# ---------- Aufbau ----------
|
|||
|
|
|
|||
|
|
@classmethod
|
|||
|
|
def build(
|
|||
|
|
cls,
|
|||
|
|
root: Path | str,
|
|||
|
|
kind_of=None,
|
|||
|
|
chunk_size: int = CHUNK_SIZE,
|
|||
|
|
revision: int = 0,
|
|||
|
|
) -> ContentManifest:
|
|||
|
|
"""Baut das Manifest aus einem Verzeichnis (§6.4 SHA-256).
|
|||
|
|
|
|||
|
|
kind_of: callable(rel_path) → "media"|"plugin"|"project";
|
|||
|
|
Default: alles KIND_MEDIA.
|
|||
|
|
"""
|
|||
|
|
root_path = Path(root)
|
|||
|
|
if not root_path.is_dir():
|
|||
|
|
raise FileNotFoundError(f"Manifest-Root fehlt: {root_path}")
|
|||
|
|
entries: dict[str, ManifestEntry] = {}
|
|||
|
|
for dirpath, _dirnames, filenames in os.walk(root_path):
|
|||
|
|
for name in sorted(filenames):
|
|||
|
|
abs_path = Path(dirpath) / name
|
|||
|
|
if not abs_path.is_file():
|
|||
|
|
continue
|
|||
|
|
rel = abs_path.relative_to(root_path).as_posix()
|
|||
|
|
if ".." in PurePosixPath(rel).parts:
|
|||
|
|
continue # defensive: nie Traversal ins Manifest
|
|||
|
|
file_hash, chunks = hash_file(abs_path, chunk_size=chunk_size)
|
|||
|
|
kind = kind_of(rel) if kind_of else KIND_MEDIA
|
|||
|
|
entries[rel] = ManifestEntry(
|
|||
|
|
rel_path=rel,
|
|||
|
|
kind=kind,
|
|||
|
|
size_bytes=abs_path.stat().st_size,
|
|||
|
|
sha256=file_hash,
|
|||
|
|
chunk_hashes=tuple(chunks),
|
|||
|
|
)
|
|||
|
|
return cls(revision=revision, entries=entries)
|
|||
|
|
|
|||
|
|
# ---------- Verifikation (§6.4 Hashprüfung) ----------
|
|||
|
|
|
|||
|
|
def verify_file(self, root: Path | str, rel_path: str) -> bool:
|
|||
|
|
"""Prüft Größe + Hash einer Datei gegen den Manifest-Eintrag."""
|
|||
|
|
entry = self._entries.get(rel_path)
|
|||
|
|
if entry is None:
|
|||
|
|
return False
|
|||
|
|
abs_path = Path(root) / rel_path
|
|||
|
|
if not abs_path.is_file():
|
|||
|
|
return False
|
|||
|
|
stat = abs_path.stat()
|
|||
|
|
if stat.st_size != entry.size_bytes:
|
|||
|
|
return False
|
|||
|
|
file_hash, _chunks = hash_file(abs_path)
|
|||
|
|
return file_hash == entry.sha256
|
|||
|
|
|
|||
|
|
# ---------- Diff (§6.4 Sync-Basis) ----------
|
|||
|
|
|
|||
|
|
def diff(self, other: ContentManifest) -> ManifestDiff:
|
|||
|
|
"""Was muss von self nach other übertragen/gelöscht werden?"""
|
|||
|
|
mine = set(self._entries)
|
|||
|
|
theirs = set(other._entries)
|
|||
|
|
added = frozenset(theirs - mine)
|
|||
|
|
removed = frozenset(mine - theirs)
|
|||
|
|
changed = frozenset(
|
|||
|
|
rel
|
|||
|
|
for rel in mine & theirs
|
|||
|
|
if self._entries[rel].sha256 != other._entries[rel].sha256
|
|||
|
|
)
|
|||
|
|
return ManifestDiff(added=added, removed=removed, changed=changed)
|
|||
|
|
|
|||
|
|
# ---------- Serialisierung ----------
|
|||
|
|
|
|||
|
|
def to_dict(self) -> dict:
|
|||
|
|
return {
|
|||
|
|
"manifest_id": self.manifest_id,
|
|||
|
|
"revision": self.revision,
|
|||
|
|
"entries": {
|
|||
|
|
rel: entry.to_dict()
|
|||
|
|
for rel, entry in sorted(self._entries.items())
|
|||
|
|
},
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
@classmethod
|
|||
|
|
def from_dict(cls, data: dict) -> ContentManifest:
|
|||
|
|
entries = {
|
|||
|
|
rel: ManifestEntry.from_dict(entry)
|
|||
|
|
for rel, entry in data.get("entries", {}).items()
|
|||
|
|
}
|
|||
|
|
return cls(
|
|||
|
|
manifest_id=data.get("manifest_id"),
|
|||
|
|
revision=int(data.get("revision", 0)),
|
|||
|
|
entries=entries,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def __len__(self) -> int:
|
|||
|
|
return len(self._entries)
|