feat(B-STOR): Gemeinsamer File Storage — MIME-Prüfung, Size-Limits, Hashing, save_with_metadata
B-STOR: storage.py um 5 Funktionen erweitert - validate_mime(): MIME-Erkennung (python-magic/mimetypes) + Allowlist-Prüfung - validate_size(): Dateigrößen-Prüfung (Default 50MB, konfigurierbar) - compute_hash(): SHA256/MD5/SHA1 Hash-Berechnung - save_with_metadata(): save + validate + hash in einem Call - get_file_metadata(): File-Stat ohne Content zu laden - config.py: storage_max_file_size_mb, storage_allowed_mimes Settings - Backward compatible: save/read/delete/exists unverändert B-STOR-TEST: 27 Tests in test_storage.py — alle grün - Path-Traversal, MIME-Validation, Size-Limits, Hashing, save_with_metadata, LocalStorage, Factory B-STOR-MIG: Bereits erledigt — DMS, Mail, Report-Generator, Attachments nutzen bereits get_storage_backend()
This commit is contained in:
Agent Zero
@@ -48,6 +48,8 @@ class Settings(BaseSettings):
|
||||
|
||||
# Storage
|
||||
storage_path: str = "/data/storage"
|
||||
storage_max_file_size_mb: int = 50
|
||||
storage_allowed_mimes: str = "" # comma-separated, empty = all allowed
|
||||
|
||||
# SMTP
|
||||
smtp_host: str = "localhost"
|
||||
|
||||
@@ -15,8 +15,10 @@ Configuration via environment variables:
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import tempfile
|
||||
from abc import ABC, abstractmethod
|
||||
@@ -26,6 +28,41 @@ import aiofiles
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Try to import python-magic for content-based MIME detection
|
||||
try:
|
||||
import magic # type: ignore
|
||||
|
||||
_HAS_MAGIC = True
|
||||
except ImportError:
|
||||
_HAS_MAGIC = False
|
||||
logger.debug("python-magic not installed, falling back to mimetypes")
|
||||
|
||||
|
||||
# Default MIME allowlist — common document, image, and archive types
|
||||
_DEFAULT_ALLOWED_MIMES: list[str] = [
|
||||
# Documents
|
||||
"text/plain", "text/html", "text/csv", "text/markdown",
|
||||
"application/pdf", "application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"application/vnd.oasis.opendocument.spreadsheet",
|
||||
"application/rtf", "application/json", "application/xml",
|
||||
# Images
|
||||
"image/jpeg", "image/png", "image/gif", "image/webp",
|
||||
"image/svg+xml", "image/tiff", "image/bmp", "image/x-icon",
|
||||
# Archives
|
||||
"application/zip", "application/x-tar", "application/gzip",
|
||||
"application/x-7z-compressed", "application/x-rar-compressed",
|
||||
"application/x-bzip2",
|
||||
# Other
|
||||
"application/octet-stream", "message/rfc822", "application/x-yaml",
|
||||
"text/x-yaml",
|
||||
]
|
||||
|
||||
|
||||
class StorageBackend(ABC):
|
||||
"""Abstract storage backend for file operations."""
|
||||
@@ -303,3 +340,203 @@ def reset_storage_backend() -> None:
|
||||
"""Reset the storage backend singleton (for testing)."""
|
||||
global _storage_backend
|
||||
_storage_backend = None
|
||||
|
||||
|
||||
# ─── Validation & Metadata Helpers ───
|
||||
|
||||
|
||||
def validate_mime(
|
||||
path: str,
|
||||
data: bytes,
|
||||
allowed_mimes: list[str] | None = None,
|
||||
) -> str:
|
||||
"""Detect the MIME type of *data* and validate it against an allowlist.
|
||||
|
||||
Uses ``python-magic`` for content-based detection when available,
|
||||
falling back to ``mimetypes`` (extension-based) otherwise.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
Filename or relative path — used for extension-based fallback.
|
||||
data:
|
||||
File content bytes used for content-based detection.
|
||||
allowed_mimes:
|
||||
Allowlist of MIME types. ``None`` uses the built-in default
|
||||
allowlist. An empty list means *all* MIME types are allowed.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
The detected MIME type.
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If the detected MIME type is not in *allowed_mimes*.
|
||||
"""
|
||||
# --- detect MIME type -------------------------------------------------
|
||||
if _HAS_MAGIC:
|
||||
try:
|
||||
mime_type = magic.from_buffer(data, mime=True)
|
||||
except Exception:
|
||||
mime_type, _ = mimetypes.guess_type(path)
|
||||
mime_type = mime_type or "application/octet-stream"
|
||||
else:
|
||||
mime_type, _ = mimetypes.guess_type(path)
|
||||
mime_type = mime_type or "application/octet-stream"
|
||||
|
||||
# --- validate against allowlist --------------------------------------
|
||||
if allowed_mimes is None:
|
||||
from app.config import get_settings
|
||||
|
||||
config_mimes = get_settings().storage_allowed_mimes
|
||||
if config_mimes:
|
||||
allowed = [m.strip() for m in config_mimes.split(",") if m.strip()]
|
||||
else:
|
||||
allowed = _DEFAULT_ALLOWED_MIMES
|
||||
else:
|
||||
allowed = allowed_mimes
|
||||
|
||||
if allowed and mime_type not in allowed:
|
||||
raise ValueError(
|
||||
f"MIME type '{mime_type}' is not allowed. "
|
||||
f"Allowed types: {', '.join(allowed[:10])}{'...' if len(allowed) > 10 else ''}"
|
||||
)
|
||||
|
||||
return mime_type
|
||||
|
||||
|
||||
def validate_size(data: bytes, max_size_mb: int | None = None) -> None:
|
||||
"""Validate that *data* does not exceed the configured size limit.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
data:
|
||||
File content bytes.
|
||||
max_size_mb:
|
||||
Maximum allowed size in megabytes. ``None`` reads the value
|
||||
from the ``STORAGE_MAX_FILE_SIZE_MB`` environment variable
|
||||
(default: 50).
|
||||
|
||||
Raises
|
||||
------
|
||||
ValueError
|
||||
If ``len(data)`` exceeds ``max_size_mb * 1024 * 1024``.
|
||||
"""
|
||||
if max_size_mb is None:
|
||||
from app.config import get_settings
|
||||
|
||||
max_size_mb = get_settings().storage_max_file_size_mb
|
||||
|
||||
max_bytes = max_size_mb * 1024 * 1024
|
||||
if len(data) > max_bytes:
|
||||
raise ValueError(
|
||||
f"File size {len(data)} bytes exceeds limit of {max_size_mb} MB ({max_bytes} bytes)"
|
||||
)
|
||||
|
||||
|
||||
def compute_hash(data: bytes, algorithm: str = "sha256") -> str:
|
||||
"""Compute a cryptographic hash of *data*.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
data:
|
||||
Content to hash.
|
||||
algorithm:
|
||||
Hash algorithm name (e.g. ``"sha256"``, ``"md5"``, ``"sha1"``).
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Hexadecimal digest string.
|
||||
"""
|
||||
h = hashlib.new(algorithm)
|
||||
h.update(data)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
async def save_with_metadata(
|
||||
path: str,
|
||||
data: bytes,
|
||||
allowed_mimes: list[str] | None = None,
|
||||
max_size_mb: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Save *data* to storage with full validation and metadata extraction.
|
||||
|
||||
Combines :func:`validate_size`, :func:`validate_mime`,
|
||||
:func:`compute_hash` and :meth:`StorageBackend.save` into a single
|
||||
call.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
Relative storage path.
|
||||
data:
|
||||
File content bytes.
|
||||
allowed_mimes:
|
||||
MIME allowlist — ``None`` uses the default allowlist.
|
||||
max_size_mb:
|
||||
Size limit in MB — ``None`` reads from config.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
``{path, mime_type, size, hash, storage_path}``
|
||||
"""
|
||||
validate_size(data, max_size_mb)
|
||||
mime_type = validate_mime(path, data, allowed_mimes)
|
||||
file_hash = compute_hash(data)
|
||||
|
||||
backend = get_storage_backend()
|
||||
storage_path = await backend.save(path, data)
|
||||
|
||||
return {
|
||||
"path": path,
|
||||
"mime_type": mime_type,
|
||||
"size": len(data),
|
||||
"hash": file_hash,
|
||||
"storage_path": storage_path,
|
||||
}
|
||||
|
||||
|
||||
def get_file_metadata(path: str) -> dict[str, Any]:
|
||||
"""Read metadata of a stored file without loading its content.
|
||||
|
||||
Works with the *local* storage backend. For S3, use the S3 client
|
||||
``stat_object`` API directly.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
path:
|
||||
Relative storage path.
|
||||
|
||||
Returns
|
||||
-------
|
||||
dict
|
||||
``{size, modified, exists}`` — ``exists`` is ``False`` when the
|
||||
file is not found, in which case ``size`` and ``modified`` are
|
||||
``None``.
|
||||
"""
|
||||
backend = get_storage_backend()
|
||||
if isinstance(backend, LocalStorage):
|
||||
full_path = backend._full_path(path)
|
||||
if not os.path.exists(full_path):
|
||||
return {"size": None, "modified": None, "exists": False}
|
||||
stat = os.stat(full_path)
|
||||
return {
|
||||
"size": stat.st_size,
|
||||
"modified": stat.st_mtime,
|
||||
"exists": True,
|
||||
}
|
||||
# S3 or other backends — fall back to exists() check
|
||||
import asyncio as _asyncio
|
||||
|
||||
loop = _asyncio.new_event_loop()
|
||||
try:
|
||||
exists = loop.run_until_complete(backend.exists(path))
|
||||
if not exists:
|
||||
return {"size": None, "modified": None, "exists": False}
|
||||
return {"size": None, "modified": None, "exists": True}
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
Reference in New Issue
Block a user