Agent Zero
abbe7a18fc
- P0: hooks.py 3-tuple fix, trigger_dispatcher Contract, contacts/plugin unregister_actions_by_owner - P0: 5 test files — check_permission mocks removed, hardcoded DB credential → env var - P1: attachment_service DmsFile via Contract helper, restore_registry/history_hooks dedup - P1: mail/plugin restore unregister, mcp_client datetime.now(UTC), saved_views/filters patterns - P1: ProtectedRoute fail-closed, 13 test assertion fixes (bcrypt, DB-URLs, SECRET_KEYs) - P2: deprecated notifications → post_system_message (3 files), forgejo Base, report_generator lazy import - P2: webhooks permissions, deps.py/roles.py plugin perms removed, import_export default - P2: address/tags/entity_links patterns removed, worker.py Contract-Umgehungen fixed - P2: 28 frontend TODOs (hardcoded constants, deprecated notification API) - P3: dead code, duplicates, deprecated imports, private attr, __import__ inline - P3: 8 frontend TODOs (LucideIcons, inline styles, XSS, i18n) - ruff: 838 → 0 (612 auto-fix + 246 manual + 27 F821 regression fix) - F821: 30 → 0 (AutomationDefinition, DmsFile, user_id, Path, Any, String) - Contract-Umgehungen: 2 neue gefunden (worker.py:169, worker.py:280) und gefixt
544 lines
18 KiB
Python
544 lines
18 KiB
Python
"""Abstract storage backend — supports local filesystem and S3-compatible storage.
|
|
|
|
Configuration via environment variables:
|
|
- STORAGE_BACKEND: "local" (default) or "s3"
|
|
- STORAGE_PATH: Local storage base path (default: /data/uploads)
|
|
- S3_ENDPOINT: S3-compatible endpoint URL
|
|
- S3_BUCKET: Bucket name
|
|
- S3_ACCESS_KEY: Access key
|
|
- S3_SECRET_KEY: Secret key
|
|
- S3_REGION: Region (default: us-east-1)
|
|
- S3_SECURE: Use HTTPS (default: true)
|
|
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import io
|
|
import logging
|
|
import mimetypes
|
|
import os
|
|
import tempfile
|
|
from abc import ABC, abstractmethod
|
|
from collections.abc import AsyncIterator
|
|
from typing import Any
|
|
|
|
import aiofiles
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Try to import python-magic for content-based MIME detection
|
|
try:
|
|
import magic # type: ignore
|
|
|
|
_HAS_MAGIC = True
|
|
except ImportError:
|
|
_HAS_MAGIC = False
|
|
logger.debug("python-magic not installed, falling back to mimetypes")
|
|
|
|
|
|
# Default MIME allowlist — common document, image, and archive types
|
|
_DEFAULT_ALLOWED_MIMES: list[str] = [
|
|
# Documents
|
|
"text/plain", "text/html", "text/csv", "text/markdown",
|
|
"application/pdf", "application/msword",
|
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
"application/vnd.ms-excel",
|
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
"application/vnd.ms-powerpoint",
|
|
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
|
"application/vnd.oasis.opendocument.text",
|
|
"application/vnd.oasis.opendocument.spreadsheet",
|
|
"application/rtf", "application/json", "application/xml",
|
|
# Images
|
|
"image/jpeg", "image/png", "image/gif", "image/webp",
|
|
"image/svg+xml", "image/tiff", "image/bmp", "image/x-icon",
|
|
# Archives
|
|
"application/zip", "application/x-tar", "application/gzip",
|
|
"application/x-7z-compressed", "application/x-rar-compressed",
|
|
"application/x-bzip2",
|
|
# Other
|
|
"application/octet-stream", "message/rfc822", "application/x-yaml",
|
|
"text/x-yaml",
|
|
]
|
|
|
|
|
|
class StorageBackend(ABC):
|
|
"""Abstract storage backend for file operations."""
|
|
|
|
@abstractmethod
|
|
async def save(self, path: str, data: bytes) -> str:
|
|
"""Save data to storage at the given path. Returns the full storage path."""
|
|
...
|
|
|
|
@abstractmethod
|
|
async def save_stream(self, path: str, chunk_aiter: AsyncIterator[bytes]) -> int:
|
|
"""Stream chunks to storage. Returns total bytes written."""
|
|
...
|
|
|
|
@abstractmethod
|
|
async def read(self, path: str) -> bytes:
|
|
"""Read data from storage at the given path."""
|
|
...
|
|
|
|
@abstractmethod
|
|
async def delete(self, path: str) -> bool:
|
|
"""Delete a file from storage. Returns True if deleted, False if not found."""
|
|
...
|
|
|
|
@abstractmethod
|
|
async def exists(self, path: str) -> bool:
|
|
"""Check if a file exists in storage."""
|
|
...
|
|
|
|
@abstractmethod
|
|
async def get_url(self, path: str, expires: int = 3600) -> str:
|
|
"""Get a URL for accessing the file (presigned URL for S3, file path for local)."""
|
|
...
|
|
|
|
@abstractmethod
|
|
async def list_files(self, prefix: str) -> list[str]:
|
|
"""List all file paths under the given prefix."""
|
|
...
|
|
|
|
|
|
class LocalStorage(StorageBackend):
|
|
"""Local filesystem storage backend."""
|
|
|
|
def __init__(self, base_path: str | None = None) -> None:
|
|
self.base_path = base_path or os.environ.get("STORAGE_PATH", "/data/uploads")
|
|
os.makedirs(self.base_path, exist_ok=True)
|
|
|
|
def _full_path(self, path: str) -> str:
|
|
"""Get the full filesystem path with path traversal protection."""
|
|
# Normalize and ensure the path stays within base_path
|
|
full = os.path.normpath(os.path.join(self.base_path, path))
|
|
if not full.startswith(os.path.normpath(self.base_path)):
|
|
raise ValueError(f"Path traversal detected: {path}")
|
|
return full
|
|
|
|
async def save(self, path: str, data: bytes) -> str:
|
|
full_path = self._full_path(path)
|
|
os.makedirs(os.path.dirname(full_path), exist_ok=True)
|
|
async with aiofiles.open(full_path, "wb") as f:
|
|
await f.write(data)
|
|
logger.debug("LocalStorage: saved %s (%d bytes)", path, len(data))
|
|
return path
|
|
|
|
async def save_stream(self, path: str, chunk_aiter: AsyncIterator[bytes]) -> int:
|
|
"""Stream chunks directly to a local file. Returns total bytes written."""
|
|
full_path = self._full_path(path)
|
|
os.makedirs(os.path.dirname(full_path), exist_ok=True)
|
|
total = 0
|
|
async with aiofiles.open(full_path, "wb") as f:
|
|
async for chunk in chunk_aiter:
|
|
await f.write(chunk)
|
|
total += len(chunk)
|
|
logger.debug("LocalStorage: streamed %s (%d bytes)", path, total)
|
|
return total
|
|
|
|
async def read(self, path: str) -> bytes:
|
|
full_path = self._full_path(path)
|
|
async with aiofiles.open(full_path, "rb") as f:
|
|
return await f.read()
|
|
|
|
async def delete(self, path: str) -> bool:
|
|
full_path = self._full_path(path)
|
|
if os.path.exists(full_path): # noqa: ASYNC240
|
|
os.remove(full_path)
|
|
return True
|
|
return False
|
|
|
|
async def exists(self, path: str) -> bool:
|
|
return os.path.exists(self._full_path(path)) # noqa: ASYNC240
|
|
|
|
async def get_url(self, path: str, expires: int = 3600) -> str:
|
|
"""Return a relative URL path for the file (not the filesystem path)."""
|
|
# Return a relative path that can be served by the app
|
|
return f"/api/v1/dms/files/{path}"
|
|
|
|
async def list_files(self, prefix: str) -> list[str]:
|
|
full_prefix = self._full_path(prefix)
|
|
if not os.path.isdir(full_prefix): # noqa: ASYNC240
|
|
return []
|
|
result: list[str] = []
|
|
for root, _dirs, files in os.walk(full_prefix): # noqa: ASYNC240
|
|
for fname in files:
|
|
rel = os.path.relpath(os.path.join(root, fname), self.base_path) # noqa: ASYNC240
|
|
result.append(rel)
|
|
return result
|
|
|
|
|
|
class S3Storage(StorageBackend):
|
|
"""S3-compatible storage backend (works with AWS S3, MinIO, etc.)."""
|
|
|
|
def __init__(
|
|
self,
|
|
endpoint: str | None = None,
|
|
bucket: str | None = None,
|
|
access_key: str | None = None,
|
|
secret_key: str | None = None,
|
|
region: str | None = None,
|
|
secure: bool | None = None,
|
|
) -> None:
|
|
self.endpoint = endpoint or os.environ.get("S3_ENDPOINT", "")
|
|
self.bucket = bucket or os.environ.get("S3_BUCKET", "")
|
|
self.access_key = access_key or os.environ.get("S3_ACCESS_KEY", "")
|
|
self.secret_key = secret_key or os.environ.get("S3_SECRET_KEY", "")
|
|
self.region = region or os.environ.get("S3_REGION", "us-east-1")
|
|
self.secure = secure if secure is not None else os.environ.get("S3_SECURE", "true").lower() == "true"
|
|
self._client: Any = None # lazy init
|
|
|
|
def _get_client(self) -> Any:
|
|
"""Lazy-initialize the S3 client (minio or boto3)."""
|
|
if self._client is not None:
|
|
return self._client
|
|
|
|
try:
|
|
from minio import Minio # type: ignore
|
|
|
|
self._client = Minio(
|
|
endpoint=self.endpoint.replace("https://", "").replace("http://", ""),
|
|
access_key=self.access_key,
|
|
secret_key=self.secret_key,
|
|
secure=self.secure,
|
|
region=self.region,
|
|
)
|
|
# Ensure bucket exists
|
|
if not self._client.bucket_exists(self.bucket):
|
|
self._client.make_bucket(self.bucket)
|
|
logger.info("S3Storage: connected to %s, bucket=%s", self.endpoint, self.bucket)
|
|
return self._client
|
|
except ImportError:
|
|
logger.error("S3Storage: minio package not installed. Install with: pip install minio")
|
|
raise
|
|
except Exception as e:
|
|
logger.error("S3Storage: failed to connect to %s: %s", self.endpoint, e)
|
|
raise
|
|
|
|
# ── Sync helper methods (called via asyncio.to_thread) ──────────────────
|
|
|
|
def _save_sync(self, path: str, data: bytes) -> str:
|
|
client = self._get_client()
|
|
client.put_object(
|
|
bucket_name=self.bucket,
|
|
object_name=path,
|
|
data=io.BytesIO(data),
|
|
length=len(data),
|
|
)
|
|
return path
|
|
|
|
def _put_file_sync(self, object_name: str, file_path: str) -> str:
|
|
client = self._get_client()
|
|
client.fput_object(self.bucket, object_name, file_path)
|
|
return object_name
|
|
|
|
def _read_sync(self, path: str) -> bytes:
|
|
client = self._get_client()
|
|
response = client.get_object(self.bucket, path)
|
|
try:
|
|
return response.read()
|
|
finally:
|
|
response.close()
|
|
response.release_conn()
|
|
|
|
def _delete_sync(self, path: str) -> bool:
|
|
client = self._get_client()
|
|
try:
|
|
client.remove_object(self.bucket, path)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def _exists_sync(self, path: str) -> bool:
|
|
client = self._get_client()
|
|
try:
|
|
client.stat_object(self.bucket, path)
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
def _get_url_sync(self, path: str, expires: int) -> str:
|
|
from datetime import timedelta
|
|
|
|
client = self._get_client()
|
|
return client.presigned_get_object(self.bucket, path, expires=timedelta(seconds=expires))
|
|
|
|
def _list_files_sync(self, prefix: str) -> list[str]:
|
|
client = self._get_client()
|
|
objects = client.list_objects(self.bucket, prefix=prefix, recursive=True)
|
|
return [obj.object_name for obj in objects]
|
|
|
|
# ── Async public API (wraps sync calls in asyncio.to_thread) ─────────────
|
|
|
|
async def save(self, path: str, data: bytes) -> str:
|
|
result = await asyncio.to_thread(self._save_sync, path, data)
|
|
logger.debug("S3Storage: saved %s (%d bytes)", path, len(data))
|
|
return result
|
|
|
|
async def save_stream(self, path: str, chunk_aiter: AsyncIterator[bytes]) -> int:
|
|
"""Stream chunks to a temp file, then upload to S3 via fput_object.
|
|
|
|
This avoids loading the entire file into RAM. The temp file is
|
|
cleaned up after upload.
|
|
"""
|
|
tmp_fd, tmp_path = tempfile.mkstemp(prefix="s3_upload_")
|
|
os.close(tmp_fd)
|
|
total = 0
|
|
try:
|
|
async with aiofiles.open(tmp_path, "wb") as f:
|
|
async for chunk in chunk_aiter:
|
|
await f.write(chunk)
|
|
total += len(chunk)
|
|
await asyncio.to_thread(self._put_file_sync, path, tmp_path)
|
|
logger.debug("S3Storage: streamed %s (%d bytes)", path, total)
|
|
return total
|
|
finally:
|
|
if os.path.exists(tmp_path): # noqa: ASYNC240
|
|
try:
|
|
os.remove(tmp_path)
|
|
except OSError:
|
|
logger.warning("S3Storage: failed to clean up temp file %s", tmp_path)
|
|
|
|
async def read(self, path: str) -> bytes:
|
|
return await asyncio.to_thread(self._read_sync, path)
|
|
|
|
async def delete(self, path: str) -> bool:
|
|
return await asyncio.to_thread(self._delete_sync, path)
|
|
|
|
async def exists(self, path: str) -> bool:
|
|
return await asyncio.to_thread(self._exists_sync, path)
|
|
|
|
async def get_url(self, path: str, expires: int = 3600) -> str:
|
|
return await asyncio.to_thread(self._get_url_sync, path, expires)
|
|
|
|
async def list_files(self, prefix: str) -> list[str]:
|
|
return await asyncio.to_thread(self._list_files_sync, prefix)
|
|
|
|
|
|
# ─── Factory ───
|
|
|
|
_storage_backend: StorageBackend | None = None
|
|
|
|
|
|
def get_storage_backend() -> StorageBackend:
|
|
"""Get the configured storage backend singleton."""
|
|
global _storage_backend
|
|
if _storage_backend is None:
|
|
backend_type = os.environ.get("STORAGE_BACKEND", "local").lower()
|
|
if backend_type == "s3":
|
|
_storage_backend = S3Storage()
|
|
logger.info("Storage backend: S3 (%s)", os.environ.get("S3_ENDPOINT", ""))
|
|
else:
|
|
_storage_backend = LocalStorage()
|
|
logger.info("Storage backend: Local (%s)", os.environ.get("STORAGE_PATH", "/data/uploads"))
|
|
return _storage_backend
|
|
|
|
|
|
def reset_storage_backend() -> None:
|
|
"""Reset the storage backend singleton (for testing)."""
|
|
global _storage_backend
|
|
_storage_backend = None
|
|
|
|
|
|
# ─── Validation & Metadata Helpers ───
|
|
|
|
|
|
def validate_mime(
|
|
path: str,
|
|
data: bytes,
|
|
allowed_mimes: list[str] | None = None,
|
|
) -> str:
|
|
"""Detect the MIME type of *data* and validate it against an allowlist.
|
|
|
|
Uses ``python-magic`` for content-based detection when available,
|
|
falling back to ``mimetypes`` (extension-based) otherwise.
|
|
|
|
Parameters
|
|
----------
|
|
path:
|
|
Filename or relative path — used for extension-based fallback.
|
|
data:
|
|
File content bytes used for content-based detection.
|
|
allowed_mimes:
|
|
Allowlist of MIME types. ``None`` uses the built-in default
|
|
allowlist. An empty list means *all* MIME types are allowed.
|
|
|
|
Returns
|
|
-------
|
|
str
|
|
The detected MIME type.
|
|
|
|
Raises
|
|
------
|
|
ValueError
|
|
If the detected MIME type is not in *allowed_mimes*.
|
|
"""
|
|
# --- detect MIME type -------------------------------------------------
|
|
if _HAS_MAGIC:
|
|
try:
|
|
mime_type = magic.from_buffer(data, mime=True)
|
|
except Exception:
|
|
mime_type, _ = mimetypes.guess_type(path)
|
|
mime_type = mime_type or "application/octet-stream"
|
|
else:
|
|
mime_type, _ = mimetypes.guess_type(path)
|
|
mime_type = mime_type or "application/octet-stream"
|
|
|
|
# --- validate against allowlist --------------------------------------
|
|
if allowed_mimes is None:
|
|
from app.config import get_settings
|
|
|
|
config_mimes = get_settings().storage_allowed_mimes
|
|
if config_mimes:
|
|
allowed = [m.strip() for m in config_mimes.split(",") if m.strip()]
|
|
else:
|
|
allowed = _DEFAULT_ALLOWED_MIMES
|
|
else:
|
|
allowed = allowed_mimes
|
|
|
|
if allowed and mime_type not in allowed:
|
|
raise ValueError(
|
|
f"MIME type '{mime_type}' is not allowed. "
|
|
f"Allowed types: {', '.join(allowed[:10])}{'...' if len(allowed) > 10 else ''}"
|
|
)
|
|
|
|
return mime_type
|
|
|
|
|
|
def validate_size(data: bytes, max_size_mb: int | None = None) -> None:
|
|
"""Validate that *data* does not exceed the configured size limit.
|
|
|
|
Parameters
|
|
----------
|
|
data:
|
|
File content bytes.
|
|
max_size_mb:
|
|
Maximum allowed size in megabytes. ``None`` reads the value
|
|
from the ``STORAGE_MAX_FILE_SIZE_MB`` environment variable
|
|
(default: 50).
|
|
|
|
Raises
|
|
------
|
|
ValueError
|
|
If ``len(data)`` exceeds ``max_size_mb * 1024 * 1024``.
|
|
"""
|
|
if max_size_mb is None:
|
|
from app.config import get_settings
|
|
|
|
max_size_mb = get_settings().storage_max_file_size_mb
|
|
|
|
max_bytes = max_size_mb * 1024 * 1024
|
|
if len(data) > max_bytes:
|
|
raise ValueError(
|
|
f"File size {len(data)} bytes exceeds limit of {max_size_mb} MB ({max_bytes} bytes)"
|
|
)
|
|
|
|
|
|
def compute_hash(data: bytes, algorithm: str = "sha256") -> str:
|
|
"""Compute a cryptographic hash of *data*.
|
|
|
|
Parameters
|
|
----------
|
|
data:
|
|
Content to hash.
|
|
algorithm:
|
|
Hash algorithm name (e.g. ``"sha256"``, ``"md5"``, ``"sha1"``).
|
|
|
|
Returns
|
|
-------
|
|
str
|
|
Hexadecimal digest string.
|
|
"""
|
|
h = hashlib.new(algorithm)
|
|
h.update(data)
|
|
return h.hexdigest()
|
|
|
|
|
|
async def save_with_metadata(
|
|
path: str,
|
|
data: bytes,
|
|
allowed_mimes: list[str] | None = None,
|
|
max_size_mb: int | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Save *data* to storage with full validation and metadata extraction.
|
|
|
|
Combines :func:`validate_size`, :func:`validate_mime`,
|
|
:func:`compute_hash` and :meth:`StorageBackend.save` into a single
|
|
call.
|
|
|
|
Parameters
|
|
----------
|
|
path:
|
|
Relative storage path.
|
|
data:
|
|
File content bytes.
|
|
allowed_mimes:
|
|
MIME allowlist — ``None`` uses the default allowlist.
|
|
max_size_mb:
|
|
Size limit in MB — ``None`` reads from config.
|
|
|
|
Returns
|
|
-------
|
|
dict
|
|
``{path, mime_type, size, hash, storage_path}``
|
|
"""
|
|
validate_size(data, max_size_mb)
|
|
mime_type = validate_mime(path, data, allowed_mimes)
|
|
file_hash = compute_hash(data)
|
|
|
|
backend = get_storage_backend()
|
|
storage_path = await backend.save(path, data)
|
|
|
|
return {
|
|
"path": path,
|
|
"mime_type": mime_type,
|
|
"size": len(data),
|
|
"hash": file_hash,
|
|
"storage_path": storage_path,
|
|
}
|
|
|
|
|
|
def get_file_metadata(path: str) -> dict[str, Any]:
|
|
"""Read metadata of a stored file without loading its content.
|
|
|
|
Works with the *local* storage backend. For S3, use the S3 client
|
|
``stat_object`` API directly.
|
|
|
|
Parameters
|
|
----------
|
|
path:
|
|
Relative storage path.
|
|
|
|
Returns
|
|
-------
|
|
dict
|
|
``{size, modified, exists}`` — ``exists`` is ``False`` when the
|
|
file is not found, in which case ``size`` and ``modified`` are
|
|
``None``.
|
|
"""
|
|
backend = get_storage_backend()
|
|
if isinstance(backend, LocalStorage):
|
|
full_path = backend._full_path(path)
|
|
if not os.path.exists(full_path):
|
|
return {"size": None, "modified": None, "exists": False}
|
|
stat = os.stat(full_path)
|
|
return {
|
|
"size": stat.st_size,
|
|
"modified": stat.st_mtime,
|
|
"exists": True,
|
|
}
|
|
# S3 or other backends — fall back to exists() check
|
|
import asyncio as _asyncio
|
|
|
|
loop = _asyncio.new_event_loop()
|
|
try:
|
|
exists = loop.run_until_complete(backend.exists(path))
|
|
if not exists:
|
|
return {"size": None, "modified": None, "exists": False}
|
|
return {"size": None, "modified": None, "exists": True}
|
|
finally:
|
|
loop.close()
|