"""Abstract storage backend — supports local filesystem and S3-compatible storage. Configuration via environment variables: - STORAGE_BACKEND: "local" (default) or "s3" - STORAGE_PATH: Local storage base path (default: /data/uploads) - S3_ENDPOINT: S3-compatible endpoint URL - S3_BUCKET: Bucket name - S3_ACCESS_KEY: Access key - S3_SECRET_KEY: Secret key - S3_REGION: Region (default: us-east-1) - S3_SECURE: Use HTTPS (default: true) """ from __future__ import annotations import asyncio import hashlib import io import logging import mimetypes import os import tempfile from abc import ABC, abstractmethod from collections.abc import AsyncIterator from typing import Any import aiofiles logger = logging.getLogger(__name__) # Try to import python-magic for content-based MIME detection try: import magic # type: ignore _HAS_MAGIC = True except ImportError: _HAS_MAGIC = False logger.debug("python-magic not installed, falling back to mimetypes") # Default MIME allowlist — common document, image, and archive types _DEFAULT_ALLOWED_MIMES: list[str] = [ # Documents "text/plain", "text/html", "text/csv", "text/markdown", "application/pdf", "application/msword", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.ms-powerpoint", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.oasis.opendocument.text", "application/vnd.oasis.opendocument.spreadsheet", "application/rtf", "application/json", "application/xml", # Images "image/jpeg", "image/png", "image/gif", "image/webp", "image/svg+xml", "image/tiff", "image/bmp", "image/x-icon", # Archives "application/zip", "application/x-tar", "application/gzip", "application/x-7z-compressed", "application/x-rar-compressed", "application/x-bzip2", # Other "application/octet-stream", "message/rfc822", "application/x-yaml", "text/x-yaml", ] class StorageBackend(ABC): """Abstract storage backend for file operations.""" @abstractmethod async def save(self, path: str, data: bytes) -> str: """Save data to storage at the given path. Returns the full storage path.""" ... @abstractmethod async def save_stream(self, path: str, chunk_aiter: AsyncIterator[bytes]) -> int: """Stream chunks to storage. Returns total bytes written.""" ... @abstractmethod async def read(self, path: str) -> bytes: """Read data from storage at the given path.""" ... @abstractmethod async def delete(self, path: str) -> bool: """Delete a file from storage. Returns True if deleted, False if not found.""" ... @abstractmethod async def exists(self, path: str) -> bool: """Check if a file exists in storage.""" ... @abstractmethod async def get_url(self, path: str, expires: int = 3600) -> str: """Get a URL for accessing the file (presigned URL for S3, file path for local).""" ... @abstractmethod async def list_files(self, prefix: str) -> list[str]: """List all file paths under the given prefix.""" ... class LocalStorage(StorageBackend): """Local filesystem storage backend.""" def __init__(self, base_path: str | None = None) -> None: self.base_path = base_path or os.environ.get("STORAGE_PATH", "/data/uploads") os.makedirs(self.base_path, exist_ok=True) def _full_path(self, path: str) -> str: """Get the full filesystem path with path traversal protection.""" # Normalize and ensure the path stays within base_path full = os.path.normpath(os.path.join(self.base_path, path)) if not full.startswith(os.path.normpath(self.base_path)): raise ValueError(f"Path traversal detected: {path}") return full async def save(self, path: str, data: bytes) -> str: full_path = self._full_path(path) os.makedirs(os.path.dirname(full_path), exist_ok=True) async with aiofiles.open(full_path, "wb") as f: await f.write(data) logger.debug("LocalStorage: saved %s (%d bytes)", path, len(data)) return path async def save_stream(self, path: str, chunk_aiter: AsyncIterator[bytes]) -> int: """Stream chunks directly to a local file. Returns total bytes written.""" full_path = self._full_path(path) os.makedirs(os.path.dirname(full_path), exist_ok=True) total = 0 async with aiofiles.open(full_path, "wb") as f: async for chunk in chunk_aiter: await f.write(chunk) total += len(chunk) logger.debug("LocalStorage: streamed %s (%d bytes)", path, total) return total async def read(self, path: str) -> bytes: full_path = self._full_path(path) async with aiofiles.open(full_path, "rb") as f: return await f.read() async def delete(self, path: str) -> bool: full_path = self._full_path(path) if os.path.exists(full_path): # noqa: ASYNC240 os.remove(full_path) return True return False async def exists(self, path: str) -> bool: return os.path.exists(self._full_path(path)) # noqa: ASYNC240 async def get_url(self, path: str, expires: int = 3600) -> str: """Return a relative URL path for the file (not the filesystem path).""" # Return a relative path that can be served by the app return f"/api/v1/dms/files/{path}" async def list_files(self, prefix: str) -> list[str]: full_prefix = self._full_path(prefix) if not os.path.isdir(full_prefix): # noqa: ASYNC240 return [] result: list[str] = [] for root, _dirs, files in os.walk(full_prefix): # noqa: ASYNC240 for fname in files: rel = os.path.relpath(os.path.join(root, fname), self.base_path) # noqa: ASYNC240 result.append(rel) return result class S3Storage(StorageBackend): """S3-compatible storage backend (works with AWS S3, MinIO, etc.).""" def __init__( self, endpoint: str | None = None, bucket: str | None = None, access_key: str | None = None, secret_key: str | None = None, region: str | None = None, secure: bool | None = None, ) -> None: self.endpoint = endpoint or os.environ.get("S3_ENDPOINT", "") self.bucket = bucket or os.environ.get("S3_BUCKET", "") self.access_key = access_key or os.environ.get("S3_ACCESS_KEY", "") self.secret_key = secret_key or os.environ.get("S3_SECRET_KEY", "") self.region = region or os.environ.get("S3_REGION", "us-east-1") self.secure = secure if secure is not None else os.environ.get("S3_SECURE", "true").lower() == "true" self._client: Any = None # lazy init def _get_client(self) -> Any: """Lazy-initialize the S3 client (minio or boto3).""" if self._client is not None: return self._client try: from minio import Minio # type: ignore self._client = Minio( endpoint=self.endpoint.replace("https://", "").replace("http://", ""), access_key=self.access_key, secret_key=self.secret_key, secure=self.secure, region=self.region, ) # Ensure bucket exists if not self._client.bucket_exists(self.bucket): self._client.make_bucket(self.bucket) logger.info("S3Storage: connected to %s, bucket=%s", self.endpoint, self.bucket) return self._client except ImportError: logger.error("S3Storage: minio package not installed. Install with: pip install minio") raise except Exception as e: logger.error("S3Storage: failed to connect to %s: %s", self.endpoint, e) raise # ── Sync helper methods (called via asyncio.to_thread) ────────────────── def _save_sync(self, path: str, data: bytes) -> str: client = self._get_client() client.put_object( bucket_name=self.bucket, object_name=path, data=io.BytesIO(data), length=len(data), ) return path def _put_file_sync(self, object_name: str, file_path: str) -> str: client = self._get_client() client.fput_object(self.bucket, object_name, file_path) return object_name def _read_sync(self, path: str) -> bytes: client = self._get_client() response = client.get_object(self.bucket, path) try: return response.read() finally: response.close() response.release_conn() def _delete_sync(self, path: str) -> bool: client = self._get_client() try: client.remove_object(self.bucket, path) return True except Exception: return False def _exists_sync(self, path: str) -> bool: client = self._get_client() try: client.stat_object(self.bucket, path) return True except Exception: return False def _get_url_sync(self, path: str, expires: int) -> str: from datetime import timedelta client = self._get_client() return client.presigned_get_object(self.bucket, path, expires=timedelta(seconds=expires)) def _list_files_sync(self, prefix: str) -> list[str]: client = self._get_client() objects = client.list_objects(self.bucket, prefix=prefix, recursive=True) return [obj.object_name for obj in objects] # ── Async public API (wraps sync calls in asyncio.to_thread) ───────────── async def save(self, path: str, data: bytes) -> str: result = await asyncio.to_thread(self._save_sync, path, data) logger.debug("S3Storage: saved %s (%d bytes)", path, len(data)) return result async def save_stream(self, path: str, chunk_aiter: AsyncIterator[bytes]) -> int: """Stream chunks to a temp file, then upload to S3 via fput_object. This avoids loading the entire file into RAM. The temp file is cleaned up after upload. """ tmp_fd, tmp_path = tempfile.mkstemp(prefix="s3_upload_") os.close(tmp_fd) total = 0 try: async with aiofiles.open(tmp_path, "wb") as f: async for chunk in chunk_aiter: await f.write(chunk) total += len(chunk) await asyncio.to_thread(self._put_file_sync, path, tmp_path) logger.debug("S3Storage: streamed %s (%d bytes)", path, total) return total finally: if os.path.exists(tmp_path): # noqa: ASYNC240 try: os.remove(tmp_path) except OSError: logger.warning("S3Storage: failed to clean up temp file %s", tmp_path) async def read(self, path: str) -> bytes: return await asyncio.to_thread(self._read_sync, path) async def delete(self, path: str) -> bool: return await asyncio.to_thread(self._delete_sync, path) async def exists(self, path: str) -> bool: return await asyncio.to_thread(self._exists_sync, path) async def get_url(self, path: str, expires: int = 3600) -> str: return await asyncio.to_thread(self._get_url_sync, path, expires) async def list_files(self, prefix: str) -> list[str]: return await asyncio.to_thread(self._list_files_sync, prefix) # ─── Factory ─── _storage_backend: StorageBackend | None = None def get_storage_backend() -> StorageBackend: """Get the configured storage backend singleton.""" global _storage_backend if _storage_backend is None: backend_type = os.environ.get("STORAGE_BACKEND", "local").lower() if backend_type == "s3": _storage_backend = S3Storage() logger.info("Storage backend: S3 (%s)", os.environ.get("S3_ENDPOINT", "")) else: _storage_backend = LocalStorage() logger.info("Storage backend: Local (%s)", os.environ.get("STORAGE_PATH", "/data/uploads")) return _storage_backend def reset_storage_backend() -> None: """Reset the storage backend singleton (for testing).""" global _storage_backend _storage_backend = None # ─── Validation & Metadata Helpers ─── def validate_mime( path: str, data: bytes, allowed_mimes: list[str] | None = None, ) -> str: """Detect the MIME type of *data* and validate it against an allowlist. Uses ``python-magic`` for content-based detection when available, falling back to ``mimetypes`` (extension-based) otherwise. Parameters ---------- path: Filename or relative path — used for extension-based fallback. data: File content bytes used for content-based detection. allowed_mimes: Allowlist of MIME types. ``None`` uses the built-in default allowlist. An empty list means *all* MIME types are allowed. Returns ------- str The detected MIME type. Raises ------ ValueError If the detected MIME type is not in *allowed_mimes*. """ # --- detect MIME type ------------------------------------------------- if _HAS_MAGIC: try: mime_type = magic.from_buffer(data, mime=True) except Exception: mime_type, _ = mimetypes.guess_type(path) mime_type = mime_type or "application/octet-stream" else: mime_type, _ = mimetypes.guess_type(path) mime_type = mime_type or "application/octet-stream" # --- validate against allowlist -------------------------------------- if allowed_mimes is None: from app.config import get_settings config_mimes = get_settings().storage_allowed_mimes if config_mimes: allowed = [m.strip() for m in config_mimes.split(",") if m.strip()] else: allowed = _DEFAULT_ALLOWED_MIMES else: allowed = allowed_mimes if allowed and mime_type not in allowed: raise ValueError( f"MIME type '{mime_type}' is not allowed. " f"Allowed types: {', '.join(allowed[:10])}{'...' if len(allowed) > 10 else ''}" ) return mime_type def validate_size(data: bytes, max_size_mb: int | None = None) -> None: """Validate that *data* does not exceed the configured size limit. Parameters ---------- data: File content bytes. max_size_mb: Maximum allowed size in megabytes. ``None`` reads the value from the ``STORAGE_MAX_FILE_SIZE_MB`` environment variable (default: 50). Raises ------ ValueError If ``len(data)`` exceeds ``max_size_mb * 1024 * 1024``. """ if max_size_mb is None: from app.config import get_settings max_size_mb = get_settings().storage_max_file_size_mb max_bytes = max_size_mb * 1024 * 1024 if len(data) > max_bytes: raise ValueError( f"File size {len(data)} bytes exceeds limit of {max_size_mb} MB ({max_bytes} bytes)" ) def compute_hash(data: bytes, algorithm: str = "sha256") -> str: """Compute a cryptographic hash of *data*. Parameters ---------- data: Content to hash. algorithm: Hash algorithm name (e.g. ``"sha256"``, ``"md5"``, ``"sha1"``). Returns ------- str Hexadecimal digest string. """ h = hashlib.new(algorithm) h.update(data) return h.hexdigest() async def save_with_metadata( path: str, data: bytes, allowed_mimes: list[str] | None = None, max_size_mb: int | None = None, ) -> dict[str, Any]: """Save *data* to storage with full validation and metadata extraction. Combines :func:`validate_size`, :func:`validate_mime`, :func:`compute_hash` and :meth:`StorageBackend.save` into a single call. Parameters ---------- path: Relative storage path. data: File content bytes. allowed_mimes: MIME allowlist — ``None`` uses the default allowlist. max_size_mb: Size limit in MB — ``None`` reads from config. Returns ------- dict ``{path, mime_type, size, hash, storage_path}`` """ validate_size(data, max_size_mb) mime_type = validate_mime(path, data, allowed_mimes) file_hash = compute_hash(data) backend = get_storage_backend() storage_path = await backend.save(path, data) return { "path": path, "mime_type": mime_type, "size": len(data), "hash": file_hash, "storage_path": storage_path, } async def get_file_metadata_async(path: str) -> dict[str, Any]: """Awaitable variant of :func:`get_file_metadata` (ARCH-052). Safe to call from inside a running event loop — never creates a nested one. For local storage this is plain filesystem access; for S3 and other async backends the backend's ``exists()`` is awaited. """ backend = get_storage_backend() if isinstance(backend, LocalStorage): full_path = backend._full_path(path) if not os.path.exists(full_path): return {"size": None, "modified": None, "exists": False} stat = os.stat(full_path) return { "size": stat.st_size, "modified": stat.st_mtime, "exists": True, } # S3 or other async backends — await the backend directly if not await backend.exists(path): return {"size": None, "modified": None, "exists": False} return {"size": None, "modified": None, "exists": True} def get_file_metadata(path: str) -> dict[str, Any]: """Read metadata of a stored file without loading its content. Works with the *local* storage backend without touching the event loop. For S3 and other async-only backends this drives the check through ``asyncio.run``; calling it from inside a running event loop raises ``RuntimeError`` — use :func:`get_file_metadata_async` there instead (ARCH-052). Parameters ---------- path: Relative storage path. Returns ------- dict ``{size, modified, exists}`` — ``exists`` is ``False`` when the file is not found, in which case ``size`` and ``modified`` are ``None``. """ backend = get_storage_backend() if isinstance(backend, LocalStorage): full_path = backend._full_path(path) if not os.path.exists(full_path): return {"size": None, "modified": None, "exists": False} stat = os.stat(full_path) return { "size": stat.st_size, "modified": stat.st_mtime, "exists": True, } # Async-only backend outside a running loop is fine; inside one we # must never build a nested event loop. try: asyncio.get_running_loop() except RuntimeError: pass else: raise RuntimeError( "get_file_metadata() cannot be used with async storage backends " "inside a running event loop — use get_file_metadata_async()" ) return asyncio.run(get_file_metadata_async(path))