leocrm/app/core/storage.py

"""Abstract storage backend — supports local filesystem and S3-compatible storage.

Configuration via environment variables:
- STORAGE_BACKEND: "local" (default) or "s3"
- STORAGE_PATH: Local storage base path (default: /data/uploads)
- S3_ENDPOINT: S3-compatible endpoint URL
- S3_BUCKET: Bucket name
- S3_ACCESS_KEY: Access key
- S3_SECRET_KEY: Secret key
- S3_REGION: Region (default: us-east-1)
- S3_SECURE: Use HTTPS (default: true)

"""

from __future__ import annotations

import asyncio
import hashlib
import io
import logging
import mimetypes
import os
import tempfile
from abc import ABC, abstractmethod
from typing import Any, AsyncIterator

import aiofiles

logger = logging.getLogger(__name__)

# Try to import python-magic for content-based MIME detection
try:
    import magic  # type: ignore

    _HAS_MAGIC = True
except ImportError:
    _HAS_MAGIC = False
    logger.debug("python-magic not installed, falling back to mimetypes")


# Default MIME allowlist — common document, image, and archive types
_DEFAULT_ALLOWED_MIMES: list[str] = [
    # Documents
    "text/plain", "text/html", "text/csv", "text/markdown",
    "application/pdf", "application/msword",
    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
    "application/vnd.ms-excel",
    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
    "application/vnd.ms-powerpoint",
    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
    "application/vnd.oasis.opendocument.text",
    "application/vnd.oasis.opendocument.spreadsheet",
    "application/rtf", "application/json", "application/xml",
    # Images
    "image/jpeg", "image/png", "image/gif", "image/webp",
    "image/svg+xml", "image/tiff", "image/bmp", "image/x-icon",
    # Archives
    "application/zip", "application/x-tar", "application/gzip",
    "application/x-7z-compressed", "application/x-rar-compressed",
    "application/x-bzip2",
    # Other
    "application/octet-stream", "message/rfc822", "application/x-yaml",
    "text/x-yaml",
]


class StorageBackend(ABC):
    """Abstract storage backend for file operations."""

    @abstractmethod
    async def save(self, path: str, data: bytes) -> str:
        """Save data to storage at the given path. Returns the full storage path."""
        ...

    @abstractmethod
    async def save_stream(self, path: str, chunk_aiter: AsyncIterator[bytes]) -> int:
        """Stream chunks to storage. Returns total bytes written."""
        ...

    @abstractmethod
    async def read(self, path: str) -> bytes:
        """Read data from storage at the given path."""
        ...

    @abstractmethod
    async def delete(self, path: str) -> bool:
        """Delete a file from storage. Returns True if deleted, False if not found."""
        ...

    @abstractmethod
    async def exists(self, path: str) -> bool:
        """Check if a file exists in storage."""
        ...

    @abstractmethod
    async def get_url(self, path: str, expires: int = 3600) -> str:
        """Get a URL for accessing the file (presigned URL for S3, file path for local)."""
        ...

    @abstractmethod
    async def list_files(self, prefix: str) -> list[str]:
        """List all file paths under the given prefix."""
        ...


class LocalStorage(StorageBackend):
    """Local filesystem storage backend."""

    def __init__(self, base_path: str | None = None) -> None:
        self.base_path = base_path or os.environ.get("STORAGE_PATH", "/data/uploads")
        os.makedirs(self.base_path, exist_ok=True)

    def _full_path(self, path: str) -> str:
        """Get the full filesystem path with path traversal protection."""
        # Normalize and ensure the path stays within base_path
        full = os.path.normpath(os.path.join(self.base_path, path))
        if not full.startswith(os.path.normpath(self.base_path)):
            raise ValueError(f"Path traversal detected: {path}")
        return full

    async def save(self, path: str, data: bytes) -> str:
        full_path = self._full_path(path)
        os.makedirs(os.path.dirname(full_path), exist_ok=True)
        async with aiofiles.open(full_path, "wb") as f:
            await f.write(data)
        logger.debug("LocalStorage: saved %s (%d bytes)", path, len(data))
        return path

    async def save_stream(self, path: str, chunk_aiter: AsyncIterator[bytes]) -> int:
        """Stream chunks directly to a local file. Returns total bytes written."""
        full_path = self._full_path(path)
        os.makedirs(os.path.dirname(full_path), exist_ok=True)
        total = 0
        async with aiofiles.open(full_path, "wb") as f:
            async for chunk in chunk_aiter:
                await f.write(chunk)
                total += len(chunk)
        logger.debug("LocalStorage: streamed %s (%d bytes)", path, total)
        return total

    async def read(self, path: str) -> bytes:
        full_path = self._full_path(path)
        async with aiofiles.open(full_path, "rb") as f:
            return await f.read()

    async def delete(self, path: str) -> bool:
        full_path = self._full_path(path)
        if os.path.exists(full_path):
            os.remove(full_path)
            return True
        return False

    async def exists(self, path: str) -> bool:
        return os.path.exists(self._full_path(path))

    async def get_url(self, path: str, expires: int = 3600) -> str:
        """Return a relative URL path for the file (not the filesystem path)."""
        # Return a relative path that can be served by the app
        return f"/api/v1/dms/files/{path}"

    async def list_files(self, prefix: str) -> list[str]:
        full_prefix = self._full_path(prefix)
        if not os.path.isdir(full_prefix):
            return []
        result: list[str] = []
        for root, _dirs, files in os.walk(full_prefix):
            for fname in files:
                rel = os.path.relpath(os.path.join(root, fname), self.base_path)
                result.append(rel)
        return result


class S3Storage(StorageBackend):
    """S3-compatible storage backend (works with AWS S3, MinIO, etc.)."""

    def __init__(
        self,
        endpoint: str | None = None,
        bucket: str | None = None,
        access_key: str | None = None,
        secret_key: str | None = None,
        region: str | None = None,
        secure: bool | None = None,
    ) -> None:
        self.endpoint = endpoint or os.environ.get("S3_ENDPOINT", "")
        self.bucket = bucket or os.environ.get("S3_BUCKET", "")
        self.access_key = access_key or os.environ.get("S3_ACCESS_KEY", "")
        self.secret_key = secret_key or os.environ.get("S3_SECRET_KEY", "")
        self.region = region or os.environ.get("S3_REGION", "us-east-1")
        self.secure = secure if secure is not None else os.environ.get("S3_SECURE", "true").lower() == "true"
        self._client: Any = None  # lazy init

    def _get_client(self) -> Any:
        """Lazy-initialize the S3 client (minio or boto3)."""
        if self._client is not None:
            return self._client

        try:
            from minio import Minio  # type: ignore

            self._client = Minio(
                endpoint=self.endpoint.replace("https://", "").replace("http://", ""),
                access_key=self.access_key,
                secret_key=self.secret_key,
                secure=self.secure,
                region=self.region,
            )
            # Ensure bucket exists
            if not self._client.bucket_exists(self.bucket):
                self._client.make_bucket(self.bucket)
            logger.info("S3Storage: connected to %s, bucket=%s", self.endpoint, self.bucket)
            return self._client
        except ImportError:
            logger.error("S3Storage: minio package not installed. Install with: pip install minio")
            raise
        except Exception as e:
            logger.error("S3Storage: failed to connect to %s: %s", self.endpoint, e)
            raise

    # ── Sync helper methods (called via asyncio.to_thread) ──────────────────

    def _save_sync(self, path: str, data: bytes) -> str:
        client = self._get_client()
        client.put_object(
            bucket_name=self.bucket,
            object_name=path,
            data=io.BytesIO(data),
            length=len(data),
        )
        return path

    def _put_file_sync(self, object_name: str, file_path: str) -> str:
        client = self._get_client()
        client.fput_object(self.bucket, object_name, file_path)
        return object_name

    def _read_sync(self, path: str) -> bytes:
        client = self._get_client()
        response = client.get_object(self.bucket, path)
        try:
            return response.read()
        finally:
            response.close()
            response.release_conn()

    def _delete_sync(self, path: str) -> bool:
        client = self._get_client()
        try:
            client.remove_object(self.bucket, path)
            return True
        except Exception:
            return False

    def _exists_sync(self, path: str) -> bool:
        client = self._get_client()
        try:
            client.stat_object(self.bucket, path)
            return True
        except Exception:
            return False

    def _get_url_sync(self, path: str, expires: int) -> str:
        from datetime import timedelta

        client = self._get_client()
        return client.presigned_get_object(self.bucket, path, expires=timedelta(seconds=expires))

    def _list_files_sync(self, prefix: str) -> list[str]:
        client = self._get_client()
        objects = client.list_objects(self.bucket, prefix=prefix, recursive=True)
        return [obj.object_name for obj in objects]

    # ── Async public API (wraps sync calls in asyncio.to_thread) ─────────────

    async def save(self, path: str, data: bytes) -> str:
        result = await asyncio.to_thread(self._save_sync, path, data)
        logger.debug("S3Storage: saved %s (%d bytes)", path, len(data))
        return result

    async def save_stream(self, path: str, chunk_aiter: AsyncIterator[bytes]) -> int:
        """Stream chunks to a temp file, then upload to S3 via fput_object.

        This avoids loading the entire file into RAM. The temp file is
        cleaned up after upload.
        """
        tmp_fd, tmp_path = tempfile.mkstemp(prefix="s3_upload_")
        os.close(tmp_fd)
        total = 0
        try:
            async with aiofiles.open(tmp_path, "wb") as f:
                async for chunk in chunk_aiter:
                    await f.write(chunk)
                    total += len(chunk)
            await asyncio.to_thread(self._put_file_sync, path, tmp_path)
            logger.debug("S3Storage: streamed %s (%d bytes)", path, total)
            return total
        finally:
            if os.path.exists(tmp_path):
                try:
                    os.remove(tmp_path)
                except OSError:
                    logger.warning("S3Storage: failed to clean up temp file %s", tmp_path)

    async def read(self, path: str) -> bytes:
        return await asyncio.to_thread(self._read_sync, path)

    async def delete(self, path: str) -> bool:
        return await asyncio.to_thread(self._delete_sync, path)

    async def exists(self, path: str) -> bool:
        return await asyncio.to_thread(self._exists_sync, path)

    async def get_url(self, path: str, expires: int = 3600) -> str:
        return await asyncio.to_thread(self._get_url_sync, path, expires)

    async def list_files(self, prefix: str) -> list[str]:
        return await asyncio.to_thread(self._list_files_sync, prefix)


# ─── Factory ───

_storage_backend: StorageBackend | None = None


def get_storage_backend() -> StorageBackend:
    """Get the configured storage backend singleton."""
    global _storage_backend
    if _storage_backend is None:
        backend_type = os.environ.get("STORAGE_BACKEND", "local").lower()
        if backend_type == "s3":
            _storage_backend = S3Storage()
            logger.info("Storage backend: S3 (%s)", os.environ.get("S3_ENDPOINT", ""))
        else:
            _storage_backend = LocalStorage()
            logger.info("Storage backend: Local (%s)", os.environ.get("STORAGE_PATH", "/data/uploads"))
    return _storage_backend


def reset_storage_backend() -> None:
    """Reset the storage backend singleton (for testing)."""
    global _storage_backend
    _storage_backend = None


# ─── Validation & Metadata Helpers ───


def validate_mime(
    path: str,
    data: bytes,
    allowed_mimes: list[str] | None = None,
) -> str:
    """Detect the MIME type of *data* and validate it against an allowlist.

    Uses ``python-magic`` for content-based detection when available,
    falling back to ``mimetypes`` (extension-based) otherwise.

    Parameters
    ----------
    path:
        Filename or relative path — used for extension-based fallback.
    data:
        File content bytes used for content-based detection.
    allowed_mimes:
        Allowlist of MIME types.  ``None`` uses the built-in default
        allowlist.  An empty list means *all* MIME types are allowed.

    Returns
    -------
    str
        The detected MIME type.

    Raises
    ------
    ValueError
        If the detected MIME type is not in *allowed_mimes*.
    """
    # --- detect MIME type -------------------------------------------------
    if _HAS_MAGIC:
        try:
            mime_type = magic.from_buffer(data, mime=True)
        except Exception:
            mime_type, _ = mimetypes.guess_type(path)
            mime_type = mime_type or "application/octet-stream"
    else:
        mime_type, _ = mimetypes.guess_type(path)
        mime_type = mime_type or "application/octet-stream"

    # --- validate against allowlist --------------------------------------
    if allowed_mimes is None:
        from app.config import get_settings

        config_mimes = get_settings().storage_allowed_mimes
        if config_mimes:
            allowed = [m.strip() for m in config_mimes.split(",") if m.strip()]
        else:
            allowed = _DEFAULT_ALLOWED_MIMES
    else:
        allowed = allowed_mimes

    if allowed and mime_type not in allowed:
        raise ValueError(
            f"MIME type '{mime_type}' is not allowed. "
            f"Allowed types: {', '.join(allowed[:10])}{'...' if len(allowed) > 10 else ''}"
        )

    return mime_type


def validate_size(data: bytes, max_size_mb: int | None = None) -> None:
    """Validate that *data* does not exceed the configured size limit.

    Parameters
    ----------
    data:
        File content bytes.
    max_size_mb:
        Maximum allowed size in megabytes.  ``None`` reads the value
        from the ``STORAGE_MAX_FILE_SIZE_MB`` environment variable
        (default: 50).

    Raises
    ------
    ValueError
        If ``len(data)`` exceeds ``max_size_mb * 1024 * 1024``.
    """
    if max_size_mb is None:
        from app.config import get_settings

        max_size_mb = get_settings().storage_max_file_size_mb

    max_bytes = max_size_mb * 1024 * 1024
    if len(data) > max_bytes:
        raise ValueError(
            f"File size {len(data)} bytes exceeds limit of {max_size_mb} MB ({max_bytes} bytes)"
        )


def compute_hash(data: bytes, algorithm: str = "sha256") -> str:
    """Compute a cryptographic hash of *data*.

    Parameters
    ----------
    data:
        Content to hash.
    algorithm:
        Hash algorithm name (e.g. ``"sha256"``, ``"md5"``, ``"sha1"``).

    Returns
    -------
    str
        Hexadecimal digest string.
    """
    h = hashlib.new(algorithm)
    h.update(data)
    return h.hexdigest()


async def save_with_metadata(
    path: str,
    data: bytes,
    allowed_mimes: list[str] | None = None,
    max_size_mb: int | None = None,
) -> dict[str, Any]:
    """Save *data* to storage with full validation and metadata extraction.

    Combines :func:`validate_size`, :func:`validate_mime`,
    :func:`compute_hash` and :meth:`StorageBackend.save` into a single
    call.

    Parameters
    ----------
    path:
        Relative storage path.
    data:
        File content bytes.
    allowed_mimes:
        MIME allowlist — ``None`` uses the default allowlist.
    max_size_mb:
        Size limit in MB — ``None`` reads from config.

    Returns
    -------
    dict
        ``{path, mime_type, size, hash, storage_path}``
    """
    validate_size(data, max_size_mb)
    mime_type = validate_mime(path, data, allowed_mimes)
    file_hash = compute_hash(data)

    backend = get_storage_backend()
    storage_path = await backend.save(path, data)

    return {
        "path": path,
        "mime_type": mime_type,
        "size": len(data),
        "hash": file_hash,
        "storage_path": storage_path,
    }


def get_file_metadata(path: str) -> dict[str, Any]:
    """Read metadata of a stored file without loading its content.

    Works with the *local* storage backend.  For S3, use the S3 client
    ``stat_object`` API directly.

    Parameters
    ----------
    path:
        Relative storage path.

    Returns
    -------
    dict
        ``{size, modified, exists}`` — ``exists`` is ``False`` when the
        file is not found, in which case ``size`` and ``modified`` are
        ``None``.
    """
    backend = get_storage_backend()
    if isinstance(backend, LocalStorage):
        full_path = backend._full_path(path)
        if not os.path.exists(full_path):
            return {"size": None, "modified": None, "exists": False}
        stat = os.stat(full_path)
        return {
            "size": stat.st_size,
            "modified": stat.st_mtime,
            "exists": True,
        }
    # S3 or other backends — fall back to exists() check
    import asyncio as _asyncio

    loop = _asyncio.new_event_loop()
    try:
        exists = loop.run_until_complete(backend.exists(path))
        if not exists:
            return {"size": None, "modified": None, "exists": False}
        return {"size": None, "modified": None, "exists": True}
    finally:
        loop.close()