leocrm/app/services/import_export_helpers.py

"""Shared import/export helpers — reusable parsing, writing, mapping, validation.

These building blocks are used by import_export_service.py and can be leveraged
by any plugin that needs custom import/export handlers.
"""

from __future__ import annotations

import csv
import io
import json
import logging
import re
from typing import Any

logger = logging.getLogger(__name__)

# ─── Encoding detection ──────────────────────────────────────────────────────

_BOMS = (
    ("utf-8-sig", b"\xef\xbb\xbf"),
    ("utf-16-le", b"\xff\xfe"),
    ("utf-16-be", b"\xfe\xff"),
)


def _detect_encoding(content: bytes) -> str:
    """Detect the most likely text encoding from raw bytes."""
    for encoding, bom in _BOMS:
        if content.startswith(bom):
            return encoding
    # Try UTF-8 first, fall back to latin-1 (never fails)
    try:
        content.decode("utf-8")
        return "utf-8"
    except UnicodeDecodeError:
        return "latin-1"


# ─── Parsers ─────────────────────────────────────────────────────────────────

def parse_csv(content: bytes) -> list[dict[str, str]]:
    """Parse CSV content (bytes) into a list of dicts.

    Handles encoding detection (UTF-8, UTF-8 BOM, UTF-16, latin-1).
    """
    encoding = _detect_encoding(content)
    text = content.decode(encoding)
    reader = csv.DictReader(io.StringIO(text))
    return [dict(row) for row in reader]


def parse_json(content: bytes) -> list[dict]:
    """Parse JSON content (bytes) into a list of dicts.

    Accepts either a JSON array of objects or a single object.
    """
    encoding = _detect_encoding(content)
    text = content.decode(encoding)
    data = json.loads(text)
    if isinstance(data, dict):
        return [data]
    if isinstance(data, list):
        return data
    raise ValueError(f"Expected JSON array or object, got {type(data).__name__}")


def parse_xlsx(content: bytes) -> list[dict[str, str]]:
    """Parse XLSX content (bytes) into a list of dicts.

    Uses openpyxl. First row is treated as headers.
    """
    try:
        import openpyxl
    except ImportError as exc:
        raise ImportError("openpyxl is required for XLSX parsing") from exc

    wb = openpyxl.load_workbook(io.BytesIO(content), read_only=True, data_only=True)
    ws = wb.active
    rows_iter = ws.iter_rows(values_only=True)
    try:
        headers = [str(h).strip() if h is not None else "" for h in next(rows_iter)]
    except StopIteration:
        wb.close()
        return []
    result: list[dict[str, str]] = []
    for row in rows_iter:
        if row is None or all(c is None or str(c).strip() == "" for c in row):
            continue
        row_dict = {}
        for idx, header in enumerate(headers):
            if not header:
                continue
            val = row[idx] if idx < len(row) else None
            row_dict[header] = str(val).strip() if val is not None else ""
        result.append(row_dict)
    wb.close()
    return result


# ─── Writers ─────────────────────────────────────────────────────────────────

def write_csv(rows: list[dict], headers: list[str]) -> bytes:
    """Write a list of dicts to CSV bytes using the given header order."""
    output = io.StringIO()
    writer = csv.DictWriter(output, fieldnames=headers, extrasaction="ignore")
    writer.writeheader()
    for row in rows:
        writer.writerow({h: row.get(h, "") for h in headers})
    return output.getvalue().encode("utf-8")


def write_json(rows: list[dict]) -> bytes:
    """Write a list of dicts to JSON bytes (UTF-8, indented)."""
    return json.dumps(rows, indent=2, ensure_ascii=False, default=str).encode("utf-8")


def write_xlsx(rows: list[dict], headers: list[str]) -> bytes:
    """Write a list of dicts to XLSX bytes using the given header order."""
    try:
        import openpyxl
    except ImportError as exc:
        raise ImportError("openpyxl is required for XLSX writing") from exc

    wb = openpyxl.Workbook()
    ws = wb.active
    ws.append(headers)
    for row in rows:
        ws.append([row.get(h, "") for h in headers])
    buf = io.BytesIO()
    wb.save(buf)
    buf.seek(0)
    data = buf.read()
    wb.close()
    return data


# ─── Field mapping ───────────────────────────────────────────────────────────

def map_fields(row: dict, mapping: dict[str, str]) -> dict:
    """Map source columns to target fields.

    Args:
        row: Source row dict with original column names.
        mapping: Dict of source_column -> target_field.
                 If a source column is not in the row, it is skipped.

    Returns:
        New dict with target field names as keys.
    """
    result: dict[str, Any] = {}
    for source_col, target_field in mapping.items():
        if source_col in row:
            result[target_field] = row[source_col]
    return result


def suggest_mapping(source_columns: list[str], target_fields: list[str]) -> dict[str, str]:
    """Auto-suggest a mapping from source columns to target fields.

    Uses case-insensitive exact match and common alias matching.
    """
    # Common aliases: source column variants -> canonical target
    aliases: dict[str, str] = {
        "first_name": "firstname",
        "last_name": "surname",
        "last_name": "surname",
        "email": "email",
        "email_address": "email",
        "phone": "phone",
        "mobile": "mobile",
        "phone_2": "mobile",
        "position": "function",
        "title": "function",
        "job_title": "function",
        "company": "name",
        "company_name": "name",
        "organization": "name",
        "web": "website",
        "url": "website",
        "homepage": "website",
        "zip": "postalcode",
        "postal_code": "postalcode",
        "zip_code": "postalcode",
        "city": "city",
        "country": "country",
        "department": "department",
        "industry": "industry",
    }

    target_lower = {f.lower(): f for f in target_fields}
    mapping: dict[str, str] = {}
    for col in source_columns:
        col_lower = col.lower().strip()
        # Direct match
        if col_lower in target_lower:
            mapping[col] = target_lower[col_lower]
            continue
        # Alias match
        alias_target = aliases.get(col_lower)
        if alias_target and alias_target in target_lower:
            mapping[col] = target_lower[alias_target]
            continue
        # Fuzzy: target contains source or vice versa
        for tl, tf in target_lower.items():
            if col_lower in tl or tl in col_lower:
                mapping[col] = tf
                break
    return mapping


# ─── Validation ──────────────────────────────────────────────────────────────

# Basic email regex (pragmatic, not RFC-perfect)
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")


def validate_row(
    row: dict,
    required: list[str],
    validators: dict[str, Any] | None = None,
) -> list[str]:
    """Validate a single row against required fields and optional validators.

    Args:
        row: The row dict to validate.
        required: List of field names that must be non-empty.
        validators: Optional dict of field_name -> validator spec.
            Supported specs:
              - {"type": "email"} — must match email pattern
              - {"type": "url"} — must look like a URL
              - {"type": "int"} — must be parseable as integer
              - {"max_length": N} — string length <= N
              - {"min_length": N} — string length >= N

    Returns:
        List of error message strings (empty if valid).
    """
    errors: list[str] = []
    for field in required:
        val = str(row.get(field, "")).strip()
        if not val:
            errors.append(f"Missing required field: {field}")

    if validators:
        for field, spec in validators.items():
            val = str(row.get(field, "")).strip()
            if not val:
                continue  # Skip empty optional fields
            vtype = spec.get("type")
            if vtype == "email" and not _EMAIL_RE.match(val):
                errors.append(f"Invalid email format in field: {field}")
            elif vtype == "url" and not (val.startswith("http://") or val.startswith("https://")):
                errors.append(f"Invalid URL format in field: {field}")
            elif vtype == "int":
                try:
                    int(val)
                except ValueError:
                    errors.append(f"Field {field} must be an integer")
            max_len = spec.get("max_length")
            if max_len and len(val) > max_len:
                errors.append(f"Field {field} exceeds max length {max_len}")
            min_len = spec.get("min_length")
            if min_len and len(val) < min_len:
                errors.append(f"Field {field} below min length {min_len}")
    return errors


# ─── Error report ────────────────────────────────────────────────────────────

def build_error_report(errors: list[dict]) -> dict:
    """Build a structured error report from a list of per-row errors.

    Args:
        errors: List of dicts, each with keys:
            - row: int (1-based row number)
            - field: str (optional, which field failed)
            - message: str (error description)

    Returns:
        Dict with:
            - total_errors: int
            - errors: list of the input dicts (normalized)
    """
    normalized: list[dict] = []
    for e in errors:
        normalized.append({
            "row": e.get("row", 0),
            "field": e.get("field", ""),
            "message": e.get("message", e.get("error", "")),
        })
    return {
        "total_errors": len(normalized),
        "errors": normalized,
    }


def build_import_result(
    total: int,
    succeeded: int,
    failed: int,
    errors: list[dict] | None = None,
    created: list[dict] | None = None,
    dry_run: bool = False,
) -> dict[str, Any]:
    """Build a standardized import result dict with error report."""
    error_report = build_error_report(errors or [])
    status = "success"
    if failed > 0 and succeeded > 0:
        status = "partial_success"
    elif failed > 0 and succeeded == 0:
        status = "failed"
    return {
        "total": total,
        "succeeded": succeeded,
        "failed": failed,
        "status": status,
        "dry_run": dry_run,
        "error_report": error_report,
        "created": created or [],
    }


# ─── Format detection ────────────────────────────────────────────────────────

def detect_format(filename: str, content: bytes) -> str:
    """Detect file format from filename extension and content magic bytes."""
    name_lower = filename.lower()
    if name_lower.endswith(".csv"):
        return "csv"
    if name_lower.endswith(".json"):
        return "json"
    if name_lower.endswith(".xlsx") or name_lower.endswith(".xls"):
        return "xlsx"
    # Magic bytes
    if content.startswith(b"PK\x03\x04"):
        return "xlsx"  # XLSX is a ZIP archive
    if content.lstrip().startswith(b"[") or content.lstrip().startswith(b"{"):
        return "json"
    return "csv"  # Default fallback


def parse_file(filename: str, content: bytes) -> list[dict[str, str]]:
    """Auto-detect format and parse file content into list of dicts."""
    fmt = detect_format(filename, content)
    if fmt == "csv":
        return parse_csv(content)
    if fmt == "json":
        return parse_json(content)
    if fmt == "xlsx":
        return parse_xlsx(content)
    raise ValueError(f"Unsupported file format: {fmt}")