leocrm/app/services/import_export_service.py

"""Import/export orchestrator — generic engine delegating to plugin contracts.

W4a (Spec #359): The core orchestrator owns the format mechanics (via the
format registry provided by format plugins) and the security policy
(sensitive-data filter, tenant scoping, audit). The domain logic (columns,
normalization, validation, persistence) lives in the owning plugin's
contract contribution (``importexport_entities()`` + ``ie_*`` methods).

New plugins can offer import/export for their entities by contributing the
contract methods — without a single core change.

Function signatures are preserved from the former contact-specific service
so existing callers and the test suite continue to work unchanged.
"""

from __future__ import annotations

import logging
from typing import Any

from sqlalchemy.ext.asyncio import AsyncSession

from app.plugins.builtins.contracts import get_contract
from app.plugins.registry import get_registry
from app.services import import_export_helpers as helpers

logger = logging.getLogger(__name__)

# Threshold for background job processing (rows above this go to ARQ)
BACKGROUND_JOB_THRESHOLD = 1000


def _find_ie_contract(entity_type: str):
    """Find the plugin contract offering import/export for ``entity_type``.

    Iterates all discovered plugins generically — no hardcoded plugin names.
    """
    for plugin_name in get_registry().list_discovered():
        contract = get_contract(plugin_name)
        entities_fn = getattr(contract, "importexport_entities", None) if contract else None
        if entities_fn is None:
            continue
        if entity_type in entities_fn():
            return contract
    return None


def _parse_csv(content: str | bytes) -> list[dict[str, str]]:
    """Parse file content into list of dicts (delegates to helpers with format detection).

    Despite the name, this handles CSV, JSON, and XLSX formats by detecting
    from content.  This maintains backward compatibility with existing callers.
    """
    if isinstance(content, str):
        content = content.encode("utf-8")
    fmt = helpers.detect_format("upload", content)
    if fmt == "json":
        return helpers.parse_json(content)
    elif fmt == "xlsx":
        return helpers.parse_xlsx(content)
    return helpers.parse_csv(content)


def _validate_row(row: dict[str, str], required: list[str]) -> list[str]:
    """Validate a single row (delegates to helpers for backward compat)."""
    return helpers.validate_row(row, required)


# ─── Import (generic, contract-driven) ──────────────────────────────────────

async def _generic_import(
    db: AsyncSession,
    tenant_id: Any,
    user_id: Any,
    csv_content: str | bytes,
    entity_type: str,
    dry_run: bool = False,
    field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
    """Generic import flow: parse → normalize → validate → persist.

    Domain logic (columns, normalizers, validators, persistence) comes from
    the plugin contract; the orchestrator handles flow control and policy.
    """
    contract = _find_ie_contract(entity_type)
    if contract is None:
        return helpers.build_import_result(
            total=0,
            succeeded=0,
            failed=1,
            errors=[{"row": 0, "field": "", "message": f"Unknown entity_type: {entity_type}"}],
        )

    rows = _parse_csv(csv_content)
    total = len(rows)
    errors: list[dict] = []
    valid_rows: list[dict] = []

    for idx, row in enumerate(rows, start=1):
        if field_mapping:
            row = helpers.map_fields(row, field_mapping)
        row = contract.ie_normalize_row(entity_type, row)
        ok, err_msg = contract.ie_row_valid(entity_type, row)
        if not ok:
            errors.append({"row": idx, "field": "", "message": err_msg})
            continue
        row_errors = helpers.validate_row(
            row, contract.ie_required(entity_type), contract.ie_validators(entity_type)
        )
        if row_errors:
            for e in row_errors:
                errors.append({"row": idx, "field": "", "message": e})
        else:
            valid_rows.append(row)

    failed_row_count = len({e["row"] for e in errors})

    if dry_run:
        return helpers.build_import_result(
            total=total,
            succeeded=len(valid_rows),
            failed=failed_row_count,
            errors=errors,
            dry_run=True,
        )

    created: list[dict] = []
    failed: list[dict] = []
    succeeded = 0

    for idx, row in enumerate(valid_rows, start=1):
        try:
            record = await contract.ie_persist_row(db, tenant_id, user_id, entity_type, row)
            created.append(record)
            succeeded += 1
        except Exception as exc:
            logger.warning("Import %s: row %d failed: %s", entity_type, idx, exc)
            await db.rollback()
            failed.append({"row": idx, "field": "", "message": str(exc)})

    if succeeded > 0:
        try:
            await db.commit()
        except Exception as exc:
            logger.error("Import %s: commit failed: %s", entity_type, exc)
            await db.rollback()
            return helpers.build_import_result(
                total=total,
                succeeded=0,
                failed=len(valid_rows),
                errors=errors + [{"row": 0, "field": "", "message": f"Commit failed: {exc}"}],
                created=[],
            )

    all_failed_rows = {e["row"] for e in errors} | {f["row"] for f in failed}
    return helpers.build_import_result(
        total=total,
        succeeded=succeeded,
        failed=len(all_failed_rows),
        errors=errors + failed,
        created=created,
    )


async def import_companies(
    db: AsyncSession,
    tenant_id: Any,
    user_id: Any,
    csv_content: str | bytes,
    dry_run: bool = False,
    field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
    """Import companies from CSV (delegates to the owning plugin's contract)."""
    return await _generic_import(
        db, tenant_id, user_id, csv_content, "companies",
        dry_run=dry_run, field_mapping=field_mapping,
    )


async def import_contacts(
    db: AsyncSession,
    tenant_id: Any,
    user_id: Any,
    csv_content: str | bytes,
    dry_run: bool = False,
    field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
    """Import contacts from CSV (delegates to the owning plugin's contract)."""
    return await _generic_import(
        db, tenant_id, user_id, csv_content, "contacts",
        dry_run=dry_run, field_mapping=field_mapping,
    )


async def import_csv(
    db: AsyncSession,
    tenant_id: Any,
    user_id: Any,
    csv_content: str | bytes,
    entity_type: str,
    dry_run: bool = False,
    field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
    """Generic import dispatcher based on entity_type."""
    return await _generic_import(
        db, tenant_id, user_id, csv_content, entity_type,
        dry_run=dry_run, field_mapping=field_mapping,
    )


# ─── Export (generic, contract-driven) ──────────────────────────────────────

async def _generic_export_csv(
    db: AsyncSession,
    tenant_id: Any,
    entity_type: str,
    user_id: Any = None,
    is_system_admin: bool = False,
) -> str:
    """Generic CSV export via the owning plugin's contract."""
    contract = _find_ie_contract(entity_type)
    if contract is None:
        return helpers.write_csv([], []).decode("utf-8")
    headers, rows = await contract.ie_fetch_rows(
        db, tenant_id, entity_type, user_id=user_id, is_system_admin=is_system_admin,
    )
    return helpers.write_csv(rows, headers).decode("utf-8")


async def export_contacts_csv(
    db: AsyncSession,
    tenant_id: Any,
    user_id: Any = None,
    is_system_admin: bool = False,
) -> str:
    """Export contacts as CSV string (delegates to contacts contract)."""
    return await _generic_export_csv(db, tenant_id, "contacts", user_id, is_system_admin)


async def export_companies_csv(
    db: AsyncSession,
    tenant_id: Any,
    user_id: Any = None,
    is_system_admin: bool = False,
) -> str:
    """Export companies as CSV string (delegates to contacts contract)."""
    return await _generic_export_csv(db, tenant_id, "companies", user_id, is_system_admin)


# ─── Preview & Validate (generic, contract-driven) ──────────────────────────

def preview_import(
    filename: str,
    content: bytes,
    entity_type: str,
) -> dict[str, Any]:
    """Parse a file and return preview data (first 10 rows) + mapping suggestion."""
    rows = helpers.parse_file(filename, content)
    columns = list(rows[0].keys()) if rows else []
    preview_rows = rows[:10]

    contract = _find_ie_contract(entity_type)
    target_fields = contract.ie_target_fields(entity_type) if contract else []
    mapping_suggestion = helpers.suggest_mapping(columns, target_fields)

    return {
        "total_rows": len(rows),
        "columns": columns,
        "preview_rows": preview_rows,
        "mapping_suggestion": mapping_suggestion,
        "target_fields": target_fields,
    }


def validate_import(
    filename: str,
    content: bytes,
    entity_type: str,
    field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
    """Validate all rows against mapping and return error report without importing."""
    rows = helpers.parse_file(filename, content)
    total = len(rows)
    errors: list[dict] = []
    valid_count = 0

    contract = _find_ie_contract(entity_type)
    if contract is None:
        return helpers.build_import_result(
            total=total,
            succeeded=0,
            failed=total,
            errors=[{"row": 0, "field": "", "message": f"Unknown entity_type: {entity_type}"}],
            dry_run=True,
        )

    for idx, row in enumerate(rows, start=1):
        if field_mapping:
            row = helpers.map_fields(row, field_mapping)
        row = contract.ie_normalize_row(entity_type, row)
        ok, err_msg = contract.ie_row_valid(entity_type, row)
        if not ok:
            errors.append({"row": idx, "field": "", "message": err_msg})
            continue
        row_errors = helpers.validate_row(
            row, contract.ie_required(entity_type), contract.ie_validators(entity_type)
        )
        if row_errors:
            for e in row_errors:
                errors.append({"row": idx, "field": "", "message": e})
        else:
            valid_count += 1

    failed_row_count = len({e["row"] for e in errors})
    return helpers.build_import_result(
        total=total,
        succeeded=valid_count,
        failed=failed_row_count,
        errors=errors,
        dry_run=True,
    )