"""Import/export orchestrator — generic engine delegating to plugin contracts. W4a (Spec #359): The core orchestrator owns the format mechanics (via the format registry provided by format plugins) and the security policy (sensitive-data filter, tenant scoping, audit). The domain logic (columns, normalization, validation, persistence) lives in the owning plugin's contract contribution (``importexport_entities()`` + ``ie_*`` methods). New plugins can offer import/export for their entities by contributing the contract methods — without a single core change. Function signatures are preserved from the former contact-specific service so existing callers and the test suite continue to work unchanged. """ from __future__ import annotations import logging from typing import Any from sqlalchemy.ext.asyncio import AsyncSession from app.plugins.builtins.contracts import get_contract from app.plugins.registry import get_registry from app.services import import_export_helpers as helpers logger = logging.getLogger(__name__) # Threshold for background job processing (rows above this go to ARQ) BACKGROUND_JOB_THRESHOLD = 1000 def _find_ie_contract(entity_type: str): """Find the plugin contract offering import/export for ``entity_type``. Iterates all discovered plugins generically — no hardcoded plugin names. """ for plugin_name in get_registry().list_discovered(): contract = get_contract(plugin_name) entities_fn = getattr(contract, "importexport_entities", None) if contract else None if entities_fn is None: continue if entity_type in entities_fn(): return contract return None def _parse_csv(content: str | bytes) -> list[dict[str, str]]: """Parse file content into list of dicts (delegates to helpers with format detection). Despite the name, this handles CSV, JSON, and XLSX formats by detecting from content. This maintains backward compatibility with existing callers. """ if isinstance(content, str): content = content.encode("utf-8") fmt = helpers.detect_format("upload", content) if fmt == "json": return helpers.parse_json(content) elif fmt == "xlsx": return helpers.parse_xlsx(content) return helpers.parse_csv(content) def _validate_row(row: dict[str, str], required: list[str]) -> list[str]: """Validate a single row (delegates to helpers for backward compat).""" return helpers.validate_row(row, required) # ─── Import (generic, contract-driven) ────────────────────────────────────── async def _generic_import( db: AsyncSession, tenant_id: Any, user_id: Any, csv_content: str | bytes, entity_type: str, dry_run: bool = False, field_mapping: dict[str, str] | None = None, ) -> dict[str, Any]: """Generic import flow: parse → normalize → validate → persist. Domain logic (columns, normalizers, validators, persistence) comes from the plugin contract; the orchestrator handles flow control and policy. """ contract = _find_ie_contract(entity_type) if contract is None: return helpers.build_import_result( total=0, succeeded=0, failed=1, errors=[{"row": 0, "field": "", "message": f"Unknown entity_type: {entity_type}"}], ) rows = _parse_csv(csv_content) total = len(rows) errors: list[dict] = [] valid_rows: list[dict] = [] for idx, row in enumerate(rows, start=1): if field_mapping: row = helpers.map_fields(row, field_mapping) row = contract.ie_normalize_row(entity_type, row) ok, err_msg = contract.ie_row_valid(entity_type, row) if not ok: errors.append({"row": idx, "field": "", "message": err_msg}) continue row_errors = helpers.validate_row( row, contract.ie_required(entity_type), contract.ie_validators(entity_type) ) if row_errors: for e in row_errors: errors.append({"row": idx, "field": "", "message": e}) else: valid_rows.append(row) failed_row_count = len({e["row"] for e in errors}) if dry_run: return helpers.build_import_result( total=total, succeeded=len(valid_rows), failed=failed_row_count, errors=errors, dry_run=True, ) created: list[dict] = [] failed: list[dict] = [] succeeded = 0 for idx, row in enumerate(valid_rows, start=1): try: record = await contract.ie_persist_row(db, tenant_id, user_id, entity_type, row) created.append(record) succeeded += 1 except Exception as exc: logger.warning("Import %s: row %d failed: %s", entity_type, idx, exc) await db.rollback() failed.append({"row": idx, "field": "", "message": str(exc)}) if succeeded > 0: try: await db.commit() except Exception as exc: logger.error("Import %s: commit failed: %s", entity_type, exc) await db.rollback() return helpers.build_import_result( total=total, succeeded=0, failed=len(valid_rows), errors=errors + [{"row": 0, "field": "", "message": f"Commit failed: {exc}"}], created=[], ) all_failed_rows = {e["row"] for e in errors} | {f["row"] for f in failed} return helpers.build_import_result( total=total, succeeded=succeeded, failed=len(all_failed_rows), errors=errors + failed, created=created, ) async def import_companies( db: AsyncSession, tenant_id: Any, user_id: Any, csv_content: str | bytes, dry_run: bool = False, field_mapping: dict[str, str] | None = None, ) -> dict[str, Any]: """Import companies from CSV (delegates to the owning plugin's contract).""" return await _generic_import( db, tenant_id, user_id, csv_content, "companies", dry_run=dry_run, field_mapping=field_mapping, ) async def import_contacts( db: AsyncSession, tenant_id: Any, user_id: Any, csv_content: str | bytes, dry_run: bool = False, field_mapping: dict[str, str] | None = None, ) -> dict[str, Any]: """Import contacts from CSV (delegates to the owning plugin's contract).""" return await _generic_import( db, tenant_id, user_id, csv_content, "contacts", dry_run=dry_run, field_mapping=field_mapping, ) async def import_csv( db: AsyncSession, tenant_id: Any, user_id: Any, csv_content: str | bytes, entity_type: str, dry_run: bool = False, field_mapping: dict[str, str] | None = None, ) -> dict[str, Any]: """Generic import dispatcher based on entity_type.""" return await _generic_import( db, tenant_id, user_id, csv_content, entity_type, dry_run=dry_run, field_mapping=field_mapping, ) # ─── Export (generic, contract-driven) ────────────────────────────────────── async def _generic_export_csv( db: AsyncSession, tenant_id: Any, entity_type: str, user_id: Any = None, is_system_admin: bool = False, ) -> str: """Generic CSV export via the owning plugin's contract.""" contract = _find_ie_contract(entity_type) if contract is None: return helpers.write_csv([], []).decode("utf-8") headers, rows = await contract.ie_fetch_rows( db, tenant_id, entity_type, user_id=user_id, is_system_admin=is_system_admin, ) return helpers.write_csv(rows, headers).decode("utf-8") async def export_contacts_csv( db: AsyncSession, tenant_id: Any, user_id: Any = None, is_system_admin: bool = False, ) -> str: """Export contacts as CSV string (delegates to contacts contract).""" return await _generic_export_csv(db, tenant_id, "contacts", user_id, is_system_admin) async def export_companies_csv( db: AsyncSession, tenant_id: Any, user_id: Any = None, is_system_admin: bool = False, ) -> str: """Export companies as CSV string (delegates to contacts contract).""" return await _generic_export_csv(db, tenant_id, "companies", user_id, is_system_admin) # ─── Preview & Validate (generic, contract-driven) ────────────────────────── def preview_import( filename: str, content: bytes, entity_type: str, ) -> dict[str, Any]: """Parse a file and return preview data (first 10 rows) + mapping suggestion.""" rows = helpers.parse_file(filename, content) columns = list(rows[0].keys()) if rows else [] preview_rows = rows[:10] contract = _find_ie_contract(entity_type) target_fields = contract.ie_target_fields(entity_type) if contract else [] mapping_suggestion = helpers.suggest_mapping(columns, target_fields) return { "total_rows": len(rows), "columns": columns, "preview_rows": preview_rows, "mapping_suggestion": mapping_suggestion, "target_fields": target_fields, } def validate_import( filename: str, content: bytes, entity_type: str, field_mapping: dict[str, str] | None = None, ) -> dict[str, Any]: """Validate all rows against mapping and return error report without importing.""" rows = helpers.parse_file(filename, content) total = len(rows) errors: list[dict] = [] valid_count = 0 contract = _find_ie_contract(entity_type) if contract is None: return helpers.build_import_result( total=total, succeeded=0, failed=total, errors=[{"row": 0, "field": "", "message": f"Unknown entity_type: {entity_type}"}], dry_run=True, ) for idx, row in enumerate(rows, start=1): if field_mapping: row = helpers.map_fields(row, field_mapping) row = contract.ie_normalize_row(entity_type, row) ok, err_msg = contract.ie_row_valid(entity_type, row) if not ok: errors.append({"row": idx, "field": "", "message": err_msg}) continue row_errors = helpers.validate_row( row, contract.ie_required(entity_type), contract.ie_validators(entity_type) ) if row_errors: for e in row_errors: errors.append({"row": idx, "field": "", "message": e}) else: valid_count += 1 failed_row_count = len({e["row"] for e in errors}) return helpers.build_import_result( total=total, succeeded=valid_count, failed=failed_row_count, errors=errors, dry_run=True, )