"""Shared import/export helpers — reusable parsing, writing, mapping, validation. These building blocks are used by import_export_service.py and can be leveraged by any plugin that needs custom import/export handlers. """ from __future__ import annotations import csv import io import json import logging import re from typing import Any logger = logging.getLogger(__name__) # ─── Encoding detection ────────────────────────────────────────────────────── _BOMS = ( ("utf-8-sig", b"\xef\xbb\xbf"), ("utf-16-le", b"\xff\xfe"), ("utf-16-be", b"\xfe\xff"), ) def _detect_encoding(content: bytes) -> str: """Detect the most likely text encoding from raw bytes.""" for encoding, bom in _BOMS: if content.startswith(bom): return encoding # Try UTF-8 first, fall back to latin-1 (never fails) try: content.decode("utf-8") return "utf-8" except UnicodeDecodeError: return "latin-1" # ─── Parsers ───────────────────────────────────────────────────────────────── def parse_csv(content: bytes) -> list[dict[str, str]]: """Parse CSV content (bytes) into a list of dicts. Handles encoding detection (UTF-8, UTF-8 BOM, UTF-16, latin-1). """ encoding = _detect_encoding(content) text = content.decode(encoding) reader = csv.DictReader(io.StringIO(text)) return [dict(row) for row in reader] def parse_json(content: bytes) -> list[dict]: """Parse JSON content (bytes) into a list of dicts. Accepts either a JSON array of objects or a single object. """ encoding = _detect_encoding(content) text = content.decode(encoding) data = json.loads(text) if isinstance(data, dict): return [data] if isinstance(data, list): return data raise ValueError(f"Expected JSON array or object, got {type(data).__name__}") def parse_xlsx(content: bytes) -> list[dict[str, str]]: """Parse XLSX content (bytes) into a list of dicts. Uses openpyxl. First row is treated as headers. """ try: import openpyxl except ImportError as exc: raise ImportError("openpyxl is required for XLSX parsing") from exc wb = openpyxl.load_workbook(io.BytesIO(content), read_only=True, data_only=True) ws = wb.active rows_iter = ws.iter_rows(values_only=True) try: headers = [str(h).strip() if h is not None else "" for h in next(rows_iter)] except StopIteration: wb.close() return [] result: list[dict[str, str]] = [] for row in rows_iter: if row is None or all(c is None or str(c).strip() == "" for c in row): continue row_dict = {} for idx, header in enumerate(headers): if not header: continue val = row[idx] if idx < len(row) else None row_dict[header] = str(val).strip() if val is not None else "" result.append(row_dict) wb.close() return result # ─── Writers ───────────────────────────────────────────────────────────────── def write_csv(rows: list[dict], headers: list[str]) -> bytes: """Write a list of dicts to CSV bytes using the given header order.""" output = io.StringIO() writer = csv.DictWriter(output, fieldnames=headers, extrasaction="ignore") writer.writeheader() for row in rows: writer.writerow({h: row.get(h, "") for h in headers}) return output.getvalue().encode("utf-8") def write_json(rows: list[dict]) -> bytes: """Write a list of dicts to JSON bytes (UTF-8, indented).""" return json.dumps(rows, indent=2, ensure_ascii=False, default=str).encode("utf-8") def write_xlsx(rows: list[dict], headers: list[str]) -> bytes: """Write a list of dicts to XLSX bytes using the given header order.""" try: import openpyxl except ImportError as exc: raise ImportError("openpyxl is required for XLSX writing") from exc wb = openpyxl.Workbook() ws = wb.active ws.append(headers) for row in rows: ws.append([row.get(h, "") for h in headers]) buf = io.BytesIO() wb.save(buf) buf.seek(0) data = buf.read() wb.close() return data # ─── Field mapping ─────────────────────────────────────────────────────────── def map_fields(row: dict, mapping: dict[str, str]) -> dict: """Map source columns to target fields. Args: row: Source row dict with original column names. mapping: Dict of source_column -> target_field. If a source column is not in the row, it is skipped. Returns: New dict with target field names as keys. """ result: dict[str, Any] = {} for source_col, target_field in mapping.items(): if source_col in row: result[target_field] = row[source_col] return result def suggest_mapping(source_columns: list[str], target_fields: list[str]) -> dict[str, str]: """Auto-suggest a mapping from source columns to target fields. Uses case-insensitive exact match and common alias matching. """ # Common aliases: source column variants -> canonical target aliases: dict[str, str] = { "first_name": "firstname", "last_name": "surname", "email": "email", "email_address": "email", "phone": "phone", "mobile": "mobile", "phone_2": "mobile", "position": "function", "title": "function", "job_title": "function", "company": "name", "company_name": "name", "organization": "name", "web": "website", "url": "website", "homepage": "website", "zip": "postalcode", "postal_code": "postalcode", "zip_code": "postalcode", "city": "city", "country": "country", "department": "department", "industry": "industry", } target_lower = {f.lower(): f for f in target_fields} mapping: dict[str, str] = {} for col in source_columns: col_lower = col.lower().strip() # Direct match if col_lower in target_lower: mapping[col] = target_lower[col_lower] continue # Alias match alias_target = aliases.get(col_lower) if alias_target and alias_target in target_lower: mapping[col] = target_lower[alias_target] continue # Fuzzy: target contains source or vice versa for tl, tf in target_lower.items(): if col_lower in tl or tl in col_lower: mapping[col] = tf break return mapping # ─── Validation ────────────────────────────────────────────────────────────── # Basic email regex (pragmatic, not RFC-perfect) _EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") def validate_row( row: dict, required: list[str], validators: dict[str, Any] | None = None, ) -> list[str]: """Validate a single row against required fields and optional validators. Args: row: The row dict to validate. required: List of field names that must be non-empty. validators: Optional dict of field_name -> validator spec. Supported specs: - {"type": "email"} — must match email pattern - {"type": "url"} — must look like a URL - {"type": "int"} — must be parseable as integer - {"max_length": N} — string length <= N - {"min_length": N} — string length >= N Returns: List of error message strings (empty if valid). """ errors: list[str] = [] for field in required: val = str(row.get(field, "")).strip() if not val: errors.append(f"Missing required field: {field}") if validators: for field, spec in validators.items(): val = str(row.get(field, "")).strip() if not val: continue # Skip empty optional fields vtype = spec.get("type") if vtype == "email" and not _EMAIL_RE.match(val): errors.append(f"Invalid email format in field: {field}") elif vtype == "url" and not (val.startswith("http://") or val.startswith("https://")): errors.append(f"Invalid URL format in field: {field}") elif vtype == "int": try: int(val) except ValueError: errors.append(f"Field {field} must be an integer") max_len = spec.get("max_length") if max_len and len(val) > max_len: errors.append(f"Field {field} exceeds max length {max_len}") min_len = spec.get("min_length") if min_len and len(val) < min_len: errors.append(f"Field {field} below min length {min_len}") return errors # ─── Error report ──────────────────────────────────────────────────────────── def build_error_report(errors: list[dict]) -> dict: """Build a structured error report from a list of per-row errors. Args: errors: List of dicts, each with keys: - row: int (1-based row number) - field: str (optional, which field failed) - message: str (error description) Returns: Dict with: - total_errors: int - errors: list of the input dicts (normalized) """ normalized: list[dict] = [] for e in errors: normalized.append({ "row": e.get("row", 0), "field": e.get("field", ""), "message": e.get("message", e.get("error", "")), }) return { "total_errors": len(normalized), "errors": normalized, } def build_import_result( total: int, succeeded: int, failed: int, errors: list[dict] | None = None, created: list[dict] | None = None, dry_run: bool = False, ) -> dict[str, Any]: """Build a standardized import result dict with error report.""" error_report = build_error_report(errors or []) status = "success" if failed > 0 and succeeded > 0: status = "partial_success" elif failed > 0 and succeeded == 0: status = "failed" return { "total": total, "succeeded": succeeded, "failed": failed, "status": status, "dry_run": dry_run, "error_report": error_report, "created": created or [], } # ─── Format detection ──────────────────────────────────────────────────────── def detect_format(filename: str, content: bytes) -> str: """Detect file format from filename extension and content magic bytes.""" name_lower = filename.lower() if name_lower.endswith(".csv"): return "csv" if name_lower.endswith(".json"): return "json" if name_lower.endswith(".xlsx") or name_lower.endswith(".xls"): return "xlsx" # Magic bytes if content.startswith(b"PK\x03\x04"): return "xlsx" # XLSX is a ZIP archive if content.lstrip().startswith(b"[") or content.lstrip().startswith(b"{"): return "json" return "csv" # Default fallback def parse_file(filename: str, content: bytes) -> list[dict[str, str]]: """Auto-detect format and parse file content into list of dicts.""" fmt = detect_format(filename, content) if fmt == "csv": return parse_csv(content) if fmt == "json": return parse_json(content) if fmt == "xlsx": return parse_xlsx(content) raise ValueError(f"Unsupported file format: {fmt}")