feat(C.5): Modularer Import/Export — Shared Helpers, Preview/Mapping, Background Jobs, Partial-Failure
C5-BASE: app/services/import_export_helpers.py (NEU, 352 Zeilen)
- parse_csv/json/xlsx, write_csv/json/xlsx, map_fields, suggest_mapping
- validate_row, build_error_report, build_import_result, detect_format
C5-PREVIEW: POST /import/preview + POST /import/validate
- Preview gibt erste 10 Zeilen + Spalten + Mapping-Vorschlag
- Validate gibt Fehler-Report ohne Import
C5-JOB: app/services/import_export_jobs.py (NEU, 165 Zeilen)
- ARQ Background Job für Files >1000 Zeilen
- Job-Status: pending/processing/completed/partial_success/failed
- GET /import/status/{job_id} — Status + Progress + Fehler-Report
- Partial-Failure: try/except pro Zeile, fehlerhafte gesammelt, erfolgreiche committet
C5-CONTACT+C5-COMPANY: Handler auf Shared Helpers umgestellt
C5-UI: ImportWizard.tsx (5 Steps: Upload→Preview→Validation→Review→Result)
C5-TEST: 45 Tests in test_import_export.py — alle grün
C5-DOC: Plugin-Dev-Guide Kapitel 31 (Import/Export Handler)
This commit is contained in:
Agent Zero
@@ -0,0 +1,352 @@
|
||||
"""Shared import/export helpers — reusable parsing, writing, mapping, validation.
|
||||
|
||||
These building blocks are used by import_export_service.py and can be leveraged
|
||||
by any plugin that needs custom import/export handlers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ─── Encoding detection ──────────────────────────────────────────────────────
|
||||
|
||||
_BOMS = (
|
||||
("utf-8-sig", b"\xef\xbb\xbf"),
|
||||
("utf-16-le", b"\xff\xfe"),
|
||||
("utf-16-be", b"\xfe\xff"),
|
||||
)
|
||||
|
||||
|
||||
def _detect_encoding(content: bytes) -> str:
|
||||
"""Detect the most likely text encoding from raw bytes."""
|
||||
for encoding, bom in _BOMS:
|
||||
if content.startswith(bom):
|
||||
return encoding
|
||||
# Try UTF-8 first, fall back to latin-1 (never fails)
|
||||
try:
|
||||
content.decode("utf-8")
|
||||
return "utf-8"
|
||||
except UnicodeDecodeError:
|
||||
return "latin-1"
|
||||
|
||||
|
||||
# ─── Parsers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def parse_csv(content: bytes) -> list[dict[str, str]]:
|
||||
"""Parse CSV content (bytes) into a list of dicts.
|
||||
|
||||
Handles encoding detection (UTF-8, UTF-8 BOM, UTF-16, latin-1).
|
||||
"""
|
||||
encoding = _detect_encoding(content)
|
||||
text = content.decode(encoding)
|
||||
reader = csv.DictReader(io.StringIO(text))
|
||||
return [dict(row) for row in reader]
|
||||
|
||||
|
||||
def parse_json(content: bytes) -> list[dict]:
|
||||
"""Parse JSON content (bytes) into a list of dicts.
|
||||
|
||||
Accepts either a JSON array of objects or a single object.
|
||||
"""
|
||||
encoding = _detect_encoding(content)
|
||||
text = content.decode(encoding)
|
||||
data = json.loads(text)
|
||||
if isinstance(data, dict):
|
||||
return [data]
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
raise ValueError(f"Expected JSON array or object, got {type(data).__name__}")
|
||||
|
||||
|
||||
def parse_xlsx(content: bytes) -> list[dict[str, str]]:
|
||||
"""Parse XLSX content (bytes) into a list of dicts.
|
||||
|
||||
Uses openpyxl. First row is treated as headers.
|
||||
"""
|
||||
try:
|
||||
import openpyxl
|
||||
except ImportError as exc:
|
||||
raise ImportError("openpyxl is required for XLSX parsing") from exc
|
||||
|
||||
wb = openpyxl.load_workbook(io.BytesIO(content), read_only=True, data_only=True)
|
||||
ws = wb.active
|
||||
rows_iter = ws.iter_rows(values_only=True)
|
||||
try:
|
||||
headers = [str(h).strip() if h is not None else "" for h in next(rows_iter)]
|
||||
except StopIteration:
|
||||
wb.close()
|
||||
return []
|
||||
result: list[dict[str, str]] = []
|
||||
for row in rows_iter:
|
||||
if row is None or all(c is None or str(c).strip() == "" for c in row):
|
||||
continue
|
||||
row_dict = {}
|
||||
for idx, header in enumerate(headers):
|
||||
if not header:
|
||||
continue
|
||||
val = row[idx] if idx < len(row) else None
|
||||
row_dict[header] = str(val).strip() if val is not None else ""
|
||||
result.append(row_dict)
|
||||
wb.close()
|
||||
return result
|
||||
|
||||
|
||||
# ─── Writers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def write_csv(rows: list[dict], headers: list[str]) -> bytes:
|
||||
"""Write a list of dicts to CSV bytes using the given header order."""
|
||||
output = io.StringIO()
|
||||
writer = csv.DictWriter(output, fieldnames=headers, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
writer.writerow({h: row.get(h, "") for h in headers})
|
||||
return output.getvalue().encode("utf-8")
|
||||
|
||||
|
||||
def write_json(rows: list[dict]) -> bytes:
|
||||
"""Write a list of dicts to JSON bytes (UTF-8, indented)."""
|
||||
return json.dumps(rows, indent=2, ensure_ascii=False, default=str).encode("utf-8")
|
||||
|
||||
|
||||
def write_xlsx(rows: list[dict], headers: list[str]) -> bytes:
|
||||
"""Write a list of dicts to XLSX bytes using the given header order."""
|
||||
try:
|
||||
import openpyxl
|
||||
except ImportError as exc:
|
||||
raise ImportError("openpyxl is required for XLSX writing") from exc
|
||||
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.append(headers)
|
||||
for row in rows:
|
||||
ws.append([row.get(h, "") for h in headers])
|
||||
buf = io.BytesIO()
|
||||
wb.save(buf)
|
||||
buf.seek(0)
|
||||
data = buf.read()
|
||||
wb.close()
|
||||
return data
|
||||
|
||||
|
||||
# ─── Field mapping ───────────────────────────────────────────────────────────
|
||||
|
||||
def map_fields(row: dict, mapping: dict[str, str]) -> dict:
|
||||
"""Map source columns to target fields.
|
||||
|
||||
Args:
|
||||
row: Source row dict with original column names.
|
||||
mapping: Dict of source_column -> target_field.
|
||||
If a source column is not in the row, it is skipped.
|
||||
|
||||
Returns:
|
||||
New dict with target field names as keys.
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
for source_col, target_field in mapping.items():
|
||||
if source_col in row:
|
||||
result[target_field] = row[source_col]
|
||||
return result
|
||||
|
||||
|
||||
def suggest_mapping(source_columns: list[str], target_fields: list[str]) -> dict[str, str]:
|
||||
"""Auto-suggest a mapping from source columns to target fields.
|
||||
|
||||
Uses case-insensitive exact match and common alias matching.
|
||||
"""
|
||||
# Common aliases: source column variants -> canonical target
|
||||
aliases: dict[str, str] = {
|
||||
"first_name": "firstname",
|
||||
"last_name": "surname",
|
||||
"last_name": "surname",
|
||||
"email": "email",
|
||||
"email_address": "email",
|
||||
"phone": "phone",
|
||||
"mobile": "mobile",
|
||||
"phone_2": "mobile",
|
||||
"position": "function",
|
||||
"title": "function",
|
||||
"job_title": "function",
|
||||
"company": "name",
|
||||
"company_name": "name",
|
||||
"organization": "name",
|
||||
"web": "website",
|
||||
"url": "website",
|
||||
"homepage": "website",
|
||||
"zip": "postalcode",
|
||||
"postal_code": "postalcode",
|
||||
"zip_code": "postalcode",
|
||||
"city": "city",
|
||||
"country": "country",
|
||||
"department": "department",
|
||||
"industry": "industry",
|
||||
}
|
||||
|
||||
target_lower = {f.lower(): f for f in target_fields}
|
||||
mapping: dict[str, str] = {}
|
||||
for col in source_columns:
|
||||
col_lower = col.lower().strip()
|
||||
# Direct match
|
||||
if col_lower in target_lower:
|
||||
mapping[col] = target_lower[col_lower]
|
||||
continue
|
||||
# Alias match
|
||||
alias_target = aliases.get(col_lower)
|
||||
if alias_target and alias_target in target_lower:
|
||||
mapping[col] = target_lower[alias_target]
|
||||
continue
|
||||
# Fuzzy: target contains source or vice versa
|
||||
for tl, tf in target_lower.items():
|
||||
if col_lower in tl or tl in col_lower:
|
||||
mapping[col] = tf
|
||||
break
|
||||
return mapping
|
||||
|
||||
|
||||
# ─── Validation ──────────────────────────────────────────────────────────────
|
||||
|
||||
# Basic email regex (pragmatic, not RFC-perfect)
|
||||
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||||
|
||||
|
||||
def validate_row(
|
||||
row: dict,
|
||||
required: list[str],
|
||||
validators: dict[str, Any] | None = None,
|
||||
) -> list[str]:
|
||||
"""Validate a single row against required fields and optional validators.
|
||||
|
||||
Args:
|
||||
row: The row dict to validate.
|
||||
required: List of field names that must be non-empty.
|
||||
validators: Optional dict of field_name -> validator spec.
|
||||
Supported specs:
|
||||
- {"type": "email"} — must match email pattern
|
||||
- {"type": "url"} — must look like a URL
|
||||
- {"type": "int"} — must be parseable as integer
|
||||
- {"max_length": N} — string length <= N
|
||||
- {"min_length": N} — string length >= N
|
||||
|
||||
Returns:
|
||||
List of error message strings (empty if valid).
|
||||
"""
|
||||
errors: list[str] = []
|
||||
for field in required:
|
||||
val = str(row.get(field, "")).strip()
|
||||
if not val:
|
||||
errors.append(f"Missing required field: {field}")
|
||||
|
||||
if validators:
|
||||
for field, spec in validators.items():
|
||||
val = str(row.get(field, "")).strip()
|
||||
if not val:
|
||||
continue # Skip empty optional fields
|
||||
vtype = spec.get("type")
|
||||
if vtype == "email" and not _EMAIL_RE.match(val):
|
||||
errors.append(f"Invalid email format in field: {field}")
|
||||
elif vtype == "url" and not (val.startswith("http://") or val.startswith("https://")):
|
||||
errors.append(f"Invalid URL format in field: {field}")
|
||||
elif vtype == "int":
|
||||
try:
|
||||
int(val)
|
||||
except ValueError:
|
||||
errors.append(f"Field {field} must be an integer")
|
||||
max_len = spec.get("max_length")
|
||||
if max_len and len(val) > max_len:
|
||||
errors.append(f"Field {field} exceeds max length {max_len}")
|
||||
min_len = spec.get("min_length")
|
||||
if min_len and len(val) < min_len:
|
||||
errors.append(f"Field {field} below min length {min_len}")
|
||||
return errors
|
||||
|
||||
|
||||
# ─── Error report ────────────────────────────────────────────────────────────
|
||||
|
||||
def build_error_report(errors: list[dict]) -> dict:
|
||||
"""Build a structured error report from a list of per-row errors.
|
||||
|
||||
Args:
|
||||
errors: List of dicts, each with keys:
|
||||
- row: int (1-based row number)
|
||||
- field: str (optional, which field failed)
|
||||
- message: str (error description)
|
||||
|
||||
Returns:
|
||||
Dict with:
|
||||
- total_errors: int
|
||||
- errors: list of the input dicts (normalized)
|
||||
"""
|
||||
normalized: list[dict] = []
|
||||
for e in errors:
|
||||
normalized.append({
|
||||
"row": e.get("row", 0),
|
||||
"field": e.get("field", ""),
|
||||
"message": e.get("message", e.get("error", "")),
|
||||
})
|
||||
return {
|
||||
"total_errors": len(normalized),
|
||||
"errors": normalized,
|
||||
}
|
||||
|
||||
|
||||
def build_import_result(
|
||||
total: int,
|
||||
succeeded: int,
|
||||
failed: int,
|
||||
errors: list[dict] | None = None,
|
||||
created: list[dict] | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a standardized import result dict with error report."""
|
||||
error_report = build_error_report(errors or [])
|
||||
status = "success"
|
||||
if failed > 0 and succeeded > 0:
|
||||
status = "partial_success"
|
||||
elif failed > 0 and succeeded == 0:
|
||||
status = "failed"
|
||||
return {
|
||||
"total": total,
|
||||
"succeeded": succeeded,
|
||||
"failed": failed,
|
||||
"status": status,
|
||||
"dry_run": dry_run,
|
||||
"error_report": error_report,
|
||||
"created": created or [],
|
||||
}
|
||||
|
||||
|
||||
# ─── Format detection ────────────────────────────────────────────────────────
|
||||
|
||||
def detect_format(filename: str, content: bytes) -> str:
|
||||
"""Detect file format from filename extension and content magic bytes."""
|
||||
name_lower = filename.lower()
|
||||
if name_lower.endswith(".csv"):
|
||||
return "csv"
|
||||
if name_lower.endswith(".json"):
|
||||
return "json"
|
||||
if name_lower.endswith(".xlsx") or name_lower.endswith(".xls"):
|
||||
return "xlsx"
|
||||
# Magic bytes
|
||||
if content.startswith(b"PK\x03\x04"):
|
||||
return "xlsx" # XLSX is a ZIP archive
|
||||
if content.lstrip().startswith(b"[") or content.lstrip().startswith(b"{"):
|
||||
return "json"
|
||||
return "csv" # Default fallback
|
||||
|
||||
|
||||
def parse_file(filename: str, content: bytes) -> list[dict[str, str]]:
|
||||
"""Auto-detect format and parse file content into list of dicts."""
|
||||
fmt = detect_format(filename, content)
|
||||
if fmt == "csv":
|
||||
return parse_csv(content)
|
||||
if fmt == "json":
|
||||
return parse_json(content)
|
||||
if fmt == "xlsx":
|
||||
return parse_xlsx(content)
|
||||
raise ValueError(f"Unsupported file format: {fmt}")
|
||||
Reference in New Issue
Block a user