352 lines
12 KiB
Python
352 lines
12 KiB
Python
|
"""Shared import/export helpers — reusable parsing, writing, mapping, validation.
|
||
|
|
|
||
|
|
These building blocks are used by import_export_service.py and can be leveraged
|
||
|
|
by any plugin that needs custom import/export handlers.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import csv
|
||
|
|
import io
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import re
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
# ─── Encoding detection ──────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
_BOMS = (
|
||
|
|
("utf-8-sig", b"\xef\xbb\xbf"),
|
||
|
|
("utf-16-le", b"\xff\xfe"),
|
||
|
|
("utf-16-be", b"\xfe\xff"),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _detect_encoding(content: bytes) -> str:
|
||
|
|
"""Detect the most likely text encoding from raw bytes."""
|
||
|
|
for encoding, bom in _BOMS:
|
||
|
|
if content.startswith(bom):
|
||
|
|
return encoding
|
||
|
|
# Try UTF-8 first, fall back to latin-1 (never fails)
|
||
|
|
try:
|
||
|
|
content.decode("utf-8")
|
||
|
|
return "utf-8"
|
||
|
|
except UnicodeDecodeError:
|
||
|
|
return "latin-1"
|
||
|
|
|
||
|
|
|
||
|
|
# ─── Parsers ─────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
def parse_csv(content: bytes) -> list[dict[str, str]]:
|
||
|
|
"""Parse CSV content (bytes) into a list of dicts.
|
||
|
|
|
||
|
|
Handles encoding detection (UTF-8, UTF-8 BOM, UTF-16, latin-1).
|
||
|
|
"""
|
||
|
|
encoding = _detect_encoding(content)
|
||
|
|
text = content.decode(encoding)
|
||
|
|
reader = csv.DictReader(io.StringIO(text))
|
||
|
|
return [dict(row) for row in reader]
|
||
|
|
|
||
|
|
|
||
|
|
def parse_json(content: bytes) -> list[dict]:
|
||
|
|
"""Parse JSON content (bytes) into a list of dicts.
|
||
|
|
|
||
|
|
Accepts either a JSON array of objects or a single object.
|
||
|
|
"""
|
||
|
|
encoding = _detect_encoding(content)
|
||
|
|
text = content.decode(encoding)
|
||
|
|
data = json.loads(text)
|
||
|
|
if isinstance(data, dict):
|
||
|
|
return [data]
|
||
|
|
if isinstance(data, list):
|
||
|
|
return data
|
||
|
|
raise ValueError(f"Expected JSON array or object, got {type(data).__name__}")
|
||
|
|
|
||
|
|
|
||
|
|
def parse_xlsx(content: bytes) -> list[dict[str, str]]:
|
||
|
|
"""Parse XLSX content (bytes) into a list of dicts.
|
||
|
|
|
||
|
|
Uses openpyxl. First row is treated as headers.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
import openpyxl
|
||
|
|
except ImportError as exc:
|
||
|
|
raise ImportError("openpyxl is required for XLSX parsing") from exc
|
||
|
|
|
||
|
|
wb = openpyxl.load_workbook(io.BytesIO(content), read_only=True, data_only=True)
|
||
|
|
ws = wb.active
|
||
|
|
rows_iter = ws.iter_rows(values_only=True)
|
||
|
|
try:
|
||
|
|
headers = [str(h).strip() if h is not None else "" for h in next(rows_iter)]
|
||
|
|
except StopIteration:
|
||
|
|
wb.close()
|
||
|
|
return []
|
||
|
|
result: list[dict[str, str]] = []
|
||
|
|
for row in rows_iter:
|
||
|
|
if row is None or all(c is None or str(c).strip() == "" for c in row):
|
||
|
|
continue
|
||
|
|
row_dict = {}
|
||
|
|
for idx, header in enumerate(headers):
|
||
|
|
if not header:
|
||
|
|
continue
|
||
|
|
val = row[idx] if idx < len(row) else None
|
||
|
|
row_dict[header] = str(val).strip() if val is not None else ""
|
||
|
|
result.append(row_dict)
|
||
|
|
wb.close()
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
# ─── Writers ─────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
def write_csv(rows: list[dict], headers: list[str]) -> bytes:
|
||
|
|
"""Write a list of dicts to CSV bytes using the given header order."""
|
||
|
|
output = io.StringIO()
|
||
|
|
writer = csv.DictWriter(output, fieldnames=headers, extrasaction="ignore")
|
||
|
|
writer.writeheader()
|
||
|
|
for row in rows:
|
||
|
|
writer.writerow({h: row.get(h, "") for h in headers})
|
||
|
|
return output.getvalue().encode("utf-8")
|
||
|
|
|
||
|
|
|
||
|
|
def write_json(rows: list[dict]) -> bytes:
|
||
|
|
"""Write a list of dicts to JSON bytes (UTF-8, indented)."""
|
||
|
|
return json.dumps(rows, indent=2, ensure_ascii=False, default=str).encode("utf-8")
|
||
|
|
|
||
|
|
|
||
|
|
def write_xlsx(rows: list[dict], headers: list[str]) -> bytes:
|
||
|
|
"""Write a list of dicts to XLSX bytes using the given header order."""
|
||
|
|
try:
|
||
|
|
import openpyxl
|
||
|
|
except ImportError as exc:
|
||
|
|
raise ImportError("openpyxl is required for XLSX writing") from exc
|
||
|
|
|
||
|
|
wb = openpyxl.Workbook()
|
||
|
|
ws = wb.active
|
||
|
|
ws.append(headers)
|
||
|
|
for row in rows:
|
||
|
|
ws.append([row.get(h, "") for h in headers])
|
||
|
|
buf = io.BytesIO()
|
||
|
|
wb.save(buf)
|
||
|
|
buf.seek(0)
|
||
|
|
data = buf.read()
|
||
|
|
wb.close()
|
||
|
|
return data
|
||
|
|
|
||
|
|
|
||
|
|
# ─── Field mapping ───────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
def map_fields(row: dict, mapping: dict[str, str]) -> dict:
|
||
|
|
"""Map source columns to target fields.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
row: Source row dict with original column names.
|
||
|
|
mapping: Dict of source_column -> target_field.
|
||
|
|
If a source column is not in the row, it is skipped.
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
New dict with target field names as keys.
|
||
|
|
"""
|
||
|
|
result: dict[str, Any] = {}
|
||
|
|
for source_col, target_field in mapping.items():
|
||
|
|
if source_col in row:
|
||
|
|
result[target_field] = row[source_col]
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def suggest_mapping(source_columns: list[str], target_fields: list[str]) -> dict[str, str]:
|
||
|
|
"""Auto-suggest a mapping from source columns to target fields.
|
||
|
|
|
||
|
|
Uses case-insensitive exact match and common alias matching.
|
||
|
|
"""
|
||
|
|
# Common aliases: source column variants -> canonical target
|
||
|
|
aliases: dict[str, str] = {
|
||
|
|
"first_name": "firstname",
|
||
|
|
"last_name": "surname",
|
||
|
|
"email": "email",
|
||
|
|
"email_address": "email",
|
||
|
|
"phone": "phone",
|
||
|
|
"mobile": "mobile",
|
||
|
|
"phone_2": "mobile",
|
||
|
|
"position": "function",
|
||
|
|
"title": "function",
|
||
|
|
"job_title": "function",
|
||
|
|
"company": "name",
|
||
|
|
"company_name": "name",
|
||
|
|
"organization": "name",
|
||
|
|
"web": "website",
|
||
|
|
"url": "website",
|
||
|
|
"homepage": "website",
|
||
|
|
"zip": "postalcode",
|
||
|
|
"postal_code": "postalcode",
|
||
|
|
"zip_code": "postalcode",
|
||
|
|
"city": "city",
|
||
|
|
"country": "country",
|
||
|
|
"department": "department",
|
||
|
|
"industry": "industry",
|
||
|
|
}
|
||
|
|
|
||
|
|
target_lower = {f.lower(): f for f in target_fields}
|
||
|
|
mapping: dict[str, str] = {}
|
||
|
|
for col in source_columns:
|
||
|
|
col_lower = col.lower().strip()
|
||
|
|
# Direct match
|
||
|
|
if col_lower in target_lower:
|
||
|
|
mapping[col] = target_lower[col_lower]
|
||
|
|
continue
|
||
|
|
# Alias match
|
||
|
|
alias_target = aliases.get(col_lower)
|
||
|
|
if alias_target and alias_target in target_lower:
|
||
|
|
mapping[col] = target_lower[alias_target]
|
||
|
|
continue
|
||
|
|
# Fuzzy: target contains source or vice versa
|
||
|
|
for tl, tf in target_lower.items():
|
||
|
|
if col_lower in tl or tl in col_lower:
|
||
|
|
mapping[col] = tf
|
||
|
|
break
|
||
|
|
return mapping
|
||
|
|
|
||
|
|
|
||
|
|
# ─── Validation ──────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
# Basic email regex (pragmatic, not RFC-perfect)
|
||
|
|
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
|
||
|
|
|
||
|
|
|
||
|
|
def validate_row(
|
||
|
|
row: dict,
|
||
|
|
required: list[str],
|
||
|
|
validators: dict[str, Any] | None = None,
|
||
|
|
) -> list[str]:
|
||
|
|
"""Validate a single row against required fields and optional validators.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
row: The row dict to validate.
|
||
|
|
required: List of field names that must be non-empty.
|
||
|
|
validators: Optional dict of field_name -> validator spec.
|
||
|
|
Supported specs:
|
||
|
|
- {"type": "email"} — must match email pattern
|
||
|
|
- {"type": "url"} — must look like a URL
|
||
|
|
- {"type": "int"} — must be parseable as integer
|
||
|
|
- {"max_length": N} — string length <= N
|
||
|
|
- {"min_length": N} — string length >= N
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
List of error message strings (empty if valid).
|
||
|
|
"""
|
||
|
|
errors: list[str] = []
|
||
|
|
for field in required:
|
||
|
|
val = str(row.get(field, "")).strip()
|
||
|
|
if not val:
|
||
|
|
errors.append(f"Missing required field: {field}")
|
||
|
|
|
||
|
|
if validators:
|
||
|
|
for field, spec in validators.items():
|
||
|
|
val = str(row.get(field, "")).strip()
|
||
|
|
if not val:
|
||
|
|
continue # Skip empty optional fields
|
||
|
|
vtype = spec.get("type")
|
||
|
|
if vtype == "email" and not _EMAIL_RE.match(val):
|
||
|
|
errors.append(f"Invalid email format in field: {field}")
|
||
|
|
elif vtype == "url" and not (val.startswith("http://") or val.startswith("https://")):
|
||
|
|
errors.append(f"Invalid URL format in field: {field}")
|
||
|
|
elif vtype == "int":
|
||
|
|
try:
|
||
|
|
int(val)
|
||
|
|
except ValueError:
|
||
|
|
errors.append(f"Field {field} must be an integer")
|
||
|
|
max_len = spec.get("max_length")
|
||
|
|
if max_len and len(val) > max_len:
|
||
|
|
errors.append(f"Field {field} exceeds max length {max_len}")
|
||
|
|
min_len = spec.get("min_length")
|
||
|
|
if min_len and len(val) < min_len:
|
||
|
|
errors.append(f"Field {field} below min length {min_len}")
|
||
|
|
return errors
|
||
|
|
|
||
|
|
|
||
|
|
# ─── Error report ────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
def build_error_report(errors: list[dict]) -> dict:
|
||
|
|
"""Build a structured error report from a list of per-row errors.
|
||
|
|
|
||
|
|
Args:
|
||
|
|
errors: List of dicts, each with keys:
|
||
|
|
- row: int (1-based row number)
|
||
|
|
- field: str (optional, which field failed)
|
||
|
|
- message: str (error description)
|
||
|
|
|
||
|
|
Returns:
|
||
|
|
Dict with:
|
||
|
|
- total_errors: int
|
||
|
|
- errors: list of the input dicts (normalized)
|
||
|
|
"""
|
||
|
|
normalized: list[dict] = []
|
||
|
|
for e in errors:
|
||
|
|
normalized.append({
|
||
|
|
"row": e.get("row", 0),
|
||
|
|
"field": e.get("field", ""),
|
||
|
|
"message": e.get("message", e.get("error", "")),
|
||
|
|
})
|
||
|
|
return {
|
||
|
|
"total_errors": len(normalized),
|
||
|
|
"errors": normalized,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def build_import_result(
|
||
|
|
total: int,
|
||
|
|
succeeded: int,
|
||
|
|
failed: int,
|
||
|
|
errors: list[dict] | None = None,
|
||
|
|
created: list[dict] | None = None,
|
||
|
|
dry_run: bool = False,
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
"""Build a standardized import result dict with error report."""
|
||
|
|
error_report = build_error_report(errors or [])
|
||
|
|
status = "success"
|
||
|
|
if failed > 0 and succeeded > 0:
|
||
|
|
status = "partial_success"
|
||
|
|
elif failed > 0 and succeeded == 0:
|
||
|
|
status = "failed"
|
||
|
|
return {
|
||
|
|
"total": total,
|
||
|
|
"succeeded": succeeded,
|
||
|
|
"failed": failed,
|
||
|
|
"status": status,
|
||
|
|
"dry_run": dry_run,
|
||
|
|
"error_report": error_report,
|
||
|
|
"created": created or [],
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
# ─── Format detection ────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
def detect_format(filename: str, content: bytes) -> str:
|
||
|
|
"""Detect file format from filename extension and content magic bytes."""
|
||
|
|
name_lower = filename.lower()
|
||
|
|
if name_lower.endswith(".csv"):
|
||
|
|
return "csv"
|
||
|
|
if name_lower.endswith(".json"):
|
||
|
|
return "json"
|
||
|
|
if name_lower.endswith(".xlsx") or name_lower.endswith(".xls"):
|
||
|
|
return "xlsx"
|
||
|
|
# Magic bytes
|
||
|
|
if content.startswith(b"PK\x03\x04"):
|
||
|
|
return "xlsx" # XLSX is a ZIP archive
|
||
|
|
if content.lstrip().startswith(b"[") or content.lstrip().startswith(b"{"):
|
||
|
|
return "json"
|
||
|
|
return "csv" # Default fallback
|
||
|
|
|
||
|
|
|
||
|
|
def parse_file(filename: str, content: bytes) -> list[dict[str, str]]:
|
||
|
|
"""Auto-detect format and parse file content into list of dicts."""
|
||
|
|
fmt = detect_format(filename, content)
|
||
|
|
if fmt == "csv":
|
||
|
|
return parse_csv(content)
|
||
|
|
if fmt == "json":
|
||
|
|
return parse_json(content)
|
||
|
|
if fmt == "xlsx":
|
||
|
|
return parse_xlsx(content)
|
||
|
|
raise ValueError(f"Unsupported file format: {fmt}")
|