feat(C.5): Modularer Import/Export — Shared Helpers, Preview/Mapping, Background Jobs, Partial-Failure

C5-BASE: app/services/import_export_helpers.py (NEU, 352 Zeilen)
- parse_csv/json/xlsx, write_csv/json/xlsx, map_fields, suggest_mapping
- validate_row, build_error_report, build_import_result, detect_format

C5-PREVIEW: POST /import/preview + POST /import/validate
- Preview gibt erste 10 Zeilen + Spalten + Mapping-Vorschlag
- Validate gibt Fehler-Report ohne Import

C5-JOB: app/services/import_export_jobs.py (NEU, 165 Zeilen)
- ARQ Background Job für Files >1000 Zeilen
- Job-Status: pending/processing/completed/partial_success/failed
- GET /import/status/{job_id} — Status + Progress + Fehler-Report
- Partial-Failure: try/except pro Zeile, fehlerhafte gesammelt, erfolgreiche committet

C5-CONTACT+C5-COMPANY: Handler auf Shared Helpers umgestellt
C5-UI: ImportWizard.tsx (5 Steps: Upload→Preview→Validation→Review→Result)
C5-TEST: 45 Tests in test_import_export.py — alle grün
C5-DOC: Plugin-Dev-Guide Kapitel 31 (Import/Export Handler)
This commit is contained in:
Agent Zero
2026-08-13 22:43:33 +02:00
parent fd14e0076b
commit e7ae0ad5ce
10 changed files with 2630 additions and 439 deletions
+1
View File
@@ -225,6 +225,7 @@ def _lazy_register_plugin_jobs() -> None:
"app.plugins.builtins.automation.agent_runner", "app.plugins.builtins.automation.agent_runner",
"app.plugins.builtins.automation.execution_engine", "app.plugins.builtins.automation.execution_engine",
"app.plugins.builtins.tasks.jobs", "app.plugins.builtins.tasks.jobs",
"app.services.import_export_jobs",
] ]
for mod_name in plugin_job_modules: for mod_name in plugin_job_modules:
try: try:
+255 -39
View File
@@ -1,8 +1,9 @@
"""Import/export routes — CSV import, dry-run preview, CSV/XLSX export.""" """Import/export routes — CSV/JSON/XLSX import, preview, validate, background jobs, CSV/XLSX export."""
from __future__ import annotations from __future__ import annotations
import io import io
import json
import uuid import uuid
from fastapi import ( from fastapi import (
@@ -20,27 +21,75 @@ from starlette.responses import StreamingResponse
from app.core.db import get_db from app.core.db import get_db
from app.deps import require_permission from app.deps import require_permission
from app.services import import_export_service from app.services import import_export_service
from app.services.import_export_helpers import (
detect_format,
parse_file,
write_csv,
write_xlsx,
)
router = APIRouter(prefix="/api/v1", tags=["import_export"]) router = APIRouter(prefix="/api/v1", tags=["import_export"])
# Row threshold for background job processing
_BACKGROUND_THRESHOLD = 1000
@router.post("/import") @router.post("/import")
async def import_csv( async def import_csv(
file: UploadFile = File(...), file: UploadFile = File(...),
entity_type: str = Form("companies"), entity_type: str = Form("companies"),
field_mapping: str = Form(None),
db: AsyncSession = Depends(get_db), db: AsyncSession = Depends(get_db),
current_user: dict = Depends(require_permission("import_export:write")), current_user: dict = Depends(require_permission("import_export:write")),
): ):
"""Import companies or contacts from CSV file. """Import companies or contacts from CSV/JSON/XLSX file.
entity_type: 'companies' or 'contacts'. entity_type: 'companies' or 'contacts'.
field_mapping: Optional JSON string of source_column -> target_field mapping.
For files > 1000 rows, the import runs as a background ARQ job.
""" """
tenant_id = uuid.UUID(current_user["tenant_id"]) tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"]) user_id = uuid.UUID(current_user["user_id"])
content = await file.read() content = await file.read()
csv_content = content.decode("utf-8") csv_content = content.decode("utf-8", errors="replace")
# Parse mapping if provided
mapping = None
if field_mapping:
try:
mapping = json.loads(field_mapping)
except json.JSONDecodeError:
raise HTTPException(status_code=400, detail="Invalid field_mapping JSON")
# Check row count for background processing
try:
rows = parse_file(file.filename or "upload.csv", content)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}")
if len(rows) > _BACKGROUND_THRESHOLD:
# Enqueue as background job
from app.services.import_export_jobs import create_import_job
try:
job_id = await create_import_job(
entity_type=entity_type,
csv_content=csv_content,
tenant_id=tenant_id,
user_id=user_id,
field_mapping=mapping,
)
return {
"status": "pending",
"job_id": job_id,
"message": f"Import enqueued as background job ({len(rows)} rows)",
"total": len(rows),
}
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to enqueue import job: {exc}")
# Synchronous import for smaller files
result = await import_export_service.import_csv( result = await import_export_service.import_csv(
db, db,
tenant_id, tenant_id,
@@ -48,6 +97,7 @@ async def import_csv(
csv_content, csv_content,
entity_type=entity_type, entity_type=entity_type,
dry_run=False, dry_run=False,
field_mapping=mapping,
) )
return result return result
@@ -56,24 +106,188 @@ async def import_csv(
async def import_csv_preview( async def import_csv_preview(
file: UploadFile = File(...), file: UploadFile = File(...),
entity_type: str = Form("companies"), entity_type: str = Form("companies"),
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(require_permission("import_export:read")), current_user: dict = Depends(require_permission("import_export:read")),
): ):
"""Preview CSV import (dry-run — no DB changes).""" """Preview CSV import (dry-run — no DB changes).
Returns first 10 rows, detected columns, and mapping suggestion.
"""
content = await file.read()
try:
result = import_export_service.preview_import(
filename=file.filename or "upload.csv",
content=content,
entity_type=entity_type,
)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}")
return result
@router.post("/import/validate")
async def import_validate(
file: UploadFile = File(...),
entity_type: str = Form("companies"),
field_mapping: str = Form(None),
current_user: dict = Depends(require_permission("import_export:read")),
):
"""Validate all rows against mapping and return error report without importing."""
content = await file.read()
mapping = None
if field_mapping:
try:
mapping = json.loads(field_mapping)
except json.JSONDecodeError:
raise HTTPException(status_code=400, detail="Invalid field_mapping JSON")
try:
result = import_export_service.validate_import(
filename=file.filename or "upload.csv",
content=content,
entity_type=entity_type,
field_mapping=mapping,
)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Failed to validate file: {exc}")
return result
@router.get("/import/status/{job_id}")
async def import_job_status(
job_id: str,
current_user: dict = Depends(require_permission("import_export:read")),
):
"""Get status of a background import job."""
from app.services.import_export_jobs import get_import_job_status
status = await get_import_job_status(job_id)
if status is None:
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
return status
@router.post("/import/contacts")
async def import_contacts_route(
file: UploadFile = File(...),
field_mapping: str = Form(None),
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(require_permission("import_export:write")),
):
"""Import contacts from CSV/JSON/XLSX file.
For files > 1000 rows, runs as background ARQ job.
"""
tenant_id = uuid.UUID(current_user["tenant_id"]) tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"]) user_id = uuid.UUID(current_user["user_id"])
content = await file.read() content = await file.read()
csv_content = content.decode("utf-8") csv_content = content.decode("utf-8", errors="replace")
result = await import_export_service.import_csv( mapping = None
if field_mapping:
try:
mapping = json.loads(field_mapping)
except json.JSONDecodeError:
raise HTTPException(status_code=400, detail="Invalid field_mapping JSON")
# Check row count for background processing
try:
rows = parse_file(file.filename or "upload.csv", content)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}")
if len(rows) > _BACKGROUND_THRESHOLD:
from app.services.import_export_jobs import create_import_job
try:
job_id = await create_import_job(
entity_type="contacts",
csv_content=csv_content,
tenant_id=tenant_id,
user_id=user_id,
field_mapping=mapping,
)
return {
"status": "pending",
"job_id": job_id,
"message": f"Import enqueued as background job ({len(rows)} rows)",
"total": len(rows),
}
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to enqueue import job: {exc}")
result = await import_export_service.import_contacts(
db, db,
tenant_id, tenant_id,
user_id, user_id,
csv_content, csv_content,
entity_type=entity_type, dry_run=False,
dry_run=True, field_mapping=mapping,
)
return result
@router.post("/import/companies")
async def import_companies_route(
file: UploadFile = File(...),
field_mapping: str = Form(None),
db: AsyncSession = Depends(get_db),
current_user: dict = Depends(require_permission("import_export:write")),
):
"""Import companies from CSV/JSON/XLSX file.
For files > 1000 rows, runs as background ARQ job.
"""
tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"])
content = await file.read()
csv_content = content.decode("utf-8", errors="replace")
mapping = None
if field_mapping:
try:
mapping = json.loads(field_mapping)
except json.JSONDecodeError:
raise HTTPException(status_code=400, detail="Invalid field_mapping JSON")
# Check row count for background processing
try:
rows = parse_file(file.filename or "upload.csv", content)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}")
if len(rows) > _BACKGROUND_THRESHOLD:
from app.services.import_export_jobs import create_import_job
try:
job_id = await create_import_job(
entity_type="companies",
csv_content=csv_content,
tenant_id=tenant_id,
user_id=user_id,
field_mapping=mapping,
)
return {
"status": "pending",
"job_id": job_id,
"message": f"Import enqueued as background job ({len(rows)} rows)",
"total": len(rows),
}
except Exception as exc:
raise HTTPException(status_code=500, detail=f"Failed to enqueue import job: {exc}")
result = await import_export_service.import_companies(
db,
tenant_id,
user_id,
csv_content,
dry_run=False,
field_mapping=mapping,
) )
return result return result
@@ -81,24 +295,21 @@ async def import_csv_preview(
@router.get("/export") @router.get("/export")
async def export_data( async def export_data(
entity_type: str = Query("contacts", pattern="^(contacts|companies)$"), entity_type: str = Query("contacts", pattern="^(contacts|companies)$"),
format: str = Query("csv", pattern="^(csv|xlsx)$"), format: str = Query("csv", pattern="^(csv|xlsx|json)$"),
db: AsyncSession = Depends(get_db), db: AsyncSession = Depends(get_db),
current_user: dict = Depends(require_permission("import_export:read")), current_user: dict = Depends(require_permission("import_export:read")),
): ):
"""Export contacts or companies as CSV or XLSX file download. """Export contacts or companies as CSV, XLSX, or JSON file download.
Query params: Query params:
- entity_type: 'contacts' or 'companies' (default: contacts) - entity_type: 'contacts' or 'companies' (default: contacts)
- format: 'csv' or 'xlsx' (default: csv) - format: 'csv', 'xlsx', or 'json' (default: csv)
""" """
tenant_id = uuid.UUID(current_user["tenant_id"]) tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"]) user_id = uuid.UUID(current_user["user_id"])
is_system_admin = current_user.get("is_system_admin", False) is_system_admin = current_user.get("is_system_admin", False)
# Determine filename based on entity_type # Fetch data from the appropriate service function
filename = f"{entity_type}_export.csv"
# Fetch CSV data from the appropriate service function
if entity_type == "contacts": if entity_type == "contacts":
csv_data = await import_export_service.export_contacts_csv( csv_data = await import_export_service.export_contacts_csv(
db, tenant_id, user_id=user_id, is_system_admin=is_system_admin db, tenant_id, user_id=user_id, is_system_admin=is_system_admin
@@ -110,29 +321,22 @@ async def export_data(
else: else:
raise HTTPException(status_code=400, detail=f"Unsupported entity_type: {entity_type}") raise HTTPException(status_code=400, detail=f"Unsupported entity_type: {entity_type}")
# XLSX format handling # Parse CSV data back to rows for format conversion
import csv as _csv
reader = _csv.reader(io.StringIO(csv_data))
all_rows = list(reader)
if not all_rows:
raise HTTPException(status_code=404, detail="No data to export")
headers = all_rows[0]
data_rows = [dict(zip(headers, row)) for row in all_rows[1:]]
if format == "xlsx": if format == "xlsx":
try: try:
import openpyxl xlsx_bytes = write_xlsx(data_rows, headers)
wb = openpyxl.Workbook()
ws = wb.active
ws.title = entity_type.capitalize()
# Parse the CSV string and populate the worksheet
import csv as _csv
reader = _csv.reader(io.StringIO(csv_data))
for row in reader:
ws.append(row)
xlsx_buffer = io.BytesIO()
wb.save(xlsx_buffer)
xlsx_buffer.seek(0)
xlsx_filename = f"{entity_type}_export.xlsx" xlsx_filename = f"{entity_type}_export.xlsx"
return StreamingResponse( return StreamingResponse(
xlsx_buffer, io.BytesIO(xlsx_bytes),
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
headers={"Content-Disposition": f'attachment; filename="{xlsx_filename}"'}, headers={"Content-Disposition": f'attachment; filename="{xlsx_filename}"'},
) )
@@ -140,7 +344,7 @@ async def export_data(
# openpyxl not available — fall back to CSV with a warning header # openpyxl not available — fall back to CSV with a warning header
media_type = "text/csv" media_type = "text/csv"
headers = { headers = {
"Content-Disposition": f'attachment; filename="{filename}"', "Content-Disposition": f'attachment; filename="{entity_type}_export.csv"',
"X-Export-Warning": "openpyxl not installed, falling back to CSV format", "X-Export-Warning": "openpyxl not installed, falling back to CSV format",
} }
return StreamingResponse( return StreamingResponse(
@@ -149,9 +353,21 @@ async def export_data(
headers=headers, headers=headers,
) )
if format == "json":
from app.services.import_export_helpers import write_json
json_bytes = write_json(data_rows)
json_filename = f"{entity_type}_export.json"
return StreamingResponse(
io.BytesIO(json_bytes),
media_type="application/json",
headers={"Content-Disposition": f'attachment; filename="{json_filename}"'},
)
# Default: CSV format # Default: CSV format
csv_filename = f"{entity_type}_export.csv"
return StreamingResponse( return StreamingResponse(
iter([csv_data.encode("utf-8")]), iter([csv_data.encode("utf-8")]),
media_type="text/csv", media_type="text/csv",
headers={"Content-Disposition": f'attachment; filename="{filename}"'}, headers={"Content-Disposition": f'attachment; filename="{csv_filename}"'},
) )
+352
View File
@@ -0,0 +1,352 @@
"""Shared import/export helpers — reusable parsing, writing, mapping, validation.
These building blocks are used by import_export_service.py and can be leveraged
by any plugin that needs custom import/export handlers.
"""
from __future__ import annotations
import csv
import io
import json
import logging
import re
from typing import Any
logger = logging.getLogger(__name__)
# ─── Encoding detection ──────────────────────────────────────────────────────
_BOMS = (
("utf-8-sig", b"\xef\xbb\xbf"),
("utf-16-le", b"\xff\xfe"),
("utf-16-be", b"\xfe\xff"),
)
def _detect_encoding(content: bytes) -> str:
"""Detect the most likely text encoding from raw bytes."""
for encoding, bom in _BOMS:
if content.startswith(bom):
return encoding
# Try UTF-8 first, fall back to latin-1 (never fails)
try:
content.decode("utf-8")
return "utf-8"
except UnicodeDecodeError:
return "latin-1"
# ─── Parsers ─────────────────────────────────────────────────────────────────
def parse_csv(content: bytes) -> list[dict[str, str]]:
"""Parse CSV content (bytes) into a list of dicts.
Handles encoding detection (UTF-8, UTF-8 BOM, UTF-16, latin-1).
"""
encoding = _detect_encoding(content)
text = content.decode(encoding)
reader = csv.DictReader(io.StringIO(text))
return [dict(row) for row in reader]
def parse_json(content: bytes) -> list[dict]:
"""Parse JSON content (bytes) into a list of dicts.
Accepts either a JSON array of objects or a single object.
"""
encoding = _detect_encoding(content)
text = content.decode(encoding)
data = json.loads(text)
if isinstance(data, dict):
return [data]
if isinstance(data, list):
return data
raise ValueError(f"Expected JSON array or object, got {type(data).__name__}")
def parse_xlsx(content: bytes) -> list[dict[str, str]]:
"""Parse XLSX content (bytes) into a list of dicts.
Uses openpyxl. First row is treated as headers.
"""
try:
import openpyxl
except ImportError as exc:
raise ImportError("openpyxl is required for XLSX parsing") from exc
wb = openpyxl.load_workbook(io.BytesIO(content), read_only=True, data_only=True)
ws = wb.active
rows_iter = ws.iter_rows(values_only=True)
try:
headers = [str(h).strip() if h is not None else "" for h in next(rows_iter)]
except StopIteration:
wb.close()
return []
result: list[dict[str, str]] = []
for row in rows_iter:
if row is None or all(c is None or str(c).strip() == "" for c in row):
continue
row_dict = {}
for idx, header in enumerate(headers):
if not header:
continue
val = row[idx] if idx < len(row) else None
row_dict[header] = str(val).strip() if val is not None else ""
result.append(row_dict)
wb.close()
return result
# ─── Writers ─────────────────────────────────────────────────────────────────
def write_csv(rows: list[dict], headers: list[str]) -> bytes:
"""Write a list of dicts to CSV bytes using the given header order."""
output = io.StringIO()
writer = csv.DictWriter(output, fieldnames=headers, extrasaction="ignore")
writer.writeheader()
for row in rows:
writer.writerow({h: row.get(h, "") for h in headers})
return output.getvalue().encode("utf-8")
def write_json(rows: list[dict]) -> bytes:
"""Write a list of dicts to JSON bytes (UTF-8, indented)."""
return json.dumps(rows, indent=2, ensure_ascii=False, default=str).encode("utf-8")
def write_xlsx(rows: list[dict], headers: list[str]) -> bytes:
"""Write a list of dicts to XLSX bytes using the given header order."""
try:
import openpyxl
except ImportError as exc:
raise ImportError("openpyxl is required for XLSX writing") from exc
wb = openpyxl.Workbook()
ws = wb.active
ws.append(headers)
for row in rows:
ws.append([row.get(h, "") for h in headers])
buf = io.BytesIO()
wb.save(buf)
buf.seek(0)
data = buf.read()
wb.close()
return data
# ─── Field mapping ───────────────────────────────────────────────────────────
def map_fields(row: dict, mapping: dict[str, str]) -> dict:
"""Map source columns to target fields.
Args:
row: Source row dict with original column names.
mapping: Dict of source_column -> target_field.
If a source column is not in the row, it is skipped.
Returns:
New dict with target field names as keys.
"""
result: dict[str, Any] = {}
for source_col, target_field in mapping.items():
if source_col in row:
result[target_field] = row[source_col]
return result
def suggest_mapping(source_columns: list[str], target_fields: list[str]) -> dict[str, str]:
"""Auto-suggest a mapping from source columns to target fields.
Uses case-insensitive exact match and common alias matching.
"""
# Common aliases: source column variants -> canonical target
aliases: dict[str, str] = {
"first_name": "firstname",
"last_name": "surname",
"last_name": "surname",
"email": "email",
"email_address": "email",
"phone": "phone",
"mobile": "mobile",
"phone_2": "mobile",
"position": "function",
"title": "function",
"job_title": "function",
"company": "name",
"company_name": "name",
"organization": "name",
"web": "website",
"url": "website",
"homepage": "website",
"zip": "postalcode",
"postal_code": "postalcode",
"zip_code": "postalcode",
"city": "city",
"country": "country",
"department": "department",
"industry": "industry",
}
target_lower = {f.lower(): f for f in target_fields}
mapping: dict[str, str] = {}
for col in source_columns:
col_lower = col.lower().strip()
# Direct match
if col_lower in target_lower:
mapping[col] = target_lower[col_lower]
continue
# Alias match
alias_target = aliases.get(col_lower)
if alias_target and alias_target in target_lower:
mapping[col] = target_lower[alias_target]
continue
# Fuzzy: target contains source or vice versa
for tl, tf in target_lower.items():
if col_lower in tl or tl in col_lower:
mapping[col] = tf
break
return mapping
# ─── Validation ──────────────────────────────────────────────────────────────
# Basic email regex (pragmatic, not RFC-perfect)
_EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
def validate_row(
row: dict,
required: list[str],
validators: dict[str, Any] | None = None,
) -> list[str]:
"""Validate a single row against required fields and optional validators.
Args:
row: The row dict to validate.
required: List of field names that must be non-empty.
validators: Optional dict of field_name -> validator spec.
Supported specs:
- {"type": "email"} — must match email pattern
- {"type": "url"} — must look like a URL
- {"type": "int"} — must be parseable as integer
- {"max_length": N} — string length <= N
- {"min_length": N} — string length >= N
Returns:
List of error message strings (empty if valid).
"""
errors: list[str] = []
for field in required:
val = str(row.get(field, "")).strip()
if not val:
errors.append(f"Missing required field: {field}")
if validators:
for field, spec in validators.items():
val = str(row.get(field, "")).strip()
if not val:
continue # Skip empty optional fields
vtype = spec.get("type")
if vtype == "email" and not _EMAIL_RE.match(val):
errors.append(f"Invalid email format in field: {field}")
elif vtype == "url" and not (val.startswith("http://") or val.startswith("https://")):
errors.append(f"Invalid URL format in field: {field}")
elif vtype == "int":
try:
int(val)
except ValueError:
errors.append(f"Field {field} must be an integer")
max_len = spec.get("max_length")
if max_len and len(val) > max_len:
errors.append(f"Field {field} exceeds max length {max_len}")
min_len = spec.get("min_length")
if min_len and len(val) < min_len:
errors.append(f"Field {field} below min length {min_len}")
return errors
# ─── Error report ────────────────────────────────────────────────────────────
def build_error_report(errors: list[dict]) -> dict:
"""Build a structured error report from a list of per-row errors.
Args:
errors: List of dicts, each with keys:
- row: int (1-based row number)
- field: str (optional, which field failed)
- message: str (error description)
Returns:
Dict with:
- total_errors: int
- errors: list of the input dicts (normalized)
"""
normalized: list[dict] = []
for e in errors:
normalized.append({
"row": e.get("row", 0),
"field": e.get("field", ""),
"message": e.get("message", e.get("error", "")),
})
return {
"total_errors": len(normalized),
"errors": normalized,
}
def build_import_result(
total: int,
succeeded: int,
failed: int,
errors: list[dict] | None = None,
created: list[dict] | None = None,
dry_run: bool = False,
) -> dict[str, Any]:
"""Build a standardized import result dict with error report."""
error_report = build_error_report(errors or [])
status = "success"
if failed > 0 and succeeded > 0:
status = "partial_success"
elif failed > 0 and succeeded == 0:
status = "failed"
return {
"total": total,
"succeeded": succeeded,
"failed": failed,
"status": status,
"dry_run": dry_run,
"error_report": error_report,
"created": created or [],
}
# ─── Format detection ────────────────────────────────────────────────────────
def detect_format(filename: str, content: bytes) -> str:
"""Detect file format from filename extension and content magic bytes."""
name_lower = filename.lower()
if name_lower.endswith(".csv"):
return "csv"
if name_lower.endswith(".json"):
return "json"
if name_lower.endswith(".xlsx") or name_lower.endswith(".xls"):
return "xlsx"
# Magic bytes
if content.startswith(b"PK\x03\x04"):
return "xlsx" # XLSX is a ZIP archive
if content.lstrip().startswith(b"[") or content.lstrip().startswith(b"{"):
return "json"
return "csv" # Default fallback
def parse_file(filename: str, content: bytes) -> list[dict[str, str]]:
"""Auto-detect format and parse file content into list of dicts."""
fmt = detect_format(filename, content)
if fmt == "csv":
return parse_csv(content)
if fmt == "json":
return parse_json(content)
if fmt == "xlsx":
return parse_xlsx(content)
raise ValueError(f"Unsupported file format: {fmt}")
+165
View File
@@ -0,0 +1,165 @@
"""ARQ background jobs for large import operations.
When an import file exceeds BACKGROUND_JOB_THRESHOLD rows, the import is
enqueued as an ARQ job instead of running synchronously. Job status is
tracked in Redis with the key pattern 'leocrm:import_job:{job_id}'.
"""
from __future__ import annotations
import json
import logging
import uuid
from typing import Any
from app.core.job_registry import register_job
from app.services import import_export_service
logger = logging.getLogger(__name__)
_JOB_KEY_PREFIX = "leocrm:import_job"
_JOB_TTL = 3600 # 1 hour
def _job_key(job_id: str) -> str:
return f"{_JOB_KEY_PREFIX}:{job_id}"
async def _set_job_status(
redis_client: Any,
job_id: str,
status: str,
**extra: Any,
) -> None:
"""Store job status in Redis with TTL."""
data = {"status": status, **extra}
await redis_client.set(_job_key(job_id), json.dumps(data, default=str), ex=_JOB_TTL)
async def _get_job_status(redis_client: Any, job_id: str) -> dict[str, Any] | None:
"""Retrieve job status from Redis."""
raw = await redis_client.get(_job_key(job_id))
if raw is None:
return None
if isinstance(raw, bytes):
raw = raw.decode("utf-8")
return json.loads(raw)
async def import_background_job(ctx: dict[str, Any], job_id: str, params: dict[str, Any]) -> dict[str, Any]:
"""ARQ job function: run import in background with status tracking.
Args:
ctx: ARQ worker context.
job_id: Unique job identifier.
params: Dict with keys:
- entity_type: 'contacts' or 'companies'
- csv_content: File content as string
- tenant_id: Tenant UUID string
- user_id: User UUID string
- field_mapping: Optional dict[str, str]
"""
from app.core.auth import get_redis
from app.core.db import get_worker_session_factory
from app.core.db import set_tenant_context
redis_client = get_redis()
entity_type = params["entity_type"]
csv_content = params["csv_content"]
tenant_id = uuid.UUID(params["tenant_id"])
user_id = uuid.UUID(params["user_id"])
field_mapping = params.get("field_mapping")
# Set initial status
await _set_job_status(redis_client, job_id, "processing", progress=0, total=0)
factory = get_worker_session_factory()
try:
async with factory() as db:
# Set tenant context for RLS
await set_tenant_context(db, tenant_id)
result = await import_export_service.import_csv(
db,
tenant_id,
user_id,
csv_content,
entity_type=entity_type,
dry_run=False,
field_mapping=field_mapping,
)
# Determine final status
status = result.get("status", "completed")
if status == "success":
final_status = "completed"
elif status == "partial_success":
final_status = "partial_success"
else:
final_status = "failed"
await _set_job_status(
redis_client,
job_id,
final_status,
progress=100,
total=result.get("total", 0),
succeeded=result.get("succeeded", 0),
failed=result.get("failed", 0),
error_report=result.get("error_report", {}),
created_count=len(result.get("created", [])),
)
return result
except Exception as exc:
logger.error("Import background job %s failed: %s", job_id, exc, exc_info=True)
await _set_job_status(
redis_client,
job_id,
"failed",
progress=0,
total=0,
error=str(exc),
)
raise
# Register the job so it appears in WorkerSettings.functions
register_job("import_background", import_background_job)
async def get_import_job_status(job_id: str) -> dict[str, Any] | None:
"""Public helper to get job status from Redis."""
from app.core.auth import get_redis
redis_client = get_redis()
return await _get_job_status(redis_client, job_id)
async def create_import_job(
entity_type: str,
csv_content: str,
tenant_id: uuid.UUID,
user_id: uuid.UUID,
field_mapping: dict[str, str] | None = None,
) -> str:
"""Enqueue an import as ARQ background job and return job_id."""
from app.core.jobs import enqueue_job
job_id = str(uuid.uuid4())
params = {
"entity_type": entity_type,
"csv_content": csv_content,
"tenant_id": str(tenant_id),
"user_id": str(user_id),
"field_mapping": field_mapping,
}
# Pre-set status as pending
from app.core.auth import get_redis
redis_client = get_redis()
await _set_job_status(redis_client, job_id, "pending", progress=0, total=0)
await enqueue_job("import_background", job_id, params)
return job_id
+359 -167
View File
@@ -1,4 +1,8 @@
"""Import/export service — CSV import with dry-run preview, CSV/XLSX export. """Import/export service — CSV/JSON/XLSX import with dry-run preview, partial-failure, CSV/XLSX export.
Uses shared helpers from import_export_helpers.py for parsing, writing, mapping,
and validation. Implements partial-failure semantics: valid rows are committed
while invalid rows are collected into a structured error report.
Uses unified Contact model fields: firstname, surname, email_1, phone_1, phone_2, function. Uses unified Contact model fields: firstname, surname, email_1, phone_1, phone_2, function.
Company import creates Contact with type='company'. Company import creates Contact with type='company'.
@@ -6,8 +10,7 @@ Company import creates Contact with type='company'.
from __future__ import annotations from __future__ import annotations
import csv import logging
import io
import uuid import uuid
from typing import Any from typing import Any
@@ -17,203 +20,322 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.audit import log_audit from app.core.audit import log_audit
from app.core.visibility import apply_visibility_filter from app.core.visibility import apply_visibility_filter
from app.models.contact import Contact from app.models.contact import Contact
from app.services import import_export_helpers as helpers
from app.services.contact_service import _serialize_contact as _contact_to_dict from app.services.contact_service import _serialize_contact as _contact_to_dict
logger = logging.getLogger(__name__)
# Expected CSV columns for each entity type # Expected CSV columns for each entity type
# Company import creates Contact with type='company' using name field # Company import creates Contact with type='company' using name field
COMPANY_COLUMNS = ["name", "industry", "phone", "email", "website"] COMPANY_COLUMNS = ["name", "industry", "phone", "email", "website"]
# Contact import uses unified Contact fields # Contact import uses unified Contact fields
CONTACT_COLUMNS = ["firstname", "surname", "email", "phone", "mobile", "function", "department"] CONTACT_COLUMNS = ["firstname", "surname", "email", "phone", "mobile", "function", "department"]
# Target fields for mapping suggestions
CONTACT_TARGET_FIELDS = ["firstname", "surname", "email", "phone", "mobile", "function", "department"]
COMPANY_TARGET_FIELDS = ["name", "industry", "phone", "email", "website"]
def _parse_csv(content: str) -> list[dict[str, str]]: # Validators for each entity type
"""Parse CSV content into list of dicts.""" CONTACT_VALIDATORS: dict[str, dict] = {
reader = csv.DictReader(io.StringIO(content)) "email": {"type": "email"},
return [dict(row) for row in reader] }
COMPANY_VALIDATORS: dict[str, dict] = {
"email": {"type": "email"},
"website": {"type": "url"},
}
# Threshold for background job processing (rows above this go to ARQ)
BACKGROUND_JOB_THRESHOLD = 1000
def _parse_csv(content: str | bytes) -> list[dict[str, str]]:
"""Parse file content into list of dicts (delegates to helpers with format detection).
Despite the name, this handles CSV, JSON, and XLSX formats by detecting
from content. This maintains backward compatibility with existing callers.
"""
if isinstance(content, str):
content = content.encode("utf-8")
# Detect format from content magic bytes
fmt = helpers.detect_format("upload", content)
if fmt == "json":
return helpers.parse_json(content)
elif fmt == "xlsx":
return helpers.parse_xlsx(content)
return helpers.parse_csv(content)
def _validate_row(row: dict[str, str], required: list[str]) -> list[str]: def _validate_row(row: dict[str, str], required: list[str]) -> list[str]:
"""Validate a single row. Returns list of error messages (empty if valid).""" """Validate a single row (delegates to helpers for backward compat)."""
errors = [] return helpers.validate_row(row, required)
for col in required:
val = row.get(col, "").strip()
if not val: def _normalize_contact_row(row: dict[str, str]) -> dict[str, str]:
errors.append(f"Missing required field: {col}") """Normalize contact row to unified field names."""
return errors firstname = (row.get("firstname") or row.get("first_name") or "").strip()
surname = (row.get("surname") or row.get("last_name") or "").strip()
row["firstname"] = firstname
row["surname"] = surname
# Map old column names to unified ones
if "email" not in row and "email_address" in row:
row["email"] = row["email_address"]
if "mobile" not in row and "phone_2" in row:
row["mobile"] = row["phone_2"]
if "function" not in row and "position" in row:
row["function"] = row["position"]
return row
def _normalize_company_row(row: dict[str, str]) -> dict[str, str]:
"""Normalize company row to unified field names."""
name = (row.get("name") or row.get("company") or row.get("company_name") or "").strip()
row["name"] = name
if "email" not in row and "email_address" in row:
row["email"] = row["email_address"]
if "website" not in row and "url" in row:
row["website"] = row["url"]
if "website" not in row and "homepage" in row:
row["website"] = row["homepage"]
return row
async def import_companies( async def import_companies(
db: AsyncSession, db: AsyncSession,
tenant_id: uuid.UUID, tenant_id: uuid.UUID,
user_id: uuid.UUID, user_id: uuid.UUID,
csv_content: str, csv_content: str | bytes,
dry_run: bool = False, dry_run: bool = False,
field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Import companies from CSV as Contact with type='company'. """Import companies from CSV as Contact with type='company'.
Uses unified Contact model: name field for company name, email_1/phone_1 for contact info. Uses unified Contact model: name field for company name, email_1/phone_1 for contact info.
Implements partial-failure: valid rows committed, invalid rows collected in error report.
""" """
rows = _parse_csv(csv_content) rows = _parse_csv(csv_content)
total = len(rows) total = len(rows)
valid_rows = [] errors: list[dict] = []
errors = [] valid_rows: list[dict] = []
for idx, row in enumerate(rows, start=1): for idx, row in enumerate(rows, start=1):
row_errors = _validate_row(row, ["name"]) # Apply field mapping if provided
if field_mapping:
row = helpers.map_fields(row, field_mapping)
row = _normalize_company_row(row)
row_errors = helpers.validate_row(row, ["name"], COMPANY_VALIDATORS)
if row_errors: if row_errors:
for e in row_errors: for e in row_errors:
errors.append({"row": idx, "error": e}) errors.append({"row": idx, "field": "", "message": e})
else: else:
valid_rows.append(row) valid_rows.append(row)
# Count unique failed rows (a row may have multiple errors)
failed_row_count = len({e["row"] for e in errors})
if dry_run: if dry_run:
return { return helpers.build_import_result(
"total": total, total=total,
"valid": len(valid_rows), succeeded=len(valid_rows),
"invalid": len(errors), failed=failed_row_count,
"errors": errors, errors=errors,
"created": [], dry_run=True,
"dry_run": True,
}
created = []
for row in valid_rows:
contact = Contact(
tenant_id=tenant_id,
type="company",
name=row["name"].strip(),
displayname=row["name"].strip(),
email_1=row.get("email", "").strip() or None,
phone_1=row.get("phone", "").strip() or None,
website=row.get("website", "").strip() or None,
owner_id=user_id,
created_by=user_id,
updated_by=user_id,
) )
db.add(contact)
await db.flush()
await log_audit(
db,
tenant_id,
user_id,
"import",
"contact",
contact.id,
changes={"name": contact.name, "type": "company"},
)
created.append(_contact_to_dict(contact))
return { created: list[dict] = []
"total": total, failed: list[dict] = []
"valid": len(valid_rows), succeeded = 0
"invalid": len(errors),
"errors": errors, for idx, row in enumerate(valid_rows, start=1):
"created": created, try:
"dry_run": False, contact = Contact(
} tenant_id=tenant_id,
type="company",
name=row["name"].strip(),
displayname=row["name"].strip(),
email_1=row.get("email", "").strip() or None,
phone_1=row.get("phone", "").strip() or None,
website=row.get("website", "").strip() or None,
owner_id=user_id,
created_by=user_id,
updated_by=user_id,
)
db.add(contact)
await db.flush()
await log_audit(
db,
tenant_id,
user_id,
"import",
"contact",
contact.id,
changes={"name": contact.name, "type": "company"},
)
created.append(_contact_to_dict(contact))
succeeded += 1
except Exception as exc:
logger.warning("Import companies: row %d failed: %s", idx, exc)
await db.rollback()
failed.append({"row": idx, "field": "", "message": str(exc)})
if succeeded > 0:
try:
await db.commit()
except Exception as exc:
logger.error("Import companies: commit failed: %s", exc)
await db.rollback()
return helpers.build_import_result(
total=total,
succeeded=0,
failed=len(valid_rows),
errors=errors + [{"row": 0, "field": "", "message": f"Commit failed: {exc}"}],
created=[],
)
# Count unique failed rows from validation errors + runtime failures
all_failed_rows = {e["row"] for e in errors} | {f["row"] for f in failed}
return helpers.build_import_result(
total=total,
succeeded=succeeded,
failed=len(all_failed_rows),
errors=errors + failed,
created=created,
)
async def import_contacts( async def import_contacts(
db: AsyncSession, db: AsyncSession,
tenant_id: uuid.UUID, tenant_id: uuid.UUID,
user_id: uuid.UUID, user_id: uuid.UUID,
csv_content: str, csv_content: str | bytes,
dry_run: bool = False, dry_run: bool = False,
field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Import contacts from CSV using unified Contact model fields. """Import contacts from CSV using unified Contact model fields.
CSV columns: firstname, surname, email, phone, mobile, function, department. CSV columns: firstname, surname, email, phone, mobile, function, department.
Maps to Contact fields: firstname, surname, email_1, phone_1, phone_2, function. Maps to Contact fields: firstname, surname, email_1, phone_1, phone_2, function.
Implements partial-failure: valid rows committed, invalid rows collected in error report.
""" """
rows = _parse_csv(csv_content) rows = _parse_csv(csv_content)
total = len(rows) total = len(rows)
valid_rows = [] errors: list[dict] = []
errors = [] valid_rows: list[dict] = []
for idx, row in enumerate(rows, start=1): for idx, row in enumerate(rows, start=1):
# Accept both old (first_name/last_name) and new (firstname/surname) column names # Apply field mapping if provided
firstname = (row.get("firstname") or row.get("first_name") or "").strip() if field_mapping:
surname = (row.get("surname") or row.get("last_name") or "").strip() row = helpers.map_fields(row, field_mapping)
if not firstname and not surname: row = _normalize_contact_row(row)
errors.append({"row": idx, "error": "Missing required field: firstname or surname"}) # Validate: at least one of firstname or surname required
if not row["firstname"] and not row["surname"]:
errors.append({"row": idx, "field": "", "message": "Missing required field: firstname or surname"})
continue
row_errors = helpers.validate_row(row, [], CONTACT_VALIDATORS)
if row_errors:
for e in row_errors:
errors.append({"row": idx, "field": "", "message": e})
else: else:
# Normalize row to use unified field names
row["firstname"] = firstname
row["surname"] = surname
valid_rows.append(row) valid_rows.append(row)
# Count unique failed rows (a row may have multiple errors)
failed_row_count = len({e["row"] for e in errors})
if dry_run: if dry_run:
return { return helpers.build_import_result(
"total": total, total=total,
"valid": len(valid_rows), succeeded=len(valid_rows),
"invalid": len(errors), failed=failed_row_count,
"errors": errors, errors=errors,
"created": [], dry_run=True,
"dry_run": True,
}
created = []
for row in valid_rows:
contact = Contact(
tenant_id=tenant_id,
type="person",
firstname=row["firstname"].strip() or None,
surname=row["surname"].strip() or None,
displayname=f"{row['firstname']} {row['surname']}",
email_1=row.get("email", "").strip() or None,
phone_1=row.get("phone", "").strip() or None,
phone_2=row.get("mobile", "").strip() or None,
owner_id=user_id,
created_by=user_id,
updated_by=user_id,
) )
db.add(contact)
await db.flush()
await log_audit(
db,
tenant_id,
user_id,
"import",
"contact",
contact.id,
changes={"firstname": contact.firstname, "surname": contact.surname},
)
created.append(_contact_to_dict(contact))
return { created: list[dict] = []
"total": total, failed: list[dict] = []
"valid": len(valid_rows), succeeded = 0
"invalid": len(errors),
"errors": errors, for idx, row in enumerate(valid_rows, start=1):
"created": created, try:
"dry_run": False, contact = Contact(
} tenant_id=tenant_id,
type="person",
firstname=row["firstname"].strip() or None,
surname=row["surname"].strip() or None,
displayname=f"{row['firstname']} {row['surname']}".strip(),
email_1=row.get("email", "").strip() or None,
phone_1=row.get("phone", "").strip() or None,
phone_2=row.get("mobile", "").strip() or None,
owner_id=user_id,
created_by=user_id,
updated_by=user_id,
)
db.add(contact)
await db.flush()
await log_audit(
db,
tenant_id,
user_id,
"import",
"contact",
contact.id,
changes={"firstname": contact.firstname, "surname": contact.surname},
)
created.append(_contact_to_dict(contact))
succeeded += 1
except Exception as exc:
logger.warning("Import contacts: row %d failed: %s", idx, exc)
await db.rollback()
failed.append({"row": idx, "field": "", "message": str(exc)})
if succeeded > 0:
try:
await db.commit()
except Exception as exc:
logger.error("Import contacts: commit failed: %s", exc)
await db.rollback()
return helpers.build_import_result(
total=total,
succeeded=0,
failed=len(valid_rows),
errors=errors + [{"row": 0, "field": "", "message": f"Commit failed: {exc}"}],
created=[],
)
# Count unique failed rows from validation errors + runtime failures
all_failed_rows = {e["row"] for e in errors} | {f["row"] for f in failed}
return helpers.build_import_result(
total=total,
succeeded=succeeded,
failed=len(all_failed_rows),
errors=errors + failed,
created=created,
)
async def import_csv( async def import_csv(
db: AsyncSession, db: AsyncSession,
tenant_id: uuid.UUID, tenant_id: uuid.UUID,
user_id: uuid.UUID, user_id: uuid.UUID,
csv_content: str, csv_content: str | bytes,
entity_type: str, entity_type: str,
dry_run: bool = False, dry_run: bool = False,
field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Generic CSV import dispatcher based on entity_type ('companies' or 'contacts').""" """Generic CSV import dispatcher based on entity_type ('companies' or 'contacts')."""
if entity_type == "companies": if entity_type == "companies":
return await import_companies(db, tenant_id, user_id, csv_content, dry_run=dry_run) return await import_companies(db, tenant_id, user_id, csv_content, dry_run=dry_run, field_mapping=field_mapping)
elif entity_type == "contacts": elif entity_type == "contacts":
return await import_contacts(db, tenant_id, user_id, csv_content, dry_run=dry_run) return await import_contacts(db, tenant_id, user_id, csv_content, dry_run=dry_run, field_mapping=field_mapping)
else: else:
return { return helpers.build_import_result(
"total": 0, total=0,
"valid": 0, succeeded=0,
"invalid": 0, failed=1,
"errors": [{"row": 0, "error": f"Unknown entity_type: {entity_type}"}], errors=[{"row": 0, "field": "", "message": f"Unknown entity_type: {entity_type}"}],
"created": [], )
"dry_run": dry_run,
}
# ─── Export functions ────────────────────────────────────────────────────────
async def export_contacts_csv( async def export_contacts_csv(
db: AsyncSession, db: AsyncSession,
tenant_id: uuid.UUID, tenant_id: uuid.UUID,
@@ -236,28 +358,24 @@ async def export_contacts_csv(
result = await db.execute(q) result = await db.execute(q)
contacts = result.scalars().all() contacts = result.scalars().all()
output = io.StringIO() headers = ["id", "type", "firstname", "surname", "name", "email", "phone", "mobile", "city", "postalcode", "country"]
writer = csv.writer(output) rows = [
writer.writerow( {
["id", "type", "firstname", "surname", "name", "email", "phone", "mobile", "city", "postalcode", "country"] "id": str(c.id),
) "type": c.type or "person",
for c in contacts: "firstname": c.firstname or "",
writer.writerow( "surname": c.surname or "",
[ "name": c.name or "",
str(c.id), "email": c.email_1 or "",
c.type or "person", "phone": c.phone_1 or "",
c.firstname or "", "mobile": c.phone_2 or "",
c.surname or "", "city": c.mailing_city or "",
c.name or "", "postalcode": c.mailing_postalcode or "",
c.email_1 or "", "country": c.mailing_country or "",
c.phone_1 or "", }
c.phone_2 or "", for c in contacts
c.mailing_city or "", ]
c.mailing_postalcode or "", return helpers.write_csv(rows, headers).decode("utf-8")
c.mailing_country or "",
]
)
return output.getvalue()
async def export_companies_csv( async def export_companies_csv(
@@ -283,23 +401,97 @@ async def export_companies_csv(
result = await db.execute(q) result = await db.execute(q)
companies = result.scalars().all() companies = result.scalars().all()
output = io.StringIO() headers = ["id", "type", "name", "email", "phone", "website", "city", "postalcode", "country"]
writer = csv.writer(output) rows = [
writer.writerow( {
["id", "type", "name", "email", "phone", "website", "city", "postalcode", "country"] "id": str(c.id),
"type": c.type or "company",
"name": c.name or "",
"email": c.email_1 or "",
"phone": c.phone_1 or "",
"website": c.website or "",
"city": c.mailing_city or "",
"postalcode": c.mailing_postalcode or "",
"country": c.mailing_country or "",
}
for c in companies
]
return helpers.write_csv(rows, headers).decode("utf-8")
# ─── Preview & Validate ──────────────────────────────────────────────────────
def preview_import(
filename: str,
content: bytes,
entity_type: str,
) -> dict[str, Any]:
"""Parse a file and return preview data (first 10 rows) + mapping suggestion.
Does NOT touch the database.
"""
rows = helpers.parse_file(filename, content)
columns = list(rows[0].keys()) if rows else []
preview_rows = rows[:10]
if entity_type == "contacts":
target_fields = CONTACT_TARGET_FIELDS
elif entity_type == "companies":
target_fields = COMPANY_TARGET_FIELDS
else:
target_fields = []
mapping_suggestion = helpers.suggest_mapping(columns, target_fields)
return {
"total_rows": len(rows),
"columns": columns,
"preview_rows": preview_rows,
"mapping_suggestion": mapping_suggestion,
"target_fields": target_fields,
}
def validate_import(
filename: str,
content: bytes,
entity_type: str,
field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
"""Validate all rows against mapping and return error report without importing."""
rows = helpers.parse_file(filename, content)
total = len(rows)
errors: list[dict] = []
valid_count = 0
for idx, row in enumerate(rows, start=1):
if field_mapping:
row = helpers.map_fields(row, field_mapping)
if entity_type == "contacts":
row = _normalize_contact_row(row)
if not row["firstname"] and not row["surname"]:
errors.append({"row": idx, "field": "", "message": "Missing required field: firstname or surname"})
continue
row_errors = helpers.validate_row(row, [], CONTACT_VALIDATORS)
elif entity_type == "companies":
row = _normalize_company_row(row)
row_errors = helpers.validate_row(row, ["name"], COMPANY_VALIDATORS)
else:
errors.append({"row": idx, "field": "", "message": f"Unknown entity_type: {entity_type}"})
continue
if row_errors:
for e in row_errors:
errors.append({"row": idx, "field": "", "message": e})
else:
valid_count += 1
# Count unique failed rows
failed_row_count = len({e["row"] for e in errors})
return helpers.build_import_result(
total=total,
succeeded=valid_count,
failed=failed_row_count,
errors=errors,
dry_run=True,
) )
for c in companies:
writer.writerow(
[
str(c.id),
c.type or "company",
c.name or "",
c.email_1 or "",
c.phone_1 or "",
c.website or "",
c.mailing_city or "",
c.mailing_postalcode or "",
c.mailing_country or "",
]
)
return output.getvalue()
+109
View File
@@ -2092,4 +2092,113 @@ Plugins deklarieren ihre API-Prefixe im Manifest (`routes.prefix`). Plugin-API-
--- ---
## 31. Import/Export Handler auf Shared Helpers
LeoCRM stellt wiederverwendbare Bausteine für Import/Export-Funktionalität bereit. Plugins können eigene Import/Export-Handler auf dieser Basis aufsetzen.
### 31.1 Shared Helpers (`app/services/import_export_helpers.py`)
Die folgenden Funktionen sind verfügbar und können von jedem Plugin importiert werden:
```python
from app.services.import_export_helpers import (
parse_csv, # CSV → list[dict] mit Encoding-Detection
parse_json, # JSON → list[dict]
parse_xlsx, # XLSX → list[dict] (openpyxl)
write_csv, # list[dict] → CSV bytes
write_json, # list[dict] → JSON bytes
write_xlsx, # list[dict] → XLSX bytes
map_fields, # Source-Column → Target-Field Mapping
suggest_mapping, # Auto-Mapping-Vorschlag
validate_row, # Zeilen-Validierung mit required + validators
build_error_report, # Strukturierter Fehler-Report
build_import_result, # Standardisiertes Import-Ergebnis
detect_format, # Format-Erkennung (csv/json/xlsx)
parse_file, # Auto-Detect + Parse
)
```
### 31.2 Eigener Import-Handler
Ein Plugin kann einen eigenen Import-Handler erstellen:
```python
from app.services.import_export_helpers import parse_file, map_fields, validate_row, build_import_result
async def import_my_entity(db, tenant_id, user_id, content: bytes, filename: str, field_mapping: dict | None = None):
rows = parse_file(filename, content)
total = len(rows)
errors = []
valid_rows = []
for idx, row in enumerate(rows, start=1):
if field_mapping:
row = map_fields(row, field_mapping)
row_errors = validate_row(row, required=["name"], validators={"email": {"type": "email"}})
if row_errors:
for e in row_errors:
errors.append({"row": idx, "field": "", "message": e})
else:
valid_rows.append(row)
# ... DB-Insert mit Partial-Failure ...
for idx, row in enumerate(valid_rows, start=1):
try:
# Insert entity
pass
except Exception as exc:
errors.append({"row": idx, "field": "", "message": str(exc)})
failed_row_count = len({e["row"] for e in errors})
return build_import_result(
total=total,
succeeded=len(valid_rows),
failed=failed_row_count,
errors=errors,
)
```
### 31.3 Partial-Failure-Semantik
Import-Handler müssen Partial-Failure implementieren:
1. **Validierung pro Zeile**: `validate_row()` prüft required fields und validators
2. **Fehlerhafte Zeilen sammeln**: Fehler werden in `errors`-Liste mit `{row, field, message}` gesammelt
3. **Erfolgreiche Zeilen committen**: Gültige Zeilen werden in DB geschrieben, pro Zeile try/except
4. **Status-Klassifizierung**: `build_import_result()` setzt Status auf `success`, `partial_success`, oder `failed`
5. **Fehler-Report**: `build_error_report()` erstellt strukturierten Report mit `total_errors` und `errors`-Liste
### 31.4 Background Processing für große Imports
Für Dateien > 1000 Zeilen soll der Import als ARQ-Job laufen:
```python
from app.services.import_export_jobs import create_import_job, get_import_job_status
# In Route:
if len(rows) > 1000:
job_id = await create_import_job(
entity_type="my_entity",
csv_content=content.decode("utf-8"),
tenant_id=tenant_id,
user_id=user_id,
field_mapping=mapping,
)
return {"status": "pending", "job_id": job_id}
```
Job-Status wird in Redis gespeichert (`leocrm:import_job:{job_id}`) mit TTL 1h.
### 31.5 Frontend-Integration
Das Frontend nutzt die API-Client-Funktionen aus `importExport.ts`:
- `previewImport(file, entityType)` — Vorschau mit Mapping-Vorschlag
- `validateImport(file, entityType, fieldMapping)` — Validierung ohne Import
- `importCsv(file, entityType, dryRun, fieldMapping)` — Import mit optionalem Mapping
- `getImportJobStatus(jobId)` — Polling für Background-Jobs
- `exportData(entityType, format)` — Export als CSV/XLSX/JSON
---
*This document is authoritative for all plugin development at LeoCRM.* *This document is authoritative for all plugin development at LeoCRM.*
+147 -8
View File
@@ -2,34 +2,87 @@ import { apiClient } from './client';
// ─── Types ────────────────────────────────────────────────────────────────── // ─── Types ──────────────────────────────────────────────────────────────────
export interface ImportError {
row: number;
field: string;
message: string;
}
export interface ErrorReport {
total_errors: number;
errors: ImportError[];
}
export interface ImportResult { export interface ImportResult {
total?: number; total?: number;
created?: number; succeeded?: number;
updated?: number; failed?: number;
status?: 'success' | 'partial_success' | 'failed' | 'pending' | 'processing' | 'completed';
dry_run?: boolean;
error_report?: ErrorReport;
created?: Record<string, unknown>[];
errors?: string[]; errors?: string[];
warnings?: string[]; warnings?: string[];
rows?: Record<string, unknown>[]; rows?: Record<string, unknown>[];
preview?: boolean; preview?: boolean;
// Legacy compat
valid?: number;
invalid?: number;
job_id?: string;
message?: string;
}
export interface PreviewResult {
total_rows: number;
columns: string[];
preview_rows: Record<string, string>[];
mapping_suggestion: Record<string, string>;
target_fields: string[];
}
export interface ValidateResult {
total: number;
succeeded: number;
failed: number;
status: string;
dry_run: boolean;
error_report: ErrorReport;
created: Record<string, unknown>[];
}
export interface JobStatus {
status: 'pending' | 'processing' | 'completed' | 'partial_success' | 'failed';
progress?: number;
total?: number;
succeeded?: number;
failed?: number;
error_report?: ErrorReport;
created_count?: number;
error?: string;
} }
export type EntityType = 'companies' | 'contacts'; export type EntityType = 'companies' | 'contacts';
export type ExportFormat = 'csv' | 'xlsx'; export type ExportFormat = 'csv' | 'xlsx' | 'json';
// ─── Import ───────────────────────────────────────────────────────────────── // ─── Import ─────────────────────────────────────────────────────────────────
/** /**
* Import a CSV file. When dryRun is true, a preview is returned without * Import a CSV/JSON/XLSX file. When dryRun is true, a preview is returned without
* writing to the database. * writing to the database.
*/ */
export async function importCsv( export async function importCsv(
file: File, file: File,
entityType: string, entityType: string,
dryRun: boolean dryRun: boolean,
fieldMapping?: Record<string, string>,
): Promise<ImportResult> { ): Promise<ImportResult> {
const url = dryRun ? '/import/preview' : '/import'; const url = dryRun ? '/import/preview' : '/import';
const formData = new FormData(); const formData = new FormData();
formData.append('file', file); formData.append('file', file);
formData.append('entity_type', entityType); formData.append('entity_type', entityType);
if (fieldMapping) {
formData.append('field_mapping', JSON.stringify(fieldMapping));
}
const response = await apiClient.post<ImportResult>(url, formData, { const response = await apiClient.post<ImportResult>(url, formData, {
headers: { 'Content-Type': 'multipart/form-data' }, headers: { 'Content-Type': 'multipart/form-data' },
@@ -37,14 +90,98 @@ export async function importCsv(
return response.data; return response.data;
} }
/**
* Preview a file: parse, return first 10 rows + columns + mapping suggestion.
*/
export async function previewImport(
file: File,
entityType: string,
): Promise<PreviewResult> {
const formData = new FormData();
formData.append('file', file);
formData.append('entity_type', entityType);
const response = await apiClient.post<PreviewResult>('/import/preview', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
});
return response.data;
}
/**
* Validate a file against mapping: returns error report without importing.
*/
export async function validateImport(
file: File,
entityType: string,
fieldMapping?: Record<string, string>,
): Promise<ValidateResult> {
const formData = new FormData();
formData.append('file', file);
formData.append('entity_type', entityType);
if (fieldMapping) {
formData.append('field_mapping', JSON.stringify(fieldMapping));
}
const response = await apiClient.post<ValidateResult>('/import/validate', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
});
return response.data;
}
/**
* Import contacts specifically.
*/
export async function importContacts(
file: File,
fieldMapping?: Record<string, string>,
): Promise<ImportResult> {
const formData = new FormData();
formData.append('file', file);
if (fieldMapping) {
formData.append('field_mapping', JSON.stringify(fieldMapping));
}
const response = await apiClient.post<ImportResult>('/import/contacts', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
});
return response.data;
}
/**
* Import companies specifically.
*/
export async function importCompanies(
file: File,
fieldMapping?: Record<string, string>,
): Promise<ImportResult> {
const formData = new FormData();
formData.append('file', file);
if (fieldMapping) {
formData.append('field_mapping', JSON.stringify(fieldMapping));
}
const response = await apiClient.post<ImportResult>('/import/companies', formData, {
headers: { 'Content-Type': 'multipart/form-data' },
});
return response.data;
}
/**
* Get status of a background import job.
*/
export async function getImportJobStatus(jobId: string): Promise<JobStatus> {
const response = await apiClient.get<JobStatus>(`/import/status/${jobId}`);
return response.data;
}
// ─── Export ───────────────────────────────────────────────────────────────── // ─── Export ─────────────────────────────────────────────────────────────────
/** /**
* Export data as CSV or XLSX and trigger a browser download. * Export data as CSV, XLSX, or JSON and trigger a browser download.
*/ */
export async function exportData( export async function exportData(
entityType: string, entityType: string,
format: string format: string,
): Promise<void> { ): Promise<void> {
const url = `/export?entity_type=${encodeURIComponent(entityType)}&format=${encodeURIComponent(format)}`; const url = `/export?entity_type=${encodeURIComponent(entityType)}&format=${encodeURIComponent(format)}`;
const response = await apiClient.get(url, { responseType: 'blob' }); const response = await apiClient.get(url, { responseType: 'blob' });
@@ -52,7 +189,9 @@ export async function exportData(
const blob = new Blob([response.data], { const blob = new Blob([response.data], {
type: format === 'xlsx' type: format === 'xlsx'
? 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ? 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
: 'text/csv', : format === 'json'
? 'application/json'
: 'text/csv',
}); });
const blobUrl = URL.createObjectURL(blob); const blobUrl = URL.createObjectURL(blob);
@@ -18,6 +18,7 @@ const ENTITY_OPTIONS = [
const FORMAT_OPTIONS = [ const FORMAT_OPTIONS = [
{ value: 'csv', label: 'CSV' }, { value: 'csv', label: 'CSV' },
{ value: 'xlsx', label: 'XLSX' }, { value: 'xlsx', label: 'XLSX' },
{ value: 'json', label: 'JSON' },
]; ];
// ─── Component ────────────────────────────────────────────────────────────── // ─── Component ──────────────────────────────────────────────────────────────
File diff suppressed because it is too large Load Diff
+590 -64
View File
@@ -1,12 +1,28 @@
"""Import/export tests — ACs 20-21: CSV import, dry-run preview.""" """Import/export tests — helpers, service partial-failure, preview/validate routes, exports."""
from __future__ import annotations from __future__ import annotations
import json
import pytest import pytest
from httpx import AsyncClient from httpx import AsyncClient
from app.services.import_export_helpers import (
parse_csv,
parse_json,
write_csv,
write_json,
map_fields,
validate_row,
build_error_report,
build_import_result,
suggest_mapping,
detect_format,
)
from app.services import import_export_service
from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users
# ─── Test data ───────────────────────────────────────────────────────────────
CSV_COMPANIES = """name,industry,phone,email,website CSV_COMPANIES = """name,industry,phone,email,website
ImportCorp,IT,123456,import@example.com,https://import.example ImportCorp,IT,123456,import@example.com,https://import.example
TechImport,Finance,654321,tech@example.com,https://tech.example TechImport,Finance,654321,tech@example.com,https://tech.example
@@ -22,13 +38,230 @@ Alice,Wonderland,alice@example.com,123,456,Manager,Sales
Bob,Builder,bob@example.com,789,012,Developer,Tech Bob,Builder,bob@example.com,789,012,Developer,Tech
""" """
CSV_CONTACTS_PARTIAL = """firstname,surname,email,phone
Alice,Wonderland,alice@example.com,123
,,bad@example.com,456
Bob,Builder,bob@example.com,789
,,not-an-email,999
Charlie,Chaplin,charlie@example.com,555
"""
CSV_COMPANIES_PARTIAL = """name,email,phone
ValidCorp1,corp1@example.com,123
,corp2@example.com,456
ValidCorp2,corp2@example.com,789
,bad-email,999
ValidCorp3,corp3@example.com,111
"""
JSON_CONTACTS = json.dumps([
{"firstname": "Json", "surname": "User", "email": "json@example.com"},
{"firstname": "Another", "surname": "Person", "email": "another@example.com"},
])
# ─── Helper unit tests ───────────────────────────────────────────────────────
class TestHelpers:
"""Unit tests for import_export_helpers functions."""
def test_parse_csv_basic(self):
"""parse_csv parses CSV bytes into list of dicts."""
content = b"name,email\nAlice,alice@example.com\nBob,bob@example.com\n"
rows = parse_csv(content)
assert len(rows) == 2
assert rows[0]["name"] == "Alice"
assert rows[0]["email"] == "alice@example.com"
assert rows[1]["name"] == "Bob"
def test_parse_csv_with_bom(self):
"""parse_csv handles UTF-8 BOM."""
content = b"\xef\xbb\xbfname,email\nAlice,alice@example.com\n"
rows = parse_csv(content)
assert len(rows) == 1
assert rows[0]["name"] == "Alice"
def test_parse_json_array(self):
"""parse_json parses JSON array into list of dicts."""
content = json.dumps([{"a": "1"}, {"a": "2"}]).encode()
rows = parse_json(content)
assert len(rows) == 2
assert rows[0]["a"] == "1"
assert rows[1]["a"] == "2"
def test_parse_json_single_object(self):
"""parse_json wraps single JSON object into list."""
content = json.dumps({"a": "1"}).encode()
rows = parse_json(content)
assert len(rows) == 1
assert rows[0]["a"] == "1"
def test_write_csv_basic(self):
"""write_csv produces correct CSV bytes."""
rows = [{"name": "Alice", "email": "alice@example.com"}]
result = write_csv(rows, ["name", "email"])
text = result.decode("utf-8")
assert "name,email" in text
assert "Alice,alice@example.com" in text
def test_write_csv_missing_fields(self):
"""write_csv fills missing fields with empty string."""
rows = [{"name": "Alice"}]
result = write_csv(rows, ["name", "email", "phone"])
text = result.decode("utf-8")
assert "Alice," in text
def test_write_json_basic(self):
"""write_json produces correct JSON bytes."""
rows = [{"name": "Alice"}, {"name": "Bob"}]
result = write_json(rows)
data = json.loads(result)
assert len(data) == 2
assert data[0]["name"] == "Alice"
assert data[1]["name"] == "Bob"
def test_map_fields_basic(self):
"""map_fields maps source columns to target fields."""
row = {"first_name": "Alice", "last_name": "Wonder", "email": "alice@example.com"}
mapping = {"first_name": "firstname", "last_name": "surname", "email": "email"}
result = map_fields(row, mapping)
assert result["firstname"] == "Alice"
assert result["surname"] == "Wonder"
assert result["email"] == "alice@example.com"
def test_map_fields_skip_missing(self):
"""map_fields skips source columns not in row."""
row = {"first_name": "Alice"}
mapping = {"first_name": "firstname", "last_name": "surname"}
result = map_fields(row, mapping)
assert "firstname" in result
assert "surname" not in result
def test_validate_row_required(self):
"""validate_row detects missing required fields."""
row = {"name": "", "email": "test@example.com"}
errors = validate_row(row, ["name"])
assert len(errors) == 1
assert "name" in errors[0]
def test_validate_row_valid(self):
"""validate_row returns empty list for valid row."""
row = {"name": "Alice", "email": "alice@example.com"}
errors = validate_row(row, ["name"], {"email": {"type": "email"}})
assert len(errors) == 0
def test_validate_row_email_validator(self):
"""validate_row detects invalid email."""
row = {"name": "Alice", "email": "not-an-email"}
errors = validate_row(row, ["name"], {"email": {"type": "email"}})
assert len(errors) == 1
assert "email" in errors[0]
def test_validate_row_url_validator(self):
"""validate_row detects invalid URL."""
row = {"website": "not-a-url"}
errors = validate_row(row, [], {"website": {"type": "url"}})
assert len(errors) == 1
assert "URL" in errors[0]
def test_validate_row_max_length(self):
"""validate_row enforces max_length."""
row = {"name": "A" * 100}
errors = validate_row(row, [], {"name": {"max_length": 50}})
assert len(errors) == 1
assert "max length" in errors[0]
def test_build_error_report_structure(self):
"""build_error_report returns correct structure."""
errors = [
{"row": 1, "field": "name", "message": "Missing required"},
{"row": 2, "field": "email", "message": "Invalid email"},
]
report = build_error_report(errors)
assert report["total_errors"] == 2
assert len(report["errors"]) == 2
assert report["errors"][0]["row"] == 1
assert report["errors"][0]["message"] == "Missing required"
def test_build_error_report_empty(self):
"""build_error_report handles empty list."""
report = build_error_report([])
assert report["total_errors"] == 0
assert report["errors"] == []
def test_build_import_result_success(self):
"""build_import_result returns 'success' status when all succeed."""
result = build_import_result(total=10, succeeded=10, failed=0)
assert result["status"] == "success"
assert result["succeeded"] == 10
assert result["failed"] == 0
def test_build_import_result_partial(self):
"""build_import_result returns 'partial_success' when some fail."""
result = build_import_result(total=10, succeeded=7, failed=3)
assert result["status"] == "partial_success"
def test_build_import_result_all_failed(self):
"""build_import_result returns 'failed' when all fail."""
result = build_import_result(total=10, succeeded=0, failed=10)
assert result["status"] == "failed"
def test_suggest_mapping_exact_match(self):
"""suggest_mapping matches exact column names."""
mapping = suggest_mapping(["firstname", "surname", "email"], ["firstname", "surname", "email"])
assert mapping["firstname"] == "firstname"
assert mapping["surname"] == "surname"
assert mapping["email"] == "email"
def test_suggest_mapping_alias(self):
"""suggest_mapping matches common aliases."""
mapping = suggest_mapping(["first_name", "last_name"], ["firstname", "surname"])
assert mapping["first_name"] == "firstname"
assert mapping["last_name"] == "surname"
def test_detect_format_csv(self):
"""detect_format identifies CSV from filename."""
assert detect_format("data.csv", b"name,email\n") == "csv"
def test_detect_format_json(self):
"""detect_format identifies JSON from filename."""
assert detect_format("data.json", b"[]") == "json"
def test_detect_format_xlsx(self):
"""detect_format identifies XLSX from filename."""
assert detect_format("data.xlsx", b"PK\x03\x04") == "xlsx"
# ─── Service partial-failure tests ───────────────────────────────────────────
@pytest.mark.asyncio @pytest.mark.asyncio
class TestImportCompanies: class TestImportCompaniesPartialFailure:
"""AC 20: CSV import for companies.""" """Import companies with partial-failure semantics."""
async def test_import_companies_csv_returns_200(self, client: AsyncClient, db_session): async def test_import_companies_partial_failure(self, client: AsyncClient, db_session):
"""AC 20: POST /api/v1/import CSV + entity_type=companies -> 200 + result.""" """Import companies: 3 success, 2 failed (empty name + invalid email)."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES_PARTIAL.encode(), "text/csv")}
data = {"entity_type": "companies"}
resp = await client.post(
"/api/v1/import",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 5
assert result["succeeded"] == 3
assert result["failed"] == 2
assert result["status"] == "partial_success"
assert len(result["created"]) == 3
# 2 failed rows, but 3 total error messages (row 4 has 2 errors)
assert result["error_report"]["total_errors"] == 3
async def test_import_companies_all_valid(self, client: AsyncClient, db_session):
"""Import companies: all rows valid."""
await seed_tenant_and_users(db_session) await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com") await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")} files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
@@ -42,13 +275,288 @@ class TestImportCompanies:
assert resp.status_code == 200 assert resp.status_code == 200
result = resp.json() result = resp.json()
assert result["total"] == 2 assert result["total"] == 2
assert result["valid"] == 2 assert result["succeeded"] == 2
assert result["invalid"] == 0 assert result["failed"] == 0
assert result["status"] == "success"
@pytest.mark.asyncio
class TestImportContactsPartialFailure:
"""Import contacts with partial-failure semantics."""
async def test_import_contacts_partial_failure(self, client: AsyncClient, db_session):
"""Import contacts: 3 success, 2 failed (empty names + invalid email)."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("contacts.csv", CSV_CONTACTS_PARTIAL.encode(), "text/csv")}
data = {"entity_type": "contacts"}
resp = await client.post(
"/api/v1/import",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 5
assert result["succeeded"] == 3
assert result["failed"] == 2
assert result["status"] == "partial_success"
assert len(result["created"]) == 3
async def test_import_contacts_all_valid(self, client: AsyncClient, db_session):
"""Import contacts: all rows valid."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
data = {"entity_type": "contacts"}
resp = await client.post(
"/api/v1/import",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 2
assert result["succeeded"] == 2
assert result["status"] == "success"
# ─── Preview & Validate route tests ──────────────────────────────────────────
@pytest.mark.asyncio
class TestImportPreview:
"""Preview endpoint returns first 10 rows + mapping suggestion."""
async def test_preview_returns_columns_and_rows(self, client: AsyncClient, db_session):
"""POST /import/preview returns columns, preview_rows, mapping_suggestion."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
data = {"entity_type": "companies"}
resp = await client.post(
"/api/v1/import/preview",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total_rows"] == 2
assert "name" in result["columns"]
assert "email" in result["columns"]
assert len(result["preview_rows"]) == 2
assert "mapping_suggestion" in result
assert "target_fields" in result
async def test_preview_contacts_mapping_suggestion(self, client: AsyncClient, db_session):
"""Preview contacts: mapping suggestion maps first_name→firstname."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
data = {"entity_type": "contacts"}
resp = await client.post(
"/api/v1/import/preview",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
mapping = result["mapping_suggestion"]
# first_name should map to firstname
assert mapping.get("first_name") == "firstname"
# last_name should map to surname
assert mapping.get("last_name") == "surname"
async def test_preview_no_db_changes(self, client: AsyncClient, db_session):
"""Preview does not create any records."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
data = {"entity_type": "companies"}
resp = await client.post(
"/api/v1/import/preview",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
# Verify no companies were created
list_resp = await client.get("/api/v1/companies", headers=ORIGIN_HEADER)
names = [item["name"] for item in list_resp.json()["items"]]
assert "ImportCorp" not in names
assert "TechImport" not in names
@pytest.mark.asyncio
class TestImportValidate:
"""Validate endpoint returns error report without importing."""
async def test_validate_returns_error_report(self, client: AsyncClient, db_session):
"""POST /import/validate returns error report for invalid rows."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES_PARTIAL.encode(), "text/csv")}
data = {"entity_type": "companies"}
resp = await client.post(
"/api/v1/import/validate",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 5
assert result["succeeded"] == 3
assert result["failed"] == 2
assert result["dry_run"] is True
# 2 failed rows, but 3 total error messages (row 4 has 2 errors)
assert result["error_report"]["total_errors"] == 3
async def test_validate_no_db_changes(self, client: AsyncClient, db_session):
"""Validate does not create any records."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
data = {"entity_type": "companies"}
resp = await client.post(
"/api/v1/import/validate",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
list_resp = await client.get("/api/v1/companies", headers=ORIGIN_HEADER)
names = [item["name"] for item in list_resp.json()["items"]]
assert "ImportCorp" not in names
async def test_validate_with_field_mapping(self, client: AsyncClient, db_session):
"""Validate with explicit field mapping."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
mapping = json.dumps({
"first_name": "firstname",
"last_name": "surname",
"email": "email",
"phone": "phone",
"mobile": "mobile",
"position": "function",
"department": "department",
})
data = {"entity_type": "contacts", "field_mapping": mapping}
resp = await client.post(
"/api/v1/import/validate",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 2
assert result["succeeded"] == 2
assert result["failed"] == 0
# ─── Job status route tests ──────────────────────────────────────────────────
@pytest.mark.asyncio
class TestImportJobStatus:
"""Job status endpoint."""
async def test_job_status_not_found(self, client: AsyncClient, db_session):
"""GET /import/status/{nonexistent_id} returns 404."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
resp = await client.get(
"/api/v1/import/status/nonexistent-job-id",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 404
# ─── Export route tests ──────────────────────────────────────────────────────
@pytest.mark.asyncio
class TestExport:
"""Export endpoints."""
async def test_export_contacts_csv(self, client: AsyncClient, db_session):
"""GET /export?entity_type=contacts&format=csv returns CSV file."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
resp = await client.get(
"/api/v1/export?entity_type=contacts&format=csv",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
assert "text/csv" in resp.headers.get("content-type", "")
async def test_export_companies_csv(self, client: AsyncClient, db_session):
"""GET /export?entity_type=companies&format=csv returns CSV file."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
resp = await client.get(
"/api/v1/export?entity_type=companies&format=csv",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
assert "text/csv" in resp.headers.get("content-type", "")
async def test_export_contacts_xlsx(self, client: AsyncClient, db_session):
"""GET /export?entity_type=contacts&format=xlsx returns XLSX file."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
resp = await client.get(
"/api/v1/export?entity_type=contacts&format=xlsx",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
ct = resp.headers.get("content-type", "")
# Could be xlsx or csv fallback if openpyxl missing (but it's installed)
assert "spreadsheet" in ct or "text/csv" in ct
async def test_export_contacts_json(self, client: AsyncClient, db_session):
"""GET /export?entity_type=contacts&format=json returns JSON file."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
resp = await client.get(
"/api/v1/export?entity_type=contacts&format=json",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
assert "application/json" in resp.headers.get("content-type", "")
# ─── Direct import route tests (legacy compat) ───────────────────────────────
@pytest.mark.asyncio
class TestImportCompanies:
"""AC 20: CSV import for companies (backward compat)."""
async def test_import_companies_csv_returns_200(self, client: AsyncClient, db_session):
"""POST /api/v1/import CSV + entity_type=companies -> 200 + result."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
data = {"entity_type": "companies"}
resp = await client.post(
"/api/v1/import",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 2
assert result["succeeded"] == 2
assert result["failed"] == 0
assert len(result["created"]) == 2 assert len(result["created"]) == 2
assert result["dry_run"] is False
async def test_import_companies_with_invalid_rows(self, client: AsyncClient, db_session): async def test_import_companies_with_invalid_rows(self, client: AsyncClient, db_session):
"""Import CSV with some invalid rows — should report errors but import valid ones.""" """Import CSV with some invalid rows — partial success."""
await seed_tenant_and_users(db_session) await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com") await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES_INVALID.encode(), "text/csv")} files = {"file": ("companies.csv", CSV_COMPANIES_INVALID.encode(), "text/csv")}
@@ -62,64 +570,15 @@ class TestImportCompanies:
assert resp.status_code == 200 assert resp.status_code == 200
result = resp.json() result = resp.json()
assert result["total"] == 2 assert result["total"] == 2
assert result["valid"] == 1 assert result["succeeded"] == 1
assert result["invalid"] == 1 assert result["failed"] == 1
assert len(result["errors"]) == 1 assert result["status"] == "partial_success"
assert len(result["created"]) == 1 assert len(result["created"]) == 1
@pytest.mark.asyncio
class TestImportPreview:
"""AC 21: Dry-run preview (no DB changes)."""
async def test_import_preview_no_db_changes(self, client: AsyncClient, db_session):
"""AC 21: POST /api/v1/import/preview CSV -> 200 + dry-run (no DB changes)."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
data = {"entity_type": "companies"}
resp = await client.post(
"/api/v1/import/preview",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 2
assert result["valid"] == 2
assert result["dry_run"] is True
assert len(result["created"]) == 0 # No actual creations
# Verify no companies were actually created
list_resp = await client.get("/api/v1/companies", headers=ORIGIN_HEADER)
names = [item["name"] for item in list_resp.json()["items"]]
assert "ImportCorp" not in names
assert "TechImport" not in names
async def test_import_preview_contacts_no_db_changes(self, client: AsyncClient, db_session):
"""Preview import for contacts — dry-run."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
data = {"entity_type": "contacts"}
resp = await client.post(
"/api/v1/import/preview",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 2
assert result["valid"] == 2
assert result["dry_run"] is True
assert len(result["created"]) == 0
@pytest.mark.asyncio @pytest.mark.asyncio
class TestImportContacts: class TestImportContacts:
"""Import contacts via CSV.""" """Import contacts via CSV (backward compat)."""
async def test_import_contacts_csv_returns_200(self, client: AsyncClient, db_session): async def test_import_contacts_csv_returns_200(self, client: AsyncClient, db_session):
"""Import contacts via CSV.""" """Import contacts via CSV."""
@@ -136,9 +595,76 @@ class TestImportContacts:
assert resp.status_code == 200 assert resp.status_code == 200
result = resp.json() result = resp.json()
assert result["total"] == 2 assert result["total"] == 2
assert result["valid"] == 2 assert result["succeeded"] == 2
assert result["invalid"] == 0 assert result["failed"] == 0
assert len(result["created"]) == 2 assert len(result["created"]) == 2
# Verify contacts appear in list # Verify contacts appear in list
list_resp = await client.get("/api/v1/contacts", headers=ORIGIN_HEADER) list_resp = await client.get("/api/v1/contacts", headers=ORIGIN_HEADER)
assert list_resp.json()["total"] >= 2 assert list_resp.json()["total"] >= 2
# ─── JSON import test ────────────────────────────────────────────────────────
@pytest.mark.asyncio
class TestImportJson:
"""Import from JSON format."""
async def test_import_contacts_json(self, client: AsyncClient, db_session):
"""Import contacts from JSON file."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("contacts.json", JSON_CONTACTS.encode(), "application/json")}
data = {"entity_type": "contacts"}
resp = await client.post(
"/api/v1/import",
files=files,
data=data,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 2
assert result["succeeded"] == 2
assert len(result["created"]) == 2
# ─── Dedicated import routes ─────────────────────────────────────────────────
@pytest.mark.asyncio
class TestImportContactsRoute:
"""POST /import/contacts dedicated route."""
async def test_import_contacts_route(self, client: AsyncClient, db_session):
"""POST /import/contacts imports contacts."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
resp = await client.post(
"/api/v1/import/contacts",
files=files,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 2
assert result["succeeded"] == 2
@pytest.mark.asyncio
class TestImportCompaniesRoute:
"""POST /import/companies dedicated route."""
async def test_import_companies_route(self, client: AsyncClient, db_session):
"""POST /import/companies imports companies."""
await seed_tenant_and_users(db_session)
await login_client(client, "admin@tenanta.com")
files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
resp = await client.post(
"/api/v1/import/companies",
files=files,
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
result = resp.json()
assert result["total"] == 2
assert result["succeeded"] == 2