Agent Zero
e7ae0ad5ce
C5-BASE: app/services/import_export_helpers.py (NEU, 352 Zeilen)
- parse_csv/json/xlsx, write_csv/json/xlsx, map_fields, suggest_mapping
- validate_row, build_error_report, build_import_result, detect_format
C5-PREVIEW: POST /import/preview + POST /import/validate
- Preview gibt erste 10 Zeilen + Spalten + Mapping-Vorschlag
- Validate gibt Fehler-Report ohne Import
C5-JOB: app/services/import_export_jobs.py (NEU, 165 Zeilen)
- ARQ Background Job für Files >1000 Zeilen
- Job-Status: pending/processing/completed/partial_success/failed
- GET /import/status/{job_id} — Status + Progress + Fehler-Report
- Partial-Failure: try/except pro Zeile, fehlerhafte gesammelt, erfolgreiche committet
C5-CONTACT+C5-COMPANY: Handler auf Shared Helpers umgestellt
C5-UI: ImportWizard.tsx (5 Steps: Upload→Preview→Validation→Review→Result)
C5-TEST: 45 Tests in test_import_export.py — alle grün
C5-DOC: Plugin-Dev-Guide Kapitel 31 (Import/Export Handler)
374 lines
12 KiB
Python
374 lines
12 KiB
Python
"""Import/export routes — CSV/JSON/XLSX import, preview, validate, background jobs, CSV/XLSX export."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import uuid
|
|
|
|
from fastapi import (
|
|
APIRouter,
|
|
Depends,
|
|
File,
|
|
Form,
|
|
HTTPException,
|
|
Query,
|
|
UploadFile,
|
|
)
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from starlette.responses import StreamingResponse
|
|
|
|
from app.core.db import get_db
|
|
from app.deps import require_permission
|
|
from app.services import import_export_service
|
|
from app.services.import_export_helpers import (
|
|
detect_format,
|
|
parse_file,
|
|
write_csv,
|
|
write_xlsx,
|
|
)
|
|
|
|
router = APIRouter(prefix="/api/v1", tags=["import_export"])
|
|
|
|
# Row threshold for background job processing
|
|
_BACKGROUND_THRESHOLD = 1000
|
|
|
|
|
|
@router.post("/import")
|
|
async def import_csv(
|
|
file: UploadFile = File(...),
|
|
entity_type: str = Form("companies"),
|
|
field_mapping: str = Form(None),
|
|
db: AsyncSession = Depends(get_db),
|
|
current_user: dict = Depends(require_permission("import_export:write")),
|
|
):
|
|
"""Import companies or contacts from CSV/JSON/XLSX file.
|
|
|
|
entity_type: 'companies' or 'contacts'.
|
|
field_mapping: Optional JSON string of source_column -> target_field mapping.
|
|
For files > 1000 rows, the import runs as a background ARQ job.
|
|
"""
|
|
tenant_id = uuid.UUID(current_user["tenant_id"])
|
|
user_id = uuid.UUID(current_user["user_id"])
|
|
|
|
content = await file.read()
|
|
csv_content = content.decode("utf-8", errors="replace")
|
|
|
|
# Parse mapping if provided
|
|
mapping = None
|
|
if field_mapping:
|
|
try:
|
|
mapping = json.loads(field_mapping)
|
|
except json.JSONDecodeError:
|
|
raise HTTPException(status_code=400, detail="Invalid field_mapping JSON")
|
|
|
|
# Check row count for background processing
|
|
try:
|
|
rows = parse_file(file.filename or "upload.csv", content)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}")
|
|
|
|
if len(rows) > _BACKGROUND_THRESHOLD:
|
|
# Enqueue as background job
|
|
from app.services.import_export_jobs import create_import_job
|
|
|
|
try:
|
|
job_id = await create_import_job(
|
|
entity_type=entity_type,
|
|
csv_content=csv_content,
|
|
tenant_id=tenant_id,
|
|
user_id=user_id,
|
|
field_mapping=mapping,
|
|
)
|
|
return {
|
|
"status": "pending",
|
|
"job_id": job_id,
|
|
"message": f"Import enqueued as background job ({len(rows)} rows)",
|
|
"total": len(rows),
|
|
}
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=500, detail=f"Failed to enqueue import job: {exc}")
|
|
|
|
# Synchronous import for smaller files
|
|
result = await import_export_service.import_csv(
|
|
db,
|
|
tenant_id,
|
|
user_id,
|
|
csv_content,
|
|
entity_type=entity_type,
|
|
dry_run=False,
|
|
field_mapping=mapping,
|
|
)
|
|
return result
|
|
|
|
|
|
@router.post("/import/preview")
|
|
async def import_csv_preview(
|
|
file: UploadFile = File(...),
|
|
entity_type: str = Form("companies"),
|
|
current_user: dict = Depends(require_permission("import_export:read")),
|
|
):
|
|
"""Preview CSV import (dry-run — no DB changes).
|
|
|
|
Returns first 10 rows, detected columns, and mapping suggestion.
|
|
"""
|
|
content = await file.read()
|
|
|
|
try:
|
|
result = import_export_service.preview_import(
|
|
filename=file.filename or "upload.csv",
|
|
content=content,
|
|
entity_type=entity_type,
|
|
)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}")
|
|
|
|
return result
|
|
|
|
|
|
@router.post("/import/validate")
|
|
async def import_validate(
|
|
file: UploadFile = File(...),
|
|
entity_type: str = Form("companies"),
|
|
field_mapping: str = Form(None),
|
|
current_user: dict = Depends(require_permission("import_export:read")),
|
|
):
|
|
"""Validate all rows against mapping and return error report without importing."""
|
|
content = await file.read()
|
|
|
|
mapping = None
|
|
if field_mapping:
|
|
try:
|
|
mapping = json.loads(field_mapping)
|
|
except json.JSONDecodeError:
|
|
raise HTTPException(status_code=400, detail="Invalid field_mapping JSON")
|
|
|
|
try:
|
|
result = import_export_service.validate_import(
|
|
filename=file.filename or "upload.csv",
|
|
content=content,
|
|
entity_type=entity_type,
|
|
field_mapping=mapping,
|
|
)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=f"Failed to validate file: {exc}")
|
|
|
|
return result
|
|
|
|
|
|
@router.get("/import/status/{job_id}")
|
|
async def import_job_status(
|
|
job_id: str,
|
|
current_user: dict = Depends(require_permission("import_export:read")),
|
|
):
|
|
"""Get status of a background import job."""
|
|
from app.services.import_export_jobs import get_import_job_status
|
|
|
|
status = await get_import_job_status(job_id)
|
|
if status is None:
|
|
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
|
return status
|
|
|
|
|
|
@router.post("/import/contacts")
|
|
async def import_contacts_route(
|
|
file: UploadFile = File(...),
|
|
field_mapping: str = Form(None),
|
|
db: AsyncSession = Depends(get_db),
|
|
current_user: dict = Depends(require_permission("import_export:write")),
|
|
):
|
|
"""Import contacts from CSV/JSON/XLSX file.
|
|
|
|
For files > 1000 rows, runs as background ARQ job.
|
|
"""
|
|
tenant_id = uuid.UUID(current_user["tenant_id"])
|
|
user_id = uuid.UUID(current_user["user_id"])
|
|
|
|
content = await file.read()
|
|
csv_content = content.decode("utf-8", errors="replace")
|
|
|
|
mapping = None
|
|
if field_mapping:
|
|
try:
|
|
mapping = json.loads(field_mapping)
|
|
except json.JSONDecodeError:
|
|
raise HTTPException(status_code=400, detail="Invalid field_mapping JSON")
|
|
|
|
# Check row count for background processing
|
|
try:
|
|
rows = parse_file(file.filename or "upload.csv", content)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}")
|
|
|
|
if len(rows) > _BACKGROUND_THRESHOLD:
|
|
from app.services.import_export_jobs import create_import_job
|
|
|
|
try:
|
|
job_id = await create_import_job(
|
|
entity_type="contacts",
|
|
csv_content=csv_content,
|
|
tenant_id=tenant_id,
|
|
user_id=user_id,
|
|
field_mapping=mapping,
|
|
)
|
|
return {
|
|
"status": "pending",
|
|
"job_id": job_id,
|
|
"message": f"Import enqueued as background job ({len(rows)} rows)",
|
|
"total": len(rows),
|
|
}
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=500, detail=f"Failed to enqueue import job: {exc}")
|
|
|
|
result = await import_export_service.import_contacts(
|
|
db,
|
|
tenant_id,
|
|
user_id,
|
|
csv_content,
|
|
dry_run=False,
|
|
field_mapping=mapping,
|
|
)
|
|
return result
|
|
|
|
|
|
@router.post("/import/companies")
|
|
async def import_companies_route(
|
|
file: UploadFile = File(...),
|
|
field_mapping: str = Form(None),
|
|
db: AsyncSession = Depends(get_db),
|
|
current_user: dict = Depends(require_permission("import_export:write")),
|
|
):
|
|
"""Import companies from CSV/JSON/XLSX file.
|
|
|
|
For files > 1000 rows, runs as background ARQ job.
|
|
"""
|
|
tenant_id = uuid.UUID(current_user["tenant_id"])
|
|
user_id = uuid.UUID(current_user["user_id"])
|
|
|
|
content = await file.read()
|
|
csv_content = content.decode("utf-8", errors="replace")
|
|
|
|
mapping = None
|
|
if field_mapping:
|
|
try:
|
|
mapping = json.loads(field_mapping)
|
|
except json.JSONDecodeError:
|
|
raise HTTPException(status_code=400, detail="Invalid field_mapping JSON")
|
|
|
|
# Check row count for background processing
|
|
try:
|
|
rows = parse_file(file.filename or "upload.csv", content)
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}")
|
|
|
|
if len(rows) > _BACKGROUND_THRESHOLD:
|
|
from app.services.import_export_jobs import create_import_job
|
|
|
|
try:
|
|
job_id = await create_import_job(
|
|
entity_type="companies",
|
|
csv_content=csv_content,
|
|
tenant_id=tenant_id,
|
|
user_id=user_id,
|
|
field_mapping=mapping,
|
|
)
|
|
return {
|
|
"status": "pending",
|
|
"job_id": job_id,
|
|
"message": f"Import enqueued as background job ({len(rows)} rows)",
|
|
"total": len(rows),
|
|
}
|
|
except Exception as exc:
|
|
raise HTTPException(status_code=500, detail=f"Failed to enqueue import job: {exc}")
|
|
|
|
result = await import_export_service.import_companies(
|
|
db,
|
|
tenant_id,
|
|
user_id,
|
|
csv_content,
|
|
dry_run=False,
|
|
field_mapping=mapping,
|
|
)
|
|
return result
|
|
|
|
|
|
@router.get("/export")
|
|
async def export_data(
|
|
entity_type: str = Query("contacts", pattern="^(contacts|companies)$"),
|
|
format: str = Query("csv", pattern="^(csv|xlsx|json)$"),
|
|
db: AsyncSession = Depends(get_db),
|
|
current_user: dict = Depends(require_permission("import_export:read")),
|
|
):
|
|
"""Export contacts or companies as CSV, XLSX, or JSON file download.
|
|
|
|
Query params:
|
|
- entity_type: 'contacts' or 'companies' (default: contacts)
|
|
- format: 'csv', 'xlsx', or 'json' (default: csv)
|
|
"""
|
|
tenant_id = uuid.UUID(current_user["tenant_id"])
|
|
user_id = uuid.UUID(current_user["user_id"])
|
|
is_system_admin = current_user.get("is_system_admin", False)
|
|
|
|
# Fetch data from the appropriate service function
|
|
if entity_type == "contacts":
|
|
csv_data = await import_export_service.export_contacts_csv(
|
|
db, tenant_id, user_id=user_id, is_system_admin=is_system_admin
|
|
)
|
|
elif entity_type == "companies":
|
|
csv_data = await import_export_service.export_companies_csv(
|
|
db, tenant_id, user_id=user_id, is_system_admin=is_system_admin
|
|
)
|
|
else:
|
|
raise HTTPException(status_code=400, detail=f"Unsupported entity_type: {entity_type}")
|
|
|
|
# Parse CSV data back to rows for format conversion
|
|
import csv as _csv
|
|
|
|
reader = _csv.reader(io.StringIO(csv_data))
|
|
all_rows = list(reader)
|
|
if not all_rows:
|
|
raise HTTPException(status_code=404, detail="No data to export")
|
|
headers = all_rows[0]
|
|
data_rows = [dict(zip(headers, row)) for row in all_rows[1:]]
|
|
|
|
if format == "xlsx":
|
|
try:
|
|
xlsx_bytes = write_xlsx(data_rows, headers)
|
|
xlsx_filename = f"{entity_type}_export.xlsx"
|
|
return StreamingResponse(
|
|
io.BytesIO(xlsx_bytes),
|
|
media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
headers={"Content-Disposition": f'attachment; filename="{xlsx_filename}"'},
|
|
)
|
|
except ImportError:
|
|
# openpyxl not available — fall back to CSV with a warning header
|
|
media_type = "text/csv"
|
|
headers = {
|
|
"Content-Disposition": f'attachment; filename="{entity_type}_export.csv"',
|
|
"X-Export-Warning": "openpyxl not installed, falling back to CSV format",
|
|
}
|
|
return StreamingResponse(
|
|
iter([csv_data.encode("utf-8")]),
|
|
media_type=media_type,
|
|
headers=headers,
|
|
)
|
|
|
|
if format == "json":
|
|
from app.services.import_export_helpers import write_json
|
|
|
|
json_bytes = write_json(data_rows)
|
|
json_filename = f"{entity_type}_export.json"
|
|
return StreamingResponse(
|
|
io.BytesIO(json_bytes),
|
|
media_type="application/json",
|
|
headers={"Content-Disposition": f'attachment; filename="{json_filename}"'},
|
|
)
|
|
|
|
# Default: CSV format
|
|
csv_filename = f"{entity_type}_export.csv"
|
|
return StreamingResponse(
|
|
iter([csv_data.encode("utf-8")]),
|
|
media_type="text/csv",
|
|
headers={"Content-Disposition": f'attachment; filename="{csv_filename}"'},
|
|
)
|