feat(C.5): Modularer Import/Export — Shared Helpers, Preview/Mapping, Background Jobs, Partial-Failure
C5-BASE: app/services/import_export_helpers.py (NEU, 352 Zeilen)
- parse_csv/json/xlsx, write_csv/json/xlsx, map_fields, suggest_mapping
- validate_row, build_error_report, build_import_result, detect_format
C5-PREVIEW: POST /import/preview + POST /import/validate
- Preview gibt erste 10 Zeilen + Spalten + Mapping-Vorschlag
- Validate gibt Fehler-Report ohne Import
C5-JOB: app/services/import_export_jobs.py (NEU, 165 Zeilen)
- ARQ Background Job für Files >1000 Zeilen
- Job-Status: pending/processing/completed/partial_success/failed
- GET /import/status/{job_id} — Status + Progress + Fehler-Report
- Partial-Failure: try/except pro Zeile, fehlerhafte gesammelt, erfolgreiche committet
C5-CONTACT+C5-COMPANY: Handler auf Shared Helpers umgestellt
C5-UI: ImportWizard.tsx (5 Steps: Upload→Preview→Validation→Review→Result)
C5-TEST: 45 Tests in test_import_export.py — alle grün
C5-DOC: Plugin-Dev-Guide Kapitel 31 (Import/Export Handler)
This commit is contained in:
@@ -0,0 +1,165 @@
|
||||
"""ARQ background jobs for large import operations.
|
||||
|
||||
When an import file exceeds BACKGROUND_JOB_THRESHOLD rows, the import is
|
||||
enqueued as an ARQ job instead of running synchronously. Job status is
|
||||
tracked in Redis with the key pattern 'leocrm:import_job:{job_id}'.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from app.core.job_registry import register_job
|
||||
from app.services import import_export_service
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_JOB_KEY_PREFIX = "leocrm:import_job"
|
||||
_JOB_TTL = 3600 # 1 hour
|
||||
|
||||
|
||||
def _job_key(job_id: str) -> str:
|
||||
return f"{_JOB_KEY_PREFIX}:{job_id}"
|
||||
|
||||
|
||||
async def _set_job_status(
|
||||
redis_client: Any,
|
||||
job_id: str,
|
||||
status: str,
|
||||
**extra: Any,
|
||||
) -> None:
|
||||
"""Store job status in Redis with TTL."""
|
||||
data = {"status": status, **extra}
|
||||
await redis_client.set(_job_key(job_id), json.dumps(data, default=str), ex=_JOB_TTL)
|
||||
|
||||
|
||||
async def _get_job_status(redis_client: Any, job_id: str) -> dict[str, Any] | None:
|
||||
"""Retrieve job status from Redis."""
|
||||
raw = await redis_client.get(_job_key(job_id))
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, bytes):
|
||||
raw = raw.decode("utf-8")
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
async def import_background_job(ctx: dict[str, Any], job_id: str, params: dict[str, Any]) -> dict[str, Any]:
|
||||
"""ARQ job function: run import in background with status tracking.
|
||||
|
||||
Args:
|
||||
ctx: ARQ worker context.
|
||||
job_id: Unique job identifier.
|
||||
params: Dict with keys:
|
||||
- entity_type: 'contacts' or 'companies'
|
||||
- csv_content: File content as string
|
||||
- tenant_id: Tenant UUID string
|
||||
- user_id: User UUID string
|
||||
- field_mapping: Optional dict[str, str]
|
||||
"""
|
||||
from app.core.auth import get_redis
|
||||
from app.core.db import get_worker_session_factory
|
||||
from app.core.db import set_tenant_context
|
||||
|
||||
redis_client = get_redis()
|
||||
entity_type = params["entity_type"]
|
||||
csv_content = params["csv_content"]
|
||||
tenant_id = uuid.UUID(params["tenant_id"])
|
||||
user_id = uuid.UUID(params["user_id"])
|
||||
field_mapping = params.get("field_mapping")
|
||||
|
||||
# Set initial status
|
||||
await _set_job_status(redis_client, job_id, "processing", progress=0, total=0)
|
||||
|
||||
factory = get_worker_session_factory()
|
||||
try:
|
||||
async with factory() as db:
|
||||
# Set tenant context for RLS
|
||||
await set_tenant_context(db, tenant_id)
|
||||
|
||||
result = await import_export_service.import_csv(
|
||||
db,
|
||||
tenant_id,
|
||||
user_id,
|
||||
csv_content,
|
||||
entity_type=entity_type,
|
||||
dry_run=False,
|
||||
field_mapping=field_mapping,
|
||||
)
|
||||
|
||||
# Determine final status
|
||||
status = result.get("status", "completed")
|
||||
if status == "success":
|
||||
final_status = "completed"
|
||||
elif status == "partial_success":
|
||||
final_status = "partial_success"
|
||||
else:
|
||||
final_status = "failed"
|
||||
|
||||
await _set_job_status(
|
||||
redis_client,
|
||||
job_id,
|
||||
final_status,
|
||||
progress=100,
|
||||
total=result.get("total", 0),
|
||||
succeeded=result.get("succeeded", 0),
|
||||
failed=result.get("failed", 0),
|
||||
error_report=result.get("error_report", {}),
|
||||
created_count=len(result.get("created", [])),
|
||||
)
|
||||
return result
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Import background job %s failed: %s", job_id, exc, exc_info=True)
|
||||
await _set_job_status(
|
||||
redis_client,
|
||||
job_id,
|
||||
"failed",
|
||||
progress=0,
|
||||
total=0,
|
||||
error=str(exc),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
# Register the job so it appears in WorkerSettings.functions
|
||||
register_job("import_background", import_background_job)
|
||||
|
||||
|
||||
async def get_import_job_status(job_id: str) -> dict[str, Any] | None:
|
||||
"""Public helper to get job status from Redis."""
|
||||
from app.core.auth import get_redis
|
||||
|
||||
redis_client = get_redis()
|
||||
return await _get_job_status(redis_client, job_id)
|
||||
|
||||
|
||||
async def create_import_job(
|
||||
entity_type: str,
|
||||
csv_content: str,
|
||||
tenant_id: uuid.UUID,
|
||||
user_id: uuid.UUID,
|
||||
field_mapping: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
"""Enqueue an import as ARQ background job and return job_id."""
|
||||
from app.core.jobs import enqueue_job
|
||||
|
||||
job_id = str(uuid.uuid4())
|
||||
params = {
|
||||
"entity_type": entity_type,
|
||||
"csv_content": csv_content,
|
||||
"tenant_id": str(tenant_id),
|
||||
"user_id": str(user_id),
|
||||
"field_mapping": field_mapping,
|
||||
}
|
||||
|
||||
# Pre-set status as pending
|
||||
from app.core.auth import get_redis
|
||||
|
||||
redis_client = get_redis()
|
||||
await _set_job_status(redis_client, job_id, "pending", progress=0, total=0)
|
||||
|
||||
await enqueue_job("import_background", job_id, params)
|
||||
return job_id
|
||||
Reference in New Issue
Block a user