"""Import/export routes — CSV/JSON/XLSX import, preview, validate, background jobs, CSV/XLSX export.""" from __future__ import annotations import io import json import uuid from fastapi import ( APIRouter, Depends, File, Form, HTTPException, Query, UploadFile, ) from sqlalchemy.ext.asyncio import AsyncSession from starlette.responses import StreamingResponse from app.core.db import get_db from app.deps import require_permission from app.services import import_export_service from app.services.import_export_helpers import ( parse_file, write_xlsx, ) router = APIRouter(prefix="/api/v1", tags=["import_export"]) # Row threshold for background job processing _BACKGROUND_THRESHOLD = 1000 @router.post("/import") async def import_csv( file: UploadFile = File(...), entity_type: str = Form(...), field_mapping: str = Form(None), db: AsyncSession = Depends(get_db), current_user: dict = Depends(require_permission("import_export:write")), ): """Import companies or contacts from CSV/JSON/XLSX file. entity_type: 'companies' or 'contacts'. field_mapping: Optional JSON string of source_column -> target_field mapping. For files > 1000 rows, the import runs as a background ARQ job. """ tenant_id = uuid.UUID(current_user["tenant_id"]) user_id = uuid.UUID(current_user["user_id"]) content = await file.read() csv_content = content.decode("utf-8", errors="replace") # Parse mapping if provided mapping = None if field_mapping: try: mapping = json.loads(field_mapping) except json.JSONDecodeError: raise HTTPException(status_code=400, detail="Invalid field_mapping JSON") from None # Check row count for background processing try: rows = parse_file(file.filename or "upload.csv", content) except Exception as exc: raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}") from exc if len(rows) > _BACKGROUND_THRESHOLD: # Enqueue as background job from app.services.import_export_jobs import create_import_job try: job_id = await create_import_job( entity_type=entity_type, csv_content=csv_content, tenant_id=tenant_id, user_id=user_id, field_mapping=mapping, ) return { "status": "pending", "job_id": job_id, "message": f"Import enqueued as background job ({len(rows)} rows)", "total": len(rows), } except Exception as exc: raise HTTPException(status_code=500, detail=f"Failed to enqueue import job: {exc}") from exc # Synchronous import for smaller files result = await import_export_service.import_csv( db, tenant_id, user_id, csv_content, entity_type=entity_type, dry_run=False, field_mapping=mapping, ) return result @router.post("/import/preview") async def import_csv_preview( file: UploadFile = File(...), entity_type: str = Form(...), current_user: dict = Depends(require_permission("import_export:read")), ): """Preview CSV import (dry-run — no DB changes). Returns first 10 rows, detected columns, and mapping suggestion. """ content = await file.read() try: result = import_export_service.preview_import( filename=file.filename or "upload.csv", content=content, entity_type=entity_type, ) except Exception as exc: raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}") from exc return result @router.post("/import/validate") async def import_validate( file: UploadFile = File(...), entity_type: str = Form(...), field_mapping: str = Form(None), current_user: dict = Depends(require_permission("import_export:read")), ): """Validate all rows against mapping and return error report without importing.""" content = await file.read() mapping = None if field_mapping: try: mapping = json.loads(field_mapping) except json.JSONDecodeError: raise HTTPException(status_code=400, detail="Invalid field_mapping JSON") from None try: result = import_export_service.validate_import( filename=file.filename or "upload.csv", content=content, entity_type=entity_type, field_mapping=mapping, ) except Exception as exc: raise HTTPException(status_code=400, detail=f"Failed to validate file: {exc}") from exc return result @router.get("/import/status/{job_id}") async def import_job_status( job_id: str, current_user: dict = Depends(require_permission("import_export:read")), ): """Get status of a background import job.""" from app.services.import_export_jobs import get_import_job_status status = await get_import_job_status(job_id) if status is None: raise HTTPException(status_code=404, detail=f"Job {job_id} not found") return status @router.post("/import/contacts") async def import_contacts_route( file: UploadFile = File(...), field_mapping: str = Form(None), db: AsyncSession = Depends(get_db), current_user: dict = Depends(require_permission("import_export:write")), ): """Import contacts from CSV/JSON/XLSX file. For files > 1000 rows, runs as background ARQ job. """ tenant_id = uuid.UUID(current_user["tenant_id"]) user_id = uuid.UUID(current_user["user_id"]) content = await file.read() csv_content = content.decode("utf-8", errors="replace") mapping = None if field_mapping: try: mapping = json.loads(field_mapping) except json.JSONDecodeError: raise HTTPException(status_code=400, detail="Invalid field_mapping JSON") from None # Check row count for background processing try: rows = parse_file(file.filename or "upload.csv", content) except Exception as exc: raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}") from exc if len(rows) > _BACKGROUND_THRESHOLD: from app.services.import_export_jobs import create_import_job try: job_id = await create_import_job( entity_type="contacts", csv_content=csv_content, tenant_id=tenant_id, user_id=user_id, field_mapping=mapping, ) return { "status": "pending", "job_id": job_id, "message": f"Import enqueued as background job ({len(rows)} rows)", "total": len(rows), } except Exception as exc: raise HTTPException(status_code=500, detail=f"Failed to enqueue import job: {exc}") from exc result = await import_export_service.import_contacts( db, tenant_id, user_id, csv_content, dry_run=False, field_mapping=mapping, ) return result @router.post("/import/companies") async def import_companies_route( file: UploadFile = File(...), field_mapping: str = Form(None), db: AsyncSession = Depends(get_db), current_user: dict = Depends(require_permission("import_export:write")), ): """Import companies from CSV/JSON/XLSX file. For files > 1000 rows, runs as background ARQ job. """ tenant_id = uuid.UUID(current_user["tenant_id"]) user_id = uuid.UUID(current_user["user_id"]) content = await file.read() csv_content = content.decode("utf-8", errors="replace") mapping = None if field_mapping: try: mapping = json.loads(field_mapping) except json.JSONDecodeError: raise HTTPException(status_code=400, detail="Invalid field_mapping JSON") from None # Check row count for background processing try: rows = parse_file(file.filename or "upload.csv", content) except Exception as exc: raise HTTPException(status_code=400, detail=f"Failed to parse file: {exc}") from exc if len(rows) > _BACKGROUND_THRESHOLD: from app.services.import_export_jobs import create_import_job try: job_id = await create_import_job( entity_type="companies", csv_content=csv_content, tenant_id=tenant_id, user_id=user_id, field_mapping=mapping, ) return { "status": "pending", "job_id": job_id, "message": f"Import enqueued as background job ({len(rows)} rows)", "total": len(rows), } except Exception as exc: raise HTTPException(status_code=500, detail=f"Failed to enqueue import job: {exc}") from exc result = await import_export_service.import_companies( db, tenant_id, user_id, csv_content, dry_run=False, field_mapping=mapping, ) return result @router.get("/export") async def export_data( entity_type: str = Query("contacts", pattern="^(contacts|companies)$"), format: str = Query("csv", pattern="^(csv|xlsx|json)$"), db: AsyncSession = Depends(get_db), current_user: dict = Depends(require_permission("import_export:read")), ): """Export contacts or companies as CSV, XLSX, or JSON file download. Query params: - entity_type: 'contacts' or 'companies' (default: contacts) - format: 'csv', 'xlsx', or 'json' (default: csv) """ tenant_id = uuid.UUID(current_user["tenant_id"]) user_id = uuid.UUID(current_user["user_id"]) is_system_admin = current_user.get("is_system_admin", False) # Fetch data from the appropriate service function if entity_type == "contacts": csv_data = await import_export_service.export_contacts_csv( db, tenant_id, user_id=user_id, is_system_admin=is_system_admin ) elif entity_type == "companies": csv_data = await import_export_service.export_companies_csv( db, tenant_id, user_id=user_id, is_system_admin=is_system_admin ) else: raise HTTPException(status_code=400, detail=f"Unsupported entity_type: {entity_type}") # Parse CSV data back to rows for format conversion import csv as _csv reader = _csv.reader(io.StringIO(csv_data)) all_rows = list(reader) if not all_rows: raise HTTPException(status_code=404, detail="No data to export") headers = all_rows[0] data_rows = [dict(zip(headers, row, strict=False)) for row in all_rows[1:]] if format == "xlsx": try: xlsx_bytes = write_xlsx(data_rows, headers) xlsx_filename = f"{entity_type}_export.xlsx" return StreamingResponse( io.BytesIO(xlsx_bytes), media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", headers={"Content-Disposition": f'attachment; filename="{xlsx_filename}"'}, ) except ImportError: # openpyxl not available — fall back to CSV with a warning header media_type = "text/csv" headers = { "Content-Disposition": f'attachment; filename="{entity_type}_export.csv"', "X-Export-Warning": "openpyxl not installed, falling back to CSV format", } return StreamingResponse( iter([csv_data.encode("utf-8")]), media_type=media_type, headers=headers, ) if format == "json": from app.services.import_export_helpers import write_json json_bytes = write_json(data_rows) json_filename = f"{entity_type}_export.json" return StreamingResponse( io.BytesIO(json_bytes), media_type="application/json", headers={"Content-Disposition": f'attachment; filename="{json_filename}"'}, ) # Default: CSV format csv_filename = f"{entity_type}_export.csv" return StreamingResponse( iter([csv_data.encode("utf-8")]), media_type="text/csv", headers={"Content-Disposition": f'attachment; filename="{csv_filename}"'}, )