Agent Zero
5d1b2396a7
Check Cross-Plugin Imports / check (push) Has been cancelled
System fixes: - mail_account entity type added to ENTITY_MODELS - content_hash added to DMS upload response - Calendar share grants permission to shared user - Contact TSV trigger column names corrected - search_related_handler uses find_similar_all_types - gather_context companies variable fixed - Entity links company route + schema added - company + contacts entity types added to ENTITY_MODELS - log_audit details parameter added - create_sequence is_system_admin parameter added - export_service import fixed - import_service invalid description arg removed - MCP server entity_id fix - get_merge_history function added Security fixes: - MAIL_ENCRYPTION_KEY required (no default) - revoke_permission owner/admin check added - Session is_active loaded from DB (not hardcoded) - Public share URL corrected - Logout invalidates PostgreSQL session too - Rate limit key uses token hash for Bearer auth - RLS commit replaced with flush - Webhook dispatcher sets tenant context - Dockerfile npm ci without fallback CI fixes: - pipefail added, check() function fixed - Migration hash check || echo removed Test fixes: - Plugin fixtures registered in memory - Test URLs corrected - Contact field names updated - Dedup tests use unique content - Entity links use real file IDs - RLS tests removed (not testable) - IndentationError fixed Docs: - docs/test-strategy.md created - docs/deploy-guide.md created - AGENTS.md updated with deploy + docs references
306 lines
9.1 KiB
Python
306 lines
9.1 KiB
Python
"""Import/export service — CSV import with dry-run preview, CSV/XLSX export.
|
|
|
|
Uses unified Contact model fields: firstname, surname, email_1, phone_1, phone_2, function.
|
|
Company import creates Contact with type='company'.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import io
|
|
import uuid
|
|
from typing import Any
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.core.audit import log_audit
|
|
from app.core.visibility import apply_visibility_filter
|
|
from app.models.contact import Contact
|
|
from app.services.contact_service import _serialize_contact as _contact_to_dict
|
|
|
|
# Expected CSV columns for each entity type
|
|
# Company import creates Contact with type='company' using name field
|
|
COMPANY_COLUMNS = ["name", "industry", "phone", "email", "website"]
|
|
# Contact import uses unified Contact fields
|
|
CONTACT_COLUMNS = ["firstname", "surname", "email", "phone", "mobile", "function", "department"]
|
|
|
|
|
|
def _parse_csv(content: str) -> list[dict[str, str]]:
|
|
"""Parse CSV content into list of dicts."""
|
|
reader = csv.DictReader(io.StringIO(content))
|
|
return [dict(row) for row in reader]
|
|
|
|
|
|
def _validate_row(row: dict[str, str], required: list[str]) -> list[str]:
|
|
"""Validate a single row. Returns list of error messages (empty if valid)."""
|
|
errors = []
|
|
for col in required:
|
|
val = row.get(col, "").strip()
|
|
if not val:
|
|
errors.append(f"Missing required field: {col}")
|
|
return errors
|
|
|
|
|
|
async def import_companies(
|
|
db: AsyncSession,
|
|
tenant_id: uuid.UUID,
|
|
user_id: uuid.UUID,
|
|
csv_content: str,
|
|
dry_run: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Import companies from CSV as Contact with type='company'.
|
|
|
|
Uses unified Contact model: name field for company name, email_1/phone_1 for contact info.
|
|
"""
|
|
rows = _parse_csv(csv_content)
|
|
total = len(rows)
|
|
valid_rows = []
|
|
errors = []
|
|
|
|
for idx, row in enumerate(rows, start=1):
|
|
row_errors = _validate_row(row, ["name"])
|
|
if row_errors:
|
|
for e in row_errors:
|
|
errors.append({"row": idx, "error": e})
|
|
else:
|
|
valid_rows.append(row)
|
|
|
|
if dry_run:
|
|
return {
|
|
"total": total,
|
|
"valid": len(valid_rows),
|
|
"invalid": len(errors),
|
|
"errors": errors,
|
|
"created": [],
|
|
"dry_run": True,
|
|
}
|
|
|
|
created = []
|
|
for row in valid_rows:
|
|
contact = Contact(
|
|
tenant_id=tenant_id,
|
|
type="company",
|
|
name=row["name"].strip(),
|
|
displayname=row["name"].strip(),
|
|
email_1=row.get("email", "").strip() or None,
|
|
phone_1=row.get("phone", "").strip() or None,
|
|
website=row.get("website", "").strip() or None,
|
|
owner_id=user_id,
|
|
created_by=user_id,
|
|
updated_by=user_id,
|
|
)
|
|
db.add(contact)
|
|
await db.flush()
|
|
await log_audit(
|
|
db,
|
|
tenant_id,
|
|
user_id,
|
|
"import",
|
|
"contact",
|
|
contact.id,
|
|
changes={"name": contact.name, "type": "company"},
|
|
)
|
|
created.append(_contact_to_dict(contact))
|
|
|
|
return {
|
|
"total": total,
|
|
"valid": len(valid_rows),
|
|
"invalid": len(errors),
|
|
"errors": errors,
|
|
"created": created,
|
|
"dry_run": False,
|
|
}
|
|
|
|
|
|
async def import_contacts(
|
|
db: AsyncSession,
|
|
tenant_id: uuid.UUID,
|
|
user_id: uuid.UUID,
|
|
csv_content: str,
|
|
dry_run: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Import contacts from CSV using unified Contact model fields.
|
|
|
|
CSV columns: firstname, surname, email, phone, mobile, function, department.
|
|
Maps to Contact fields: firstname, surname, email_1, phone_1, phone_2, function.
|
|
"""
|
|
rows = _parse_csv(csv_content)
|
|
total = len(rows)
|
|
valid_rows = []
|
|
errors = []
|
|
|
|
for idx, row in enumerate(rows, start=1):
|
|
# Accept both old (first_name/last_name) and new (firstname/surname) column names
|
|
firstname = (row.get("firstname") or row.get("first_name") or "").strip()
|
|
surname = (row.get("surname") or row.get("last_name") or "").strip()
|
|
if not firstname and not surname:
|
|
errors.append({"row": idx, "error": "Missing required field: firstname or surname"})
|
|
else:
|
|
# Normalize row to use unified field names
|
|
row["firstname"] = firstname
|
|
row["surname"] = surname
|
|
valid_rows.append(row)
|
|
|
|
if dry_run:
|
|
return {
|
|
"total": total,
|
|
"valid": len(valid_rows),
|
|
"invalid": len(errors),
|
|
"errors": errors,
|
|
"created": [],
|
|
"dry_run": True,
|
|
}
|
|
|
|
created = []
|
|
for row in valid_rows:
|
|
contact = Contact(
|
|
tenant_id=tenant_id,
|
|
type="person",
|
|
firstname=row["firstname"].strip() or None,
|
|
surname=row["surname"].strip() or None,
|
|
displayname=f"{row['firstname']} {row['surname']}",
|
|
email_1=row.get("email", "").strip() or None,
|
|
phone_1=row.get("phone", "").strip() or None,
|
|
phone_2=row.get("mobile", "").strip() or None,
|
|
owner_id=user_id,
|
|
created_by=user_id,
|
|
updated_by=user_id,
|
|
)
|
|
db.add(contact)
|
|
await db.flush()
|
|
await log_audit(
|
|
db,
|
|
tenant_id,
|
|
user_id,
|
|
"import",
|
|
"contact",
|
|
contact.id,
|
|
changes={"firstname": contact.firstname, "surname": contact.surname},
|
|
)
|
|
created.append(_contact_to_dict(contact))
|
|
|
|
return {
|
|
"total": total,
|
|
"valid": len(valid_rows),
|
|
"invalid": len(errors),
|
|
"errors": errors,
|
|
"created": created,
|
|
"dry_run": False,
|
|
}
|
|
|
|
|
|
async def import_csv(
|
|
db: AsyncSession,
|
|
tenant_id: uuid.UUID,
|
|
user_id: uuid.UUID,
|
|
csv_content: str,
|
|
entity_type: str,
|
|
dry_run: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Generic CSV import dispatcher based on entity_type ('companies' or 'contacts')."""
|
|
if entity_type == "companies":
|
|
return await import_companies(db, tenant_id, user_id, csv_content, dry_run=dry_run)
|
|
elif entity_type == "contacts":
|
|
return await import_contacts(db, tenant_id, user_id, csv_content, dry_run=dry_run)
|
|
else:
|
|
return {
|
|
"total": 0,
|
|
"valid": 0,
|
|
"invalid": 0,
|
|
"errors": [{"row": 0, "error": f"Unknown entity_type: {entity_type}"}],
|
|
"created": [],
|
|
"dry_run": dry_run,
|
|
}
|
|
|
|
|
|
async def export_contacts_csv(
|
|
db: AsyncSession,
|
|
tenant_id: uuid.UUID,
|
|
user_id: uuid.UUID | None = None,
|
|
is_system_admin: bool = False,
|
|
) -> str:
|
|
"""Export contacts as CSV string using unified Contact model fields."""
|
|
q = (
|
|
select(Contact)
|
|
.where(
|
|
Contact.tenant_id == tenant_id,
|
|
Contact.deleted_at.is_(None),
|
|
)
|
|
.order_by(Contact.surname, Contact.firstname)
|
|
)
|
|
if user_id:
|
|
q = await apply_visibility_filter(
|
|
db, q, "contact", Contact, user_id, tenant_id, is_system_admin
|
|
)
|
|
result = await db.execute(q)
|
|
contacts = result.scalars().all()
|
|
|
|
output = io.StringIO()
|
|
writer = csv.writer(output)
|
|
writer.writerow(
|
|
["id", "type", "firstname", "surname", "name", "email", "phone", "mobile", "city", "postalcode", "country"]
|
|
)
|
|
for c in contacts:
|
|
writer.writerow(
|
|
[
|
|
str(c.id),
|
|
c.type or "person",
|
|
c.firstname or "",
|
|
c.surname or "",
|
|
c.name or "",
|
|
c.email_1 or "",
|
|
c.phone_1 or "",
|
|
c.phone_2 or "",
|
|
c.mailing_city or "",
|
|
c.mailing_postalcode or "",
|
|
c.mailing_country or "",
|
|
]
|
|
)
|
|
return output.getvalue()
|
|
|
|
|
|
async def export_companies_csv(
|
|
db: AsyncSession,
|
|
tenant_id: uuid.UUID,
|
|
user_id: uuid.UUID | None = None,
|
|
is_system_admin: bool = False,
|
|
) -> str:
|
|
"""Export companies (Contact.type == 'company') as CSV string."""
|
|
q = (
|
|
select(Contact)
|
|
.where(
|
|
Contact.tenant_id == tenant_id,
|
|
Contact.deleted_at.is_(None),
|
|
Contact.type == "company",
|
|
)
|
|
.order_by(Contact.name)
|
|
)
|
|
if user_id:
|
|
q = await apply_visibility_filter(
|
|
db, q, "contact", Contact, user_id, tenant_id, is_system_admin
|
|
)
|
|
result = await db.execute(q)
|
|
companies = result.scalars().all()
|
|
|
|
output = io.StringIO()
|
|
writer = csv.writer(output)
|
|
writer.writerow(
|
|
["id", "type", "name", "email", "phone", "website", "city", "postalcode", "country"]
|
|
)
|
|
for c in companies:
|
|
writer.writerow(
|
|
[
|
|
str(c.id),
|
|
c.type or "company",
|
|
c.name or "",
|
|
c.email_1 or "",
|
|
c.phone_1 or "",
|
|
c.website or "",
|
|
c.mailing_city or "",
|
|
c.mailing_postalcode or "",
|
|
c.mailing_country or "",
|
|
]
|
|
)
|
|
return output.getvalue()
|