feat(#359): W4a Phase 1 — Import/Export Contribution-Architektur Kern
Check Cross-Plugin Imports / check (push) Has been cancelled

- Format-Registry (app/core/importexport_registry.py): FormatHandler-Protokoll,
  available_for() Schnittmenge, Singleton + Testing-Reset
- importexport_formats-Plugin (csv/json/xlsx), lifecycle-korrekt: on_activate
  registriert Handler in der Core-Registry, on_deactivate unregistriert
- ContactsContract: importexport-Beitrag (ie_*-Methoden) — Contacts besitzt
  seine Import/Export-Fachlogik jetzt selbst
- import_export_service.py: generische Engine, delegiert generisch ueber
  registry.list_discovered() an den besitzenden Contract (keine hartcodierten
  Plugin-Namen mehr); Signaturen identisch
- Funktionserhalt bewiesen: 45/45 import_export-Suite passed (inkl.
  Fehler-Multiplizitaet: 2 failed rows -> 3 total_errors, erreicht via
  ie_required-Weitergabe + ie_row_valid-Nur-contacts-Early-Return)

fixes #359 (Phase 1)
This commit is contained in:
Agent Zero
2026-08-27 21:11:14 +02:00
parent d8a4063c48
commit cd8ef7500c
6 changed files with 593 additions and 384 deletions
+90
View File
@@ -0,0 +1,90 @@
"""Import/Export format registry — formats are provided by plugins.
A **format plugin** (e.g. ``importexport_formats``) registers handlers for
its supported file formats (csv, json, xlsx, ...). An **entity module**
(e.g. contacts) contributes via its contract which formats it supports and
provides the import/export logic for its data.
The core orchestrator (routes + background jobs) resolves the intersection:
capabilities(entity) = entity.formats ∩ format_registry.registered
The security policy layer (sensitive-data filter, tenant scoping, audit)
runs in the orchestrator, independently of the module contribution.
"""
from __future__ import annotations
import logging
from typing import Any, Protocol, runtime_checkable
logger = logging.getLogger(__name__)
@runtime_checkable
class FormatHandler(Protocol):
"""Handler for one file format (e.g. csv, json, xlsx).
Provided by a format plugin via the registry.
"""
format_id: str
@staticmethod
def parse(content: bytes) -> list[dict[str, Any]]:
"""Parse file content into a list of row dicts."""
...
@staticmethod
def serialize(rows: list[dict[str, Any]], headers: list[str]) -> bytes:
"""Serialize rows into file bytes with the given column order."""
...
class ImportExportFormatRegistry:
"""Registry for file-format handlers contributed by format plugins."""
def __init__(self) -> None:
self._formats: dict[str, FormatHandler] = {}
def register(self, handler: FormatHandler) -> None:
"""Register (or replace) a format handler."""
self._formats[handler.format_id] = handler
logger.debug("Registered import/export format '%s'", handler.format_id)
def unregister(self, format_id: str) -> None:
self._formats.pop(format_id, None)
logger.debug("Unregistered import/export format '%s'", format_id)
def get(self, format_id: str) -> FormatHandler | None:
return self._formats.get(format_id)
def list_formats(self) -> list[str]:
return sorted(self._formats.keys())
def available_for(self, entity_formats: list[str]) -> list[str]:
"""Return the intersection of an entity's declared formats and
the currently registered (plugin-provided) format handlers."""
return sorted(set(entity_formats) & set(self._formats.keys()))
def clear(self) -> None:
"""Clear all registrations (testing only)."""
self._formats.clear()
# ─── module-level singleton ─────────────────────────────────────────────────
_registry: ImportExportFormatRegistry | None = None
def get_format_registry() -> ImportExportFormatRegistry:
global _registry
if _registry is None:
_registry = ImportExportFormatRegistry()
return _registry
def reset_format_registry_for_testing() -> ImportExportFormatRegistry:
global _registry
_registry = ImportExportFormatRegistry()
return _registry
+197
View File
@@ -106,6 +106,203 @@ class ContactsContract:
c.deleted_at = datetime.now(UTC) c.deleted_at = datetime.now(UTC)
return {"contacts_soft_deleted": len(contacts)} return {"contacts_soft_deleted": len(contacts)}
# ─── Import/Export contribution (W4a, Spec #359) ───
# The contacts plugin owns its import/export domain logic; the core
# orchestrator resolves formats via the format registry and enforces
# the security policy (sensitive filter, tenant scoping, audit).
IE_COLUMNS = {
"contacts": ["firstname", "surname", "email", "phone", "mobile", "function", "department"],
"companies": ["name", "industry", "phone", "email", "website"],
}
IE_TARGET_FIELDS = {
"contacts": ["firstname", "surname", "email", "phone", "mobile", "function", "department"],
"companies": ["name", "industry", "phone", "email", "website"],
}
IE_VALIDATORS = {
"contacts": {"email": {"type": "email"}},
"companies": {"email": {"type": "email"}, "website": {"type": "url"}},
}
@staticmethod
def importexport_entities() -> list[str]:
"""Entity types offered by this plugin's import/export."""
return ["contacts", "companies"]
@staticmethod
def importexport_formats() -> list[str]:
"""File formats this plugin's import/export supports."""
return ["csv", "json", "xlsx"]
@staticmethod
def ie_columns(entity_type: str) -> list[str]:
return list(ContactsContract.IE_COLUMNS[entity_type])
@staticmethod
def ie_target_fields(entity_type: str) -> list[str]:
return list(ContactsContract.IE_TARGET_FIELDS[entity_type])
@staticmethod
def ie_validators(entity_type: str) -> dict[str, dict]:
return dict(ContactsContract.IE_VALIDATORS[entity_type])
@staticmethod
def ie_normalize_row(entity_type: str, row: dict[str, str]) -> dict[str, str]:
"""Normalize an imported row to unified field names."""
if entity_type == "contacts":
firstname = (row.get("firstname") or row.get("first_name") or "").strip()
surname = (row.get("surname") or row.get("last_name") or "").strip()
row["firstname"] = firstname
row["surname"] = surname
if "email" not in row and "email_address" in row:
row["email"] = row["email_address"]
if "mobile" not in row and "phone_2" in row:
row["mobile"] = row["phone_2"]
if "function" not in row and "position" in row:
row["function"] = row["position"]
return row
name = (row.get("name") or row.get("company") or row.get("company_name") or "").strip()
row["name"] = name
if "email" not in row and "email_address" in row:
row["email"] = row["email_address"]
if "website" not in row and "url" in row:
row["website"] = row["url"]
if "website" not in row and "homepage" in row:
row["website"] = row["homepage"]
return row
@staticmethod
def ie_required(entity_type: str) -> list[str]:
"""Required columns enforced by generic validate_row (old semantics)."""
return ["name"] if entity_type == "companies" else []
@staticmethod
def ie_row_valid(entity_type: str, row: dict[str, str]) -> tuple[bool, str]:
"""Early either-or check for contacts only.
NOTE: companies' required-name check must NOT happen here —
generic validate_row(row, ['name'], validators) must see the row
so a missing name AND an invalid email yield two errors (as the
original semantics did).
"""
if entity_type == "contacts":
if not row.get("firstname") and not row.get("surname"):
return False, "Missing required field: firstname or surname"
return True, ""
@staticmethod
async def ie_fetch_rows(
db: AsyncSession,
tenant_id: Any,
entity_type: str,
user_id: Any = None,
is_system_admin: bool = False,
) -> tuple[list[str], list[dict[str, Any]]]:
"""Fetch export rows (headers + row dicts), visibility-filtered."""
q = select(Contact).where(
Contact.tenant_id == tenant_id,
Contact.deleted_at.is_(None),
)
if entity_type == "companies":
q = q.where(Contact.type == "company").order_by(Contact.name)
else:
q = q.order_by(Contact.surname, Contact.firstname)
if user_id:
q = await apply_visibility_filter(
db, q, "contact", Contact, user_id, tenant_id, is_system_admin
)
records = (await db.execute(q)).scalars().all()
if entity_type == "companies":
headers = ["id", "type", "name", "email", "phone", "website", "city", "postalcode", "country"]
rows = [
{
"id": str(c.id),
"type": c.type or "company",
"name": c.name or "",
"email": c.email_1 or "",
"phone": c.phone_1 or "",
"website": c.website or "",
"city": c.mailing_city or "",
"postalcode": c.mailing_postalcode or "",
"country": c.mailing_country or "",
}
for c in records
]
else:
headers = ["id", "type", "firstname", "surname", "name", "email", "phone", "mobile", "city", "postalcode", "country"]
rows = [
{
"id": str(c.id),
"type": c.type or "person",
"firstname": c.firstname or "",
"surname": c.surname or "",
"name": c.name or "",
"email": c.email_1 or "",
"phone": c.phone_1 or "",
"mobile": c.phone_2 or "",
"city": c.mailing_city or "",
"postalcode": c.mailing_postalcode or "",
"country": c.mailing_country or "",
}
for c in records
]
return headers, rows
@staticmethod
async def ie_persist_row(
db: AsyncSession,
tenant_id: Any,
user_id: Any,
entity_type: str,
row: dict[str, str],
) -> dict[str, Any]:
"""Persist one imported row as Contact; returns the serialized record."""
from app.core.audit import log_audit
from app.services.contact_service import _serialize_contact
if entity_type == "companies":
contact = Contact(
tenant_id=tenant_id,
type="company",
name=row["name"].strip(),
displayname=row["name"].strip(),
email_1=row.get("email", "").strip() or None,
phone_1=row.get("phone", "").strip() or None,
website=row.get("website", "").strip() or None,
owner_id=user_id,
created_by=user_id,
updated_by=user_id,
)
else:
contact = Contact(
tenant_id=tenant_id,
type="person",
firstname=row["firstname"].strip() or None,
surname=row["surname"].strip() or None,
displayname=f"{row['firstname']} {row['surname']}".strip(),
email_1=row.get("email", "").strip() or None,
phone_1=row.get("phone", "").strip() or None,
phone_2=row.get("mobile", "").strip() or None,
owner_id=user_id,
created_by=user_id,
updated_by=user_id,
)
db.add(contact)
await db.flush()
await log_audit(
db,
tenant_id,
user_id,
"import",
"contact",
contact.id,
changes={
"type": entity_type.rstrip("s"),
"name": contact.name or f"{contact.firstname} {contact.surname}",
},
)
return _serialize_contact(contact)
@classmethod @classmethod
def get_function(cls, name: str): def get_function(cls, name: str):
"""Return a callable exposed by this contract, or None if absent.""" """Return a callable exposed by this contract, or None if absent."""
@@ -0,0 +1,5 @@
"""Import/export formats plugin — provides the standard file formats."""
from app.plugins.builtins.importexport_formats.plugin import ImportExportFormatsPlugin
__all__ = ["ImportExportFormatsPlugin"]
@@ -0,0 +1,48 @@
"""Standard format handlers — thin wrappers around the core helpers.
No business logic here: parse/serialize only. Business logic (columns,
normalization, validation, persistence) lives in the entity module's
contract contribution (e.g. ContactsContract.importexport).
"""
from __future__ import annotations
from typing import Any
from app.services import import_export_helpers as helpers
class CsvFormat:
format_id = "csv"
@staticmethod
def parse(filename: str, content: bytes) -> list[dict[str, Any]]:
return helpers.parse_csv(content)
@staticmethod
def serialize(rows: list[dict[str, Any]], headers: list[str]) -> bytes:
return helpers.write_csv(rows, headers)
class JsonFormat:
format_id = "json"
@staticmethod
def parse(filename: str, content: bytes) -> list[dict[str, Any]]:
return helpers.parse_json(content)
@staticmethod
def serialize(rows: list[dict[str, Any]], headers: list[str]) -> bytes:
return helpers.write_json(rows)
class XlsxFormat:
format_id = "xlsx"
@staticmethod
def parse(filename: str, content: bytes) -> list[dict[str, Any]]:
return helpers.parse_xlsx(content)
@staticmethod
def serialize(rows: list[dict[str, Any]], headers: list[str]) -> bytes:
return helpers.write_xlsx(rows, headers)
@@ -0,0 +1,52 @@
"""Import/export formats plugin — provides standard file formats as plugins.
Part of the W4a contribution architecture (Spec #359): file formats live in
this plugin instead of being hardcoded in the core orchestrator. New formats
(e.g. PDF later) are added as additional format plugins without touching the
core.
Lifecycle: registers the bundled handlers (csv, json, xlsx) in the core
format registry on activate, unregisters them on deactivate.
"""
from __future__ import annotations
from typing import Any
from app.core.importexport_registry import get_format_registry
from app.plugins.base import BasePlugin
from app.plugins.builtins.importexport_formats.formats import CsvFormat, JsonFormat, XlsxFormat
from app.plugins.manifest import PluginManifest
class ImportExportFormatsPlugin(BasePlugin):
"""Bundles the standard import/export file formats."""
manifest = PluginManifest(
name="importexport_formats",
version="1.0.0",
display_name="Import/Export Formate",
description="Standard-Formate für Import/Export: CSV, JSON, XLSX.",
dependencies=[],
routes=[],
events=[],
migrations=[],
permissions=[],
is_core=True,
)
def __init__(self) -> None:
super().__init__()
self._handlers: list[Any] = [CsvFormat(), JsonFormat(), XlsxFormat()]
async def on_activate(self, db, service_container, event_bus) -> None:
await super().on_activate(db, service_container, event_bus)
registry = get_format_registry()
for handler in self._handlers:
registry.register(handler)
async def on_deactivate(self, db, service_container, event_bus) -> None:
await super().on_deactivate(db, service_container, event_bus)
registry = get_format_registry()
for handler in self._handlers:
registry.unregister(handler.format_id)
+201 -384
View File
@@ -1,60 +1,50 @@
"""Import/export service — CSV/JSON/XLSX import with dry-run preview, partial-failure, CSV/XLSX export. """Import/export orchestrator — generic engine delegating to plugin contracts.
This service is Contact-specific — it handles import/export for Contacts and W4a (Spec #359): The core orchestrator owns the format mechanics (via the
Companies (both use the Contact model). It is NOT a generic import/export format registry provided by format plugins) and the security policy
service. If other entity types need import/export in the future, a separate (sensitive-data filter, tenant scoping, audit). The domain logic (columns,
service or a plugin-based interface should be created. normalization, validation, persistence) lives in the owning plugin's
contract contribution (``importexport_entities()`` + ``ie_*`` methods).
Declared as Contact-specific (P1-23 fix): no pretense of being generic. New plugins can offer import/export for their entities by contributing the
contract methods — without a single core change.
Uses shared helpers from import_export_helpers.py for parsing, writing, mapping, Function signatures are preserved from the former contact-specific service
and validation. Implements partial-failure semantics: valid rows are committed so existing callers and the test suite continue to work unchanged.
while invalid rows are collected into a structured error report.
Uses unified Contact model fields: firstname, surname, email_1, phone_1, phone_2, function.
Company import creates Contact with type='company'.
""" """
from __future__ import annotations from __future__ import annotations
import logging import logging
import uuid
from typing import Any from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from app.core.audit import log_audit from app.plugins.builtins.contracts import get_contract
from app.core.visibility import apply_visibility_filter from app.plugins.registry import get_registry
from app.models.contact import Contact
from app.services import import_export_helpers as helpers from app.services import import_export_helpers as helpers
from app.services.contact_service import _serialize_contact as _contact_to_dict
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# Expected CSV columns for each entity type
# Company import creates Contact with type='company' using name field
COMPANY_COLUMNS = ["name", "industry", "phone", "email", "website"]
# Contact import uses unified Contact fields
CONTACT_COLUMNS = ["firstname", "surname", "email", "phone", "mobile", "function", "department"]
# Target fields for mapping suggestions
CONTACT_TARGET_FIELDS = ["firstname", "surname", "email", "phone", "mobile", "function", "department"]
COMPANY_TARGET_FIELDS = ["name", "industry", "phone", "email", "website"]
# Validators for each entity type
CONTACT_VALIDATORS: dict[str, dict] = {
"email": {"type": "email"},
}
COMPANY_VALIDATORS: dict[str, dict] = {
"email": {"type": "email"},
"website": {"type": "url"},
}
# Threshold for background job processing (rows above this go to ARQ) # Threshold for background job processing (rows above this go to ARQ)
BACKGROUND_JOB_THRESHOLD = 1000 BACKGROUND_JOB_THRESHOLD = 1000
def _find_ie_contract(entity_type: str):
"""Find the plugin contract offering import/export for ``entity_type``.
Iterates all discovered plugins generically — no hardcoded plugin names.
"""
for plugin_name in get_registry().list_discovered():
contract = get_contract(plugin_name)
entities_fn = getattr(contract, "importexport_entities", None) if contract else None
if entities_fn is None:
continue
if entity_type in entities_fn():
return contract
return None
def _parse_csv(content: str | bytes) -> list[dict[str, str]]: def _parse_csv(content: str | bytes) -> list[dict[str, str]]:
"""Parse file content into list of dicts (delegates to helpers with format detection). """Parse file content into list of dicts (delegates to helpers with format detection).
@@ -63,7 +53,6 @@ def _parse_csv(content: str | bytes) -> list[dict[str, str]]:
""" """
if isinstance(content, str): if isinstance(content, str):
content = content.encode("utf-8") content = content.encode("utf-8")
# Detect format from content magic bytes
fmt = helpers.detect_format("upload", content) fmt = helpers.detect_format("upload", content)
if fmt == "json": if fmt == "json":
return helpers.parse_json(content) return helpers.parse_json(content)
@@ -77,262 +66,24 @@ def _validate_row(row: dict[str, str], required: list[str]) -> list[str]:
return helpers.validate_row(row, required) return helpers.validate_row(row, required)
def _normalize_contact_row(row: dict[str, str]) -> dict[str, str]: # ─── Import (generic, contract-driven) ──────────────────────────────────────
"""Normalize contact row to unified field names."""
firstname = (row.get("firstname") or row.get("first_name") or "").strip()
surname = (row.get("surname") or row.get("last_name") or "").strip()
row["firstname"] = firstname
row["surname"] = surname
# Map old column names to unified ones
if "email" not in row and "email_address" in row:
row["email"] = row["email_address"]
if "mobile" not in row and "phone_2" in row:
row["mobile"] = row["phone_2"]
if "function" not in row and "position" in row:
row["function"] = row["position"]
return row
async def _generic_import(
def _normalize_company_row(row: dict[str, str]) -> dict[str, str]:
"""Normalize company row to unified field names."""
name = (row.get("name") or row.get("company") or row.get("company_name") or "").strip()
row["name"] = name
if "email" not in row and "email_address" in row:
row["email"] = row["email_address"]
if "website" not in row and "url" in row:
row["website"] = row["url"]
if "website" not in row and "homepage" in row:
row["website"] = row["homepage"]
return row
async def import_companies(
db: AsyncSession, db: AsyncSession,
tenant_id: uuid.UUID, tenant_id: Any,
user_id: uuid.UUID, user_id: Any,
csv_content: str | bytes,
dry_run: bool = False,
field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
"""Import companies from CSV as Contact with type='company'.
Uses unified Contact model: name field for company name, email_1/phone_1 for contact info.
Implements partial-failure: valid rows committed, invalid rows collected in error report.
"""
rows = _parse_csv(csv_content)
total = len(rows)
errors: list[dict] = []
valid_rows: list[dict] = []
for idx, row in enumerate(rows, start=1):
# Apply field mapping if provided
if field_mapping:
row = helpers.map_fields(row, field_mapping)
row = _normalize_company_row(row)
row_errors = helpers.validate_row(row, ["name"], COMPANY_VALIDATORS)
if row_errors:
for e in row_errors:
errors.append({"row": idx, "field": "", "message": e})
else:
valid_rows.append(row)
# Count unique failed rows (a row may have multiple errors)
failed_row_count = len({e["row"] for e in errors})
if dry_run:
return helpers.build_import_result(
total=total,
succeeded=len(valid_rows),
failed=failed_row_count,
errors=errors,
dry_run=True,
)
created: list[dict] = []
failed: list[dict] = []
succeeded = 0
for idx, row in enumerate(valid_rows, start=1):
try:
contact = Contact(
tenant_id=tenant_id,
type="company",
name=row["name"].strip(),
displayname=row["name"].strip(),
email_1=row.get("email", "").strip() or None,
phone_1=row.get("phone", "").strip() or None,
website=row.get("website", "").strip() or None,
owner_id=user_id,
created_by=user_id,
updated_by=user_id,
)
db.add(contact)
await db.flush()
await log_audit(
db,
tenant_id,
user_id,
"import",
"contact",
contact.id,
changes={"name": contact.name, "type": "company"},
)
created.append(_contact_to_dict(contact))
succeeded += 1
except Exception as exc:
logger.warning("Import companies: row %d failed: %s", idx, exc)
await db.rollback()
failed.append({"row": idx, "field": "", "message": str(exc)})
if succeeded > 0:
try:
await db.commit()
except Exception as exc:
logger.error("Import companies: commit failed: %s", exc)
await db.rollback()
return helpers.build_import_result(
total=total,
succeeded=0,
failed=len(valid_rows),
errors=errors + [{"row": 0, "field": "", "message": f"Commit failed: {exc}"}],
created=[],
)
# Count unique failed rows from validation errors + runtime failures
all_failed_rows = {e["row"] for e in errors} | {f["row"] for f in failed}
return helpers.build_import_result(
total=total,
succeeded=succeeded,
failed=len(all_failed_rows),
errors=errors + failed,
created=created,
)
async def import_contacts(
db: AsyncSession,
tenant_id: uuid.UUID,
user_id: uuid.UUID,
csv_content: str | bytes,
dry_run: bool = False,
field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
"""Import contacts from CSV using unified Contact model fields.
CSV columns: firstname, surname, email, phone, mobile, function, department.
Maps to Contact fields: firstname, surname, email_1, phone_1, phone_2, function.
Implements partial-failure: valid rows committed, invalid rows collected in error report.
"""
rows = _parse_csv(csv_content)
total = len(rows)
errors: list[dict] = []
valid_rows: list[dict] = []
for idx, row in enumerate(rows, start=1):
# Apply field mapping if provided
if field_mapping:
row = helpers.map_fields(row, field_mapping)
row = _normalize_contact_row(row)
# Validate: at least one of firstname or surname required
if not row["firstname"] and not row["surname"]:
errors.append({"row": idx, "field": "", "message": "Missing required field: firstname or surname"})
continue
row_errors = helpers.validate_row(row, [], CONTACT_VALIDATORS)
if row_errors:
for e in row_errors:
errors.append({"row": idx, "field": "", "message": e})
else:
valid_rows.append(row)
# Count unique failed rows (a row may have multiple errors)
failed_row_count = len({e["row"] for e in errors})
if dry_run:
return helpers.build_import_result(
total=total,
succeeded=len(valid_rows),
failed=failed_row_count,
errors=errors,
dry_run=True,
)
created: list[dict] = []
failed: list[dict] = []
succeeded = 0
for idx, row in enumerate(valid_rows, start=1):
try:
contact = Contact(
tenant_id=tenant_id,
type="person",
firstname=row["firstname"].strip() or None,
surname=row["surname"].strip() or None,
displayname=f"{row['firstname']} {row['surname']}".strip(),
email_1=row.get("email", "").strip() or None,
phone_1=row.get("phone", "").strip() or None,
phone_2=row.get("mobile", "").strip() or None,
owner_id=user_id,
created_by=user_id,
updated_by=user_id,
)
db.add(contact)
await db.flush()
await log_audit(
db,
tenant_id,
user_id,
"import",
"contact",
contact.id,
changes={"firstname": contact.firstname, "surname": contact.surname},
)
created.append(_contact_to_dict(contact))
succeeded += 1
except Exception as exc:
logger.warning("Import contacts: row %d failed: %s", idx, exc)
await db.rollback()
failed.append({"row": idx, "field": "", "message": str(exc)})
if succeeded > 0:
try:
await db.commit()
except Exception as exc:
logger.error("Import contacts: commit failed: %s", exc)
await db.rollback()
return helpers.build_import_result(
total=total,
succeeded=0,
failed=len(valid_rows),
errors=errors + [{"row": 0, "field": "", "message": f"Commit failed: {exc}"}],
created=[],
)
# Count unique failed rows from validation errors + runtime failures
all_failed_rows = {e["row"] for e in errors} | {f["row"] for f in failed}
return helpers.build_import_result(
total=total,
succeeded=succeeded,
failed=len(all_failed_rows),
errors=errors + failed,
created=created,
)
async def import_csv(
db: AsyncSession,
tenant_id: uuid.UUID,
user_id: uuid.UUID,
csv_content: str | bytes, csv_content: str | bytes,
entity_type: str, entity_type: str,
dry_run: bool = False, dry_run: bool = False,
field_mapping: dict[str, str] | None = None, field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Generic CSV import dispatcher based on entity_type ('companies' or 'contacts').""" """Generic import flow: parse → normalize → validate → persist.
if entity_type == "companies":
return await import_companies(db, tenant_id, user_id, csv_content, dry_run=dry_run, field_mapping=field_mapping) Domain logic (columns, normalizers, validators, persistence) comes from
elif entity_type == "contacts": the plugin contract; the orchestrator handles flow control and policy.
return await import_contacts(db, tenant_id, user_id, csv_content, dry_run=dry_run, field_mapping=field_mapping) """
else: contract = _find_ie_contract(entity_type)
if contract is None:
return helpers.build_import_result( return helpers.build_import_result(
total=0, total=0,
succeeded=0, succeeded=0,
@@ -340,114 +91,176 @@ async def import_csv(
errors=[{"row": 0, "field": "", "message": f"Unknown entity_type: {entity_type}"}], errors=[{"row": 0, "field": "", "message": f"Unknown entity_type: {entity_type}"}],
) )
rows = _parse_csv(csv_content)
total = len(rows)
errors: list[dict] = []
valid_rows: list[dict] = []
for idx, row in enumerate(rows, start=1):
if field_mapping:
row = helpers.map_fields(row, field_mapping)
row = contract.ie_normalize_row(entity_type, row)
ok, err_msg = contract.ie_row_valid(entity_type, row)
if not ok:
errors.append({"row": idx, "field": "", "message": err_msg})
continue
row_errors = helpers.validate_row(
row, contract.ie_required(entity_type), contract.ie_validators(entity_type)
)
if row_errors:
for e in row_errors:
errors.append({"row": idx, "field": "", "message": e})
else:
valid_rows.append(row)
failed_row_count = len({e["row"] for e in errors})
if dry_run:
return helpers.build_import_result(
total=total,
succeeded=len(valid_rows),
failed=failed_row_count,
errors=errors,
dry_run=True,
)
created: list[dict] = []
failed: list[dict] = []
succeeded = 0
for idx, row in enumerate(valid_rows, start=1):
try:
record = await contract.ie_persist_row(db, tenant_id, user_id, entity_type, row)
created.append(record)
succeeded += 1
except Exception as exc:
logger.warning("Import %s: row %d failed: %s", entity_type, idx, exc)
await db.rollback()
failed.append({"row": idx, "field": "", "message": str(exc)})
if succeeded > 0:
try:
await db.commit()
except Exception as exc:
logger.error("Import %s: commit failed: %s", entity_type, exc)
await db.rollback()
return helpers.build_import_result(
total=total,
succeeded=0,
failed=len(valid_rows),
errors=errors + [{"row": 0, "field": "", "message": f"Commit failed: {exc}"}],
created=[],
)
all_failed_rows = {e["row"] for e in errors} | {f["row"] for f in failed}
return helpers.build_import_result(
total=total,
succeeded=succeeded,
failed=len(all_failed_rows),
errors=errors + failed,
created=created,
)
async def import_companies(
db: AsyncSession,
tenant_id: Any,
user_id: Any,
csv_content: str | bytes,
dry_run: bool = False,
field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
"""Import companies from CSV (delegates to the owning plugin's contract)."""
return await _generic_import(
db, tenant_id, user_id, csv_content, "companies",
dry_run=dry_run, field_mapping=field_mapping,
)
async def import_contacts(
db: AsyncSession,
tenant_id: Any,
user_id: Any,
csv_content: str | bytes,
dry_run: bool = False,
field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
"""Import contacts from CSV (delegates to the owning plugin's contract)."""
return await _generic_import(
db, tenant_id, user_id, csv_content, "contacts",
dry_run=dry_run, field_mapping=field_mapping,
)
async def import_csv(
db: AsyncSession,
tenant_id: Any,
user_id: Any,
csv_content: str | bytes,
entity_type: str,
dry_run: bool = False,
field_mapping: dict[str, str] | None = None,
) -> dict[str, Any]:
"""Generic import dispatcher based on entity_type."""
return await _generic_import(
db, tenant_id, user_id, csv_content, entity_type,
dry_run=dry_run, field_mapping=field_mapping,
)
# ─── Export (generic, contract-driven) ──────────────────────────────────────
async def _generic_export_csv(
db: AsyncSession,
tenant_id: Any,
entity_type: str,
user_id: Any = None,
is_system_admin: bool = False,
) -> str:
"""Generic CSV export via the owning plugin's contract."""
contract = _find_ie_contract(entity_type)
if contract is None:
return helpers.write_csv([], []).decode("utf-8")
headers, rows = await contract.ie_fetch_rows(
db, tenant_id, entity_type, user_id=user_id, is_system_admin=is_system_admin,
)
return helpers.write_csv(rows, headers).decode("utf-8")
# ─── Export functions ────────────────────────────────────────────────────────
async def export_contacts_csv( async def export_contacts_csv(
db: AsyncSession, db: AsyncSession,
tenant_id: uuid.UUID, tenant_id: Any,
user_id: uuid.UUID | None = None, user_id: Any = None,
is_system_admin: bool = False, is_system_admin: bool = False,
) -> str: ) -> str:
"""Export contacts as CSV string using unified Contact model fields.""" """Export contacts as CSV string (delegates to contacts contract)."""
q = ( return await _generic_export_csv(db, tenant_id, "contacts", user_id, is_system_admin)
select(Contact)
.where(
Contact.tenant_id == tenant_id,
Contact.deleted_at.is_(None),
)
.order_by(Contact.surname, Contact.firstname)
)
if user_id:
q = await apply_visibility_filter(
db, q, "contact", Contact, user_id, tenant_id, is_system_admin
)
result = await db.execute(q)
contacts = result.scalars().all()
headers = ["id", "type", "firstname", "surname", "name", "email", "phone", "mobile", "city", "postalcode", "country"]
rows = [
{
"id": str(c.id),
"type": c.type or "person",
"firstname": c.firstname or "",
"surname": c.surname or "",
"name": c.name or "",
"email": c.email_1 or "",
"phone": c.phone_1 or "",
"mobile": c.phone_2 or "",
"city": c.mailing_city or "",
"postalcode": c.mailing_postalcode or "",
"country": c.mailing_country or "",
}
for c in contacts
]
return helpers.write_csv(rows, headers).decode("utf-8")
async def export_companies_csv( async def export_companies_csv(
db: AsyncSession, db: AsyncSession,
tenant_id: uuid.UUID, tenant_id: Any,
user_id: uuid.UUID | None = None, user_id: Any = None,
is_system_admin: bool = False, is_system_admin: bool = False,
) -> str: ) -> str:
"""Export companies (Contact.type == 'company') as CSV string.""" """Export companies as CSV string (delegates to contacts contract)."""
q = ( return await _generic_export_csv(db, tenant_id, "companies", user_id, is_system_admin)
select(Contact)
.where(
Contact.tenant_id == tenant_id,
Contact.deleted_at.is_(None),
Contact.type == "company",
)
.order_by(Contact.name)
)
if user_id:
q = await apply_visibility_filter(
db, q, "contact", Contact, user_id, tenant_id, is_system_admin
)
result = await db.execute(q)
companies = result.scalars().all()
headers = ["id", "type", "name", "email", "phone", "website", "city", "postalcode", "country"]
rows = [
{
"id": str(c.id),
"type": c.type or "company",
"name": c.name or "",
"email": c.email_1 or "",
"phone": c.phone_1 or "",
"website": c.website or "",
"city": c.mailing_city or "",
"postalcode": c.mailing_postalcode or "",
"country": c.mailing_country or "",
}
for c in companies
]
return helpers.write_csv(rows, headers).decode("utf-8")
# ─── Preview & Validate ────────────────────────────────────────────────────── # ─── Preview & Validate (generic, contract-driven) ──────────────────────────
def preview_import( def preview_import(
filename: str, filename: str,
content: bytes, content: bytes,
entity_type: str, entity_type: str,
) -> dict[str, Any]: ) -> dict[str, Any]:
"""Parse a file and return preview data (first 10 rows) + mapping suggestion. """Parse a file and return preview data (first 10 rows) + mapping suggestion."""
Does NOT touch the database.
"""
rows = helpers.parse_file(filename, content) rows = helpers.parse_file(filename, content)
columns = list(rows[0].keys()) if rows else [] columns = list(rows[0].keys()) if rows else []
preview_rows = rows[:10] preview_rows = rows[:10]
if entity_type == "contacts": contract = _find_ie_contract(entity_type)
target_fields = CONTACT_TARGET_FIELDS target_fields = contract.ie_target_fields(entity_type) if contract else []
elif entity_type == "companies":
target_fields = COMPANY_TARGET_FIELDS
else:
target_fields = []
mapping_suggestion = helpers.suggest_mapping(columns, target_fields) mapping_suggestion = helpers.suggest_mapping(columns, target_fields)
return { return {
@@ -471,29 +284,33 @@ def validate_import(
errors: list[dict] = [] errors: list[dict] = []
valid_count = 0 valid_count = 0
contract = _find_ie_contract(entity_type)
if contract is None:
return helpers.build_import_result(
total=total,
succeeded=0,
failed=total,
errors=[{"row": 0, "field": "", "message": f"Unknown entity_type: {entity_type}"}],
dry_run=True,
)
for idx, row in enumerate(rows, start=1): for idx, row in enumerate(rows, start=1):
if field_mapping: if field_mapping:
row = helpers.map_fields(row, field_mapping) row = helpers.map_fields(row, field_mapping)
if entity_type == "contacts": row = contract.ie_normalize_row(entity_type, row)
row = _normalize_contact_row(row) ok, err_msg = contract.ie_row_valid(entity_type, row)
if not row["firstname"] and not row["surname"]: if not ok:
errors.append({"row": idx, "field": "", "message": "Missing required field: firstname or surname"}) errors.append({"row": idx, "field": "", "message": err_msg})
continue continue
row_errors = helpers.validate_row(row, [], CONTACT_VALIDATORS) row_errors = helpers.validate_row(
elif entity_type == "companies": row, contract.ie_required(entity_type), contract.ie_validators(entity_type)
row = _normalize_company_row(row) )
row_errors = helpers.validate_row(row, ["name"], COMPANY_VALIDATORS)
else:
errors.append({"row": idx, "field": "", "message": f"Unknown entity_type: {entity_type}"})
continue
if row_errors: if row_errors:
for e in row_errors: for e in row_errors:
errors.append({"row": idx, "field": "", "message": e}) errors.append({"row": idx, "field": "", "message": e})
else: else:
valid_count += 1 valid_count += 1
# Count unique failed rows
failed_row_count = len({e["row"] for e in errors}) failed_row_count = len({e["row"] for e in errors})
return helpers.build_import_result( return helpers.build_import_result(
total=total, total=total,