"""Contact deduplication / merge service. This service is Contact-specific — it handles duplicate detection and merging for the Contact entity only. It is NOT a generic dedup service. If other entity types need dedup in the future, a separate service or a plugin-based interface should be created. Declared as Contact-specific (P1-22 fix): no pretense of being generic. """ from __future__ import annotations import uuid from datetime import UTC from typing import Any from sqlalchemy import func, select, text from sqlalchemy.ext.asyncio import AsyncSession from app.models.contact import Contact from app.models.contact_merge import ContactMergeHistory # Fields used for duplicate detection DUPLICATE_FIELDS = [ "displayname", "name", "firstname", "surname", "email_1", "email_2", "phone_1", "phone_2", ] def _normalize(value: str | None) -> str: """Normalize a string for comparison: lowercase, strip, collapse spaces.""" if not value: return "" return " ".join(value.lower().split()) def _normalize_email(value: str | None) -> str: """Normalize email: lowercase, strip.""" if not value: return "" return value.lower().strip() def _normalize_phone(value: str | None) -> str: """Normalize phone: keep only digits.""" if not value: return "" return "".join(c for c in value if c.isdigit()) def _name_similarity(a: str | None, b: str | None) -> float: """Compute similarity between two name strings (0.0 - 1.0). Uses a simple token-based Jaccard similarity. """ na = set(_normalize(a).split()) nb = set(_normalize(b).split()) if not na or not nb: return 0.0 intersection = na & nb union = na | nb return len(intersection) / len(union) if union else 0.0 async def find_duplicates( db: AsyncSession, tenant_id: uuid.UUID, threshold: float = 0.7, limit: int = 50, ) -> list[dict[str, Any]]: """Find potential duplicate contacts within a tenant. Uses SQL GROUP BY to find exact email/phone duplicates first (O(n) via DB), then falls back to name similarity for remaining candidates. Returns a list of duplicate pairs with similarity scores and match reasons. """ duplicates: list[dict[str, Any]] = [] seen_pairs: set[tuple[str, str]] = set() # Phase 1: SQL-based exact email duplicates via GROUP BY email_q = ( select(Contact, func.count().over(partition_by=func.lower(Contact.email_1)).label("cnt")) .where( Contact.tenant_id == tenant_id, Contact.deleted_at.is_(None), Contact.email_1.isnot(None), Contact.email_1 != "", ) .order_by(Contact.email_1, Contact.displayname) ) email_result = await db.execute(email_q) email_rows = email_result.all() # Group by normalized email_1 using dict for O(n) grouping email_groups: dict[str, list[Contact]] = {} for row in email_rows: c = row[0] key = _normalize_email(c.email_1) email_groups.setdefault(key, []).append(c) for _email_key, group in email_groups.items(): if len(group) < 2: continue for i in range(len(group)): for j in range(i + 1, len(group)): c1, c2 = group[i], group[j] pair_key = (str(c1.id), str(c2.id)) if pair_key in seen_pairs: continue seen_pairs.add(pair_key) duplicates.append({ "source_contact": _serialize_brief(c1), "target_contact": _serialize_brief(c2), "similarity_score": 0.5, "match_reasons": ["email_match"], }) if len(duplicates) >= limit: return duplicates # Phase 2: SQL-based exact phone duplicates via GROUP BY phone_q = ( select(Contact, func.count().over(partition_by=func.regexp_replace(Contact.phone_1, '[^0-9]', '', 'g')).label("cnt")) .where( Contact.tenant_id == tenant_id, Contact.deleted_at.is_(None), Contact.phone_1.isnot(None), Contact.phone_1 != "", ) .order_by(Contact.phone_1, Contact.displayname) ) phone_result = await db.execute(phone_q) phone_rows = phone_result.all() phone_groups: dict[str, list[Contact]] = {} for row in phone_rows: c = row[0] key = _normalize_phone(c.phone_1) phone_groups.setdefault(key, []).append(c) for _phone_key, group in phone_groups.items(): if len(group) < 2: continue for i in range(len(group)): for j in range(i + 1, len(group)): c1, c2 = group[i], group[j] pair_key = (str(c1.id), str(c2.id)) if pair_key in seen_pairs: continue seen_pairs.add(pair_key) duplicates.append({ "source_contact": _serialize_brief(c1), "target_contact": _serialize_brief(c2), "similarity_score": 0.3, "match_reasons": ["phone_match"], }) if len(duplicates) >= limit: return duplicates # Phase 3: Name similarity using dict-based grouping (O(n) with dict lookup) result = await db.execute( select(Contact) .where( Contact.tenant_id == tenant_id, Contact.deleted_at.is_(None), ) .order_by(Contact.displayname) ) contacts = result.scalars().all() # Build a dict of normalized names for O(1) lookup name_map: dict[str, list[Contact]] = {} for c in contacts: norm = _normalize(c.displayname) if norm: name_map.setdefault(norm, []).append(c) # Find contacts with same normalized name for _norm_name, group in name_map.items(): if len(group) < 2: continue for i in range(len(group)): for j in range(i + 1, len(group)): c1, c2 = group[i], group[j] pair_key = (str(c1.id), str(c2.id)) if pair_key in seen_pairs: continue seen_pairs.add(pair_key) duplicates.append({ "source_contact": _serialize_brief(c1), "target_contact": _serialize_brief(c2), "similarity_score": 1.0, "match_reasons": ["name_similarity:1.00"], }) if len(duplicates) >= limit: return duplicates return duplicates def _serialize_brief(c: Contact) -> dict: """Serialize a contact briefly for duplicate display.""" return { "id": str(c.id), "type": c.type, "displayname": c.displayname, "name": c.name, "firstname": c.firstname, "surname": c.surname, "email_1": c.email_1, "email_2": c.email_2, "phone_1": c.phone_1, "phone_2": c.phone_2, "mailing_city": c.mailing_city, "mailing_postalcode": c.mailing_postalcode, "created_at": c.created_at.isoformat() if c.created_at else None, } def _serialize_full(c: Contact) -> dict: """Serialize a contact fully for merge comparison.""" return { "id": str(c.id), "type": c.type, "displayname": c.displayname, "name": c.name, "firstname": c.firstname, "surname": c.surname, "suffix": c.suffix, "email_1": c.email_1, "email_2": c.email_2, "phone_1": c.phone_1, "phone_2": c.phone_2, "website": c.website, "mailing_street": c.mailing_street, "mailing_postalcode": c.mailing_postalcode, "mailing_city": c.mailing_city, "mailing_country": c.mailing_country, "note": c.projectnote, "tags": c.tags, "code": c.code, "vat_code": c.vat_code, } async def merge_contacts( db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID, source_id: str, target_id: str, field_overrides: dict[str, Any] | None = None, note: str | None = None, ) -> dict[str, Any]: """Merge source contact into target contact. 1. Apply field overrides (if provided) to target contact. 2. Re-point entity_links from source to target. 3. Re-point tag_assignments from source to target. 4. Soft-delete the source contact. 5. Record merge history. Returns the merge history record and updated target contact. """ source_uuid = uuid.UUID(source_id) target_uuid = uuid.UUID(target_id) # Fetch both contacts result = await db.execute( select(Contact).where( Contact.id == source_uuid, Contact.tenant_id == tenant_id, Contact.deleted_at.is_(None), ) ) source = result.scalar_one_or_none() if not source: raise ValueError("Source contact not found") result = await db.execute( select(Contact).where( Contact.id == target_uuid, Contact.tenant_id == tenant_id, Contact.deleted_at.is_(None), ) ) target = result.scalar_one_or_none() if not target: raise ValueError("Target contact not found") # Apply field overrides to target if field_overrides: for key, value in field_overrides.items(): if hasattr(target, key): setattr(target, key, value) # Re-point entity_links from source to target await db.execute( text( "UPDATE entity_links SET entity_id = :target_id " "WHERE entity_id = :source_id AND tenant_id = :tenant_id" ), {"target_id": target_uuid, "source_id": source_uuid, "tenant_id": tenant_id}, ) # Re-point tag_assignments from source to target await db.execute( text( "UPDATE tag_assignments SET entity_id = :target_id " "WHERE entity_id = :source_id AND tenant_id = :tenant_id" ), {"target_id": target_uuid, "source_id": source_uuid, "tenant_id": tenant_id}, ) # Re-point contact_persons from source to target await db.execute( text( "UPDATE contactpersons SET contact_id = :target_id " "WHERE contact_id = :source_id AND tenant_id = :tenant_id" ), {"target_id": target_uuid, "source_id": source_uuid, "tenant_id": tenant_id}, ) # Soft-delete the source contact from datetime import datetime source.deleted_at = datetime.now(UTC) # Record merge history history = ContactMergeHistory( tenant_id=tenant_id, merged_by=user_id, source_contact_id=source_uuid, target_contact_id=target_uuid, note=note, ) db.add(history) await db.flush() # Determine which fields were actually overridden merged_fields = field_overrides or {} if not merged_fields: # Auto-merge: fill empty target fields from source for attr in ("email_1", "email_2", "phone_1", "phone_2", "website", "mailing_street", "mailing_postalcode", "mailing_city", "mailing_country", "code", "vat_code"): target_val = getattr(target, attr, None) source_val = getattr(source, attr, None) if not target_val and source_val: setattr(target, attr, source_val) merged_fields[attr] = source_val history.merged_fields = merged_fields await db.flush() return { "history": { "id": str(history.id), "source_id": source_id, "target_id": target_id, "note": note, "merged_fields": merged_fields, "created_at": history.created_at.isoformat() if history.created_at else None, }, "target_contact": _serialize_full(target), } async def get_merge_history( db: AsyncSession, tenant_id: uuid.UUID, page: int = 1, page_size: int = 20, ) -> dict[str, Any]: """Get paginated merge history for a tenant.""" offset = (page - 1) * page_size result = await db.execute( select(ContactMergeHistory) .where(ContactMergeHistory.tenant_id == tenant_id) .order_by(ContactMergeHistory.created_at.desc()) .offset(offset) .limit(page_size) ) records = result.scalars().all() total_result = await db.execute( select(func.count()).select_from(ContactMergeHistory) .where(ContactMergeHistory.tenant_id == tenant_id) ) total = total_result.scalar() or 0 return { "items": [ { "id": str(r.id), "source_contact_id": str(r.source_contact_id), "target_contact_id": str(r.target_contact_id), "merged_by": str(r.merged_by) if r.merged_by else None, "note": r.note, "merged_fields": r.merged_fields or {}, "created_at": r.created_at.isoformat() if r.created_at else None, } for r in records ], "total": total, "page": page, "page_size": page_size, }