leocrm/app/plugins/builtins/ai_proactive/services.py

"""Proactive Engine — core logic for context-aware AI suggestions.

Handles context changes, gathers entity data, generates LLM-powered
suggestions, pushes via SSE, and manages suggestion lifecycle.
"""

from __future__ import annotations

import os
import asyncio
import json
import logging
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any

import litellm
from sqlalchemy import func, select, text
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.cache import get_cache
from app.core.db import create_db_session, get_session_factory
from app.core.notifications import create_notification
from app.models.audit import AuditLog
from app.models.company import Company
from app.models.contact import CompanyContact, Contact
from app.plugins.builtins.ai_proactive.models import (
    ContextLog,
    ProactiveSettings,
    ProactiveSuggestion,
)
logger = logging.getLogger(__name__)

litellm.suppress_debug_info = True

async def _get_llm_api_key(db: AsyncSession, tenant_id: uuid.UUID) -> tuple[str | None, str | None]:
    """Get API key and base_url from the default AI provider in the DB.

    Falls back to API_KEY_OLLAMA_CLOUD env var if no provider found.
    Returns (api_key, base_url).
    """
    try:
        from app.plugins.builtins.ai_assistant.services import get_default_provider
        provider = await get_default_provider(db, tenant_id)
        if provider and provider.api_key:
            return provider.api_key, provider.base_url
    except Exception:
        logger.debug("Failed to get provider from DB, falling back to env")
    # Fallback to env var
    env_key = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
    return (env_key if env_key else None), None

# ─── SSE Push Infrastructure ───

_sse_queues: dict[str, asyncio.Queue[dict[str, Any]]] = {}


def get_sse_queue(user_id: str) -> asyncio.Queue[dict[str, Any]]:
    """Get or create SSE queue for a user."""
    if user_id not in _sse_queues:
        _sse_queues[user_id] = asyncio.Queue()
    return _sse_queues[user_id]


async def push_suggestion(user_id: str, suggestion: dict[str, Any]) -> None:
    """Push suggestion to user's SSE queue."""
    queue = get_sse_queue(user_id)
    await queue.put(suggestion)


# ─── Rate Limiting ───


async def is_rate_limited(
    tenant_id: uuid.UUID, user_id: uuid.UUID, rate_limit_seconds: int
) -> bool:
    """Check if user is rate-limited using Redis.

    Returns True if rate-limited (key exists), False otherwise.
    Sets a key with TTL = rate_limit_seconds on first call.
    """
    try:
        r = get_cache()
        key = f"ai_proactive:rate:{user_id}"
        existing = await r.get(key)
        if existing is not None:
            return True
        await r.setex(key, rate_limit_seconds, "1")
        return False
    except Exception:
        logger.exception("Rate-limit check failed, allowing request")
        return False


# ─── Settings Helpers ───


async def get_user_settings(
    db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID
) -> ProactiveSettings:
    """Get proactive AI settings for user, create defaults if not exist."""
    result = await db.execute(
        select(ProactiveSettings)
        .where(ProactiveSettings.tenant_id == tenant_id)
        .where(ProactiveSettings.user_id == user_id)
        .limit(1)
    )
    settings = result.scalar_one_or_none()
    if settings is None:
        settings = ProactiveSettings(
            tenant_id=tenant_id,
            user_id=user_id,           enabled=True,
            suggestion_categories=["mail", "tasks", "contacts", "companies", "insights"],
            confidence_threshold=0.5,
            rate_limit_seconds=10,
            model="ollama/deepseek-v4",
        )
        db.add(settings)
        await db.flush()
    return settings


# ─── Context Gathering ───


def _serialize_row(row: Any) -> dict[str, Any]:
    """Serialize a SQLAlchemy model instance to a dict."""
    if row is None:
        return {}
    result: dict[str, Any] = {}
    for column in row.__table__.columns:
        val = getattr(row, column.name)
        if isinstance(val, datetime):
            result[column.name] = val.isoformat()
        elif isinstance(val, uuid.UUID):
            result[column.name] = str(val)
        else:
            result[column.name] = val
    return result


async def gather_context(
    db: AsyncSession, entity_type: str, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> dict[str, Any]:
    """Collect context data for an entity.

    Gathers related data from contacts, companies, mails, calendar events,
    audit logs, and semantically similar entities via unified_search.
    """
    context: dict[str, Any] = {
        "entity_type": entity_type,
        "entity_id": str(entity_id),
    }

    if entity_type == "contact":
        # Contact data
        result = await db.execute(
            select(Contact)
            .where(Contact.id == entity_id)
            .where(Contact.tenant_id == tenant_id)
            .limit(1)
        )
        contact = result.scalar_one_or_none()
        context["contact"] = _serialize_row(contact) if contact else None

        # Last 10 mails
        from app.plugins.builtins.mail.models import Mail

        mail_result = await db.execute(
            select(Mail)
            .where(Mail.contact_id == entity_id)
            .where(Mail.tenant_id == tenant_id)
            .order_by(Mail.received_at.desc())
            .limit(10)
        )
        context["mails"] = [_serialize_row(m) for m in mail_result.scalars().all()]

        # Company via company_contacts
        cc_result = await db.execute(
            select(CompanyContact)
            .where(CompanyContact.contact_id == entity_id)
            .where(CompanyContact.tenant_id == tenant_id)
            .limit(5)
        )
        companies: list[dict[str, Any]] = []
        for cc in cc_result.scalars().all():
            comp_result = await db.execute(
                select(Company)
                .where(Company.id == cc.company_id)
                .where(Company.tenant_id == tenant_id)
                .limit(1)
            )
            comp = comp_result.scalar_one_or_none()
            if comp:
                comp_data = _serialize_row(comp)
                comp_data["role_at_company"] = cc.role_at_company
                comp_data["is_primary"] = cc.is_primary
                companies.append(comp_data)
        context["company"] = companies[0] if companies else None
        context["companies"] = companies

        # Upcoming calendar events
        from app.plugins.builtins.calendar.models import CalendarEntry, CalendarEntryLink

        now = datetime.now(UTC)
        event_result = await db.execute(
            select(CalendarEntry)
            .join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id)
            .where(CalendarEntryLink.entity_type == "contact")
            .where(CalendarEntryLink.entity_id == entity_id)
            .where(CalendarEntry.tenant_id == tenant_id)
            .where(CalendarEntry.start_at > now)
            .order_by(CalendarEntry.start_at.asc())
            .limit(5)
        )
        context["events"] = [_serialize_row(e) for e in event_result.scalars().all()]

        # Last 20 audit log entries
        audit_result = await db.execute(
            select(AuditLog)
            .where(AuditLog.entity_id == entity_id)
            .where(AuditLog.tenant_id == tenant_id)
            .order_by(AuditLog.timestamp.desc())
            .limit(20)
        )
        context["activities"] = [_serialize_row(a) for a in audit_result.scalars().all()]

    elif entity_type == "mail":
        from app.plugins.builtins.mail.models import Mail

        result = await db.execute(
            select(Mail)
            .where(Mail.id == entity_id)
            .where(Mail.tenant_id == tenant_id)
            .limit(1)
        )
        mail = result.scalar_one_or_none()
        context["mail"] = _serialize_row(mail) if mail else None

        if mail and mail.contact_id:
            contact_result = await db.execute(
                select(Contact)
                .where(Contact.id == mail.contact_id)
                .where(Contact.tenant_id == tenant_id)
                .limit(1)
            )
            contact = contact_result.scalar_one_or_none()
            context["contact"] = _serialize_row(contact) if contact else None

        if mail and mail.thread_id:
            thread_result = await db.execute(
                select(Mail)
                .where(Mail.thread_id == mail.thread_id)
                .where(Mail.tenant_id == tenant_id)
                .order_by(Mail.received_at.asc())
                .limit(20)
            )
            context["thread"] = [_serialize_row(m) for m in thread_result.scalars().all()]

        if mail and mail.company_id:
            comp_result = await db.execute(
                select(Company)
                .where(Company.id == mail.company_id)
                .where(Company.tenant_id == tenant_id)
                .limit(1)
            )
            comp = comp_result.scalar_one_or_none()
            context["company"] = _serialize_row(comp) if comp else None

    elif entity_type == "company":
        result = await db.execute(
            select(Company)
            .where(Company.id == entity_id)
            .where(Company.tenant_id == tenant_id)
            .limit(1)
        )
        company = result.scalar_one_or_none()
        context["company"] = _serialize_row(company) if company else None

        # Contacts via company_contacts
        cc_result = await db.execute(
            select(CompanyContact)
            .where(CompanyContact.company_id == entity_id)
            .where(CompanyContact.tenant_id == tenant_id)
        )
        contacts: list[dict[str, Any]] = []
        for cc in cc_result.scalars().all():
            contact_result = await db.execute(
                select(Contact)
                .where(Contact.id == cc.contact_id)
                .where(Contact.tenant_id == tenant_id)
                .limit(1)
            )
            contact = contact_result.scalar_one_or_none()
            if contact:
                contact_data = _serialize_row(contact)
                contact_data["role_at_company"] = cc.role_at_company
                contact_data["is_primary"] = cc.is_primary
                contacts.append(contact_data)
        context["contacts"] = contacts

        # Mails for this company
        from app.plugins.builtins.mail.models import Mail

        mail_result = await db.execute(
            select(Mail)
            .where(Mail.company_id == entity_id)
            .where(Mail.tenant_id == tenant_id)
            .order_by(Mail.received_at.desc())
            .limit(10)
        )
        context["mails"] = [_serialize_row(m) for m in mail_result.scalars().all()]

        # Upcoming events
        from app.plugins.builtins.calendar.models import CalendarEntry, CalendarEntryLink

        now = datetime.now(UTC)
        event_result = await db.execute(
            select(CalendarEntry)
            .join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id)
            .where(CalendarEntryLink.entity_type == "company")
            .where(CalendarEntryLink.entity_id == entity_id)
            .where(CalendarEntry.tenant_id == tenant_id)
            .where(CalendarEntry.start_at > now)
            .order_by(CalendarEntry.start_at.asc())
            .limit(5)
        )
        context["events"] = [_serialize_row(e) for e in event_result.scalars().all()]

    elif entity_type == "file":
        # Basic file info via raw SQL (file model may vary)
        try:
            file_result = await db.execute(
                text("SELECT * FROM files WHERE id = :fid AND tenant_id = :tid"),
                {"fid": entity_id, "tid": tenant_id},
            )
            file_row = file_result.mappings().first()
            context["file"] = dict(file_row) if file_row else None
        except Exception:
            context["file"] = None

        # Linked entities via entity_links (if table exists)
        try:
            links_result = await db.execute(
                text(
                    "SELECT * FROM entity_links WHERE entity_id = :eid AND tenant_id = :tid LIMIT 20"
                ),
                {"eid": entity_id, "tid": tenant_id},
            )
            context["linked_entities"] = [dict(r) for r in links_result.mappings().all()]
        except Exception:
            context["linked_entities"] = []

    # Semantically similar entities via unified_search
    try:
        from app.plugins.builtins.unified_search.search_engine import (
            find_similar_all_types,
        )

        context["similar"] = await find_similar_all_types(
            db, entity_type, entity_id, tenant_id, limit=3
        )
    except Exception:
        context["similar"] = {}

    return context


# ─── Suggestion Generation ───


SYSTEM_PROMPT = """Du bist ein proaktiver KI-Assistent für ein CRM. Analysiere den Kontext und generiere Vorschläge.

Antworte mit JSON:
{
  "suggestion_type": "info|warning|action|insight",
  "title": "Kurzer Titel (max 200 Zeichen)",
  "content": "Beschreibung des Vorschlags",
  "confidence": 0.0-1.0,
  "actions": [{"method": "GET|POST|PUT|DELETE", "path": "/api/v1/...", "body": {}, "description": "..."}]
}

- suggestion_type: info=Information, warning=Warnung, action=Aktionsvorschlag, insight=Erkenntnis
- actions: Vorgeschlagene CRM-Aktionen die der User ausführen kann
- confidence: Wie sicher bist du dir (0.0=unsicher, 1.0=sehr sicher)
- Antworte nur mit gültigem JSON, kein Markdown"""


async def generate_suggestion(
    context_data: dict[str, Any], settings: ProactiveSettings,
    db: AsyncSession | None = None, tenant_id: uuid.UUID | None = None,
) -> dict[str, Any] | None:
    """LLM generates a suggestion from context data.

    Returns dict with suggestion_type, title, content, confidence, actions
    or None on failure.
    """
    model = settings.model or "ollama/deepseek-v4"

    # Get API key from DB (like ai_assistant does) or fall back to env
    api_key = None
    api_base = None
    if db and tenant_id:
        api_key, api_base = await _get_llm_api_key(db, tenant_id)
    if not api_key:
        api_key = os.environ.get('API_KEY_OLLAMA_CLOUD', '') or None

    litellm_kwargs: dict[str, Any] = dict(
        model=model,
        messages=[
            {"role": "system", "content": SYSTEM_PROMPT},
            {
                "role": "user",
                "content": json.dumps(context_data, default=str, ensure_ascii=False),
            },
        ],
        temperature=0.3,
        max_tokens=500,
        response_format={"type": "json_object"},
    )
    if api_key:
        litellm_kwargs["api_key"] = api_key
    if api_base:
        litellm_kwargs["api_base"] = api_base

    try:
        response = await litellm.acompletion(**litellm_kwargs)
        content = response.choices[0].message.content
        if not content:
            return None
        result = json.loads(content)
        # Validate required fields
        if not result.get("title") or not result.get("content"):
            return None
        # Ensure actions is a list
        if not isinstance(result.get("actions"), list):
            result["actions"] = []
        # Clamp confidence
        confidence = result.get("confidence", 0.5)
        try:
            confidence = float(confidence)
        except (TypeError, ValueError):
            confidence = 0.5
        result["confidence"] = max(0.0, min(1.0, confidence))
        # Validate suggestion_type
        valid_types = {"info", "warning", "action", "insight"}
        if result.get("suggestion_type") not in valid_types:
            result["suggestion_type"] = "info"
        return result
    except Exception:
        logger.exception("Failed to generate suggestion via LLM")
        return None


# ─── Main Handler ───


async def handle_context_change(payload: dict[str, Any]) -> None:
    """Main handler: Context-Change → Suggestion generation.

    1. Rate-limit check
    2. Settings check (enabled?)
    3. Gather context data
    4. Generate suggestion via LLM
    5. Confidence threshold check
    6. Save suggestion to DB
    7. Push via SSE
    8. Notification for urgent suggestions
    9. Enqueue background deep analysis job
    """
    user_id_str = payload.get("user_id")
    tenant_id_str = payload.get("tenant_id")
    entity_type = payload.get("entity_type")
    entity_id_str = payload.get("entity_id")

    if not user_id_str or not tenant_id_str or not entity_type:
        logger.warning("handle_context_change: missing required fields in payload")
        return

    try:
        tenant_id = uuid.UUID(tenant_id_str)
        user_id = uuid.UUID(user_id_str)
    except (ValueError, TypeError):
        logger.warning("handle_context_change: invalid UUID in payload")
        return

    entity_id: uuid.UUID | None = None
    if entity_id_str:
        try:
            entity_id = uuid.UUID(entity_id_str)
        except (ValueError, TypeError):
            entity_id = None

    if entity_id is None:
        logger.debug("handle_context_change: no entity_id, skipping")
        return

    async with create_db_session(tenant_id) as db:
        # Get settings
        settings = await get_user_settings(db, tenant_id, user_id)
        if not settings.enabled:
            logger.debug("handle_context_change: proactive AI disabled for user")
            return

        # Rate limit check
        if await is_rate_limited(tenant_id, user_id, settings.rate_limit_seconds):
            logger.debug("handle_context_change: rate limited")
            return

        # Gather context
        context_data = await gather_context(db, entity_type, entity_id, tenant_id)

        # Generate suggestion
        suggestion_data = await generate_suggestion(context_data, settings, db, tenant_id)
        if suggestion_data is None:
            logger.debug("handle_context_change: no suggestion generated")
            return

        # Confidence threshold check
        if suggestion_data["confidence"] < settings.confidence_threshold:
            logger.debug(
                "handle_context_change: confidence %s below threshold %s",
                suggestion_data["confidence"],
                settings.confidence_threshold,
            )
            return

        # Save suggestion
        suggestion = ProactiveSuggestion(
            tenant_id=tenant_id,
            user_id=user_id,
            entity_type=entity_type,
            entity_id=entity_id,
            suggestion_type=suggestion_data["suggestion_type"],
            title=suggestion_data["title"],
            content=suggestion_data["content"],
            confidence=suggestion_data["confidence"],
            actions=suggestion_data["actions"],
            context_snapshot=context_data,
        )
        db.add(suggestion)
        await db.flush()

        # Build response dict for SSE
        suggestion_dict = {
            "id": str(suggestion.id),
            "entity_type": suggestion.entity_type,
            "entity_id": str(suggestion.entity_id) if suggestion.entity_id else None,
            "suggestion_type": suggestion.suggestion_type,
            "title": suggestion.title,
            "content": suggestion.content,
            "confidence": suggestion.confidence,
            "actions": suggestion.actions,
            "created_at": suggestion.created_at.isoformat() if suggestion.created_at else None,
            "is_dismissed": suggestion.is_dismissed,
            "is_acted_upon": suggestion.is_acted_upon,
        }

        # Push via SSE
        await push_suggestion(str(user_id), suggestion_dict)

        # Notification for urgent suggestions
        if suggestion.suggestion_type == "warning":
            try:
                await create_notification(
                    db,
                    tenant_id,
                    user_id,
                    type="ai_suggestion_urgent",
                    title=suggestion.title,
                    body=suggestion.content[:200],
                )
            except Exception:
                logger.exception("Failed to create notification")
        else:
            try:
                await create_notification(
                    db,
                    tenant_id,
                    user_id,
                    type="ai_suggestion",
                    title=suggestion.title,
                    body=suggestion.content[:200],
                )
            except Exception:
                logger.exception("Failed to create notification")

        await db.commit()

    # Enqueue background deep analysis job
    try:
        from app.core.jobs import enqueue_job

        await enqueue_job(
            "deep_analysis",
            entity_type,
            str(entity_id),
            str(user_id),
            str(tenant_id),
        )
    except Exception:
        logger.exception("Failed to enqueue deep_analysis job")


# ─── Suggestion CRUD ───


async def get_active_suggestions(
    db: AsyncSession,
    tenant_id: uuid.UUID,
    user_id: uuid.UUID,
    entity_type: str | None = None,
    entity_id: uuid.UUID | None = None,
    limit: int = 10,
) -> list[ProactiveSuggestion]:
    """Get active (non-dismissed, not expired) suggestions for user."""
    now = datetime.now(UTC)
    stmt = (
        select(ProactiveSuggestion)
        .where(ProactiveSuggestion.tenant_id == tenant_id)
        .where(ProactiveSuggestion.user_id == user_id)
        .where(ProactiveSuggestion.is_dismissed == False)  # noqa: E712
        .where(
            (ProactiveSuggestion.expires_at.is_(None))
            | (ProactiveSuggestion.expires_at > now)
        )
    )
    if entity_type:
        stmt = stmt.where(ProactiveSuggestion.entity_type == entity_type)
    if entity_id:
        stmt = stmt.where(ProactiveSuggestion.entity_id == entity_id)
    stmt = stmt.order_by(ProactiveSuggestion.created_at.desc()).limit(limit)
    result = await db.execute(stmt)
    return list(result.scalars().all())


async def mark_dismissed(
    db: AsyncSession,
    suggestion_id: uuid.UUID,
    user_id: uuid.UUID,
    tenant_id: uuid.UUID,
) -> bool:
    """Mark suggestion as dismissed. Returns True if found and updated."""
    result = await db.execute(
        select(ProactiveSuggestion)
        .where(ProactiveSuggestion.id == suggestion_id)
        .where(ProactiveSuggestion.tenant_id == tenant_id)
        .where(ProactiveSuggestion.user_id == user_id)
        .limit(1)
    )
    suggestion = result.scalar_one_or_none()
    if suggestion is None:
        return False
    suggestion.is_dismissed = True
    await db.flush()
    return True


async def execute_suggested_action(
    db: AsyncSession,
    suggestion_id: uuid.UUID,
    action_index: int,
    user_id: uuid.UUID,
    tenant_id: uuid.UUID,
    user_context: dict[str, Any],
) -> dict[str, Any]:
    """Execute a suggested action.

    1. Load suggestion
    2. Get action from actions[action_index]
    3. Execute via internal HTTP call (httpx)
    4. Mark suggestion as is_acted_upon=True
    5. Return result
    """
    import httpx

    result = await db.execute(
        select(ProactiveSuggestion)
        .where(ProactiveSuggestion.id == suggestion_id)
        .where(ProactiveSuggestion.tenant_id == tenant_id)
        .where(ProactiveSuggestion.user_id == user_id)
        .limit(1)
    )
    suggestion = result.scalar_one_or_none()
    if suggestion is None:
        return {"success": False, "error": "Suggestion not found"}

    actions = suggestion.actions or []
    if action_index < 0 or action_index >= len(actions):
        return {"success": False, "error": "Invalid action index"}

    action = actions[action_index]
    method = action.get("method", "GET").upper()
    path = action.get("path", "")
    body = action.get("body")

    if not path:
        return {"success": False, "error": "No path in action"}

    # Build internal URL
    from app.config import get_settings

    settings = get_settings()
    base_url = getattr(settings, "internal_base_url", "http://localhost:8000")
    url = f"{base_url}{path}"

    # Build headers from user context (session cookie)
    headers: dict[str, str] = {"Content-Type": "application/json"}
    cookie_name = getattr(settings, "session_cookie_name", "session")
    session_id = user_context.get("session_id", "")
    if session_id:
        headers["Cookie"] = f"{cookie_name}={session_id}"

    try:
        async with httpx.AsyncClient(timeout=30.0) as client:
            resp = await client.request(
                method,
                url,
                json=body if body else None,
                headers=headers,
            )
        data = None
        try:
            data = resp.json()
        except Exception:
            data = {"status_code": resp.status_code, "text": resp.text}
        if resp.status_code < 400:
            suggestion.is_acted_upon = True
            await db.flush()
            return {"success": True, "data": data}
        return {"success": False, "error": f"HTTP {resp.status_code}", "data": data}
    except Exception as e:
        logger.exception("Failed to execute suggested action")
        return {"success": False, "error": str(e)}


# ─── Stats ───


async def get_stats(
    db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID
) -> dict[str, Any]:
    """Get proactive AI usage statistics for a user."""
    base_filter = (
        ProactiveSuggestion.tenant_id == tenant_id,
        ProactiveSuggestion.user_id == user_id,
    )
    total_result = await db.execute(
        select(func.count()).select_from(ProactiveSuggestion).where(*base_filter)
    )
    total = total_result.scalar() or 0

    dismissed_result = await db.execute(
        select(func.count())
        .select_from(ProactiveSuggestion)
        .where(*base_filter, ProactiveSuggestion.is_dismissed == True)  # noqa: E712
    )
    dismissed = dismissed_result.scalar() or 0

    acted_result = await db.execute(
        select(func.count())
        .select_from(ProactiveSuggestion)
        .where(*base_filter, ProactiveSuggestion.is_acted_upon == True)  # noqa: E712
    )
    acted_upon = acted_result.scalar() or 0

    active = total - dismissed
    dismiss_rate = (dismissed / total) if total > 0 else 0.0
    act_rate = (acted_upon / total) if total > 0 else 0.0

    return {
        "total_suggestions": total,
        "dismissed": dismissed,
        "acted_upon": acted_upon,
        "active": active,
        "dismiss_rate": round(dismiss_rate, 4),
        "act_rate": round(act_rate, 4),
    }