From 8cebb4f4e9959b68fc490e4e5742b4ea1f0d17ae Mon Sep 17 00:00:00 2001 From: Agent Zero Date: Sat, 18 Jul 2026 11:21:51 +0200 Subject: [PATCH] feat: unified_search + ai_proactive plugins with Ollama Cloud DeepSeek V4 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - unified_search: Hybride Suche (PostgreSQL FTS + pgvector + RRF Fusion) - 5 Search Providers (Contact, Company, Mail, File, Event) - KI Query Understanding (Fuzzy, Facetten via LiteLLM) - DMS Text-Extraction (PDF, DOCX, XLSX, PPTX) - Embedding Pipeline (ollama/nomic-embed-text, 768 Dim) - Background Jobs für Indexierung - Plugin-basierte Provider Registry - ai_proactive: Proaktiver KI-Agent - Context-Tracking (Frontend → Backend → Event Bus) - Proactive Engine mit LLM Suggestion-Generierung - SSE Real-time Push an Frontend - 6 AI Tools für Tool Registry - Rate-Limiting + User Settings - Deep Analysis Background Jobs - Frontend Integration: - useAIContext Hook, SuggestionSidebar, SuggestionBadge - ProactiveAISettings Page, Search API Client - Globale Suche auf neue API umgestellt - Tests: test_unified_search.py + test_ai_proactive.py (alle bestanden) - Config: Ollama Cloud DeepSeek V4 als Default, konfigurierbar - Dependencies: PyMuPDF, python-docx, python-pptx, pgvector - Bugfixes: notification type_key length, migration IF NOT EXISTS --- .env.example | 8 + app/models/notification.py | 2 +- app/plugins/builtins/ai_proactive/__init__.py | 7 + .../builtins/ai_proactive/context_tools.py | 335 ++++++++ .../ai_proactive/frontend/AISettings.tsx | 157 ++++ .../ai_proactive/frontend/SuggestionBadge.tsx | 59 ++ .../ai_proactive/frontend/SuggestionCard.tsx | 94 +++ .../frontend/SuggestionSidebar.tsx | 109 +++ .../builtins/ai_proactive/frontend/api.ts | 72 ++ .../ai_proactive/frontend/useAIContext.ts | 24 + app/plugins/builtins/ai_proactive/jobs.py | 292 +++++++ .../ai_proactive/migrations/0001_initial.sql | 50 ++ app/plugins/builtins/ai_proactive/models.py | 111 +++ app/plugins/builtins/ai_proactive/plugin.py | 107 +++ app/plugins/builtins/ai_proactive/routes.py | 317 ++++++++ app/plugins/builtins/ai_proactive/schemas.py | 103 +++ app/plugins/builtins/ai_proactive/services.py | 748 ++++++++++++++++++ .../builtins/unified_search/__init__.py | 7 + .../builtins/unified_search/embedding.py | 131 +++ app/plugins/builtins/unified_search/jobs.py | 246 ++++++ .../migrations/0001_initial.sql | 154 ++++ .../migrations/0002_embeddings.sql | 24 + app/plugins/builtins/unified_search/models.py | 50 ++ app/plugins/builtins/unified_search/plugin.py | 137 ++++ .../unified_search/provider_registry.py | 133 ++++ .../unified_search/providers/__init__.py | 3 + .../providers/company_provider.py | 114 +++ .../providers/contact_provider.py | 121 +++ .../providers/event_provider.py | 114 +++ .../unified_search/providers/file_provider.py | 111 +++ .../unified_search/providers/mail_provider.py | 111 +++ .../unified_search/query_understanding.py | 101 +++ app/plugins/builtins/unified_search/routes.py | 316 ++++++++ .../builtins/unified_search/schemas.py | 71 ++ .../builtins/unified_search/search_engine.py | 252 ++++++ .../unified_search/text_extraction.py | 126 +++ frontend/src/api/aiProactive.ts | 72 ++ frontend/src/api/hooks.ts | 42 +- frontend/src/api/search.ts | 38 + .../src/components/ai/SuggestionBadge.tsx | 59 ++ frontend/src/components/ai/SuggestionCard.tsx | 94 +++ .../src/components/ai/SuggestionSidebar.tsx | 109 +++ frontend/src/components/layout/AppShell.tsx | 11 + frontend/src/components/layout/TopBar.tsx | 6 +- frontend/src/hooks/useAIContext.ts | 24 + frontend/src/pages/ProactiveAISettings.tsx | 157 ++++ frontend/src/pages/Settings.tsx | 1 + frontend/src/routes/index.tsx | 2 + frontend/src/store/uiStore.ts | 4 + requirements.txt | 6 + tests/test_ai_proactive.py | 440 +++++++++++ tests/test_unified_search.py | 552 +++++++++++++ 52 files changed, 6493 insertions(+), 41 deletions(-) create mode 100644 app/plugins/builtins/ai_proactive/__init__.py create mode 100644 app/plugins/builtins/ai_proactive/context_tools.py create mode 100644 app/plugins/builtins/ai_proactive/frontend/AISettings.tsx create mode 100644 app/plugins/builtins/ai_proactive/frontend/SuggestionBadge.tsx create mode 100644 app/plugins/builtins/ai_proactive/frontend/SuggestionCard.tsx create mode 100644 app/plugins/builtins/ai_proactive/frontend/SuggestionSidebar.tsx create mode 100644 app/plugins/builtins/ai_proactive/frontend/api.ts create mode 100644 app/plugins/builtins/ai_proactive/frontend/useAIContext.ts create mode 100644 app/plugins/builtins/ai_proactive/jobs.py create mode 100644 app/plugins/builtins/ai_proactive/migrations/0001_initial.sql create mode 100644 app/plugins/builtins/ai_proactive/models.py create mode 100644 app/plugins/builtins/ai_proactive/plugin.py create mode 100644 app/plugins/builtins/ai_proactive/routes.py create mode 100644 app/plugins/builtins/ai_proactive/schemas.py create mode 100644 app/plugins/builtins/ai_proactive/services.py create mode 100644 app/plugins/builtins/unified_search/__init__.py create mode 100644 app/plugins/builtins/unified_search/embedding.py create mode 100644 app/plugins/builtins/unified_search/jobs.py create mode 100644 app/plugins/builtins/unified_search/migrations/0001_initial.sql create mode 100644 app/plugins/builtins/unified_search/migrations/0002_embeddings.sql create mode 100644 app/plugins/builtins/unified_search/models.py create mode 100644 app/plugins/builtins/unified_search/plugin.py create mode 100644 app/plugins/builtins/unified_search/provider_registry.py create mode 100644 app/plugins/builtins/unified_search/providers/__init__.py create mode 100644 app/plugins/builtins/unified_search/providers/company_provider.py create mode 100644 app/plugins/builtins/unified_search/providers/contact_provider.py create mode 100644 app/plugins/builtins/unified_search/providers/event_provider.py create mode 100644 app/plugins/builtins/unified_search/providers/file_provider.py create mode 100644 app/plugins/builtins/unified_search/providers/mail_provider.py create mode 100644 app/plugins/builtins/unified_search/query_understanding.py create mode 100644 app/plugins/builtins/unified_search/routes.py create mode 100644 app/plugins/builtins/unified_search/schemas.py create mode 100644 app/plugins/builtins/unified_search/search_engine.py create mode 100644 app/plugins/builtins/unified_search/text_extraction.py create mode 100644 frontend/src/api/aiProactive.ts create mode 100644 frontend/src/api/search.ts create mode 100644 frontend/src/components/ai/SuggestionBadge.tsx create mode 100644 frontend/src/components/ai/SuggestionCard.tsx create mode 100644 frontend/src/components/ai/SuggestionSidebar.tsx create mode 100644 frontend/src/hooks/useAIContext.ts create mode 100644 frontend/src/pages/ProactiveAISettings.tsx create mode 100644 tests/test_ai_proactive.py create mode 100644 tests/test_unified_search.py diff --git a/.env.example b/.env.example index 9a1abe3..f907690 100644 --- a/.env.example +++ b/.env.example @@ -54,3 +54,11 @@ RATE_LIMIT_RESET_CONFIRM_MAX=5 RATE_LIMIT_RESET_CONFIRM_WINDOW=3600 RATE_LIMIT_GENERAL_MAX=60 RATE_LIMIT_GENERAL_WINDOW=60 + +# === AI / Search === +# Ollama Cloud API Key (für LiteLLM) +API_KEY_OLLAMA_CLOUD= +# Embedding Modell (default: ollama/nomic-embed-text) +SEARCH_EMBEDDING_MODEL=ollama/nomic-embed-text +# LLM Modell für Query Understanding (default: ollama/deepseek-v4) +SEARCH_LLM_MODEL=ollama/deepseek-v4 diff --git a/app/models/notification.py b/app/models/notification.py index 4869967..62b713f 100644 --- a/app/models/notification.py +++ b/app/models/notification.py @@ -53,7 +53,7 @@ class NotificationType(Base): id: Mapped[uuid.UUID] = mapped_column( PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4 ) - type_key: Mapped[str] = mapped_column(String(20), nullable=False, unique=True) + type_key: Mapped[str] = mapped_column(String(100), nullable=False, unique=True) plugin_name: Mapped[str] = mapped_column(String(100), nullable=False) category: Mapped[str] = mapped_column(String(50), nullable=False, default="general") label: Mapped[str] = mapped_column(String(200), nullable=False) diff --git a/app/plugins/builtins/ai_proactive/__init__.py b/app/plugins/builtins/ai_proactive/__init__.py new file mode 100644 index 0000000..dc5b333 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/__init__.py @@ -0,0 +1,7 @@ +"""Proactive AI plugin package.""" + +from __future__ import annotations + +from app.plugins.builtins.ai_proactive.plugin import AIProactivePlugin + +__all__ = ["AIProactivePlugin"] diff --git a/app/plugins/builtins/ai_proactive/context_tools.py b/app/plugins/builtins/ai_proactive/context_tools.py new file mode 100644 index 0000000..66aa0b9 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/context_tools.py @@ -0,0 +1,335 @@ +"""AI Tools for the Tool Registry — context-aware tools registered +by the ai_proactive plugin for use by the AI Assistant. + +Each tool is an async function (arguments: dict, context: dict) -> str +that performs SQL queries and returns JSON-string results. +""" + +from __future__ import annotations + +import json +import logging +import uuid +from typing import Any + +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.db import create_db_session +from app.models.audit import AuditLog +from app.models.contact import Contact +from app.plugins.builtins.mail.models import Mail + +logger = logging.getLogger(__name__) + + +def _serialize(row: Any) -> dict[str, Any]: + """Serialize a SQLAlchemy model instance to a dict.""" + if row is None: + return {} + result: dict[str, Any] = {} + for column in row.__table__.columns: + val = getattr(row, column.name) + if hasattr(val, "isoformat"): + result[column.name] = val.isoformat() + elif isinstance(val, uuid.UUID): + result[column.name] = str(val) + else: + result[column.name] = val + return result + + +async def _get_db_and_tenant(context: dict[str, Any]) -> tuple[AsyncSession, uuid.UUID, uuid.UUID]: + """Extract tenant_id, user_id from context and create a DB session.""" + tenant_id = uuid.UUID(context.get("tenant_id", "")) + user_id = uuid.UUID(context.get("user_id", "")) + db = context.get("db") + if db is not None and isinstance(db, AsyncSession): + return db, tenant_id, user_id + # Fallback: create standalone session + raise RuntimeError("No db session in context") + + +# ─── Tool Handlers ─── + + +async def get_contact_mails_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str: + """Get all mails for a contact, ordered by received_at DESC.""" + try: + db, tenant_id, _ = await _get_db_and_tenant(context) + contact_id = uuid.UUID(arguments["contact_id"]) + limit = arguments.get("limit", 10) + result = await db.execute( + select(Mail) + .where(Mail.contact_id == contact_id) + .where(Mail.tenant_id == tenant_id) + .order_by(Mail.received_at.desc()) + .limit(limit) + ) + mails = [_serialize(m) for m in result.scalars().all()] + return json.dumps({"mails": mails, "count": len(mails)}, default=str) + except Exception as e: + logger.exception("get_contact_mails_handler failed") + return json.dumps({"error": str(e)}) + + +async def get_contact_history_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str: + """Get audit log history for an entity, ordered by timestamp DESC.""" + try: + db, tenant_id, _ = await _get_db_and_tenant(context) + entity_id = uuid.UUID(arguments["entity_id"]) + limit = arguments.get("limit", 20) + result = await db.execute( + select(AuditLog) + .where(AuditLog.entity_id == entity_id) + .where(AuditLog.tenant_id == tenant_id) + .order_by(AuditLog.timestamp.desc()) + .limit(limit) + ) + entries = [_serialize(a) for a in result.scalars().all()] + return json.dumps({"activities": entries, "count": len(entries)}, default=str) + except Exception as e: + logger.exception("get_contact_history_handler failed") + return json.dumps({"error": str(e)}) + + +async def search_related_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str: + """Find semantically similar entities via unified_search.""" + try: + db, tenant_id, _ = await _get_db_and_tenant(context) + entity_type = arguments["entity_type"] + entity_id = uuid.UUID(arguments["entity_id"]) + limit = arguments.get("limit", 5) + + from app.plugins.builtins.unified_search.search_engine import ( + find_similar_all_types, + ) + + similar = await find_similar_all_types( + db, entity_type, entity_id, tenant_id, limit=limit + ) + return json.dumps({"similar": similar}, default=str) + except Exception as e: + logger.exception("search_related_handler failed") + return json.dumps({"error": str(e)}) + + +async def summarize_mail_thread_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str: + """Summarize a mail thread using LLM.""" + try: + import litellm + + db, tenant_id, _ = await _get_db_and_tenant(context) + thread_id = arguments["thread_id"] + limit = arguments.get("limit", 20) + + result = await db.execute( + select(Mail) + .where(Mail.thread_id == thread_id) + .where(Mail.tenant_id == tenant_id) + .order_by(Mail.received_at.asc()) + .limit(limit) + ) + mails = result.scalars().all() + if not mails: + return json.dumps({"summary": "No mails found in thread", "count": 0}) + + thread_text = "\n\n".join( + f"From: {m.from_address}\nSubject: {m.subject}\nDate: {m.received_at}\nBody: {m.body_text[:500]}" + for m in mails + ) + + response = await litellm.acompletion( + model="gpt-4o-mini", + messages=[ + { + "role": "system", + "content": "Fasse diesen E-Mail-Thread präzise zusammen. Antworte auf Deutsch.", + }, + {"role": "user", "content": thread_text}, + ], + temperature=0.3, + max_tokens=300, + ) + summary = response.choices[0].message.content or "" + return json.dumps({"summary": summary, "count": len(mails)}, default=str) + except Exception as e: + logger.exception("summarize_mail_thread_handler failed") + return json.dumps({"error": str(e)}) + + +async def get_open_tasks_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str: + """Get open tasks (calendar entries) for a contact or company.""" + try: + from datetime import UTC, datetime + + from app.plugins.builtins.calendar.models import ( + CalendarEntry, + CalendarEntryLink, + ) + + db, tenant_id, _ = await _get_db_and_tenant(context) + entity_type = arguments["entity_type"] + entity_id = uuid.UUID(arguments["entity_id"]) + now = datetime.now(UTC) + + result = await db.execute( + select(CalendarEntry) + .join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id) + .where(CalendarEntryLink.entity_type == entity_type) + .where(CalendarEntryLink.entity_id == entity_id) + .where(CalendarEntry.tenant_id == tenant_id) + .where(CalendarEntry.start_at > now) + .where(CalendarEntry.status == "open") + .order_by(CalendarEntry.start_at.asc()) + .limit(20) + ) + tasks = [_serialize(e) for e in result.scalars().all()] + return json.dumps({"tasks": tasks, "count": len(tasks)}, default=str) + except Exception as e: + logger.exception("get_open_tasks_handler failed") + return json.dumps({"error": str(e)}) + + +async def hybrid_search_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str: + """Perform hybrid search via unified_search search_engine.""" + try: + from app.plugins.builtins.unified_search.search_engine import hybrid_search + + db, tenant_id, _ = await _get_db_and_tenant(context) + query = arguments["query"] + entity_types = arguments.get("entity_types") + limit = arguments.get("limit", 20) + + query_analysis = { + "normalized_query": query, + "semantic_terms": [], + } + results = await hybrid_search( + db, query_analysis, tenant_id, entity_types=entity_types, limit=limit + ) + return json.dumps({"results": results, "count": len(results)}, default=str) + except Exception as e: + logger.exception("hybrid_search_handler failed") + return json.dumps({"error": str(e)}) + + +# ─── Registration ─── + + +def register_context_tools(registry) -> None: + """Register AI tools into the existing Tool Registry.""" + + # Tool 1: get_contact_mails + registry.register( + name="get_contact_mails", + description="Alle Mails eines Kontakts abrufen, sortiert nach Datum absteigend", + parameters={ + "type": "object", + "properties": { + "contact_id": {"type": "string", "description": "UUID des Kontakts"}, + "limit": {"type": "integer", "default": 10, "description": "Max results"}, + }, + "required": ["contact_id"], + }, + handler=get_contact_mails_handler, + plugin_name="ai_proactive", + required_permission="mail:read", + category="context", + ) + + # Tool 2: get_contact_history + registry.register( + name="get_contact_history", + description="Aktivitätsverlauf (Audit Log) für eine Entity abrufen", + parameters={ + "type": "object", + "properties": { + "entity_id": {"type": "string", "description": "UUID der Entity"}, + "limit": {"type": "integer", "default": 20, "description": "Max results"}, + }, + "required": ["entity_id"], + }, + handler=get_contact_history_handler, + plugin_name="ai_proactive", + required_permission="ai_proactive:read", + category="context", + ) + + # Tool 3: search_related + registry.register( + name="search_related", + description="Semantisch ähnliche Entities über unified_search finden", + parameters={ + "type": "object", + "properties": { + "entity_type": {"type": "string", "description": "contact, company, mail, file, event"}, + "entity_id": {"type": "string", "description": "UUID der Entity"}, + "limit": {"type": "integer", "default": 5, "description": "Max results per type"}, + }, + "required": ["entity_type", "entity_id"], + }, + handler=search_related_handler, + plugin_name="ai_proactive", + required_permission="search:read", + category="context", + ) + + # Tool 4: summarize_mail_thread + registry.register( + name="summarize_mail_thread", + description="E-Mail-Thread per LLM zusammenfassen", + parameters={ + "type": "object", + "properties": { + "thread_id": {"type": "string", "description": "Thread ID"}, + "limit": {"type": "integer", "default": 20, "description": "Max mails to include"}, + }, + "required": ["thread_id"], + }, + handler=summarize_mail_thread_handler, + plugin_name="ai_proactive", + required_permission="mail:read", + category="context", + ) + + # Tool 5: get_open_tasks + registry.register( + name="get_open_tasks", + description="Offene Aufgaben (Kalender-Termine) für einen Kontakt oder eine Firma abrufen", + parameters={ + "type": "object", + "properties": { + "entity_type": {"type": "string", "description": "contact or company"}, + "entity_id": {"type": "string", "description": "UUID der Entity"}, + }, + "required": ["entity_type", "entity_id"], + }, + handler=get_open_tasks_handler, + plugin_name="ai_proactive", + required_permission="ai_proactive:read", + category="context", + ) + + # Tool 6: hybrid_search + registry.register( + name="hybrid_search", + description="Hybride Volltext- und Semantiksuche über alle CRM-Entities via unified_search", + parameters={ + "type": "object", + "properties": { + "query": {"type": "string", "description": "Suchanfrage"}, + "entity_types": { + "type": "array", + "items": {"type": "string"}, + "description": "Optional: filter by entity types", + }, + "limit": {"type": "integer", "default": 20, "description": "Max results"}, + }, + "required": ["query"], + }, + handler=hybrid_search_handler, + plugin_name="ai_proactive", + required_permission="search:read", + category="context", + ) diff --git a/app/plugins/builtins/ai_proactive/frontend/AISettings.tsx b/app/plugins/builtins/ai_proactive/frontend/AISettings.tsx new file mode 100644 index 0000000..9ff3647 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/frontend/AISettings.tsx @@ -0,0 +1,157 @@ +import { useProactiveSettings } from './api'; + +const categoryLabels: Record = { + mail: 'E-Mails', + tasks: 'Aufgaben', + contacts: 'Kontakte', + companies: 'Firmen', + insights: 'Erkenntnisse', +}; + +const modelOptions = [ + { value: 'ollama/deepseek-v4', label: 'DeepSeek V4 (empfohlen)' }, + { value: 'ollama/deepseek-v4-pro', label: 'DeepSeek V4 Pro (präzise)' }, + { value: 'ollama/llama3.2', label: 'Llama 3.2 (Alternative)' }, + { value: 'ollama/gpt-4o-mini', label: 'GPT-4o mini via Ollama' }, + { value: 'gpt-4o-mini', label: 'GPT-4o mini (Direct OpenAI)' }, +]; + +export function AISettings() { + const { settings, update } = useProactiveSettings(); + + if (!settings) { + return ( +
+
Lade Einstellungen...
+
+ ); + } + + const toggleCategory = async (cat: string) => { + const current = settings.suggestion_categories || []; + const newCats = current.includes(cat) + ? current.filter((c: string) => c !== cat) + : [...current, cat]; + await update({ suggestion_categories: newCats }); + }; + + return ( +
+ {/* Enable/Disable */} +
+
+

Proaktive KI

+

Aktiviert oder deaktiviert alle Vorschläge

+
+ +
+ + {/* Categories */} +
+

Kategorien

+

Für welche Bereiche sollen Vorschläge generiert werden?

+
+ {Object.entries(categoryLabels).map(([key, label]) => { + const checked = settings.suggestion_categories?.includes(key); + return ( + + ); + })} +
+
+ + {/* Confidence Threshold */} +
+
+

Confidence Schwellwert

+ + {Math.round((settings.confidence_threshold || 0.5) * 100)}% + +
+

+ Nur Vorschläge mit höherer Confidence anzeigen +

+ update({ confidence_threshold: parseFloat(e.target.value) })} + className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500" + /> +
+ Alle + Sehr sicher +
+
+ + {/* Rate Limit */} +
+
+

Rate Limit

+ + {settings.rate_limit_seconds || 10}s + +
+

+ Mindestabstand zwischen Vorschlägen +

+ update({ rate_limit_seconds: parseInt(e.target.value) })} + className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500" + /> +
+ 5s + 60s +
+
+ + {/* Model Selection */} +
+

KI Modell (Ollama Cloud)

+

Wähle das Modell für Vorschläge. DeepSeek V4 ist empfohlen für beste Performance/Kosten.

+ +
+
+ ); +} diff --git a/app/plugins/builtins/ai_proactive/frontend/SuggestionBadge.tsx b/app/plugins/builtins/ai_proactive/frontend/SuggestionBadge.tsx new file mode 100644 index 0000000..bdabaa5 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/frontend/SuggestionBadge.tsx @@ -0,0 +1,59 @@ +import { useState, useEffect } from 'react'; +import { apiClient } from '@/api/client'; + +interface SuggestionBadgeProps { + onClick: () => void; +} + +export function SuggestionBadge({ onClick }: SuggestionBadgeProps) { + const [count, setCount] = useState(0); + const [pulsing, setPulsing] = useState(false); + + useEffect(() => { + // Initial count + apiClient.get('/api/v1/ai-proactive/suggestions').then(r => { + setCount(r.data.items.length); + }).catch(() => {}); + + // SSE for real-time updates + const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream'); + eventSource.onmessage = () => { + setCount(prev => prev + 1); + setPulsing(true); + setTimeout(() => setPulsing(false), 2000); + }; + + return () => eventSource.close(); + }, []); + + if (count === 0) { + return ( + + ); + } + + return ( + + ); +} diff --git a/app/plugins/builtins/ai_proactive/frontend/SuggestionCard.tsx b/app/plugins/builtins/ai_proactive/frontend/SuggestionCard.tsx new file mode 100644 index 0000000..09050e4 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/frontend/SuggestionCard.tsx @@ -0,0 +1,94 @@ +import { useState } from 'react'; +import type { Suggestion } from './api'; + +const typeConfig = { + info: { icon: '💡', color: 'blue', bg: 'bg-blue-50', border: 'border-blue-200', text: 'text-blue-700', bar: 'bg-blue-500' }, + warning: { icon: '⚠️', color: 'orange', bg: 'bg-orange-50', border: 'border-orange-200', text: 'text-orange-700', bar: 'bg-orange-500' }, + action: { icon: '✅', color: 'green', bg: 'bg-green-50', border: 'border-green-200', text: 'text-green-700', bar: 'bg-green-500' }, + insight: { icon: '🔍', color: 'purple', bg: 'bg-purple-50', border: 'border-purple-200', text: 'text-purple-700', bar: 'bg-purple-500' }, +}; + +interface SuggestionCardProps { + suggestion: Suggestion; + onDismiss: (id: string) => void; + onAct: (id: string, actionIndex: number) => void; +} + +export function SuggestionCard({ suggestion, onDismiss, onAct }: SuggestionCardProps) { + const [expanded, setExpanded] = useState(false); + const config = typeConfig[suggestion.suggestion_type] || typeConfig.info; + const confidencePercent = Math.round(suggestion.confidence * 100); + + return ( +
+ {/* Header */} +
+
+ {config.icon} +

{suggestion.title}

+
+ +
+ + {/* Content */} +

{suggestion.content}

+ + {/* Confidence Bar */} +
+
+
+
+ {confidencePercent}% +
+ + {/* Actions */} + {suggestion.actions && suggestion.actions.length > 0 && ( +
+ {suggestion.actions.slice(0, expanded ? undefined : 2).map((action, idx) => ( + + ))} + {suggestion.actions.length > 2 && ( + + )} +
+ )} + + {/* Acted upon badge */} + {suggestion.is_acted_upon && ( +
+ ✓ Ausgeführt +
+ )} +
+ ); +} diff --git a/app/plugins/builtins/ai_proactive/frontend/SuggestionSidebar.tsx b/app/plugins/builtins/ai_proactive/frontend/SuggestionSidebar.tsx new file mode 100644 index 0000000..b81f24d --- /dev/null +++ b/app/plugins/builtins/ai_proactive/frontend/SuggestionSidebar.tsx @@ -0,0 +1,109 @@ +import { useState } from 'react'; +import { useSuggestions } from './api'; +import { SuggestionCard } from './SuggestionCard'; + +interface SuggestionSidebarProps { + isOpen: boolean; + onClose: () => void; +} + +export function SuggestionSidebar({ isOpen, onClose }: SuggestionSidebarProps) { + const { suggestions, connected, dismiss, act } = useSuggestions(); + const [filter, setFilter] = useState('all'); + + const filteredSuggestions = filter === 'all' + ? suggestions + : suggestions.filter(s => s.suggestion_type === filter); + + const filterOptions = [ + { value: 'all', label: 'Alle' }, + { value: 'info', label: 'Info' }, + { value: 'warning', label: 'Warnung' }, + { value: 'action', label: 'Aktion' }, + { value: 'insight', label: 'Erkenntnis' }, + ]; + + return ( + <> + {/* Overlay */} + {isOpen && ( +
+ )} + + {/* Sidebar */} + + + ); +} diff --git a/app/plugins/builtins/ai_proactive/frontend/api.ts b/app/plugins/builtins/ai_proactive/frontend/api.ts new file mode 100644 index 0000000..e94bc28 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/frontend/api.ts @@ -0,0 +1,72 @@ +import { useState, useEffect, useCallback } from 'react'; +import { apiClient } from '@/api/client'; + +export interface Suggestion { + id: string; + entity_type: string; + entity_id: string | null; + suggestion_type: 'info' | 'warning' | 'action' | 'insight'; + title: string; + content: string; + confidence: number; + actions: Array<{ method: string; path: string; body: any; description: string }>; + created_at: string; + is_dismissed: boolean; + is_acted_upon: boolean; +} + +export function useSuggestions() { + const [suggestions, setSuggestions] = useState([]); + const [connected, setConnected] = useState(false); + + useEffect(() => { + // Initial load + apiClient.get('/api/v1/ai-proactive/suggestions').then(r => { + setSuggestions(r.data.items); + }).catch(() => {}); + + // SSE Stream + const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream'); + eventSource.onopen = () => setConnected(true); + eventSource.onerror = () => setConnected(false); + eventSource.onmessage = (e) => { + try { + const suggestion = JSON.parse(e.data); + setSuggestions(prev => [suggestion, ...prev].slice(0, 50)); + } catch {} + }; + + return () => eventSource.close(); + }, []); + + const dismiss = useCallback(async (id: string) => { + setSuggestions(prev => prev.filter(s => s.id !== id)); + await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/dismiss`).catch(() => {}); + }, []); + + const act = useCallback(async (id: string, actionIndex: number) => { + const result = await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/act`, { + action_index: actionIndex + }); + setSuggestions(prev => prev.map(s => s.id === id ? { ...s, is_acted_upon: true } : s)); + return result.data; + }, []); + + return { suggestions, connected, dismiss, act }; +} + +export function useProactiveSettings() { + const [settings, setSettings] = useState(null); + + useEffect(() => { + apiClient.get('/api/v1/ai-proactive/settings').then(r => setSettings(r.data)); + }, []); + + const update = useCallback(async (updates: any) => { + const r = await apiClient.put('/api/v1/ai-proactive/settings', updates); + setSettings(r.data); + return r.data; + }, []); + + return { settings, update }; +} diff --git a/app/plugins/builtins/ai_proactive/frontend/useAIContext.ts b/app/plugins/builtins/ai_proactive/frontend/useAIContext.ts new file mode 100644 index 0000000..63c37da --- /dev/null +++ b/app/plugins/builtins/ai_proactive/frontend/useAIContext.ts @@ -0,0 +1,24 @@ +import { useEffect, useRef } from 'react'; +import { useLocation } from 'react-router-dom'; +import { apiClient } from '@/api/client'; + +export function useAIContext(entityType?: string, entityId?: string, entityData?: any) { + const location = useLocation(); + const debounceRef = useRef>(); + + useEffect(() => { + if (debounceRef.current) clearTimeout(debounceRef.current); + debounceRef.current = setTimeout(() => { + apiClient.post('/api/v1/ai-proactive/context', { + page: location.pathname, + entity_type: entityType, + entity_id: entityId, + entity_data: entityData, + }).catch(() => {}); // Silent fail, don't bother user + }, 500); // Debounce 500ms + + return () => { + if (debounceRef.current) clearTimeout(debounceRef.current); + }; + }, [location.pathname, entityType, entityId, entityData]); +} diff --git a/app/plugins/builtins/ai_proactive/jobs.py b/app/plugins/builtins/ai_proactive/jobs.py new file mode 100644 index 0000000..efa3682 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/jobs.py @@ -0,0 +1,292 @@ +"""ARQ background jobs for the AI Proactive plugin. + +Deep analysis job runs after context-change for deeper analysis: +- Mail-thread summary generation +- Finding similar contacts via unified_search +- Analyzing open tasks +- Generating extended suggestion with more context +- Pushing additional suggestion if confidence > threshold +""" + +from __future__ import annotations + +import os +import json +import logging +import uuid +from typing import Any + +import litellm +from sqlalchemy import select + +from app.core.db import create_db_session +from app.core.notifications import create_notification +from app.models.audit import AuditLog +from app.models.contact import Contact +from app.plugins.builtins.ai_proactive.models import ProactiveSuggestion, ProactiveSettings +from app.plugins.builtins.ai_proactive.services import ( + _serialize_row, + generate_suggestion, + get_user_settings, + push_suggestion, +) +from app.plugins.builtins.mail.models import Mail + +logger = logging.getLogger(__name__) + +OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '') + +DEEP_SYSTEM_PROMPT = """Du bist ein proaktiver KI-Assistent für ein CRM. Du führst eine Tiefenanalyse durch. + +Analysiere den erweiterten Kontext und generiere einen detaillierten Vorschlag. +Antworte mit JSON: +{ + "suggestion_type": "info|warning|action|insight", + "title": "Kurzer Titel (max 200 Zeichen)", + "content": "Detaillierte Beschreibung mit konkreten Empfehlungen", + "confidence": 0.0-1.0, + "actions": [{"method": "GET|POST|PUT|DELETE", "path": "/api/v1/...", "body": {}, "description": "..."}] +} + +Fokussiere auf: +- Muster in der Kommunikationshistorie +- Beziehungen zwischen Entities +- Handlungsempfehlungen mit hoher Relevanz +- Antworte nur mit gültigem JSON""" + + +async def deep_analysis( + ctx: dict[str, Any], + entity_type: str, + entity_id: str, + user_id: str, + tenant_id: str, +) -> None: + """Deep Analysis Background-Job. + + Enqueued after context-change for deeper analysis: + - Mail-thread summary generation + - Similar contacts via unified_search + - Open tasks analysis + - Extended suggestion generation with more context + - If confidence > threshold: save + push additional suggestion + """ + try: + eid = uuid.UUID(entity_id) + uid = uuid.UUID(user_id) + tid = uuid.UUID(tenant_id) + except (ValueError, TypeError): + logger.warning("deep_analysis: invalid UUID parameters") + return + + async with create_db_session(tid) as db: + # Get settings + settings = await get_user_settings(db, tid, uid) + if not settings.enabled: + return + + # Gather extended context (more mails, more activities) + extended_context: dict[str, Any] = { + "entity_type": entity_type, + "entity_id": entity_id, + "deep_analysis": True, + } + + if entity_type == "contact": + # Extended: last 30 mails + mail_result = await db.execute( + select(Mail) + .where(Mail.contact_id == eid) + .where(Mail.tenant_id == tid) + .order_by(Mail.received_at.desc()) + .limit(30) + ) + extended_context["mails"] = [ + _serialize_row(m) for m in mail_result.scalars().all() + ] + + # Extended: last 50 audit entries + audit_result = await db.execute( + select(AuditLog) + .where(AuditLog.entity_id == eid) + .where(AuditLog.tenant_id == tid) + .order_by(AuditLog.timestamp.desc()) + .limit(50) + ) + extended_context["activities"] = [ + _serialize_row(a) for a in audit_result.scalars().all() + ] + + # Contact data + contact_result = await db.execute( + select(Contact) + .where(Contact.id == eid) + .where(Contact.tenant_id == tid) + .limit(1) + ) + contact = contact_result.scalar_one_or_none() + extended_context["contact"] = _serialize_row(contact) if contact else None + + elif entity_type == "company": + from app.models.company import Company, CompanyContact + + comp_result = await db.execute( + select(Company) + .where(Company.id == eid) + .where(Company.tenant_id == tid) + .limit(1) + ) + company = comp_result.scalar_one_or_none() + extended_context["company"] = _serialize_row(company) if company else None + + # All mails for company + mail_result = await db.execute( + select(Mail) + .where(Mail.company_id == eid) + .where(Mail.tenant_id == tid) + .order_by(Mail.received_at.desc()) + .limit(30) + ) + extended_context["mails"] = [ + _serialize_row(m) for m in mail_result.scalars().all() + ] + + elif entity_type == "mail": + mail_result = await db.execute( + select(Mail) + .where(Mail.id == eid) + .where(Mail.tenant_id == tid) + .limit(1) + ) + mail = mail_result.scalar_one_or_none() + extended_context["mail"] = _serialize_row(mail) if mail else None + + if mail and mail.thread_id: + thread_result = await db.execute( + select(Mail) + .where(Mail.thread_id == mail.thread_id) + .where(Mail.tenant_id == tid) + .order_by(Mail.received_at.asc()) + .limit(50) + ) + extended_context["thread"] = [ + _serialize_row(m) for m in thread_result.scalars().all() + ] + + # Similar entities via unified_search + try: + from app.plugins.builtins.unified_search.search_engine import ( + find_similar_all_types, + ) + + extended_context["similar"] = await find_similar_all_types( + db, entity_type, eid, tid, limit=5 + ) + except Exception: + extended_context["similar"] = {} + + # Generate extended suggestion with deep analysis prompt + model = settings.model or "ollama/deepseek-v4" + try: + response = await litellm.acompletion( + model=model, + messages=[ + {"role": "system", "content": DEEP_SYSTEM_PROMPT}, + { + "role": "user", + "content": json.dumps( + extended_context, default=str, ensure_ascii=False + ), + }, + ], + temperature=0.3, + max_tokens=800, + response_format={"type": "json_object"}, + api_key=OLLAMA_API_KEY, + ) + content = response.choices[0].message.content + if not content: + return + suggestion_data = json.loads(content) + if not suggestion_data.get("title") or not suggestion_data.get("content"): + return + if not isinstance(suggestion_data.get("actions"), list): + suggestion_data["actions"] = [] + confidence = suggestion_data.get("confidence", 0.5) + try: + confidence = float(confidence) + except (TypeError, ValueError): + confidence = 0.5 + suggestion_data["confidence"] = max(0.0, min(1.0, confidence)) + valid_types = {"info", "warning", "action", "insight"} + if suggestion_data.get("suggestion_type") not in valid_types: + suggestion_data["suggestion_type"] = "insight" + except Exception: + logger.exception("deep_analysis: LLM call failed") + return + + # Confidence threshold check + if suggestion_data["confidence"] < settings.confidence_threshold: + logger.debug( + "deep_analysis: confidence %s below threshold %s", + suggestion_data["confidence"], + settings.confidence_threshold, + ) + return + + # Save extended suggestion + suggestion = ProactiveSuggestion( + tenant_id=tid, + user_id=uid, + entity_type=entity_type, + entity_id=eid, + suggestion_type=suggestion_data["suggestion_type"], + title=f"[Tiefenanalyse] {suggestion_data['title']}", + content=suggestion_data["content"], + confidence=suggestion_data["confidence"], + actions=suggestion_data["actions"], + context_snapshot=extended_context, + ) + db.add(suggestion) + await db.flush() + + suggestion_dict = { + "id": str(suggestion.id), + "entity_type": suggestion.entity_type, + "entity_id": str(suggestion.entity_id) if suggestion.entity_id else None, + "suggestion_type": suggestion.suggestion_type, + "title": suggestion.title, + "content": suggestion.content, + "confidence": suggestion.confidence, + "actions": suggestion.actions, + "created_at": suggestion.created_at.isoformat() + if suggestion.created_at + else None, + "is_dismissed": suggestion.is_dismissed, + "is_acted_upon": suggestion.is_acted_upon, + } + + # Push via SSE + await push_suggestion(str(uid), suggestion_dict) + + # Notification for urgent suggestions + if suggestion.suggestion_type == "warning": + try: + await create_notification( + db, + tid, + uid, + type="ai_suggestion_urgent", + title=suggestion.title, + body=suggestion.content[:200], + ) + except Exception: + logger.exception("deep_analysis: notification failed") + + await db.commit() + logger.info( + "deep_analysis: generated suggestion %s for %s/%s", + suggestion.id, + entity_type, + entity_id, + ) diff --git a/app/plugins/builtins/ai_proactive/migrations/0001_initial.sql b/app/plugins/builtins/ai_proactive/migrations/0001_initial.sql new file mode 100644 index 0000000..61c4e18 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/migrations/0001_initial.sql @@ -0,0 +1,50 @@ +-- ai_proactive plugin: suggestions, context log, user settings + +-- Suggestions Store +CREATE TABLE IF NOT EXISTS ai_proactive_suggestions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tenant_id UUID NOT NULL, + user_id UUID NOT NULL, + entity_type VARCHAR(50) NOT NULL, + entity_id UUID, + suggestion_type VARCHAR(50) NOT NULL DEFAULT 'info', -- info, warning, action, insight + title VARCHAR(200) NOT NULL, + content TEXT NOT NULL, + confidence FLOAT NOT NULL DEFAULT 0.5, + actions JSONB NOT NULL DEFAULT '[]', -- [{method, path, body, description}] + is_dismissed BOOLEAN NOT NULL DEFAULT FALSE, + is_acted_upon BOOLEAN NOT NULL DEFAULT FALSE, + context_snapshot JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ +); +CREATE INDEX IF NOT EXISTS ix_suggestions_user ON ai_proactive_suggestions (tenant_id, user_id, is_dismissed, created_at DESC); +CREATE INDEX IF NOT EXISTS ix_suggestions_entity ON ai_proactive_suggestions (tenant_id, entity_type, entity_id); + +-- Context Log (was der User wann angesehen hat) +CREATE TABLE IF NOT EXISTS ai_proactive_context_log ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tenant_id UUID NOT NULL, + user_id UUID NOT NULL, + page VARCHAR(200) NOT NULL, + entity_type VARCHAR(50), + entity_id UUID, + entity_data JSONB NOT NULL DEFAULT '{}', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); +CREATE INDEX IF NOT EXISTS ix_context_log_user_time ON ai_proactive_context_log (tenant_id, user_id, created_at DESC); + +-- User Settings für proactive AI +CREATE TABLE IF NOT EXISTS ai_proactive_settings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tenant_id UUID NOT NULL, + user_id UUID NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + suggestion_categories JSONB NOT NULL DEFAULT '["mail","tasks","contacts","companies","insights"]', + confidence_threshold FLOAT NOT NULL DEFAULT 0.5, + rate_limit_seconds INT NOT NULL DEFAULT 10, + model VARCHAR(100) NOT NULL DEFAULT 'gpt-4o-mini', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(tenant_id, user_id) +); diff --git a/app/plugins/builtins/ai_proactive/models.py b/app/plugins/builtins/ai_proactive/models.py new file mode 100644 index 0000000..570c897 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/models.py @@ -0,0 +1,111 @@ +"""SQLAlchemy models for the AI Proactive plugin.""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +from sqlalchemy import ( + Boolean, + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + Text, +) +from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID +from sqlalchemy.orm import Mapped, mapped_column + +from app.core.db import Base, TenantMixin + + +class ProactiveSuggestion(Base, TenantMixin): + """A proactive AI suggestion generated from user context.""" + + __tablename__ = "ai_proactive_suggestions" + __table_args__ = ( + Index( + "ix_suggestions_user", + "tenant_id", + "user_id", + "is_dismissed", + "created_at", + ), + Index("ix_suggestions_entity", "tenant_id", "entity_type", "entity_id"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + user_id: Mapped[uuid.UUID] = mapped_column( + PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + entity_type: Mapped[str] = mapped_column(String(50), nullable=False) + entity_id: Mapped[uuid.UUID | None] = mapped_column(PGUUID(as_uuid=True), nullable=True) + suggestion_type: Mapped[str] = mapped_column( + String(50), nullable=False, default="info" + ) + title: Mapped[str] = mapped_column(String(200), nullable=False) + content: Mapped[str] = mapped_column(Text, nullable=False) + confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.5) + actions: Mapped[list[Any]] = mapped_column(JSONB, nullable=False, default=list) + is_dismissed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_acted_upon: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + context_snapshot: Mapped[dict[str, Any]] = mapped_column( + JSONB, nullable=False, default=dict + ) + expires_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) + + +class ContextLog(Base, TenantMixin): + """Log of user context changes (page views, entity selections).""" + + __tablename__ = "ai_proactive_context_log" + __table_args__ = ( + Index("ix_context_log_user_time", "tenant_id", "user_id", "created_at"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + user_id: Mapped[uuid.UUID] = mapped_column( + PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + page: Mapped[str] = mapped_column(String(200), nullable=False) + entity_type: Mapped[str | None] = mapped_column(String(50), nullable=True) + entity_id: Mapped[uuid.UUID | None] = mapped_column(PGUUID(as_uuid=True), nullable=True) + entity_data: Mapped[dict[str, Any]] = mapped_column( + JSONB, nullable=False, default=dict + ) + + +class ProactiveSettings(Base, TenantMixin): + """Per-user settings for proactive AI.""" + + __tablename__ = "ai_proactive_settings" + __table_args__ = ( + Index("ix_proactive_settings_user", "tenant_id", "user_id"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + user_id: Mapped[uuid.UUID] = mapped_column( + PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + suggestion_categories: Mapped[list[Any]] = mapped_column( + JSONB, nullable=False, default=lambda: ["mail", "tasks", "contacts", "companies", "insights"] + ) + confidence_threshold: Mapped[float] = mapped_column( + Float, nullable=False, default=0.5 + ) + rate_limit_seconds: Mapped[int] = mapped_column( + Integer, nullable=False, default=10 + ) + model: Mapped[str] = mapped_column(String(100), nullable=False, default="ollama/deepseek-v4") diff --git a/app/plugins/builtins/ai_proactive/plugin.py b/app/plugins/builtins/ai_proactive/plugin.py new file mode 100644 index 0000000..6ffae6a --- /dev/null +++ b/app/plugins/builtins/ai_proactive/plugin.py @@ -0,0 +1,107 @@ +"""Proactive AI Agent plugin — context-aware suggestions built on +unified_search and ai_assistant. + +Listens to context-change events, gathers entity data, generates LLM-powered +suggestions, pushes them via SSE, and registers AI tools for the assistant. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from app.plugins.base import BasePlugin +from app.plugins.manifest import PluginManifest, PluginRouteDef + +logger = logging.getLogger(__name__) + + +class AIProactivePlugin(BasePlugin): + """Proactive KI Agent that monitors user context and suggests actions.""" + + manifest = PluginManifest( + name="ai_proactive", + version="1.0.0", + display_name="Proaktiver KI Agent", + description=( + "Überwacht den User-Kontext, generiert proaktiv Vorschläge per LLM " + "und pusht diese via SSE. Nutzt unified_search und ai_assistant." + ), + dependencies=["ai_assistant", "unified_search"], + routes=[ + PluginRouteDef( + path="/api/v1/ai-proactive", + module="app.plugins.builtins.ai_proactive.routes", + router_attr="router", + ), + ], + events=["context.view_changed", "context.entity_selected"], + migrations=["0001_initial.sql"], + permissions=[ + "ai_proactive:read", + "ai_proactive:write", + "ai_proactive:config", + ], + is_core=False, + ) + + async def on_activate(self, db, service_container, event_bus) -> None: + """Register context tools and subscribe to events.""" + await super().on_activate(db, service_container, event_bus) + try: + from app.plugins.builtins.ai_proactive.context_tools import ( + register_context_tools, + ) + from app.plugins.builtins.ai_assistant.tool_registry import ( + get_tool_registry, + ) + + register_context_tools(get_tool_registry()) + logger.info("AI Proactive context tools registered") + except Exception: + logger.exception("Failed to register AI Proactive context tools") + + async def on_deactivate(self, db, service_container, event_bus) -> None: + """Unregister tools and event listeners.""" + try: + from app.plugins.builtins.ai_assistant.tool_registry import ( + get_tool_registry, + ) + + get_tool_registry().unregister_plugin("ai_proactive") + logger.info("AI Proactive context tools unregistered") + except Exception: + logger.exception("Failed to unregister AI Proactive context tools") + await super().on_deactivate(db, service_container, event_bus) + + # ─── Event Handlers ─── + + async def on_context_view_changed(self, payload: dict[str, Any]) -> None: + """Handle context.view_changed event.""" + from app.plugins.builtins.ai_proactive.services import handle_context_change + + await handle_context_change(payload) + + async def on_context_entity_selected(self, payload: dict[str, Any]) -> None: + """Handle context.entity_selected event.""" + from app.plugins.builtins.ai_proactive.services import handle_context_change + + await handle_context_change(payload) + + def get_notification_types(self) -> list[dict[str, Any]]: + return [ + { + "type_key": "ai_suggestion", + "category": "ai", + "label": "KI Vorschlag", + "description": "Proaktiver KI-Vorschlag", + "is_enabled_by_default": True, + }, + { + "type_key": "ai_suggestion_urgent", + "category": "ai", + "label": "Dringender KI Vorschlag", + "description": "Dringender proaktiver KI-Vorschlag", + "is_enabled_by_default": True, + }, + ] diff --git a/app/plugins/builtins/ai_proactive/routes.py b/app/plugins/builtins/ai_proactive/routes.py new file mode 100644 index 0000000..ced2ef3 --- /dev/null +++ b/app/plugins/builtins/ai_proactive/routes.py @@ -0,0 +1,317 @@ +"""API routes for the AI Proactive plugin. + +Endpoints: context tracking, suggestions CRUD, SSE stream, settings, stats. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import uuid +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import StreamingResponse +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.db import get_db, set_tenant_context +from app.core.event_bus import get_event_bus +from app.deps import get_current_user, require_permission +from app.plugins.builtins.ai_proactive.models import ( + ContextLog, + ProactiveSettings, + ProactiveSuggestion, +) +from app.plugins.builtins.ai_proactive.schemas import ( + ActRequest, + ActResponse, + ContextReport, + SettingsResponse, + SettingsUpdate, + SuggestionListResponse, + SuggestionResponse, + StatsResponse, +) +from app.plugins.builtins.ai_proactive.services import ( + execute_suggested_action, + get_active_suggestions, + get_sse_queue, + get_stats, + get_user_settings, + mark_dismissed, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/v1/ai-proactive", tags=["ai-proactive"]) + + +def _suggestion_to_response(s: ProactiveSuggestion) -> SuggestionResponse: + """Convert a ProactiveSuggestion model to SuggestionResponse.""" + return SuggestionResponse( + id=str(s.id), + entity_type=s.entity_type, + entity_id=str(s.entity_id) if s.entity_id else None, + suggestion_type=s.suggestion_type, + title=s.title, + content=s.content, + confidence=s.confidence, + actions=s.actions or [], + created_at=s.created_at, + is_dismissed=s.is_dismissed, + is_acted_upon=s.is_acted_upon, + ) + + +def _settings_to_response(s: ProactiveSettings) -> SettingsResponse: + """Convert ProactiveSettings model to SettingsResponse.""" + return SettingsResponse( + enabled=s.enabled, + suggestion_categories=s.suggestion_categories or [], + confidence_threshold=s.confidence_threshold, + rate_limit_seconds=s.rate_limit_seconds, + model=s.model, + ) + + +# ─── Context Tracking ─── + + +@router.post("/context", dependencies=[Depends(require_permission("ai_proactive:read"))]) +async def report_context( + context: ContextReport, + current_user: dict[str, Any] = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Frontend reports current context (page, entity). + + Stores context log entry and publishes event for proactive engine. + """ + tenant_id = uuid.UUID(current_user["tenant_id"]) + user_id = uuid.UUID(current_user["user_id"]) + + # Parse entity_id if present + entity_id: uuid.UUID | None = None + if context.entity_id: + try: + entity_id = uuid.UUID(context.entity_id) + except (ValueError, TypeError): + entity_id = None + + # Store context log + log_entry = ContextLog( + tenant_id=tenant_id, + user_id=user_id, + page=context.page, + entity_type=context.entity_type, + entity_id=entity_id, + entity_data=context.entity_data or {}, + ) + db.add(log_entry) + await db.flush() + + # Publish event + event_bus = get_event_bus() + event_name = ( + "context.entity_selected" + if context.entity_type and context.entity_id + else "context.view_changed" + ) + payload = { + "user_id": str(user_id), + "tenant_id": str(tenant_id), + "page": context.page, + "entity_type": context.entity_type, + "entity_id": str(entity_id) if entity_id else None, + "entity_data": context.entity_data or {}, + } + await event_bus.publish(event_name, payload) + + return {"status": "ok"} + + +# ─── Suggestions ─── + + +@router.get( + "/suggestions", + dependencies=[Depends(require_permission("ai_proactive:read"))], + response_model=SuggestionListResponse, +) +async def get_suggestions( + entity_type: str | None = Query(None), + entity_id: str | None = Query(None), + limit: int = Query(10, ge=1, le=100), + current_user: dict[str, Any] = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Get active suggestions for the current user.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + user_id = uuid.UUID(current_user["user_id"]) + + eid: uuid.UUID | None = None + if entity_id: + try: + eid = uuid.UUID(entity_id) + except (ValueError, TypeError): + eid = None + + suggestions = await get_active_suggestions( + db, tenant_id, user_id, entity_type=entity_type, entity_id=eid, limit=limit + ) + items = [_suggestion_to_response(s) for s in suggestions] + return SuggestionListResponse(items=items, total=len(items)) + + +@router.post( + "/suggestions/{suggestion_id}/dismiss", + dependencies=[Depends(require_permission("ai_proactive:write"))], +) +async def dismiss_suggestion( + suggestion_id: str, + current_user: dict[str, Any] = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Mark a suggestion as dismissed.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + user_id = uuid.UUID(current_user["user_id"]) + + try: + sid = uuid.UUID(suggestion_id) + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="Invalid suggestion ID") + + success = await mark_dismissed(db, sid, user_id, tenant_id) + if not success: + raise HTTPException(status_code=404, detail="Suggestion not found") + return {"status": "ok"} + + +@router.post( + "/suggestions/{suggestion_id}/act", + dependencies=[Depends(require_permission("ai_proactive:write"))], + response_model=ActResponse, +) +async def act_on_suggestion( + suggestion_id: str, + action: ActRequest, + current_user: dict[str, Any] = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Execute a suggested action.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + user_id = uuid.UUID(current_user["user_id"]) + + try: + sid = uuid.UUID(suggestion_id) + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="Invalid suggestion ID") + + result = await execute_suggested_action( + db, sid, action.action_index, user_id, tenant_id, current_user + ) + return ActResponse( + success=result.get("success", False), + data=result.get("data"), + error=result.get("error"), + ) + + +# ─── SSE Stream ─── + + +@router.get( + "/suggestions/stream", + dependencies=[Depends(require_permission("ai_proactive:read"))], +) +async def stream_suggestions( + current_user: dict[str, Any] = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """SSE stream: push new suggestions in real-time. + + Uses an asyncio.Queue per user. Heartbeat every 30s. + """ + user_id = str(current_user["user_id"]) + + async def event_generator(): + queue = get_sse_queue(user_id) + while True: + try: + suggestion = await asyncio.wait_for(queue.get(), timeout=30) + yield f"data: {json.dumps(suggestion, default=str)}\n\n" + except asyncio.TimeoutError: + yield ": keepalive\n\n" + + return StreamingResponse( + event_generator(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + + +# ─── Settings ─── + + +@router.get( + "/settings", + dependencies=[Depends(require_permission("ai_proactive:read"))], + response_model=SettingsResponse, +) +async def get_settings( + current_user: dict[str, Any] = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Get proactive AI settings for the current user.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + user_id = uuid.UUID(current_user["user_id"]) + settings = await get_user_settings(db, tenant_id, user_id) + return _settings_to_response(settings) + + +@router.put( + "/settings", + dependencies=[Depends(require_permission("ai_proactive:config"))], + response_model=SettingsResponse, +) +async def update_settings( + settings_update: SettingsUpdate, + current_user: dict[str, Any] = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Update proactive AI settings for the current user.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + user_id = uuid.UUID(current_user["user_id"]) + + settings = await get_user_settings(db, tenant_id, user_id) + + update_data = settings_update.model_dump(exclude_unset=True) + for field, val in update_data.items(): + setattr(settings, field, val) + + await db.flush() + return _settings_to_response(settings) + + +# ─── Stats ─── + + +@router.get( + "/stats", + dependencies=[Depends(require_permission("ai_proactive:read"))], + response_model=StatsResponse, +) +async def get_stats_endpoint( + current_user: dict[str, Any] = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +): + """Get proactive AI usage statistics for the current user.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + user_id = uuid.UUID(current_user["user_id"]) + stats = await get_stats(db, tenant_id, user_id) + return StatsResponse(**stats) diff --git a/app/plugins/builtins/ai_proactive/schemas.py b/app/plugins/builtins/ai_proactive/schemas.py new file mode 100644 index 0000000..1fae8fe --- /dev/null +++ b/app/plugins/builtins/ai_proactive/schemas.py @@ -0,0 +1,103 @@ +"""Pydantic v2 schemas for the AI Proactive plugin API.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field + + +class ContextReport(BaseModel): + """Frontend reports the current page/entity context.""" + + page: str = Field(..., description="Current page path") + entity_type: str | None = Field(None, description="Entity type being viewed") + entity_id: str | None = Field(None, description="Entity ID being viewed") + entity_data: dict[str, Any] | None = Field( + None, description="Additional entity metadata" + ) + + +class SuggestionAction(BaseModel): + """A suggested CRM action.""" + + method: str = Field(..., description="HTTP method") + path: str = Field(..., description="API path") + body: dict[str, Any] | None = Field(None, description="Request body") + description: str = Field(..., description="Human-readable description") + + +class SuggestionResponse(BaseModel): + """API response for a single suggestion.""" + + id: str + entity_type: str + entity_id: str | None + suggestion_type: str + title: str + content: str + confidence: float + actions: list[dict[str, Any]] + created_at: datetime + is_dismissed: bool + is_acted_upon: bool = False + + +class SuggestionListResponse(BaseModel): + """Paginated list of suggestions.""" + + items: list[SuggestionResponse] + total: int + + +class ActRequest(BaseModel): + """Execute a suggested action by index.""" + + action_index: int = Field(..., ge=0, description="Index into actions array") + + +class ActResponse(BaseModel): + """Result of executing a suggested action.""" + + success: bool + data: dict[str, Any] | None = None + error: str | None = None + + +class SettingsResponse(BaseModel): + """Proactive AI settings for the current user.""" + + enabled: bool + suggestion_categories: list[str] + confidence_threshold: float + rate_limit_seconds: int + model: str + available_models: list[str] = Field(default_factory=lambda: [ + 'ollama/deepseek-v4', + 'ollama/deepseek-v4-pro', + 'ollama/llama3.2', + 'ollama/gpt-4o-mini', + 'gpt-4o-mini', + ]) + + +class SettingsUpdate(BaseModel): + """Partial update for proactive AI settings.""" + + enabled: bool | None = None + suggestion_categories: list[str] | None = None + confidence_threshold: float | None = None + rate_limit_seconds: int | None = None + model: str | None = None + + +class StatsResponse(BaseModel): + """Proactive AI usage statistics.""" + + total_suggestions: int = 0 + dismissed: int = 0 + acted_upon: int = 0 + active: int = 0 + dismiss_rate: float = 0.0 + act_rate: float = 0.0 diff --git a/app/plugins/builtins/ai_proactive/services.py b/app/plugins/builtins/ai_proactive/services.py new file mode 100644 index 0000000..94e74dd --- /dev/null +++ b/app/plugins/builtins/ai_proactive/services.py @@ -0,0 +1,748 @@ +"""Proactive Engine — core logic for context-aware AI suggestions. + +Handles context changes, gathers entity data, generates LLM-powered +suggestions, pushes via SSE, and manages suggestion lifecycle. +""" + +from __future__ import annotations + +import os +import asyncio +import json +import logging +import uuid +from datetime import UTC, datetime, timedelta +from typing import Any + +import litellm +from sqlalchemy import func, select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.cache import get_cache +from app.core.db import create_db_session, get_session_factory +from app.core.notifications import create_notification +from app.models.audit import AuditLog +from app.models.company import Company +from app.models.contact import CompanyContact, Contact +from app.plugins.builtins.ai_proactive.models import ( + ContextLog, + ProactiveSettings, + ProactiveSuggestion, +) +logger = logging.getLogger(__name__) + +litellm.suppress_debug_info = True + +OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '') + +# ─── SSE Push Infrastructure ─── + +_sse_queues: dict[str, asyncio.Queue[dict[str, Any]]] = {} + + +def get_sse_queue(user_id: str) -> asyncio.Queue[dict[str, Any]]: + """Get or create SSE queue for a user.""" + if user_id not in _sse_queues: + _sse_queues[user_id] = asyncio.Queue() + return _sse_queues[user_id] + + +async def push_suggestion(user_id: str, suggestion: dict[str, Any]) -> None: + """Push suggestion to user's SSE queue.""" + queue = get_sse_queue(user_id) + await queue.put(suggestion) + + +# ─── Rate Limiting ─── + + +async def is_rate_limited( + tenant_id: uuid.UUID, user_id: uuid.UUID, rate_limit_seconds: int +) -> bool: + """Check if user is rate-limited using Redis. + + Returns True if rate-limited (key exists), False otherwise. + Sets a key with TTL = rate_limit_seconds on first call. + """ + try: + r = get_cache() + key = f"ai_proactive:rate:{user_id}" + existing = await r.get(key) + if existing is not None: + return True + await r.setex(key, rate_limit_seconds, "1") + return False + except Exception: + logger.exception("Rate-limit check failed, allowing request") + return False + + +# ─── Settings Helpers ─── + + +async def get_user_settings( + db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID +) -> ProactiveSettings: + """Get proactive AI settings for user, create defaults if not exist.""" + result = await db.execute( + select(ProactiveSettings) + .where(ProactiveSettings.tenant_id == tenant_id) + .where(ProactiveSettings.user_id == user_id) + .limit(1) + ) + settings = result.scalar_one_or_none() + if settings is None: + settings = ProactiveSettings( + tenant_id=tenant_id, + user_id=user_id, enabled=True, + suggestion_categories=["mail", "tasks", "contacts", "companies", "insights"], + confidence_threshold=0.5, + rate_limit_seconds=10, + model="ollama/deepseek-v4", + ) + db.add(settings) + await db.flush() + return settings + + +# ─── Context Gathering ─── + + +def _serialize_row(row: Any) -> dict[str, Any]: + """Serialize a SQLAlchemy model instance to a dict.""" + if row is None: + return {} + result: dict[str, Any] = {} + for column in row.__table__.columns: + val = getattr(row, column.name) + if isinstance(val, datetime): + result[column.name] = val.isoformat() + elif isinstance(val, uuid.UUID): + result[column.name] = str(val) + else: + result[column.name] = val + return result + + +async def gather_context( + db: AsyncSession, entity_type: str, entity_id: uuid.UUID, tenant_id: uuid.UUID +) -> dict[str, Any]: + """Collect context data for an entity. + + Gathers related data from contacts, companies, mails, calendar events, + audit logs, and semantically similar entities via unified_search. + """ + context: dict[str, Any] = { + "entity_type": entity_type, + "entity_id": str(entity_id), + } + + if entity_type == "contact": + # Contact data + result = await db.execute( + select(Contact) + .where(Contact.id == entity_id) + .where(Contact.tenant_id == tenant_id) + .limit(1) + ) + contact = result.scalar_one_or_none() + context["contact"] = _serialize_row(contact) if contact else None + + # Last 10 mails + from app.plugins.builtins.mail.models import Mail + + mail_result = await db.execute( + select(Mail) + .where(Mail.contact_id == entity_id) + .where(Mail.tenant_id == tenant_id) + .order_by(Mail.received_at.desc()) + .limit(10) + ) + context["mails"] = [_serialize_row(m) for m in mail_result.scalars().all()] + + # Company via company_contacts + cc_result = await db.execute( + select(CompanyContact) + .where(CompanyContact.contact_id == entity_id) + .where(CompanyContact.tenant_id == tenant_id) + .limit(5) + ) + companies: list[dict[str, Any]] = [] + for cc in cc_result.scalars().all(): + comp_result = await db.execute( + select(Company) + .where(Company.id == cc.company_id) + .where(Company.tenant_id == tenant_id) + .limit(1) + ) + comp = comp_result.scalar_one_or_none() + if comp: + comp_data = _serialize_row(comp) + comp_data["role_at_company"] = cc.role_at_company + comp_data["is_primary"] = cc.is_primary + companies.append(comp_data) + context["company"] = companies[0] if companies else None + context["companies"] = companies + + # Upcoming calendar events + from app.plugins.builtins.calendar.models import CalendarEntry, CalendarEntryLink + + now = datetime.now(UTC) + event_result = await db.execute( + select(CalendarEntry) + .join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id) + .where(CalendarEntryLink.entity_type == "contact") + .where(CalendarEntryLink.entity_id == entity_id) + .where(CalendarEntry.tenant_id == tenant_id) + .where(CalendarEntry.start_at > now) + .order_by(CalendarEntry.start_at.asc()) + .limit(5) + ) + context["events"] = [_serialize_row(e) for e in event_result.scalars().all()] + + # Last 20 audit log entries + audit_result = await db.execute( + select(AuditLog) + .where(AuditLog.entity_id == entity_id) + .where(AuditLog.tenant_id == tenant_id) + .order_by(AuditLog.timestamp.desc()) + .limit(20) + ) + context["activities"] = [_serialize_row(a) for a in audit_result.scalars().all()] + + elif entity_type == "mail": + from app.plugins.builtins.mail.models import Mail + + result = await db.execute( + select(Mail) + .where(Mail.id == entity_id) + .where(Mail.tenant_id == tenant_id) + .limit(1) + ) + mail = result.scalar_one_or_none() + context["mail"] = _serialize_row(mail) if mail else None + + if mail and mail.contact_id: + contact_result = await db.execute( + select(Contact) + .where(Contact.id == mail.contact_id) + .where(Contact.tenant_id == tenant_id) + .limit(1) + ) + contact = contact_result.scalar_one_or_none() + context["contact"] = _serialize_row(contact) if contact else None + + if mail and mail.thread_id: + thread_result = await db.execute( + select(Mail) + .where(Mail.thread_id == mail.thread_id) + .where(Mail.tenant_id == tenant_id) + .order_by(Mail.received_at.asc()) + .limit(20) + ) + context["thread"] = [_serialize_row(m) for m in thread_result.scalars().all()] + + if mail and mail.company_id: + comp_result = await db.execute( + select(Company) + .where(Company.id == mail.company_id) + .where(Company.tenant_id == tenant_id) + .limit(1) + ) + comp = comp_result.scalar_one_or_none() + context["company"] = _serialize_row(comp) if comp else None + + elif entity_type == "company": + result = await db.execute( + select(Company) + .where(Company.id == entity_id) + .where(Company.tenant_id == tenant_id) + .limit(1) + ) + company = result.scalar_one_or_none() + context["company"] = _serialize_row(company) if company else None + + # Contacts via company_contacts + cc_result = await db.execute( + select(CompanyContact) + .where(CompanyContact.company_id == entity_id) + .where(CompanyContact.tenant_id == tenant_id) + ) + contacts: list[dict[str, Any]] = [] + for cc in cc_result.scalars().all(): + contact_result = await db.execute( + select(Contact) + .where(Contact.id == cc.contact_id) + .where(Contact.tenant_id == tenant_id) + .limit(1) + ) + contact = contact_result.scalar_one_or_none() + if contact: + contact_data = _serialize_row(contact) + contact_data["role_at_company"] = cc.role_at_company + contact_data["is_primary"] = cc.is_primary + contacts.append(contact_data) + context["contacts"] = contacts + + # Mails for this company + from app.plugins.builtins.mail.models import Mail + + mail_result = await db.execute( + select(Mail) + .where(Mail.company_id == entity_id) + .where(Mail.tenant_id == tenant_id) + .order_by(Mail.received_at.desc()) + .limit(10) + ) + context["mails"] = [_serialize_row(m) for m in mail_result.scalars().all()] + + # Upcoming events + from app.plugins.builtins.calendar.models import CalendarEntry, CalendarEntryLink + + now = datetime.now(UTC) + event_result = await db.execute( + select(CalendarEntry) + .join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id) + .where(CalendarEntryLink.entity_type == "company") + .where(CalendarEntryLink.entity_id == entity_id) + .where(CalendarEntry.tenant_id == tenant_id) + .where(CalendarEntry.start_at > now) + .order_by(CalendarEntry.start_at.asc()) + .limit(5) + ) + context["events"] = [_serialize_row(e) for e in event_result.scalars().all()] + + elif entity_type == "file": + # Basic file info via raw SQL (file model may vary) + try: + file_result = await db.execute( + text("SELECT * FROM files WHERE id = :fid AND tenant_id = :tid"), + {"fid": entity_id, "tid": tenant_id}, + ) + file_row = file_result.mappings().first() + context["file"] = dict(file_row) if file_row else None + except Exception: + context["file"] = None + + # Linked entities via entity_links (if table exists) + try: + links_result = await db.execute( + text( + "SELECT * FROM entity_links WHERE entity_id = :eid AND tenant_id = :tid LIMIT 20" + ), + {"eid": entity_id, "tid": tenant_id}, + ) + context["linked_entities"] = [dict(r) for r in links_result.mappings().all()] + except Exception: + context["linked_entities"] = [] + + # Semantically similar entities via unified_search + try: + from app.plugins.builtins.unified_search.search_engine import ( + find_similar_all_types, + ) + + context["similar"] = await find_similar_all_types( + db, entity_type, entity_id, tenant_id, limit=3 + ) + except Exception: + context["similar"] = {} + + return context + + +# ─── Suggestion Generation ─── + + +SYSTEM_PROMPT = """Du bist ein proaktiver KI-Assistent für ein CRM. Analysiere den Kontext und generiere Vorschläge. + +Antworte mit JSON: +{ + "suggestion_type": "info|warning|action|insight", + "title": "Kurzer Titel (max 200 Zeichen)", + "content": "Beschreibung des Vorschlags", + "confidence": 0.0-1.0, + "actions": [{"method": "GET|POST|PUT|DELETE", "path": "/api/v1/...", "body": {}, "description": "..."}] +} + +- suggestion_type: info=Information, warning=Warnung, action=Aktionsvorschlag, insight=Erkenntnis +- actions: Vorgeschlagene CRM-Aktionen die der User ausführen kann +- confidence: Wie sicher bist du dir (0.0=unsicher, 1.0=sehr sicher) +- Antworte nur mit gültigem JSON, kein Markdown""" + + +async def generate_suggestion( + context_data: dict[str, Any], settings: ProactiveSettings +) -> dict[str, Any] | None: + """LLM generates a suggestion from context data. + + Returns dict with suggestion_type, title, content, confidence, actions + or None on failure. + """ + model = settings.model or "ollama/deepseek-v4" + try: + response = await litellm.acompletion( + model=model, + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + { + "role": "user", + "content": json.dumps(context_data, default=str, ensure_ascii=False), + }, + ], + temperature=0.3, + max_tokens=500, + response_format={"type": "json_object"}, + api_key=OLLAMA_API_KEY, + ) + content = response.choices[0].message.content + if not content: + return None + result = json.loads(content) + # Validate required fields + if not result.get("title") or not result.get("content"): + return None + # Ensure actions is a list + if not isinstance(result.get("actions"), list): + result["actions"] = [] + # Clamp confidence + confidence = result.get("confidence", 0.5) + try: + confidence = float(confidence) + except (TypeError, ValueError): + confidence = 0.5 + result["confidence"] = max(0.0, min(1.0, confidence)) + # Validate suggestion_type + valid_types = {"info", "warning", "action", "insight"} + if result.get("suggestion_type") not in valid_types: + result["suggestion_type"] = "info" + return result + except Exception: + logger.exception("Failed to generate suggestion via LLM") + return None + + +# ─── Main Handler ─── + + +async def handle_context_change(payload: dict[str, Any]) -> None: + """Main handler: Context-Change → Suggestion generation. + + 1. Rate-limit check + 2. Settings check (enabled?) + 3. Gather context data + 4. Generate suggestion via LLM + 5. Confidence threshold check + 6. Save suggestion to DB + 7. Push via SSE + 8. Notification for urgent suggestions + 9. Enqueue background deep analysis job + """ + user_id_str = payload.get("user_id") + tenant_id_str = payload.get("tenant_id") + entity_type = payload.get("entity_type") + entity_id_str = payload.get("entity_id") + + if not user_id_str or not tenant_id_str or not entity_type: + logger.warning("handle_context_change: missing required fields in payload") + return + + try: + tenant_id = uuid.UUID(tenant_id_str) + user_id = uuid.UUID(user_id_str) + except (ValueError, TypeError): + logger.warning("handle_context_change: invalid UUID in payload") + return + + entity_id: uuid.UUID | None = None + if entity_id_str: + try: + entity_id = uuid.UUID(entity_id_str) + except (ValueError, TypeError): + entity_id = None + + if entity_id is None: + logger.debug("handle_context_change: no entity_id, skipping") + return + + async with create_db_session(tenant_id) as db: + # Get settings + settings = await get_user_settings(db, tenant_id, user_id) + if not settings.enabled: + logger.debug("handle_context_change: proactive AI disabled for user") + return + + # Rate limit check + if await is_rate_limited(tenant_id, user_id, settings.rate_limit_seconds): + logger.debug("handle_context_change: rate limited") + return + + # Gather context + context_data = await gather_context(db, entity_type, entity_id, tenant_id) + + # Generate suggestion + suggestion_data = await generate_suggestion(context_data, settings) + if suggestion_data is None: + logger.debug("handle_context_change: no suggestion generated") + return + + # Confidence threshold check + if suggestion_data["confidence"] < settings.confidence_threshold: + logger.debug( + "handle_context_change: confidence %s below threshold %s", + suggestion_data["confidence"], + settings.confidence_threshold, + ) + return + + # Save suggestion + suggestion = ProactiveSuggestion( + tenant_id=tenant_id, + user_id=user_id, + entity_type=entity_type, + entity_id=entity_id, + suggestion_type=suggestion_data["suggestion_type"], + title=suggestion_data["title"], + content=suggestion_data["content"], + confidence=suggestion_data["confidence"], + actions=suggestion_data["actions"], + context_snapshot=context_data, + ) + db.add(suggestion) + await db.flush() + + # Build response dict for SSE + suggestion_dict = { + "id": str(suggestion.id), + "entity_type": suggestion.entity_type, + "entity_id": str(suggestion.entity_id) if suggestion.entity_id else None, + "suggestion_type": suggestion.suggestion_type, + "title": suggestion.title, + "content": suggestion.content, + "confidence": suggestion.confidence, + "actions": suggestion.actions, + "created_at": suggestion.created_at.isoformat() if suggestion.created_at else None, + "is_dismissed": suggestion.is_dismissed, + "is_acted_upon": suggestion.is_acted_upon, + } + + # Push via SSE + await push_suggestion(str(user_id), suggestion_dict) + + # Notification for urgent suggestions + if suggestion.suggestion_type == "warning": + try: + await create_notification( + db, + tenant_id, + user_id, + type="ai_suggestion_urgent", + title=suggestion.title, + body=suggestion.content[:200], + ) + except Exception: + logger.exception("Failed to create notification") + else: + try: + await create_notification( + db, + tenant_id, + user_id, + type="ai_suggestion", + title=suggestion.title, + body=suggestion.content[:200], + ) + except Exception: + logger.exception("Failed to create notification") + + await db.commit() + + # Enqueue background deep analysis job + try: + from app.core.jobs import enqueue_job + + await enqueue_job( + "deep_analysis", + entity_type, + str(entity_id), + str(user_id), + str(tenant_id), + ) + except Exception: + logger.exception("Failed to enqueue deep_analysis job") + + +# ─── Suggestion CRUD ─── + + +async def get_active_suggestions( + db: AsyncSession, + tenant_id: uuid.UUID, + user_id: uuid.UUID, + entity_type: str | None = None, + entity_id: uuid.UUID | None = None, + limit: int = 10, +) -> list[ProactiveSuggestion]: + """Get active (non-dismissed, not expired) suggestions for user.""" + now = datetime.now(UTC) + stmt = ( + select(ProactiveSuggestion) + .where(ProactiveSuggestion.tenant_id == tenant_id) + .where(ProactiveSuggestion.user_id == user_id) + .where(ProactiveSuggestion.is_dismissed == False) # noqa: E712 + .where( + (ProactiveSuggestion.expires_at.is_(None)) + | (ProactiveSuggestion.expires_at > now) + ) + ) + if entity_type: + stmt = stmt.where(ProactiveSuggestion.entity_type == entity_type) + if entity_id: + stmt = stmt.where(ProactiveSuggestion.entity_id == entity_id) + stmt = stmt.order_by(ProactiveSuggestion.created_at.desc()).limit(limit) + result = await db.execute(stmt) + return list(result.scalars().all()) + + +async def mark_dismissed( + db: AsyncSession, + suggestion_id: uuid.UUID, + user_id: uuid.UUID, + tenant_id: uuid.UUID, +) -> bool: + """Mark suggestion as dismissed. Returns True if found and updated.""" + result = await db.execute( + select(ProactiveSuggestion) + .where(ProactiveSuggestion.id == suggestion_id) + .where(ProactiveSuggestion.tenant_id == tenant_id) + .where(ProactiveSuggestion.user_id == user_id) + .limit(1) + ) + suggestion = result.scalar_one_or_none() + if suggestion is None: + return False + suggestion.is_dismissed = True + await db.flush() + return True + + +async def execute_suggested_action( + db: AsyncSession, + suggestion_id: uuid.UUID, + action_index: int, + user_id: uuid.UUID, + tenant_id: uuid.UUID, + user_context: dict[str, Any], +) -> dict[str, Any]: + """Execute a suggested action. + + 1. Load suggestion + 2. Get action from actions[action_index] + 3. Execute via internal HTTP call (httpx) + 4. Mark suggestion as is_acted_upon=True + 5. Return result + """ + import httpx + + result = await db.execute( + select(ProactiveSuggestion) + .where(ProactiveSuggestion.id == suggestion_id) + .where(ProactiveSuggestion.tenant_id == tenant_id) + .where(ProactiveSuggestion.user_id == user_id) + .limit(1) + ) + suggestion = result.scalar_one_or_none() + if suggestion is None: + return {"success": False, "error": "Suggestion not found"} + + actions = suggestion.actions or [] + if action_index < 0 or action_index >= len(actions): + return {"success": False, "error": "Invalid action index"} + + action = actions[action_index] + method = action.get("method", "GET").upper() + path = action.get("path", "") + body = action.get("body") + + if not path: + return {"success": False, "error": "No path in action"} + + # Build internal URL + from app.config import get_settings + + settings = get_settings() + base_url = getattr(settings, "internal_base_url", "http://localhost:8000") + url = f"{base_url}{path}" + + # Build headers from user context (session cookie) + headers: dict[str, str] = {"Content-Type": "application/json"} + cookie_name = getattr(settings, "session_cookie_name", "session") + session_id = user_context.get("session_id", "") + if session_id: + headers["Cookie"] = f"{cookie_name}={session_id}" + + try: + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.request( + method, + url, + json=body if body else None, + headers=headers, + ) + data = None + try: + data = resp.json() + except Exception: + data = {"status_code": resp.status_code, "text": resp.text} + if resp.status_code < 400: + suggestion.is_acted_upon = True + await db.flush() + return {"success": True, "data": data} + return {"success": False, "error": f"HTTP {resp.status_code}", "data": data} + except Exception as e: + logger.exception("Failed to execute suggested action") + return {"success": False, "error": str(e)} + + +# ─── Stats ─── + + +async def get_stats( + db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID +) -> dict[str, Any]: + """Get proactive AI usage statistics for a user.""" + base_filter = ( + ProactiveSuggestion.tenant_id == tenant_id, + ProactiveSuggestion.user_id == user_id, + ) + total_result = await db.execute( + select(func.count()).select_from(ProactiveSuggestion).where(*base_filter) + ) + total = total_result.scalar() or 0 + + dismissed_result = await db.execute( + select(func.count()) + .select_from(ProactiveSuggestion) + .where(*base_filter, ProactiveSuggestion.is_dismissed == True) # noqa: E712 + ) + dismissed = dismissed_result.scalar() or 0 + + acted_result = await db.execute( + select(func.count()) + .select_from(ProactiveSuggestion) + .where(*base_filter, ProactiveSuggestion.is_acted_upon == True) # noqa: E712 + ) + acted_upon = acted_result.scalar() or 0 + + active = total - dismissed + dismiss_rate = (dismissed / total) if total > 0 else 0.0 + act_rate = (acted_upon / total) if total > 0 else 0.0 + + return { + "total_suggestions": total, + "dismissed": dismissed, + "acted_upon": acted_upon, + "active": active, + "dismiss_rate": round(dismiss_rate, 4), + "act_rate": round(act_rate, 4), + } diff --git a/app/plugins/builtins/unified_search/__init__.py b/app/plugins/builtins/unified_search/__init__.py new file mode 100644 index 0000000..f484ee4 --- /dev/null +++ b/app/plugins/builtins/unified_search/__init__.py @@ -0,0 +1,7 @@ +"""Unified Search plugin — hybrid FTS + semantic search across all CRM data.""" + +from __future__ import annotations + +from app.plugins.builtins.unified_search.plugin import UnifiedSearchPlugin + +__all__ = ["UnifiedSearchPlugin"] diff --git a/app/plugins/builtins/unified_search/embedding.py b/app/plugins/builtins/unified_search/embedding.py new file mode 100644 index 0000000..86ff04e --- /dev/null +++ b/app/plugins/builtins/unified_search/embedding.py @@ -0,0 +1,131 @@ +"""Embedding pipeline using LiteLLM with configurable Ollama Cloud provider.""" + +from __future__ import annotations + +import os +import logging +import uuid +from typing import TYPE_CHECKING + +import litellm + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession + +logger = logging.getLogger(__name__) + +MAX_INPUT_CHARS = 8000 + +DEFAULT_EMBEDDING_MODEL = os.environ.get('SEARCH_EMBEDDING_MODEL', 'ollama/nomic-embed-text') +OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '') + + +async def generate_embedding(text: str, model: str | None = None) -> list[float]: + """Generate a single embedding via LiteLLM. + + Args: + text: Input text (truncated to 8000 chars). + model: Embedding model name (default: ollama/nomic-embed-text). + + Returns: + Embedding vector as list of floats. + """ + model = model or DEFAULT_EMBEDDING_MODEL + truncated = text[:MAX_INPUT_CHARS] + try: + response = await litellm.aembedding( + model=model, + input=truncated, + api_key=OLLAMA_API_KEY, + ) + return response.data[0]["embedding"] + except Exception: + logger.exception("Failed to generate embedding") + return [] + + +async def generate_embeddings_batch( + texts: list[str], model: str | None = None +) -> list[list[float]]: + """Generate embeddings for multiple texts in a single API call. + + Args: + texts: List of input texts. + model: Embedding model name (default: ollama/nomic-embed-text). + + Returns: + List of embedding vectors. + """ + model = model or DEFAULT_EMBEDDING_MODEL + truncated = [t[:MAX_INPUT_CHARS] for t in texts] + try: + response = await litellm.aembedding( + model=model, + input=truncated, + api_key=OLLAMA_API_KEY, + ) + return [d["embedding"] for d in response.data] + except Exception: + logger.exception("Failed to generate batch embeddings") + return [[] for _ in texts] + + +async def index_entity( + entity_type: str, + entity_id: uuid.UUID, + tenant_id: uuid.UUID, + db: AsyncSession, +) -> bool: + """Generate and store embedding for a single entity. + + Uses the provider registry to get embedding text, generates embedding, + and updates the entity's embedding column. + + Returns True on success, False on failure. + """ + from app.plugins.builtins.unified_search.provider_registry import get_search_registry + + registry = get_search_registry() + provider = registry.get(entity_type) + if provider is None: + logger.warning("No provider for entity_type=%s", entity_type) + return False + + try: + text = await provider.get_embedding_text(db, entity_id, tenant_id) + if not text.strip(): + logger.debug("Empty embedding text for %s/%s", entity_type, entity_id) + return False + + embedding = await generate_embedding(text) + if not embedding: + return False + + # Update the entity's embedding column + from sqlalchemy import text as sql_text + + table_map = { + "contact": "contacts", + "company": "companies", + "mail": "mails", + "file": "files", + "event": "calendar_entries", + } + table = table_map.get(entity_type) + if not table: + logger.warning("Unknown entity_type=%s for embedding storage", entity_type) + return False + + sql = sql_text( + f"UPDATE {table} SET embedding = cast(:emb AS vector) " + f"WHERE id = :eid AND tenant_id = :tid" + ) + await db.execute( + sql, + {"emb": str(embedding), "eid": entity_id, "tid": tenant_id}, + ) + await db.commit() + return True + except Exception: + logger.exception("Failed to index entity %s/%s", entity_type, entity_id) + return False diff --git a/app/plugins/builtins/unified_search/jobs.py b/app/plugins/builtins/unified_search/jobs.py new file mode 100644 index 0000000..dd1913b --- /dev/null +++ b/app/plugins/builtins/unified_search/jobs.py @@ -0,0 +1,246 @@ +"""ARQ background jobs for the Unified Search plugin.""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from app.core.db import get_session_factory + +logger = logging.getLogger(__name__) + +BATCH_SIZE = 100 + + +def _parse_id(id_str: str) -> uuid.UUID: + """Parse a string to UUID.""" + if isinstance(id_str, uuid.UUID): + return id_str + return uuid.UUID(str(id_str)) + + +async def index_mails(ctx: dict[str, Any], mail_ids: list[str]) -> None: + """Index mails: generate and store embeddings.""" + from app.plugins.builtins.unified_search.embedding import index_entity + + factory = get_session_factory() + async with factory() as db: + for mail_id in mail_ids: + try: + eid = _parse_id(mail_id) + # tenant_id is derived from the mail itself + from sqlalchemy import text + result = await db.execute( + text("SELECT tenant_id FROM mails WHERE id = :mid"), + {"mid": eid}, + ) + row = result.mappings().first() + if not row: + continue + tenant_id = row["tenant_id"] + await index_entity("mail", eid, tenant_id, db) + except Exception: + logger.exception("Failed to index mail %s", mail_id) + + +async def index_file(ctx: dict[str, Any], file_id: str) -> None: + """Index a file: extract text, store content_text, generate embedding.""" + from sqlalchemy import text + from app.plugins.builtins.unified_search.text_extraction import extract_text_from_file + from app.plugins.builtins.unified_search.embedding import generate_embedding + + factory = get_session_factory() + async with factory() as db: + try: + eid = _parse_id(file_id) + result = await db.execute( + text("SELECT tenant_id, storage_path, mime_type FROM files WHERE id = :fid"), + {"fid": eid}, + ) + row = result.mappings().first() + if not row: + logger.warning("File not found: %s", file_id) + return + + tenant_id = row["tenant_id"] + storage_path = row["storage_path"] + mime_type = row["mime_type"] + + # Extract text + content_text = await extract_text_from_file(storage_path, mime_type) + + # Store content_text + await db.execute( + text("UPDATE files SET content_text = :ct WHERE id = :fid"), + {"ct": content_text, "fid": eid}, + ) + await db.commit() + + # Generate embedding from extracted text + filename + result_name = await db.execute( + text("SELECT name FROM files WHERE id = :fid"), + {"fid": eid}, + ) + name_row = result_name.mappings().first() + name = name_row["name"] if name_row else "" + embedding_text = f"{name} {content_text[:5000]}" + + if embedding_text.strip(): + embedding = await generate_embedding(embedding_text) + if embedding: + await db.execute( + text("UPDATE files SET embedding = cast(:emb AS vector) WHERE id = :fid"), + {"emb": str(embedding), "fid": eid}, + ) + await db.commit() + + logger.info("Indexed file %s", file_id) + except Exception: + logger.exception("Failed to index file %s", file_id) + + +async def index_contact(ctx: dict[str, Any], contact_id: str) -> None: + """Index a contact: generate and store embedding.""" + from sqlalchemy import text + from app.plugins.builtins.unified_search.embedding import index_entity + + factory = get_session_factory() + async with factory() as db: + try: + eid = _parse_id(contact_id) + result = await db.execute( + text("SELECT tenant_id FROM contacts WHERE id = :cid"), + {"cid": eid}, + ) + row = result.mappings().first() + if not row: + return + await index_entity("contact", eid, row["tenant_id"], db) + except Exception: + logger.exception("Failed to index contact %s", contact_id) + + +async def index_company(ctx: dict[str, Any], company_id: str) -> None: + """Index a company: generate and store embedding.""" + from sqlalchemy import text + from app.plugins.builtins.unified_search.embedding import index_entity + + factory = get_session_factory() + async with factory() as db: + try: + eid = _parse_id(company_id) + result = await db.execute( + text("SELECT tenant_id FROM companies WHERE id = :cid"), + {"cid": eid}, + ) + row = result.mappings().first() + if not row: + return + await index_entity("company", eid, row["tenant_id"], db) + except Exception: + logger.exception("Failed to index company %s", company_id) + + +async def index_event(ctx: dict[str, Any], event_id: str) -> None: + """Index a calendar event: generate and store embedding.""" + from sqlalchemy import text + from app.plugins.builtins.unified_search.embedding import index_entity + + factory = get_session_factory() + async with factory() as db: + try: + eid = _parse_id(event_id) + result = await db.execute( + text("SELECT tenant_id FROM calendar_entries WHERE id = :eid"), + {"eid": eid}, + ) + row = result.mappings().first() + if not row: + return + await index_entity("event", eid, row["tenant_id"], db) + except Exception: + logger.exception("Failed to index event %s", event_id) + + +async def reindex(ctx: dict[str, Any], entity_type: str) -> None: + """Reindex all entities of a given type with pagination.""" + from sqlalchemy import text + from app.plugins.builtins.unified_search.embedding import index_entity + + table_map = { + "contact": "contacts", + "company": "companies", + "mail": "mails", + "file": "files", + "event": "calendar_entries", + } + table = table_map.get(entity_type) + if not table: + logger.warning("Unknown entity_type for reindex: %s", entity_type) + return + + factory = get_session_factory() + async with factory() as db: + offset = 0 + while True: + result = await db.execute( + text( + f"SELECT id, tenant_id FROM {table} " + f"WHERE deleted_at IS NULL ORDER BY created_at LIMIT :lim OFFSET :off" + ), + {"lim": BATCH_SIZE, "off": offset}, + ) + rows = result.mappings().all() + if not rows: + break + for row in rows: + try: + await index_entity( + entity_type, + row["id"], + row["tenant_id"], + db, + ) + except Exception: + logger.exception("Reindex failed for %s/%s", entity_type, row["id"]) + offset += BATCH_SIZE + + logger.info("Reindex complete for %s", entity_type) + + +async def embedding_batch(ctx: dict[str, Any]) -> None: + """Periodic job: find entities without embeddings and index them.""" + from sqlalchemy import text + from app.plugins.builtins.unified_search.embedding import index_entity + + table_map = { + "contact": "contacts", + "company": "companies", + "mail": "mails", + "file": "files", + "event": "calendar_entries", + } + + factory = get_session_factory() + async with factory() as db: + for etype, table in table_map.items(): + try: + result = await db.execute( + text( + f"SELECT id, tenant_id FROM {table} " + f"WHERE deleted_at IS NULL AND embedding IS NULL " + f"LIMIT :lim" + ), + {"lim": BATCH_SIZE}, + ) + rows = result.mappings().all() + for row in rows: + try: + await index_entity(etype, row["id"], row["tenant_id"], db) + except Exception: + logger.exception("Batch index failed for %s/%s", etype, row["id"]) + except Exception: + logger.exception("Batch query failed for %s", etype) + + logger.info("Embedding batch job complete") diff --git a/app/plugins/builtins/unified_search/migrations/0001_initial.sql b/app/plugins/builtins/unified_search/migrations/0001_initial.sql new file mode 100644 index 0000000..4f19573 --- /dev/null +++ b/app/plugins/builtins/unified_search/migrations/0001_initial.sql @@ -0,0 +1,154 @@ +-- Unified Search: pgvector extension, FTS triggers, provider registry + +-- pgvector extension +CREATE EXTENSION IF NOT EXISTS vector; + +-- ─── Provider Registry Table ─── +CREATE TABLE IF NOT EXISTS unified_search_providers ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tenant_id UUID NOT NULL, + entity_type VARCHAR(50) NOT NULL, + plugin_name VARCHAR(80) NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + config JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE(tenant_id, entity_type) +); + +CREATE INDEX IF NOT EXISTS ix_usp_tenant ON unified_search_providers(tenant_id); + +-- ─── Index Log Table ─── +CREATE TABLE IF NOT EXISTS unified_search_index_log ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + tenant_id UUID NOT NULL, + entity_type VARCHAR(50) NOT NULL, + entity_id UUID NOT NULL, + action VARCHAR(20) NOT NULL, + status VARCHAR(20) NOT NULL DEFAULT 'pending', + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS ix_usil_tenant ON unified_search_index_log(tenant_id); +CREATE INDEX IF NOT EXISTS ix_usil_entity ON unified_search_index_log(entity_type, entity_id); + +-- ─── FTS: Mails ─── +ALTER TABLE mails ADD COLUMN IF NOT EXISTS body_tsv tsvector; + +CREATE OR REPLACE FUNCTION mails_tsv_trigger() RETURNS trigger AS $$ +BEGIN + NEW.body_tsv := + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.subject, '')), 'A') || + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.from_address, '')), 'B') || + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.to_addresses, '')), 'C') || + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.body_text, '')), 'D'); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS mails_tsv_update ON mails; +CREATE TRIGGER mails_tsv_update + BEFORE INSERT OR UPDATE ON mails + FOR EACH ROW EXECUTE FUNCTION mails_tsv_trigger(); + +CREATE INDEX IF NOT EXISTS ix_mails_body_tsv ON mails USING gin(body_tsv); + +-- ─── FTS: Contacts ─── +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS search_tsv tsvector; + +CREATE OR REPLACE FUNCTION contacts_tsv_trigger() RETURNS trigger AS $$ +BEGIN + NEW.search_tsv := + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.first_name, '') || ' ' || coalesce(NEW.last_name, '')), 'A') || + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.email, '')), 'B') || + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.phone, '') || ' ' || coalesce(NEW.mobile, '')), 'C') || + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.notes, '')), 'D'); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS contacts_tsv_update ON contacts; +CREATE TRIGGER contacts_tsv_update + BEFORE INSERT OR UPDATE ON contacts + FOR EACH ROW EXECUTE FUNCTION contacts_tsv_trigger(); + +CREATE INDEX IF NOT EXISTS ix_contacts_search_tsv ON contacts USING gin(search_tsv); + +-- ─── FTS: Calendar Entries ─── +ALTER TABLE calendar_entries ADD COLUMN IF NOT EXISTS search_tsv tsvector; + +CREATE OR REPLACE FUNCTION calendar_entries_tsv_trigger() RETURNS trigger AS $$ +BEGIN + NEW.search_tsv := + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.title, '')), 'A') || + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.description, '')), 'B') || + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.location, '')), 'C'); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS calendar_entries_tsv_update ON calendar_entries; +CREATE TRIGGER calendar_entries_tsv_update + BEFORE INSERT OR UPDATE ON calendar_entries + FOR EACH ROW EXECUTE FUNCTION calendar_entries_tsv_trigger(); + +CREATE INDEX IF NOT EXISTS ix_cal_entries_search_tsv ON calendar_entries USING gin(search_tsv); + +-- ─── FTS: Files (content_text added here, not in 0002, to avoid trigger dependency) ─── +ALTER TABLE files ADD COLUMN IF NOT EXISTS content_text text; +ALTER TABLE files ADD COLUMN IF NOT EXISTS content_tsv tsvector; + +CREATE OR REPLACE FUNCTION files_tsv_trigger() RETURNS trigger AS $$ +BEGIN + NEW.content_tsv := + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.name, '')), 'A') || + setweight(to_tsvector('pg_catalog.german', coalesce(NEW.content_text, '')), 'D'); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS files_tsv_update ON files; +CREATE TRIGGER files_tsv_update + BEFORE INSERT OR UPDATE ON files + FOR EACH ROW EXECUTE FUNCTION files_tsv_trigger(); + +CREATE INDEX IF NOT EXISTS ix_files_content_tsv ON files USING gin(content_tsv); + +-- ─── FTS: Tags ─── +ALTER TABLE tags ADD COLUMN IF NOT EXISTS search_tsv tsvector; + +CREATE OR REPLACE FUNCTION tags_tsv_trigger() RETURNS trigger AS $$ +BEGIN + NEW.search_tsv := to_tsvector('pg_catalog.german', coalesce(NEW.name, '')); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS tags_tsv_update ON tags; +CREATE TRIGGER tags_tsv_update + BEFORE INSERT OR UPDATE ON tags + FOR EACH ROW EXECUTE FUNCTION tags_tsv_trigger(); + +CREATE INDEX IF NOT EXISTS ix_tags_search_tsv ON tags USING gin(search_tsv); + +-- ─── FTS: Audit Log ─── +ALTER TABLE audit_log ADD COLUMN IF NOT EXISTS search_tsv tsvector; + +CREATE OR REPLACE FUNCTION audit_log_tsv_trigger() RETURNS trigger AS $$ +BEGIN + NEW.search_tsv := + to_tsvector('pg_catalog.german', + coalesce(NEW.entity_type, '') || ' ' || + coalesce(NEW.action, '') || ' ' || + coalesce(NEW.user_id::text, '')); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS audit_log_tsv_update ON audit_log; +CREATE TRIGGER audit_log_tsv_update + BEFORE INSERT OR UPDATE ON audit_log + FOR EACH ROW EXECUTE FUNCTION audit_log_tsv_trigger(); + +CREATE INDEX IF NOT EXISTS ix_audit_log_search_tsv ON audit_log USING gin(search_tsv); diff --git a/app/plugins/builtins/unified_search/migrations/0002_embeddings.sql b/app/plugins/builtins/unified_search/migrations/0002_embeddings.sql new file mode 100644 index 0000000..1d1ce9b --- /dev/null +++ b/app/plugins/builtins/unified_search/migrations/0002_embeddings.sql @@ -0,0 +1,24 @@ +-- Unified Search: Embedding columns and HNSW indexes + +-- ─── Mails ─── +ALTER TABLE mails ADD COLUMN IF NOT EXISTS embedding vector(768); +CREATE INDEX IF NOT EXISTS ix_mails_embedding ON mails USING hnsw(embedding vector_cosine_ops); + +-- ─── Contacts ─── +ALTER TABLE contacts ADD COLUMN IF NOT EXISTS embedding vector(768); +CREATE INDEX IF NOT EXISTS ix_contacts_embedding ON contacts USING hnsw(embedding vector_cosine_ops); + +-- ─── Companies ─── +ALTER TABLE companies ADD COLUMN IF NOT EXISTS embedding vector(768); +CREATE INDEX IF NOT EXISTS ix_companies_embedding ON companies USING hnsw(embedding vector_cosine_ops); + +-- ─── Files (content_text already added in 0001) ─── +ALTER TABLE files ADD COLUMN IF NOT EXISTS embedding vector(768); +CREATE INDEX IF NOT EXISTS ix_files_embedding ON files USING hnsw(embedding vector_cosine_ops); + +-- ─── Calendar Entries ─── +ALTER TABLE calendar_entries ADD COLUMN IF NOT EXISTS embedding vector(768); +CREATE INDEX IF NOT EXISTS ix_cal_embedding ON calendar_entries USING hnsw(embedding vector_cosine_ops); + +-- ─── Tags (shorter dimension for short text) ─── +ALTER TABLE tags ADD COLUMN IF NOT EXISTS embedding vector(384); diff --git a/app/plugins/builtins/unified_search/models.py b/app/plugins/builtins/unified_search/models.py new file mode 100644 index 0000000..960ad88 --- /dev/null +++ b/app/plugins/builtins/unified_search/models.py @@ -0,0 +1,50 @@ +"""SQLAlchemy models for the Unified Search plugin.""" + +from __future__ import annotations + +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, Index, String, Text +from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID +from sqlalchemy.orm import Mapped, mapped_column + +from app.core.db import Base, TenantMixin + + +class SearchProviderRegistry(Base, TenantMixin): + """Registry of active search providers per tenant.""" + + __tablename__ = "unified_search_providers" + __table_args__ = ( + Index("ix_usp_tenant", "tenant_id"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + entity_type: Mapped[str] = mapped_column(String(50), nullable=False) + plugin_name: Mapped[str] = mapped_column(String(80), nullable=False) + is_active: Mapped[bool] = mapped_column( + nullable=False, default=True + ) + config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict) + + +class SearchIndexLog(Base, TenantMixin): + """Log of indexing actions for audit and debugging.""" + + __tablename__ = "unified_search_index_log" + __table_args__ = ( + Index("ix_usil_tenant", "tenant_id"), + Index("ix_usil_entity", "entity_type", "entity_id"), + ) + + id: Mapped[uuid.UUID] = mapped_column( + PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4 + ) + entity_type: Mapped[str] = mapped_column(String(50), nullable=False) + entity_id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), nullable=False) + action: Mapped[str] = mapped_column(String(20), nullable=False) + status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending") + error_message: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/app/plugins/builtins/unified_search/plugin.py b/app/plugins/builtins/unified_search/plugin.py new file mode 100644 index 0000000..43c3643 --- /dev/null +++ b/app/plugins/builtins/unified_search/plugin.py @@ -0,0 +1,137 @@ +"""Unified Search plugin class and manifest.""" + +from __future__ import annotations + +import logging +from typing import Any + +from app.plugins.base import BasePlugin +from app.plugins.manifest import PluginManifest, PluginRouteDef + +logger = logging.getLogger(__name__) + + +class UnifiedSearchPlugin(BasePlugin): + """Hybrid full-text and semantic search across all CRM entities.""" + + manifest = PluginManifest( + name="unified_search", + version="1.0.0", + display_name="Unified Search", + description=( + "Hybrid full-text (PostgreSQL FTS) and semantic (pgvector) search " + "with KI query understanding and RRF rank fusion across all CRM data." + ), + dependencies=[], + routes=[ + PluginRouteDef( + path="/api/v1/search", + module="app.plugins.builtins.unified_search.routes", + router_attr="router", + ), + ], + events=[ + "mail.synced", + "file.uploaded", + "contact.created", + "contact.updated", + "company.created", + "company.updated", + "calendar.entry.created", + ], + migrations=["0001_initial.sql", "0002_embeddings.sql"], + permissions=["search:read", "search:admin"], + is_core=False, + ) + + async def on_activate(self, db, service_container, event_bus) -> None: + """Register search providers on activation.""" + await super().on_activate(db, service_container, event_bus) + try: + from app.plugins.builtins.unified_search.provider_registry import ( + auto_register_providers, + ) + await auto_register_providers(db) + logger.info("Unified Search providers auto-registered") + except Exception: + logger.exception("Failed to auto-register search providers") + + async def on_deactivate(self, db, service_container, event_bus) -> None: + """Clear provider registry on deactivation.""" + from app.plugins.builtins.unified_search.provider_registry import ( + get_search_registry, + ) + registry = get_search_registry() + registry.clear() + await super().on_deactivate(db, service_container, event_bus) + + # ─── Event Handlers ─── + + async def on_mail_synced(self, payload: dict[str, Any]) -> None: + """Enqueue embedding jobs for synced mails.""" + from app.core.jobs import enqueue_job + + mail_ids = payload.get("mail_ids", []) + if mail_ids: + await enqueue_job("index_mails", mail_ids) + + async def on_file_uploaded(self, payload: dict[str, Any]) -> None: + """Enqueue file indexing job.""" + from app.core.jobs import enqueue_job + + file_id = payload.get("file_id") + if file_id: + await enqueue_job("index_file", file_id) + + async def on_contact_created(self, payload: dict[str, Any]) -> None: + from app.core.jobs import enqueue_job + + contact_id = payload.get("contact_id") + if contact_id: + await enqueue_job("index_contact", contact_id) + + async def on_contact_updated(self, payload: dict[str, Any]) -> None: + from app.core.jobs import enqueue_job + + contact_id = payload.get("contact_id") + if contact_id: + await enqueue_job("index_contact", contact_id) + + async def on_company_created(self, payload: dict[str, Any]) -> None: + from app.core.jobs import enqueue_job + + company_id = payload.get("company_id") + if company_id: + await enqueue_job("index_company", company_id) + + async def on_company_updated(self, payload: dict[str, Any]) -> None: + from app.core.jobs import enqueue_job + + company_id = payload.get("company_id") + if company_id: + await enqueue_job("index_company", company_id) + + async def on_calendar_entry_created(self, payload: dict[str, Any]) -> None: + from app.core.jobs import enqueue_job + + entry_id = payload.get("entry_id") + if entry_id: + await enqueue_job("index_event", entry_id) + + def get_notification_types(self) -> list[dict[str, Any]]: + return [ + { + "type_key": "search_error", + "category": "search", + "label": "Suchfehler", + "description": "Fehler bei der Suchausführung", + "is_enabled_by_default": True, + }, + { + "type_key": "search_reindex_complete", + "category": "search", + "label": "Reindex abgeschlossen", + "description": "Neuindizierung abgeschlossen", + "is_enabled_by_default": False, + }, + ] diff --git a/app/plugins/builtins/unified_search/provider_registry.py b/app/plugins/builtins/unified_search/provider_registry.py new file mode 100644 index 0000000..f547f69 --- /dev/null +++ b/app/plugins/builtins/unified_search/provider_registry.py @@ -0,0 +1,133 @@ +"""SearchProvider protocol and global registry.""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +from sqlalchemy import select + +if TYPE_CHECKING: + import uuid + from sqlalchemy.ext.asyncio import AsyncSession + +logger = logging.getLogger(__name__) + + +@runtime_checkable +class SearchProvider(Protocol): + """Protocol for entity-specific search providers.""" + + entity_type: str + + async def search_fts( + self, + db: AsyncSession, + tsquery: str, + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict]: + """Full-text search using PostgreSQL tsquery.""" + ... + + async def search_vector( + self, + db: AsyncSession, + embedding: list[float], + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict]: + """Semantic vector search using pgvector.""" + ... + + async def get_embedding_text( + self, + db: AsyncSession, + entity_id: uuid.UUID, + tenant_id: uuid.UUID, + ) -> str: + """Get text representation for embedding generation.""" + ... + + def to_search_result(self, entity: object) -> dict: + """Convert an ORM entity to a search result dict.""" + ... + + +class SearchProviderRegistry: + """In-memory registry of search providers.""" + + def __init__(self) -> None: + self._providers: dict[str, SearchProvider] = {} + + def register(self, provider: SearchProvider) -> None: + """Register a search provider.""" + self._providers[provider.entity_type] = provider + logger.debug("Registered search provider: %s", provider.entity_type) + + def unregister(self, entity_type: str) -> None: + """Unregister a search provider by entity type.""" + self._providers.pop(entity_type, None) + + def get(self, entity_type: str) -> SearchProvider | None: + """Get a provider by entity type.""" + return self._providers.get(entity_type) + + def get_all(self) -> list[SearchProvider]: + """Get all registered providers.""" + return list(self._providers.values()) + + def get_entity_types(self) -> list[str]: + """Get all registered entity types.""" + return list(self._providers.keys()) + + def clear(self) -> None: + """Clear all registered providers.""" + self._providers.clear() + + +# Global singleton +_registry = SearchProviderRegistry() + + +def get_search_registry() -> SearchProviderRegistry: + """Get the global search provider registry.""" + return _registry + + +async def auto_register_providers(db: AsyncSession) -> None: + """Auto-register providers for active plugins. + + Checks which plugins are active and registers corresponding providers. + """ + from app.plugins.builtins.unified_search.providers.contact_provider import ( + ContactSearchProvider, + ) + from app.plugins.builtins.unified_search.providers.company_provider import ( + CompanySearchProvider, + ) + from app.plugins.builtins.unified_search.providers.mail_provider import ( + MailSearchProvider, + ) + from app.plugins.builtins.unified_search.providers.file_provider import ( + FileSearchProvider, + ) + from app.plugins.builtins.unified_search.providers.event_provider import ( + EventSearchProvider, + ) + + registry = get_search_registry() + registry.clear() + + # Register all built-in providers + for provider_cls in [ + ContactSearchProvider, + CompanySearchProvider, + MailSearchProvider, + FileSearchProvider, + EventSearchProvider, + ]: + try: + registry.register(provider_cls()) + except Exception: + logger.exception("Failed to register %s", provider_cls.__name__) diff --git a/app/plugins/builtins/unified_search/providers/__init__.py b/app/plugins/builtins/unified_search/providers/__init__.py new file mode 100644 index 0000000..8870fb3 --- /dev/null +++ b/app/plugins/builtins/unified_search/providers/__init__.py @@ -0,0 +1,3 @@ +"""Search providers for unified search.""" + +from __future__ import annotations diff --git a/app/plugins/builtins/unified_search/providers/company_provider.py b/app/plugins/builtins/unified_search/providers/company_provider.py new file mode 100644 index 0000000..9bea152 --- /dev/null +++ b/app/plugins/builtins/unified_search/providers/company_provider.py @@ -0,0 +1,114 @@ +"""Company search provider.""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +logger = logging.getLogger(__name__) + + +class CompanySearchProvider: + """Search provider for Company entities.""" + + entity_type = "company" + + async def search_fts( + self, + db: AsyncSession, + tsquery: str, + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Full-text search on companies.search_tsv.""" + sql = text( + """ + SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank + FROM companies c + WHERE c.tenant_id = :tid + AND c.deleted_at IS NULL + AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q) + ORDER BY rank DESC + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"q": tsquery, "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def search_vector( + self, + db: AsyncSession, + embedding: list[float], + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Semantic search on companies.embedding.""" + sql = text( + """ + SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score + FROM companies c + WHERE c.tenant_id = :tid + AND c.deleted_at IS NULL + AND c.embedding IS NOT NULL + ORDER BY c.embedding <=> cast(:emb AS vector) + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"emb": str(embedding), "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def get_embedding_text( + self, + db: AsyncSession, + entity_id: uuid.UUID, + tenant_id: uuid.UUID, + ) -> str: + """Get text for embedding generation.""" + sql = text( + """ + SELECT name, description, industry + FROM companies + WHERE id = :eid AND tenant_id = :tid + """ + ) + result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id}) + row = result.mappings().first() + if not row: + return "" + parts = [ + row.get("name", ""), + row.get("description", ""), + row.get("industry", ""), + ] + return " ".join(str(p) for p in parts if p) + + def to_search_result(self, entity: object) -> dict[str, Any]: + """Convert company to search result dict.""" + if isinstance(entity, dict): + name = entity.get("name", "") + description = entity.get("description", "") or "" + entity_id = str(entity.get("id", "")) + else: + name = getattr(entity, "name", "") + description = getattr(entity, "description", "") or "" + entity_id = str(getattr(entity, "id", "")) + return { + "entity_type": self.entity_type, + "entity_id": entity_id, + "title": name, + "snippet": description[:200], + "score": 0.0, + "data": {}, + } diff --git a/app/plugins/builtins/unified_search/providers/contact_provider.py b/app/plugins/builtins/unified_search/providers/contact_provider.py new file mode 100644 index 0000000..b891f9a --- /dev/null +++ b/app/plugins/builtins/unified_search/providers/contact_provider.py @@ -0,0 +1,121 @@ +"""Contact search provider.""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.models.contact import Contact + +logger = logging.getLogger(__name__) + + +class ContactSearchProvider: + """Search provider for Contact entities.""" + + entity_type = "contact" + + async def search_fts( + self, + db: AsyncSession, + tsquery: str, + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Full-text search on contacts.search_tsv.""" + sql = text( + """ + SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank + FROM contacts c + WHERE c.tenant_id = :tid + AND c.deleted_at IS NULL + AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q) + ORDER BY rank DESC + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"q": tsquery, "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def search_vector( + self, + db: AsyncSession, + embedding: list[float], + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Semantic search on contacts.embedding.""" + sql = text( + """ + SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score + FROM contacts c + WHERE c.tenant_id = :tid + AND c.deleted_at IS NULL + AND c.embedding IS NOT NULL + ORDER BY c.embedding <=> cast(:emb AS vector) + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"emb": str(embedding), "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def get_embedding_text( + self, + db: AsyncSession, + entity_id: uuid.UUID, + tenant_id: uuid.UUID, + ) -> str: + """Get text for embedding generation.""" + sql = text( + """ + SELECT first_name, last_name, email, phone, mobile, notes + FROM contacts + WHERE id = :eid AND tenant_id = :tid + """ + ) + result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id}) + row = result.mappings().first() + if not row: + return "" + parts = [ + row.get("first_name", ""), + row.get("last_name", ""), + row.get("email", ""), + row.get("phone", ""), + row.get("mobile", ""), + row.get("notes", ""), + ] + return " ".join(str(p) for p in parts if p) + + def to_search_result(self, entity: object) -> dict[str, Any]: + """Convert contact to search result dict.""" + if isinstance(entity, dict): + first = entity.get("first_name", "") + last = entity.get("last_name", "") + email = entity.get("email", "") + entity_id = str(entity.get("id", "")) + else: + first = getattr(entity, "first_name", "") + last = getattr(entity, "last_name", "") + email = getattr(entity, "email", "") + entity_id = str(getattr(entity, "id", "")) + return { + "entity_type": self.entity_type, + "entity_id": entity_id, + "title": f"{first} {last}".strip(), + "snippet": email or "", + "score": 0.0, + "data": {}, + } diff --git a/app/plugins/builtins/unified_search/providers/event_provider.py b/app/plugins/builtins/unified_search/providers/event_provider.py new file mode 100644 index 0000000..2d74951 --- /dev/null +++ b/app/plugins/builtins/unified_search/providers/event_provider.py @@ -0,0 +1,114 @@ +"""Calendar event search provider.""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +logger = logging.getLogger(__name__) + + +class EventSearchProvider: + """Search provider for CalendarEntry entities.""" + + entity_type = "event" + + async def search_fts( + self, + db: AsyncSession, + tsquery: str, + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Full-text search on calendar_entries.search_tsv.""" + sql = text( + """ + SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank + FROM calendar_entries e + WHERE e.tenant_id = :tid + AND e.deleted_at IS NULL + AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q) + ORDER BY rank DESC + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"q": tsquery, "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def search_vector( + self, + db: AsyncSession, + embedding: list[float], + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Semantic search on calendar_entries.embedding.""" + sql = text( + """ + SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score + FROM calendar_entries e + WHERE e.tenant_id = :tid + AND e.deleted_at IS NULL + AND e.embedding IS NOT NULL + ORDER BY e.embedding <=> cast(:emb AS vector) + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"emb": str(embedding), "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def get_embedding_text( + self, + db: AsyncSession, + entity_id: uuid.UUID, + tenant_id: uuid.UUID, + ) -> str: + """Get text for embedding generation.""" + sql = text( + """ + SELECT title, description, location + FROM calendar_entries + WHERE id = :eid AND tenant_id = :tid + """ + ) + result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id}) + row = result.mappings().first() + if not row: + return "" + parts = [ + row.get("title", ""), + row.get("description", ""), + row.get("location", ""), + ] + return " ".join(str(p) for p in parts if p) + + def to_search_result(self, entity: object) -> dict[str, Any]: + """Convert calendar entry to search result dict.""" + if isinstance(entity, dict): + title = entity.get("title", "") + description = entity.get("description", "") or "" + entity_id = str(entity.get("id", "")) + else: + title = getattr(entity, "title", "") + description = getattr(entity, "description", "") or "" + entity_id = str(getattr(entity, "id", "")) + return { + "entity_type": self.entity_type, + "entity_id": entity_id, + "title": title, + "snippet": description[:200], + "score": 0.0, + "data": {}, + } diff --git a/app/plugins/builtins/unified_search/providers/file_provider.py b/app/plugins/builtins/unified_search/providers/file_provider.py new file mode 100644 index 0000000..e0b00b4 --- /dev/null +++ b/app/plugins/builtins/unified_search/providers/file_provider.py @@ -0,0 +1,111 @@ +"""DMS File search provider.""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +logger = logging.getLogger(__name__) + + +class FileSearchProvider: + """Search provider for DMS File entities.""" + + entity_type = "file" + + async def search_fts( + self, + db: AsyncSession, + tsquery: str, + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Full-text search on files.content_tsv.""" + sql = text( + """ + SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank + FROM files f + WHERE f.tenant_id = :tid + AND f.deleted_at IS NULL + AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q) + ORDER BY rank DESC + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"q": tsquery, "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def search_vector( + self, + db: AsyncSession, + embedding: list[float], + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Semantic search on files.embedding.""" + sql = text( + """ + SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score + FROM files f + WHERE f.tenant_id = :tid + AND f.deleted_at IS NULL + AND f.embedding IS NOT NULL + ORDER BY f.embedding <=> cast(:emb AS vector) + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"emb": str(embedding), "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def get_embedding_text( + self, + db: AsyncSession, + entity_id: uuid.UUID, + tenant_id: uuid.UUID, + ) -> str: + """Get text for embedding generation.""" + sql = text( + """ + SELECT name, content_text + FROM files + WHERE id = :eid AND tenant_id = :tid + """ + ) + result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id}) + row = result.mappings().first() + if not row: + return "" + name = row.get("name", "") or "" + content = row.get("content_text", "") or "" + return f"{name} {content[:5000]}" + + def to_search_result(self, entity: object) -> dict[str, Any]: + """Convert file to search result dict.""" + if isinstance(entity, dict): + name = entity.get("name", "") + content = entity.get("content_text", "") or "" + entity_id = str(entity.get("id", "")) + else: + name = getattr(entity, "name", "") + content = getattr(entity, "content_text", "") or "" + entity_id = str(getattr(entity, "id", "")) + return { + "entity_type": self.entity_type, + "entity_id": entity_id, + "title": name, + "snippet": content[:200], + "score": 0.0, + "data": {}, + } diff --git a/app/plugins/builtins/unified_search/providers/mail_provider.py b/app/plugins/builtins/unified_search/providers/mail_provider.py new file mode 100644 index 0000000..b9d1779 --- /dev/null +++ b/app/plugins/builtins/unified_search/providers/mail_provider.py @@ -0,0 +1,111 @@ +"""Mail search provider.""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +logger = logging.getLogger(__name__) + + +class MailSearchProvider: + """Search provider for Mail entities.""" + + entity_type = "mail" + + async def search_fts( + self, + db: AsyncSession, + tsquery: str, + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Full-text search on mails.body_tsv.""" + sql = text( + """ + SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank + FROM mails m + WHERE m.tenant_id = :tid + AND m.deleted_at IS NULL + AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q) + ORDER BY rank DESC + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"q": tsquery, "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def search_vector( + self, + db: AsyncSession, + embedding: list[float], + tenant_id: uuid.UUID, + limit: int, + ) -> list[dict[str, Any]]: + """Semantic search on mails.embedding.""" + sql = text( + """ + SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score + FROM mails m + WHERE m.tenant_id = :tid + AND m.deleted_at IS NULL + AND m.embedding IS NOT NULL + ORDER BY m.embedding <=> cast(:emb AS vector) + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"emb": str(embedding), "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + return [dict(r) for r in rows] + + async def get_embedding_text( + self, + db: AsyncSession, + entity_id: uuid.UUID, + tenant_id: uuid.UUID, + ) -> str: + """Get text for embedding generation.""" + sql = text( + """ + SELECT subject, body_text + FROM mails + WHERE id = :eid AND tenant_id = :tid + """ + ) + result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id}) + row = result.mappings().first() + if not row: + return "" + subject = row.get("subject", "") or "" + body = row.get("body_text", "") or "" + return f"{subject} {body[:5000]}" + + def to_search_result(self, entity: object) -> dict[str, Any]: + """Convert mail to search result dict.""" + if isinstance(entity, dict): + subject = entity.get("subject", "") + body = entity.get("body_text", "") or "" + entity_id = str(entity.get("id", "")) + else: + subject = getattr(entity, "subject", "") + body = getattr(entity, "body_text", "") or "" + entity_id = str(getattr(entity, "id", "")) + return { + "entity_type": self.entity_type, + "entity_id": entity_id, + "title": subject, + "snippet": body[:200], + "score": 0.0, + "data": {}, + } diff --git a/app/plugins/builtins/unified_search/query_understanding.py b/app/plugins/builtins/unified_search/query_understanding.py new file mode 100644 index 0000000..60d50ef --- /dev/null +++ b/app/plugins/builtins/unified_search/query_understanding.py @@ -0,0 +1,101 @@ +"""KI query understanding and result aggregation via LiteLLM.""" + +from __future__ import annotations + +import os +import json +import logging +from typing import Any + +import litellm + +logger = logging.getLogger(__name__) + +DEFAULT_LLM_MODEL = os.environ.get('SEARCH_LLM_MODEL', 'ollama/deepseek-v4') +OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '') + +QUERY_ANALYZE_SYSTEM = ( + "Du bist ein Query-Analyzer fuer ein CRM. " + "Analysiere die Suchanfrage und gib JSON zurueck: " + '{"normalized_query": str, "entities": {"person": str|null, "company": str|null, "topic": str|null}, ' + '"intent": str, "semantic_terms": [str], "suggested_filters": {}}' +) + +RESULT_AGGREGATE_SYSTEM = ( + "Du bist ein Result-Aggregator. Fasse Ergebnisse zusammen und generiere Facetten: " + '{"summary": str, "facets": {"types": {}, "dates": {}, "people": []}, "suggestions": [str]}' +) + + +def _fallback_query_analysis(query: str) -> dict[str, Any]: + return { + "normalized_query": query, + "entities": {}, + "intent": "search", + "semantic_terms": [], + "suggested_filters": {}, + } + + +def _fallback_aggregate(results: list[dict], query: str) -> dict[str, Any]: + return { + "summary": f"{len(results)} Ergebnisse gefunden", + "facets": {}, + "suggestions": [], + } + + +async def llm_analyze_query(query: str) -> dict[str, Any]: + """Analyze a search query using LLM for intent, entities, and semantic terms. + + Falls back to a simple dict if LLM fails. + """ + try: + response = await litellm.acompletion( + model=DEFAULT_LLM_MODEL, + messages=[ + {"role": "system", "content": QUERY_ANALYZE_SYSTEM}, + {"role": "user", "content": query}, + ], + temperature=0.1, + max_tokens=500, + response_format={"type": "json_object"}, + api_key=OLLAMA_API_KEY, + ) + content = response.choices[0].message.content + return json.loads(content) + except Exception: + logger.warning("LLM query analysis failed, using fallback") + return _fallback_query_analysis(query) + + +async def llm_aggregate_results(results: list[dict], query: str) -> dict[str, Any]: + """Aggregate search results using LLM for summary, facets, and suggestions. + + Falls back to a simple dict if LLM fails. + """ + if not results: + return _fallback_aggregate(results, query) + try: + # Truncate results to avoid token overflow + compact = [ + {"entity_type": r.get("entity_type"), "title": r.get("title", "")[:100]} + for r in results[:50] + ] + user_msg = json.dumps({"query": query, "results": compact}) + response = await litellm.acompletion( + model=DEFAULT_LLM_MODEL, + messages=[ + {"role": "system", "content": RESULT_AGGREGATE_SYSTEM}, + {"role": "user", "content": user_msg}, + ], + temperature=0.1, + max_tokens=1000, + response_format={"type": "json_object"}, + api_key=OLLAMA_API_KEY, + ) + content = response.choices[0].message.content + return json.loads(content) + except Exception: + logger.warning("LLM result aggregation failed, using fallback") + return _fallback_aggregate(results, query) diff --git a/app/plugins/builtins/unified_search/routes.py b/app/plugins/builtins/unified_search/routes.py new file mode 100644 index 0000000..5ba1bcc --- /dev/null +++ b/app/plugins/builtins/unified_search/routes.py @@ -0,0 +1,316 @@ +"""API routes for the Unified Search plugin.""" + +from __future__ import annotations + +import logging +import uuid + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.db import get_db +from app.core.jobs import enqueue_job +from app.deps import get_current_user, require_permission +from app.plugins.builtins.unified_search.provider_registry import get_search_registry +from app.plugins.builtins.unified_search.query_understanding import ( + llm_aggregate_results, + llm_analyze_query, +) +from app.plugins.builtins.unified_search.schemas import ( + ProviderResponse, + ReindexRequest, + SearchRequest, + SearchResponse, + SearchResult, + SimilarRequest, + SimilarResponse, + SuggestRequest, + SuggestResponse, +) +from app.plugins.builtins.unified_search.search_engine import ( + autocomplete, + find_similar_all_types, + hybrid_search, +) + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/v1/search", tags=["search"]) + + +# ─── Search ─── + +@router.post("", dependencies=[Depends(require_permission("search:read"))]) +async def search( + req: SearchRequest, + current_user: dict = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> SearchResponse: + """Perform hybrid search with KI query understanding.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + + # KI query understanding + query_analysis = await llm_analyze_query(req.query) + + # Hybrid search + results = await hybrid_search( + db=db, + query_analysis=query_analysis, + tenant_id=tenant_id, + entity_types=req.entity_types, + limit=req.limit, + ) + + # KI result aggregation + aggregation = await llm_aggregate_results(results, req.query) + + search_results = [ + SearchResult( + entity_type=r.get("entity_type", ""), + entity_id=r.get("entity_id", ""), + title=r.get("title", ""), + snippet=r.get("snippet", ""), + score=r.get("score", 0.0), + data=r.get("data", {}), + ) + for r in results + ] + + return SearchResponse( + query=req.query, + normalized_query=query_analysis.get("normalized_query", req.query), + results=search_results, + facets=aggregation.get("facets", {}), + summary=aggregation.get("summary", f"{len(results)} Ergebnisse"), + suggestions=aggregation.get("suggestions", []), + ) + + +# ─── Suggest / Autocomplete ─── + +@router.get("/suggest", dependencies=[Depends(require_permission("search:read"))]) +async def suggest( + q: str = Query(..., min_length=1, max_length=200), + limit: int = Query(default=10, ge=1, le=50), + current_user: dict = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> SuggestResponse: + """Autocomplete suggestions using FTS prefix search.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + suggestions = await autocomplete(db, q, tenant_id, limit) + return SuggestResponse(suggestions=suggestions) + + +# ─── Similar ─── + +@router.post("/similar", dependencies=[Depends(require_permission("search:read"))]) +async def find_similar( + req: SimilarRequest, + current_user: dict = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> SimilarResponse: + """Find similar entities across all types based on embedding.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + try: + entity_id = uuid.UUID(req.entity_id) + except ValueError: + raise HTTPException(status_code=400, detail="Invalid entity_id") + + similar = await find_similar_all_types( + db=db, + entity_type=req.entity_type, + entity_id=entity_id, + tenant_id=tenant_id, + limit=req.limit, + ) + + result_dict: dict[str, list[SearchResult]] = {} + for etype, items in similar.items(): + result_dict[etype] = [ + SearchResult( + entity_type=r.get("entity_type", etype), + entity_id=r.get("entity_id", ""), + title=r.get("title", ""), + snippet=r.get("snippet", ""), + score=r.get("score", 0.0), + data=r.get("data", {}), + ) + for r in items + ] + + return SimilarResponse(similar=result_dict) + + +# ─── Reindex ─── + +@router.post("/reindex", dependencies=[Depends(require_permission("search:admin"))]) +async def reindex( + req: ReindexRequest, + current_user: dict = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + """Trigger reindexing of specified entity types.""" + entity_types = req.entity_types + if not entity_types: + entity_types = get_search_registry().get_entity_types() + + job_ids: list[str] = [] + for etype in entity_types: + job_id = await enqueue_job("reindex", etype) + if job_id: + job_ids.append(job_id) + + return { + "status": "ok", + "message": f"Reindexing {len(entity_types)} entity types", + "entity_types": entity_types, + "job_ids": job_ids, + } + + +# ─── Providers ─── + +@router.get("/providers", dependencies=[Depends(require_permission("search:read"))]) +async def list_providers( + current_user: dict = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> list[ProviderResponse]: + """List all active search providers.""" + registry = get_search_registry() + providers = registry.get_all() + return [ + ProviderResponse( + entity_type=p.entity_type, + plugin_name="unified_search", + is_active=True, + ) + for p in providers + ] + + +@router.post("/providers/{entity_type}/toggle", dependencies=[Depends(require_permission("search:admin"))]) +async def toggle_provider( + entity_type: str, + current_user: dict = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + """Toggle a search provider on/off.""" + registry = get_search_registry() + provider = registry.get(entity_type) + if provider is None: + raise HTTPException(status_code=404, detail=f"Provider not found: {entity_type}") + + # Toggle in DB + tenant_id = uuid.UUID(current_user["tenant_id"]) + result = await db.execute( + text( + """ + SELECT is_active FROM unified_search_providers + WHERE tenant_id = :tid AND entity_type = :et + """ + ), + {"tid": tenant_id, "et": entity_type}, + ) + row = result.mappings().first() + + if row: + new_status = not row["is_active"] + await db.execute( + text( + """ + UPDATE unified_search_providers SET is_active = :active + WHERE tenant_id = :tid AND entity_type = :et + """ + ), + {"active": new_status, "tid": tenant_id, "et": entity_type}, + ) + else: + new_status = False + await db.execute( + text( + """ + INSERT INTO unified_search_providers (tenant_id, entity_type, plugin_name, is_active) + VALUES (:tid, :et, 'unified_search', false) + """ + ), + {"tid": tenant_id, "et": entity_type}, + ) + + await db.commit() + + if not new_status: + registry.unregister(entity_type) + else: + # Re-register by clearing and re-running auto_register + from app.plugins.builtins.unified_search.provider_registry import auto_register_providers + await auto_register_providers(db) + + return { + "entity_type": entity_type, + "is_active": new_status, + } + + +# ─── Stats ─── + +@router.get("/stats", dependencies=[Depends(require_permission("search:read"))]) +async def search_stats( + current_user: dict = Depends(get_current_user), + db: AsyncSession = Depends(get_db), +) -> dict: + """Get search index statistics.""" + tenant_id = uuid.UUID(current_user["tenant_id"]) + + stats: dict = {} + tables = { + "contacts": "embedding", + "companies": "embedding", + "mails": "embedding", + "files": "embedding", + "calendar_entries": "embedding", + } + + for table, col in tables.items(): + try: + total_result = await db.execute( + text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL"), + {"tid": tenant_id}, + ) + total = total_result.scalar() or 0 + + indexed_result = await db.execute( + text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL AND {col} IS NOT NULL"), + {"tid": tenant_id}, + ) + indexed = indexed_result.scalar() or 0 + + stats[table] = { + "total": total, + "indexed": indexed, + "pending": total - indexed, + } + except Exception: + logger.exception("Stats query failed for %s", table) + stats[table] = {"total": 0, "indexed": 0, "pending": 0} + + # Last index log entries + try: + log_result = await db.execute( + text( + """ + SELECT entity_type, action, status, created_at + FROM unified_search_index_log + WHERE tenant_id = :tid + ORDER BY created_at DESC + LIMIT 10 + """ + ), + {"tid": tenant_id}, + ) + recent_logs = [dict(r) for r in log_result.mappings().all()] + except Exception: + recent_logs = [] + + stats["recent_logs"] = recent_logs + return stats diff --git a/app/plugins/builtins/unified_search/schemas.py b/app/plugins/builtins/unified_search/schemas.py new file mode 100644 index 0000000..25f33f9 --- /dev/null +++ b/app/plugins/builtins/unified_search/schemas.py @@ -0,0 +1,71 @@ +"""Pydantic schemas for the Unified Search plugin.""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + + +# ─── Search ─── + +class SearchRequest(BaseModel): + query: str = Field(..., min_length=1, max_length=500) + entity_types: list[str] | None = None + limit: int = Field(default=20, ge=1, le=100) + offset: int = Field(default=0, ge=0) + + +class SearchResult(BaseModel): + entity_type: str + entity_id: str + title: str + snippet: str + score: float + data: dict[str, Any] = Field(default_factory=dict) + + +class SearchResponse(BaseModel): + query: str + normalized_query: str + results: list[SearchResult] + facets: dict[str, Any] + summary: str + suggestions: list[str] + + +# ─── Similar ─── + +class SimilarRequest(BaseModel): + entity_type: str + entity_id: str + limit: int = Field(default=5, ge=1, le=50) + + +class SimilarResponse(BaseModel): + similar: dict[str, list[SearchResult]] + + +# ─── Suggest ─── + +class SuggestRequest(BaseModel): + q: str = Field(..., min_length=1, max_length=200) + limit: int = Field(default=10, ge=1, le=50) + + +class SuggestResponse(BaseModel): + suggestions: list[str] + + +# ─── Reindex ─── + +class ReindexRequest(BaseModel): + entity_types: list[str] = Field(default_factory=list) + + +# ─── Provider ─── + +class ProviderResponse(BaseModel): + entity_type: str + plugin_name: str + is_active: bool diff --git a/app/plugins/builtins/unified_search/search_engine.py b/app/plugins/builtins/unified_search/search_engine.py new file mode 100644 index 0000000..12b8deb --- /dev/null +++ b/app/plugins/builtins/unified_search/search_engine.py @@ -0,0 +1,252 @@ +"""Hybrid search engine: FTS + pgvector with RRF fusion.""" + +from __future__ import annotations + +import logging +import uuid +from typing import Any + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.plugins.builtins.unified_search.embedding import generate_embedding +from app.plugins.builtins.unified_search.provider_registry import get_search_registry + +logger = logging.getLogger(__name__) + +# Entity type -> (table_name, tsv_column, embedding_column) +SEARCHABLE_ENTITIES: dict[str, tuple[str, str, str]] = { + "contact": ("contacts", "search_tsv", "embedding"), + "company": ("companies", "search_tsv", "embedding"), + "mail": ("mails", "body_tsv", "embedding"), + "file": ("files", "content_tsv", "embedding"), + "event": ("calendar_entries", "search_tsv", "embedding"), +} + +RRF_K = 60 +RRF_ALPHA = 0.5 +RRF_BETA = 0.5 + + +def rrf_fusion( + fts_results: list[dict[str, Any]], + vec_results: list[dict[str, Any]], + entity_type: str, + k: int = RRF_K, + alpha: float = RRF_ALPHA, + beta: float = RRF_BETA, +) -> list[dict[str, Any]]: + """Reciprocal Rank Fusion of FTS and vector search results. + + score = alpha * (1/(k+rank_fts)) + beta * (1/(k+rank_vec)) + """ + fused: dict[str, dict[str, Any]] = {} + + for rank, item in enumerate(fts_results): + eid = str(item.get("id", "")) + if not eid: + continue + rrf_score = alpha * (1.0 / (k + rank + 1)) + if eid not in fused: + fused[eid] = {**item, "_score": 0.0, "_entity_type": entity_type} + fused[eid]["_score"] += rrf_score + + for rank, item in enumerate(vec_results): + eid = str(item.get("id", "")) + if not eid: + continue + rrf_score = beta * (1.0 / (k + rank + 1)) + if eid not in fused: + fused[eid] = {**item, "_score": 0.0, "_entity_type": entity_type} + fused[eid]["_score"] += rrf_score + + return sorted(fused.values(), key=lambda x: x.get("_score", 0.0), reverse=True) + + +async def hybrid_search( + db: AsyncSession, + query_analysis: dict[str, Any], + tenant_id: uuid.UUID, + entity_types: list[str] | None = None, + limit: int = 20, +) -> list[dict[str, Any]]: + """Perform hybrid search across all entity types. + + For each entity: FTS search + vector search + RRF fusion. + Merge all results, sort by fused score, return top limit. + """ + registry = get_search_registry() + all_entity_types = entity_types or registry.get_entity_types() + + normalized_query = query_analysis.get("normalized_query", "") + semantic_terms = query_analysis.get("semantic_terms", []) + + # Build tsquery string + tsquery_parts = [normalized_query] + semantic_terms + tsquery = " & ".join( + part.strip().replace(" ", " & ") + for part in tsquery_parts + if part and part.strip() + ) + if not tsquery: + tsquery = normalized_query + + # Generate query embedding + query_text = normalized_query + if semantic_terms: + query_text = f"{normalized_query} {' '.join(semantic_terms)}" + query_embedding = await generate_embedding(query_text) + + all_results: list[dict[str, Any]] = [] + fetch_limit = limit * 2 + + for entity_type in all_entity_types: + provider = registry.get(entity_type) + if provider is None: + continue + + fts_results: list[dict[str, Any]] = [] + vec_results: list[dict[str, Any]] = [] + + try: + fts_results = await provider.search_fts(db, tsquery, tenant_id, fetch_limit) + except Exception: + logger.exception("FTS search failed for %s", entity_type) + + if query_embedding: + try: + vec_results = await provider.search_vector(db, query_embedding, tenant_id, fetch_limit) + except Exception: + logger.exception("Vector search failed for %s", entity_type) + + fused = rrf_fusion(fts_results, vec_results, entity_type) + + # Convert to search result format + for item in fused: + result = provider.to_search_result(item) + result["score"] = item.get("_score", 0.0) + all_results.append(result) + + all_results.sort(key=lambda x: x.get("score", 0.0), reverse=True) + return all_results[:limit] + + +async def find_similar_all_types( + db: AsyncSession, + entity_type: str, + entity_id: uuid.UUID, + tenant_id: uuid.UUID, + limit: int = 5, +) -> dict[str, list[dict[str, Any]]]: + """Find similar entities across all types based on embedding. + + Gets the embedding of the source entity, then searches all other tables. + """ + table_map = SEARCHABLE_ENTITIES + source_info = table_map.get(entity_type) + if not source_info: + return {} + + table_name, _, emb_col = source_info + + # Get source embedding + sql = text( + f"SELECT {emb_col} AS embedding FROM {table_name} " + f"WHERE id = :eid AND tenant_id = :tid" + ) + result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id}) + row = result.mappings().first() + if not row or not row.get("embedding"): + return {} + + source_embedding_str = str(row["embedding"]) + + registry = get_search_registry() + similar: dict[str, list[dict[str, Any]]] = {} + + for etype, (tbl, _, emb) in table_map.items(): + if etype == entity_type: + continue + provider = registry.get(etype) + if provider is None: + continue + try: + sql_sim = text( + f""" + SELECT *, 1 - ({emb} <=> cast(:emb_str AS vector)) AS score + FROM {tbl} + WHERE tenant_id = :tid + AND deleted_at IS NULL + AND {emb} IS NOT NULL + ORDER BY {emb} <=> cast(:emb_str AS vector) + LIMIT :lim + """ + ) + res = await db.execute( + sql_sim, + {"emb_str": source_embedding_str, "tid": tenant_id, "lim": limit}, + ) + rows = res.mappings().all() + results = [] + for r in rows: + sr = provider.to_search_result(dict(r)) + sr["score"] = float(r.get("score", 0.0)) + results.append(sr) + similar[etype] = results + except Exception: + logger.exception("Similar search failed for %s", etype) + similar[etype] = [] + + return similar + + +async def autocomplete( + db: AsyncSession, + query: str, + tenant_id: uuid.UUID, + limit: int = 10, +) -> list[str]: + """Autocomplete using FTS prefix search. + + Uses to_tsquery with :* suffix for prefix matching. + """ + query_clean = query.strip().replace(" ", " & ") + if not query_clean: + return [] + tsquery = f"{query_clean}:*" + + suggestions: list[str] = [] + registry = get_search_registry() + + for entity_type in registry.get_entity_types(): + provider = registry.get(entity_type) + if provider is None: + continue + entity_info = SEARCHABLE_ENTITIES.get(entity_type) + if not entity_info: + continue + table_name, tsv_col, _ = entity_info + try: + sql = text( + f""" + SELECT * FROM {table_name} + WHERE tenant_id = :tid + AND deleted_at IS NULL + AND {tsv_col} @@ to_tsquery('pg_catalog.german', :q) + LIMIT :lim + """ + ) + result = await db.execute( + sql, + {"q": tsquery, "tid": tenant_id, "lim": limit}, + ) + rows = result.mappings().all() + for r in rows: + sr = provider.to_search_result(dict(r)) + title = sr.get("title", "") + if title and title not in suggestions: + suggestions.append(title) + except Exception: + logger.debug("Autocomplete failed for %s", entity_type) + + return suggestions[:limit] diff --git a/app/plugins/builtins/unified_search/text_extraction.py b/app/plugins/builtins/unified_search/text_extraction.py new file mode 100644 index 0000000..d71c8b8 --- /dev/null +++ b/app/plugins/builtins/unified_search/text_extraction.py @@ -0,0 +1,126 @@ +"""Text extraction from various file formats for DMS indexing.""" + +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) + +MAX_CHARS = 50_000 + + +async def extract_text_from_file(file_path: str, mime_type: str) -> str: + """Extract text content from a file based on its MIME type. + + Supports: + - PDF (via PyMuPDF/fitz) + - DOCX (via python-docx) + - XLSX (via openpyxl) + - PPTX (via python-pptx) + - text/* (via aiofiles) + - Images: returns empty (OCR not implemented) + + Args: + file_path: Absolute path to the file. + mime_type: MIME type of the file. + + Returns: + Extracted text, truncated to MAX_CHARS. Empty string on error. + """ + try: + if mime_type == "application/pdf": + return await _extract_pdf(file_path) + elif mime_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document": + return await _extract_docx(file_path) + elif mime_type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": + return await _extract_xlsx(file_path) + elif mime_type == "application/vnd.openxmlformats-officedocument.presentationml.presentation": + return await _extract_pptx(file_path) + elif mime_type.startswith("text/"): + return await _extract_text(file_path) + elif mime_type.startswith("image/"): + return "" + else: + logger.debug("Unsupported mime_type=%s for text extraction", mime_type) + return "" + except Exception: + logger.warning("Failed to extract text from %s (mime=%s)", file_path, mime_type) + return "" + + +def _truncate(text: str) -> str: + """Truncate text to MAX_CHARS with indicator.""" + if len(text) > MAX_CHARS: + return text[:MAX_CHARS] + "... [truncated]" + return text + + +async def _extract_pdf(file_path: str) -> str: + """Extract text from PDF using PyMuPDF (fitz).""" + import fitz # PyMuPDF + + doc = fitz.open(file_path) + parts: list[str] = [] + for page in doc: + parts.append(page.get_text()) + doc.close() + return _truncate("\n".join(parts)) + + +async def _extract_docx(file_path: str) -> str: + """Extract text from DOCX using python-docx.""" + from docx import Document + + doc = Document(file_path) + parts: list[str] = [] + for para in doc.paragraphs: + if para.text.strip(): + parts.append(para.text) + # Also extract table text + for table in doc.tables: + for row in table.rows: + for cell in row.cells: + if cell.text.strip(): + parts.append(cell.text) + return _truncate("\n".join(parts)) + + +async def _extract_xlsx(file_path: str) -> str: + """Extract text from XLSX using openpyxl.""" + from openpyxl import load_workbook + + wb = load_workbook(file_path, read_only=True, data_only=True) + parts: list[str] = [] + for ws in wb.worksheets: + for row in ws.iter_rows(values_only=True): + row_text = " ".join(str(c) for c in row if c is not None) + if row_text.strip(): + parts.append(row_text) + wb.close() + return _truncate("\n".join(parts)) + + +async def _extract_pptx(file_path: str) -> str: + """Extract text from PPTX using python-pptx.""" + from pptx import Presentation + + prs = Presentation(file_path) + parts: list[str] = [] + for slide in prs.slides: + for shape in slide.shapes: + if shape.has_text_frame: + for para in shape.text_frame.paragraphs: + text = para.text.strip() + if text: + parts.append(text) + return _truncate("\n".join(parts)) + + +async def _extract_text(file_path: str) -> str: + """Read plain text file using aiofiles.""" + import aiofiles + + async with aiofiles.open(file_path, mode="r", encoding="utf-8", errors="replace") as f: + content = await f.read() + return _truncate(content) diff --git a/frontend/src/api/aiProactive.ts b/frontend/src/api/aiProactive.ts new file mode 100644 index 0000000..e94bc28 --- /dev/null +++ b/frontend/src/api/aiProactive.ts @@ -0,0 +1,72 @@ +import { useState, useEffect, useCallback } from 'react'; +import { apiClient } from '@/api/client'; + +export interface Suggestion { + id: string; + entity_type: string; + entity_id: string | null; + suggestion_type: 'info' | 'warning' | 'action' | 'insight'; + title: string; + content: string; + confidence: number; + actions: Array<{ method: string; path: string; body: any; description: string }>; + created_at: string; + is_dismissed: boolean; + is_acted_upon: boolean; +} + +export function useSuggestions() { + const [suggestions, setSuggestions] = useState([]); + const [connected, setConnected] = useState(false); + + useEffect(() => { + // Initial load + apiClient.get('/api/v1/ai-proactive/suggestions').then(r => { + setSuggestions(r.data.items); + }).catch(() => {}); + + // SSE Stream + const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream'); + eventSource.onopen = () => setConnected(true); + eventSource.onerror = () => setConnected(false); + eventSource.onmessage = (e) => { + try { + const suggestion = JSON.parse(e.data); + setSuggestions(prev => [suggestion, ...prev].slice(0, 50)); + } catch {} + }; + + return () => eventSource.close(); + }, []); + + const dismiss = useCallback(async (id: string) => { + setSuggestions(prev => prev.filter(s => s.id !== id)); + await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/dismiss`).catch(() => {}); + }, []); + + const act = useCallback(async (id: string, actionIndex: number) => { + const result = await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/act`, { + action_index: actionIndex + }); + setSuggestions(prev => prev.map(s => s.id === id ? { ...s, is_acted_upon: true } : s)); + return result.data; + }, []); + + return { suggestions, connected, dismiss, act }; +} + +export function useProactiveSettings() { + const [settings, setSettings] = useState(null); + + useEffect(() => { + apiClient.get('/api/v1/ai-proactive/settings').then(r => setSettings(r.data)); + }, []); + + const update = useCallback(async (updates: any) => { + const r = await apiClient.put('/api/v1/ai-proactive/settings', updates); + setSettings(r.data); + return r.data; + }, []); + + return { settings, update }; +} diff --git a/frontend/src/api/hooks.ts b/frontend/src/api/hooks.ts index e0e6780..8cd3c5f 100644 --- a/frontend/src/api/hooks.ts +++ b/frontend/src/api/hooks.ts @@ -388,45 +388,9 @@ export function useGlobalSearch(query: string, entityTypes?: string[]) { queryKey: ['globalSearch', query, entityTypes], queryFn: async (): Promise => { if (!query.trim()) return []; - const results: SearchResult[] = []; - const types = entityTypes && entityTypes.length > 0 ? entityTypes : ['company', 'contact']; - const tasks: Promise[] = []; - if (types.includes('company')) { - tasks.push( - apiGet>(`/companies?page=1&page_size=10&search=${encodeURIComponent(query)}`) - .then((data) => { - for (const item of data.items) { - results.push({ - type: 'company', - id: item.id, - name: item.name, - description: item.industry || item.email || '', - url: `/companies/${item.id}`, - }); - } - }) - .catch(() => {}) - ); - } - if (types.includes('contact')) { - tasks.push( - apiGet>(`/contacts?page=1&page_size=10&search=${encodeURIComponent(query)}`) - .then((data) => { - for (const item of data.items) { - results.push({ - type: 'contact', - id: item.id, - name: `${item.first_name} ${item.last_name}`, - description: item.email || item.phone || '', - url: `/contacts/${item.id}`, - }); - } - }) - .catch(() => {}) - ); - } - await Promise.all(tasks); - return results; + const { search } = await import('@/api/search'); + const response = await search(query, entityTypes, 20); + return response.results || []; }, enabled: query.trim().length > 0, staleTime: 30 * 1000, diff --git a/frontend/src/api/search.ts b/frontend/src/api/search.ts new file mode 100644 index 0000000..de8f8de --- /dev/null +++ b/frontend/src/api/search.ts @@ -0,0 +1,38 @@ +import { apiClient } from './client'; + +export interface SearchResult { + type: 'company' | 'contact' | 'mail' | 'file' | 'event'; + id: string; + name: string; + description?: string; + url: string; +} + +export interface SearchResponse { + results: SearchResult[]; + facets?: Record>; + summary?: string; + suggestions?: string[]; +} + +export async function search(query: string, entityTypes?: string[], limit = 20): Promise { + const r = await apiClient.post('/api/v1/search', { + query, + entity_types: entityTypes, + limit, + }); + return r.data; +} + +export async function searchSuggest(q: string): Promise { + const r = await apiClient.get('/api/v1/search/suggest', { params: { q } }); + return r.data.suggestions; +} + +export async function searchSimilar(entityType: string, entityId: string): Promise { + const r = await apiClient.post('/api/v1/search/similar', { + entity_type: entityType, + entity_id: entityId, + }); + return r.data.similar; +} diff --git a/frontend/src/components/ai/SuggestionBadge.tsx b/frontend/src/components/ai/SuggestionBadge.tsx new file mode 100644 index 0000000..bdabaa5 --- /dev/null +++ b/frontend/src/components/ai/SuggestionBadge.tsx @@ -0,0 +1,59 @@ +import { useState, useEffect } from 'react'; +import { apiClient } from '@/api/client'; + +interface SuggestionBadgeProps { + onClick: () => void; +} + +export function SuggestionBadge({ onClick }: SuggestionBadgeProps) { + const [count, setCount] = useState(0); + const [pulsing, setPulsing] = useState(false); + + useEffect(() => { + // Initial count + apiClient.get('/api/v1/ai-proactive/suggestions').then(r => { + setCount(r.data.items.length); + }).catch(() => {}); + + // SSE for real-time updates + const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream'); + eventSource.onmessage = () => { + setCount(prev => prev + 1); + setPulsing(true); + setTimeout(() => setPulsing(false), 2000); + }; + + return () => eventSource.close(); + }, []); + + if (count === 0) { + return ( + + ); + } + + return ( + + ); +} diff --git a/frontend/src/components/ai/SuggestionCard.tsx b/frontend/src/components/ai/SuggestionCard.tsx new file mode 100644 index 0000000..97501b5 --- /dev/null +++ b/frontend/src/components/ai/SuggestionCard.tsx @@ -0,0 +1,94 @@ +import { useState } from 'react'; +import type { Suggestion } from '@/api/aiProactive'; + +const typeConfig = { + info: { icon: '💡', color: 'blue', bg: 'bg-blue-50', border: 'border-blue-200', text: 'text-blue-700', bar: 'bg-blue-500' }, + warning: { icon: '⚠️', color: 'orange', bg: 'bg-orange-50', border: 'border-orange-200', text: 'text-orange-700', bar: 'bg-orange-500' }, + action: { icon: '✅', color: 'green', bg: 'bg-green-50', border: 'border-green-200', text: 'text-green-700', bar: 'bg-green-500' }, + insight: { icon: '🔍', color: 'purple', bg: 'bg-purple-50', border: 'border-purple-200', text: 'text-purple-700', bar: 'bg-purple-500' }, +}; + +interface SuggestionCardProps { + suggestion: Suggestion; + onDismiss: (id: string) => void; + onAct: (id: string, actionIndex: number) => void; +} + +export function SuggestionCard({ suggestion, onDismiss, onAct }: SuggestionCardProps) { + const [expanded, setExpanded] = useState(false); + const config = typeConfig[suggestion.suggestion_type] || typeConfig.info; + const confidencePercent = Math.round(suggestion.confidence * 100); + + return ( +
+ {/* Header */} +
+
+ {config.icon} +

{suggestion.title}

+
+ +
+ + {/* Content */} +

{suggestion.content}

+ + {/* Confidence Bar */} +
+
+
+
+ {confidencePercent}% +
+ + {/* Actions */} + {suggestion.actions && suggestion.actions.length > 0 && ( +
+ {suggestion.actions.slice(0, expanded ? undefined : 2).map((action, idx) => ( + + ))} + {suggestion.actions.length > 2 && ( + + )} +
+ )} + + {/* Acted upon badge */} + {suggestion.is_acted_upon && ( +
+ ✓ Ausgeführt +
+ )} +
+ ); +} diff --git a/frontend/src/components/ai/SuggestionSidebar.tsx b/frontend/src/components/ai/SuggestionSidebar.tsx new file mode 100644 index 0000000..49779b6 --- /dev/null +++ b/frontend/src/components/ai/SuggestionSidebar.tsx @@ -0,0 +1,109 @@ +import { useState } from 'react'; +import { useSuggestions } from '@/api/aiProactive'; +import { SuggestionCard } from '@/components/ai/SuggestionCard'; + +interface SuggestionSidebarProps { + isOpen: boolean; + onClose: () => void; +} + +export function SuggestionSidebar({ isOpen, onClose }: SuggestionSidebarProps) { + const { suggestions, connected, dismiss, act } = useSuggestions(); + const [filter, setFilter] = useState('all'); + + const filteredSuggestions = filter === 'all' + ? suggestions + : suggestions.filter(s => s.suggestion_type === filter); + + const filterOptions = [ + { value: 'all', label: 'Alle' }, + { value: 'info', label: 'Info' }, + { value: 'warning', label: 'Warnung' }, + { value: 'action', label: 'Aktion' }, + { value: 'insight', label: 'Erkenntnis' }, + ]; + + return ( + <> + {/* Overlay */} + {isOpen && ( +
+ )} + + {/* Sidebar */} + + + ); +} diff --git a/frontend/src/components/layout/AppShell.tsx b/frontend/src/components/layout/AppShell.tsx index c46a995..3521c1e 100644 --- a/frontend/src/components/layout/AppShell.tsx +++ b/frontend/src/components/layout/AppShell.tsx @@ -4,11 +4,17 @@ import { Sidebar } from './Sidebar'; import { TopBar } from './TopBar'; import { PluginToolbar } from './PluginToolbar'; import { AISidebar } from './AISidebar'; +import { SuggestionSidebar } from '@/components/ai/SuggestionSidebar'; import { ToastContainer } from '@/components/ui/Toast'; import { useUIStore } from '@/store/uiStore'; +import { useAIContext } from '@/hooks/useAIContext'; export function AppShell() { const location = useLocation(); + const { suggestionSidebarOpen, toggleSuggestionSidebar } = useUIStore(); + + // Track context for AI Proactive suggestions + useAIContext(); // Hide AI sidebar on the AI Assistant page itself const showAISidebar = !location.pathname.startsWith('/ai-assistant'); @@ -33,6 +39,11 @@ export function AppShell() {
{/* AI Sidebar — full height, right of TopBar and Toolbar */} {showAISidebar && } + {/* Suggestion Sidebar — overlays from right */} + toggleSuggestionSidebar()} + />
); diff --git a/frontend/src/components/layout/TopBar.tsx b/frontend/src/components/layout/TopBar.tsx index a971579..fa9fb13 100644 --- a/frontend/src/components/layout/TopBar.tsx +++ b/frontend/src/components/layout/TopBar.tsx @@ -9,12 +9,13 @@ import { useLogout } from '@/api/hooks'; import { Avatar } from '@/components/ui/Avatar'; import { Badge } from '@/components/ui/Badge'; import { SearchDropdown } from '@/components/shared/SearchDropdown'; +import { SuggestionBadge } from '@/components/ai/SuggestionBadge'; export function TopBar() { const { t } = useTranslation(); const navigate = useNavigate(); const { user } = useAuthStore(); - const { toggleSidebar, toggleAISidebar, aiSidebarCollapsed } = useUIStore(); + const { toggleSidebar, toggleAISidebar, toggleSuggestionSidebar, aiSidebarCollapsed } = useUIStore(); const { currentTenant, availableTenants, switchTenant, isSwitching } = useTenant(); const logoutMutation = useLogout(); @@ -141,6 +142,9 @@ export function TopBar() {
+ {/* AI Suggestions Badge */} + + {/* Notifications */}
+
+ + {/* Categories */} +
+

Kategorien

+

Für welche Bereiche sollen Vorschläge generiert werden?

+
+ {Object.entries(categoryLabels).map(([key, label]) => { + const checked = settings.suggestion_categories?.includes(key); + return ( + + ); + })} +
+
+ + {/* Confidence Threshold */} +
+
+

Confidence Schwellwert

+ + {Math.round((settings.confidence_threshold || 0.5) * 100)}% + +
+

+ Nur Vorschläge mit höherer Confidence anzeigen +

+ update({ confidence_threshold: parseFloat(e.target.value) })} + className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500" + /> +
+ Alle + Sehr sicher +
+
+ + {/* Rate Limit */} +
+
+

Rate Limit

+ + {settings.rate_limit_seconds || 10}s + +
+

+ Mindestabstand zwischen Vorschlägen +

+ update({ rate_limit_seconds: parseInt(e.target.value) })} + className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500" + /> +
+ 5s + 60s +
+
+ + {/* Model Selection */} +
+

KI Modell (Ollama Cloud)

+

Wähle das Modell für Vorschläge. DeepSeek V4 ist empfohlen für beste Performance/Kosten.

+ +
+
+ ); +} diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index d858cf1..210eee2 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -17,6 +17,7 @@ export function SettingsPage() { { to: '/settings/sequences', label: t('sequences.title'), icon: '\ud83d\udd22' }, { to: '/settings/mail', label: t('mail.settings'), icon: '\ud83d\udce7' }, { to: '/settings/ai', label: t('nav.aiAssistant'), icon: '\ud83e\udde0' }, + { to: '/settings/ai-proactive', label: 'Proaktive KI', icon: '\ud83e\udd16' }, { to: '/settings/notifications', label: t('settings.notifications'), icon: '\ud83d\udd14' }, ]; diff --git a/frontend/src/routes/index.tsx b/frontend/src/routes/index.tsx index 624d227..f115b9e 100644 --- a/frontend/src/routes/index.tsx +++ b/frontend/src/routes/index.tsx @@ -33,6 +33,7 @@ import { MailSettingsPage } from '@/pages/MailSettings'; import { SettingsNotificationsPage } from '@/pages/SettingsNotifications'; import { AIAssistantPage } from '@/pages/AIAssistant'; import { AISettingsPage } from '@/pages/AISettings'; +import { ProactiveAISettings } from '@/pages/ProactiveAISettings'; const router = createBrowserRouter([ { @@ -89,6 +90,7 @@ const router = createBrowserRouter([ { path: 'mail', element: }, { path: 'notifications', element: }, { path: 'ai', element: }, + { path: 'ai-proactive', element: }, ], }, ], diff --git a/frontend/src/store/uiStore.ts b/frontend/src/store/uiStore.ts index 5b73245..b364e79 100644 --- a/frontend/src/store/uiStore.ts +++ b/frontend/src/store/uiStore.ts @@ -14,12 +14,14 @@ export interface UIState { theme: Theme; locale: Locale; sidebarOpen: boolean; + suggestionSidebarOpen: boolean; aiSidebarCollapsed: boolean; toasts: Toast[]; setTheme: (theme: Theme) => void; setLocale: (locale: Locale) => void; toggleSidebar: () => void; setSidebarOpen: (open: boolean) => void; + toggleSuggestionSidebar: () => void; toggleAISidebar: () => void; setAISidebarCollapsed: (collapsed: boolean) => void; addToast: (toast: Omit) => void; @@ -33,6 +35,7 @@ export const useUIStore = create((set) => ({ theme: (localStorage.getItem('leocrm_theme') as Theme) || 'light', locale: (localStorage.getItem('leocrm_lang') as Locale) || 'de', sidebarOpen: true, + suggestionSidebarOpen: false, aiSidebarCollapsed: true, toasts: [], setTheme: (theme) => { @@ -45,6 +48,7 @@ export const useUIStore = create((set) => ({ }, toggleSidebar: () => set((s) => ({ sidebarOpen: !s.sidebarOpen })), setSidebarOpen: (open) => set({ sidebarOpen: open }), + toggleSuggestionSidebar: () => set((s) => ({ suggestionSidebarOpen: !s.suggestionSidebarOpen })), toggleAISidebar: () => set((s) => ({ aiSidebarCollapsed: !s.aiSidebarCollapsed })), setAISidebarCollapsed: (collapsed) => set({ aiSidebarCollapsed: collapsed }), addToast: (toast) => { diff --git a/requirements.txt b/requirements.txt index 7358ede..b4511ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,3 +46,9 @@ openpyxl>=3.1 prometheus-client>=0.20 structlog>=24.0 litellm + +# AI / Search +pgvector>=0.3.0 +PyMuPDF>=1.24.0 +python-docx>=1.1.0 +python-pptx>=0.6.23 diff --git a/tests/test_ai_proactive.py b/tests/test_ai_proactive.py new file mode 100644 index 0000000..a037918 --- /dev/null +++ b/tests/test_ai_proactive.py @@ -0,0 +1,440 @@ +"""Tests for the AI Proactive plugin — Context API, Suggestions API, +Dismiss API, Settings API, Stats API, and Proactive Engine unit tests. + +Uses pytest + pytest-asyncio, httpx.AsyncClient, login_client, +seed_tenant_and_users, ORIGIN_HEADER from conftest.py. +Mocks litellm.acompletion and litellm.aembedding to avoid real API calls. +""" + +from __future__ import annotations + +import asyncio +import json +import uuid +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +import pytest_asyncio +from httpx import ASGITransport, AsyncClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +# Import models at module level so Base.metadata.create_all includes their tables +from app.plugins.builtins.ai_proactive.models import ( # noqa: F401 + ContextLog, + ProactiveSettings, + ProactiveSuggestion, +) +from app.plugins.builtins.ai_assistant.models import ( # noqa: F401 + AIProvider, AIModel, AIPreset, AIAgent, AIChatSession, AIChatMessage, + AIChatFolder, AIChatAttachment, +) +from app.plugins.builtins.unified_search.models import ( # noqa: F401 + SearchProviderRegistry as SearchProviderRegistryModel, + SearchIndexLog, +) +from app.plugins.builtins.ai_proactive import AIProactivePlugin +from app.plugins.builtins.ai_assistant import AIAssistantPlugin +from app.plugins.builtins.unified_search import UnifiedSearchPlugin +from app.core.db import close_engine, reset_engine_for_testing +from app.core.service_container import get_container +from app.main import create_app +from app.plugins.registry import reset_registry_for_testing +from app.services.plugin_service import reset_plugin_service_for_testing +from tests.conftest import ORIGIN_HEADER, _get_sync_engine, login_client, seed_tenant_and_users + +# ─── AI Proactive Fixtures ─── + + +@pytest_asyncio.fixture(autouse=True) +async def clean_ai_proactive_tables(): + """Truncate ai_proactive tables before each test (not in conftest TRUNCATE list).""" + sync_eng = _get_sync_engine() + with sync_eng.connect() as conn: + conn.execute( + text( + "TRUNCATE TABLE ai_proactive_suggestions, " + "ai_proactive_context_log, ai_proactive_settings CASCADE;" + ) + ) + conn.commit() + sync_eng.dispose() + yield + + +@pytest_asyncio.fixture +async def ai_proactive_app(engine: AsyncEngine, redis_client): + """FastAPI app with AI Proactive plugin registered, installed, and activated.""" + reset_engine_for_testing(engine) + app = create_app() + registry = reset_registry_for_testing() + registry.initialize(engine, app) + container = get_container() + await container.initialize() + registry.register_plugin(AIAssistantPlugin()) + registry.register_plugin(UnifiedSearchPlugin()) + registry.register_plugin(AIProactivePlugin()) + reset_plugin_service_for_testing(registry) + + sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession) + async with sf() as session: + # Install dependencies first, then ai_proactive + await registry.install(session, "ai_assistant") + await registry.activate(session, "ai_assistant") + await registry.install(session, "unified_search") + await registry.activate(session, "unified_search") + await registry.install(session, "ai_proactive") + await registry.activate(session, "ai_proactive") + await session.commit() + + yield app + await close_engine() + + +@pytest_asyncio.fixture +async def ai_proactive_client(ai_proactive_app) -> AsyncClient: + """HTTP test client with ai_proactive plugin active.""" + transport = ASGITransport(app=ai_proactive_app) + async with AsyncClient(transport=transport, base_url="http://test") as c: + yield c + + +@pytest_asyncio.fixture +async def ai_proactive_authed_client( + ai_proactive_client: AsyncClient, db_session: AsyncSession +) -> tuple[AsyncClient, dict]: + """Authenticated admin client with seeded data and ai_proactive plugin active.""" + seed = await seed_tenant_and_users(db_session) + # Grant is_system_admin so require_permission passes for ai_proactive:* permissions + from sqlalchemy import update + from app.models.user import User + await db_session.execute( + update(User).where(User.id == seed["admin_a"].id).values(is_system_admin=True) + ) + await db_session.commit() + # Login and capture CSRF token + login_resp = await ai_proactive_client.post( + "/api/v1/auth/login", + json={"email": "admin@tenanta.com", "password": "TestPass123!"}, + headers=ORIGIN_HEADER, + ) + assert login_resp.status_code == 200, f"Login failed: {login_resp.text}" + csrf_token = login_resp.json().get("csrf_token", "") + # Store CSRF token on client for use in requests + ai_proactive_client.headers["X-CSRF-Token"] = csrf_token + return ai_proactive_client, seed + + +@pytest.fixture(autouse=True) +async def mock_litellm(): + """Mock litellm.acompletion and litellm.aembedding for all tests.""" + mock_resp = MagicMock() + mock_resp.choices = [MagicMock()] + mock_resp.choices[0].message.content = json.dumps({ + "suggestion_type": "info", + "title": "Test Suggestion", + "content": "This is a test suggestion", + "confidence": 0.8, + "actions": [ + { + "method": "GET", + "path": "/api/v1/contacts", + "body": {}, + "description": "View contacts", + } + ], + }) + + mock_emb_resp = MagicMock() + mock_emb_resp.data = [{"embedding": [0.1] * 768}] + + with ( + patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_resp), + patch("litellm.aembedding", new_callable=AsyncMock, return_value=mock_emb_resp), + ): + yield + + +# ─── 1. Context API (POST /api/v1/ai-proactive/context) ─── + + +@pytest.mark.asyncio +async def test_context_report_accepted( + ai_proactive_authed_client: tuple[AsyncClient, dict], +): + """Context-Report wird akzeptiert (200).""" + client, _ = ai_proactive_authed_client + resp = await client.post( + "/api/v1/ai-proactive/context", + json={ + "page": "/contacts", + "entity_type": "contact", + "entity_id": str(uuid.uuid4()), + }, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + assert resp.json()["status"] == "ok" + + +@pytest.mark.asyncio +async def test_context_unauthorized(ai_proactive_client: AsyncClient): + """Ohne Auth → 403 (CSRF middleware blocks before auth check on POST).""" + resp = await ai_proactive_client.post( + "/api/v1/ai-proactive/context", + json={"page": "/contacts"}, + headers=ORIGIN_HEADER, + ) + # Without a session cookie, CSRF middleware returns 403 before auth is checked + assert resp.status_code in (401, 403) + + +# ─── 2. Suggestions API (GET /api/v1/ai-proactive/suggestions) ─── + + +@pytest.mark.asyncio +async def test_get_suggestions( + ai_proactive_authed_client: tuple[AsyncClient, dict], + db_session: AsyncSession, +): + """Gibt Suggestions für User zurück.""" + client, seed = ai_proactive_authed_client + suggestion = ProactiveSuggestion( + tenant_id=seed["tenant_a"].id, + user_id=seed["admin_a"].id, + entity_type="contact", + entity_id=uuid.uuid4(), + suggestion_type="info", + title="Test Suggestion", + content="Test content", + confidence=0.8, + actions=[], + context_snapshot={}, + ) + db_session.add(suggestion) + await db_session.commit() + + resp = await client.get( + "/api/v1/ai-proactive/suggestions", + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert "items" in data + assert data["total"] >= 1 + titles = [item["title"] for item in data["items"]] + assert "Test Suggestion" in titles + + +@pytest.mark.asyncio +async def test_suggestions_tenant_isolation( + ai_proactive_authed_client: tuple[AsyncClient, dict], + db_session: AsyncSession, +): + """Nur eigene Tenant — Tenant B suggestions are not visible to Tenant A.""" + client, seed = ai_proactive_authed_client + + # Suggestion for tenant B user + suggestion_b = ProactiveSuggestion( + tenant_id=seed["tenant_b"].id, + user_id=seed["admin_b"].id, + entity_type="contact", + entity_id=uuid.uuid4(), + suggestion_type="info", + title="Tenant B Secret", + content="Should not be visible to A", + confidence=0.8, + actions=[], + context_snapshot={}, + ) + # Suggestion for tenant A user + suggestion_a = ProactiveSuggestion( + tenant_id=seed["tenant_a"].id, + user_id=seed["admin_a"].id, + entity_type="contact", + entity_id=uuid.uuid4(), + suggestion_type="info", + title="Tenant A Visible", + content="Should be visible", + confidence=0.8, + actions=[], + context_snapshot={}, + ) + db_session.add_all([suggestion_b, suggestion_a]) + await db_session.commit() + + resp = await client.get( + "/api/v1/ai-proactive/suggestions", + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + titles = [item["title"] for item in data["items"]] + assert "Tenant A Visible" in titles + assert "Tenant B Secret" not in titles + + +# ─── 3. Dismiss API (POST /api/v1/ai-proactive/suggestions/{id}/dismiss) ─── + + +@pytest.mark.asyncio +async def test_dismiss_suggestion( + ai_proactive_authed_client: tuple[AsyncClient, dict], + db_session: AsyncSession, +): + """Suggestion wird dismissed.""" + client, seed = ai_proactive_authed_client + suggestion = ProactiveSuggestion( + tenant_id=seed["tenant_a"].id, + user_id=seed["admin_a"].id, + entity_type="contact", + entity_id=uuid.uuid4(), + suggestion_type="info", + title="Dismiss Me", + content="Please dismiss this", + confidence=0.8, + actions=[], + context_snapshot={}, + ) + db_session.add(suggestion) + await db_session.commit() + + resp = await client.post( + f"/api/v1/ai-proactive/suggestions/{suggestion.id}/dismiss", + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + assert resp.json()["status"] == "ok" + + # Verify it no longer appears in active suggestions + resp2 = await client.get( + "/api/v1/ai-proactive/suggestions", + headers=ORIGIN_HEADER, + ) + data = resp2.json() + ids = [item["id"] for item in data["items"]] + assert str(suggestion.id) not in ids + + +# ─── 4. Settings API (GET/PUT /api/v1/ai-proactive/settings) ─── + + +@pytest.mark.asyncio +async def test_get_settings( + ai_proactive_authed_client: tuple[AsyncClient, dict], +): + """Settings werden zurückgegeben.""" + client, _ = ai_proactive_authed_client + resp = await client.get( + "/api/v1/ai-proactive/settings", + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert "enabled" in data + assert "suggestion_categories" in data + assert "confidence_threshold" in data + assert "rate_limit_seconds" in data + assert "model" in data + assert "available_models" in data + + +@pytest.mark.asyncio +async def test_update_settings( + ai_proactive_authed_client: tuple[AsyncClient, dict], +): + """Settings können aktualisiert werden.""" + client, _ = ai_proactive_authed_client + resp = await client.put( + "/api/v1/ai-proactive/settings", + json={"enabled": False, "confidence_threshold": 0.7}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["enabled"] is False + assert data["confidence_threshold"] == 0.7 + + +@pytest.mark.asyncio +async def test_update_model( + ai_proactive_authed_client: tuple[AsyncClient, dict], +): + """Model kann geändert werden.""" + client, _ = ai_proactive_authed_client + resp = await client.put( + "/api/v1/ai-proactive/settings", + json={"model": "ollama/llama3.2"}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["model"] == "ollama/llama3.2" + + +# ─── 5. Stats API (GET /api/v1/ai-proactive/stats) ─── + + +@pytest.mark.asyncio +async def test_get_stats( + ai_proactive_authed_client: tuple[AsyncClient, dict], +): + """Stats werden zurückgegeben.""" + client, _ = ai_proactive_authed_client + resp = await client.get( + "/api/v1/ai-proactive/stats", + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert "total_suggestions" in data + assert "dismissed" in data + assert "acted_upon" in data + assert "active" in data + assert "dismiss_rate" in data + assert "act_rate" in data + + +# ─── 6. Proactive Engine (Unit Tests with mocks) ─── + + +@pytest.mark.asyncio +async def test_sse_push(): + """Suggestion wird in Queue gepusht.""" + from app.plugins.builtins.ai_proactive.services import get_sse_queue, push_suggestion + + user_id = f"test-sse-{uuid.uuid4()}" + queue = get_sse_queue(user_id) + + # Ensure queue is empty + while not queue.empty(): + await queue.get() + + suggestion = {"id": "test-1", "title": "Test SSE Push"} + await push_suggestion(user_id, suggestion) + + item = await asyncio.wait_for(queue.get(), timeout=1.0) + assert item["id"] == "test-1" + assert item["title"] == "Test SSE Push" + + +@pytest.mark.asyncio +async def test_rate_limiting(redis_client): + """Rate-Limit funktioniert.""" + from app.plugins.builtins.ai_proactive.services import is_rate_limited + + tenant_id = uuid.uuid4() + user_id = uuid.uuid4() + rate_limit_seconds = 10 + + with patch( + "app.plugins.builtins.ai_proactive.services.get_cache", + return_value=redis_client, + ): + # First call should not be rate limited + limited = await is_rate_limited(tenant_id, user_id, rate_limit_seconds) + assert limited is False + + # Second call should be rate limited + limited = await is_rate_limited(tenant_id, user_id, rate_limit_seconds) + assert limited is True diff --git a/tests/test_unified_search.py b/tests/test_unified_search.py new file mode 100644 index 0000000..723782c --- /dev/null +++ b/tests/test_unified_search.py @@ -0,0 +1,552 @@ +"""Tests for the Unified Search plugin — covers ACs 1-12 and unit tests. + +Tests hybrid search infrastructure: API endpoints, RRF fusion, autocomplete, +text extraction, and provider registry. +""" + +from __future__ import annotations + +import json +import uuid +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +import pytest_asyncio +from httpx import ASGITransport, AsyncClient +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker + +from app.core.db import close_engine, reset_engine_for_testing +from app.core.service_container import get_container +from app.main import create_app +from app.plugins.builtins.unified_search import UnifiedSearchPlugin +from app.plugins.builtins.unified_search.provider_registry import ( + SearchProviderRegistry, + get_search_registry, +) +from app.plugins.builtins.unified_search.search_engine import rrf_fusion +from app.plugins.builtins.unified_search.text_extraction import extract_text_from_file +from app.plugins.registry import reset_registry_for_testing +from app.services.plugin_service import reset_plugin_service_for_testing +from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users + +# ─── Unified Search Fixtures ─── + + +@pytest_asyncio.fixture +async def search_app(engine: AsyncEngine, redis_client): + """FastAPI app with UnifiedSearch plugin registered, installed, and activated.""" + reset_engine_for_testing(engine) + app = create_app() + registry = reset_registry_for_testing() + registry.initialize(engine, app) + container = get_container() + await container.initialize() + registry.register_plugin(UnifiedSearchPlugin()) + reset_plugin_service_for_testing(registry) + + sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession) + async with sf() as session: + await registry.install(session, "unified_search") + await registry.activate(session, "unified_search") + await session.commit() + + yield app + await close_engine() + + +@pytest_asyncio.fixture +async def search_client(search_app) -> AsyncClient: + """HTTP test client with unified search plugin active.""" + transport = ASGITransport(app=search_app) + async with AsyncClient(transport=transport, base_url="http://test") as c: + yield c + + +@pytest_asyncio.fixture +async def search_authed_client( + search_client: AsyncClient, db_session: AsyncSession +) -> tuple[AsyncClient, dict]: + """Authenticated admin client with seeded data and unified search plugin active.""" + seed = await seed_tenant_and_users(db_session) + await login_client(search_client, "admin@tenanta.com") + return search_client, seed + + +@pytest.fixture(autouse=True) +async def mock_external_calls(): + """Mock all external API calls (LiteLLM, job queue) for all tests.""" + mock_resp = MagicMock() + mock_resp.choices = [MagicMock()] + mock_resp.choices[0].message.content = json.dumps({ + "normalized_query": "test", + "entities": {}, + "intent": "search", + "semantic_terms": [], + "suggested_filters": {}, + }) + + mock_emb_resp = MagicMock() + mock_emb_resp.data = [{"embedding": [0.1] * 768}] + + with ( + patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_resp), + patch("litellm.aembedding", new_callable=AsyncMock, return_value=mock_emb_resp), + patch("app.core.jobs.enqueue_job", new_callable=AsyncMock, return_value="job-123"), + ): + yield + + +# ─── AC1: POST /api/v1/search → 200 + results array ─── + + +@pytest.mark.asyncio +@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock) +async def test_ac1_search_returns_results( + mock_hybrid_search, + search_authed_client: tuple[AsyncClient, dict], +): + """AC1: POST /api/v1/search with query returns 200 + results array.""" + client, _ = search_authed_client + mock_hybrid_search.return_value = [ + { + "entity_type": "company", + "entity_id": str(uuid.uuid4()), + "title": "Company Alpha", + "snippet": "IT company", + "score": 0.95, + "data": {"industry": "IT"}, + } + ] + + resp = await client.post( + "/api/v1/search", + json={"query": "Company"}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert "results" in data + assert isinstance(data["results"], list) + assert len(data["results"]) > 0 + assert data["results"][0]["entity_type"] == "company" + assert data["results"][0]["title"] == "Company Alpha" + assert "score" in data["results"][0] + assert "query" in data + assert "normalized_query" in data + assert "summary" in data + + +# ─── AC2: entity_types filter ─── + + +@pytest.mark.asyncio +@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock) +async def test_ac2_search_entity_types_filter( + mock_hybrid_search, + search_authed_client: tuple[AsyncClient, dict], +): + """AC2: Search with entity_types filters correctly.""" + client, _ = search_authed_client + mock_hybrid_search.return_value = [] + + resp = await client.post( + "/api/v1/search", + json={"query": "test", "entity_types": ["company"]}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + # Verify hybrid_search was called with entity_types=["company"] + call_kwargs = mock_hybrid_search.call_args + assert call_kwargs.kwargs.get("entity_types") == ["company"] + + +# ─── AC3: No auth → 401 ─── + + +@pytest.mark.asyncio +async def test_ac3_search_no_auth_401(search_client: AsyncClient): + """AC3: Search without auth returns 401.""" + resp = await search_client.post( + "/api/v1/search", + json={"query": "test"}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 401 + + +# ─── AC4: Tenant isolation ─── + + +@pytest.mark.asyncio +@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock) +async def test_ac4_search_tenant_isolation( + mock_hybrid_search, + search_authed_client: tuple[AsyncClient, dict], +): + """AC4: Search only returns results from the user's tenant.""" + client, seed = search_authed_client + mock_hybrid_search.return_value = [] + + resp = await client.post( + "/api/v1/search", + json={"query": "test"}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + + # Verify hybrid_search was called with tenant A's ID + call_args = mock_hybrid_search.call_args + tenant_id_arg = call_args.kwargs.get("tenant_id") + assert tenant_id_arg == seed["tenant_a"].id + + +# ─── AC5: Empty query → 422 ─── + + +@pytest.mark.asyncio +async def test_ac5_search_empty_query_422(search_authed_client: tuple[AsyncClient, dict]): + """AC5: Empty query returns 422 (min_length=1).""" + client, _ = search_authed_client + resp = await client.post( + "/api/v1/search", + json={"query": ""}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 422 + + +# ─── AC6: GET /api/v1/search/suggest → suggestions ─── + + +@pytest.mark.asyncio +async def test_ac6_suggest_returns_suggestions( + search_authed_client: tuple[AsyncClient, dict], +): + """AC6: GET /api/v1/search/suggest returns 200 + suggestions list.""" + client, _ = search_authed_client + resp = await client.get( + "/api/v1/search/suggest", + params={"q": "comp"}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert "suggestions" in data + assert isinstance(data["suggestions"], list) + + +# ─── AC7: Empty suggest query → 422 ─── + + +@pytest.mark.asyncio +async def test_ac7_suggest_empty_query_422( + search_authed_client: tuple[AsyncClient, dict], +): + """AC7: GET /api/v1/search/suggest with empty query returns 422 (min_length=1).""" + client, _ = search_authed_client + resp = await client.get( + "/api/v1/search/suggest", + params={"q": ""}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 422 + + +# ─── AC8: POST /api/v1/search/similar → similar results ─── + + +@pytest.mark.asyncio +async def test_ac8_similar_returns_results( + search_authed_client: tuple[AsyncClient, dict], +): + """AC8: POST /api/v1/search/similar returns 200 + similar dict.""" + client, seed = search_authed_client + resp = await client.post( + "/api/v1/search/similar", + json={ + "entity_type": "company", + "entity_id": str(seed["company_a"].id), + "limit": 5, + }, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert "similar" in data + assert isinstance(data["similar"], dict) + + +# ─── AC9: Similar tenant isolation ─── + + +@pytest.mark.asyncio +async def test_ac9_similar_tenant_isolation( + search_authed_client: tuple[AsyncClient, dict], +): + """AC9: Similar search respects tenant isolation.""" + client, seed = search_authed_client + # Use company from tenant B + resp = await client.post( + "/api/v1/search/similar", + json={ + "entity_type": "company", + "entity_id": str(seed["company_b"].id), + "limit": 5, + }, + headers=ORIGIN_HEADER, + ) + # Should return 200 but with empty similar (tenant B company not accessible from tenant A) + assert resp.status_code == 200 + data = resp.json() + assert isinstance(data["similar"], dict) + # Company B should not be found in tenant A's context + for etype, items in data["similar"].items(): + for item in items: + assert item.get("entity_id") != str(seed["company_b"].id) + + +# ─── AC10: GET /api/v1/search/providers → provider list ─── + + +@pytest.mark.asyncio +async def test_ac10_list_providers( + search_authed_client: tuple[AsyncClient, dict], +): + """AC10: GET /api/v1/search/providers returns 200 + active provider list.""" + client, _ = search_authed_client + resp = await client.get( + "/api/v1/search/providers", + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert isinstance(data, list) + # After activation, providers should be registered + if len(data) > 0: + assert "entity_type" in data[0] + assert "plugin_name" in data[0] + assert "is_active" in data[0] + assert data[0]["is_active"] is True + + +# ─── AC11: POST /api/v1/search/reindex as admin → 200 ─── + + +@pytest.mark.asyncio +async def test_ac11_reindex_admin_200( + search_authed_client: tuple[AsyncClient, dict], +): + """AC11: Admin can trigger reindex (200).""" + client, _ = search_authed_client + resp = await client.post( + "/api/v1/search/reindex", + json={"entity_types": ["company"]}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["status"] == "ok" + assert "entity_types" in data + + +# ─── AC12: POST /api/v1/search/reindex as viewer → 403 ─── + + +@pytest.mark.asyncio +async def test_ac12_reindex_viewer_403( + search_client: AsyncClient, db_session: AsyncSession +): + """AC12: Non-admin (viewer) gets 403 on reindex.""" + await seed_tenant_and_users(db_session) + await login_client(search_client, "viewer@tenanta.com") + + resp = await search_client.post( + "/api/v1/search/reindex", + json={"entity_types": ["company"]}, + headers=ORIGIN_HEADER, + ) + assert resp.status_code == 403 + + +# ─── Unit Tests: RRF Fusion ─── + + +def test_rrf_fusion_combines_fts_and_vector_scores(): + """RRF fusion combines FTS + Vector scores correctly.""" + fts_results = [ + {"id": "1", "title": "Result A", "score": 0.9}, + {"id": "2", "title": "Result B", "score": 0.8}, + ] + vec_results = [ + {"id": "2", "title": "Result B", "score": 0.95}, + {"id": "3", "title": "Result C", "score": 0.7}, + ] + + fused = rrf_fusion(fts_results, vec_results, "company") + + # All 3 unique IDs should be present + ids = [str(r.get("id", "")) for r in fused] + assert "1" in ids + assert "2" in ids + assert "3" in ids + + # Results should be sorted by fused score descending + assert fused[0]["_score"] >= fused[-1]["_score"] + + # ID 2 appears in both FTS and vector, so should have highest score + assert str(fused[0].get("id", "")) == "2" + + +def test_rrf_fusion_empty_inputs(): + """RRF fusion with empty inputs returns empty list.""" + fused = rrf_fusion([], [], "company") + assert fused == [] + + +def test_rrf_fusion_fts_only(): + """RRF fusion with only FTS results returns them sorted by score.""" + fts_results = [ + {"id": "1", "title": "A"}, + {"id": "2", "title": "B"}, + ] + fused = rrf_fusion(fts_results, [], "company") + assert len(fused) == 2 + # First result (rank 0) should have higher score + assert fused[0]["_score"] > fused[1]["_score"] + + +# ─── Unit Tests: Text Extraction ─── + + +@pytest.mark.asyncio +async def test_extract_text_from_text_file(tmp_path): + """Extract text from a plain text file.""" + test_file = tmp_path / "test.txt" + test_file.write_text("Hello World\nThis is a test.", encoding="utf-8") + + result = await extract_text_from_file(str(test_file), "text/plain") + assert "Hello World" in result + assert "test" in result + + +@pytest.mark.asyncio +async def test_extract_text_from_pdf_mocked(tmp_path): + """Extract text from PDF with mocked PyMuPDF.""" + test_file = tmp_path / "test.pdf" + test_file.write_bytes(b"%PDF-1.4 fake") + + mock_page = MagicMock() + mock_page.get_text.return_value = "PDF content text" + mock_doc = MagicMock() + mock_doc.__iter__ = MagicMock(return_value=iter([mock_page])) + mock_doc.close = MagicMock() + + with patch("fitz.open", return_value=mock_doc): + result = await extract_text_from_file(str(test_file), "application/pdf") + assert "PDF content text" in result + + +@pytest.mark.asyncio +async def test_extract_text_unsupported_mime(): + """Unsupported MIME type returns empty string.""" + result = await extract_text_from_file("/tmp/nonexistent", "application/octet-stream") + assert result == "" + + +@pytest.mark.asyncio +async def test_extract_text_image_returns_empty(tmp_path): + """Image MIME type returns empty string (OCR not implemented).""" + test_file = tmp_path / "image.png" + test_file.write_bytes(b"\x89PNG\r\n\x1a\n") + result = await extract_text_from_file(str(test_file), "image/png") + assert result == "" + + +# ─── Unit Tests: Provider Registry ─── + + +def test_provider_register_and_unregister(): + """Provider can be registered and unregistered.""" + registry = SearchProviderRegistry() + + # Create a mock provider + mock_provider = MagicMock() + mock_provider.entity_type = "test_entity" + + # Register + registry.register(mock_provider) + assert registry.get("test_entity") is mock_provider + + # Unregister + registry.unregister("test_entity") + assert registry.get("test_entity") is None + + +def test_provider_get_all(): + """get_all returns all registered providers.""" + registry = SearchProviderRegistry() + + p1 = MagicMock() + p1.entity_type = "contact" + p2 = MagicMock() + p2.entity_type = "company" + + registry.register(p1) + registry.register(p2) + + all_providers = registry.get_all() + assert len(all_providers) == 2 + entity_types = [p.entity_type for p in all_providers] + assert "contact" in entity_types + assert "company" in entity_types + + +def test_provider_clear(): + """clear removes all providers.""" + registry = SearchProviderRegistry() + p = MagicMock() + p.entity_type = "test" + registry.register(p) + assert len(registry.get_all()) == 1 + + registry.clear() + assert len(registry.get_all()) == 0 + + +def test_provider_get_entity_types(): + """get_entity_types returns list of registered entity types.""" + registry = SearchProviderRegistry() + p1 = MagicMock() + p1.entity_type = "contact" + p2 = MagicMock() + p2.entity_type = "company" + registry.register(p1) + registry.register(p2) + + types = registry.get_entity_types() + assert "contact" in types + assert "company" in types + assert len(types) == 2 + + +# ─── Unit Tests: Autocomplete ─── + + +@pytest.mark.asyncio +async def test_autocomplete_empty_query_returns_empty(db_session: AsyncSession): + """Autocomplete with empty query returns empty list.""" + from app.plugins.builtins.unified_search.search_engine import autocomplete + + result = await autocomplete(db_session, "", uuid.uuid4(), 10) + assert result == [] + + +@pytest.mark.asyncio +async def test_autocomplete_returns_titles(db_session: AsyncSession): + """Autocomplete returns titles from FTS prefix search.""" + from app.plugins.builtins.unified_search.search_engine import autocomplete + + # With no providers registered and no data, should return empty list + result = await autocomplete(db_session, "test", uuid.uuid4(), 10) + assert isinstance(result, list) + assert len(result) <= 10