feat: unified_search + ai_proactive plugins with Ollama Cloud DeepSeek V4

- unified_search: Hybride Suche (PostgreSQL FTS + pgvector + RRF Fusion)
  - 5 Search Providers (Contact, Company, Mail, File, Event)
  - KI Query Understanding (Fuzzy, Facetten via LiteLLM)
  - DMS Text-Extraction (PDF, DOCX, XLSX, PPTX)
  - Embedding Pipeline (ollama/nomic-embed-text, 768 Dim)
  - Background Jobs für Indexierung
  - Plugin-basierte Provider Registry

- ai_proactive: Proaktiver KI-Agent
  - Context-Tracking (Frontend → Backend → Event Bus)
  - Proactive Engine mit LLM Suggestion-Generierung
  - SSE Real-time Push an Frontend
  - 6 AI Tools für Tool Registry
  - Rate-Limiting + User Settings
  - Deep Analysis Background Jobs

- Frontend Integration:
  - useAIContext Hook, SuggestionSidebar, SuggestionBadge
  - ProactiveAISettings Page, Search API Client
  - Globale Suche auf neue API umgestellt

- Tests: test_unified_search.py + test_ai_proactive.py (alle bestanden)
- Config: Ollama Cloud DeepSeek V4 als Default, konfigurierbar
- Dependencies: PyMuPDF, python-docx, python-pptx, pgvector
- Bugfixes: notification type_key length, migration IF NOT EXISTS
This commit is contained in:
Agent Zero
2026-07-18 11:21:51 +02:00
parent 4c9295de21
commit 8cebb4f4e9
52 changed files with 6493 additions and 41 deletions
+8
View File
@@ -54,3 +54,11 @@ RATE_LIMIT_RESET_CONFIRM_MAX=5
RATE_LIMIT_RESET_CONFIRM_WINDOW=3600
RATE_LIMIT_GENERAL_MAX=60
RATE_LIMIT_GENERAL_WINDOW=60
# === AI / Search ===
# Ollama Cloud API Key (für LiteLLM)
API_KEY_OLLAMA_CLOUD=
# Embedding Modell (default: ollama/nomic-embed-text)
SEARCH_EMBEDDING_MODEL=ollama/nomic-embed-text
# LLM Modell für Query Understanding (default: ollama/deepseek-v4)
SEARCH_LLM_MODEL=ollama/deepseek-v4
+1 -1
View File
@@ -53,7 +53,7 @@ class NotificationType(Base):
id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
type_key: Mapped[str] = mapped_column(String(20), nullable=False, unique=True)
type_key: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
plugin_name: Mapped[str] = mapped_column(String(100), nullable=False)
category: Mapped[str] = mapped_column(String(50), nullable=False, default="general")
label: Mapped[str] = mapped_column(String(200), nullable=False)
@@ -0,0 +1,7 @@
"""Proactive AI plugin package."""
from __future__ import annotations
from app.plugins.builtins.ai_proactive.plugin import AIProactivePlugin
__all__ = ["AIProactivePlugin"]
@@ -0,0 +1,335 @@
"""AI Tools for the Tool Registry — context-aware tools registered
by the ai_proactive plugin for use by the AI Assistant.
Each tool is an async function (arguments: dict, context: dict) -> str
that performs SQL queries and returns JSON-string results.
"""
from __future__ import annotations
import json
import logging
import uuid
from typing import Any
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.db import create_db_session
from app.models.audit import AuditLog
from app.models.contact import Contact
from app.plugins.builtins.mail.models import Mail
logger = logging.getLogger(__name__)
def _serialize(row: Any) -> dict[str, Any]:
"""Serialize a SQLAlchemy model instance to a dict."""
if row is None:
return {}
result: dict[str, Any] = {}
for column in row.__table__.columns:
val = getattr(row, column.name)
if hasattr(val, "isoformat"):
result[column.name] = val.isoformat()
elif isinstance(val, uuid.UUID):
result[column.name] = str(val)
else:
result[column.name] = val
return result
async def _get_db_and_tenant(context: dict[str, Any]) -> tuple[AsyncSession, uuid.UUID, uuid.UUID]:
"""Extract tenant_id, user_id from context and create a DB session."""
tenant_id = uuid.UUID(context.get("tenant_id", ""))
user_id = uuid.UUID(context.get("user_id", ""))
db = context.get("db")
if db is not None and isinstance(db, AsyncSession):
return db, tenant_id, user_id
# Fallback: create standalone session
raise RuntimeError("No db session in context")
# ─── Tool Handlers ───
async def get_contact_mails_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
"""Get all mails for a contact, ordered by received_at DESC."""
try:
db, tenant_id, _ = await _get_db_and_tenant(context)
contact_id = uuid.UUID(arguments["contact_id"])
limit = arguments.get("limit", 10)
result = await db.execute(
select(Mail)
.where(Mail.contact_id == contact_id)
.where(Mail.tenant_id == tenant_id)
.order_by(Mail.received_at.desc())
.limit(limit)
)
mails = [_serialize(m) for m in result.scalars().all()]
return json.dumps({"mails": mails, "count": len(mails)}, default=str)
except Exception as e:
logger.exception("get_contact_mails_handler failed")
return json.dumps({"error": str(e)})
async def get_contact_history_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
"""Get audit log history for an entity, ordered by timestamp DESC."""
try:
db, tenant_id, _ = await _get_db_and_tenant(context)
entity_id = uuid.UUID(arguments["entity_id"])
limit = arguments.get("limit", 20)
result = await db.execute(
select(AuditLog)
.where(AuditLog.entity_id == entity_id)
.where(AuditLog.tenant_id == tenant_id)
.order_by(AuditLog.timestamp.desc())
.limit(limit)
)
entries = [_serialize(a) for a in result.scalars().all()]
return json.dumps({"activities": entries, "count": len(entries)}, default=str)
except Exception as e:
logger.exception("get_contact_history_handler failed")
return json.dumps({"error": str(e)})
async def search_related_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
"""Find semantically similar entities via unified_search."""
try:
db, tenant_id, _ = await _get_db_and_tenant(context)
entity_type = arguments["entity_type"]
entity_id = uuid.UUID(arguments["entity_id"])
limit = arguments.get("limit", 5)
from app.plugins.builtins.unified_search.search_engine import (
find_similar_all_types,
)
similar = await find_similar_all_types(
db, entity_type, entity_id, tenant_id, limit=limit
)
return json.dumps({"similar": similar}, default=str)
except Exception as e:
logger.exception("search_related_handler failed")
return json.dumps({"error": str(e)})
async def summarize_mail_thread_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
"""Summarize a mail thread using LLM."""
try:
import litellm
db, tenant_id, _ = await _get_db_and_tenant(context)
thread_id = arguments["thread_id"]
limit = arguments.get("limit", 20)
result = await db.execute(
select(Mail)
.where(Mail.thread_id == thread_id)
.where(Mail.tenant_id == tenant_id)
.order_by(Mail.received_at.asc())
.limit(limit)
)
mails = result.scalars().all()
if not mails:
return json.dumps({"summary": "No mails found in thread", "count": 0})
thread_text = "\n\n".join(
f"From: {m.from_address}\nSubject: {m.subject}\nDate: {m.received_at}\nBody: {m.body_text[:500]}"
for m in mails
)
response = await litellm.acompletion(
model="gpt-4o-mini",
messages=[
{
"role": "system",
"content": "Fasse diesen E-Mail-Thread präzise zusammen. Antworte auf Deutsch.",
},
{"role": "user", "content": thread_text},
],
temperature=0.3,
max_tokens=300,
)
summary = response.choices[0].message.content or ""
return json.dumps({"summary": summary, "count": len(mails)}, default=str)
except Exception as e:
logger.exception("summarize_mail_thread_handler failed")
return json.dumps({"error": str(e)})
async def get_open_tasks_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
"""Get open tasks (calendar entries) for a contact or company."""
try:
from datetime import UTC, datetime
from app.plugins.builtins.calendar.models import (
CalendarEntry,
CalendarEntryLink,
)
db, tenant_id, _ = await _get_db_and_tenant(context)
entity_type = arguments["entity_type"]
entity_id = uuid.UUID(arguments["entity_id"])
now = datetime.now(UTC)
result = await db.execute(
select(CalendarEntry)
.join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id)
.where(CalendarEntryLink.entity_type == entity_type)
.where(CalendarEntryLink.entity_id == entity_id)
.where(CalendarEntry.tenant_id == tenant_id)
.where(CalendarEntry.start_at > now)
.where(CalendarEntry.status == "open")
.order_by(CalendarEntry.start_at.asc())
.limit(20)
)
tasks = [_serialize(e) for e in result.scalars().all()]
return json.dumps({"tasks": tasks, "count": len(tasks)}, default=str)
except Exception as e:
logger.exception("get_open_tasks_handler failed")
return json.dumps({"error": str(e)})
async def hybrid_search_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
"""Perform hybrid search via unified_search search_engine."""
try:
from app.plugins.builtins.unified_search.search_engine import hybrid_search
db, tenant_id, _ = await _get_db_and_tenant(context)
query = arguments["query"]
entity_types = arguments.get("entity_types")
limit = arguments.get("limit", 20)
query_analysis = {
"normalized_query": query,
"semantic_terms": [],
}
results = await hybrid_search(
db, query_analysis, tenant_id, entity_types=entity_types, limit=limit
)
return json.dumps({"results": results, "count": len(results)}, default=str)
except Exception as e:
logger.exception("hybrid_search_handler failed")
return json.dumps({"error": str(e)})
# ─── Registration ───
def register_context_tools(registry) -> None:
"""Register AI tools into the existing Tool Registry."""
# Tool 1: get_contact_mails
registry.register(
name="get_contact_mails",
description="Alle Mails eines Kontakts abrufen, sortiert nach Datum absteigend",
parameters={
"type": "object",
"properties": {
"contact_id": {"type": "string", "description": "UUID des Kontakts"},
"limit": {"type": "integer", "default": 10, "description": "Max results"},
},
"required": ["contact_id"],
},
handler=get_contact_mails_handler,
plugin_name="ai_proactive",
required_permission="mail:read",
category="context",
)
# Tool 2: get_contact_history
registry.register(
name="get_contact_history",
description="Aktivitätsverlauf (Audit Log) für eine Entity abrufen",
parameters={
"type": "object",
"properties": {
"entity_id": {"type": "string", "description": "UUID der Entity"},
"limit": {"type": "integer", "default": 20, "description": "Max results"},
},
"required": ["entity_id"],
},
handler=get_contact_history_handler,
plugin_name="ai_proactive",
required_permission="ai_proactive:read",
category="context",
)
# Tool 3: search_related
registry.register(
name="search_related",
description="Semantisch ähnliche Entities über unified_search finden",
parameters={
"type": "object",
"properties": {
"entity_type": {"type": "string", "description": "contact, company, mail, file, event"},
"entity_id": {"type": "string", "description": "UUID der Entity"},
"limit": {"type": "integer", "default": 5, "description": "Max results per type"},
},
"required": ["entity_type", "entity_id"],
},
handler=search_related_handler,
plugin_name="ai_proactive",
required_permission="search:read",
category="context",
)
# Tool 4: summarize_mail_thread
registry.register(
name="summarize_mail_thread",
description="E-Mail-Thread per LLM zusammenfassen",
parameters={
"type": "object",
"properties": {
"thread_id": {"type": "string", "description": "Thread ID"},
"limit": {"type": "integer", "default": 20, "description": "Max mails to include"},
},
"required": ["thread_id"],
},
handler=summarize_mail_thread_handler,
plugin_name="ai_proactive",
required_permission="mail:read",
category="context",
)
# Tool 5: get_open_tasks
registry.register(
name="get_open_tasks",
description="Offene Aufgaben (Kalender-Termine) für einen Kontakt oder eine Firma abrufen",
parameters={
"type": "object",
"properties": {
"entity_type": {"type": "string", "description": "contact or company"},
"entity_id": {"type": "string", "description": "UUID der Entity"},
},
"required": ["entity_type", "entity_id"],
},
handler=get_open_tasks_handler,
plugin_name="ai_proactive",
required_permission="ai_proactive:read",
category="context",
)
# Tool 6: hybrid_search
registry.register(
name="hybrid_search",
description="Hybride Volltext- und Semantiksuche über alle CRM-Entities via unified_search",
parameters={
"type": "object",
"properties": {
"query": {"type": "string", "description": "Suchanfrage"},
"entity_types": {
"type": "array",
"items": {"type": "string"},
"description": "Optional: filter by entity types",
},
"limit": {"type": "integer", "default": 20, "description": "Max results"},
},
"required": ["query"],
},
handler=hybrid_search_handler,
plugin_name="ai_proactive",
required_permission="search:read",
category="context",
)
@@ -0,0 +1,157 @@
import { useProactiveSettings } from './api';
const categoryLabels: Record<string, string> = {
mail: 'E-Mails',
tasks: 'Aufgaben',
contacts: 'Kontakte',
companies: 'Firmen',
insights: 'Erkenntnisse',
};
const modelOptions = [
{ value: 'ollama/deepseek-v4', label: 'DeepSeek V4 (empfohlen)' },
{ value: 'ollama/deepseek-v4-pro', label: 'DeepSeek V4 Pro (präzise)' },
{ value: 'ollama/llama3.2', label: 'Llama 3.2 (Alternative)' },
{ value: 'ollama/gpt-4o-mini', label: 'GPT-4o mini via Ollama' },
{ value: 'gpt-4o-mini', label: 'GPT-4o mini (Direct OpenAI)' },
];
export function AISettings() {
const { settings, update } = useProactiveSettings();
if (!settings) {
return (
<div className="flex items-center justify-center p-8 text-gray-400">
<div className="animate-pulse">Lade Einstellungen...</div>
</div>
);
}
const toggleCategory = async (cat: string) => {
const current = settings.suggestion_categories || [];
const newCats = current.includes(cat)
? current.filter((c: string) => c !== cat)
: [...current, cat];
await update({ suggestion_categories: newCats });
};
return (
<div className="max-w-2xl space-y-6">
{/* Enable/Disable */}
<div className="flex items-center justify-between p-4 bg-white rounded-lg border border-gray-200">
<div>
<h3 className="font-semibold text-gray-800">Proaktive KI</h3>
<p className="text-sm text-gray-500">Aktiviert oder deaktiviert alle Vorschläge</p>
</div>
<button
onClick={() => update({ enabled: !settings.enabled })}
className={`relative inline-flex h-6 w-11 items-center rounded-full transition-colors ${
settings.enabled ? 'bg-blue-500' : 'bg-gray-300'
}`}
>
<span
className={`inline-block h-4 w-4 transform rounded-full bg-white transition-transform ${
settings.enabled ? 'translate-x-6' : 'translate-x-1'
}`}
/>
</button>
</div>
{/* Categories */}
<div className="p-4 bg-white rounded-lg border border-gray-200">
<h3 className="font-semibold text-gray-800 mb-3">Kategorien</h3>
<p className="text-sm text-gray-500 mb-4">Für welche Bereiche sollen Vorschläge generiert werden?</p>
<div className="grid grid-cols-2 gap-3">
{Object.entries(categoryLabels).map(([key, label]) => {
const checked = settings.suggestion_categories?.includes(key);
return (
<label
key={key}
className={`flex items-center gap-2 p-3 rounded-lg border cursor-pointer transition-colors ${
checked ? 'bg-blue-50 border-blue-300' : 'bg-gray-50 border-gray-200 hover:bg-gray-100'
}`}
>
<input
type="checkbox"
checked={checked || false}
onChange={() => toggleCategory(key)}
className="w-4 h-4 rounded text-blue-500 focus:ring-blue-400"
/>
<span className="text-sm text-gray-700">{label}</span>
</label>
);
})}
</div>
</div>
{/* Confidence Threshold */}
<div className="p-4 bg-white rounded-lg border border-gray-200">
<div className="flex items-center justify-between mb-2">
<h3 className="font-semibold text-gray-800">Confidence Schwellwert</h3>
<span className="text-sm font-mono text-blue-600">
{Math.round((settings.confidence_threshold || 0.5) * 100)}%
</span>
</div>
<p className="text-sm text-gray-500 mb-3">
Nur Vorschläge mit höherer Confidence anzeigen
</p>
<input
type="range"
min="0"
max="1"
step="0.05"
value={settings.confidence_threshold || 0.5}
onChange={(e) => update({ confidence_threshold: parseFloat(e.target.value) })}
className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500"
/>
<div className="flex justify-between text-xs text-gray-400 mt-1">
<span>Alle</span>
<span>Sehr sicher</span>
</div>
</div>
{/* Rate Limit */}
<div className="p-4 bg-white rounded-lg border border-gray-200">
<div className="flex items-center justify-between mb-2">
<h3 className="font-semibold text-gray-800">Rate Limit</h3>
<span className="text-sm font-mono text-blue-600">
{settings.rate_limit_seconds || 10}s
</span>
</div>
<p className="text-sm text-gray-500 mb-3">
Mindestabstand zwischen Vorschlägen
</p>
<input
type="range"
min="5"
max="60"
step="5"
value={settings.rate_limit_seconds || 10}
onChange={(e) => update({ rate_limit_seconds: parseInt(e.target.value) })}
className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500"
/>
<div className="flex justify-between text-xs text-gray-400 mt-1">
<span>5s</span>
<span>60s</span>
</div>
</div>
{/* Model Selection */}
<div className="p-4 bg-white rounded-lg border border-gray-200">
<h3 className="font-semibold text-gray-800 mb-2">KI Modell (Ollama Cloud)</h3>
<p className="text-sm text-gray-500 mb-3">Wähle das Modell für Vorschläge. DeepSeek V4 ist empfohlen für beste Performance/Kosten.</p>
<select
value={settings.model || 'ollama/deepseek-v4'}
onChange={(e) => update({ model: e.target.value })}
className="w-full px-3 py-2 rounded-lg border border-gray-200 bg-white text-sm text-gray-700 focus:ring-2 focus:ring-blue-400 focus:border-blue-400 outline-none"
>
{modelOptions.map(opt => (
<option key={opt.value} value={opt.value}>
{opt.label}
</option>
))}
</select>
</div>
</div>
);
}
@@ -0,0 +1,59 @@
import { useState, useEffect } from 'react';
import { apiClient } from '@/api/client';
interface SuggestionBadgeProps {
onClick: () => void;
}
export function SuggestionBadge({ onClick }: SuggestionBadgeProps) {
const [count, setCount] = useState(0);
const [pulsing, setPulsing] = useState(false);
useEffect(() => {
// Initial count
apiClient.get('/api/v1/ai-proactive/suggestions').then(r => {
setCount(r.data.items.length);
}).catch(() => {});
// SSE for real-time updates
const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream');
eventSource.onmessage = () => {
setCount(prev => prev + 1);
setPulsing(true);
setTimeout(() => setPulsing(false), 2000);
};
return () => eventSource.close();
}, []);
if (count === 0) {
return (
<button
onClick={onClick}
className="relative p-2 text-gray-500 hover:text-gray-700 transition-colors"
title="KI Vorschläge"
>
🤖
</button>
);
}
return (
<button
onClick={onClick}
className={`relative p-2 text-gray-500 hover:text-gray-700 transition-colors ${
pulsing ? 'animate-pulse' : ''
}`}
title={`${count} KI Vorschläge`}
>
🤖
<span
className={`absolute -top-0 -right-0 min-w-[18px] h-[18px] flex items-center justify-center text-[10px] font-bold text-white rounded-full ${
pulsing ? 'bg-red-500 animate-bounce' : 'bg-blue-500'
}`}
>
{count > 99 ? '99+' : count}
</span>
</button>
);
}
@@ -0,0 +1,94 @@
import { useState } from 'react';
import type { Suggestion } from './api';
const typeConfig = {
info: { icon: '💡', color: 'blue', bg: 'bg-blue-50', border: 'border-blue-200', text: 'text-blue-700', bar: 'bg-blue-500' },
warning: { icon: '⚠️', color: 'orange', bg: 'bg-orange-50', border: 'border-orange-200', text: 'text-orange-700', bar: 'bg-orange-500' },
action: { icon: '✅', color: 'green', bg: 'bg-green-50', border: 'border-green-200', text: 'text-green-700', bar: 'bg-green-500' },
insight: { icon: '🔍', color: 'purple', bg: 'bg-purple-50', border: 'border-purple-200', text: 'text-purple-700', bar: 'bg-purple-500' },
};
interface SuggestionCardProps {
suggestion: Suggestion;
onDismiss: (id: string) => void;
onAct: (id: string, actionIndex: number) => void;
}
export function SuggestionCard({ suggestion, onDismiss, onAct }: SuggestionCardProps) {
const [expanded, setExpanded] = useState(false);
const config = typeConfig[suggestion.suggestion_type] || typeConfig.info;
const confidencePercent = Math.round(suggestion.confidence * 100);
return (
<div
className={`animate-slide-in ${config.bg} ${config.border} border rounded-lg p-4 mb-3 shadow-sm hover:shadow-md transition-shadow`}
>
{/* Header */}
<div className="flex items-start justify-between mb-2">
<div className="flex items-center gap-2">
<span className="text-lg">{config.icon}</span>
<h3 className={`font-semibold text-sm ${config.text}`}>{suggestion.title}</h3>
</div>
<button
onClick={() => onDismiss(suggestion.id)}
className="text-gray-400 hover:text-gray-600 transition-colors"
title="Ignorieren"
>
</button>
</div>
{/* Content */}
<p className="text-sm text-gray-600 mb-3 line-clamp-3">{suggestion.content}</p>
{/* Confidence Bar */}
<div className="flex items-center gap-2 mb-3">
<div className="flex-1 h-1.5 bg-gray-200 rounded-full overflow-hidden">
<div
className={`h-full ${config.bar} rounded-full transition-all duration-500`}
style={{ width: `${confidencePercent}%` }}
/>
</div>
<span className="text-xs text-gray-500 font-mono">{confidencePercent}%</span>
</div>
{/* Actions */}
{suggestion.actions && suggestion.actions.length > 0 && (
<div className="space-y-2">
{suggestion.actions.slice(0, expanded ? undefined : 2).map((action, idx) => (
<button
key={idx}
onClick={() => onAct(suggestion.id, idx)}
disabled={suggestion.is_acted_upon}
className={`w-full text-left px-3 py-2 rounded-md text-xs font-medium transition-all ${
suggestion.is_acted_upon
? 'bg-gray-100 text-gray-400 cursor-not-allowed'
: 'bg-white hover:bg-gray-50 text-gray-700 border border-gray-200 hover:border-gray-300'
}`}
>
<span className="font-mono text-[10px] uppercase mr-1 px-1 py-0.5 bg-gray-100 rounded">
{action.method}
</span>
{action.description}
</button>
))}
{suggestion.actions.length > 2 && (
<button
onClick={() => setExpanded(!expanded)}
className="text-xs text-gray-500 hover:text-gray-700"
>
{expanded ? 'Weniger' : `${suggestion.actions.length - 2} weitere Aktionen`}
</button>
)}
</div>
)}
{/* Acted upon badge */}
{suggestion.is_acted_upon && (
<div className="mt-2 text-xs text-green-600 font-medium flex items-center gap-1">
Ausgeführt
</div>
)}
</div>
);
}
@@ -0,0 +1,109 @@
import { useState } from 'react';
import { useSuggestions } from './api';
import { SuggestionCard } from './SuggestionCard';
interface SuggestionSidebarProps {
isOpen: boolean;
onClose: () => void;
}
export function SuggestionSidebar({ isOpen, onClose }: SuggestionSidebarProps) {
const { suggestions, connected, dismiss, act } = useSuggestions();
const [filter, setFilter] = useState<string>('all');
const filteredSuggestions = filter === 'all'
? suggestions
: suggestions.filter(s => s.suggestion_type === filter);
const filterOptions = [
{ value: 'all', label: 'Alle' },
{ value: 'info', label: 'Info' },
{ value: 'warning', label: 'Warnung' },
{ value: 'action', label: 'Aktion' },
{ value: 'insight', label: 'Erkenntnis' },
];
return (
<>
{/* Overlay */}
{isOpen && (
<div
className="fixed inset-0 bg-black/20 z-40 lg:hidden"
onClick={onClose}
/>
)}
{/* Sidebar */}
<aside
className={`fixed right-0 top-0 h-full w-96 bg-white shadow-2xl z-50 transform transition-transform duration-300 flex flex-col ${
isOpen ? 'translate-x-0' : 'translate-x-full'
}`}
>
{/* Header */}
<div className="flex items-center justify-between p-4 border-b border-gray-200">
<div className="flex items-center gap-2">
<h2 className="font-semibold text-gray-800">KI Vorschläge</h2>
{connected && (
<span className="flex items-center gap-1 text-xs text-green-600">
<span className="w-2 h-2 bg-green-500 rounded-full animate-pulse" />
Live
</span>
)}
</div>
<button
onClick={onClose}
className="text-gray-400 hover:text-gray-600 transition-colors p-1"
>
</button>
</div>
{/* Filter */}
<div className="flex gap-1 p-3 border-b border-gray-100 overflow-x-auto">
{filterOptions.map(opt => (
<button
key={opt.value}
onClick={() => setFilter(opt.value)}
className={`px-3 py-1 rounded-full text-xs font-medium whitespace-nowrap transition-colors ${
filter === opt.value
? 'bg-blue-500 text-white'
: 'bg-gray-100 text-gray-600 hover:bg-gray-200'
}`}
>
{opt.label}
</button>
))}
</div>
{/* Content */}
<div className="flex-1 overflow-y-auto p-4">
{filteredSuggestions.length === 0 ? (
<div className="flex flex-col items-center justify-center h-full text-gray-400">
<div className="text-4xl mb-3">🤖</div>
<p className="text-sm">Keine Vorschläge vorhanden</p>
<p className="text-xs mt-1">Die KI analysiert deinen Kontext...</p>
</div>
) : (
<div>
{filteredSuggestions.map(suggestion => (
<SuggestionCard
n key={suggestion.id}
suggestion={suggestion}
onDismiss={dismiss}
onAct={act}
/>
))}
</div>
)}
</div>
{/* Footer */}
<div className="p-3 border-t border-gray-200 text-center">
<span className="text-xs text-gray-400">
{suggestions.length} Vorschläge insgesamt
</span>
</div>
</aside>
</>
);
}
@@ -0,0 +1,72 @@
import { useState, useEffect, useCallback } from 'react';
import { apiClient } from '@/api/client';
export interface Suggestion {
id: string;
entity_type: string;
entity_id: string | null;
suggestion_type: 'info' | 'warning' | 'action' | 'insight';
title: string;
content: string;
confidence: number;
actions: Array<{ method: string; path: string; body: any; description: string }>;
created_at: string;
is_dismissed: boolean;
is_acted_upon: boolean;
}
export function useSuggestions() {
const [suggestions, setSuggestions] = useState<Suggestion[]>([]);
const [connected, setConnected] = useState(false);
useEffect(() => {
// Initial load
apiClient.get('/api/v1/ai-proactive/suggestions').then(r => {
setSuggestions(r.data.items);
}).catch(() => {});
// SSE Stream
const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream');
eventSource.onopen = () => setConnected(true);
eventSource.onerror = () => setConnected(false);
eventSource.onmessage = (e) => {
try {
const suggestion = JSON.parse(e.data);
setSuggestions(prev => [suggestion, ...prev].slice(0, 50));
} catch {}
};
return () => eventSource.close();
}, []);
const dismiss = useCallback(async (id: string) => {
setSuggestions(prev => prev.filter(s => s.id !== id));
await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/dismiss`).catch(() => {});
}, []);
const act = useCallback(async (id: string, actionIndex: number) => {
const result = await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/act`, {
action_index: actionIndex
});
setSuggestions(prev => prev.map(s => s.id === id ? { ...s, is_acted_upon: true } : s));
return result.data;
}, []);
return { suggestions, connected, dismiss, act };
}
export function useProactiveSettings() {
const [settings, setSettings] = useState<any>(null);
useEffect(() => {
apiClient.get('/api/v1/ai-proactive/settings').then(r => setSettings(r.data));
}, []);
const update = useCallback(async (updates: any) => {
const r = await apiClient.put('/api/v1/ai-proactive/settings', updates);
setSettings(r.data);
return r.data;
}, []);
return { settings, update };
}
@@ -0,0 +1,24 @@
import { useEffect, useRef } from 'react';
import { useLocation } from 'react-router-dom';
import { apiClient } from '@/api/client';
export function useAIContext(entityType?: string, entityId?: string, entityData?: any) {
const location = useLocation();
const debounceRef = useRef<ReturnType<typeof setTimeout>>();
useEffect(() => {
if (debounceRef.current) clearTimeout(debounceRef.current);
debounceRef.current = setTimeout(() => {
apiClient.post('/api/v1/ai-proactive/context', {
page: location.pathname,
entity_type: entityType,
entity_id: entityId,
entity_data: entityData,
}).catch(() => {}); // Silent fail, don't bother user
}, 500); // Debounce 500ms
return () => {
if (debounceRef.current) clearTimeout(debounceRef.current);
};
}, [location.pathname, entityType, entityId, entityData]);
}
+292
View File
@@ -0,0 +1,292 @@
"""ARQ background jobs for the AI Proactive plugin.
Deep analysis job runs after context-change for deeper analysis:
- Mail-thread summary generation
- Finding similar contacts via unified_search
- Analyzing open tasks
- Generating extended suggestion with more context
- Pushing additional suggestion if confidence > threshold
"""
from __future__ import annotations
import os
import json
import logging
import uuid
from typing import Any
import litellm
from sqlalchemy import select
from app.core.db import create_db_session
from app.core.notifications import create_notification
from app.models.audit import AuditLog
from app.models.contact import Contact
from app.plugins.builtins.ai_proactive.models import ProactiveSuggestion, ProactiveSettings
from app.plugins.builtins.ai_proactive.services import (
_serialize_row,
generate_suggestion,
get_user_settings,
push_suggestion,
)
from app.plugins.builtins.mail.models import Mail
logger = logging.getLogger(__name__)
OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
DEEP_SYSTEM_PROMPT = """Du bist ein proaktiver KI-Assistent für ein CRM. Du führst eine Tiefenanalyse durch.
Analysiere den erweiterten Kontext und generiere einen detaillierten Vorschlag.
Antworte mit JSON:
{
"suggestion_type": "info|warning|action|insight",
"title": "Kurzer Titel (max 200 Zeichen)",
"content": "Detaillierte Beschreibung mit konkreten Empfehlungen",
"confidence": 0.0-1.0,
"actions": [{"method": "GET|POST|PUT|DELETE", "path": "/api/v1/...", "body": {}, "description": "..."}]
}
Fokussiere auf:
- Muster in der Kommunikationshistorie
- Beziehungen zwischen Entities
- Handlungsempfehlungen mit hoher Relevanz
- Antworte nur mit gültigem JSON"""
async def deep_analysis(
ctx: dict[str, Any],
entity_type: str,
entity_id: str,
user_id: str,
tenant_id: str,
) -> None:
"""Deep Analysis Background-Job.
Enqueued after context-change for deeper analysis:
- Mail-thread summary generation
- Similar contacts via unified_search
- Open tasks analysis
- Extended suggestion generation with more context
- If confidence > threshold: save + push additional suggestion
"""
try:
eid = uuid.UUID(entity_id)
uid = uuid.UUID(user_id)
tid = uuid.UUID(tenant_id)
except (ValueError, TypeError):
logger.warning("deep_analysis: invalid UUID parameters")
return
async with create_db_session(tid) as db:
# Get settings
settings = await get_user_settings(db, tid, uid)
if not settings.enabled:
return
# Gather extended context (more mails, more activities)
extended_context: dict[str, Any] = {
"entity_type": entity_type,
"entity_id": entity_id,
"deep_analysis": True,
}
if entity_type == "contact":
# Extended: last 30 mails
mail_result = await db.execute(
select(Mail)
.where(Mail.contact_id == eid)
.where(Mail.tenant_id == tid)
.order_by(Mail.received_at.desc())
.limit(30)
)
extended_context["mails"] = [
_serialize_row(m) for m in mail_result.scalars().all()
]
# Extended: last 50 audit entries
audit_result = await db.execute(
select(AuditLog)
.where(AuditLog.entity_id == eid)
.where(AuditLog.tenant_id == tid)
.order_by(AuditLog.timestamp.desc())
.limit(50)
)
extended_context["activities"] = [
_serialize_row(a) for a in audit_result.scalars().all()
]
# Contact data
contact_result = await db.execute(
select(Contact)
.where(Contact.id == eid)
.where(Contact.tenant_id == tid)
.limit(1)
)
contact = contact_result.scalar_one_or_none()
extended_context["contact"] = _serialize_row(contact) if contact else None
elif entity_type == "company":
from app.models.company import Company, CompanyContact
comp_result = await db.execute(
select(Company)
.where(Company.id == eid)
.where(Company.tenant_id == tid)
.limit(1)
)
company = comp_result.scalar_one_or_none()
extended_context["company"] = _serialize_row(company) if company else None
# All mails for company
mail_result = await db.execute(
select(Mail)
.where(Mail.company_id == eid)
.where(Mail.tenant_id == tid)
.order_by(Mail.received_at.desc())
.limit(30)
)
extended_context["mails"] = [
_serialize_row(m) for m in mail_result.scalars().all()
]
elif entity_type == "mail":
mail_result = await db.execute(
select(Mail)
.where(Mail.id == eid)
.where(Mail.tenant_id == tid)
.limit(1)
)
mail = mail_result.scalar_one_or_none()
extended_context["mail"] = _serialize_row(mail) if mail else None
if mail and mail.thread_id:
thread_result = await db.execute(
select(Mail)
.where(Mail.thread_id == mail.thread_id)
.where(Mail.tenant_id == tid)
.order_by(Mail.received_at.asc())
.limit(50)
)
extended_context["thread"] = [
_serialize_row(m) for m in thread_result.scalars().all()
]
# Similar entities via unified_search
try:
from app.plugins.builtins.unified_search.search_engine import (
find_similar_all_types,
)
extended_context["similar"] = await find_similar_all_types(
db, entity_type, eid, tid, limit=5
)
except Exception:
extended_context["similar"] = {}
# Generate extended suggestion with deep analysis prompt
model = settings.model or "ollama/deepseek-v4"
try:
response = await litellm.acompletion(
model=model,
messages=[
{"role": "system", "content": DEEP_SYSTEM_PROMPT},
{
"role": "user",
"content": json.dumps(
extended_context, default=str, ensure_ascii=False
),
},
],
temperature=0.3,
max_tokens=800,
response_format={"type": "json_object"},
api_key=OLLAMA_API_KEY,
)
content = response.choices[0].message.content
if not content:
return
suggestion_data = json.loads(content)
if not suggestion_data.get("title") or not suggestion_data.get("content"):
return
if not isinstance(suggestion_data.get("actions"), list):
suggestion_data["actions"] = []
confidence = suggestion_data.get("confidence", 0.5)
try:
confidence = float(confidence)
except (TypeError, ValueError):
confidence = 0.5
suggestion_data["confidence"] = max(0.0, min(1.0, confidence))
valid_types = {"info", "warning", "action", "insight"}
if suggestion_data.get("suggestion_type") not in valid_types:
suggestion_data["suggestion_type"] = "insight"
except Exception:
logger.exception("deep_analysis: LLM call failed")
return
# Confidence threshold check
if suggestion_data["confidence"] < settings.confidence_threshold:
logger.debug(
"deep_analysis: confidence %s below threshold %s",
suggestion_data["confidence"],
settings.confidence_threshold,
)
return
# Save extended suggestion
suggestion = ProactiveSuggestion(
tenant_id=tid,
user_id=uid,
entity_type=entity_type,
entity_id=eid,
suggestion_type=suggestion_data["suggestion_type"],
title=f"[Tiefenanalyse] {suggestion_data['title']}",
content=suggestion_data["content"],
confidence=suggestion_data["confidence"],
actions=suggestion_data["actions"],
context_snapshot=extended_context,
)
db.add(suggestion)
await db.flush()
suggestion_dict = {
"id": str(suggestion.id),
"entity_type": suggestion.entity_type,
"entity_id": str(suggestion.entity_id) if suggestion.entity_id else None,
"suggestion_type": suggestion.suggestion_type,
"title": suggestion.title,
"content": suggestion.content,
"confidence": suggestion.confidence,
"actions": suggestion.actions,
"created_at": suggestion.created_at.isoformat()
if suggestion.created_at
else None,
"is_dismissed": suggestion.is_dismissed,
"is_acted_upon": suggestion.is_acted_upon,
}
# Push via SSE
await push_suggestion(str(uid), suggestion_dict)
# Notification for urgent suggestions
if suggestion.suggestion_type == "warning":
try:
await create_notification(
db,
tid,
uid,
type="ai_suggestion_urgent",
title=suggestion.title,
body=suggestion.content[:200],
)
except Exception:
logger.exception("deep_analysis: notification failed")
await db.commit()
logger.info(
"deep_analysis: generated suggestion %s for %s/%s",
suggestion.id,
entity_type,
entity_id,
)
@@ -0,0 +1,50 @@
-- ai_proactive plugin: suggestions, context log, user settings
-- Suggestions Store
CREATE TABLE IF NOT EXISTS ai_proactive_suggestions (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id UUID NOT NULL,
user_id UUID NOT NULL,
entity_type VARCHAR(50) NOT NULL,
entity_id UUID,
suggestion_type VARCHAR(50) NOT NULL DEFAULT 'info', -- info, warning, action, insight
title VARCHAR(200) NOT NULL,
content TEXT NOT NULL,
confidence FLOAT NOT NULL DEFAULT 0.5,
actions JSONB NOT NULL DEFAULT '[]', -- [{method, path, body, description}]
is_dismissed BOOLEAN NOT NULL DEFAULT FALSE,
is_acted_upon BOOLEAN NOT NULL DEFAULT FALSE,
context_snapshot JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS ix_suggestions_user ON ai_proactive_suggestions (tenant_id, user_id, is_dismissed, created_at DESC);
CREATE INDEX IF NOT EXISTS ix_suggestions_entity ON ai_proactive_suggestions (tenant_id, entity_type, entity_id);
-- Context Log (was der User wann angesehen hat)
CREATE TABLE IF NOT EXISTS ai_proactive_context_log (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id UUID NOT NULL,
user_id UUID NOT NULL,
page VARCHAR(200) NOT NULL,
entity_type VARCHAR(50),
entity_id UUID,
entity_data JSONB NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS ix_context_log_user_time ON ai_proactive_context_log (tenant_id, user_id, created_at DESC);
-- User Settings für proactive AI
CREATE TABLE IF NOT EXISTS ai_proactive_settings (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id UUID NOT NULL,
user_id UUID NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT TRUE,
suggestion_categories JSONB NOT NULL DEFAULT '["mail","tasks","contacts","companies","insights"]',
confidence_threshold FLOAT NOT NULL DEFAULT 0.5,
rate_limit_seconds INT NOT NULL DEFAULT 10,
model VARCHAR(100) NOT NULL DEFAULT 'gpt-4o-mini',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
UNIQUE(tenant_id, user_id)
);
+111
View File
@@ -0,0 +1,111 @@
"""SQLAlchemy models for the AI Proactive plugin."""
from __future__ import annotations
import uuid
from datetime import datetime
from typing import Any
from sqlalchemy import (
Boolean,
DateTime,
Float,
ForeignKey,
Index,
Integer,
String,
Text,
)
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
from sqlalchemy.orm import Mapped, mapped_column
from app.core.db import Base, TenantMixin
class ProactiveSuggestion(Base, TenantMixin):
"""A proactive AI suggestion generated from user context."""
__tablename__ = "ai_proactive_suggestions"
__table_args__ = (
Index(
"ix_suggestions_user",
"tenant_id",
"user_id",
"is_dismissed",
"created_at",
),
Index("ix_suggestions_entity", "tenant_id", "entity_type", "entity_id"),
)
id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
user_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
entity_id: Mapped[uuid.UUID | None] = mapped_column(PGUUID(as_uuid=True), nullable=True)
suggestion_type: Mapped[str] = mapped_column(
String(50), nullable=False, default="info"
)
title: Mapped[str] = mapped_column(String(200), nullable=False)
content: Mapped[str] = mapped_column(Text, nullable=False)
confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.5)
actions: Mapped[list[Any]] = mapped_column(JSONB, nullable=False, default=list)
is_dismissed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
is_acted_upon: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
context_snapshot: Mapped[dict[str, Any]] = mapped_column(
JSONB, nullable=False, default=dict
)
expires_at: Mapped[datetime | None] = mapped_column(
DateTime(timezone=True), nullable=True
)
class ContextLog(Base, TenantMixin):
"""Log of user context changes (page views, entity selections)."""
__tablename__ = "ai_proactive_context_log"
__table_args__ = (
Index("ix_context_log_user_time", "tenant_id", "user_id", "created_at"),
)
id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
user_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
page: Mapped[str] = mapped_column(String(200), nullable=False)
entity_type: Mapped[str | None] = mapped_column(String(50), nullable=True)
entity_id: Mapped[uuid.UUID | None] = mapped_column(PGUUID(as_uuid=True), nullable=True)
entity_data: Mapped[dict[str, Any]] = mapped_column(
JSONB, nullable=False, default=dict
)
class ProactiveSettings(Base, TenantMixin):
"""Per-user settings for proactive AI."""
__tablename__ = "ai_proactive_settings"
__table_args__ = (
Index("ix_proactive_settings_user", "tenant_id", "user_id"),
)
id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
user_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False
)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
suggestion_categories: Mapped[list[Any]] = mapped_column(
JSONB, nullable=False, default=lambda: ["mail", "tasks", "contacts", "companies", "insights"]
)
confidence_threshold: Mapped[float] = mapped_column(
Float, nullable=False, default=0.5
)
rate_limit_seconds: Mapped[int] = mapped_column(
Integer, nullable=False, default=10
)
model: Mapped[str] = mapped_column(String(100), nullable=False, default="ollama/deepseek-v4")
+107
View File
@@ -0,0 +1,107 @@
"""Proactive AI Agent plugin — context-aware suggestions built on
unified_search and ai_assistant.
Listens to context-change events, gathers entity data, generates LLM-powered
suggestions, pushes them via SSE, and registers AI tools for the assistant.
"""
from __future__ import annotations
import logging
from typing import Any
from app.plugins.base import BasePlugin
from app.plugins.manifest import PluginManifest, PluginRouteDef
logger = logging.getLogger(__name__)
class AIProactivePlugin(BasePlugin):
"""Proactive KI Agent that monitors user context and suggests actions."""
manifest = PluginManifest(
name="ai_proactive",
version="1.0.0",
display_name="Proaktiver KI Agent",
description=(
"Überwacht den User-Kontext, generiert proaktiv Vorschläge per LLM "
"und pusht diese via SSE. Nutzt unified_search und ai_assistant."
),
dependencies=["ai_assistant", "unified_search"],
routes=[
PluginRouteDef(
path="/api/v1/ai-proactive",
module="app.plugins.builtins.ai_proactive.routes",
router_attr="router",
),
],
events=["context.view_changed", "context.entity_selected"],
migrations=["0001_initial.sql"],
permissions=[
"ai_proactive:read",
"ai_proactive:write",
"ai_proactive:config",
],
is_core=False,
)
async def on_activate(self, db, service_container, event_bus) -> None:
"""Register context tools and subscribe to events."""
await super().on_activate(db, service_container, event_bus)
try:
from app.plugins.builtins.ai_proactive.context_tools import (
register_context_tools,
)
from app.plugins.builtins.ai_assistant.tool_registry import (
get_tool_registry,
)
register_context_tools(get_tool_registry())
logger.info("AI Proactive context tools registered")
except Exception:
logger.exception("Failed to register AI Proactive context tools")
async def on_deactivate(self, db, service_container, event_bus) -> None:
"""Unregister tools and event listeners."""
try:
from app.plugins.builtins.ai_assistant.tool_registry import (
get_tool_registry,
)
get_tool_registry().unregister_plugin("ai_proactive")
logger.info("AI Proactive context tools unregistered")
except Exception:
logger.exception("Failed to unregister AI Proactive context tools")
await super().on_deactivate(db, service_container, event_bus)
# ─── Event Handlers ───
async def on_context_view_changed(self, payload: dict[str, Any]) -> None:
"""Handle context.view_changed event."""
from app.plugins.builtins.ai_proactive.services import handle_context_change
await handle_context_change(payload)
async def on_context_entity_selected(self, payload: dict[str, Any]) -> None:
"""Handle context.entity_selected event."""
from app.plugins.builtins.ai_proactive.services import handle_context_change
await handle_context_change(payload)
def get_notification_types(self) -> list[dict[str, Any]]:
return [
{
"type_key": "ai_suggestion",
"category": "ai",
"label": "KI Vorschlag",
"description": "Proaktiver KI-Vorschlag",
"is_enabled_by_default": True,
},
{
"type_key": "ai_suggestion_urgent",
"category": "ai",
"label": "Dringender KI Vorschlag",
"description": "Dringender proaktiver KI-Vorschlag",
"is_enabled_by_default": True,
},
]
+317
View File
@@ -0,0 +1,317 @@
"""API routes for the AI Proactive plugin.
Endpoints: context tracking, suggestions CRUD, SSE stream, settings, stats.
"""
from __future__ import annotations
import asyncio
import json
import logging
import uuid
from typing import Any
from fastapi import APIRouter, Depends, HTTPException, Query
from fastapi.responses import StreamingResponse
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.db import get_db, set_tenant_context
from app.core.event_bus import get_event_bus
from app.deps import get_current_user, require_permission
from app.plugins.builtins.ai_proactive.models import (
ContextLog,
ProactiveSettings,
ProactiveSuggestion,
)
from app.plugins.builtins.ai_proactive.schemas import (
ActRequest,
ActResponse,
ContextReport,
SettingsResponse,
SettingsUpdate,
SuggestionListResponse,
SuggestionResponse,
StatsResponse,
)
from app.plugins.builtins.ai_proactive.services import (
execute_suggested_action,
get_active_suggestions,
get_sse_queue,
get_stats,
get_user_settings,
mark_dismissed,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/v1/ai-proactive", tags=["ai-proactive"])
def _suggestion_to_response(s: ProactiveSuggestion) -> SuggestionResponse:
"""Convert a ProactiveSuggestion model to SuggestionResponse."""
return SuggestionResponse(
id=str(s.id),
entity_type=s.entity_type,
entity_id=str(s.entity_id) if s.entity_id else None,
suggestion_type=s.suggestion_type,
title=s.title,
content=s.content,
confidence=s.confidence,
actions=s.actions or [],
created_at=s.created_at,
is_dismissed=s.is_dismissed,
is_acted_upon=s.is_acted_upon,
)
def _settings_to_response(s: ProactiveSettings) -> SettingsResponse:
"""Convert ProactiveSettings model to SettingsResponse."""
return SettingsResponse(
enabled=s.enabled,
suggestion_categories=s.suggestion_categories or [],
confidence_threshold=s.confidence_threshold,
rate_limit_seconds=s.rate_limit_seconds,
model=s.model,
)
# ─── Context Tracking ───
@router.post("/context", dependencies=[Depends(require_permission("ai_proactive:read"))])
async def report_context(
context: ContextReport,
current_user: dict[str, Any] = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Frontend reports current context (page, entity).
Stores context log entry and publishes event for proactive engine.
"""
tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"])
# Parse entity_id if present
entity_id: uuid.UUID | None = None
if context.entity_id:
try:
entity_id = uuid.UUID(context.entity_id)
except (ValueError, TypeError):
entity_id = None
# Store context log
log_entry = ContextLog(
tenant_id=tenant_id,
user_id=user_id,
page=context.page,
entity_type=context.entity_type,
entity_id=entity_id,
entity_data=context.entity_data or {},
)
db.add(log_entry)
await db.flush()
# Publish event
event_bus = get_event_bus()
event_name = (
"context.entity_selected"
if context.entity_type and context.entity_id
else "context.view_changed"
)
payload = {
"user_id": str(user_id),
"tenant_id": str(tenant_id),
"page": context.page,
"entity_type": context.entity_type,
"entity_id": str(entity_id) if entity_id else None,
"entity_data": context.entity_data or {},
}
await event_bus.publish(event_name, payload)
return {"status": "ok"}
# ─── Suggestions ───
@router.get(
"/suggestions",
dependencies=[Depends(require_permission("ai_proactive:read"))],
response_model=SuggestionListResponse,
)
async def get_suggestions(
entity_type: str | None = Query(None),
entity_id: str | None = Query(None),
limit: int = Query(10, ge=1, le=100),
current_user: dict[str, Any] = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Get active suggestions for the current user."""
tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"])
eid: uuid.UUID | None = None
if entity_id:
try:
eid = uuid.UUID(entity_id)
except (ValueError, TypeError):
eid = None
suggestions = await get_active_suggestions(
db, tenant_id, user_id, entity_type=entity_type, entity_id=eid, limit=limit
)
items = [_suggestion_to_response(s) for s in suggestions]
return SuggestionListResponse(items=items, total=len(items))
@router.post(
"/suggestions/{suggestion_id}/dismiss",
dependencies=[Depends(require_permission("ai_proactive:write"))],
)
async def dismiss_suggestion(
suggestion_id: str,
current_user: dict[str, Any] = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Mark a suggestion as dismissed."""
tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"])
try:
sid = uuid.UUID(suggestion_id)
except (ValueError, TypeError):
raise HTTPException(status_code=400, detail="Invalid suggestion ID")
success = await mark_dismissed(db, sid, user_id, tenant_id)
if not success:
raise HTTPException(status_code=404, detail="Suggestion not found")
return {"status": "ok"}
@router.post(
"/suggestions/{suggestion_id}/act",
dependencies=[Depends(require_permission("ai_proactive:write"))],
response_model=ActResponse,
)
async def act_on_suggestion(
suggestion_id: str,
action: ActRequest,
current_user: dict[str, Any] = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Execute a suggested action."""
tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"])
try:
sid = uuid.UUID(suggestion_id)
except (ValueError, TypeError):
raise HTTPException(status_code=400, detail="Invalid suggestion ID")
result = await execute_suggested_action(
db, sid, action.action_index, user_id, tenant_id, current_user
)
return ActResponse(
success=result.get("success", False),
data=result.get("data"),
error=result.get("error"),
)
# ─── SSE Stream ───
@router.get(
"/suggestions/stream",
dependencies=[Depends(require_permission("ai_proactive:read"))],
)
async def stream_suggestions(
current_user: dict[str, Any] = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""SSE stream: push new suggestions in real-time.
Uses an asyncio.Queue per user. Heartbeat every 30s.
"""
user_id = str(current_user["user_id"])
async def event_generator():
queue = get_sse_queue(user_id)
while True:
try:
suggestion = await asyncio.wait_for(queue.get(), timeout=30)
yield f"data: {json.dumps(suggestion, default=str)}\n\n"
except asyncio.TimeoutError:
yield ": keepalive\n\n"
return StreamingResponse(
event_generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
)
# ─── Settings ───
@router.get(
"/settings",
dependencies=[Depends(require_permission("ai_proactive:read"))],
response_model=SettingsResponse,
)
async def get_settings(
current_user: dict[str, Any] = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Get proactive AI settings for the current user."""
tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"])
settings = await get_user_settings(db, tenant_id, user_id)
return _settings_to_response(settings)
@router.put(
"/settings",
dependencies=[Depends(require_permission("ai_proactive:config"))],
response_model=SettingsResponse,
)
async def update_settings(
settings_update: SettingsUpdate,
current_user: dict[str, Any] = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Update proactive AI settings for the current user."""
tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"])
settings = await get_user_settings(db, tenant_id, user_id)
update_data = settings_update.model_dump(exclude_unset=True)
for field, val in update_data.items():
setattr(settings, field, val)
await db.flush()
return _settings_to_response(settings)
# ─── Stats ───
@router.get(
"/stats",
dependencies=[Depends(require_permission("ai_proactive:read"))],
response_model=StatsResponse,
)
async def get_stats_endpoint(
current_user: dict[str, Any] = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
):
"""Get proactive AI usage statistics for the current user."""
tenant_id = uuid.UUID(current_user["tenant_id"])
user_id = uuid.UUID(current_user["user_id"])
stats = await get_stats(db, tenant_id, user_id)
return StatsResponse(**stats)
@@ -0,0 +1,103 @@
"""Pydantic v2 schemas for the AI Proactive plugin API."""
from __future__ import annotations
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field
class ContextReport(BaseModel):
"""Frontend reports the current page/entity context."""
page: str = Field(..., description="Current page path")
entity_type: str | None = Field(None, description="Entity type being viewed")
entity_id: str | None = Field(None, description="Entity ID being viewed")
entity_data: dict[str, Any] | None = Field(
None, description="Additional entity metadata"
)
class SuggestionAction(BaseModel):
"""A suggested CRM action."""
method: str = Field(..., description="HTTP method")
path: str = Field(..., description="API path")
body: dict[str, Any] | None = Field(None, description="Request body")
description: str = Field(..., description="Human-readable description")
class SuggestionResponse(BaseModel):
"""API response for a single suggestion."""
id: str
entity_type: str
entity_id: str | None
suggestion_type: str
title: str
content: str
confidence: float
actions: list[dict[str, Any]]
created_at: datetime
is_dismissed: bool
is_acted_upon: bool = False
class SuggestionListResponse(BaseModel):
"""Paginated list of suggestions."""
items: list[SuggestionResponse]
total: int
class ActRequest(BaseModel):
"""Execute a suggested action by index."""
action_index: int = Field(..., ge=0, description="Index into actions array")
class ActResponse(BaseModel):
"""Result of executing a suggested action."""
success: bool
data: dict[str, Any] | None = None
error: str | None = None
class SettingsResponse(BaseModel):
"""Proactive AI settings for the current user."""
enabled: bool
suggestion_categories: list[str]
confidence_threshold: float
rate_limit_seconds: int
model: str
available_models: list[str] = Field(default_factory=lambda: [
'ollama/deepseek-v4',
'ollama/deepseek-v4-pro',
'ollama/llama3.2',
'ollama/gpt-4o-mini',
'gpt-4o-mini',
])
class SettingsUpdate(BaseModel):
"""Partial update for proactive AI settings."""
enabled: bool | None = None
suggestion_categories: list[str] | None = None
confidence_threshold: float | None = None
rate_limit_seconds: int | None = None
model: str | None = None
class StatsResponse(BaseModel):
"""Proactive AI usage statistics."""
total_suggestions: int = 0
dismissed: int = 0
acted_upon: int = 0
active: int = 0
dismiss_rate: float = 0.0
act_rate: float = 0.0
@@ -0,0 +1,748 @@
"""Proactive Engine — core logic for context-aware AI suggestions.
Handles context changes, gathers entity data, generates LLM-powered
suggestions, pushes via SSE, and manages suggestion lifecycle.
"""
from __future__ import annotations
import os
import asyncio
import json
import logging
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
import litellm
from sqlalchemy import func, select, text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.cache import get_cache
from app.core.db import create_db_session, get_session_factory
from app.core.notifications import create_notification
from app.models.audit import AuditLog
from app.models.company import Company
from app.models.contact import CompanyContact, Contact
from app.plugins.builtins.ai_proactive.models import (
ContextLog,
ProactiveSettings,
ProactiveSuggestion,
)
logger = logging.getLogger(__name__)
litellm.suppress_debug_info = True
OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
# ─── SSE Push Infrastructure ───
_sse_queues: dict[str, asyncio.Queue[dict[str, Any]]] = {}
def get_sse_queue(user_id: str) -> asyncio.Queue[dict[str, Any]]:
"""Get or create SSE queue for a user."""
if user_id not in _sse_queues:
_sse_queues[user_id] = asyncio.Queue()
return _sse_queues[user_id]
async def push_suggestion(user_id: str, suggestion: dict[str, Any]) -> None:
"""Push suggestion to user's SSE queue."""
queue = get_sse_queue(user_id)
await queue.put(suggestion)
# ─── Rate Limiting ───
async def is_rate_limited(
tenant_id: uuid.UUID, user_id: uuid.UUID, rate_limit_seconds: int
) -> bool:
"""Check if user is rate-limited using Redis.
Returns True if rate-limited (key exists), False otherwise.
Sets a key with TTL = rate_limit_seconds on first call.
"""
try:
r = get_cache()
key = f"ai_proactive:rate:{user_id}"
existing = await r.get(key)
if existing is not None:
return True
await r.setex(key, rate_limit_seconds, "1")
return False
except Exception:
logger.exception("Rate-limit check failed, allowing request")
return False
# ─── Settings Helpers ───
async def get_user_settings(
db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID
) -> ProactiveSettings:
"""Get proactive AI settings for user, create defaults if not exist."""
result = await db.execute(
select(ProactiveSettings)
.where(ProactiveSettings.tenant_id == tenant_id)
.where(ProactiveSettings.user_id == user_id)
.limit(1)
)
settings = result.scalar_one_or_none()
if settings is None:
settings = ProactiveSettings(
tenant_id=tenant_id,
user_id=user_id, enabled=True,
suggestion_categories=["mail", "tasks", "contacts", "companies", "insights"],
confidence_threshold=0.5,
rate_limit_seconds=10,
model="ollama/deepseek-v4",
)
db.add(settings)
await db.flush()
return settings
# ─── Context Gathering ───
def _serialize_row(row: Any) -> dict[str, Any]:
"""Serialize a SQLAlchemy model instance to a dict."""
if row is None:
return {}
result: dict[str, Any] = {}
for column in row.__table__.columns:
val = getattr(row, column.name)
if isinstance(val, datetime):
result[column.name] = val.isoformat()
elif isinstance(val, uuid.UUID):
result[column.name] = str(val)
else:
result[column.name] = val
return result
async def gather_context(
db: AsyncSession, entity_type: str, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> dict[str, Any]:
"""Collect context data for an entity.
Gathers related data from contacts, companies, mails, calendar events,
audit logs, and semantically similar entities via unified_search.
"""
context: dict[str, Any] = {
"entity_type": entity_type,
"entity_id": str(entity_id),
}
if entity_type == "contact":
# Contact data
result = await db.execute(
select(Contact)
.where(Contact.id == entity_id)
.where(Contact.tenant_id == tenant_id)
.limit(1)
)
contact = result.scalar_one_or_none()
context["contact"] = _serialize_row(contact) if contact else None
# Last 10 mails
from app.plugins.builtins.mail.models import Mail
mail_result = await db.execute(
select(Mail)
.where(Mail.contact_id == entity_id)
.where(Mail.tenant_id == tenant_id)
.order_by(Mail.received_at.desc())
.limit(10)
)
context["mails"] = [_serialize_row(m) for m in mail_result.scalars().all()]
# Company via company_contacts
cc_result = await db.execute(
select(CompanyContact)
.where(CompanyContact.contact_id == entity_id)
.where(CompanyContact.tenant_id == tenant_id)
.limit(5)
)
companies: list[dict[str, Any]] = []
for cc in cc_result.scalars().all():
comp_result = await db.execute(
select(Company)
.where(Company.id == cc.company_id)
.where(Company.tenant_id == tenant_id)
.limit(1)
)
comp = comp_result.scalar_one_or_none()
if comp:
comp_data = _serialize_row(comp)
comp_data["role_at_company"] = cc.role_at_company
comp_data["is_primary"] = cc.is_primary
companies.append(comp_data)
context["company"] = companies[0] if companies else None
context["companies"] = companies
# Upcoming calendar events
from app.plugins.builtins.calendar.models import CalendarEntry, CalendarEntryLink
now = datetime.now(UTC)
event_result = await db.execute(
select(CalendarEntry)
.join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id)
.where(CalendarEntryLink.entity_type == "contact")
.where(CalendarEntryLink.entity_id == entity_id)
.where(CalendarEntry.tenant_id == tenant_id)
.where(CalendarEntry.start_at > now)
.order_by(CalendarEntry.start_at.asc())
.limit(5)
)
context["events"] = [_serialize_row(e) for e in event_result.scalars().all()]
# Last 20 audit log entries
audit_result = await db.execute(
select(AuditLog)
.where(AuditLog.entity_id == entity_id)
.where(AuditLog.tenant_id == tenant_id)
.order_by(AuditLog.timestamp.desc())
.limit(20)
)
context["activities"] = [_serialize_row(a) for a in audit_result.scalars().all()]
elif entity_type == "mail":
from app.plugins.builtins.mail.models import Mail
result = await db.execute(
select(Mail)
.where(Mail.id == entity_id)
.where(Mail.tenant_id == tenant_id)
.limit(1)
)
mail = result.scalar_one_or_none()
context["mail"] = _serialize_row(mail) if mail else None
if mail and mail.contact_id:
contact_result = await db.execute(
select(Contact)
.where(Contact.id == mail.contact_id)
.where(Contact.tenant_id == tenant_id)
.limit(1)
)
contact = contact_result.scalar_one_or_none()
context["contact"] = _serialize_row(contact) if contact else None
if mail and mail.thread_id:
thread_result = await db.execute(
select(Mail)
.where(Mail.thread_id == mail.thread_id)
.where(Mail.tenant_id == tenant_id)
.order_by(Mail.received_at.asc())
.limit(20)
)
context["thread"] = [_serialize_row(m) for m in thread_result.scalars().all()]
if mail and mail.company_id:
comp_result = await db.execute(
select(Company)
.where(Company.id == mail.company_id)
.where(Company.tenant_id == tenant_id)
.limit(1)
)
comp = comp_result.scalar_one_or_none()
context["company"] = _serialize_row(comp) if comp else None
elif entity_type == "company":
result = await db.execute(
select(Company)
.where(Company.id == entity_id)
.where(Company.tenant_id == tenant_id)
.limit(1)
)
company = result.scalar_one_or_none()
context["company"] = _serialize_row(company) if company else None
# Contacts via company_contacts
cc_result = await db.execute(
select(CompanyContact)
.where(CompanyContact.company_id == entity_id)
.where(CompanyContact.tenant_id == tenant_id)
)
contacts: list[dict[str, Any]] = []
for cc in cc_result.scalars().all():
contact_result = await db.execute(
select(Contact)
.where(Contact.id == cc.contact_id)
.where(Contact.tenant_id == tenant_id)
.limit(1)
)
contact = contact_result.scalar_one_or_none()
if contact:
contact_data = _serialize_row(contact)
contact_data["role_at_company"] = cc.role_at_company
contact_data["is_primary"] = cc.is_primary
contacts.append(contact_data)
context["contacts"] = contacts
# Mails for this company
from app.plugins.builtins.mail.models import Mail
mail_result = await db.execute(
select(Mail)
.where(Mail.company_id == entity_id)
.where(Mail.tenant_id == tenant_id)
.order_by(Mail.received_at.desc())
.limit(10)
)
context["mails"] = [_serialize_row(m) for m in mail_result.scalars().all()]
# Upcoming events
from app.plugins.builtins.calendar.models import CalendarEntry, CalendarEntryLink
now = datetime.now(UTC)
event_result = await db.execute(
select(CalendarEntry)
.join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id)
.where(CalendarEntryLink.entity_type == "company")
.where(CalendarEntryLink.entity_id == entity_id)
.where(CalendarEntry.tenant_id == tenant_id)
.where(CalendarEntry.start_at > now)
.order_by(CalendarEntry.start_at.asc())
.limit(5)
)
context["events"] = [_serialize_row(e) for e in event_result.scalars().all()]
elif entity_type == "file":
# Basic file info via raw SQL (file model may vary)
try:
file_result = await db.execute(
text("SELECT * FROM files WHERE id = :fid AND tenant_id = :tid"),
{"fid": entity_id, "tid": tenant_id},
)
file_row = file_result.mappings().first()
context["file"] = dict(file_row) if file_row else None
except Exception:
context["file"] = None
# Linked entities via entity_links (if table exists)
try:
links_result = await db.execute(
text(
"SELECT * FROM entity_links WHERE entity_id = :eid AND tenant_id = :tid LIMIT 20"
),
{"eid": entity_id, "tid": tenant_id},
)
context["linked_entities"] = [dict(r) for r in links_result.mappings().all()]
except Exception:
context["linked_entities"] = []
# Semantically similar entities via unified_search
try:
from app.plugins.builtins.unified_search.search_engine import (
find_similar_all_types,
)
context["similar"] = await find_similar_all_types(
db, entity_type, entity_id, tenant_id, limit=3
)
except Exception:
context["similar"] = {}
return context
# ─── Suggestion Generation ───
SYSTEM_PROMPT = """Du bist ein proaktiver KI-Assistent für ein CRM. Analysiere den Kontext und generiere Vorschläge.
Antworte mit JSON:
{
"suggestion_type": "info|warning|action|insight",
"title": "Kurzer Titel (max 200 Zeichen)",
"content": "Beschreibung des Vorschlags",
"confidence": 0.0-1.0,
"actions": [{"method": "GET|POST|PUT|DELETE", "path": "/api/v1/...", "body": {}, "description": "..."}]
}
- suggestion_type: info=Information, warning=Warnung, action=Aktionsvorschlag, insight=Erkenntnis
- actions: Vorgeschlagene CRM-Aktionen die der User ausführen kann
- confidence: Wie sicher bist du dir (0.0=unsicher, 1.0=sehr sicher)
- Antworte nur mit gültigem JSON, kein Markdown"""
async def generate_suggestion(
context_data: dict[str, Any], settings: ProactiveSettings
) -> dict[str, Any] | None:
"""LLM generates a suggestion from context data.
Returns dict with suggestion_type, title, content, confidence, actions
or None on failure.
"""
model = settings.model or "ollama/deepseek-v4"
try:
response = await litellm.acompletion(
model=model,
messages=[
{"role": "system", "content": SYSTEM_PROMPT},
{
"role": "user",
"content": json.dumps(context_data, default=str, ensure_ascii=False),
},
],
temperature=0.3,
max_tokens=500,
response_format={"type": "json_object"},
api_key=OLLAMA_API_KEY,
)
content = response.choices[0].message.content
if not content:
return None
result = json.loads(content)
# Validate required fields
if not result.get("title") or not result.get("content"):
return None
# Ensure actions is a list
if not isinstance(result.get("actions"), list):
result["actions"] = []
# Clamp confidence
confidence = result.get("confidence", 0.5)
try:
confidence = float(confidence)
except (TypeError, ValueError):
confidence = 0.5
result["confidence"] = max(0.0, min(1.0, confidence))
# Validate suggestion_type
valid_types = {"info", "warning", "action", "insight"}
if result.get("suggestion_type") not in valid_types:
result["suggestion_type"] = "info"
return result
except Exception:
logger.exception("Failed to generate suggestion via LLM")
return None
# ─── Main Handler ───
async def handle_context_change(payload: dict[str, Any]) -> None:
"""Main handler: Context-Change → Suggestion generation.
1. Rate-limit check
2. Settings check (enabled?)
3. Gather context data
4. Generate suggestion via LLM
5. Confidence threshold check
6. Save suggestion to DB
7. Push via SSE
8. Notification for urgent suggestions
9. Enqueue background deep analysis job
"""
user_id_str = payload.get("user_id")
tenant_id_str = payload.get("tenant_id")
entity_type = payload.get("entity_type")
entity_id_str = payload.get("entity_id")
if not user_id_str or not tenant_id_str or not entity_type:
logger.warning("handle_context_change: missing required fields in payload")
return
try:
tenant_id = uuid.UUID(tenant_id_str)
user_id = uuid.UUID(user_id_str)
except (ValueError, TypeError):
logger.warning("handle_context_change: invalid UUID in payload")
return
entity_id: uuid.UUID | None = None
if entity_id_str:
try:
entity_id = uuid.UUID(entity_id_str)
except (ValueError, TypeError):
entity_id = None
if entity_id is None:
logger.debug("handle_context_change: no entity_id, skipping")
return
async with create_db_session(tenant_id) as db:
# Get settings
settings = await get_user_settings(db, tenant_id, user_id)
if not settings.enabled:
logger.debug("handle_context_change: proactive AI disabled for user")
return
# Rate limit check
if await is_rate_limited(tenant_id, user_id, settings.rate_limit_seconds):
logger.debug("handle_context_change: rate limited")
return
# Gather context
context_data = await gather_context(db, entity_type, entity_id, tenant_id)
# Generate suggestion
suggestion_data = await generate_suggestion(context_data, settings)
if suggestion_data is None:
logger.debug("handle_context_change: no suggestion generated")
return
# Confidence threshold check
if suggestion_data["confidence"] < settings.confidence_threshold:
logger.debug(
"handle_context_change: confidence %s below threshold %s",
suggestion_data["confidence"],
settings.confidence_threshold,
)
return
# Save suggestion
suggestion = ProactiveSuggestion(
tenant_id=tenant_id,
user_id=user_id,
entity_type=entity_type,
entity_id=entity_id,
suggestion_type=suggestion_data["suggestion_type"],
title=suggestion_data["title"],
content=suggestion_data["content"],
confidence=suggestion_data["confidence"],
actions=suggestion_data["actions"],
context_snapshot=context_data,
)
db.add(suggestion)
await db.flush()
# Build response dict for SSE
suggestion_dict = {
"id": str(suggestion.id),
"entity_type": suggestion.entity_type,
"entity_id": str(suggestion.entity_id) if suggestion.entity_id else None,
"suggestion_type": suggestion.suggestion_type,
"title": suggestion.title,
"content": suggestion.content,
"confidence": suggestion.confidence,
"actions": suggestion.actions,
"created_at": suggestion.created_at.isoformat() if suggestion.created_at else None,
"is_dismissed": suggestion.is_dismissed,
"is_acted_upon": suggestion.is_acted_upon,
}
# Push via SSE
await push_suggestion(str(user_id), suggestion_dict)
# Notification for urgent suggestions
if suggestion.suggestion_type == "warning":
try:
await create_notification(
db,
tenant_id,
user_id,
type="ai_suggestion_urgent",
title=suggestion.title,
body=suggestion.content[:200],
)
except Exception:
logger.exception("Failed to create notification")
else:
try:
await create_notification(
db,
tenant_id,
user_id,
type="ai_suggestion",
title=suggestion.title,
body=suggestion.content[:200],
)
except Exception:
logger.exception("Failed to create notification")
await db.commit()
# Enqueue background deep analysis job
try:
from app.core.jobs import enqueue_job
await enqueue_job(
"deep_analysis",
entity_type,
str(entity_id),
str(user_id),
str(tenant_id),
)
except Exception:
logger.exception("Failed to enqueue deep_analysis job")
# ─── Suggestion CRUD ───
async def get_active_suggestions(
db: AsyncSession,
tenant_id: uuid.UUID,
user_id: uuid.UUID,
entity_type: str | None = None,
entity_id: uuid.UUID | None = None,
limit: int = 10,
) -> list[ProactiveSuggestion]:
"""Get active (non-dismissed, not expired) suggestions for user."""
now = datetime.now(UTC)
stmt = (
select(ProactiveSuggestion)
.where(ProactiveSuggestion.tenant_id == tenant_id)
.where(ProactiveSuggestion.user_id == user_id)
.where(ProactiveSuggestion.is_dismissed == False) # noqa: E712
.where(
(ProactiveSuggestion.expires_at.is_(None))
| (ProactiveSuggestion.expires_at > now)
)
)
if entity_type:
stmt = stmt.where(ProactiveSuggestion.entity_type == entity_type)
if entity_id:
stmt = stmt.where(ProactiveSuggestion.entity_id == entity_id)
stmt = stmt.order_by(ProactiveSuggestion.created_at.desc()).limit(limit)
result = await db.execute(stmt)
return list(result.scalars().all())
async def mark_dismissed(
db: AsyncSession,
suggestion_id: uuid.UUID,
user_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> bool:
"""Mark suggestion as dismissed. Returns True if found and updated."""
result = await db.execute(
select(ProactiveSuggestion)
.where(ProactiveSuggestion.id == suggestion_id)
.where(ProactiveSuggestion.tenant_id == tenant_id)
.where(ProactiveSuggestion.user_id == user_id)
.limit(1)
)
suggestion = result.scalar_one_or_none()
if suggestion is None:
return False
suggestion.is_dismissed = True
await db.flush()
return True
async def execute_suggested_action(
db: AsyncSession,
suggestion_id: uuid.UUID,
action_index: int,
user_id: uuid.UUID,
tenant_id: uuid.UUID,
user_context: dict[str, Any],
) -> dict[str, Any]:
"""Execute a suggested action.
1. Load suggestion
2. Get action from actions[action_index]
3. Execute via internal HTTP call (httpx)
4. Mark suggestion as is_acted_upon=True
5. Return result
"""
import httpx
result = await db.execute(
select(ProactiveSuggestion)
.where(ProactiveSuggestion.id == suggestion_id)
.where(ProactiveSuggestion.tenant_id == tenant_id)
.where(ProactiveSuggestion.user_id == user_id)
.limit(1)
)
suggestion = result.scalar_one_or_none()
if suggestion is None:
return {"success": False, "error": "Suggestion not found"}
actions = suggestion.actions or []
if action_index < 0 or action_index >= len(actions):
return {"success": False, "error": "Invalid action index"}
action = actions[action_index]
method = action.get("method", "GET").upper()
path = action.get("path", "")
body = action.get("body")
if not path:
return {"success": False, "error": "No path in action"}
# Build internal URL
from app.config import get_settings
settings = get_settings()
base_url = getattr(settings, "internal_base_url", "http://localhost:8000")
url = f"{base_url}{path}"
# Build headers from user context (session cookie)
headers: dict[str, str] = {"Content-Type": "application/json"}
cookie_name = getattr(settings, "session_cookie_name", "session")
session_id = user_context.get("session_id", "")
if session_id:
headers["Cookie"] = f"{cookie_name}={session_id}"
try:
async with httpx.AsyncClient(timeout=30.0) as client:
resp = await client.request(
method,
url,
json=body if body else None,
headers=headers,
)
data = None
try:
data = resp.json()
except Exception:
data = {"status_code": resp.status_code, "text": resp.text}
if resp.status_code < 400:
suggestion.is_acted_upon = True
await db.flush()
return {"success": True, "data": data}
return {"success": False, "error": f"HTTP {resp.status_code}", "data": data}
except Exception as e:
logger.exception("Failed to execute suggested action")
return {"success": False, "error": str(e)}
# ─── Stats ───
async def get_stats(
db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID
) -> dict[str, Any]:
"""Get proactive AI usage statistics for a user."""
base_filter = (
ProactiveSuggestion.tenant_id == tenant_id,
ProactiveSuggestion.user_id == user_id,
)
total_result = await db.execute(
select(func.count()).select_from(ProactiveSuggestion).where(*base_filter)
)
total = total_result.scalar() or 0
dismissed_result = await db.execute(
select(func.count())
.select_from(ProactiveSuggestion)
.where(*base_filter, ProactiveSuggestion.is_dismissed == True) # noqa: E712
)
dismissed = dismissed_result.scalar() or 0
acted_result = await db.execute(
select(func.count())
.select_from(ProactiveSuggestion)
.where(*base_filter, ProactiveSuggestion.is_acted_upon == True) # noqa: E712
)
acted_upon = acted_result.scalar() or 0
active = total - dismissed
dismiss_rate = (dismissed / total) if total > 0 else 0.0
act_rate = (acted_upon / total) if total > 0 else 0.0
return {
"total_suggestions": total,
"dismissed": dismissed,
"acted_upon": acted_upon,
"active": active,
"dismiss_rate": round(dismiss_rate, 4),
"act_rate": round(act_rate, 4),
}
@@ -0,0 +1,7 @@
"""Unified Search plugin — hybrid FTS + semantic search across all CRM data."""
from __future__ import annotations
from app.plugins.builtins.unified_search.plugin import UnifiedSearchPlugin
__all__ = ["UnifiedSearchPlugin"]
@@ -0,0 +1,131 @@
"""Embedding pipeline using LiteLLM with configurable Ollama Cloud provider."""
from __future__ import annotations
import os
import logging
import uuid
from typing import TYPE_CHECKING
import litellm
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
MAX_INPUT_CHARS = 8000
DEFAULT_EMBEDDING_MODEL = os.environ.get('SEARCH_EMBEDDING_MODEL', 'ollama/nomic-embed-text')
OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
async def generate_embedding(text: str, model: str | None = None) -> list[float]:
"""Generate a single embedding via LiteLLM.
Args:
text: Input text (truncated to 8000 chars).
model: Embedding model name (default: ollama/nomic-embed-text).
Returns:
Embedding vector as list of floats.
"""
model = model or DEFAULT_EMBEDDING_MODEL
truncated = text[:MAX_INPUT_CHARS]
try:
response = await litellm.aembedding(
model=model,
input=truncated,
api_key=OLLAMA_API_KEY,
)
return response.data[0]["embedding"]
except Exception:
logger.exception("Failed to generate embedding")
return []
async def generate_embeddings_batch(
texts: list[str], model: str | None = None
) -> list[list[float]]:
"""Generate embeddings for multiple texts in a single API call.
Args:
texts: List of input texts.
model: Embedding model name (default: ollama/nomic-embed-text).
Returns:
List of embedding vectors.
"""
model = model or DEFAULT_EMBEDDING_MODEL
truncated = [t[:MAX_INPUT_CHARS] for t in texts]
try:
response = await litellm.aembedding(
model=model,
input=truncated,
api_key=OLLAMA_API_KEY,
)
return [d["embedding"] for d in response.data]
except Exception:
logger.exception("Failed to generate batch embeddings")
return [[] for _ in texts]
async def index_entity(
entity_type: str,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
db: AsyncSession,
) -> bool:
"""Generate and store embedding for a single entity.
Uses the provider registry to get embedding text, generates embedding,
and updates the entity's embedding column.
Returns True on success, False on failure.
"""
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
registry = get_search_registry()
provider = registry.get(entity_type)
if provider is None:
logger.warning("No provider for entity_type=%s", entity_type)
return False
try:
text = await provider.get_embedding_text(db, entity_id, tenant_id)
if not text.strip():
logger.debug("Empty embedding text for %s/%s", entity_type, entity_id)
return False
embedding = await generate_embedding(text)
if not embedding:
return False
# Update the entity's embedding column
from sqlalchemy import text as sql_text
table_map = {
"contact": "contacts",
"company": "companies",
"mail": "mails",
"file": "files",
"event": "calendar_entries",
}
table = table_map.get(entity_type)
if not table:
logger.warning("Unknown entity_type=%s for embedding storage", entity_type)
return False
sql = sql_text(
f"UPDATE {table} SET embedding = cast(:emb AS vector) "
f"WHERE id = :eid AND tenant_id = :tid"
)
await db.execute(
sql,
{"emb": str(embedding), "eid": entity_id, "tid": tenant_id},
)
await db.commit()
return True
except Exception:
logger.exception("Failed to index entity %s/%s", entity_type, entity_id)
return False
+246
View File
@@ -0,0 +1,246 @@
"""ARQ background jobs for the Unified Search plugin."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from app.core.db import get_session_factory
logger = logging.getLogger(__name__)
BATCH_SIZE = 100
def _parse_id(id_str: str) -> uuid.UUID:
"""Parse a string to UUID."""
if isinstance(id_str, uuid.UUID):
return id_str
return uuid.UUID(str(id_str))
async def index_mails(ctx: dict[str, Any], mail_ids: list[str]) -> None:
"""Index mails: generate and store embeddings."""
from app.plugins.builtins.unified_search.embedding import index_entity
factory = get_session_factory()
async with factory() as db:
for mail_id in mail_ids:
try:
eid = _parse_id(mail_id)
# tenant_id is derived from the mail itself
from sqlalchemy import text
result = await db.execute(
text("SELECT tenant_id FROM mails WHERE id = :mid"),
{"mid": eid},
)
row = result.mappings().first()
if not row:
continue
tenant_id = row["tenant_id"]
await index_entity("mail", eid, tenant_id, db)
except Exception:
logger.exception("Failed to index mail %s", mail_id)
async def index_file(ctx: dict[str, Any], file_id: str) -> None:
"""Index a file: extract text, store content_text, generate embedding."""
from sqlalchemy import text
from app.plugins.builtins.unified_search.text_extraction import extract_text_from_file
from app.plugins.builtins.unified_search.embedding import generate_embedding
factory = get_session_factory()
async with factory() as db:
try:
eid = _parse_id(file_id)
result = await db.execute(
text("SELECT tenant_id, storage_path, mime_type FROM files WHERE id = :fid"),
{"fid": eid},
)
row = result.mappings().first()
if not row:
logger.warning("File not found: %s", file_id)
return
tenant_id = row["tenant_id"]
storage_path = row["storage_path"]
mime_type = row["mime_type"]
# Extract text
content_text = await extract_text_from_file(storage_path, mime_type)
# Store content_text
await db.execute(
text("UPDATE files SET content_text = :ct WHERE id = :fid"),
{"ct": content_text, "fid": eid},
)
await db.commit()
# Generate embedding from extracted text + filename
result_name = await db.execute(
text("SELECT name FROM files WHERE id = :fid"),
{"fid": eid},
)
name_row = result_name.mappings().first()
name = name_row["name"] if name_row else ""
embedding_text = f"{name} {content_text[:5000]}"
if embedding_text.strip():
embedding = await generate_embedding(embedding_text)
if embedding:
await db.execute(
text("UPDATE files SET embedding = cast(:emb AS vector) WHERE id = :fid"),
{"emb": str(embedding), "fid": eid},
)
await db.commit()
logger.info("Indexed file %s", file_id)
except Exception:
logger.exception("Failed to index file %s", file_id)
async def index_contact(ctx: dict[str, Any], contact_id: str) -> None:
"""Index a contact: generate and store embedding."""
from sqlalchemy import text
from app.plugins.builtins.unified_search.embedding import index_entity
factory = get_session_factory()
async with factory() as db:
try:
eid = _parse_id(contact_id)
result = await db.execute(
text("SELECT tenant_id FROM contacts WHERE id = :cid"),
{"cid": eid},
)
row = result.mappings().first()
if not row:
return
await index_entity("contact", eid, row["tenant_id"], db)
except Exception:
logger.exception("Failed to index contact %s", contact_id)
async def index_company(ctx: dict[str, Any], company_id: str) -> None:
"""Index a company: generate and store embedding."""
from sqlalchemy import text
from app.plugins.builtins.unified_search.embedding import index_entity
factory = get_session_factory()
async with factory() as db:
try:
eid = _parse_id(company_id)
result = await db.execute(
text("SELECT tenant_id FROM companies WHERE id = :cid"),
{"cid": eid},
)
row = result.mappings().first()
if not row:
return
await index_entity("company", eid, row["tenant_id"], db)
except Exception:
logger.exception("Failed to index company %s", company_id)
async def index_event(ctx: dict[str, Any], event_id: str) -> None:
"""Index a calendar event: generate and store embedding."""
from sqlalchemy import text
from app.plugins.builtins.unified_search.embedding import index_entity
factory = get_session_factory()
async with factory() as db:
try:
eid = _parse_id(event_id)
result = await db.execute(
text("SELECT tenant_id FROM calendar_entries WHERE id = :eid"),
{"eid": eid},
)
row = result.mappings().first()
if not row:
return
await index_entity("event", eid, row["tenant_id"], db)
except Exception:
logger.exception("Failed to index event %s", event_id)
async def reindex(ctx: dict[str, Any], entity_type: str) -> None:
"""Reindex all entities of a given type with pagination."""
from sqlalchemy import text
from app.plugins.builtins.unified_search.embedding import index_entity
table_map = {
"contact": "contacts",
"company": "companies",
"mail": "mails",
"file": "files",
"event": "calendar_entries",
}
table = table_map.get(entity_type)
if not table:
logger.warning("Unknown entity_type for reindex: %s", entity_type)
return
factory = get_session_factory()
async with factory() as db:
offset = 0
while True:
result = await db.execute(
text(
f"SELECT id, tenant_id FROM {table} "
f"WHERE deleted_at IS NULL ORDER BY created_at LIMIT :lim OFFSET :off"
),
{"lim": BATCH_SIZE, "off": offset},
)
rows = result.mappings().all()
if not rows:
break
for row in rows:
try:
await index_entity(
entity_type,
row["id"],
row["tenant_id"],
db,
)
except Exception:
logger.exception("Reindex failed for %s/%s", entity_type, row["id"])
offset += BATCH_SIZE
logger.info("Reindex complete for %s", entity_type)
async def embedding_batch(ctx: dict[str, Any]) -> None:
"""Periodic job: find entities without embeddings and index them."""
from sqlalchemy import text
from app.plugins.builtins.unified_search.embedding import index_entity
table_map = {
"contact": "contacts",
"company": "companies",
"mail": "mails",
"file": "files",
"event": "calendar_entries",
}
factory = get_session_factory()
async with factory() as db:
for etype, table in table_map.items():
try:
result = await db.execute(
text(
f"SELECT id, tenant_id FROM {table} "
f"WHERE deleted_at IS NULL AND embedding IS NULL "
f"LIMIT :lim"
),
{"lim": BATCH_SIZE},
)
rows = result.mappings().all()
for row in rows:
try:
await index_entity(etype, row["id"], row["tenant_id"], db)
except Exception:
logger.exception("Batch index failed for %s/%s", etype, row["id"])
except Exception:
logger.exception("Batch query failed for %s", etype)
logger.info("Embedding batch job complete")
@@ -0,0 +1,154 @@
-- Unified Search: pgvector extension, FTS triggers, provider registry
-- pgvector extension
CREATE EXTENSION IF NOT EXISTS vector;
-- ─── Provider Registry Table ───
CREATE TABLE IF NOT EXISTS unified_search_providers (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id UUID NOT NULL,
entity_type VARCHAR(50) NOT NULL,
plugin_name VARCHAR(80) NOT NULL,
is_active BOOLEAN NOT NULL DEFAULT TRUE,
config JSONB NOT NULL DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE(tenant_id, entity_type)
);
CREATE INDEX IF NOT EXISTS ix_usp_tenant ON unified_search_providers(tenant_id);
-- ─── Index Log Table ───
CREATE TABLE IF NOT EXISTS unified_search_index_log (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id UUID NOT NULL,
entity_type VARCHAR(50) NOT NULL,
entity_id UUID NOT NULL,
action VARCHAR(20) NOT NULL,
status VARCHAR(20) NOT NULL DEFAULT 'pending',
error_message TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS ix_usil_tenant ON unified_search_index_log(tenant_id);
CREATE INDEX IF NOT EXISTS ix_usil_entity ON unified_search_index_log(entity_type, entity_id);
-- ─── FTS: Mails ───
ALTER TABLE mails ADD COLUMN IF NOT EXISTS body_tsv tsvector;
CREATE OR REPLACE FUNCTION mails_tsv_trigger() RETURNS trigger AS $$
BEGIN
NEW.body_tsv :=
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.subject, '')), 'A') ||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.from_address, '')), 'B') ||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.to_addresses, '')), 'C') ||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.body_text, '')), 'D');
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS mails_tsv_update ON mails;
CREATE TRIGGER mails_tsv_update
BEFORE INSERT OR UPDATE ON mails
FOR EACH ROW EXECUTE FUNCTION mails_tsv_trigger();
CREATE INDEX IF NOT EXISTS ix_mails_body_tsv ON mails USING gin(body_tsv);
-- ─── FTS: Contacts ───
ALTER TABLE contacts ADD COLUMN IF NOT EXISTS search_tsv tsvector;
CREATE OR REPLACE FUNCTION contacts_tsv_trigger() RETURNS trigger AS $$
BEGIN
NEW.search_tsv :=
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.first_name, '') || ' ' || coalesce(NEW.last_name, '')), 'A') ||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.email, '')), 'B') ||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.phone, '') || ' ' || coalesce(NEW.mobile, '')), 'C') ||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.notes, '')), 'D');
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS contacts_tsv_update ON contacts;
CREATE TRIGGER contacts_tsv_update
BEFORE INSERT OR UPDATE ON contacts
FOR EACH ROW EXECUTE FUNCTION contacts_tsv_trigger();
CREATE INDEX IF NOT EXISTS ix_contacts_search_tsv ON contacts USING gin(search_tsv);
-- ─── FTS: Calendar Entries ───
ALTER TABLE calendar_entries ADD COLUMN IF NOT EXISTS search_tsv tsvector;
CREATE OR REPLACE FUNCTION calendar_entries_tsv_trigger() RETURNS trigger AS $$
BEGIN
NEW.search_tsv :=
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.title, '')), 'A') ||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.description, '')), 'B') ||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.location, '')), 'C');
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS calendar_entries_tsv_update ON calendar_entries;
CREATE TRIGGER calendar_entries_tsv_update
BEFORE INSERT OR UPDATE ON calendar_entries
FOR EACH ROW EXECUTE FUNCTION calendar_entries_tsv_trigger();
CREATE INDEX IF NOT EXISTS ix_cal_entries_search_tsv ON calendar_entries USING gin(search_tsv);
-- ─── FTS: Files (content_text added here, not in 0002, to avoid trigger dependency) ───
ALTER TABLE files ADD COLUMN IF NOT EXISTS content_text text;
ALTER TABLE files ADD COLUMN IF NOT EXISTS content_tsv tsvector;
CREATE OR REPLACE FUNCTION files_tsv_trigger() RETURNS trigger AS $$
BEGIN
NEW.content_tsv :=
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.name, '')), 'A') ||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.content_text, '')), 'D');
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS files_tsv_update ON files;
CREATE TRIGGER files_tsv_update
BEFORE INSERT OR UPDATE ON files
FOR EACH ROW EXECUTE FUNCTION files_tsv_trigger();
CREATE INDEX IF NOT EXISTS ix_files_content_tsv ON files USING gin(content_tsv);
-- ─── FTS: Tags ───
ALTER TABLE tags ADD COLUMN IF NOT EXISTS search_tsv tsvector;
CREATE OR REPLACE FUNCTION tags_tsv_trigger() RETURNS trigger AS $$
BEGIN
NEW.search_tsv := to_tsvector('pg_catalog.german', coalesce(NEW.name, ''));
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS tags_tsv_update ON tags;
CREATE TRIGGER tags_tsv_update
BEFORE INSERT OR UPDATE ON tags
FOR EACH ROW EXECUTE FUNCTION tags_tsv_trigger();
CREATE INDEX IF NOT EXISTS ix_tags_search_tsv ON tags USING gin(search_tsv);
-- ─── FTS: Audit Log ───
ALTER TABLE audit_log ADD COLUMN IF NOT EXISTS search_tsv tsvector;
CREATE OR REPLACE FUNCTION audit_log_tsv_trigger() RETURNS trigger AS $$
BEGIN
NEW.search_tsv :=
to_tsvector('pg_catalog.german',
coalesce(NEW.entity_type, '') || ' ' ||
coalesce(NEW.action, '') || ' ' ||
coalesce(NEW.user_id::text, ''));
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
DROP TRIGGER IF EXISTS audit_log_tsv_update ON audit_log;
CREATE TRIGGER audit_log_tsv_update
BEFORE INSERT OR UPDATE ON audit_log
FOR EACH ROW EXECUTE FUNCTION audit_log_tsv_trigger();
CREATE INDEX IF NOT EXISTS ix_audit_log_search_tsv ON audit_log USING gin(search_tsv);
@@ -0,0 +1,24 @@
-- Unified Search: Embedding columns and HNSW indexes
-- ─── Mails ───
ALTER TABLE mails ADD COLUMN IF NOT EXISTS embedding vector(768);
CREATE INDEX IF NOT EXISTS ix_mails_embedding ON mails USING hnsw(embedding vector_cosine_ops);
-- ─── Contacts ───
ALTER TABLE contacts ADD COLUMN IF NOT EXISTS embedding vector(768);
CREATE INDEX IF NOT EXISTS ix_contacts_embedding ON contacts USING hnsw(embedding vector_cosine_ops);
-- ─── Companies ───
ALTER TABLE companies ADD COLUMN IF NOT EXISTS embedding vector(768);
CREATE INDEX IF NOT EXISTS ix_companies_embedding ON companies USING hnsw(embedding vector_cosine_ops);
-- ─── Files (content_text already added in 0001) ───
ALTER TABLE files ADD COLUMN IF NOT EXISTS embedding vector(768);
CREATE INDEX IF NOT EXISTS ix_files_embedding ON files USING hnsw(embedding vector_cosine_ops);
-- ─── Calendar Entries ───
ALTER TABLE calendar_entries ADD COLUMN IF NOT EXISTS embedding vector(768);
CREATE INDEX IF NOT EXISTS ix_cal_embedding ON calendar_entries USING hnsw(embedding vector_cosine_ops);
-- ─── Tags (shorter dimension for short text) ───
ALTER TABLE tags ADD COLUMN IF NOT EXISTS embedding vector(384);
@@ -0,0 +1,50 @@
"""SQLAlchemy models for the Unified Search plugin."""
from __future__ import annotations
import uuid
from datetime import datetime
from sqlalchemy import DateTime, ForeignKey, Index, String, Text
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
from sqlalchemy.orm import Mapped, mapped_column
from app.core.db import Base, TenantMixin
class SearchProviderRegistry(Base, TenantMixin):
"""Registry of active search providers per tenant."""
__tablename__ = "unified_search_providers"
__table_args__ = (
Index("ix_usp_tenant", "tenant_id"),
)
id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
plugin_name: Mapped[str] = mapped_column(String(80), nullable=False)
is_active: Mapped[bool] = mapped_column(
nullable=False, default=True
)
config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
class SearchIndexLog(Base, TenantMixin):
"""Log of indexing actions for audit and debugging."""
__tablename__ = "unified_search_index_log"
__table_args__ = (
Index("ix_usil_tenant", "tenant_id"),
Index("ix_usil_entity", "entity_type", "entity_id"),
)
id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
entity_id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), nullable=False)
action: Mapped[str] = mapped_column(String(20), nullable=False)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
@@ -0,0 +1,137 @@
"""Unified Search plugin class and manifest."""
from __future__ import annotations
import logging
from typing import Any
from app.plugins.base import BasePlugin
from app.plugins.manifest import PluginManifest, PluginRouteDef
logger = logging.getLogger(__name__)
class UnifiedSearchPlugin(BasePlugin):
"""Hybrid full-text and semantic search across all CRM entities."""
manifest = PluginManifest(
name="unified_search",
version="1.0.0",
display_name="Unified Search",
description=(
"Hybrid full-text (PostgreSQL FTS) and semantic (pgvector) search "
"with KI query understanding and RRF rank fusion across all CRM data."
),
dependencies=[],
routes=[
PluginRouteDef(
path="/api/v1/search",
module="app.plugins.builtins.unified_search.routes",
router_attr="router",
),
],
events=[
"mail.synced",
"file.uploaded",
"contact.created",
"contact.updated",
"company.created",
"company.updated",
"calendar.entry.created",
],
migrations=["0001_initial.sql", "0002_embeddings.sql"],
permissions=["search:read", "search:admin"],
is_core=False,
)
async def on_activate(self, db, service_container, event_bus) -> None:
"""Register search providers on activation."""
await super().on_activate(db, service_container, event_bus)
try:
from app.plugins.builtins.unified_search.provider_registry import (
auto_register_providers,
)
await auto_register_providers(db)
logger.info("Unified Search providers auto-registered")
except Exception:
logger.exception("Failed to auto-register search providers")
async def on_deactivate(self, db, service_container, event_bus) -> None:
"""Clear provider registry on deactivation."""
from app.plugins.builtins.unified_search.provider_registry import (
get_search_registry,
)
registry = get_search_registry()
registry.clear()
await super().on_deactivate(db, service_container, event_bus)
# ─── Event Handlers ───
async def on_mail_synced(self, payload: dict[str, Any]) -> None:
"""Enqueue embedding jobs for synced mails."""
from app.core.jobs import enqueue_job
mail_ids = payload.get("mail_ids", [])
if mail_ids:
await enqueue_job("index_mails", mail_ids)
async def on_file_uploaded(self, payload: dict[str, Any]) -> None:
"""Enqueue file indexing job."""
from app.core.jobs import enqueue_job
file_id = payload.get("file_id")
if file_id:
await enqueue_job("index_file", file_id)
async def on_contact_created(self, payload: dict[str, Any]) -> None:
from app.core.jobs import enqueue_job
contact_id = payload.get("contact_id")
if contact_id:
await enqueue_job("index_contact", contact_id)
async def on_contact_updated(self, payload: dict[str, Any]) -> None:
from app.core.jobs import enqueue_job
contact_id = payload.get("contact_id")
if contact_id:
await enqueue_job("index_contact", contact_id)
async def on_company_created(self, payload: dict[str, Any]) -> None:
from app.core.jobs import enqueue_job
company_id = payload.get("company_id")
if company_id:
await enqueue_job("index_company", company_id)
async def on_company_updated(self, payload: dict[str, Any]) -> None:
from app.core.jobs import enqueue_job
company_id = payload.get("company_id")
if company_id:
await enqueue_job("index_company", company_id)
async def on_calendar_entry_created(self, payload: dict[str, Any]) -> None:
from app.core.jobs import enqueue_job
entry_id = payload.get("entry_id")
if entry_id:
await enqueue_job("index_event", entry_id)
def get_notification_types(self) -> list[dict[str, Any]]:
return [
{
"type_key": "search_error",
"category": "search",
"label": "Suchfehler",
"description": "Fehler bei der Suchausführung",
"is_enabled_by_default": True,
},
{
"type_key": "search_reindex_complete",
"category": "search",
"label": "Reindex abgeschlossen",
"description": "Neuindizierung abgeschlossen",
"is_enabled_by_default": False,
},
]
@@ -0,0 +1,133 @@
"""SearchProvider protocol and global registry."""
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Protocol, runtime_checkable
from sqlalchemy import select
if TYPE_CHECKING:
import uuid
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
@runtime_checkable
class SearchProvider(Protocol):
"""Protocol for entity-specific search providers."""
entity_type: str
async def search_fts(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
) -> list[dict]:
"""Full-text search using PostgreSQL tsquery."""
...
async def search_vector(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
) -> list[dict]:
"""Semantic vector search using pgvector."""
...
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> str:
"""Get text representation for embedding generation."""
...
def to_search_result(self, entity: object) -> dict:
"""Convert an ORM entity to a search result dict."""
...
class SearchProviderRegistry:
"""In-memory registry of search providers."""
def __init__(self) -> None:
self._providers: dict[str, SearchProvider] = {}
def register(self, provider: SearchProvider) -> None:
"""Register a search provider."""
self._providers[provider.entity_type] = provider
logger.debug("Registered search provider: %s", provider.entity_type)
def unregister(self, entity_type: str) -> None:
"""Unregister a search provider by entity type."""
self._providers.pop(entity_type, None)
def get(self, entity_type: str) -> SearchProvider | None:
"""Get a provider by entity type."""
return self._providers.get(entity_type)
def get_all(self) -> list[SearchProvider]:
"""Get all registered providers."""
return list(self._providers.values())
def get_entity_types(self) -> list[str]:
"""Get all registered entity types."""
return list(self._providers.keys())
def clear(self) -> None:
"""Clear all registered providers."""
self._providers.clear()
# Global singleton
_registry = SearchProviderRegistry()
def get_search_registry() -> SearchProviderRegistry:
"""Get the global search provider registry."""
return _registry
async def auto_register_providers(db: AsyncSession) -> None:
"""Auto-register providers for active plugins.
Checks which plugins are active and registers corresponding providers.
"""
from app.plugins.builtins.unified_search.providers.contact_provider import (
ContactSearchProvider,
)
from app.plugins.builtins.unified_search.providers.company_provider import (
CompanySearchProvider,
)
from app.plugins.builtins.unified_search.providers.mail_provider import (
MailSearchProvider,
)
from app.plugins.builtins.unified_search.providers.file_provider import (
FileSearchProvider,
)
from app.plugins.builtins.unified_search.providers.event_provider import (
EventSearchProvider,
)
registry = get_search_registry()
registry.clear()
# Register all built-in providers
for provider_cls in [
ContactSearchProvider,
CompanySearchProvider,
MailSearchProvider,
FileSearchProvider,
EventSearchProvider,
]:
try:
registry.register(provider_cls())
except Exception:
logger.exception("Failed to register %s", provider_cls.__name__)
@@ -0,0 +1,3 @@
"""Search providers for unified search."""
from __future__ import annotations
@@ -0,0 +1,114 @@
"""Company search provider."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
class CompanySearchProvider:
"""Search provider for Company entities."""
entity_type = "company"
async def search_fts(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Full-text search on companies.search_tsv."""
sql = text(
"""
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM companies c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Semantic search on companies.embedding."""
sql = text(
"""
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
FROM companies c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.embedding IS NOT NULL
ORDER BY c.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> str:
"""Get text for embedding generation."""
sql = text(
"""
SELECT name, description, industry
FROM companies
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
parts = [
row.get("name", ""),
row.get("description", ""),
row.get("industry", ""),
]
return " ".join(str(p) for p in parts if p)
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert company to search result dict."""
if isinstance(entity, dict):
name = entity.get("name", "")
description = entity.get("description", "") or ""
entity_id = str(entity.get("id", ""))
else:
name = getattr(entity, "name", "")
description = getattr(entity, "description", "") or ""
entity_id = str(getattr(entity, "id", ""))
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": name,
"snippet": description[:200],
"score": 0.0,
"data": {},
}
@@ -0,0 +1,121 @@
"""Contact search provider."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.contact import Contact
logger = logging.getLogger(__name__)
class ContactSearchProvider:
"""Search provider for Contact entities."""
entity_type = "contact"
async def search_fts(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Full-text search on contacts.search_tsv."""
sql = text(
"""
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Semantic search on contacts.embedding."""
sql = text(
"""
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.embedding IS NOT NULL
ORDER BY c.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> str:
"""Get text for embedding generation."""
sql = text(
"""
SELECT first_name, last_name, email, phone, mobile, notes
FROM contacts
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
parts = [
row.get("first_name", ""),
row.get("last_name", ""),
row.get("email", ""),
row.get("phone", ""),
row.get("mobile", ""),
row.get("notes", ""),
]
return " ".join(str(p) for p in parts if p)
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert contact to search result dict."""
if isinstance(entity, dict):
first = entity.get("first_name", "")
last = entity.get("last_name", "")
email = entity.get("email", "")
entity_id = str(entity.get("id", ""))
else:
first = getattr(entity, "first_name", "")
last = getattr(entity, "last_name", "")
email = getattr(entity, "email", "")
entity_id = str(getattr(entity, "id", ""))
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": f"{first} {last}".strip(),
"snippet": email or "",
"score": 0.0,
"data": {},
}
@@ -0,0 +1,114 @@
"""Calendar event search provider."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
class EventSearchProvider:
"""Search provider for CalendarEntry entities."""
entity_type = "event"
async def search_fts(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Full-text search on calendar_entries.search_tsv."""
sql = text(
"""
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Semantic search on calendar_entries.embedding."""
sql = text(
"""
SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.embedding IS NOT NULL
ORDER BY e.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> str:
"""Get text for embedding generation."""
sql = text(
"""
SELECT title, description, location
FROM calendar_entries
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
parts = [
row.get("title", ""),
row.get("description", ""),
row.get("location", ""),
]
return " ".join(str(p) for p in parts if p)
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert calendar entry to search result dict."""
if isinstance(entity, dict):
title = entity.get("title", "")
description = entity.get("description", "") or ""
entity_id = str(entity.get("id", ""))
else:
title = getattr(entity, "title", "")
description = getattr(entity, "description", "") or ""
entity_id = str(getattr(entity, "id", ""))
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": title,
"snippet": description[:200],
"score": 0.0,
"data": {},
}
@@ -0,0 +1,111 @@
"""DMS File search provider."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
class FileSearchProvider:
"""Search provider for DMS File entities."""
entity_type = "file"
async def search_fts(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Full-text search on files.content_tsv."""
sql = text(
"""
SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Semantic search on files.embedding."""
sql = text(
"""
SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.embedding IS NOT NULL
ORDER BY f.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> str:
"""Get text for embedding generation."""
sql = text(
"""
SELECT name, content_text
FROM files
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
name = row.get("name", "") or ""
content = row.get("content_text", "") or ""
return f"{name} {content[:5000]}"
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert file to search result dict."""
if isinstance(entity, dict):
name = entity.get("name", "")
content = entity.get("content_text", "") or ""
entity_id = str(entity.get("id", ""))
else:
name = getattr(entity, "name", "")
content = getattr(entity, "content_text", "") or ""
entity_id = str(getattr(entity, "id", ""))
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": name,
"snippet": content[:200],
"score": 0.0,
"data": {},
}
@@ -0,0 +1,111 @@
"""Mail search provider."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
class MailSearchProvider:
"""Search provider for Mail entities."""
entity_type = "mail"
async def search_fts(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Full-text search on mails.body_tsv."""
sql = text(
"""
SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
) -> list[dict[str, Any]]:
"""Semantic search on mails.embedding."""
sql = text(
"""
SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.embedding IS NOT NULL
ORDER BY m.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> str:
"""Get text for embedding generation."""
sql = text(
"""
SELECT subject, body_text
FROM mails
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
subject = row.get("subject", "") or ""
body = row.get("body_text", "") or ""
return f"{subject} {body[:5000]}"
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert mail to search result dict."""
if isinstance(entity, dict):
subject = entity.get("subject", "")
body = entity.get("body_text", "") or ""
entity_id = str(entity.get("id", ""))
else:
subject = getattr(entity, "subject", "")
body = getattr(entity, "body_text", "") or ""
entity_id = str(getattr(entity, "id", ""))
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": subject,
"snippet": body[:200],
"score": 0.0,
"data": {},
}
@@ -0,0 +1,101 @@
"""KI query understanding and result aggregation via LiteLLM."""
from __future__ import annotations
import os
import json
import logging
from typing import Any
import litellm
logger = logging.getLogger(__name__)
DEFAULT_LLM_MODEL = os.environ.get('SEARCH_LLM_MODEL', 'ollama/deepseek-v4')
OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
QUERY_ANALYZE_SYSTEM = (
"Du bist ein Query-Analyzer fuer ein CRM. "
"Analysiere die Suchanfrage und gib JSON zurueck: "
'{"normalized_query": str, "entities": {"person": str|null, "company": str|null, "topic": str|null}, '
'"intent": str, "semantic_terms": [str], "suggested_filters": {}}'
)
RESULT_AGGREGATE_SYSTEM = (
"Du bist ein Result-Aggregator. Fasse Ergebnisse zusammen und generiere Facetten: "
'{"summary": str, "facets": {"types": {}, "dates": {}, "people": []}, "suggestions": [str]}'
)
def _fallback_query_analysis(query: str) -> dict[str, Any]:
return {
"normalized_query": query,
"entities": {},
"intent": "search",
"semantic_terms": [],
"suggested_filters": {},
}
def _fallback_aggregate(results: list[dict], query: str) -> dict[str, Any]:
return {
"summary": f"{len(results)} Ergebnisse gefunden",
"facets": {},
"suggestions": [],
}
async def llm_analyze_query(query: str) -> dict[str, Any]:
"""Analyze a search query using LLM for intent, entities, and semantic terms.
Falls back to a simple dict if LLM fails.
"""
try:
response = await litellm.acompletion(
model=DEFAULT_LLM_MODEL,
messages=[
{"role": "system", "content": QUERY_ANALYZE_SYSTEM},
{"role": "user", "content": query},
],
temperature=0.1,
max_tokens=500,
response_format={"type": "json_object"},
api_key=OLLAMA_API_KEY,
)
content = response.choices[0].message.content
return json.loads(content)
except Exception:
logger.warning("LLM query analysis failed, using fallback")
return _fallback_query_analysis(query)
async def llm_aggregate_results(results: list[dict], query: str) -> dict[str, Any]:
"""Aggregate search results using LLM for summary, facets, and suggestions.
Falls back to a simple dict if LLM fails.
"""
if not results:
return _fallback_aggregate(results, query)
try:
# Truncate results to avoid token overflow
compact = [
{"entity_type": r.get("entity_type"), "title": r.get("title", "")[:100]}
for r in results[:50]
]
user_msg = json.dumps({"query": query, "results": compact})
response = await litellm.acompletion(
model=DEFAULT_LLM_MODEL,
messages=[
{"role": "system", "content": RESULT_AGGREGATE_SYSTEM},
{"role": "user", "content": user_msg},
],
temperature=0.1,
max_tokens=1000,
response_format={"type": "json_object"},
api_key=OLLAMA_API_KEY,
)
content = response.choices[0].message.content
return json.loads(content)
except Exception:
logger.warning("LLM result aggregation failed, using fallback")
return _fallback_aggregate(results, query)
@@ -0,0 +1,316 @@
"""API routes for the Unified Search plugin."""
from __future__ import annotations
import logging
import uuid
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.db import get_db
from app.core.jobs import enqueue_job
from app.deps import get_current_user, require_permission
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
from app.plugins.builtins.unified_search.query_understanding import (
llm_aggregate_results,
llm_analyze_query,
)
from app.plugins.builtins.unified_search.schemas import (
ProviderResponse,
ReindexRequest,
SearchRequest,
SearchResponse,
SearchResult,
SimilarRequest,
SimilarResponse,
SuggestRequest,
SuggestResponse,
)
from app.plugins.builtins.unified_search.search_engine import (
autocomplete,
find_similar_all_types,
hybrid_search,
)
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/v1/search", tags=["search"])
# ─── Search ───
@router.post("", dependencies=[Depends(require_permission("search:read"))])
async def search(
req: SearchRequest,
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> SearchResponse:
"""Perform hybrid search with KI query understanding."""
tenant_id = uuid.UUID(current_user["tenant_id"])
# KI query understanding
query_analysis = await llm_analyze_query(req.query)
# Hybrid search
results = await hybrid_search(
db=db,
query_analysis=query_analysis,
tenant_id=tenant_id,
entity_types=req.entity_types,
limit=req.limit,
)
# KI result aggregation
aggregation = await llm_aggregate_results(results, req.query)
search_results = [
SearchResult(
entity_type=r.get("entity_type", ""),
entity_id=r.get("entity_id", ""),
title=r.get("title", ""),
snippet=r.get("snippet", ""),
score=r.get("score", 0.0),
data=r.get("data", {}),
)
for r in results
]
return SearchResponse(
query=req.query,
normalized_query=query_analysis.get("normalized_query", req.query),
results=search_results,
facets=aggregation.get("facets", {}),
summary=aggregation.get("summary", f"{len(results)} Ergebnisse"),
suggestions=aggregation.get("suggestions", []),
)
# ─── Suggest / Autocomplete ───
@router.get("/suggest", dependencies=[Depends(require_permission("search:read"))])
async def suggest(
q: str = Query(..., min_length=1, max_length=200),
limit: int = Query(default=10, ge=1, le=50),
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> SuggestResponse:
"""Autocomplete suggestions using FTS prefix search."""
tenant_id = uuid.UUID(current_user["tenant_id"])
suggestions = await autocomplete(db, q, tenant_id, limit)
return SuggestResponse(suggestions=suggestions)
# ─── Similar ───
@router.post("/similar", dependencies=[Depends(require_permission("search:read"))])
async def find_similar(
req: SimilarRequest,
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> SimilarResponse:
"""Find similar entities across all types based on embedding."""
tenant_id = uuid.UUID(current_user["tenant_id"])
try:
entity_id = uuid.UUID(req.entity_id)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid entity_id")
similar = await find_similar_all_types(
db=db,
entity_type=req.entity_type,
entity_id=entity_id,
tenant_id=tenant_id,
limit=req.limit,
)
result_dict: dict[str, list[SearchResult]] = {}
for etype, items in similar.items():
result_dict[etype] = [
SearchResult(
entity_type=r.get("entity_type", etype),
entity_id=r.get("entity_id", ""),
title=r.get("title", ""),
snippet=r.get("snippet", ""),
score=r.get("score", 0.0),
data=r.get("data", {}),
)
for r in items
]
return SimilarResponse(similar=result_dict)
# ─── Reindex ───
@router.post("/reindex", dependencies=[Depends(require_permission("search:admin"))])
async def reindex(
req: ReindexRequest,
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> dict:
"""Trigger reindexing of specified entity types."""
entity_types = req.entity_types
if not entity_types:
entity_types = get_search_registry().get_entity_types()
job_ids: list[str] = []
for etype in entity_types:
job_id = await enqueue_job("reindex", etype)
if job_id:
job_ids.append(job_id)
return {
"status": "ok",
"message": f"Reindexing {len(entity_types)} entity types",
"entity_types": entity_types,
"job_ids": job_ids,
}
# ─── Providers ───
@router.get("/providers", dependencies=[Depends(require_permission("search:read"))])
async def list_providers(
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> list[ProviderResponse]:
"""List all active search providers."""
registry = get_search_registry()
providers = registry.get_all()
return [
ProviderResponse(
entity_type=p.entity_type,
plugin_name="unified_search",
is_active=True,
)
for p in providers
]
@router.post("/providers/{entity_type}/toggle", dependencies=[Depends(require_permission("search:admin"))])
async def toggle_provider(
entity_type: str,
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> dict:
"""Toggle a search provider on/off."""
registry = get_search_registry()
provider = registry.get(entity_type)
if provider is None:
raise HTTPException(status_code=404, detail=f"Provider not found: {entity_type}")
# Toggle in DB
tenant_id = uuid.UUID(current_user["tenant_id"])
result = await db.execute(
text(
"""
SELECT is_active FROM unified_search_providers
WHERE tenant_id = :tid AND entity_type = :et
"""
),
{"tid": tenant_id, "et": entity_type},
)
row = result.mappings().first()
if row:
new_status = not row["is_active"]
await db.execute(
text(
"""
UPDATE unified_search_providers SET is_active = :active
WHERE tenant_id = :tid AND entity_type = :et
"""
),
{"active": new_status, "tid": tenant_id, "et": entity_type},
)
else:
new_status = False
await db.execute(
text(
"""
INSERT INTO unified_search_providers (tenant_id, entity_type, plugin_name, is_active)
VALUES (:tid, :et, 'unified_search', false)
"""
),
{"tid": tenant_id, "et": entity_type},
)
await db.commit()
if not new_status:
registry.unregister(entity_type)
else:
# Re-register by clearing and re-running auto_register
from app.plugins.builtins.unified_search.provider_registry import auto_register_providers
await auto_register_providers(db)
return {
"entity_type": entity_type,
"is_active": new_status,
}
# ─── Stats ───
@router.get("/stats", dependencies=[Depends(require_permission("search:read"))])
async def search_stats(
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> dict:
"""Get search index statistics."""
tenant_id = uuid.UUID(current_user["tenant_id"])
stats: dict = {}
tables = {
"contacts": "embedding",
"companies": "embedding",
"mails": "embedding",
"files": "embedding",
"calendar_entries": "embedding",
}
for table, col in tables.items():
try:
total_result = await db.execute(
text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL"),
{"tid": tenant_id},
)
total = total_result.scalar() or 0
indexed_result = await db.execute(
text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL AND {col} IS NOT NULL"),
{"tid": tenant_id},
)
indexed = indexed_result.scalar() or 0
stats[table] = {
"total": total,
"indexed": indexed,
"pending": total - indexed,
}
except Exception:
logger.exception("Stats query failed for %s", table)
stats[table] = {"total": 0, "indexed": 0, "pending": 0}
# Last index log entries
try:
log_result = await db.execute(
text(
"""
SELECT entity_type, action, status, created_at
FROM unified_search_index_log
WHERE tenant_id = :tid
ORDER BY created_at DESC
LIMIT 10
"""
),
{"tid": tenant_id},
)
recent_logs = [dict(r) for r in log_result.mappings().all()]
except Exception:
recent_logs = []
stats["recent_logs"] = recent_logs
return stats
@@ -0,0 +1,71 @@
"""Pydantic schemas for the Unified Search plugin."""
from __future__ import annotations
from typing import Any
from pydantic import BaseModel, Field
# ─── Search ───
class SearchRequest(BaseModel):
query: str = Field(..., min_length=1, max_length=500)
entity_types: list[str] | None = None
limit: int = Field(default=20, ge=1, le=100)
offset: int = Field(default=0, ge=0)
class SearchResult(BaseModel):
entity_type: str
entity_id: str
title: str
snippet: str
score: float
data: dict[str, Any] = Field(default_factory=dict)
class SearchResponse(BaseModel):
query: str
normalized_query: str
results: list[SearchResult]
facets: dict[str, Any]
summary: str
suggestions: list[str]
# ─── Similar ───
class SimilarRequest(BaseModel):
entity_type: str
entity_id: str
limit: int = Field(default=5, ge=1, le=50)
class SimilarResponse(BaseModel):
similar: dict[str, list[SearchResult]]
# ─── Suggest ───
class SuggestRequest(BaseModel):
q: str = Field(..., min_length=1, max_length=200)
limit: int = Field(default=10, ge=1, le=50)
class SuggestResponse(BaseModel):
suggestions: list[str]
# ─── Reindex ───
class ReindexRequest(BaseModel):
entity_types: list[str] = Field(default_factory=list)
# ─── Provider ───
class ProviderResponse(BaseModel):
entity_type: str
plugin_name: str
is_active: bool
@@ -0,0 +1,252 @@
"""Hybrid search engine: FTS + pgvector with RRF fusion."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.embedding import generate_embedding
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
logger = logging.getLogger(__name__)
# Entity type -> (table_name, tsv_column, embedding_column)
SEARCHABLE_ENTITIES: dict[str, tuple[str, str, str]] = {
"contact": ("contacts", "search_tsv", "embedding"),
"company": ("companies", "search_tsv", "embedding"),
"mail": ("mails", "body_tsv", "embedding"),
"file": ("files", "content_tsv", "embedding"),
"event": ("calendar_entries", "search_tsv", "embedding"),
}
RRF_K = 60
RRF_ALPHA = 0.5
RRF_BETA = 0.5
def rrf_fusion(
fts_results: list[dict[str, Any]],
vec_results: list[dict[str, Any]],
entity_type: str,
k: int = RRF_K,
alpha: float = RRF_ALPHA,
beta: float = RRF_BETA,
) -> list[dict[str, Any]]:
"""Reciprocal Rank Fusion of FTS and vector search results.
score = alpha * (1/(k+rank_fts)) + beta * (1/(k+rank_vec))
"""
fused: dict[str, dict[str, Any]] = {}
for rank, item in enumerate(fts_results):
eid = str(item.get("id", ""))
if not eid:
continue
rrf_score = alpha * (1.0 / (k + rank + 1))
if eid not in fused:
fused[eid] = {**item, "_score": 0.0, "_entity_type": entity_type}
fused[eid]["_score"] += rrf_score
for rank, item in enumerate(vec_results):
eid = str(item.get("id", ""))
if not eid:
continue
rrf_score = beta * (1.0 / (k + rank + 1))
if eid not in fused:
fused[eid] = {**item, "_score": 0.0, "_entity_type": entity_type}
fused[eid]["_score"] += rrf_score
return sorted(fused.values(), key=lambda x: x.get("_score", 0.0), reverse=True)
async def hybrid_search(
db: AsyncSession,
query_analysis: dict[str, Any],
tenant_id: uuid.UUID,
entity_types: list[str] | None = None,
limit: int = 20,
) -> list[dict[str, Any]]:
"""Perform hybrid search across all entity types.
For each entity: FTS search + vector search + RRF fusion.
Merge all results, sort by fused score, return top limit.
"""
registry = get_search_registry()
all_entity_types = entity_types or registry.get_entity_types()
normalized_query = query_analysis.get("normalized_query", "")
semantic_terms = query_analysis.get("semantic_terms", [])
# Build tsquery string
tsquery_parts = [normalized_query] + semantic_terms
tsquery = " & ".join(
part.strip().replace(" ", " & ")
for part in tsquery_parts
if part and part.strip()
)
if not tsquery:
tsquery = normalized_query
# Generate query embedding
query_text = normalized_query
if semantic_terms:
query_text = f"{normalized_query} {' '.join(semantic_terms)}"
query_embedding = await generate_embedding(query_text)
all_results: list[dict[str, Any]] = []
fetch_limit = limit * 2
for entity_type in all_entity_types:
provider = registry.get(entity_type)
if provider is None:
continue
fts_results: list[dict[str, Any]] = []
vec_results: list[dict[str, Any]] = []
try:
fts_results = await provider.search_fts(db, tsquery, tenant_id, fetch_limit)
except Exception:
logger.exception("FTS search failed for %s", entity_type)
if query_embedding:
try:
vec_results = await provider.search_vector(db, query_embedding, tenant_id, fetch_limit)
except Exception:
logger.exception("Vector search failed for %s", entity_type)
fused = rrf_fusion(fts_results, vec_results, entity_type)
# Convert to search result format
for item in fused:
result = provider.to_search_result(item)
result["score"] = item.get("_score", 0.0)
all_results.append(result)
all_results.sort(key=lambda x: x.get("score", 0.0), reverse=True)
return all_results[:limit]
async def find_similar_all_types(
db: AsyncSession,
entity_type: str,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
limit: int = 5,
) -> dict[str, list[dict[str, Any]]]:
"""Find similar entities across all types based on embedding.
Gets the embedding of the source entity, then searches all other tables.
"""
table_map = SEARCHABLE_ENTITIES
source_info = table_map.get(entity_type)
if not source_info:
return {}
table_name, _, emb_col = source_info
# Get source embedding
sql = text(
f"SELECT {emb_col} AS embedding FROM {table_name} "
f"WHERE id = :eid AND tenant_id = :tid"
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row or not row.get("embedding"):
return {}
source_embedding_str = str(row["embedding"])
registry = get_search_registry()
similar: dict[str, list[dict[str, Any]]] = {}
for etype, (tbl, _, emb) in table_map.items():
if etype == entity_type:
continue
provider = registry.get(etype)
if provider is None:
continue
try:
sql_sim = text(
f"""
SELECT *, 1 - ({emb} <=> cast(:emb_str AS vector)) AS score
FROM {tbl}
WHERE tenant_id = :tid
AND deleted_at IS NULL
AND {emb} IS NOT NULL
ORDER BY {emb} <=> cast(:emb_str AS vector)
LIMIT :lim
"""
)
res = await db.execute(
sql_sim,
{"emb_str": source_embedding_str, "tid": tenant_id, "lim": limit},
)
rows = res.mappings().all()
results = []
for r in rows:
sr = provider.to_search_result(dict(r))
sr["score"] = float(r.get("score", 0.0))
results.append(sr)
similar[etype] = results
except Exception:
logger.exception("Similar search failed for %s", etype)
similar[etype] = []
return similar
async def autocomplete(
db: AsyncSession,
query: str,
tenant_id: uuid.UUID,
limit: int = 10,
) -> list[str]:
"""Autocomplete using FTS prefix search.
Uses to_tsquery with :* suffix for prefix matching.
"""
query_clean = query.strip().replace(" ", " & ")
if not query_clean:
return []
tsquery = f"{query_clean}:*"
suggestions: list[str] = []
registry = get_search_registry()
for entity_type in registry.get_entity_types():
provider = registry.get(entity_type)
if provider is None:
continue
entity_info = SEARCHABLE_ENTITIES.get(entity_type)
if not entity_info:
continue
table_name, tsv_col, _ = entity_info
try:
sql = text(
f"""
SELECT * FROM {table_name}
WHERE tenant_id = :tid
AND deleted_at IS NULL
AND {tsv_col} @@ to_tsquery('pg_catalog.german', :q)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
for r in rows:
sr = provider.to_search_result(dict(r))
title = sr.get("title", "")
if title and title not in suggestions:
suggestions.append(title)
except Exception:
logger.debug("Autocomplete failed for %s", entity_type)
return suggestions[:limit]
@@ -0,0 +1,126 @@
"""Text extraction from various file formats for DMS indexing."""
from __future__ import annotations
import logging
from typing import Any
logger = logging.getLogger(__name__)
MAX_CHARS = 50_000
async def extract_text_from_file(file_path: str, mime_type: str) -> str:
"""Extract text content from a file based on its MIME type.
Supports:
- PDF (via PyMuPDF/fitz)
- DOCX (via python-docx)
- XLSX (via openpyxl)
- PPTX (via python-pptx)
- text/* (via aiofiles)
- Images: returns empty (OCR not implemented)
Args:
file_path: Absolute path to the file.
mime_type: MIME type of the file.
Returns:
Extracted text, truncated to MAX_CHARS. Empty string on error.
"""
try:
if mime_type == "application/pdf":
return await _extract_pdf(file_path)
elif mime_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
return await _extract_docx(file_path)
elif mime_type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
return await _extract_xlsx(file_path)
elif mime_type == "application/vnd.openxmlformats-officedocument.presentationml.presentation":
return await _extract_pptx(file_path)
elif mime_type.startswith("text/"):
return await _extract_text(file_path)
elif mime_type.startswith("image/"):
return ""
else:
logger.debug("Unsupported mime_type=%s for text extraction", mime_type)
return ""
except Exception:
logger.warning("Failed to extract text from %s (mime=%s)", file_path, mime_type)
return ""
def _truncate(text: str) -> str:
"""Truncate text to MAX_CHARS with indicator."""
if len(text) > MAX_CHARS:
return text[:MAX_CHARS] + "... [truncated]"
return text
async def _extract_pdf(file_path: str) -> str:
"""Extract text from PDF using PyMuPDF (fitz)."""
import fitz # PyMuPDF
doc = fitz.open(file_path)
parts: list[str] = []
for page in doc:
parts.append(page.get_text())
doc.close()
return _truncate("\n".join(parts))
async def _extract_docx(file_path: str) -> str:
"""Extract text from DOCX using python-docx."""
from docx import Document
doc = Document(file_path)
parts: list[str] = []
for para in doc.paragraphs:
if para.text.strip():
parts.append(para.text)
# Also extract table text
for table in doc.tables:
for row in table.rows:
for cell in row.cells:
if cell.text.strip():
parts.append(cell.text)
return _truncate("\n".join(parts))
async def _extract_xlsx(file_path: str) -> str:
"""Extract text from XLSX using openpyxl."""
from openpyxl import load_workbook
wb = load_workbook(file_path, read_only=True, data_only=True)
parts: list[str] = []
for ws in wb.worksheets:
for row in ws.iter_rows(values_only=True):
row_text = " ".join(str(c) for c in row if c is not None)
if row_text.strip():
parts.append(row_text)
wb.close()
return _truncate("\n".join(parts))
async def _extract_pptx(file_path: str) -> str:
"""Extract text from PPTX using python-pptx."""
from pptx import Presentation
prs = Presentation(file_path)
parts: list[str] = []
for slide in prs.slides:
for shape in slide.shapes:
if shape.has_text_frame:
for para in shape.text_frame.paragraphs:
text = para.text.strip()
if text:
parts.append(text)
return _truncate("\n".join(parts))
async def _extract_text(file_path: str) -> str:
"""Read plain text file using aiofiles."""
import aiofiles
async with aiofiles.open(file_path, mode="r", encoding="utf-8", errors="replace") as f:
content = await f.read()
return _truncate(content)
+72
View File
@@ -0,0 +1,72 @@
import { useState, useEffect, useCallback } from 'react';
import { apiClient } from '@/api/client';
export interface Suggestion {
id: string;
entity_type: string;
entity_id: string | null;
suggestion_type: 'info' | 'warning' | 'action' | 'insight';
title: string;
content: string;
confidence: number;
actions: Array<{ method: string; path: string; body: any; description: string }>;
created_at: string;
is_dismissed: boolean;
is_acted_upon: boolean;
}
export function useSuggestions() {
const [suggestions, setSuggestions] = useState<Suggestion[]>([]);
const [connected, setConnected] = useState(false);
useEffect(() => {
// Initial load
apiClient.get('/api/v1/ai-proactive/suggestions').then(r => {
setSuggestions(r.data.items);
}).catch(() => {});
// SSE Stream
const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream');
eventSource.onopen = () => setConnected(true);
eventSource.onerror = () => setConnected(false);
eventSource.onmessage = (e) => {
try {
const suggestion = JSON.parse(e.data);
setSuggestions(prev => [suggestion, ...prev].slice(0, 50));
} catch {}
};
return () => eventSource.close();
}, []);
const dismiss = useCallback(async (id: string) => {
setSuggestions(prev => prev.filter(s => s.id !== id));
await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/dismiss`).catch(() => {});
}, []);
const act = useCallback(async (id: string, actionIndex: number) => {
const result = await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/act`, {
action_index: actionIndex
});
setSuggestions(prev => prev.map(s => s.id === id ? { ...s, is_acted_upon: true } : s));
return result.data;
}, []);
return { suggestions, connected, dismiss, act };
}
export function useProactiveSettings() {
const [settings, setSettings] = useState<any>(null);
useEffect(() => {
apiClient.get('/api/v1/ai-proactive/settings').then(r => setSettings(r.data));
}, []);
const update = useCallback(async (updates: any) => {
const r = await apiClient.put('/api/v1/ai-proactive/settings', updates);
setSettings(r.data);
return r.data;
}, []);
return { settings, update };
}
+3 -39
View File
@@ -388,45 +388,9 @@ export function useGlobalSearch(query: string, entityTypes?: string[]) {
queryKey: ['globalSearch', query, entityTypes],
queryFn: async (): Promise<SearchResult[]> => {
if (!query.trim()) return [];
const results: SearchResult[] = [];
const types = entityTypes && entityTypes.length > 0 ? entityTypes : ['company', 'contact'];
const tasks: Promise<void>[] = [];
if (types.includes('company')) {
tasks.push(
apiGet<PaginatedResponse<Company>>(`/companies?page=1&page_size=10&search=${encodeURIComponent(query)}`)
.then((data) => {
for (const item of data.items) {
results.push({
type: 'company',
id: item.id,
name: item.name,
description: item.industry || item.email || '',
url: `/companies/${item.id}`,
});
}
})
.catch(() => {})
);
}
if (types.includes('contact')) {
tasks.push(
apiGet<PaginatedResponse<Contact>>(`/contacts?page=1&page_size=10&search=${encodeURIComponent(query)}`)
.then((data) => {
for (const item of data.items) {
results.push({
type: 'contact',
id: item.id,
name: `${item.first_name} ${item.last_name}`,
description: item.email || item.phone || '',
url: `/contacts/${item.id}`,
});
}
})
.catch(() => {})
);
}
await Promise.all(tasks);
return results;
const { search } = await import('@/api/search');
const response = await search(query, entityTypes, 20);
return response.results || [];
},
enabled: query.trim().length > 0,
staleTime: 30 * 1000,
+38
View File
@@ -0,0 +1,38 @@
import { apiClient } from './client';
export interface SearchResult {
type: 'company' | 'contact' | 'mail' | 'file' | 'event';
id: string;
name: string;
description?: string;
url: string;
}
export interface SearchResponse {
results: SearchResult[];
facets?: Record<string, Array<{ value: string; count: number }>>;
summary?: string;
suggestions?: string[];
}
export async function search(query: string, entityTypes?: string[], limit = 20): Promise<SearchResponse> {
const r = await apiClient.post('/api/v1/search', {
query,
entity_types: entityTypes,
limit,
});
return r.data;
}
export async function searchSuggest(q: string): Promise<string[]> {
const r = await apiClient.get('/api/v1/search/suggest', { params: { q } });
return r.data.suggestions;
}
export async function searchSimilar(entityType: string, entityId: string): Promise<SearchResult[]> {
const r = await apiClient.post('/api/v1/search/similar', {
entity_type: entityType,
entity_id: entityId,
});
return r.data.similar;
}
@@ -0,0 +1,59 @@
import { useState, useEffect } from 'react';
import { apiClient } from '@/api/client';
interface SuggestionBadgeProps {
onClick: () => void;
}
export function SuggestionBadge({ onClick }: SuggestionBadgeProps) {
const [count, setCount] = useState(0);
const [pulsing, setPulsing] = useState(false);
useEffect(() => {
// Initial count
apiClient.get('/api/v1/ai-proactive/suggestions').then(r => {
setCount(r.data.items.length);
}).catch(() => {});
// SSE for real-time updates
const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream');
eventSource.onmessage = () => {
setCount(prev => prev + 1);
setPulsing(true);
setTimeout(() => setPulsing(false), 2000);
};
return () => eventSource.close();
}, []);
if (count === 0) {
return (
<button
onClick={onClick}
className="relative p-2 text-gray-500 hover:text-gray-700 transition-colors"
title="KI Vorschläge"
>
🤖
</button>
);
}
return (
<button
onClick={onClick}
className={`relative p-2 text-gray-500 hover:text-gray-700 transition-colors ${
pulsing ? 'animate-pulse' : ''
}`}
title={`${count} KI Vorschläge`}
>
🤖
<span
className={`absolute -top-0 -right-0 min-w-[18px] h-[18px] flex items-center justify-center text-[10px] font-bold text-white rounded-full ${
pulsing ? 'bg-red-500 animate-bounce' : 'bg-blue-500'
}`}
>
{count > 99 ? '99+' : count}
</span>
</button>
);
}
@@ -0,0 +1,94 @@
import { useState } from 'react';
import type { Suggestion } from '@/api/aiProactive';
const typeConfig = {
info: { icon: '💡', color: 'blue', bg: 'bg-blue-50', border: 'border-blue-200', text: 'text-blue-700', bar: 'bg-blue-500' },
warning: { icon: '⚠️', color: 'orange', bg: 'bg-orange-50', border: 'border-orange-200', text: 'text-orange-700', bar: 'bg-orange-500' },
action: { icon: '✅', color: 'green', bg: 'bg-green-50', border: 'border-green-200', text: 'text-green-700', bar: 'bg-green-500' },
insight: { icon: '🔍', color: 'purple', bg: 'bg-purple-50', border: 'border-purple-200', text: 'text-purple-700', bar: 'bg-purple-500' },
};
interface SuggestionCardProps {
suggestion: Suggestion;
onDismiss: (id: string) => void;
onAct: (id: string, actionIndex: number) => void;
}
export function SuggestionCard({ suggestion, onDismiss, onAct }: SuggestionCardProps) {
const [expanded, setExpanded] = useState(false);
const config = typeConfig[suggestion.suggestion_type] || typeConfig.info;
const confidencePercent = Math.round(suggestion.confidence * 100);
return (
<div
className={`animate-slide-in ${config.bg} ${config.border} border rounded-lg p-4 mb-3 shadow-sm hover:shadow-md transition-shadow`}
>
{/* Header */}
<div className="flex items-start justify-between mb-2">
<div className="flex items-center gap-2">
<span className="text-lg">{config.icon}</span>
<h3 className={`font-semibold text-sm ${config.text}`}>{suggestion.title}</h3>
</div>
<button
onClick={() => onDismiss(suggestion.id)}
className="text-gray-400 hover:text-gray-600 transition-colors"
title="Ignorieren"
>
</button>
</div>
{/* Content */}
<p className="text-sm text-gray-600 mb-3 line-clamp-3">{suggestion.content}</p>
{/* Confidence Bar */}
<div className="flex items-center gap-2 mb-3">
<div className="flex-1 h-1.5 bg-gray-200 rounded-full overflow-hidden">
<div
className={`h-full ${config.bar} rounded-full transition-all duration-500`}
style={{ width: `${confidencePercent}%` }}
/>
</div>
<span className="text-xs text-gray-500 font-mono">{confidencePercent}%</span>
</div>
{/* Actions */}
{suggestion.actions && suggestion.actions.length > 0 && (
<div className="space-y-2">
{suggestion.actions.slice(0, expanded ? undefined : 2).map((action, idx) => (
<button
key={idx}
onClick={() => onAct(suggestion.id, idx)}
disabled={suggestion.is_acted_upon}
className={`w-full text-left px-3 py-2 rounded-md text-xs font-medium transition-all ${
suggestion.is_acted_upon
? 'bg-gray-100 text-gray-400 cursor-not-allowed'
: 'bg-white hover:bg-gray-50 text-gray-700 border border-gray-200 hover:border-gray-300'
}`}
>
<span className="font-mono text-[10px] uppercase mr-1 px-1 py-0.5 bg-gray-100 rounded">
{action.method}
</span>
{action.description}
</button>
))}
{suggestion.actions.length > 2 && (
<button
onClick={() => setExpanded(!expanded)}
className="text-xs text-gray-500 hover:text-gray-700"
>
{expanded ? 'Weniger' : `${suggestion.actions.length - 2} weitere Aktionen`}
</button>
)}
</div>
)}
{/* Acted upon badge */}
{suggestion.is_acted_upon && (
<div className="mt-2 text-xs text-green-600 font-medium flex items-center gap-1">
Ausgeführt
</div>
)}
</div>
);
}
@@ -0,0 +1,109 @@
import { useState } from 'react';
import { useSuggestions } from '@/api/aiProactive';
import { SuggestionCard } from '@/components/ai/SuggestionCard';
interface SuggestionSidebarProps {
isOpen: boolean;
onClose: () => void;
}
export function SuggestionSidebar({ isOpen, onClose }: SuggestionSidebarProps) {
const { suggestions, connected, dismiss, act } = useSuggestions();
const [filter, setFilter] = useState<string>('all');
const filteredSuggestions = filter === 'all'
? suggestions
: suggestions.filter(s => s.suggestion_type === filter);
const filterOptions = [
{ value: 'all', label: 'Alle' },
{ value: 'info', label: 'Info' },
{ value: 'warning', label: 'Warnung' },
{ value: 'action', label: 'Aktion' },
{ value: 'insight', label: 'Erkenntnis' },
];
return (
<>
{/* Overlay */}
{isOpen && (
<div
className="fixed inset-0 bg-black/20 z-40 lg:hidden"
onClick={onClose}
/>
)}
{/* Sidebar */}
<aside
className={`fixed right-0 top-0 h-full w-96 bg-white shadow-2xl z-50 transform transition-transform duration-300 flex flex-col ${
isOpen ? 'translate-x-0' : 'translate-x-full'
}`}
>
{/* Header */}
<div className="flex items-center justify-between p-4 border-b border-gray-200">
<div className="flex items-center gap-2">
<h2 className="font-semibold text-gray-800">KI Vorschläge</h2>
{connected && (
<span className="flex items-center gap-1 text-xs text-green-600">
<span className="w-2 h-2 bg-green-500 rounded-full animate-pulse" />
Live
</span>
)}
</div>
<button
onClick={onClose}
className="text-gray-400 hover:text-gray-600 transition-colors p-1"
>
</button>
</div>
{/* Filter */}
<div className="flex gap-1 p-3 border-b border-gray-100 overflow-x-auto">
{filterOptions.map(opt => (
<button
key={opt.value}
onClick={() => setFilter(opt.value)}
className={`px-3 py-1 rounded-full text-xs font-medium whitespace-nowrap transition-colors ${
filter === opt.value
? 'bg-blue-500 text-white'
: 'bg-gray-100 text-gray-600 hover:bg-gray-200'
}`}
>
{opt.label}
</button>
))}
</div>
{/* Content */}
<div className="flex-1 overflow-y-auto p-4">
{filteredSuggestions.length === 0 ? (
<div className="flex flex-col items-center justify-center h-full text-gray-400">
<div className="text-4xl mb-3">🤖</div>
<p className="text-sm">Keine Vorschläge vorhanden</p>
<p className="text-xs mt-1">Die KI analysiert deinen Kontext...</p>
</div>
) : (
<div>
{filteredSuggestions.map(suggestion => (
<SuggestionCard
key={suggestion.id}
suggestion={suggestion}
onDismiss={dismiss}
onAct={act}
/>
))}
</div>
)}
</div>
{/* Footer */}
<div className="p-3 border-t border-gray-200 text-center">
<span className="text-xs text-gray-400">
{suggestions.length} Vorschläge insgesamt
</span>
</div>
</aside>
</>
);
}
@@ -4,11 +4,17 @@ import { Sidebar } from './Sidebar';
import { TopBar } from './TopBar';
import { PluginToolbar } from './PluginToolbar';
import { AISidebar } from './AISidebar';
import { SuggestionSidebar } from '@/components/ai/SuggestionSidebar';
import { ToastContainer } from '@/components/ui/Toast';
import { useUIStore } from '@/store/uiStore';
import { useAIContext } from '@/hooks/useAIContext';
export function AppShell() {
const location = useLocation();
const { suggestionSidebarOpen, toggleSuggestionSidebar } = useUIStore();
// Track context for AI Proactive suggestions
useAIContext();
// Hide AI sidebar on the AI Assistant page itself
const showAISidebar = !location.pathname.startsWith('/ai-assistant');
@@ -33,6 +39,11 @@ export function AppShell() {
</div>
{/* AI Sidebar — full height, right of TopBar and Toolbar */}
{showAISidebar && <AISidebar />}
{/* Suggestion Sidebar — overlays from right */}
<SuggestionSidebar
isOpen={suggestionSidebarOpen}
onClose={() => toggleSuggestionSidebar()}
/>
<ToastContainer />
</div>
);
+5 -1
View File
@@ -9,12 +9,13 @@ import { useLogout } from '@/api/hooks';
import { Avatar } from '@/components/ui/Avatar';
import { Badge } from '@/components/ui/Badge';
import { SearchDropdown } from '@/components/shared/SearchDropdown';
import { SuggestionBadge } from '@/components/ai/SuggestionBadge';
export function TopBar() {
const { t } = useTranslation();
const navigate = useNavigate();
const { user } = useAuthStore();
const { toggleSidebar, toggleAISidebar, aiSidebarCollapsed } = useUIStore();
const { toggleSidebar, toggleAISidebar, toggleSuggestionSidebar, aiSidebarCollapsed } = useUIStore();
const { currentTenant, availableTenants, switchTenant, isSwitching } = useTenant();
const logoutMutation = useLogout();
@@ -141,6 +142,9 @@ export function TopBar() {
</div>
<div className="flex items-center gap-2">
{/* AI Suggestions Badge */}
<SuggestionBadge onClick={toggleSuggestionSidebar} />
{/* Notifications */}
<div ref={notifRef} className="relative">
<button
+24
View File
@@ -0,0 +1,24 @@
import { useEffect, useRef } from 'react';
import { useLocation } from 'react-router-dom';
import { apiClient } from '@/api/client';
export function useAIContext(entityType?: string, entityId?: string, entityData?: any) {
const location = useLocation();
const debounceRef = useRef<ReturnType<typeof setTimeout>>();
useEffect(() => {
if (debounceRef.current) clearTimeout(debounceRef.current);
debounceRef.current = setTimeout(() => {
apiClient.post('/api/v1/ai-proactive/context', {
page: location.pathname,
entity_type: entityType,
entity_id: entityId,
entity_data: entityData,
}).catch(() => {}); // Silent fail, don't bother user
}, 500); // Debounce 500ms
return () => {
if (debounceRef.current) clearTimeout(debounceRef.current);
};
}, [location.pathname, entityType, entityId, entityData]);
}
+157
View File
@@ -0,0 +1,157 @@
import { useProactiveSettings } from '@/api/aiProactive';
const categoryLabels: Record<string, string> = {
mail: 'E-Mails',
tasks: 'Aufgaben',
contacts: 'Kontakte',
companies: 'Firmen',
insights: 'Erkenntnisse',
};
const modelOptions = [
{ value: 'ollama/deepseek-v4', label: 'DeepSeek V4 (empfohlen)' },
{ value: 'ollama/deepseek-v4-pro', label: 'DeepSeek V4 Pro (präzise)' },
{ value: 'ollama/llama3.2', label: 'Llama 3.2 (Alternative)' },
{ value: 'ollama/gpt-4o-mini', label: 'GPT-4o mini via Ollama' },
{ value: 'gpt-4o-mini', label: 'GPT-4o mini (Direct OpenAI)' },
];
export function ProactiveAISettings() {
const { settings, update } = useProactiveSettings();
if (!settings) {
return (
<div className="flex items-center justify-center p-8 text-gray-400">
<div className="animate-pulse">Lade Einstellungen...</div>
</div>
);
}
const toggleCategory = async (cat: string) => {
const current = settings.suggestion_categories || [];
const newCats = current.includes(cat)
? current.filter((c: string) => c !== cat)
: [...current, cat];
await update({ suggestion_categories: newCats });
};
return (
<div className="max-w-2xl space-y-6">
{/* Enable/Disable */}
<div className="flex items-center justify-between p-4 bg-white rounded-lg border border-gray-200">
<div>
<h3 className="font-semibold text-gray-800">Proaktive KI</h3>
<p className="text-sm text-gray-500">Aktiviert oder deaktiviert alle Vorschläge</p>
</div>
<button
onClick={() => update({ enabled: !settings.enabled })}
className={`relative inline-flex h-6 w-11 items-center rounded-full transition-colors ${
settings.enabled ? 'bg-blue-500' : 'bg-gray-300'
}`}
>
<span
className={`inline-block h-4 w-4 transform rounded-full bg-white transition-transform ${
settings.enabled ? 'translate-x-6' : 'translate-x-1'
}`}
/>
</button>
</div>
{/* Categories */}
<div className="p-4 bg-white rounded-lg border border-gray-200">
<h3 className="font-semibold text-gray-800 mb-3">Kategorien</h3>
<p className="text-sm text-gray-500 mb-4">Für welche Bereiche sollen Vorschläge generiert werden?</p>
<div className="grid grid-cols-2 gap-3">
{Object.entries(categoryLabels).map(([key, label]) => {
const checked = settings.suggestion_categories?.includes(key);
return (
<label
key={key}
className={`flex items-center gap-2 p-3 rounded-lg border cursor-pointer transition-colors ${
checked ? 'bg-blue-50 border-blue-300' : 'bg-gray-50 border-gray-200 hover:bg-gray-100'
}`}
>
<input
type="checkbox"
checked={checked || false}
onChange={() => toggleCategory(key)}
className="w-4 h-4 rounded text-blue-500 focus:ring-blue-400"
/>
<span className="text-sm text-gray-700">{label}</span>
</label>
);
})}
</div>
</div>
{/* Confidence Threshold */}
<div className="p-4 bg-white rounded-lg border border-gray-200">
<div className="flex items-center justify-between mb-2">
<h3 className="font-semibold text-gray-800">Confidence Schwellwert</h3>
<span className="text-sm font-mono text-blue-600">
{Math.round((settings.confidence_threshold || 0.5) * 100)}%
</span>
</div>
<p className="text-sm text-gray-500 mb-3">
Nur Vorschläge mit höherer Confidence anzeigen
</p>
<input
type="range"
min="0"
max="1"
step="0.05"
value={settings.confidence_threshold || 0.5}
onChange={(e) => update({ confidence_threshold: parseFloat(e.target.value) })}
className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500"
/>
<div className="flex justify-between text-xs text-gray-400 mt-1">
<span>Alle</span>
<span>Sehr sicher</span>
</div>
</div>
{/* Rate Limit */}
<div className="p-4 bg-white rounded-lg border border-gray-200">
<div className="flex items-center justify-between mb-2">
<h3 className="font-semibold text-gray-800">Rate Limit</h3>
<span className="text-sm font-mono text-blue-600">
{settings.rate_limit_seconds || 10}s
</span>
</div>
<p className="text-sm text-gray-500 mb-3">
Mindestabstand zwischen Vorschlägen
</p>
<input
type="range"
min="5"
max="60"
step="5"
value={settings.rate_limit_seconds || 10}
onChange={(e) => update({ rate_limit_seconds: parseInt(e.target.value) })}
className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500"
/>
<div className="flex justify-between text-xs text-gray-400 mt-1">
<span>5s</span>
<span>60s</span>
</div>
</div>
{/* Model Selection */}
<div className="p-4 bg-white rounded-lg border border-gray-200">
<h3 className="font-semibold text-gray-800 mb-2">KI Modell (Ollama Cloud)</h3>
<p className="text-sm text-gray-500 mb-3">Wähle das Modell für Vorschläge. DeepSeek V4 ist empfohlen für beste Performance/Kosten.</p>
<select
value={settings.model || 'ollama/deepseek-v4'}
onChange={(e) => update({ model: e.target.value })}
className="w-full px-3 py-2 rounded-lg border border-gray-200 bg-white text-sm text-gray-700 focus:ring-2 focus:ring-blue-400 focus:border-blue-400 outline-none"
>
{modelOptions.map(opt => (
<option key={opt.value} value={opt.value}>
{opt.label}
</option>
))}
</select>
</div>
</div>
);
}
+1
View File
@@ -17,6 +17,7 @@ export function SettingsPage() {
{ to: '/settings/sequences', label: t('sequences.title'), icon: '\ud83d\udd22' },
{ to: '/settings/mail', label: t('mail.settings'), icon: '\ud83d\udce7' },
{ to: '/settings/ai', label: t('nav.aiAssistant'), icon: '\ud83e\udde0' },
{ to: '/settings/ai-proactive', label: 'Proaktive KI', icon: '\ud83e\udd16' },
{ to: '/settings/notifications', label: t('settings.notifications'), icon: '\ud83d\udd14' },
];
+2
View File
@@ -33,6 +33,7 @@ import { MailSettingsPage } from '@/pages/MailSettings';
import { SettingsNotificationsPage } from '@/pages/SettingsNotifications';
import { AIAssistantPage } from '@/pages/AIAssistant';
import { AISettingsPage } from '@/pages/AISettings';
import { ProactiveAISettings } from '@/pages/ProactiveAISettings';
const router = createBrowserRouter([
{
@@ -89,6 +90,7 @@ const router = createBrowserRouter([
{ path: 'mail', element: <MailSettingsPage /> },
{ path: 'notifications', element: <SettingsNotificationsPage /> },
{ path: 'ai', element: <AISettingsPage /> },
{ path: 'ai-proactive', element: <ProactiveAISettings /> },
],
},
],
+4
View File
@@ -14,12 +14,14 @@ export interface UIState {
theme: Theme;
locale: Locale;
sidebarOpen: boolean;
suggestionSidebarOpen: boolean;
aiSidebarCollapsed: boolean;
toasts: Toast[];
setTheme: (theme: Theme) => void;
setLocale: (locale: Locale) => void;
toggleSidebar: () => void;
setSidebarOpen: (open: boolean) => void;
toggleSuggestionSidebar: () => void;
toggleAISidebar: () => void;
setAISidebarCollapsed: (collapsed: boolean) => void;
addToast: (toast: Omit<Toast, 'id'>) => void;
@@ -33,6 +35,7 @@ export const useUIStore = create<UIState>((set) => ({
theme: (localStorage.getItem('leocrm_theme') as Theme) || 'light',
locale: (localStorage.getItem('leocrm_lang') as Locale) || 'de',
sidebarOpen: true,
suggestionSidebarOpen: false,
aiSidebarCollapsed: true,
toasts: [],
setTheme: (theme) => {
@@ -45,6 +48,7 @@ export const useUIStore = create<UIState>((set) => ({
},
toggleSidebar: () => set((s) => ({ sidebarOpen: !s.sidebarOpen })),
setSidebarOpen: (open) => set({ sidebarOpen: open }),
toggleSuggestionSidebar: () => set((s) => ({ suggestionSidebarOpen: !s.suggestionSidebarOpen })),
toggleAISidebar: () => set((s) => ({ aiSidebarCollapsed: !s.aiSidebarCollapsed })),
setAISidebarCollapsed: (collapsed) => set({ aiSidebarCollapsed: collapsed }),
addToast: (toast) => {
+6
View File
@@ -46,3 +46,9 @@ openpyxl>=3.1
prometheus-client>=0.20
structlog>=24.0
litellm
# AI / Search
pgvector>=0.3.0
PyMuPDF>=1.24.0
python-docx>=1.1.0
python-pptx>=0.6.23
+440
View File
@@ -0,0 +1,440 @@
"""Tests for the AI Proactive plugin — Context API, Suggestions API,
Dismiss API, Settings API, Stats API, and Proactive Engine unit tests.
Uses pytest + pytest-asyncio, httpx.AsyncClient, login_client,
seed_tenant_and_users, ORIGIN_HEADER from conftest.py.
Mocks litellm.acompletion and litellm.aembedding to avoid real API calls.
"""
from __future__ import annotations
import asyncio
import json
import uuid
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import pytest_asyncio
from httpx import ASGITransport, AsyncClient
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
# Import models at module level so Base.metadata.create_all includes their tables
from app.plugins.builtins.ai_proactive.models import ( # noqa: F401
ContextLog,
ProactiveSettings,
ProactiveSuggestion,
)
from app.plugins.builtins.ai_assistant.models import ( # noqa: F401
AIProvider, AIModel, AIPreset, AIAgent, AIChatSession, AIChatMessage,
AIChatFolder, AIChatAttachment,
)
from app.plugins.builtins.unified_search.models import ( # noqa: F401
SearchProviderRegistry as SearchProviderRegistryModel,
SearchIndexLog,
)
from app.plugins.builtins.ai_proactive import AIProactivePlugin
from app.plugins.builtins.ai_assistant import AIAssistantPlugin
from app.plugins.builtins.unified_search import UnifiedSearchPlugin
from app.core.db import close_engine, reset_engine_for_testing
from app.core.service_container import get_container
from app.main import create_app
from app.plugins.registry import reset_registry_for_testing
from app.services.plugin_service import reset_plugin_service_for_testing
from tests.conftest import ORIGIN_HEADER, _get_sync_engine, login_client, seed_tenant_and_users
# ─── AI Proactive Fixtures ───
@pytest_asyncio.fixture(autouse=True)
async def clean_ai_proactive_tables():
"""Truncate ai_proactive tables before each test (not in conftest TRUNCATE list)."""
sync_eng = _get_sync_engine()
with sync_eng.connect() as conn:
conn.execute(
text(
"TRUNCATE TABLE ai_proactive_suggestions, "
"ai_proactive_context_log, ai_proactive_settings CASCADE;"
)
)
conn.commit()
sync_eng.dispose()
yield
@pytest_asyncio.fixture
async def ai_proactive_app(engine: AsyncEngine, redis_client):
"""FastAPI app with AI Proactive plugin registered, installed, and activated."""
reset_engine_for_testing(engine)
app = create_app()
registry = reset_registry_for_testing()
registry.initialize(engine, app)
container = get_container()
await container.initialize()
registry.register_plugin(AIAssistantPlugin())
registry.register_plugin(UnifiedSearchPlugin())
registry.register_plugin(AIProactivePlugin())
reset_plugin_service_for_testing(registry)
sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
async with sf() as session:
# Install dependencies first, then ai_proactive
await registry.install(session, "ai_assistant")
await registry.activate(session, "ai_assistant")
await registry.install(session, "unified_search")
await registry.activate(session, "unified_search")
await registry.install(session, "ai_proactive")
await registry.activate(session, "ai_proactive")
await session.commit()
yield app
await close_engine()
@pytest_asyncio.fixture
async def ai_proactive_client(ai_proactive_app) -> AsyncClient:
"""HTTP test client with ai_proactive plugin active."""
transport = ASGITransport(app=ai_proactive_app)
async with AsyncClient(transport=transport, base_url="http://test") as c:
yield c
@pytest_asyncio.fixture
async def ai_proactive_authed_client(
ai_proactive_client: AsyncClient, db_session: AsyncSession
) -> tuple[AsyncClient, dict]:
"""Authenticated admin client with seeded data and ai_proactive plugin active."""
seed = await seed_tenant_and_users(db_session)
# Grant is_system_admin so require_permission passes for ai_proactive:* permissions
from sqlalchemy import update
from app.models.user import User
await db_session.execute(
update(User).where(User.id == seed["admin_a"].id).values(is_system_admin=True)
)
await db_session.commit()
# Login and capture CSRF token
login_resp = await ai_proactive_client.post(
"/api/v1/auth/login",
json={"email": "admin@tenanta.com", "password": "TestPass123!"},
headers=ORIGIN_HEADER,
)
assert login_resp.status_code == 200, f"Login failed: {login_resp.text}"
csrf_token = login_resp.json().get("csrf_token", "")
# Store CSRF token on client for use in requests
ai_proactive_client.headers["X-CSRF-Token"] = csrf_token
return ai_proactive_client, seed
@pytest.fixture(autouse=True)
async def mock_litellm():
"""Mock litellm.acompletion and litellm.aembedding for all tests."""
mock_resp = MagicMock()
mock_resp.choices = [MagicMock()]
mock_resp.choices[0].message.content = json.dumps({
"suggestion_type": "info",
"title": "Test Suggestion",
"content": "This is a test suggestion",
"confidence": 0.8,
"actions": [
{
"method": "GET",
"path": "/api/v1/contacts",
"body": {},
"description": "View contacts",
}
],
})
mock_emb_resp = MagicMock()
mock_emb_resp.data = [{"embedding": [0.1] * 768}]
with (
patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_resp),
patch("litellm.aembedding", new_callable=AsyncMock, return_value=mock_emb_resp),
):
yield
# ─── 1. Context API (POST /api/v1/ai-proactive/context) ───
@pytest.mark.asyncio
async def test_context_report_accepted(
ai_proactive_authed_client: tuple[AsyncClient, dict],
):
"""Context-Report wird akzeptiert (200)."""
client, _ = ai_proactive_authed_client
resp = await client.post(
"/api/v1/ai-proactive/context",
json={
"page": "/contacts",
"entity_type": "contact",
"entity_id": str(uuid.uuid4()),
},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
assert resp.json()["status"] == "ok"
@pytest.mark.asyncio
async def test_context_unauthorized(ai_proactive_client: AsyncClient):
"""Ohne Auth → 403 (CSRF middleware blocks before auth check on POST)."""
resp = await ai_proactive_client.post(
"/api/v1/ai-proactive/context",
json={"page": "/contacts"},
headers=ORIGIN_HEADER,
)
# Without a session cookie, CSRF middleware returns 403 before auth is checked
assert resp.status_code in (401, 403)
# ─── 2. Suggestions API (GET /api/v1/ai-proactive/suggestions) ───
@pytest.mark.asyncio
async def test_get_suggestions(
ai_proactive_authed_client: tuple[AsyncClient, dict],
db_session: AsyncSession,
):
"""Gibt Suggestions für User zurück."""
client, seed = ai_proactive_authed_client
suggestion = ProactiveSuggestion(
tenant_id=seed["tenant_a"].id,
user_id=seed["admin_a"].id,
entity_type="contact",
entity_id=uuid.uuid4(),
suggestion_type="info",
title="Test Suggestion",
content="Test content",
confidence=0.8,
actions=[],
context_snapshot={},
)
db_session.add(suggestion)
await db_session.commit()
resp = await client.get(
"/api/v1/ai-proactive/suggestions",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert "items" in data
assert data["total"] >= 1
titles = [item["title"] for item in data["items"]]
assert "Test Suggestion" in titles
@pytest.mark.asyncio
async def test_suggestions_tenant_isolation(
ai_proactive_authed_client: tuple[AsyncClient, dict],
db_session: AsyncSession,
):
"""Nur eigene Tenant — Tenant B suggestions are not visible to Tenant A."""
client, seed = ai_proactive_authed_client
# Suggestion for tenant B user
suggestion_b = ProactiveSuggestion(
tenant_id=seed["tenant_b"].id,
user_id=seed["admin_b"].id,
entity_type="contact",
entity_id=uuid.uuid4(),
suggestion_type="info",
title="Tenant B Secret",
content="Should not be visible to A",
confidence=0.8,
actions=[],
context_snapshot={},
)
# Suggestion for tenant A user
suggestion_a = ProactiveSuggestion(
tenant_id=seed["tenant_a"].id,
user_id=seed["admin_a"].id,
entity_type="contact",
entity_id=uuid.uuid4(),
suggestion_type="info",
title="Tenant A Visible",
content="Should be visible",
confidence=0.8,
actions=[],
context_snapshot={},
)
db_session.add_all([suggestion_b, suggestion_a])
await db_session.commit()
resp = await client.get(
"/api/v1/ai-proactive/suggestions",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
titles = [item["title"] for item in data["items"]]
assert "Tenant A Visible" in titles
assert "Tenant B Secret" not in titles
# ─── 3. Dismiss API (POST /api/v1/ai-proactive/suggestions/{id}/dismiss) ───
@pytest.mark.asyncio
async def test_dismiss_suggestion(
ai_proactive_authed_client: tuple[AsyncClient, dict],
db_session: AsyncSession,
):
"""Suggestion wird dismissed."""
client, seed = ai_proactive_authed_client
suggestion = ProactiveSuggestion(
tenant_id=seed["tenant_a"].id,
user_id=seed["admin_a"].id,
entity_type="contact",
entity_id=uuid.uuid4(),
suggestion_type="info",
title="Dismiss Me",
content="Please dismiss this",
confidence=0.8,
actions=[],
context_snapshot={},
)
db_session.add(suggestion)
await db_session.commit()
resp = await client.post(
f"/api/v1/ai-proactive/suggestions/{suggestion.id}/dismiss",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
assert resp.json()["status"] == "ok"
# Verify it no longer appears in active suggestions
resp2 = await client.get(
"/api/v1/ai-proactive/suggestions",
headers=ORIGIN_HEADER,
)
data = resp2.json()
ids = [item["id"] for item in data["items"]]
assert str(suggestion.id) not in ids
# ─── 4. Settings API (GET/PUT /api/v1/ai-proactive/settings) ───
@pytest.mark.asyncio
async def test_get_settings(
ai_proactive_authed_client: tuple[AsyncClient, dict],
):
"""Settings werden zurückgegeben."""
client, _ = ai_proactive_authed_client
resp = await client.get(
"/api/v1/ai-proactive/settings",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert "enabled" in data
assert "suggestion_categories" in data
assert "confidence_threshold" in data
assert "rate_limit_seconds" in data
assert "model" in data
assert "available_models" in data
@pytest.mark.asyncio
async def test_update_settings(
ai_proactive_authed_client: tuple[AsyncClient, dict],
):
"""Settings können aktualisiert werden."""
client, _ = ai_proactive_authed_client
resp = await client.put(
"/api/v1/ai-proactive/settings",
json={"enabled": False, "confidence_threshold": 0.7},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert data["enabled"] is False
assert data["confidence_threshold"] == 0.7
@pytest.mark.asyncio
async def test_update_model(
ai_proactive_authed_client: tuple[AsyncClient, dict],
):
"""Model kann geändert werden."""
client, _ = ai_proactive_authed_client
resp = await client.put(
"/api/v1/ai-proactive/settings",
json={"model": "ollama/llama3.2"},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert data["model"] == "ollama/llama3.2"
# ─── 5. Stats API (GET /api/v1/ai-proactive/stats) ───
@pytest.mark.asyncio
async def test_get_stats(
ai_proactive_authed_client: tuple[AsyncClient, dict],
):
"""Stats werden zurückgegeben."""
client, _ = ai_proactive_authed_client
resp = await client.get(
"/api/v1/ai-proactive/stats",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert "total_suggestions" in data
assert "dismissed" in data
assert "acted_upon" in data
assert "active" in data
assert "dismiss_rate" in data
assert "act_rate" in data
# ─── 6. Proactive Engine (Unit Tests with mocks) ───
@pytest.mark.asyncio
async def test_sse_push():
"""Suggestion wird in Queue gepusht."""
from app.plugins.builtins.ai_proactive.services import get_sse_queue, push_suggestion
user_id = f"test-sse-{uuid.uuid4()}"
queue = get_sse_queue(user_id)
# Ensure queue is empty
while not queue.empty():
await queue.get()
suggestion = {"id": "test-1", "title": "Test SSE Push"}
await push_suggestion(user_id, suggestion)
item = await asyncio.wait_for(queue.get(), timeout=1.0)
assert item["id"] == "test-1"
assert item["title"] == "Test SSE Push"
@pytest.mark.asyncio
async def test_rate_limiting(redis_client):
"""Rate-Limit funktioniert."""
from app.plugins.builtins.ai_proactive.services import is_rate_limited
tenant_id = uuid.uuid4()
user_id = uuid.uuid4()
rate_limit_seconds = 10
with patch(
"app.plugins.builtins.ai_proactive.services.get_cache",
return_value=redis_client,
):
# First call should not be rate limited
limited = await is_rate_limited(tenant_id, user_id, rate_limit_seconds)
assert limited is False
# Second call should be rate limited
limited = await is_rate_limited(tenant_id, user_id, rate_limit_seconds)
assert limited is True
+552
View File
@@ -0,0 +1,552 @@
"""Tests for the Unified Search plugin — covers ACs 1-12 and unit tests.
Tests hybrid search infrastructure: API endpoints, RRF fusion, autocomplete,
text extraction, and provider registry.
"""
from __future__ import annotations
import json
import uuid
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import pytest_asyncio
from httpx import ASGITransport, AsyncClient
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
from app.core.db import close_engine, reset_engine_for_testing
from app.core.service_container import get_container
from app.main import create_app
from app.plugins.builtins.unified_search import UnifiedSearchPlugin
from app.plugins.builtins.unified_search.provider_registry import (
SearchProviderRegistry,
get_search_registry,
)
from app.plugins.builtins.unified_search.search_engine import rrf_fusion
from app.plugins.builtins.unified_search.text_extraction import extract_text_from_file
from app.plugins.registry import reset_registry_for_testing
from app.services.plugin_service import reset_plugin_service_for_testing
from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users
# ─── Unified Search Fixtures ───
@pytest_asyncio.fixture
async def search_app(engine: AsyncEngine, redis_client):
"""FastAPI app with UnifiedSearch plugin registered, installed, and activated."""
reset_engine_for_testing(engine)
app = create_app()
registry = reset_registry_for_testing()
registry.initialize(engine, app)
container = get_container()
await container.initialize()
registry.register_plugin(UnifiedSearchPlugin())
reset_plugin_service_for_testing(registry)
sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
async with sf() as session:
await registry.install(session, "unified_search")
await registry.activate(session, "unified_search")
await session.commit()
yield app
await close_engine()
@pytest_asyncio.fixture
async def search_client(search_app) -> AsyncClient:
"""HTTP test client with unified search plugin active."""
transport = ASGITransport(app=search_app)
async with AsyncClient(transport=transport, base_url="http://test") as c:
yield c
@pytest_asyncio.fixture
async def search_authed_client(
search_client: AsyncClient, db_session: AsyncSession
) -> tuple[AsyncClient, dict]:
"""Authenticated admin client with seeded data and unified search plugin active."""
seed = await seed_tenant_and_users(db_session)
await login_client(search_client, "admin@tenanta.com")
return search_client, seed
@pytest.fixture(autouse=True)
async def mock_external_calls():
"""Mock all external API calls (LiteLLM, job queue) for all tests."""
mock_resp = MagicMock()
mock_resp.choices = [MagicMock()]
mock_resp.choices[0].message.content = json.dumps({
"normalized_query": "test",
"entities": {},
"intent": "search",
"semantic_terms": [],
"suggested_filters": {},
})
mock_emb_resp = MagicMock()
mock_emb_resp.data = [{"embedding": [0.1] * 768}]
with (
patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_resp),
patch("litellm.aembedding", new_callable=AsyncMock, return_value=mock_emb_resp),
patch("app.core.jobs.enqueue_job", new_callable=AsyncMock, return_value="job-123"),
):
yield
# ─── AC1: POST /api/v1/search → 200 + results array ───
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
async def test_ac1_search_returns_results(
mock_hybrid_search,
search_authed_client: tuple[AsyncClient, dict],
):
"""AC1: POST /api/v1/search with query returns 200 + results array."""
client, _ = search_authed_client
mock_hybrid_search.return_value = [
{
"entity_type": "company",
"entity_id": str(uuid.uuid4()),
"title": "Company Alpha",
"snippet": "IT company",
"score": 0.95,
"data": {"industry": "IT"},
}
]
resp = await client.post(
"/api/v1/search",
json={"query": "Company"},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert "results" in data
assert isinstance(data["results"], list)
assert len(data["results"]) > 0
assert data["results"][0]["entity_type"] == "company"
assert data["results"][0]["title"] == "Company Alpha"
assert "score" in data["results"][0]
assert "query" in data
assert "normalized_query" in data
assert "summary" in data
# ─── AC2: entity_types filter ───
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
async def test_ac2_search_entity_types_filter(
mock_hybrid_search,
search_authed_client: tuple[AsyncClient, dict],
):
"""AC2: Search with entity_types filters correctly."""
client, _ = search_authed_client
mock_hybrid_search.return_value = []
resp = await client.post(
"/api/v1/search",
json={"query": "test", "entity_types": ["company"]},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
# Verify hybrid_search was called with entity_types=["company"]
call_kwargs = mock_hybrid_search.call_args
assert call_kwargs.kwargs.get("entity_types") == ["company"]
# ─── AC3: No auth → 401 ───
@pytest.mark.asyncio
async def test_ac3_search_no_auth_401(search_client: AsyncClient):
"""AC3: Search without auth returns 401."""
resp = await search_client.post(
"/api/v1/search",
json={"query": "test"},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 401
# ─── AC4: Tenant isolation ───
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
async def test_ac4_search_tenant_isolation(
mock_hybrid_search,
search_authed_client: tuple[AsyncClient, dict],
):
"""AC4: Search only returns results from the user's tenant."""
client, seed = search_authed_client
mock_hybrid_search.return_value = []
resp = await client.post(
"/api/v1/search",
json={"query": "test"},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
# Verify hybrid_search was called with tenant A's ID
call_args = mock_hybrid_search.call_args
tenant_id_arg = call_args.kwargs.get("tenant_id")
assert tenant_id_arg == seed["tenant_a"].id
# ─── AC5: Empty query → 422 ───
@pytest.mark.asyncio
async def test_ac5_search_empty_query_422(search_authed_client: tuple[AsyncClient, dict]):
"""AC5: Empty query returns 422 (min_length=1)."""
client, _ = search_authed_client
resp = await client.post(
"/api/v1/search",
json={"query": ""},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 422
# ─── AC6: GET /api/v1/search/suggest → suggestions ───
@pytest.mark.asyncio
async def test_ac6_suggest_returns_suggestions(
search_authed_client: tuple[AsyncClient, dict],
):
"""AC6: GET /api/v1/search/suggest returns 200 + suggestions list."""
client, _ = search_authed_client
resp = await client.get(
"/api/v1/search/suggest",
params={"q": "comp"},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert "suggestions" in data
assert isinstance(data["suggestions"], list)
# ─── AC7: Empty suggest query → 422 ───
@pytest.mark.asyncio
async def test_ac7_suggest_empty_query_422(
search_authed_client: tuple[AsyncClient, dict],
):
"""AC7: GET /api/v1/search/suggest with empty query returns 422 (min_length=1)."""
client, _ = search_authed_client
resp = await client.get(
"/api/v1/search/suggest",
params={"q": ""},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 422
# ─── AC8: POST /api/v1/search/similar → similar results ───
@pytest.mark.asyncio
async def test_ac8_similar_returns_results(
search_authed_client: tuple[AsyncClient, dict],
):
"""AC8: POST /api/v1/search/similar returns 200 + similar dict."""
client, seed = search_authed_client
resp = await client.post(
"/api/v1/search/similar",
json={
"entity_type": "company",
"entity_id": str(seed["company_a"].id),
"limit": 5,
},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert "similar" in data
assert isinstance(data["similar"], dict)
# ─── AC9: Similar tenant isolation ───
@pytest.mark.asyncio
async def test_ac9_similar_tenant_isolation(
search_authed_client: tuple[AsyncClient, dict],
):
"""AC9: Similar search respects tenant isolation."""
client, seed = search_authed_client
# Use company from tenant B
resp = await client.post(
"/api/v1/search/similar",
json={
"entity_type": "company",
"entity_id": str(seed["company_b"].id),
"limit": 5,
},
headers=ORIGIN_HEADER,
)
# Should return 200 but with empty similar (tenant B company not accessible from tenant A)
assert resp.status_code == 200
data = resp.json()
assert isinstance(data["similar"], dict)
# Company B should not be found in tenant A's context
for etype, items in data["similar"].items():
for item in items:
assert item.get("entity_id") != str(seed["company_b"].id)
# ─── AC10: GET /api/v1/search/providers → provider list ───
@pytest.mark.asyncio
async def test_ac10_list_providers(
search_authed_client: tuple[AsyncClient, dict],
):
"""AC10: GET /api/v1/search/providers returns 200 + active provider list."""
client, _ = search_authed_client
resp = await client.get(
"/api/v1/search/providers",
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert isinstance(data, list)
# After activation, providers should be registered
if len(data) > 0:
assert "entity_type" in data[0]
assert "plugin_name" in data[0]
assert "is_active" in data[0]
assert data[0]["is_active"] is True
# ─── AC11: POST /api/v1/search/reindex as admin → 200 ───
@pytest.mark.asyncio
async def test_ac11_reindex_admin_200(
search_authed_client: tuple[AsyncClient, dict],
):
"""AC11: Admin can trigger reindex (200)."""
client, _ = search_authed_client
resp = await client.post(
"/api/v1/search/reindex",
json={"entity_types": ["company"]},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
assert data["status"] == "ok"
assert "entity_types" in data
# ─── AC12: POST /api/v1/search/reindex as viewer → 403 ───
@pytest.mark.asyncio
async def test_ac12_reindex_viewer_403(
search_client: AsyncClient, db_session: AsyncSession
):
"""AC12: Non-admin (viewer) gets 403 on reindex."""
await seed_tenant_and_users(db_session)
await login_client(search_client, "viewer@tenanta.com")
resp = await search_client.post(
"/api/v1/search/reindex",
json={"entity_types": ["company"]},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 403
# ─── Unit Tests: RRF Fusion ───
def test_rrf_fusion_combines_fts_and_vector_scores():
"""RRF fusion combines FTS + Vector scores correctly."""
fts_results = [
{"id": "1", "title": "Result A", "score": 0.9},
{"id": "2", "title": "Result B", "score": 0.8},
]
vec_results = [
{"id": "2", "title": "Result B", "score": 0.95},
{"id": "3", "title": "Result C", "score": 0.7},
]
fused = rrf_fusion(fts_results, vec_results, "company")
# All 3 unique IDs should be present
ids = [str(r.get("id", "")) for r in fused]
assert "1" in ids
assert "2" in ids
assert "3" in ids
# Results should be sorted by fused score descending
assert fused[0]["_score"] >= fused[-1]["_score"]
# ID 2 appears in both FTS and vector, so should have highest score
assert str(fused[0].get("id", "")) == "2"
def test_rrf_fusion_empty_inputs():
"""RRF fusion with empty inputs returns empty list."""
fused = rrf_fusion([], [], "company")
assert fused == []
def test_rrf_fusion_fts_only():
"""RRF fusion with only FTS results returns them sorted by score."""
fts_results = [
{"id": "1", "title": "A"},
{"id": "2", "title": "B"},
]
fused = rrf_fusion(fts_results, [], "company")
assert len(fused) == 2
# First result (rank 0) should have higher score
assert fused[0]["_score"] > fused[1]["_score"]
# ─── Unit Tests: Text Extraction ───
@pytest.mark.asyncio
async def test_extract_text_from_text_file(tmp_path):
"""Extract text from a plain text file."""
test_file = tmp_path / "test.txt"
test_file.write_text("Hello World\nThis is a test.", encoding="utf-8")
result = await extract_text_from_file(str(test_file), "text/plain")
assert "Hello World" in result
assert "test" in result
@pytest.mark.asyncio
async def test_extract_text_from_pdf_mocked(tmp_path):
"""Extract text from PDF with mocked PyMuPDF."""
test_file = tmp_path / "test.pdf"
test_file.write_bytes(b"%PDF-1.4 fake")
mock_page = MagicMock()
mock_page.get_text.return_value = "PDF content text"
mock_doc = MagicMock()
mock_doc.__iter__ = MagicMock(return_value=iter([mock_page]))
mock_doc.close = MagicMock()
with patch("fitz.open", return_value=mock_doc):
result = await extract_text_from_file(str(test_file), "application/pdf")
assert "PDF content text" in result
@pytest.mark.asyncio
async def test_extract_text_unsupported_mime():
"""Unsupported MIME type returns empty string."""
result = await extract_text_from_file("/tmp/nonexistent", "application/octet-stream")
assert result == ""
@pytest.mark.asyncio
async def test_extract_text_image_returns_empty(tmp_path):
"""Image MIME type returns empty string (OCR not implemented)."""
test_file = tmp_path / "image.png"
test_file.write_bytes(b"\x89PNG\r\n\x1a\n")
result = await extract_text_from_file(str(test_file), "image/png")
assert result == ""
# ─── Unit Tests: Provider Registry ───
def test_provider_register_and_unregister():
"""Provider can be registered and unregistered."""
registry = SearchProviderRegistry()
# Create a mock provider
mock_provider = MagicMock()
mock_provider.entity_type = "test_entity"
# Register
registry.register(mock_provider)
assert registry.get("test_entity") is mock_provider
# Unregister
registry.unregister("test_entity")
assert registry.get("test_entity") is None
def test_provider_get_all():
"""get_all returns all registered providers."""
registry = SearchProviderRegistry()
p1 = MagicMock()
p1.entity_type = "contact"
p2 = MagicMock()
p2.entity_type = "company"
registry.register(p1)
registry.register(p2)
all_providers = registry.get_all()
assert len(all_providers) == 2
entity_types = [p.entity_type for p in all_providers]
assert "contact" in entity_types
assert "company" in entity_types
def test_provider_clear():
"""clear removes all providers."""
registry = SearchProviderRegistry()
p = MagicMock()
p.entity_type = "test"
registry.register(p)
assert len(registry.get_all()) == 1
registry.clear()
assert len(registry.get_all()) == 0
def test_provider_get_entity_types():
"""get_entity_types returns list of registered entity types."""
registry = SearchProviderRegistry()
p1 = MagicMock()
p1.entity_type = "contact"
p2 = MagicMock()
p2.entity_type = "company"
registry.register(p1)
registry.register(p2)
types = registry.get_entity_types()
assert "contact" in types
assert "company" in types
assert len(types) == 2
# ─── Unit Tests: Autocomplete ───
@pytest.mark.asyncio
async def test_autocomplete_empty_query_returns_empty(db_session: AsyncSession):
"""Autocomplete with empty query returns empty list."""
from app.plugins.builtins.unified_search.search_engine import autocomplete
result = await autocomplete(db_session, "", uuid.uuid4(), 10)
assert result == []
@pytest.mark.asyncio
async def test_autocomplete_returns_titles(db_session: AsyncSession):
"""Autocomplete returns titles from FTS prefix search."""
from app.plugins.builtins.unified_search.search_engine import autocomplete
# With no providers registered and no data, should return empty list
result = await autocomplete(db_session, "test", uuid.uuid4(), 10)
assert isinstance(result, list)
assert len(result) <= 10