feat: unified_search + ai_proactive plugins with Ollama Cloud DeepSeek V4
- unified_search: Hybride Suche (PostgreSQL FTS + pgvector + RRF Fusion) - 5 Search Providers (Contact, Company, Mail, File, Event) - KI Query Understanding (Fuzzy, Facetten via LiteLLM) - DMS Text-Extraction (PDF, DOCX, XLSX, PPTX) - Embedding Pipeline (ollama/nomic-embed-text, 768 Dim) - Background Jobs für Indexierung - Plugin-basierte Provider Registry - ai_proactive: Proaktiver KI-Agent - Context-Tracking (Frontend → Backend → Event Bus) - Proactive Engine mit LLM Suggestion-Generierung - SSE Real-time Push an Frontend - 6 AI Tools für Tool Registry - Rate-Limiting + User Settings - Deep Analysis Background Jobs - Frontend Integration: - useAIContext Hook, SuggestionSidebar, SuggestionBadge - ProactiveAISettings Page, Search API Client - Globale Suche auf neue API umgestellt - Tests: test_unified_search.py + test_ai_proactive.py (alle bestanden) - Config: Ollama Cloud DeepSeek V4 als Default, konfigurierbar - Dependencies: PyMuPDF, python-docx, python-pptx, pgvector - Bugfixes: notification type_key length, migration IF NOT EXISTS
This commit is contained in:
@@ -54,3 +54,11 @@ RATE_LIMIT_RESET_CONFIRM_MAX=5
|
||||
RATE_LIMIT_RESET_CONFIRM_WINDOW=3600
|
||||
RATE_LIMIT_GENERAL_MAX=60
|
||||
RATE_LIMIT_GENERAL_WINDOW=60
|
||||
|
||||
# === AI / Search ===
|
||||
# Ollama Cloud API Key (für LiteLLM)
|
||||
API_KEY_OLLAMA_CLOUD=
|
||||
# Embedding Modell (default: ollama/nomic-embed-text)
|
||||
SEARCH_EMBEDDING_MODEL=ollama/nomic-embed-text
|
||||
# LLM Modell für Query Understanding (default: ollama/deepseek-v4)
|
||||
SEARCH_LLM_MODEL=ollama/deepseek-v4
|
||||
|
||||
@@ -53,7 +53,7 @@ class NotificationType(Base):
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
type_key: Mapped[str] = mapped_column(String(20), nullable=False, unique=True)
|
||||
type_key: Mapped[str] = mapped_column(String(100), nullable=False, unique=True)
|
||||
plugin_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
category: Mapped[str] = mapped_column(String(50), nullable=False, default="general")
|
||||
label: Mapped[str] = mapped_column(String(200), nullable=False)
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Proactive AI plugin package."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.plugins.builtins.ai_proactive.plugin import AIProactivePlugin
|
||||
|
||||
__all__ = ["AIProactivePlugin"]
|
||||
@@ -0,0 +1,335 @@
|
||||
"""AI Tools for the Tool Registry — context-aware tools registered
|
||||
by the ai_proactive plugin for use by the AI Assistant.
|
||||
|
||||
Each tool is an async function (arguments: dict, context: dict) -> str
|
||||
that performs SQL queries and returns JSON-string results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.db import create_db_session
|
||||
from app.models.audit import AuditLog
|
||||
from app.models.contact import Contact
|
||||
from app.plugins.builtins.mail.models import Mail
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _serialize(row: Any) -> dict[str, Any]:
|
||||
"""Serialize a SQLAlchemy model instance to a dict."""
|
||||
if row is None:
|
||||
return {}
|
||||
result: dict[str, Any] = {}
|
||||
for column in row.__table__.columns:
|
||||
val = getattr(row, column.name)
|
||||
if hasattr(val, "isoformat"):
|
||||
result[column.name] = val.isoformat()
|
||||
elif isinstance(val, uuid.UUID):
|
||||
result[column.name] = str(val)
|
||||
else:
|
||||
result[column.name] = val
|
||||
return result
|
||||
|
||||
|
||||
async def _get_db_and_tenant(context: dict[str, Any]) -> tuple[AsyncSession, uuid.UUID, uuid.UUID]:
|
||||
"""Extract tenant_id, user_id from context and create a DB session."""
|
||||
tenant_id = uuid.UUID(context.get("tenant_id", ""))
|
||||
user_id = uuid.UUID(context.get("user_id", ""))
|
||||
db = context.get("db")
|
||||
if db is not None and isinstance(db, AsyncSession):
|
||||
return db, tenant_id, user_id
|
||||
# Fallback: create standalone session
|
||||
raise RuntimeError("No db session in context")
|
||||
|
||||
|
||||
# ─── Tool Handlers ───
|
||||
|
||||
|
||||
async def get_contact_mails_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
|
||||
"""Get all mails for a contact, ordered by received_at DESC."""
|
||||
try:
|
||||
db, tenant_id, _ = await _get_db_and_tenant(context)
|
||||
contact_id = uuid.UUID(arguments["contact_id"])
|
||||
limit = arguments.get("limit", 10)
|
||||
result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.contact_id == contact_id)
|
||||
.where(Mail.tenant_id == tenant_id)
|
||||
.order_by(Mail.received_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
mails = [_serialize(m) for m in result.scalars().all()]
|
||||
return json.dumps({"mails": mails, "count": len(mails)}, default=str)
|
||||
except Exception as e:
|
||||
logger.exception("get_contact_mails_handler failed")
|
||||
return json.dumps({"error": str(e)})
|
||||
|
||||
|
||||
async def get_contact_history_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
|
||||
"""Get audit log history for an entity, ordered by timestamp DESC."""
|
||||
try:
|
||||
db, tenant_id, _ = await _get_db_and_tenant(context)
|
||||
entity_id = uuid.UUID(arguments["entity_id"])
|
||||
limit = arguments.get("limit", 20)
|
||||
result = await db.execute(
|
||||
select(AuditLog)
|
||||
.where(AuditLog.entity_id == entity_id)
|
||||
.where(AuditLog.tenant_id == tenant_id)
|
||||
.order_by(AuditLog.timestamp.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
entries = [_serialize(a) for a in result.scalars().all()]
|
||||
return json.dumps({"activities": entries, "count": len(entries)}, default=str)
|
||||
except Exception as e:
|
||||
logger.exception("get_contact_history_handler failed")
|
||||
return json.dumps({"error": str(e)})
|
||||
|
||||
|
||||
async def search_related_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
|
||||
"""Find semantically similar entities via unified_search."""
|
||||
try:
|
||||
db, tenant_id, _ = await _get_db_and_tenant(context)
|
||||
entity_type = arguments["entity_type"]
|
||||
entity_id = uuid.UUID(arguments["entity_id"])
|
||||
limit = arguments.get("limit", 5)
|
||||
|
||||
from app.plugins.builtins.unified_search.search_engine import (
|
||||
find_similar_all_types,
|
||||
)
|
||||
|
||||
similar = await find_similar_all_types(
|
||||
db, entity_type, entity_id, tenant_id, limit=limit
|
||||
)
|
||||
return json.dumps({"similar": similar}, default=str)
|
||||
except Exception as e:
|
||||
logger.exception("search_related_handler failed")
|
||||
return json.dumps({"error": str(e)})
|
||||
|
||||
|
||||
async def summarize_mail_thread_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
|
||||
"""Summarize a mail thread using LLM."""
|
||||
try:
|
||||
import litellm
|
||||
|
||||
db, tenant_id, _ = await _get_db_and_tenant(context)
|
||||
thread_id = arguments["thread_id"]
|
||||
limit = arguments.get("limit", 20)
|
||||
|
||||
result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.thread_id == thread_id)
|
||||
.where(Mail.tenant_id == tenant_id)
|
||||
.order_by(Mail.received_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
mails = result.scalars().all()
|
||||
if not mails:
|
||||
return json.dumps({"summary": "No mails found in thread", "count": 0})
|
||||
|
||||
thread_text = "\n\n".join(
|
||||
f"From: {m.from_address}\nSubject: {m.subject}\nDate: {m.received_at}\nBody: {m.body_text[:500]}"
|
||||
for m in mails
|
||||
)
|
||||
|
||||
response = await litellm.acompletion(
|
||||
model="gpt-4o-mini",
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": "Fasse diesen E-Mail-Thread präzise zusammen. Antworte auf Deutsch.",
|
||||
},
|
||||
{"role": "user", "content": thread_text},
|
||||
],
|
||||
temperature=0.3,
|
||||
max_tokens=300,
|
||||
)
|
||||
summary = response.choices[0].message.content or ""
|
||||
return json.dumps({"summary": summary, "count": len(mails)}, default=str)
|
||||
except Exception as e:
|
||||
logger.exception("summarize_mail_thread_handler failed")
|
||||
return json.dumps({"error": str(e)})
|
||||
|
||||
|
||||
async def get_open_tasks_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
|
||||
"""Get open tasks (calendar entries) for a contact or company."""
|
||||
try:
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from app.plugins.builtins.calendar.models import (
|
||||
CalendarEntry,
|
||||
CalendarEntryLink,
|
||||
)
|
||||
|
||||
db, tenant_id, _ = await _get_db_and_tenant(context)
|
||||
entity_type = arguments["entity_type"]
|
||||
entity_id = uuid.UUID(arguments["entity_id"])
|
||||
now = datetime.now(UTC)
|
||||
|
||||
result = await db.execute(
|
||||
select(CalendarEntry)
|
||||
.join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id)
|
||||
.where(CalendarEntryLink.entity_type == entity_type)
|
||||
.where(CalendarEntryLink.entity_id == entity_id)
|
||||
.where(CalendarEntry.tenant_id == tenant_id)
|
||||
.where(CalendarEntry.start_at > now)
|
||||
.where(CalendarEntry.status == "open")
|
||||
.order_by(CalendarEntry.start_at.asc())
|
||||
.limit(20)
|
||||
)
|
||||
tasks = [_serialize(e) for e in result.scalars().all()]
|
||||
return json.dumps({"tasks": tasks, "count": len(tasks)}, default=str)
|
||||
except Exception as e:
|
||||
logger.exception("get_open_tasks_handler failed")
|
||||
return json.dumps({"error": str(e)})
|
||||
|
||||
|
||||
async def hybrid_search_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
|
||||
"""Perform hybrid search via unified_search search_engine."""
|
||||
try:
|
||||
from app.plugins.builtins.unified_search.search_engine import hybrid_search
|
||||
|
||||
db, tenant_id, _ = await _get_db_and_tenant(context)
|
||||
query = arguments["query"]
|
||||
entity_types = arguments.get("entity_types")
|
||||
limit = arguments.get("limit", 20)
|
||||
|
||||
query_analysis = {
|
||||
"normalized_query": query,
|
||||
"semantic_terms": [],
|
||||
}
|
||||
results = await hybrid_search(
|
||||
db, query_analysis, tenant_id, entity_types=entity_types, limit=limit
|
||||
)
|
||||
return json.dumps({"results": results, "count": len(results)}, default=str)
|
||||
except Exception as e:
|
||||
logger.exception("hybrid_search_handler failed")
|
||||
return json.dumps({"error": str(e)})
|
||||
|
||||
|
||||
# ─── Registration ───
|
||||
|
||||
|
||||
def register_context_tools(registry) -> None:
|
||||
"""Register AI tools into the existing Tool Registry."""
|
||||
|
||||
# Tool 1: get_contact_mails
|
||||
registry.register(
|
||||
name="get_contact_mails",
|
||||
description="Alle Mails eines Kontakts abrufen, sortiert nach Datum absteigend",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"contact_id": {"type": "string", "description": "UUID des Kontakts"},
|
||||
"limit": {"type": "integer", "default": 10, "description": "Max results"},
|
||||
},
|
||||
"required": ["contact_id"],
|
||||
},
|
||||
handler=get_contact_mails_handler,
|
||||
plugin_name="ai_proactive",
|
||||
required_permission="mail:read",
|
||||
category="context",
|
||||
)
|
||||
|
||||
# Tool 2: get_contact_history
|
||||
registry.register(
|
||||
name="get_contact_history",
|
||||
description="Aktivitätsverlauf (Audit Log) für eine Entity abrufen",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"entity_id": {"type": "string", "description": "UUID der Entity"},
|
||||
"limit": {"type": "integer", "default": 20, "description": "Max results"},
|
||||
},
|
||||
"required": ["entity_id"],
|
||||
},
|
||||
handler=get_contact_history_handler,
|
||||
plugin_name="ai_proactive",
|
||||
required_permission="ai_proactive:read",
|
||||
category="context",
|
||||
)
|
||||
|
||||
# Tool 3: search_related
|
||||
registry.register(
|
||||
name="search_related",
|
||||
description="Semantisch ähnliche Entities über unified_search finden",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"entity_type": {"type": "string", "description": "contact, company, mail, file, event"},
|
||||
"entity_id": {"type": "string", "description": "UUID der Entity"},
|
||||
"limit": {"type": "integer", "default": 5, "description": "Max results per type"},
|
||||
},
|
||||
"required": ["entity_type", "entity_id"],
|
||||
},
|
||||
handler=search_related_handler,
|
||||
plugin_name="ai_proactive",
|
||||
required_permission="search:read",
|
||||
category="context",
|
||||
)
|
||||
|
||||
# Tool 4: summarize_mail_thread
|
||||
registry.register(
|
||||
name="summarize_mail_thread",
|
||||
description="E-Mail-Thread per LLM zusammenfassen",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"thread_id": {"type": "string", "description": "Thread ID"},
|
||||
"limit": {"type": "integer", "default": 20, "description": "Max mails to include"},
|
||||
},
|
||||
"required": ["thread_id"],
|
||||
},
|
||||
handler=summarize_mail_thread_handler,
|
||||
plugin_name="ai_proactive",
|
||||
required_permission="mail:read",
|
||||
category="context",
|
||||
)
|
||||
|
||||
# Tool 5: get_open_tasks
|
||||
registry.register(
|
||||
name="get_open_tasks",
|
||||
description="Offene Aufgaben (Kalender-Termine) für einen Kontakt oder eine Firma abrufen",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"entity_type": {"type": "string", "description": "contact or company"},
|
||||
"entity_id": {"type": "string", "description": "UUID der Entity"},
|
||||
},
|
||||
"required": ["entity_type", "entity_id"],
|
||||
},
|
||||
handler=get_open_tasks_handler,
|
||||
plugin_name="ai_proactive",
|
||||
required_permission="ai_proactive:read",
|
||||
category="context",
|
||||
)
|
||||
|
||||
# Tool 6: hybrid_search
|
||||
registry.register(
|
||||
name="hybrid_search",
|
||||
description="Hybride Volltext- und Semantiksuche über alle CRM-Entities via unified_search",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string", "description": "Suchanfrage"},
|
||||
"entity_types": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Optional: filter by entity types",
|
||||
},
|
||||
"limit": {"type": "integer", "default": 20, "description": "Max results"},
|
||||
},
|
||||
"required": ["query"],
|
||||
},
|
||||
handler=hybrid_search_handler,
|
||||
plugin_name="ai_proactive",
|
||||
required_permission="search:read",
|
||||
category="context",
|
||||
)
|
||||
@@ -0,0 +1,157 @@
|
||||
import { useProactiveSettings } from './api';
|
||||
|
||||
const categoryLabels: Record<string, string> = {
|
||||
mail: 'E-Mails',
|
||||
tasks: 'Aufgaben',
|
||||
contacts: 'Kontakte',
|
||||
companies: 'Firmen',
|
||||
insights: 'Erkenntnisse',
|
||||
};
|
||||
|
||||
const modelOptions = [
|
||||
{ value: 'ollama/deepseek-v4', label: 'DeepSeek V4 (empfohlen)' },
|
||||
{ value: 'ollama/deepseek-v4-pro', label: 'DeepSeek V4 Pro (präzise)' },
|
||||
{ value: 'ollama/llama3.2', label: 'Llama 3.2 (Alternative)' },
|
||||
{ value: 'ollama/gpt-4o-mini', label: 'GPT-4o mini via Ollama' },
|
||||
{ value: 'gpt-4o-mini', label: 'GPT-4o mini (Direct OpenAI)' },
|
||||
];
|
||||
|
||||
export function AISettings() {
|
||||
const { settings, update } = useProactiveSettings();
|
||||
|
||||
if (!settings) {
|
||||
return (
|
||||
<div className="flex items-center justify-center p-8 text-gray-400">
|
||||
<div className="animate-pulse">Lade Einstellungen...</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const toggleCategory = async (cat: string) => {
|
||||
const current = settings.suggestion_categories || [];
|
||||
const newCats = current.includes(cat)
|
||||
? current.filter((c: string) => c !== cat)
|
||||
: [...current, cat];
|
||||
await update({ suggestion_categories: newCats });
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="max-w-2xl space-y-6">
|
||||
{/* Enable/Disable */}
|
||||
<div className="flex items-center justify-between p-4 bg-white rounded-lg border border-gray-200">
|
||||
<div>
|
||||
<h3 className="font-semibold text-gray-800">Proaktive KI</h3>
|
||||
<p className="text-sm text-gray-500">Aktiviert oder deaktiviert alle Vorschläge</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => update({ enabled: !settings.enabled })}
|
||||
className={`relative inline-flex h-6 w-11 items-center rounded-full transition-colors ${
|
||||
settings.enabled ? 'bg-blue-500' : 'bg-gray-300'
|
||||
}`}
|
||||
>
|
||||
<span
|
||||
className={`inline-block h-4 w-4 transform rounded-full bg-white transition-transform ${
|
||||
settings.enabled ? 'translate-x-6' : 'translate-x-1'
|
||||
}`}
|
||||
/>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Categories */}
|
||||
<div className="p-4 bg-white rounded-lg border border-gray-200">
|
||||
<h3 className="font-semibold text-gray-800 mb-3">Kategorien</h3>
|
||||
<p className="text-sm text-gray-500 mb-4">Für welche Bereiche sollen Vorschläge generiert werden?</p>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
{Object.entries(categoryLabels).map(([key, label]) => {
|
||||
const checked = settings.suggestion_categories?.includes(key);
|
||||
return (
|
||||
<label
|
||||
key={key}
|
||||
className={`flex items-center gap-2 p-3 rounded-lg border cursor-pointer transition-colors ${
|
||||
checked ? 'bg-blue-50 border-blue-300' : 'bg-gray-50 border-gray-200 hover:bg-gray-100'
|
||||
}`}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={checked || false}
|
||||
onChange={() => toggleCategory(key)}
|
||||
className="w-4 h-4 rounded text-blue-500 focus:ring-blue-400"
|
||||
/>
|
||||
<span className="text-sm text-gray-700">{label}</span>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Confidence Threshold */}
|
||||
<div className="p-4 bg-white rounded-lg border border-gray-200">
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<h3 className="font-semibold text-gray-800">Confidence Schwellwert</h3>
|
||||
<span className="text-sm font-mono text-blue-600">
|
||||
{Math.round((settings.confidence_threshold || 0.5) * 100)}%
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-sm text-gray-500 mb-3">
|
||||
Nur Vorschläge mit höherer Confidence anzeigen
|
||||
</p>
|
||||
<input
|
||||
type="range"
|
||||
min="0"
|
||||
max="1"
|
||||
step="0.05"
|
||||
value={settings.confidence_threshold || 0.5}
|
||||
onChange={(e) => update({ confidence_threshold: parseFloat(e.target.value) })}
|
||||
className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500"
|
||||
/>
|
||||
<div className="flex justify-between text-xs text-gray-400 mt-1">
|
||||
<span>Alle</span>
|
||||
<span>Sehr sicher</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Rate Limit */}
|
||||
<div className="p-4 bg-white rounded-lg border border-gray-200">
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<h3 className="font-semibold text-gray-800">Rate Limit</h3>
|
||||
<span className="text-sm font-mono text-blue-600">
|
||||
{settings.rate_limit_seconds || 10}s
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-sm text-gray-500 mb-3">
|
||||
Mindestabstand zwischen Vorschlägen
|
||||
</p>
|
||||
<input
|
||||
type="range"
|
||||
min="5"
|
||||
max="60"
|
||||
step="5"
|
||||
value={settings.rate_limit_seconds || 10}
|
||||
onChange={(e) => update({ rate_limit_seconds: parseInt(e.target.value) })}
|
||||
className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500"
|
||||
/>
|
||||
<div className="flex justify-between text-xs text-gray-400 mt-1">
|
||||
<span>5s</span>
|
||||
<span>60s</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Model Selection */}
|
||||
<div className="p-4 bg-white rounded-lg border border-gray-200">
|
||||
<h3 className="font-semibold text-gray-800 mb-2">KI Modell (Ollama Cloud)</h3>
|
||||
<p className="text-sm text-gray-500 mb-3">Wähle das Modell für Vorschläge. DeepSeek V4 ist empfohlen für beste Performance/Kosten.</p>
|
||||
<select
|
||||
value={settings.model || 'ollama/deepseek-v4'}
|
||||
onChange={(e) => update({ model: e.target.value })}
|
||||
className="w-full px-3 py-2 rounded-lg border border-gray-200 bg-white text-sm text-gray-700 focus:ring-2 focus:ring-blue-400 focus:border-blue-400 outline-none"
|
||||
>
|
||||
{modelOptions.map(opt => (
|
||||
<option key={opt.value} value={opt.value}>
|
||||
{opt.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { apiClient } from '@/api/client';
|
||||
|
||||
interface SuggestionBadgeProps {
|
||||
onClick: () => void;
|
||||
}
|
||||
|
||||
export function SuggestionBadge({ onClick }: SuggestionBadgeProps) {
|
||||
const [count, setCount] = useState(0);
|
||||
const [pulsing, setPulsing] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
// Initial count
|
||||
apiClient.get('/api/v1/ai-proactive/suggestions').then(r => {
|
||||
setCount(r.data.items.length);
|
||||
}).catch(() => {});
|
||||
|
||||
// SSE for real-time updates
|
||||
const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream');
|
||||
eventSource.onmessage = () => {
|
||||
setCount(prev => prev + 1);
|
||||
setPulsing(true);
|
||||
setTimeout(() => setPulsing(false), 2000);
|
||||
};
|
||||
|
||||
return () => eventSource.close();
|
||||
}, []);
|
||||
|
||||
if (count === 0) {
|
||||
return (
|
||||
<button
|
||||
onClick={onClick}
|
||||
className="relative p-2 text-gray-500 hover:text-gray-700 transition-colors"
|
||||
title="KI Vorschläge"
|
||||
>
|
||||
🤖
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<button
|
||||
onClick={onClick}
|
||||
className={`relative p-2 text-gray-500 hover:text-gray-700 transition-colors ${
|
||||
pulsing ? 'animate-pulse' : ''
|
||||
}`}
|
||||
title={`${count} KI Vorschläge`}
|
||||
>
|
||||
🤖
|
||||
<span
|
||||
className={`absolute -top-0 -right-0 min-w-[18px] h-[18px] flex items-center justify-center text-[10px] font-bold text-white rounded-full ${
|
||||
pulsing ? 'bg-red-500 animate-bounce' : 'bg-blue-500'
|
||||
}`}
|
||||
>
|
||||
{count > 99 ? '99+' : count}
|
||||
</span>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import { useState } from 'react';
|
||||
import type { Suggestion } from './api';
|
||||
|
||||
const typeConfig = {
|
||||
info: { icon: '💡', color: 'blue', bg: 'bg-blue-50', border: 'border-blue-200', text: 'text-blue-700', bar: 'bg-blue-500' },
|
||||
warning: { icon: '⚠️', color: 'orange', bg: 'bg-orange-50', border: 'border-orange-200', text: 'text-orange-700', bar: 'bg-orange-500' },
|
||||
action: { icon: '✅', color: 'green', bg: 'bg-green-50', border: 'border-green-200', text: 'text-green-700', bar: 'bg-green-500' },
|
||||
insight: { icon: '🔍', color: 'purple', bg: 'bg-purple-50', border: 'border-purple-200', text: 'text-purple-700', bar: 'bg-purple-500' },
|
||||
};
|
||||
|
||||
interface SuggestionCardProps {
|
||||
suggestion: Suggestion;
|
||||
onDismiss: (id: string) => void;
|
||||
onAct: (id: string, actionIndex: number) => void;
|
||||
}
|
||||
|
||||
export function SuggestionCard({ suggestion, onDismiss, onAct }: SuggestionCardProps) {
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const config = typeConfig[suggestion.suggestion_type] || typeConfig.info;
|
||||
const confidencePercent = Math.round(suggestion.confidence * 100);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={`animate-slide-in ${config.bg} ${config.border} border rounded-lg p-4 mb-3 shadow-sm hover:shadow-md transition-shadow`}
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="flex items-start justify-between mb-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-lg">{config.icon}</span>
|
||||
<h3 className={`font-semibold text-sm ${config.text}`}>{suggestion.title}</h3>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => onDismiss(suggestion.id)}
|
||||
className="text-gray-400 hover:text-gray-600 transition-colors"
|
||||
title="Ignorieren"
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<p className="text-sm text-gray-600 mb-3 line-clamp-3">{suggestion.content}</p>
|
||||
|
||||
{/* Confidence Bar */}
|
||||
<div className="flex items-center gap-2 mb-3">
|
||||
<div className="flex-1 h-1.5 bg-gray-200 rounded-full overflow-hidden">
|
||||
<div
|
||||
className={`h-full ${config.bar} rounded-full transition-all duration-500`}
|
||||
style={{ width: `${confidencePercent}%` }}
|
||||
/>
|
||||
</div>
|
||||
<span className="text-xs text-gray-500 font-mono">{confidencePercent}%</span>
|
||||
</div>
|
||||
|
||||
{/* Actions */}
|
||||
{suggestion.actions && suggestion.actions.length > 0 && (
|
||||
<div className="space-y-2">
|
||||
{suggestion.actions.slice(0, expanded ? undefined : 2).map((action, idx) => (
|
||||
<button
|
||||
key={idx}
|
||||
onClick={() => onAct(suggestion.id, idx)}
|
||||
disabled={suggestion.is_acted_upon}
|
||||
className={`w-full text-left px-3 py-2 rounded-md text-xs font-medium transition-all ${
|
||||
suggestion.is_acted_upon
|
||||
? 'bg-gray-100 text-gray-400 cursor-not-allowed'
|
||||
: 'bg-white hover:bg-gray-50 text-gray-700 border border-gray-200 hover:border-gray-300'
|
||||
}`}
|
||||
>
|
||||
<span className="font-mono text-[10px] uppercase mr-1 px-1 py-0.5 bg-gray-100 rounded">
|
||||
{action.method}
|
||||
</span>
|
||||
{action.description}
|
||||
</button>
|
||||
))}
|
||||
{suggestion.actions.length > 2 && (
|
||||
<button
|
||||
onClick={() => setExpanded(!expanded)}
|
||||
className="text-xs text-gray-500 hover:text-gray-700"
|
||||
>
|
||||
{expanded ? 'Weniger' : `${suggestion.actions.length - 2} weitere Aktionen`}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Acted upon badge */}
|
||||
{suggestion.is_acted_upon && (
|
||||
<div className="mt-2 text-xs text-green-600 font-medium flex items-center gap-1">
|
||||
✓ Ausgeführt
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { useState } from 'react';
|
||||
import { useSuggestions } from './api';
|
||||
import { SuggestionCard } from './SuggestionCard';
|
||||
|
||||
interface SuggestionSidebarProps {
|
||||
isOpen: boolean;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export function SuggestionSidebar({ isOpen, onClose }: SuggestionSidebarProps) {
|
||||
const { suggestions, connected, dismiss, act } = useSuggestions();
|
||||
const [filter, setFilter] = useState<string>('all');
|
||||
|
||||
const filteredSuggestions = filter === 'all'
|
||||
? suggestions
|
||||
: suggestions.filter(s => s.suggestion_type === filter);
|
||||
|
||||
const filterOptions = [
|
||||
{ value: 'all', label: 'Alle' },
|
||||
{ value: 'info', label: 'Info' },
|
||||
{ value: 'warning', label: 'Warnung' },
|
||||
{ value: 'action', label: 'Aktion' },
|
||||
{ value: 'insight', label: 'Erkenntnis' },
|
||||
];
|
||||
|
||||
return (
|
||||
<>
|
||||
{/* Overlay */}
|
||||
{isOpen && (
|
||||
<div
|
||||
className="fixed inset-0 bg-black/20 z-40 lg:hidden"
|
||||
onClick={onClose}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Sidebar */}
|
||||
<aside
|
||||
className={`fixed right-0 top-0 h-full w-96 bg-white shadow-2xl z-50 transform transition-transform duration-300 flex flex-col ${
|
||||
isOpen ? 'translate-x-0' : 'translate-x-full'
|
||||
}`}
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between p-4 border-b border-gray-200">
|
||||
<div className="flex items-center gap-2">
|
||||
<h2 className="font-semibold text-gray-800">KI Vorschläge</h2>
|
||||
{connected && (
|
||||
<span className="flex items-center gap-1 text-xs text-green-600">
|
||||
<span className="w-2 h-2 bg-green-500 rounded-full animate-pulse" />
|
||||
Live
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="text-gray-400 hover:text-gray-600 transition-colors p-1"
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Filter */}
|
||||
<div className="flex gap-1 p-3 border-b border-gray-100 overflow-x-auto">
|
||||
{filterOptions.map(opt => (
|
||||
<button
|
||||
key={opt.value}
|
||||
onClick={() => setFilter(opt.value)}
|
||||
className={`px-3 py-1 rounded-full text-xs font-medium whitespace-nowrap transition-colors ${
|
||||
filter === opt.value
|
||||
? 'bg-blue-500 text-white'
|
||||
: 'bg-gray-100 text-gray-600 hover:bg-gray-200'
|
||||
}`}
|
||||
>
|
||||
{opt.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<div className="flex-1 overflow-y-auto p-4">
|
||||
{filteredSuggestions.length === 0 ? (
|
||||
<div className="flex flex-col items-center justify-center h-full text-gray-400">
|
||||
<div className="text-4xl mb-3">🤖</div>
|
||||
<p className="text-sm">Keine Vorschläge vorhanden</p>
|
||||
<p className="text-xs mt-1">Die KI analysiert deinen Kontext...</p>
|
||||
</div>
|
||||
) : (
|
||||
<div>
|
||||
{filteredSuggestions.map(suggestion => (
|
||||
<SuggestionCard
|
||||
n key={suggestion.id}
|
||||
suggestion={suggestion}
|
||||
onDismiss={dismiss}
|
||||
onAct={act}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Footer */}
|
||||
<div className="p-3 border-t border-gray-200 text-center">
|
||||
<span className="text-xs text-gray-400">
|
||||
{suggestions.length} Vorschläge insgesamt
|
||||
</span>
|
||||
</div>
|
||||
</aside>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { apiClient } from '@/api/client';
|
||||
|
||||
export interface Suggestion {
|
||||
id: string;
|
||||
entity_type: string;
|
||||
entity_id: string | null;
|
||||
suggestion_type: 'info' | 'warning' | 'action' | 'insight';
|
||||
title: string;
|
||||
content: string;
|
||||
confidence: number;
|
||||
actions: Array<{ method: string; path: string; body: any; description: string }>;
|
||||
created_at: string;
|
||||
is_dismissed: boolean;
|
||||
is_acted_upon: boolean;
|
||||
}
|
||||
|
||||
export function useSuggestions() {
|
||||
const [suggestions, setSuggestions] = useState<Suggestion[]>([]);
|
||||
const [connected, setConnected] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
// Initial load
|
||||
apiClient.get('/api/v1/ai-proactive/suggestions').then(r => {
|
||||
setSuggestions(r.data.items);
|
||||
}).catch(() => {});
|
||||
|
||||
// SSE Stream
|
||||
const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream');
|
||||
eventSource.onopen = () => setConnected(true);
|
||||
eventSource.onerror = () => setConnected(false);
|
||||
eventSource.onmessage = (e) => {
|
||||
try {
|
||||
const suggestion = JSON.parse(e.data);
|
||||
setSuggestions(prev => [suggestion, ...prev].slice(0, 50));
|
||||
} catch {}
|
||||
};
|
||||
|
||||
return () => eventSource.close();
|
||||
}, []);
|
||||
|
||||
const dismiss = useCallback(async (id: string) => {
|
||||
setSuggestions(prev => prev.filter(s => s.id !== id));
|
||||
await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/dismiss`).catch(() => {});
|
||||
}, []);
|
||||
|
||||
const act = useCallback(async (id: string, actionIndex: number) => {
|
||||
const result = await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/act`, {
|
||||
action_index: actionIndex
|
||||
});
|
||||
setSuggestions(prev => prev.map(s => s.id === id ? { ...s, is_acted_upon: true } : s));
|
||||
return result.data;
|
||||
}, []);
|
||||
|
||||
return { suggestions, connected, dismiss, act };
|
||||
}
|
||||
|
||||
export function useProactiveSettings() {
|
||||
const [settings, setSettings] = useState<any>(null);
|
||||
|
||||
useEffect(() => {
|
||||
apiClient.get('/api/v1/ai-proactive/settings').then(r => setSettings(r.data));
|
||||
}, []);
|
||||
|
||||
const update = useCallback(async (updates: any) => {
|
||||
const r = await apiClient.put('/api/v1/ai-proactive/settings', updates);
|
||||
setSettings(r.data);
|
||||
return r.data;
|
||||
}, []);
|
||||
|
||||
return { settings, update };
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useLocation } from 'react-router-dom';
|
||||
import { apiClient } from '@/api/client';
|
||||
|
||||
export function useAIContext(entityType?: string, entityId?: string, entityData?: any) {
|
||||
const location = useLocation();
|
||||
const debounceRef = useRef<ReturnType<typeof setTimeout>>();
|
||||
|
||||
useEffect(() => {
|
||||
if (debounceRef.current) clearTimeout(debounceRef.current);
|
||||
debounceRef.current = setTimeout(() => {
|
||||
apiClient.post('/api/v1/ai-proactive/context', {
|
||||
page: location.pathname,
|
||||
entity_type: entityType,
|
||||
entity_id: entityId,
|
||||
entity_data: entityData,
|
||||
}).catch(() => {}); // Silent fail, don't bother user
|
||||
}, 500); // Debounce 500ms
|
||||
|
||||
return () => {
|
||||
if (debounceRef.current) clearTimeout(debounceRef.current);
|
||||
};
|
||||
}, [location.pathname, entityType, entityId, entityData]);
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
"""ARQ background jobs for the AI Proactive plugin.
|
||||
|
||||
Deep analysis job runs after context-change for deeper analysis:
|
||||
- Mail-thread summary generation
|
||||
- Finding similar contacts via unified_search
|
||||
- Analyzing open tasks
|
||||
- Generating extended suggestion with more context
|
||||
- Pushing additional suggestion if confidence > threshold
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.db import create_db_session
|
||||
from app.core.notifications import create_notification
|
||||
from app.models.audit import AuditLog
|
||||
from app.models.contact import Contact
|
||||
from app.plugins.builtins.ai_proactive.models import ProactiveSuggestion, ProactiveSettings
|
||||
from app.plugins.builtins.ai_proactive.services import (
|
||||
_serialize_row,
|
||||
generate_suggestion,
|
||||
get_user_settings,
|
||||
push_suggestion,
|
||||
)
|
||||
from app.plugins.builtins.mail.models import Mail
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
|
||||
|
||||
DEEP_SYSTEM_PROMPT = """Du bist ein proaktiver KI-Assistent für ein CRM. Du führst eine Tiefenanalyse durch.
|
||||
|
||||
Analysiere den erweiterten Kontext und generiere einen detaillierten Vorschlag.
|
||||
Antworte mit JSON:
|
||||
{
|
||||
"suggestion_type": "info|warning|action|insight",
|
||||
"title": "Kurzer Titel (max 200 Zeichen)",
|
||||
"content": "Detaillierte Beschreibung mit konkreten Empfehlungen",
|
||||
"confidence": 0.0-1.0,
|
||||
"actions": [{"method": "GET|POST|PUT|DELETE", "path": "/api/v1/...", "body": {}, "description": "..."}]
|
||||
}
|
||||
|
||||
Fokussiere auf:
|
||||
- Muster in der Kommunikationshistorie
|
||||
- Beziehungen zwischen Entities
|
||||
- Handlungsempfehlungen mit hoher Relevanz
|
||||
- Antworte nur mit gültigem JSON"""
|
||||
|
||||
|
||||
async def deep_analysis(
|
||||
ctx: dict[str, Any],
|
||||
entity_type: str,
|
||||
entity_id: str,
|
||||
user_id: str,
|
||||
tenant_id: str,
|
||||
) -> None:
|
||||
"""Deep Analysis Background-Job.
|
||||
|
||||
Enqueued after context-change for deeper analysis:
|
||||
- Mail-thread summary generation
|
||||
- Similar contacts via unified_search
|
||||
- Open tasks analysis
|
||||
- Extended suggestion generation with more context
|
||||
- If confidence > threshold: save + push additional suggestion
|
||||
"""
|
||||
try:
|
||||
eid = uuid.UUID(entity_id)
|
||||
uid = uuid.UUID(user_id)
|
||||
tid = uuid.UUID(tenant_id)
|
||||
except (ValueError, TypeError):
|
||||
logger.warning("deep_analysis: invalid UUID parameters")
|
||||
return
|
||||
|
||||
async with create_db_session(tid) as db:
|
||||
# Get settings
|
||||
settings = await get_user_settings(db, tid, uid)
|
||||
if not settings.enabled:
|
||||
return
|
||||
|
||||
# Gather extended context (more mails, more activities)
|
||||
extended_context: dict[str, Any] = {
|
||||
"entity_type": entity_type,
|
||||
"entity_id": entity_id,
|
||||
"deep_analysis": True,
|
||||
}
|
||||
|
||||
if entity_type == "contact":
|
||||
# Extended: last 30 mails
|
||||
mail_result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.contact_id == eid)
|
||||
.where(Mail.tenant_id == tid)
|
||||
.order_by(Mail.received_at.desc())
|
||||
.limit(30)
|
||||
)
|
||||
extended_context["mails"] = [
|
||||
_serialize_row(m) for m in mail_result.scalars().all()
|
||||
]
|
||||
|
||||
# Extended: last 50 audit entries
|
||||
audit_result = await db.execute(
|
||||
select(AuditLog)
|
||||
.where(AuditLog.entity_id == eid)
|
||||
.where(AuditLog.tenant_id == tid)
|
||||
.order_by(AuditLog.timestamp.desc())
|
||||
.limit(50)
|
||||
)
|
||||
extended_context["activities"] = [
|
||||
_serialize_row(a) for a in audit_result.scalars().all()
|
||||
]
|
||||
|
||||
# Contact data
|
||||
contact_result = await db.execute(
|
||||
select(Contact)
|
||||
.where(Contact.id == eid)
|
||||
.where(Contact.tenant_id == tid)
|
||||
.limit(1)
|
||||
)
|
||||
contact = contact_result.scalar_one_or_none()
|
||||
extended_context["contact"] = _serialize_row(contact) if contact else None
|
||||
|
||||
elif entity_type == "company":
|
||||
from app.models.company import Company, CompanyContact
|
||||
|
||||
comp_result = await db.execute(
|
||||
select(Company)
|
||||
.where(Company.id == eid)
|
||||
.where(Company.tenant_id == tid)
|
||||
.limit(1)
|
||||
)
|
||||
company = comp_result.scalar_one_or_none()
|
||||
extended_context["company"] = _serialize_row(company) if company else None
|
||||
|
||||
# All mails for company
|
||||
mail_result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.company_id == eid)
|
||||
.where(Mail.tenant_id == tid)
|
||||
.order_by(Mail.received_at.desc())
|
||||
.limit(30)
|
||||
)
|
||||
extended_context["mails"] = [
|
||||
_serialize_row(m) for m in mail_result.scalars().all()
|
||||
]
|
||||
|
||||
elif entity_type == "mail":
|
||||
mail_result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.id == eid)
|
||||
.where(Mail.tenant_id == tid)
|
||||
.limit(1)
|
||||
)
|
||||
mail = mail_result.scalar_one_or_none()
|
||||
extended_context["mail"] = _serialize_row(mail) if mail else None
|
||||
|
||||
if mail and mail.thread_id:
|
||||
thread_result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.thread_id == mail.thread_id)
|
||||
.where(Mail.tenant_id == tid)
|
||||
.order_by(Mail.received_at.asc())
|
||||
.limit(50)
|
||||
)
|
||||
extended_context["thread"] = [
|
||||
_serialize_row(m) for m in thread_result.scalars().all()
|
||||
]
|
||||
|
||||
# Similar entities via unified_search
|
||||
try:
|
||||
from app.plugins.builtins.unified_search.search_engine import (
|
||||
find_similar_all_types,
|
||||
)
|
||||
|
||||
extended_context["similar"] = await find_similar_all_types(
|
||||
db, entity_type, eid, tid, limit=5
|
||||
)
|
||||
except Exception:
|
||||
extended_context["similar"] = {}
|
||||
|
||||
# Generate extended suggestion with deep analysis prompt
|
||||
model = settings.model or "ollama/deepseek-v4"
|
||||
try:
|
||||
response = await litellm.acompletion(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": DEEP_SYSTEM_PROMPT},
|
||||
{
|
||||
"role": "user",
|
||||
"content": json.dumps(
|
||||
extended_context, default=str, ensure_ascii=False
|
||||
),
|
||||
},
|
||||
],
|
||||
temperature=0.3,
|
||||
max_tokens=800,
|
||||
response_format={"type": "json_object"},
|
||||
api_key=OLLAMA_API_KEY,
|
||||
)
|
||||
content = response.choices[0].message.content
|
||||
if not content:
|
||||
return
|
||||
suggestion_data = json.loads(content)
|
||||
if not suggestion_data.get("title") or not suggestion_data.get("content"):
|
||||
return
|
||||
if not isinstance(suggestion_data.get("actions"), list):
|
||||
suggestion_data["actions"] = []
|
||||
confidence = suggestion_data.get("confidence", 0.5)
|
||||
try:
|
||||
confidence = float(confidence)
|
||||
except (TypeError, ValueError):
|
||||
confidence = 0.5
|
||||
suggestion_data["confidence"] = max(0.0, min(1.0, confidence))
|
||||
valid_types = {"info", "warning", "action", "insight"}
|
||||
if suggestion_data.get("suggestion_type") not in valid_types:
|
||||
suggestion_data["suggestion_type"] = "insight"
|
||||
except Exception:
|
||||
logger.exception("deep_analysis: LLM call failed")
|
||||
return
|
||||
|
||||
# Confidence threshold check
|
||||
if suggestion_data["confidence"] < settings.confidence_threshold:
|
||||
logger.debug(
|
||||
"deep_analysis: confidence %s below threshold %s",
|
||||
suggestion_data["confidence"],
|
||||
settings.confidence_threshold,
|
||||
)
|
||||
return
|
||||
|
||||
# Save extended suggestion
|
||||
suggestion = ProactiveSuggestion(
|
||||
tenant_id=tid,
|
||||
user_id=uid,
|
||||
entity_type=entity_type,
|
||||
entity_id=eid,
|
||||
suggestion_type=suggestion_data["suggestion_type"],
|
||||
title=f"[Tiefenanalyse] {suggestion_data['title']}",
|
||||
content=suggestion_data["content"],
|
||||
confidence=suggestion_data["confidence"],
|
||||
actions=suggestion_data["actions"],
|
||||
context_snapshot=extended_context,
|
||||
)
|
||||
db.add(suggestion)
|
||||
await db.flush()
|
||||
|
||||
suggestion_dict = {
|
||||
"id": str(suggestion.id),
|
||||
"entity_type": suggestion.entity_type,
|
||||
"entity_id": str(suggestion.entity_id) if suggestion.entity_id else None,
|
||||
"suggestion_type": suggestion.suggestion_type,
|
||||
"title": suggestion.title,
|
||||
"content": suggestion.content,
|
||||
"confidence": suggestion.confidence,
|
||||
"actions": suggestion.actions,
|
||||
"created_at": suggestion.created_at.isoformat()
|
||||
if suggestion.created_at
|
||||
else None,
|
||||
"is_dismissed": suggestion.is_dismissed,
|
||||
"is_acted_upon": suggestion.is_acted_upon,
|
||||
}
|
||||
|
||||
# Push via SSE
|
||||
await push_suggestion(str(uid), suggestion_dict)
|
||||
|
||||
# Notification for urgent suggestions
|
||||
if suggestion.suggestion_type == "warning":
|
||||
try:
|
||||
await create_notification(
|
||||
db,
|
||||
tid,
|
||||
uid,
|
||||
type="ai_suggestion_urgent",
|
||||
title=suggestion.title,
|
||||
body=suggestion.content[:200],
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("deep_analysis: notification failed")
|
||||
|
||||
await db.commit()
|
||||
logger.info(
|
||||
"deep_analysis: generated suggestion %s for %s/%s",
|
||||
suggestion.id,
|
||||
entity_type,
|
||||
entity_id,
|
||||
)
|
||||
@@ -0,0 +1,50 @@
|
||||
-- ai_proactive plugin: suggestions, context log, user settings
|
||||
|
||||
-- Suggestions Store
|
||||
CREATE TABLE IF NOT EXISTS ai_proactive_suggestions (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tenant_id UUID NOT NULL,
|
||||
user_id UUID NOT NULL,
|
||||
entity_type VARCHAR(50) NOT NULL,
|
||||
entity_id UUID,
|
||||
suggestion_type VARCHAR(50) NOT NULL DEFAULT 'info', -- info, warning, action, insight
|
||||
title VARCHAR(200) NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
confidence FLOAT NOT NULL DEFAULT 0.5,
|
||||
actions JSONB NOT NULL DEFAULT '[]', -- [{method, path, body, description}]
|
||||
is_dismissed BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
is_acted_upon BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
context_snapshot JSONB NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
expires_at TIMESTAMPTZ
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_suggestions_user ON ai_proactive_suggestions (tenant_id, user_id, is_dismissed, created_at DESC);
|
||||
CREATE INDEX IF NOT EXISTS ix_suggestions_entity ON ai_proactive_suggestions (tenant_id, entity_type, entity_id);
|
||||
|
||||
-- Context Log (was der User wann angesehen hat)
|
||||
CREATE TABLE IF NOT EXISTS ai_proactive_context_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tenant_id UUID NOT NULL,
|
||||
user_id UUID NOT NULL,
|
||||
page VARCHAR(200) NOT NULL,
|
||||
entity_type VARCHAR(50),
|
||||
entity_id UUID,
|
||||
entity_data JSONB NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_context_log_user_time ON ai_proactive_context_log (tenant_id, user_id, created_at DESC);
|
||||
|
||||
-- User Settings für proactive AI
|
||||
CREATE TABLE IF NOT EXISTS ai_proactive_settings (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tenant_id UUID NOT NULL,
|
||||
user_id UUID NOT NULL,
|
||||
enabled BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
suggestion_categories JSONB NOT NULL DEFAULT '["mail","tasks","contacts","companies","insights"]',
|
||||
confidence_threshold FLOAT NOT NULL DEFAULT 0.5,
|
||||
rate_limit_seconds INT NOT NULL DEFAULT 10,
|
||||
model VARCHAR(100) NOT NULL DEFAULT 'gpt-4o-mini',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
UNIQUE(tenant_id, user_id)
|
||||
);
|
||||
@@ -0,0 +1,111 @@
|
||||
"""SQLAlchemy models for the AI Proactive plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.core.db import Base, TenantMixin
|
||||
|
||||
|
||||
class ProactiveSuggestion(Base, TenantMixin):
|
||||
"""A proactive AI suggestion generated from user context."""
|
||||
|
||||
__tablename__ = "ai_proactive_suggestions"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_suggestions_user",
|
||||
"tenant_id",
|
||||
"user_id",
|
||||
"is_dismissed",
|
||||
"created_at",
|
||||
),
|
||||
Index("ix_suggestions_entity", "tenant_id", "entity_type", "entity_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
entity_id: Mapped[uuid.UUID | None] = mapped_column(PGUUID(as_uuid=True), nullable=True)
|
||||
suggestion_type: Mapped[str] = mapped_column(
|
||||
String(50), nullable=False, default="info"
|
||||
)
|
||||
title: Mapped[str] = mapped_column(String(200), nullable=False)
|
||||
content: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.5)
|
||||
actions: Mapped[list[Any]] = mapped_column(JSONB, nullable=False, default=list)
|
||||
is_dismissed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
is_acted_upon: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
context_snapshot: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSONB, nullable=False, default=dict
|
||||
)
|
||||
expires_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
|
||||
|
||||
class ContextLog(Base, TenantMixin):
|
||||
"""Log of user context changes (page views, entity selections)."""
|
||||
|
||||
__tablename__ = "ai_proactive_context_log"
|
||||
__table_args__ = (
|
||||
Index("ix_context_log_user_time", "tenant_id", "user_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
page: Mapped[str] = mapped_column(String(200), nullable=False)
|
||||
entity_type: Mapped[str | None] = mapped_column(String(50), nullable=True)
|
||||
entity_id: Mapped[uuid.UUID | None] = mapped_column(PGUUID(as_uuid=True), nullable=True)
|
||||
entity_data: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSONB, nullable=False, default=dict
|
||||
)
|
||||
|
||||
|
||||
class ProactiveSettings(Base, TenantMixin):
|
||||
"""Per-user settings for proactive AI."""
|
||||
|
||||
__tablename__ = "ai_proactive_settings"
|
||||
__table_args__ = (
|
||||
Index("ix_proactive_settings_user", "tenant_id", "user_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
suggestion_categories: Mapped[list[Any]] = mapped_column(
|
||||
JSONB, nullable=False, default=lambda: ["mail", "tasks", "contacts", "companies", "insights"]
|
||||
)
|
||||
confidence_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.5
|
||||
)
|
||||
rate_limit_seconds: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=10
|
||||
)
|
||||
model: Mapped[str] = mapped_column(String(100), nullable=False, default="ollama/deepseek-v4")
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Proactive AI Agent plugin — context-aware suggestions built on
|
||||
unified_search and ai_assistant.
|
||||
|
||||
Listens to context-change events, gathers entity data, generates LLM-powered
|
||||
suggestions, pushes them via SSE, and registers AI tools for the assistant.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from app.plugins.base import BasePlugin
|
||||
from app.plugins.manifest import PluginManifest, PluginRouteDef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AIProactivePlugin(BasePlugin):
|
||||
"""Proactive KI Agent that monitors user context and suggests actions."""
|
||||
|
||||
manifest = PluginManifest(
|
||||
name="ai_proactive",
|
||||
version="1.0.0",
|
||||
display_name="Proaktiver KI Agent",
|
||||
description=(
|
||||
"Überwacht den User-Kontext, generiert proaktiv Vorschläge per LLM "
|
||||
"und pusht diese via SSE. Nutzt unified_search und ai_assistant."
|
||||
),
|
||||
dependencies=["ai_assistant", "unified_search"],
|
||||
routes=[
|
||||
PluginRouteDef(
|
||||
path="/api/v1/ai-proactive",
|
||||
module="app.plugins.builtins.ai_proactive.routes",
|
||||
router_attr="router",
|
||||
),
|
||||
],
|
||||
events=["context.view_changed", "context.entity_selected"],
|
||||
migrations=["0001_initial.sql"],
|
||||
permissions=[
|
||||
"ai_proactive:read",
|
||||
"ai_proactive:write",
|
||||
"ai_proactive:config",
|
||||
],
|
||||
is_core=False,
|
||||
)
|
||||
|
||||
async def on_activate(self, db, service_container, event_bus) -> None:
|
||||
"""Register context tools and subscribe to events."""
|
||||
await super().on_activate(db, service_container, event_bus)
|
||||
try:
|
||||
from app.plugins.builtins.ai_proactive.context_tools import (
|
||||
register_context_tools,
|
||||
)
|
||||
from app.plugins.builtins.ai_assistant.tool_registry import (
|
||||
get_tool_registry,
|
||||
)
|
||||
|
||||
register_context_tools(get_tool_registry())
|
||||
logger.info("AI Proactive context tools registered")
|
||||
except Exception:
|
||||
logger.exception("Failed to register AI Proactive context tools")
|
||||
|
||||
async def on_deactivate(self, db, service_container, event_bus) -> None:
|
||||
"""Unregister tools and event listeners."""
|
||||
try:
|
||||
from app.plugins.builtins.ai_assistant.tool_registry import (
|
||||
get_tool_registry,
|
||||
)
|
||||
|
||||
get_tool_registry().unregister_plugin("ai_proactive")
|
||||
logger.info("AI Proactive context tools unregistered")
|
||||
except Exception:
|
||||
logger.exception("Failed to unregister AI Proactive context tools")
|
||||
await super().on_deactivate(db, service_container, event_bus)
|
||||
|
||||
# ─── Event Handlers ───
|
||||
|
||||
async def on_context_view_changed(self, payload: dict[str, Any]) -> None:
|
||||
"""Handle context.view_changed event."""
|
||||
from app.plugins.builtins.ai_proactive.services import handle_context_change
|
||||
|
||||
await handle_context_change(payload)
|
||||
|
||||
async def on_context_entity_selected(self, payload: dict[str, Any]) -> None:
|
||||
"""Handle context.entity_selected event."""
|
||||
from app.plugins.builtins.ai_proactive.services import handle_context_change
|
||||
|
||||
await handle_context_change(payload)
|
||||
|
||||
def get_notification_types(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"type_key": "ai_suggestion",
|
||||
"category": "ai",
|
||||
"label": "KI Vorschlag",
|
||||
"description": "Proaktiver KI-Vorschlag",
|
||||
"is_enabled_by_default": True,
|
||||
},
|
||||
{
|
||||
"type_key": "ai_suggestion_urgent",
|
||||
"category": "ai",
|
||||
"label": "Dringender KI Vorschlag",
|
||||
"description": "Dringender proaktiver KI-Vorschlag",
|
||||
"is_enabled_by_default": True,
|
||||
},
|
||||
]
|
||||
@@ -0,0 +1,317 @@
|
||||
"""API routes for the AI Proactive plugin.
|
||||
|
||||
Endpoints: context tracking, suggestions CRUD, SSE stream, settings, stats.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.db import get_db, set_tenant_context
|
||||
from app.core.event_bus import get_event_bus
|
||||
from app.deps import get_current_user, require_permission
|
||||
from app.plugins.builtins.ai_proactive.models import (
|
||||
ContextLog,
|
||||
ProactiveSettings,
|
||||
ProactiveSuggestion,
|
||||
)
|
||||
from app.plugins.builtins.ai_proactive.schemas import (
|
||||
ActRequest,
|
||||
ActResponse,
|
||||
ContextReport,
|
||||
SettingsResponse,
|
||||
SettingsUpdate,
|
||||
SuggestionListResponse,
|
||||
SuggestionResponse,
|
||||
StatsResponse,
|
||||
)
|
||||
from app.plugins.builtins.ai_proactive.services import (
|
||||
execute_suggested_action,
|
||||
get_active_suggestions,
|
||||
get_sse_queue,
|
||||
get_stats,
|
||||
get_user_settings,
|
||||
mark_dismissed,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/v1/ai-proactive", tags=["ai-proactive"])
|
||||
|
||||
|
||||
def _suggestion_to_response(s: ProactiveSuggestion) -> SuggestionResponse:
|
||||
"""Convert a ProactiveSuggestion model to SuggestionResponse."""
|
||||
return SuggestionResponse(
|
||||
id=str(s.id),
|
||||
entity_type=s.entity_type,
|
||||
entity_id=str(s.entity_id) if s.entity_id else None,
|
||||
suggestion_type=s.suggestion_type,
|
||||
title=s.title,
|
||||
content=s.content,
|
||||
confidence=s.confidence,
|
||||
actions=s.actions or [],
|
||||
created_at=s.created_at,
|
||||
is_dismissed=s.is_dismissed,
|
||||
is_acted_upon=s.is_acted_upon,
|
||||
)
|
||||
|
||||
|
||||
def _settings_to_response(s: ProactiveSettings) -> SettingsResponse:
|
||||
"""Convert ProactiveSettings model to SettingsResponse."""
|
||||
return SettingsResponse(
|
||||
enabled=s.enabled,
|
||||
suggestion_categories=s.suggestion_categories or [],
|
||||
confidence_threshold=s.confidence_threshold,
|
||||
rate_limit_seconds=s.rate_limit_seconds,
|
||||
model=s.model,
|
||||
)
|
||||
|
||||
|
||||
# ─── Context Tracking ───
|
||||
|
||||
|
||||
@router.post("/context", dependencies=[Depends(require_permission("ai_proactive:read"))])
|
||||
async def report_context(
|
||||
context: ContextReport,
|
||||
current_user: dict[str, Any] = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Frontend reports current context (page, entity).
|
||||
|
||||
Stores context log entry and publishes event for proactive engine.
|
||||
"""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
user_id = uuid.UUID(current_user["user_id"])
|
||||
|
||||
# Parse entity_id if present
|
||||
entity_id: uuid.UUID | None = None
|
||||
if context.entity_id:
|
||||
try:
|
||||
entity_id = uuid.UUID(context.entity_id)
|
||||
except (ValueError, TypeError):
|
||||
entity_id = None
|
||||
|
||||
# Store context log
|
||||
log_entry = ContextLog(
|
||||
tenant_id=tenant_id,
|
||||
user_id=user_id,
|
||||
page=context.page,
|
||||
entity_type=context.entity_type,
|
||||
entity_id=entity_id,
|
||||
entity_data=context.entity_data or {},
|
||||
)
|
||||
db.add(log_entry)
|
||||
await db.flush()
|
||||
|
||||
# Publish event
|
||||
event_bus = get_event_bus()
|
||||
event_name = (
|
||||
"context.entity_selected"
|
||||
if context.entity_type and context.entity_id
|
||||
else "context.view_changed"
|
||||
)
|
||||
payload = {
|
||||
"user_id": str(user_id),
|
||||
"tenant_id": str(tenant_id),
|
||||
"page": context.page,
|
||||
"entity_type": context.entity_type,
|
||||
"entity_id": str(entity_id) if entity_id else None,
|
||||
"entity_data": context.entity_data or {},
|
||||
}
|
||||
await event_bus.publish(event_name, payload)
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
# ─── Suggestions ───
|
||||
|
||||
|
||||
@router.get(
|
||||
"/suggestions",
|
||||
dependencies=[Depends(require_permission("ai_proactive:read"))],
|
||||
response_model=SuggestionListResponse,
|
||||
)
|
||||
async def get_suggestions(
|
||||
entity_type: str | None = Query(None),
|
||||
entity_id: str | None = Query(None),
|
||||
limit: int = Query(10, ge=1, le=100),
|
||||
current_user: dict[str, Any] = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get active suggestions for the current user."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
user_id = uuid.UUID(current_user["user_id"])
|
||||
|
||||
eid: uuid.UUID | None = None
|
||||
if entity_id:
|
||||
try:
|
||||
eid = uuid.UUID(entity_id)
|
||||
except (ValueError, TypeError):
|
||||
eid = None
|
||||
|
||||
suggestions = await get_active_suggestions(
|
||||
db, tenant_id, user_id, entity_type=entity_type, entity_id=eid, limit=limit
|
||||
)
|
||||
items = [_suggestion_to_response(s) for s in suggestions]
|
||||
return SuggestionListResponse(items=items, total=len(items))
|
||||
|
||||
|
||||
@router.post(
|
||||
"/suggestions/{suggestion_id}/dismiss",
|
||||
dependencies=[Depends(require_permission("ai_proactive:write"))],
|
||||
)
|
||||
async def dismiss_suggestion(
|
||||
suggestion_id: str,
|
||||
current_user: dict[str, Any] = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Mark a suggestion as dismissed."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
user_id = uuid.UUID(current_user["user_id"])
|
||||
|
||||
try:
|
||||
sid = uuid.UUID(suggestion_id)
|
||||
except (ValueError, TypeError):
|
||||
raise HTTPException(status_code=400, detail="Invalid suggestion ID")
|
||||
|
||||
success = await mark_dismissed(db, sid, user_id, tenant_id)
|
||||
if not success:
|
||||
raise HTTPException(status_code=404, detail="Suggestion not found")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.post(
|
||||
"/suggestions/{suggestion_id}/act",
|
||||
dependencies=[Depends(require_permission("ai_proactive:write"))],
|
||||
response_model=ActResponse,
|
||||
)
|
||||
async def act_on_suggestion(
|
||||
suggestion_id: str,
|
||||
action: ActRequest,
|
||||
current_user: dict[str, Any] = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Execute a suggested action."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
user_id = uuid.UUID(current_user["user_id"])
|
||||
|
||||
try:
|
||||
sid = uuid.UUID(suggestion_id)
|
||||
except (ValueError, TypeError):
|
||||
raise HTTPException(status_code=400, detail="Invalid suggestion ID")
|
||||
|
||||
result = await execute_suggested_action(
|
||||
db, sid, action.action_index, user_id, tenant_id, current_user
|
||||
)
|
||||
return ActResponse(
|
||||
success=result.get("success", False),
|
||||
data=result.get("data"),
|
||||
error=result.get("error"),
|
||||
)
|
||||
|
||||
|
||||
# ─── SSE Stream ───
|
||||
|
||||
|
||||
@router.get(
|
||||
"/suggestions/stream",
|
||||
dependencies=[Depends(require_permission("ai_proactive:read"))],
|
||||
)
|
||||
async def stream_suggestions(
|
||||
current_user: dict[str, Any] = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""SSE stream: push new suggestions in real-time.
|
||||
|
||||
Uses an asyncio.Queue per user. Heartbeat every 30s.
|
||||
"""
|
||||
user_id = str(current_user["user_id"])
|
||||
|
||||
async def event_generator():
|
||||
queue = get_sse_queue(user_id)
|
||||
while True:
|
||||
try:
|
||||
suggestion = await asyncio.wait_for(queue.get(), timeout=30)
|
||||
yield f"data: {json.dumps(suggestion, default=str)}\n\n"
|
||||
except asyncio.TimeoutError:
|
||||
yield ": keepalive\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
event_generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# ─── Settings ───
|
||||
|
||||
|
||||
@router.get(
|
||||
"/settings",
|
||||
dependencies=[Depends(require_permission("ai_proactive:read"))],
|
||||
response_model=SettingsResponse,
|
||||
)
|
||||
async def get_settings(
|
||||
current_user: dict[str, Any] = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get proactive AI settings for the current user."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
user_id = uuid.UUID(current_user["user_id"])
|
||||
settings = await get_user_settings(db, tenant_id, user_id)
|
||||
return _settings_to_response(settings)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/settings",
|
||||
dependencies=[Depends(require_permission("ai_proactive:config"))],
|
||||
response_model=SettingsResponse,
|
||||
)
|
||||
async def update_settings(
|
||||
settings_update: SettingsUpdate,
|
||||
current_user: dict[str, Any] = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Update proactive AI settings for the current user."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
user_id = uuid.UUID(current_user["user_id"])
|
||||
|
||||
settings = await get_user_settings(db, tenant_id, user_id)
|
||||
|
||||
update_data = settings_update.model_dump(exclude_unset=True)
|
||||
for field, val in update_data.items():
|
||||
setattr(settings, field, val)
|
||||
|
||||
await db.flush()
|
||||
return _settings_to_response(settings)
|
||||
|
||||
|
||||
# ─── Stats ───
|
||||
|
||||
|
||||
@router.get(
|
||||
"/stats",
|
||||
dependencies=[Depends(require_permission("ai_proactive:read"))],
|
||||
response_model=StatsResponse,
|
||||
)
|
||||
async def get_stats_endpoint(
|
||||
current_user: dict[str, Any] = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Get proactive AI usage statistics for the current user."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
user_id = uuid.UUID(current_user["user_id"])
|
||||
stats = await get_stats(db, tenant_id, user_id)
|
||||
return StatsResponse(**stats)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Pydantic v2 schemas for the AI Proactive plugin API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ContextReport(BaseModel):
|
||||
"""Frontend reports the current page/entity context."""
|
||||
|
||||
page: str = Field(..., description="Current page path")
|
||||
entity_type: str | None = Field(None, description="Entity type being viewed")
|
||||
entity_id: str | None = Field(None, description="Entity ID being viewed")
|
||||
entity_data: dict[str, Any] | None = Field(
|
||||
None, description="Additional entity metadata"
|
||||
)
|
||||
|
||||
|
||||
class SuggestionAction(BaseModel):
|
||||
"""A suggested CRM action."""
|
||||
|
||||
method: str = Field(..., description="HTTP method")
|
||||
path: str = Field(..., description="API path")
|
||||
body: dict[str, Any] | None = Field(None, description="Request body")
|
||||
description: str = Field(..., description="Human-readable description")
|
||||
|
||||
|
||||
class SuggestionResponse(BaseModel):
|
||||
"""API response for a single suggestion."""
|
||||
|
||||
id: str
|
||||
entity_type: str
|
||||
entity_id: str | None
|
||||
suggestion_type: str
|
||||
title: str
|
||||
content: str
|
||||
confidence: float
|
||||
actions: list[dict[str, Any]]
|
||||
created_at: datetime
|
||||
is_dismissed: bool
|
||||
is_acted_upon: bool = False
|
||||
|
||||
|
||||
class SuggestionListResponse(BaseModel):
|
||||
"""Paginated list of suggestions."""
|
||||
|
||||
items: list[SuggestionResponse]
|
||||
total: int
|
||||
|
||||
|
||||
class ActRequest(BaseModel):
|
||||
"""Execute a suggested action by index."""
|
||||
|
||||
action_index: int = Field(..., ge=0, description="Index into actions array")
|
||||
|
||||
|
||||
class ActResponse(BaseModel):
|
||||
"""Result of executing a suggested action."""
|
||||
|
||||
success: bool
|
||||
data: dict[str, Any] | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class SettingsResponse(BaseModel):
|
||||
"""Proactive AI settings for the current user."""
|
||||
|
||||
enabled: bool
|
||||
suggestion_categories: list[str]
|
||||
confidence_threshold: float
|
||||
rate_limit_seconds: int
|
||||
model: str
|
||||
available_models: list[str] = Field(default_factory=lambda: [
|
||||
'ollama/deepseek-v4',
|
||||
'ollama/deepseek-v4-pro',
|
||||
'ollama/llama3.2',
|
||||
'ollama/gpt-4o-mini',
|
||||
'gpt-4o-mini',
|
||||
])
|
||||
|
||||
|
||||
class SettingsUpdate(BaseModel):
|
||||
"""Partial update for proactive AI settings."""
|
||||
|
||||
enabled: bool | None = None
|
||||
suggestion_categories: list[str] | None = None
|
||||
confidence_threshold: float | None = None
|
||||
rate_limit_seconds: int | None = None
|
||||
model: str | None = None
|
||||
|
||||
|
||||
class StatsResponse(BaseModel):
|
||||
"""Proactive AI usage statistics."""
|
||||
|
||||
total_suggestions: int = 0
|
||||
dismissed: int = 0
|
||||
acted_upon: int = 0
|
||||
active: int = 0
|
||||
dismiss_rate: float = 0.0
|
||||
act_rate: float = 0.0
|
||||
@@ -0,0 +1,748 @@
|
||||
"""Proactive Engine — core logic for context-aware AI suggestions.
|
||||
|
||||
Handles context changes, gathers entity data, generates LLM-powered
|
||||
suggestions, pushes via SSE, and manages suggestion lifecycle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.cache import get_cache
|
||||
from app.core.db import create_db_session, get_session_factory
|
||||
from app.core.notifications import create_notification
|
||||
from app.models.audit import AuditLog
|
||||
from app.models.company import Company
|
||||
from app.models.contact import CompanyContact, Contact
|
||||
from app.plugins.builtins.ai_proactive.models import (
|
||||
ContextLog,
|
||||
ProactiveSettings,
|
||||
ProactiveSuggestion,
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
litellm.suppress_debug_info = True
|
||||
|
||||
OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
|
||||
|
||||
# ─── SSE Push Infrastructure ───
|
||||
|
||||
_sse_queues: dict[str, asyncio.Queue[dict[str, Any]]] = {}
|
||||
|
||||
|
||||
def get_sse_queue(user_id: str) -> asyncio.Queue[dict[str, Any]]:
|
||||
"""Get or create SSE queue for a user."""
|
||||
if user_id not in _sse_queues:
|
||||
_sse_queues[user_id] = asyncio.Queue()
|
||||
return _sse_queues[user_id]
|
||||
|
||||
|
||||
async def push_suggestion(user_id: str, suggestion: dict[str, Any]) -> None:
|
||||
"""Push suggestion to user's SSE queue."""
|
||||
queue = get_sse_queue(user_id)
|
||||
await queue.put(suggestion)
|
||||
|
||||
|
||||
# ─── Rate Limiting ───
|
||||
|
||||
|
||||
async def is_rate_limited(
|
||||
tenant_id: uuid.UUID, user_id: uuid.UUID, rate_limit_seconds: int
|
||||
) -> bool:
|
||||
"""Check if user is rate-limited using Redis.
|
||||
|
||||
Returns True if rate-limited (key exists), False otherwise.
|
||||
Sets a key with TTL = rate_limit_seconds on first call.
|
||||
"""
|
||||
try:
|
||||
r = get_cache()
|
||||
key = f"ai_proactive:rate:{user_id}"
|
||||
existing = await r.get(key)
|
||||
if existing is not None:
|
||||
return True
|
||||
await r.setex(key, rate_limit_seconds, "1")
|
||||
return False
|
||||
except Exception:
|
||||
logger.exception("Rate-limit check failed, allowing request")
|
||||
return False
|
||||
|
||||
|
||||
# ─── Settings Helpers ───
|
||||
|
||||
|
||||
async def get_user_settings(
|
||||
db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID
|
||||
) -> ProactiveSettings:
|
||||
"""Get proactive AI settings for user, create defaults if not exist."""
|
||||
result = await db.execute(
|
||||
select(ProactiveSettings)
|
||||
.where(ProactiveSettings.tenant_id == tenant_id)
|
||||
.where(ProactiveSettings.user_id == user_id)
|
||||
.limit(1)
|
||||
)
|
||||
settings = result.scalar_one_or_none()
|
||||
if settings is None:
|
||||
settings = ProactiveSettings(
|
||||
tenant_id=tenant_id,
|
||||
user_id=user_id, enabled=True,
|
||||
suggestion_categories=["mail", "tasks", "contacts", "companies", "insights"],
|
||||
confidence_threshold=0.5,
|
||||
rate_limit_seconds=10,
|
||||
model="ollama/deepseek-v4",
|
||||
)
|
||||
db.add(settings)
|
||||
await db.flush()
|
||||
return settings
|
||||
|
||||
|
||||
# ─── Context Gathering ───
|
||||
|
||||
|
||||
def _serialize_row(row: Any) -> dict[str, Any]:
|
||||
"""Serialize a SQLAlchemy model instance to a dict."""
|
||||
if row is None:
|
||||
return {}
|
||||
result: dict[str, Any] = {}
|
||||
for column in row.__table__.columns:
|
||||
val = getattr(row, column.name)
|
||||
if isinstance(val, datetime):
|
||||
result[column.name] = val.isoformat()
|
||||
elif isinstance(val, uuid.UUID):
|
||||
result[column.name] = str(val)
|
||||
else:
|
||||
result[column.name] = val
|
||||
return result
|
||||
|
||||
|
||||
async def gather_context(
|
||||
db: AsyncSession, entity_type: str, entity_id: uuid.UUID, tenant_id: uuid.UUID
|
||||
) -> dict[str, Any]:
|
||||
"""Collect context data for an entity.
|
||||
|
||||
Gathers related data from contacts, companies, mails, calendar events,
|
||||
audit logs, and semantically similar entities via unified_search.
|
||||
"""
|
||||
context: dict[str, Any] = {
|
||||
"entity_type": entity_type,
|
||||
"entity_id": str(entity_id),
|
||||
}
|
||||
|
||||
if entity_type == "contact":
|
||||
# Contact data
|
||||
result = await db.execute(
|
||||
select(Contact)
|
||||
.where(Contact.id == entity_id)
|
||||
.where(Contact.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
contact = result.scalar_one_or_none()
|
||||
context["contact"] = _serialize_row(contact) if contact else None
|
||||
|
||||
# Last 10 mails
|
||||
from app.plugins.builtins.mail.models import Mail
|
||||
|
||||
mail_result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.contact_id == entity_id)
|
||||
.where(Mail.tenant_id == tenant_id)
|
||||
.order_by(Mail.received_at.desc())
|
||||
.limit(10)
|
||||
)
|
||||
context["mails"] = [_serialize_row(m) for m in mail_result.scalars().all()]
|
||||
|
||||
# Company via company_contacts
|
||||
cc_result = await db.execute(
|
||||
select(CompanyContact)
|
||||
.where(CompanyContact.contact_id == entity_id)
|
||||
.where(CompanyContact.tenant_id == tenant_id)
|
||||
.limit(5)
|
||||
)
|
||||
companies: list[dict[str, Any]] = []
|
||||
for cc in cc_result.scalars().all():
|
||||
comp_result = await db.execute(
|
||||
select(Company)
|
||||
.where(Company.id == cc.company_id)
|
||||
.where(Company.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
comp = comp_result.scalar_one_or_none()
|
||||
if comp:
|
||||
comp_data = _serialize_row(comp)
|
||||
comp_data["role_at_company"] = cc.role_at_company
|
||||
comp_data["is_primary"] = cc.is_primary
|
||||
companies.append(comp_data)
|
||||
context["company"] = companies[0] if companies else None
|
||||
context["companies"] = companies
|
||||
|
||||
# Upcoming calendar events
|
||||
from app.plugins.builtins.calendar.models import CalendarEntry, CalendarEntryLink
|
||||
|
||||
now = datetime.now(UTC)
|
||||
event_result = await db.execute(
|
||||
select(CalendarEntry)
|
||||
.join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id)
|
||||
.where(CalendarEntryLink.entity_type == "contact")
|
||||
.where(CalendarEntryLink.entity_id == entity_id)
|
||||
.where(CalendarEntry.tenant_id == tenant_id)
|
||||
.where(CalendarEntry.start_at > now)
|
||||
.order_by(CalendarEntry.start_at.asc())
|
||||
.limit(5)
|
||||
)
|
||||
context["events"] = [_serialize_row(e) for e in event_result.scalars().all()]
|
||||
|
||||
# Last 20 audit log entries
|
||||
audit_result = await db.execute(
|
||||
select(AuditLog)
|
||||
.where(AuditLog.entity_id == entity_id)
|
||||
.where(AuditLog.tenant_id == tenant_id)
|
||||
.order_by(AuditLog.timestamp.desc())
|
||||
.limit(20)
|
||||
)
|
||||
context["activities"] = [_serialize_row(a) for a in audit_result.scalars().all()]
|
||||
|
||||
elif entity_type == "mail":
|
||||
from app.plugins.builtins.mail.models import Mail
|
||||
|
||||
result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.id == entity_id)
|
||||
.where(Mail.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
mail = result.scalar_one_or_none()
|
||||
context["mail"] = _serialize_row(mail) if mail else None
|
||||
|
||||
if mail and mail.contact_id:
|
||||
contact_result = await db.execute(
|
||||
select(Contact)
|
||||
.where(Contact.id == mail.contact_id)
|
||||
.where(Contact.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
contact = contact_result.scalar_one_or_none()
|
||||
context["contact"] = _serialize_row(contact) if contact else None
|
||||
|
||||
if mail and mail.thread_id:
|
||||
thread_result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.thread_id == mail.thread_id)
|
||||
.where(Mail.tenant_id == tenant_id)
|
||||
.order_by(Mail.received_at.asc())
|
||||
.limit(20)
|
||||
)
|
||||
context["thread"] = [_serialize_row(m) for m in thread_result.scalars().all()]
|
||||
|
||||
if mail and mail.company_id:
|
||||
comp_result = await db.execute(
|
||||
select(Company)
|
||||
.where(Company.id == mail.company_id)
|
||||
.where(Company.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
comp = comp_result.scalar_one_or_none()
|
||||
context["company"] = _serialize_row(comp) if comp else None
|
||||
|
||||
elif entity_type == "company":
|
||||
result = await db.execute(
|
||||
select(Company)
|
||||
.where(Company.id == entity_id)
|
||||
.where(Company.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
company = result.scalar_one_or_none()
|
||||
context["company"] = _serialize_row(company) if company else None
|
||||
|
||||
# Contacts via company_contacts
|
||||
cc_result = await db.execute(
|
||||
select(CompanyContact)
|
||||
.where(CompanyContact.company_id == entity_id)
|
||||
.where(CompanyContact.tenant_id == tenant_id)
|
||||
)
|
||||
contacts: list[dict[str, Any]] = []
|
||||
for cc in cc_result.scalars().all():
|
||||
contact_result = await db.execute(
|
||||
select(Contact)
|
||||
.where(Contact.id == cc.contact_id)
|
||||
.where(Contact.tenant_id == tenant_id)
|
||||
.limit(1)
|
||||
)
|
||||
contact = contact_result.scalar_one_or_none()
|
||||
if contact:
|
||||
contact_data = _serialize_row(contact)
|
||||
contact_data["role_at_company"] = cc.role_at_company
|
||||
contact_data["is_primary"] = cc.is_primary
|
||||
contacts.append(contact_data)
|
||||
context["contacts"] = contacts
|
||||
|
||||
# Mails for this company
|
||||
from app.plugins.builtins.mail.models import Mail
|
||||
|
||||
mail_result = await db.execute(
|
||||
select(Mail)
|
||||
.where(Mail.company_id == entity_id)
|
||||
.where(Mail.tenant_id == tenant_id)
|
||||
.order_by(Mail.received_at.desc())
|
||||
.limit(10)
|
||||
)
|
||||
context["mails"] = [_serialize_row(m) for m in mail_result.scalars().all()]
|
||||
|
||||
# Upcoming events
|
||||
from app.plugins.builtins.calendar.models import CalendarEntry, CalendarEntryLink
|
||||
|
||||
now = datetime.now(UTC)
|
||||
event_result = await db.execute(
|
||||
select(CalendarEntry)
|
||||
.join(CalendarEntryLink, CalendarEntryLink.entry_id == CalendarEntry.id)
|
||||
.where(CalendarEntryLink.entity_type == "company")
|
||||
.where(CalendarEntryLink.entity_id == entity_id)
|
||||
.where(CalendarEntry.tenant_id == tenant_id)
|
||||
.where(CalendarEntry.start_at > now)
|
||||
.order_by(CalendarEntry.start_at.asc())
|
||||
.limit(5)
|
||||
)
|
||||
context["events"] = [_serialize_row(e) for e in event_result.scalars().all()]
|
||||
|
||||
elif entity_type == "file":
|
||||
# Basic file info via raw SQL (file model may vary)
|
||||
try:
|
||||
file_result = await db.execute(
|
||||
text("SELECT * FROM files WHERE id = :fid AND tenant_id = :tid"),
|
||||
{"fid": entity_id, "tid": tenant_id},
|
||||
)
|
||||
file_row = file_result.mappings().first()
|
||||
context["file"] = dict(file_row) if file_row else None
|
||||
except Exception:
|
||||
context["file"] = None
|
||||
|
||||
# Linked entities via entity_links (if table exists)
|
||||
try:
|
||||
links_result = await db.execute(
|
||||
text(
|
||||
"SELECT * FROM entity_links WHERE entity_id = :eid AND tenant_id = :tid LIMIT 20"
|
||||
),
|
||||
{"eid": entity_id, "tid": tenant_id},
|
||||
)
|
||||
context["linked_entities"] = [dict(r) for r in links_result.mappings().all()]
|
||||
except Exception:
|
||||
context["linked_entities"] = []
|
||||
|
||||
# Semantically similar entities via unified_search
|
||||
try:
|
||||
from app.plugins.builtins.unified_search.search_engine import (
|
||||
find_similar_all_types,
|
||||
)
|
||||
|
||||
context["similar"] = await find_similar_all_types(
|
||||
db, entity_type, entity_id, tenant_id, limit=3
|
||||
)
|
||||
except Exception:
|
||||
context["similar"] = {}
|
||||
|
||||
return context
|
||||
|
||||
|
||||
# ─── Suggestion Generation ───
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """Du bist ein proaktiver KI-Assistent für ein CRM. Analysiere den Kontext und generiere Vorschläge.
|
||||
|
||||
Antworte mit JSON:
|
||||
{
|
||||
"suggestion_type": "info|warning|action|insight",
|
||||
"title": "Kurzer Titel (max 200 Zeichen)",
|
||||
"content": "Beschreibung des Vorschlags",
|
||||
"confidence": 0.0-1.0,
|
||||
"actions": [{"method": "GET|POST|PUT|DELETE", "path": "/api/v1/...", "body": {}, "description": "..."}]
|
||||
}
|
||||
|
||||
- suggestion_type: info=Information, warning=Warnung, action=Aktionsvorschlag, insight=Erkenntnis
|
||||
- actions: Vorgeschlagene CRM-Aktionen die der User ausführen kann
|
||||
- confidence: Wie sicher bist du dir (0.0=unsicher, 1.0=sehr sicher)
|
||||
- Antworte nur mit gültigem JSON, kein Markdown"""
|
||||
|
||||
|
||||
async def generate_suggestion(
|
||||
context_data: dict[str, Any], settings: ProactiveSettings
|
||||
) -> dict[str, Any] | None:
|
||||
"""LLM generates a suggestion from context data.
|
||||
|
||||
Returns dict with suggestion_type, title, content, confidence, actions
|
||||
or None on failure.
|
||||
"""
|
||||
model = settings.model or "ollama/deepseek-v4"
|
||||
try:
|
||||
response = await litellm.acompletion(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{
|
||||
"role": "user",
|
||||
"content": json.dumps(context_data, default=str, ensure_ascii=False),
|
||||
},
|
||||
],
|
||||
temperature=0.3,
|
||||
max_tokens=500,
|
||||
response_format={"type": "json_object"},
|
||||
api_key=OLLAMA_API_KEY,
|
||||
)
|
||||
content = response.choices[0].message.content
|
||||
if not content:
|
||||
return None
|
||||
result = json.loads(content)
|
||||
# Validate required fields
|
||||
if not result.get("title") or not result.get("content"):
|
||||
return None
|
||||
# Ensure actions is a list
|
||||
if not isinstance(result.get("actions"), list):
|
||||
result["actions"] = []
|
||||
# Clamp confidence
|
||||
confidence = result.get("confidence", 0.5)
|
||||
try:
|
||||
confidence = float(confidence)
|
||||
except (TypeError, ValueError):
|
||||
confidence = 0.5
|
||||
result["confidence"] = max(0.0, min(1.0, confidence))
|
||||
# Validate suggestion_type
|
||||
valid_types = {"info", "warning", "action", "insight"}
|
||||
if result.get("suggestion_type") not in valid_types:
|
||||
result["suggestion_type"] = "info"
|
||||
return result
|
||||
except Exception:
|
||||
logger.exception("Failed to generate suggestion via LLM")
|
||||
return None
|
||||
|
||||
|
||||
# ─── Main Handler ───
|
||||
|
||||
|
||||
async def handle_context_change(payload: dict[str, Any]) -> None:
|
||||
"""Main handler: Context-Change → Suggestion generation.
|
||||
|
||||
1. Rate-limit check
|
||||
2. Settings check (enabled?)
|
||||
3. Gather context data
|
||||
4. Generate suggestion via LLM
|
||||
5. Confidence threshold check
|
||||
6. Save suggestion to DB
|
||||
7. Push via SSE
|
||||
8. Notification for urgent suggestions
|
||||
9. Enqueue background deep analysis job
|
||||
"""
|
||||
user_id_str = payload.get("user_id")
|
||||
tenant_id_str = payload.get("tenant_id")
|
||||
entity_type = payload.get("entity_type")
|
||||
entity_id_str = payload.get("entity_id")
|
||||
|
||||
if not user_id_str or not tenant_id_str or not entity_type:
|
||||
logger.warning("handle_context_change: missing required fields in payload")
|
||||
return
|
||||
|
||||
try:
|
||||
tenant_id = uuid.UUID(tenant_id_str)
|
||||
user_id = uuid.UUID(user_id_str)
|
||||
except (ValueError, TypeError):
|
||||
logger.warning("handle_context_change: invalid UUID in payload")
|
||||
return
|
||||
|
||||
entity_id: uuid.UUID | None = None
|
||||
if entity_id_str:
|
||||
try:
|
||||
entity_id = uuid.UUID(entity_id_str)
|
||||
except (ValueError, TypeError):
|
||||
entity_id = None
|
||||
|
||||
if entity_id is None:
|
||||
logger.debug("handle_context_change: no entity_id, skipping")
|
||||
return
|
||||
|
||||
async with create_db_session(tenant_id) as db:
|
||||
# Get settings
|
||||
settings = await get_user_settings(db, tenant_id, user_id)
|
||||
if not settings.enabled:
|
||||
logger.debug("handle_context_change: proactive AI disabled for user")
|
||||
return
|
||||
|
||||
# Rate limit check
|
||||
if await is_rate_limited(tenant_id, user_id, settings.rate_limit_seconds):
|
||||
logger.debug("handle_context_change: rate limited")
|
||||
return
|
||||
|
||||
# Gather context
|
||||
context_data = await gather_context(db, entity_type, entity_id, tenant_id)
|
||||
|
||||
# Generate suggestion
|
||||
suggestion_data = await generate_suggestion(context_data, settings)
|
||||
if suggestion_data is None:
|
||||
logger.debug("handle_context_change: no suggestion generated")
|
||||
return
|
||||
|
||||
# Confidence threshold check
|
||||
if suggestion_data["confidence"] < settings.confidence_threshold:
|
||||
logger.debug(
|
||||
"handle_context_change: confidence %s below threshold %s",
|
||||
suggestion_data["confidence"],
|
||||
settings.confidence_threshold,
|
||||
)
|
||||
return
|
||||
|
||||
# Save suggestion
|
||||
suggestion = ProactiveSuggestion(
|
||||
tenant_id=tenant_id,
|
||||
user_id=user_id,
|
||||
entity_type=entity_type,
|
||||
entity_id=entity_id,
|
||||
suggestion_type=suggestion_data["suggestion_type"],
|
||||
title=suggestion_data["title"],
|
||||
content=suggestion_data["content"],
|
||||
confidence=suggestion_data["confidence"],
|
||||
actions=suggestion_data["actions"],
|
||||
context_snapshot=context_data,
|
||||
)
|
||||
db.add(suggestion)
|
||||
await db.flush()
|
||||
|
||||
# Build response dict for SSE
|
||||
suggestion_dict = {
|
||||
"id": str(suggestion.id),
|
||||
"entity_type": suggestion.entity_type,
|
||||
"entity_id": str(suggestion.entity_id) if suggestion.entity_id else None,
|
||||
"suggestion_type": suggestion.suggestion_type,
|
||||
"title": suggestion.title,
|
||||
"content": suggestion.content,
|
||||
"confidence": suggestion.confidence,
|
||||
"actions": suggestion.actions,
|
||||
"created_at": suggestion.created_at.isoformat() if suggestion.created_at else None,
|
||||
"is_dismissed": suggestion.is_dismissed,
|
||||
"is_acted_upon": suggestion.is_acted_upon,
|
||||
}
|
||||
|
||||
# Push via SSE
|
||||
await push_suggestion(str(user_id), suggestion_dict)
|
||||
|
||||
# Notification for urgent suggestions
|
||||
if suggestion.suggestion_type == "warning":
|
||||
try:
|
||||
await create_notification(
|
||||
db,
|
||||
tenant_id,
|
||||
user_id,
|
||||
type="ai_suggestion_urgent",
|
||||
title=suggestion.title,
|
||||
body=suggestion.content[:200],
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to create notification")
|
||||
else:
|
||||
try:
|
||||
await create_notification(
|
||||
db,
|
||||
tenant_id,
|
||||
user_id,
|
||||
type="ai_suggestion",
|
||||
title=suggestion.title,
|
||||
body=suggestion.content[:200],
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to create notification")
|
||||
|
||||
await db.commit()
|
||||
|
||||
# Enqueue background deep analysis job
|
||||
try:
|
||||
from app.core.jobs import enqueue_job
|
||||
|
||||
await enqueue_job(
|
||||
"deep_analysis",
|
||||
entity_type,
|
||||
str(entity_id),
|
||||
str(user_id),
|
||||
str(tenant_id),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to enqueue deep_analysis job")
|
||||
|
||||
|
||||
# ─── Suggestion CRUD ───
|
||||
|
||||
|
||||
async def get_active_suggestions(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
user_id: uuid.UUID,
|
||||
entity_type: str | None = None,
|
||||
entity_id: uuid.UUID | None = None,
|
||||
limit: int = 10,
|
||||
) -> list[ProactiveSuggestion]:
|
||||
"""Get active (non-dismissed, not expired) suggestions for user."""
|
||||
now = datetime.now(UTC)
|
||||
stmt = (
|
||||
select(ProactiveSuggestion)
|
||||
.where(ProactiveSuggestion.tenant_id == tenant_id)
|
||||
.where(ProactiveSuggestion.user_id == user_id)
|
||||
.where(ProactiveSuggestion.is_dismissed == False) # noqa: E712
|
||||
.where(
|
||||
(ProactiveSuggestion.expires_at.is_(None))
|
||||
| (ProactiveSuggestion.expires_at > now)
|
||||
)
|
||||
)
|
||||
if entity_type:
|
||||
stmt = stmt.where(ProactiveSuggestion.entity_type == entity_type)
|
||||
if entity_id:
|
||||
stmt = stmt.where(ProactiveSuggestion.entity_id == entity_id)
|
||||
stmt = stmt.order_by(ProactiveSuggestion.created_at.desc()).limit(limit)
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
|
||||
async def mark_dismissed(
|
||||
db: AsyncSession,
|
||||
suggestion_id: uuid.UUID,
|
||||
user_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
) -> bool:
|
||||
"""Mark suggestion as dismissed. Returns True if found and updated."""
|
||||
result = await db.execute(
|
||||
select(ProactiveSuggestion)
|
||||
.where(ProactiveSuggestion.id == suggestion_id)
|
||||
.where(ProactiveSuggestion.tenant_id == tenant_id)
|
||||
.where(ProactiveSuggestion.user_id == user_id)
|
||||
.limit(1)
|
||||
)
|
||||
suggestion = result.scalar_one_or_none()
|
||||
if suggestion is None:
|
||||
return False
|
||||
suggestion.is_dismissed = True
|
||||
await db.flush()
|
||||
return True
|
||||
|
||||
|
||||
async def execute_suggested_action(
|
||||
db: AsyncSession,
|
||||
suggestion_id: uuid.UUID,
|
||||
action_index: int,
|
||||
user_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
user_context: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Execute a suggested action.
|
||||
|
||||
1. Load suggestion
|
||||
2. Get action from actions[action_index]
|
||||
3. Execute via internal HTTP call (httpx)
|
||||
4. Mark suggestion as is_acted_upon=True
|
||||
5. Return result
|
||||
"""
|
||||
import httpx
|
||||
|
||||
result = await db.execute(
|
||||
select(ProactiveSuggestion)
|
||||
.where(ProactiveSuggestion.id == suggestion_id)
|
||||
.where(ProactiveSuggestion.tenant_id == tenant_id)
|
||||
.where(ProactiveSuggestion.user_id == user_id)
|
||||
.limit(1)
|
||||
)
|
||||
suggestion = result.scalar_one_or_none()
|
||||
if suggestion is None:
|
||||
return {"success": False, "error": "Suggestion not found"}
|
||||
|
||||
actions = suggestion.actions or []
|
||||
if action_index < 0 or action_index >= len(actions):
|
||||
return {"success": False, "error": "Invalid action index"}
|
||||
|
||||
action = actions[action_index]
|
||||
method = action.get("method", "GET").upper()
|
||||
path = action.get("path", "")
|
||||
body = action.get("body")
|
||||
|
||||
if not path:
|
||||
return {"success": False, "error": "No path in action"}
|
||||
|
||||
# Build internal URL
|
||||
from app.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
base_url = getattr(settings, "internal_base_url", "http://localhost:8000")
|
||||
url = f"{base_url}{path}"
|
||||
|
||||
# Build headers from user context (session cookie)
|
||||
headers: dict[str, str] = {"Content-Type": "application/json"}
|
||||
cookie_name = getattr(settings, "session_cookie_name", "session")
|
||||
session_id = user_context.get("session_id", "")
|
||||
if session_id:
|
||||
headers["Cookie"] = f"{cookie_name}={session_id}"
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
resp = await client.request(
|
||||
method,
|
||||
url,
|
||||
json=body if body else None,
|
||||
headers=headers,
|
||||
)
|
||||
data = None
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception:
|
||||
data = {"status_code": resp.status_code, "text": resp.text}
|
||||
if resp.status_code < 400:
|
||||
suggestion.is_acted_upon = True
|
||||
await db.flush()
|
||||
return {"success": True, "data": data}
|
||||
return {"success": False, "error": f"HTTP {resp.status_code}", "data": data}
|
||||
except Exception as e:
|
||||
logger.exception("Failed to execute suggested action")
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
|
||||
# ─── Stats ───
|
||||
|
||||
|
||||
async def get_stats(
|
||||
db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID
|
||||
) -> dict[str, Any]:
|
||||
"""Get proactive AI usage statistics for a user."""
|
||||
base_filter = (
|
||||
ProactiveSuggestion.tenant_id == tenant_id,
|
||||
ProactiveSuggestion.user_id == user_id,
|
||||
)
|
||||
total_result = await db.execute(
|
||||
select(func.count()).select_from(ProactiveSuggestion).where(*base_filter)
|
||||
)
|
||||
total = total_result.scalar() or 0
|
||||
|
||||
dismissed_result = await db.execute(
|
||||
select(func.count())
|
||||
.select_from(ProactiveSuggestion)
|
||||
.where(*base_filter, ProactiveSuggestion.is_dismissed == True) # noqa: E712
|
||||
)
|
||||
dismissed = dismissed_result.scalar() or 0
|
||||
|
||||
acted_result = await db.execute(
|
||||
select(func.count())
|
||||
.select_from(ProactiveSuggestion)
|
||||
.where(*base_filter, ProactiveSuggestion.is_acted_upon == True) # noqa: E712
|
||||
)
|
||||
acted_upon = acted_result.scalar() or 0
|
||||
|
||||
active = total - dismissed
|
||||
dismiss_rate = (dismissed / total) if total > 0 else 0.0
|
||||
act_rate = (acted_upon / total) if total > 0 else 0.0
|
||||
|
||||
return {
|
||||
"total_suggestions": total,
|
||||
"dismissed": dismissed,
|
||||
"acted_upon": acted_upon,
|
||||
"active": active,
|
||||
"dismiss_rate": round(dismiss_rate, 4),
|
||||
"act_rate": round(act_rate, 4),
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Unified Search plugin — hybrid FTS + semantic search across all CRM data."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.plugins.builtins.unified_search.plugin import UnifiedSearchPlugin
|
||||
|
||||
__all__ = ["UnifiedSearchPlugin"]
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Embedding pipeline using LiteLLM with configurable Ollama Cloud provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import logging
|
||||
import uuid
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import litellm
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_INPUT_CHARS = 8000
|
||||
|
||||
DEFAULT_EMBEDDING_MODEL = os.environ.get('SEARCH_EMBEDDING_MODEL', 'ollama/nomic-embed-text')
|
||||
OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
|
||||
|
||||
|
||||
async def generate_embedding(text: str, model: str | None = None) -> list[float]:
|
||||
"""Generate a single embedding via LiteLLM.
|
||||
|
||||
Args:
|
||||
text: Input text (truncated to 8000 chars).
|
||||
model: Embedding model name (default: ollama/nomic-embed-text).
|
||||
|
||||
Returns:
|
||||
Embedding vector as list of floats.
|
||||
"""
|
||||
model = model or DEFAULT_EMBEDDING_MODEL
|
||||
truncated = text[:MAX_INPUT_CHARS]
|
||||
try:
|
||||
response = await litellm.aembedding(
|
||||
model=model,
|
||||
input=truncated,
|
||||
api_key=OLLAMA_API_KEY,
|
||||
)
|
||||
return response.data[0]["embedding"]
|
||||
except Exception:
|
||||
logger.exception("Failed to generate embedding")
|
||||
return []
|
||||
|
||||
|
||||
async def generate_embeddings_batch(
|
||||
texts: list[str], model: str | None = None
|
||||
) -> list[list[float]]:
|
||||
"""Generate embeddings for multiple texts in a single API call.
|
||||
|
||||
Args:
|
||||
texts: List of input texts.
|
||||
model: Embedding model name (default: ollama/nomic-embed-text).
|
||||
|
||||
Returns:
|
||||
List of embedding vectors.
|
||||
"""
|
||||
model = model or DEFAULT_EMBEDDING_MODEL
|
||||
truncated = [t[:MAX_INPUT_CHARS] for t in texts]
|
||||
try:
|
||||
response = await litellm.aembedding(
|
||||
model=model,
|
||||
input=truncated,
|
||||
api_key=OLLAMA_API_KEY,
|
||||
)
|
||||
return [d["embedding"] for d in response.data]
|
||||
except Exception:
|
||||
logger.exception("Failed to generate batch embeddings")
|
||||
return [[] for _ in texts]
|
||||
|
||||
|
||||
async def index_entity(
|
||||
entity_type: str,
|
||||
entity_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
db: AsyncSession,
|
||||
) -> bool:
|
||||
"""Generate and store embedding for a single entity.
|
||||
|
||||
Uses the provider registry to get embedding text, generates embedding,
|
||||
and updates the entity's embedding column.
|
||||
|
||||
Returns True on success, False on failure.
|
||||
"""
|
||||
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
|
||||
|
||||
registry = get_search_registry()
|
||||
provider = registry.get(entity_type)
|
||||
if provider is None:
|
||||
logger.warning("No provider for entity_type=%s", entity_type)
|
||||
return False
|
||||
|
||||
try:
|
||||
text = await provider.get_embedding_text(db, entity_id, tenant_id)
|
||||
if not text.strip():
|
||||
logger.debug("Empty embedding text for %s/%s", entity_type, entity_id)
|
||||
return False
|
||||
|
||||
embedding = await generate_embedding(text)
|
||||
if not embedding:
|
||||
return False
|
||||
|
||||
# Update the entity's embedding column
|
||||
from sqlalchemy import text as sql_text
|
||||
|
||||
table_map = {
|
||||
"contact": "contacts",
|
||||
"company": "companies",
|
||||
"mail": "mails",
|
||||
"file": "files",
|
||||
"event": "calendar_entries",
|
||||
}
|
||||
table = table_map.get(entity_type)
|
||||
if not table:
|
||||
logger.warning("Unknown entity_type=%s for embedding storage", entity_type)
|
||||
return False
|
||||
|
||||
sql = sql_text(
|
||||
f"UPDATE {table} SET embedding = cast(:emb AS vector) "
|
||||
f"WHERE id = :eid AND tenant_id = :tid"
|
||||
)
|
||||
await db.execute(
|
||||
sql,
|
||||
{"emb": str(embedding), "eid": entity_id, "tid": tenant_id},
|
||||
)
|
||||
await db.commit()
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("Failed to index entity %s/%s", entity_type, entity_id)
|
||||
return False
|
||||
@@ -0,0 +1,246 @@
|
||||
"""ARQ background jobs for the Unified Search plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from app.core.db import get_session_factory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
BATCH_SIZE = 100
|
||||
|
||||
|
||||
def _parse_id(id_str: str) -> uuid.UUID:
|
||||
"""Parse a string to UUID."""
|
||||
if isinstance(id_str, uuid.UUID):
|
||||
return id_str
|
||||
return uuid.UUID(str(id_str))
|
||||
|
||||
|
||||
async def index_mails(ctx: dict[str, Any], mail_ids: list[str]) -> None:
|
||||
"""Index mails: generate and store embeddings."""
|
||||
from app.plugins.builtins.unified_search.embedding import index_entity
|
||||
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
for mail_id in mail_ids:
|
||||
try:
|
||||
eid = _parse_id(mail_id)
|
||||
# tenant_id is derived from the mail itself
|
||||
from sqlalchemy import text
|
||||
result = await db.execute(
|
||||
text("SELECT tenant_id FROM mails WHERE id = :mid"),
|
||||
{"mid": eid},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
continue
|
||||
tenant_id = row["tenant_id"]
|
||||
await index_entity("mail", eid, tenant_id, db)
|
||||
except Exception:
|
||||
logger.exception("Failed to index mail %s", mail_id)
|
||||
|
||||
|
||||
async def index_file(ctx: dict[str, Any], file_id: str) -> None:
|
||||
"""Index a file: extract text, store content_text, generate embedding."""
|
||||
from sqlalchemy import text
|
||||
from app.plugins.builtins.unified_search.text_extraction import extract_text_from_file
|
||||
from app.plugins.builtins.unified_search.embedding import generate_embedding
|
||||
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
try:
|
||||
eid = _parse_id(file_id)
|
||||
result = await db.execute(
|
||||
text("SELECT tenant_id, storage_path, mime_type FROM files WHERE id = :fid"),
|
||||
{"fid": eid},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
logger.warning("File not found: %s", file_id)
|
||||
return
|
||||
|
||||
tenant_id = row["tenant_id"]
|
||||
storage_path = row["storage_path"]
|
||||
mime_type = row["mime_type"]
|
||||
|
||||
# Extract text
|
||||
content_text = await extract_text_from_file(storage_path, mime_type)
|
||||
|
||||
# Store content_text
|
||||
await db.execute(
|
||||
text("UPDATE files SET content_text = :ct WHERE id = :fid"),
|
||||
{"ct": content_text, "fid": eid},
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
# Generate embedding from extracted text + filename
|
||||
result_name = await db.execute(
|
||||
text("SELECT name FROM files WHERE id = :fid"),
|
||||
{"fid": eid},
|
||||
)
|
||||
name_row = result_name.mappings().first()
|
||||
name = name_row["name"] if name_row else ""
|
||||
embedding_text = f"{name} {content_text[:5000]}"
|
||||
|
||||
if embedding_text.strip():
|
||||
embedding = await generate_embedding(embedding_text)
|
||||
if embedding:
|
||||
await db.execute(
|
||||
text("UPDATE files SET embedding = cast(:emb AS vector) WHERE id = :fid"),
|
||||
{"emb": str(embedding), "fid": eid},
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
logger.info("Indexed file %s", file_id)
|
||||
except Exception:
|
||||
logger.exception("Failed to index file %s", file_id)
|
||||
|
||||
|
||||
async def index_contact(ctx: dict[str, Any], contact_id: str) -> None:
|
||||
"""Index a contact: generate and store embedding."""
|
||||
from sqlalchemy import text
|
||||
from app.plugins.builtins.unified_search.embedding import index_entity
|
||||
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
try:
|
||||
eid = _parse_id(contact_id)
|
||||
result = await db.execute(
|
||||
text("SELECT tenant_id FROM contacts WHERE id = :cid"),
|
||||
{"cid": eid},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return
|
||||
await index_entity("contact", eid, row["tenant_id"], db)
|
||||
except Exception:
|
||||
logger.exception("Failed to index contact %s", contact_id)
|
||||
|
||||
|
||||
async def index_company(ctx: dict[str, Any], company_id: str) -> None:
|
||||
"""Index a company: generate and store embedding."""
|
||||
from sqlalchemy import text
|
||||
from app.plugins.builtins.unified_search.embedding import index_entity
|
||||
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
try:
|
||||
eid = _parse_id(company_id)
|
||||
result = await db.execute(
|
||||
text("SELECT tenant_id FROM companies WHERE id = :cid"),
|
||||
{"cid": eid},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return
|
||||
await index_entity("company", eid, row["tenant_id"], db)
|
||||
except Exception:
|
||||
logger.exception("Failed to index company %s", company_id)
|
||||
|
||||
|
||||
async def index_event(ctx: dict[str, Any], event_id: str) -> None:
|
||||
"""Index a calendar event: generate and store embedding."""
|
||||
from sqlalchemy import text
|
||||
from app.plugins.builtins.unified_search.embedding import index_entity
|
||||
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
try:
|
||||
eid = _parse_id(event_id)
|
||||
result = await db.execute(
|
||||
text("SELECT tenant_id FROM calendar_entries WHERE id = :eid"),
|
||||
{"eid": eid},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return
|
||||
await index_entity("event", eid, row["tenant_id"], db)
|
||||
except Exception:
|
||||
logger.exception("Failed to index event %s", event_id)
|
||||
|
||||
|
||||
async def reindex(ctx: dict[str, Any], entity_type: str) -> None:
|
||||
"""Reindex all entities of a given type with pagination."""
|
||||
from sqlalchemy import text
|
||||
from app.plugins.builtins.unified_search.embedding import index_entity
|
||||
|
||||
table_map = {
|
||||
"contact": "contacts",
|
||||
"company": "companies",
|
||||
"mail": "mails",
|
||||
"file": "files",
|
||||
"event": "calendar_entries",
|
||||
}
|
||||
table = table_map.get(entity_type)
|
||||
if not table:
|
||||
logger.warning("Unknown entity_type for reindex: %s", entity_type)
|
||||
return
|
||||
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
offset = 0
|
||||
while True:
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"SELECT id, tenant_id FROM {table} "
|
||||
f"WHERE deleted_at IS NULL ORDER BY created_at LIMIT :lim OFFSET :off"
|
||||
),
|
||||
{"lim": BATCH_SIZE, "off": offset},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
try:
|
||||
await index_entity(
|
||||
entity_type,
|
||||
row["id"],
|
||||
row["tenant_id"],
|
||||
db,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Reindex failed for %s/%s", entity_type, row["id"])
|
||||
offset += BATCH_SIZE
|
||||
|
||||
logger.info("Reindex complete for %s", entity_type)
|
||||
|
||||
|
||||
async def embedding_batch(ctx: dict[str, Any]) -> None:
|
||||
"""Periodic job: find entities without embeddings and index them."""
|
||||
from sqlalchemy import text
|
||||
from app.plugins.builtins.unified_search.embedding import index_entity
|
||||
|
||||
table_map = {
|
||||
"contact": "contacts",
|
||||
"company": "companies",
|
||||
"mail": "mails",
|
||||
"file": "files",
|
||||
"event": "calendar_entries",
|
||||
}
|
||||
|
||||
factory = get_session_factory()
|
||||
async with factory() as db:
|
||||
for etype, table in table_map.items():
|
||||
try:
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"SELECT id, tenant_id FROM {table} "
|
||||
f"WHERE deleted_at IS NULL AND embedding IS NULL "
|
||||
f"LIMIT :lim"
|
||||
),
|
||||
{"lim": BATCH_SIZE},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
for row in rows:
|
||||
try:
|
||||
await index_entity(etype, row["id"], row["tenant_id"], db)
|
||||
except Exception:
|
||||
logger.exception("Batch index failed for %s/%s", etype, row["id"])
|
||||
except Exception:
|
||||
logger.exception("Batch query failed for %s", etype)
|
||||
|
||||
logger.info("Embedding batch job complete")
|
||||
@@ -0,0 +1,154 @@
|
||||
-- Unified Search: pgvector extension, FTS triggers, provider registry
|
||||
|
||||
-- pgvector extension
|
||||
CREATE EXTENSION IF NOT EXISTS vector;
|
||||
|
||||
-- ─── Provider Registry Table ───
|
||||
CREATE TABLE IF NOT EXISTS unified_search_providers (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tenant_id UUID NOT NULL,
|
||||
entity_type VARCHAR(50) NOT NULL,
|
||||
plugin_name VARCHAR(80) NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
config JSONB NOT NULL DEFAULT '{}'::jsonb,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
UNIQUE(tenant_id, entity_type)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_usp_tenant ON unified_search_providers(tenant_id);
|
||||
|
||||
-- ─── Index Log Table ───
|
||||
CREATE TABLE IF NOT EXISTS unified_search_index_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
tenant_id UUID NOT NULL,
|
||||
entity_type VARCHAR(50) NOT NULL,
|
||||
entity_id UUID NOT NULL,
|
||||
action VARCHAR(20) NOT NULL,
|
||||
status VARCHAR(20) NOT NULL DEFAULT 'pending',
|
||||
error_message TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_usil_tenant ON unified_search_index_log(tenant_id);
|
||||
CREATE INDEX IF NOT EXISTS ix_usil_entity ON unified_search_index_log(entity_type, entity_id);
|
||||
|
||||
-- ─── FTS: Mails ───
|
||||
ALTER TABLE mails ADD COLUMN IF NOT EXISTS body_tsv tsvector;
|
||||
|
||||
CREATE OR REPLACE FUNCTION mails_tsv_trigger() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.body_tsv :=
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.subject, '')), 'A') ||
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.from_address, '')), 'B') ||
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.to_addresses, '')), 'C') ||
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.body_text, '')), 'D');
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS mails_tsv_update ON mails;
|
||||
CREATE TRIGGER mails_tsv_update
|
||||
BEFORE INSERT OR UPDATE ON mails
|
||||
FOR EACH ROW EXECUTE FUNCTION mails_tsv_trigger();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_mails_body_tsv ON mails USING gin(body_tsv);
|
||||
|
||||
-- ─── FTS: Contacts ───
|
||||
ALTER TABLE contacts ADD COLUMN IF NOT EXISTS search_tsv tsvector;
|
||||
|
||||
CREATE OR REPLACE FUNCTION contacts_tsv_trigger() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_tsv :=
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.first_name, '') || ' ' || coalesce(NEW.last_name, '')), 'A') ||
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.email, '')), 'B') ||
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.phone, '') || ' ' || coalesce(NEW.mobile, '')), 'C') ||
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.notes, '')), 'D');
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS contacts_tsv_update ON contacts;
|
||||
CREATE TRIGGER contacts_tsv_update
|
||||
BEFORE INSERT OR UPDATE ON contacts
|
||||
FOR EACH ROW EXECUTE FUNCTION contacts_tsv_trigger();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_contacts_search_tsv ON contacts USING gin(search_tsv);
|
||||
|
||||
-- ─── FTS: Calendar Entries ───
|
||||
ALTER TABLE calendar_entries ADD COLUMN IF NOT EXISTS search_tsv tsvector;
|
||||
|
||||
CREATE OR REPLACE FUNCTION calendar_entries_tsv_trigger() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_tsv :=
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.title, '')), 'A') ||
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.description, '')), 'B') ||
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.location, '')), 'C');
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS calendar_entries_tsv_update ON calendar_entries;
|
||||
CREATE TRIGGER calendar_entries_tsv_update
|
||||
BEFORE INSERT OR UPDATE ON calendar_entries
|
||||
FOR EACH ROW EXECUTE FUNCTION calendar_entries_tsv_trigger();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_cal_entries_search_tsv ON calendar_entries USING gin(search_tsv);
|
||||
|
||||
-- ─── FTS: Files (content_text added here, not in 0002, to avoid trigger dependency) ───
|
||||
ALTER TABLE files ADD COLUMN IF NOT EXISTS content_text text;
|
||||
ALTER TABLE files ADD COLUMN IF NOT EXISTS content_tsv tsvector;
|
||||
|
||||
CREATE OR REPLACE FUNCTION files_tsv_trigger() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.content_tsv :=
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.name, '')), 'A') ||
|
||||
setweight(to_tsvector('pg_catalog.german', coalesce(NEW.content_text, '')), 'D');
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS files_tsv_update ON files;
|
||||
CREATE TRIGGER files_tsv_update
|
||||
BEFORE INSERT OR UPDATE ON files
|
||||
FOR EACH ROW EXECUTE FUNCTION files_tsv_trigger();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_files_content_tsv ON files USING gin(content_tsv);
|
||||
|
||||
-- ─── FTS: Tags ───
|
||||
ALTER TABLE tags ADD COLUMN IF NOT EXISTS search_tsv tsvector;
|
||||
|
||||
CREATE OR REPLACE FUNCTION tags_tsv_trigger() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_tsv := to_tsvector('pg_catalog.german', coalesce(NEW.name, ''));
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS tags_tsv_update ON tags;
|
||||
CREATE TRIGGER tags_tsv_update
|
||||
BEFORE INSERT OR UPDATE ON tags
|
||||
FOR EACH ROW EXECUTE FUNCTION tags_tsv_trigger();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_tags_search_tsv ON tags USING gin(search_tsv);
|
||||
|
||||
-- ─── FTS: Audit Log ───
|
||||
ALTER TABLE audit_log ADD COLUMN IF NOT EXISTS search_tsv tsvector;
|
||||
|
||||
CREATE OR REPLACE FUNCTION audit_log_tsv_trigger() RETURNS trigger AS $$
|
||||
BEGIN
|
||||
NEW.search_tsv :=
|
||||
to_tsvector('pg_catalog.german',
|
||||
coalesce(NEW.entity_type, '') || ' ' ||
|
||||
coalesce(NEW.action, '') || ' ' ||
|
||||
coalesce(NEW.user_id::text, ''));
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS audit_log_tsv_update ON audit_log;
|
||||
CREATE TRIGGER audit_log_tsv_update
|
||||
BEFORE INSERT OR UPDATE ON audit_log
|
||||
FOR EACH ROW EXECUTE FUNCTION audit_log_tsv_trigger();
|
||||
|
||||
CREATE INDEX IF NOT EXISTS ix_audit_log_search_tsv ON audit_log USING gin(search_tsv);
|
||||
@@ -0,0 +1,24 @@
|
||||
-- Unified Search: Embedding columns and HNSW indexes
|
||||
|
||||
-- ─── Mails ───
|
||||
ALTER TABLE mails ADD COLUMN IF NOT EXISTS embedding vector(768);
|
||||
CREATE INDEX IF NOT EXISTS ix_mails_embedding ON mails USING hnsw(embedding vector_cosine_ops);
|
||||
|
||||
-- ─── Contacts ───
|
||||
ALTER TABLE contacts ADD COLUMN IF NOT EXISTS embedding vector(768);
|
||||
CREATE INDEX IF NOT EXISTS ix_contacts_embedding ON contacts USING hnsw(embedding vector_cosine_ops);
|
||||
|
||||
-- ─── Companies ───
|
||||
ALTER TABLE companies ADD COLUMN IF NOT EXISTS embedding vector(768);
|
||||
CREATE INDEX IF NOT EXISTS ix_companies_embedding ON companies USING hnsw(embedding vector_cosine_ops);
|
||||
|
||||
-- ─── Files (content_text already added in 0001) ───
|
||||
ALTER TABLE files ADD COLUMN IF NOT EXISTS embedding vector(768);
|
||||
CREATE INDEX IF NOT EXISTS ix_files_embedding ON files USING hnsw(embedding vector_cosine_ops);
|
||||
|
||||
-- ─── Calendar Entries ───
|
||||
ALTER TABLE calendar_entries ADD COLUMN IF NOT EXISTS embedding vector(768);
|
||||
CREATE INDEX IF NOT EXISTS ix_cal_embedding ON calendar_entries USING hnsw(embedding vector_cosine_ops);
|
||||
|
||||
-- ─── Tags (shorter dimension for short text) ───
|
||||
ALTER TABLE tags ADD COLUMN IF NOT EXISTS embedding vector(384);
|
||||
@@ -0,0 +1,50 @@
|
||||
"""SQLAlchemy models for the Unified Search plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Index, String, Text
|
||||
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from app.core.db import Base, TenantMixin
|
||||
|
||||
|
||||
class SearchProviderRegistry(Base, TenantMixin):
|
||||
"""Registry of active search providers per tenant."""
|
||||
|
||||
__tablename__ = "unified_search_providers"
|
||||
__table_args__ = (
|
||||
Index("ix_usp_tenant", "tenant_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
plugin_name: Mapped[str] = mapped_column(String(80), nullable=False)
|
||||
is_active: Mapped[bool] = mapped_column(
|
||||
nullable=False, default=True
|
||||
)
|
||||
config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
|
||||
|
||||
|
||||
class SearchIndexLog(Base, TenantMixin):
|
||||
"""Log of indexing actions for audit and debugging."""
|
||||
|
||||
__tablename__ = "unified_search_index_log"
|
||||
__table_args__ = (
|
||||
Index("ix_usil_tenant", "tenant_id"),
|
||||
Index("ix_usil_entity", "entity_type", "entity_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
|
||||
)
|
||||
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
entity_id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), nullable=False)
|
||||
action: Mapped[str] = mapped_column(String(20), nullable=False)
|
||||
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Unified Search plugin class and manifest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from app.plugins.base import BasePlugin
|
||||
from app.plugins.manifest import PluginManifest, PluginRouteDef
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UnifiedSearchPlugin(BasePlugin):
|
||||
"""Hybrid full-text and semantic search across all CRM entities."""
|
||||
|
||||
manifest = PluginManifest(
|
||||
name="unified_search",
|
||||
version="1.0.0",
|
||||
display_name="Unified Search",
|
||||
description=(
|
||||
"Hybrid full-text (PostgreSQL FTS) and semantic (pgvector) search "
|
||||
"with KI query understanding and RRF rank fusion across all CRM data."
|
||||
),
|
||||
dependencies=[],
|
||||
routes=[
|
||||
PluginRouteDef(
|
||||
path="/api/v1/search",
|
||||
module="app.plugins.builtins.unified_search.routes",
|
||||
router_attr="router",
|
||||
),
|
||||
],
|
||||
events=[
|
||||
"mail.synced",
|
||||
"file.uploaded",
|
||||
"contact.created",
|
||||
"contact.updated",
|
||||
"company.created",
|
||||
"company.updated",
|
||||
"calendar.entry.created",
|
||||
],
|
||||
migrations=["0001_initial.sql", "0002_embeddings.sql"],
|
||||
permissions=["search:read", "search:admin"],
|
||||
is_core=False,
|
||||
)
|
||||
|
||||
async def on_activate(self, db, service_container, event_bus) -> None:
|
||||
"""Register search providers on activation."""
|
||||
await super().on_activate(db, service_container, event_bus)
|
||||
try:
|
||||
from app.plugins.builtins.unified_search.provider_registry import (
|
||||
auto_register_providers,
|
||||
)
|
||||
await auto_register_providers(db)
|
||||
logger.info("Unified Search providers auto-registered")
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-register search providers")
|
||||
|
||||
async def on_deactivate(self, db, service_container, event_bus) -> None:
|
||||
"""Clear provider registry on deactivation."""
|
||||
from app.plugins.builtins.unified_search.provider_registry import (
|
||||
get_search_registry,
|
||||
)
|
||||
registry = get_search_registry()
|
||||
registry.clear()
|
||||
await super().on_deactivate(db, service_container, event_bus)
|
||||
|
||||
# ─── Event Handlers ───
|
||||
|
||||
async def on_mail_synced(self, payload: dict[str, Any]) -> None:
|
||||
"""Enqueue embedding jobs for synced mails."""
|
||||
from app.core.jobs import enqueue_job
|
||||
|
||||
mail_ids = payload.get("mail_ids", [])
|
||||
if mail_ids:
|
||||
await enqueue_job("index_mails", mail_ids)
|
||||
|
||||
async def on_file_uploaded(self, payload: dict[str, Any]) -> None:
|
||||
"""Enqueue file indexing job."""
|
||||
from app.core.jobs import enqueue_job
|
||||
|
||||
file_id = payload.get("file_id")
|
||||
if file_id:
|
||||
await enqueue_job("index_file", file_id)
|
||||
|
||||
async def on_contact_created(self, payload: dict[str, Any]) -> None:
|
||||
from app.core.jobs import enqueue_job
|
||||
|
||||
contact_id = payload.get("contact_id")
|
||||
if contact_id:
|
||||
await enqueue_job("index_contact", contact_id)
|
||||
|
||||
async def on_contact_updated(self, payload: dict[str, Any]) -> None:
|
||||
from app.core.jobs import enqueue_job
|
||||
|
||||
contact_id = payload.get("contact_id")
|
||||
if contact_id:
|
||||
await enqueue_job("index_contact", contact_id)
|
||||
|
||||
async def on_company_created(self, payload: dict[str, Any]) -> None:
|
||||
from app.core.jobs import enqueue_job
|
||||
|
||||
company_id = payload.get("company_id")
|
||||
if company_id:
|
||||
await enqueue_job("index_company", company_id)
|
||||
|
||||
async def on_company_updated(self, payload: dict[str, Any]) -> None:
|
||||
from app.core.jobs import enqueue_job
|
||||
|
||||
company_id = payload.get("company_id")
|
||||
if company_id:
|
||||
await enqueue_job("index_company", company_id)
|
||||
|
||||
async def on_calendar_entry_created(self, payload: dict[str, Any]) -> None:
|
||||
from app.core.jobs import enqueue_job
|
||||
|
||||
entry_id = payload.get("entry_id")
|
||||
if entry_id:
|
||||
await enqueue_job("index_event", entry_id)
|
||||
|
||||
def get_notification_types(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"type_key": "search_error",
|
||||
"category": "search",
|
||||
"label": "Suchfehler",
|
||||
"description": "Fehler bei der Suchausführung",
|
||||
"is_enabled_by_default": True,
|
||||
},
|
||||
{
|
||||
"type_key": "search_reindex_complete",
|
||||
"category": "search",
|
||||
"label": "Reindex abgeschlossen",
|
||||
"description": "Neuindizierung abgeschlossen",
|
||||
"is_enabled_by_default": False,
|
||||
},
|
||||
]
|
||||
@@ -0,0 +1,133 @@
|
||||
"""SearchProvider protocol and global registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import uuid
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class SearchProvider(Protocol):
|
||||
"""Protocol for entity-specific search providers."""
|
||||
|
||||
entity_type: str
|
||||
|
||||
async def search_fts(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
tsquery: str,
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict]:
|
||||
"""Full-text search using PostgreSQL tsquery."""
|
||||
...
|
||||
|
||||
async def search_vector(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
embedding: list[float],
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict]:
|
||||
"""Semantic vector search using pgvector."""
|
||||
...
|
||||
|
||||
async def get_embedding_text(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
entity_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
) -> str:
|
||||
"""Get text representation for embedding generation."""
|
||||
...
|
||||
|
||||
def to_search_result(self, entity: object) -> dict:
|
||||
"""Convert an ORM entity to a search result dict."""
|
||||
...
|
||||
|
||||
|
||||
class SearchProviderRegistry:
|
||||
"""In-memory registry of search providers."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._providers: dict[str, SearchProvider] = {}
|
||||
|
||||
def register(self, provider: SearchProvider) -> None:
|
||||
"""Register a search provider."""
|
||||
self._providers[provider.entity_type] = provider
|
||||
logger.debug("Registered search provider: %s", provider.entity_type)
|
||||
|
||||
def unregister(self, entity_type: str) -> None:
|
||||
"""Unregister a search provider by entity type."""
|
||||
self._providers.pop(entity_type, None)
|
||||
|
||||
def get(self, entity_type: str) -> SearchProvider | None:
|
||||
"""Get a provider by entity type."""
|
||||
return self._providers.get(entity_type)
|
||||
|
||||
def get_all(self) -> list[SearchProvider]:
|
||||
"""Get all registered providers."""
|
||||
return list(self._providers.values())
|
||||
|
||||
def get_entity_types(self) -> list[str]:
|
||||
"""Get all registered entity types."""
|
||||
return list(self._providers.keys())
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Clear all registered providers."""
|
||||
self._providers.clear()
|
||||
|
||||
|
||||
# Global singleton
|
||||
_registry = SearchProviderRegistry()
|
||||
|
||||
|
||||
def get_search_registry() -> SearchProviderRegistry:
|
||||
"""Get the global search provider registry."""
|
||||
return _registry
|
||||
|
||||
|
||||
async def auto_register_providers(db: AsyncSession) -> None:
|
||||
"""Auto-register providers for active plugins.
|
||||
|
||||
Checks which plugins are active and registers corresponding providers.
|
||||
"""
|
||||
from app.plugins.builtins.unified_search.providers.contact_provider import (
|
||||
ContactSearchProvider,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.providers.company_provider import (
|
||||
CompanySearchProvider,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.providers.mail_provider import (
|
||||
MailSearchProvider,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.providers.file_provider import (
|
||||
FileSearchProvider,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.providers.event_provider import (
|
||||
EventSearchProvider,
|
||||
)
|
||||
|
||||
registry = get_search_registry()
|
||||
registry.clear()
|
||||
|
||||
# Register all built-in providers
|
||||
for provider_cls in [
|
||||
ContactSearchProvider,
|
||||
CompanySearchProvider,
|
||||
MailSearchProvider,
|
||||
FileSearchProvider,
|
||||
EventSearchProvider,
|
||||
]:
|
||||
try:
|
||||
registry.register(provider_cls())
|
||||
except Exception:
|
||||
logger.exception("Failed to register %s", provider_cls.__name__)
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Search providers for unified search."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Company search provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CompanySearchProvider:
|
||||
"""Search provider for Company entities."""
|
||||
|
||||
entity_type = "company"
|
||||
|
||||
async def search_fts(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
tsquery: str,
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Full-text search on companies.search_tsv."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
|
||||
FROM companies c
|
||||
WHERE c.tenant_id = :tid
|
||||
AND c.deleted_at IS NULL
|
||||
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
|
||||
ORDER BY rank DESC
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"q": tsquery, "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def search_vector(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
embedding: list[float],
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Semantic search on companies.embedding."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
|
||||
FROM companies c
|
||||
WHERE c.tenant_id = :tid
|
||||
AND c.deleted_at IS NULL
|
||||
AND c.embedding IS NOT NULL
|
||||
ORDER BY c.embedding <=> cast(:emb AS vector)
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def get_embedding_text(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
entity_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
) -> str:
|
||||
"""Get text for embedding generation."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT name, description, industry
|
||||
FROM companies
|
||||
WHERE id = :eid AND tenant_id = :tid
|
||||
"""
|
||||
)
|
||||
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return ""
|
||||
parts = [
|
||||
row.get("name", ""),
|
||||
row.get("description", ""),
|
||||
row.get("industry", ""),
|
||||
]
|
||||
return " ".join(str(p) for p in parts if p)
|
||||
|
||||
def to_search_result(self, entity: object) -> dict[str, Any]:
|
||||
"""Convert company to search result dict."""
|
||||
if isinstance(entity, dict):
|
||||
name = entity.get("name", "")
|
||||
description = entity.get("description", "") or ""
|
||||
entity_id = str(entity.get("id", ""))
|
||||
else:
|
||||
name = getattr(entity, "name", "")
|
||||
description = getattr(entity, "description", "") or ""
|
||||
entity_id = str(getattr(entity, "id", ""))
|
||||
return {
|
||||
"entity_type": self.entity_type,
|
||||
"entity_id": entity_id,
|
||||
"title": name,
|
||||
"snippet": description[:200],
|
||||
"score": 0.0,
|
||||
"data": {},
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Contact search provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.contact import Contact
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ContactSearchProvider:
|
||||
"""Search provider for Contact entities."""
|
||||
|
||||
entity_type = "contact"
|
||||
|
||||
async def search_fts(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
tsquery: str,
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Full-text search on contacts.search_tsv."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
|
||||
FROM contacts c
|
||||
WHERE c.tenant_id = :tid
|
||||
AND c.deleted_at IS NULL
|
||||
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
|
||||
ORDER BY rank DESC
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"q": tsquery, "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def search_vector(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
embedding: list[float],
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Semantic search on contacts.embedding."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
|
||||
FROM contacts c
|
||||
WHERE c.tenant_id = :tid
|
||||
AND c.deleted_at IS NULL
|
||||
AND c.embedding IS NOT NULL
|
||||
ORDER BY c.embedding <=> cast(:emb AS vector)
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def get_embedding_text(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
entity_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
) -> str:
|
||||
"""Get text for embedding generation."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT first_name, last_name, email, phone, mobile, notes
|
||||
FROM contacts
|
||||
WHERE id = :eid AND tenant_id = :tid
|
||||
"""
|
||||
)
|
||||
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return ""
|
||||
parts = [
|
||||
row.get("first_name", ""),
|
||||
row.get("last_name", ""),
|
||||
row.get("email", ""),
|
||||
row.get("phone", ""),
|
||||
row.get("mobile", ""),
|
||||
row.get("notes", ""),
|
||||
]
|
||||
return " ".join(str(p) for p in parts if p)
|
||||
|
||||
def to_search_result(self, entity: object) -> dict[str, Any]:
|
||||
"""Convert contact to search result dict."""
|
||||
if isinstance(entity, dict):
|
||||
first = entity.get("first_name", "")
|
||||
last = entity.get("last_name", "")
|
||||
email = entity.get("email", "")
|
||||
entity_id = str(entity.get("id", ""))
|
||||
else:
|
||||
first = getattr(entity, "first_name", "")
|
||||
last = getattr(entity, "last_name", "")
|
||||
email = getattr(entity, "email", "")
|
||||
entity_id = str(getattr(entity, "id", ""))
|
||||
return {
|
||||
"entity_type": self.entity_type,
|
||||
"entity_id": entity_id,
|
||||
"title": f"{first} {last}".strip(),
|
||||
"snippet": email or "",
|
||||
"score": 0.0,
|
||||
"data": {},
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Calendar event search provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class EventSearchProvider:
|
||||
"""Search provider for CalendarEntry entities."""
|
||||
|
||||
entity_type = "event"
|
||||
|
||||
async def search_fts(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
tsquery: str,
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Full-text search on calendar_entries.search_tsv."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
|
||||
FROM calendar_entries e
|
||||
WHERE e.tenant_id = :tid
|
||||
AND e.deleted_at IS NULL
|
||||
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
|
||||
ORDER BY rank DESC
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"q": tsquery, "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def search_vector(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
embedding: list[float],
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Semantic search on calendar_entries.embedding."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score
|
||||
FROM calendar_entries e
|
||||
WHERE e.tenant_id = :tid
|
||||
AND e.deleted_at IS NULL
|
||||
AND e.embedding IS NOT NULL
|
||||
ORDER BY e.embedding <=> cast(:emb AS vector)
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def get_embedding_text(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
entity_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
) -> str:
|
||||
"""Get text for embedding generation."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT title, description, location
|
||||
FROM calendar_entries
|
||||
WHERE id = :eid AND tenant_id = :tid
|
||||
"""
|
||||
)
|
||||
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return ""
|
||||
parts = [
|
||||
row.get("title", ""),
|
||||
row.get("description", ""),
|
||||
row.get("location", ""),
|
||||
]
|
||||
return " ".join(str(p) for p in parts if p)
|
||||
|
||||
def to_search_result(self, entity: object) -> dict[str, Any]:
|
||||
"""Convert calendar entry to search result dict."""
|
||||
if isinstance(entity, dict):
|
||||
title = entity.get("title", "")
|
||||
description = entity.get("description", "") or ""
|
||||
entity_id = str(entity.get("id", ""))
|
||||
else:
|
||||
title = getattr(entity, "title", "")
|
||||
description = getattr(entity, "description", "") or ""
|
||||
entity_id = str(getattr(entity, "id", ""))
|
||||
return {
|
||||
"entity_type": self.entity_type,
|
||||
"entity_id": entity_id,
|
||||
"title": title,
|
||||
"snippet": description[:200],
|
||||
"score": 0.0,
|
||||
"data": {},
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
"""DMS File search provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileSearchProvider:
|
||||
"""Search provider for DMS File entities."""
|
||||
|
||||
entity_type = "file"
|
||||
|
||||
async def search_fts(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
tsquery: str,
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Full-text search on files.content_tsv."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
|
||||
FROM files f
|
||||
WHERE f.tenant_id = :tid
|
||||
AND f.deleted_at IS NULL
|
||||
AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q)
|
||||
ORDER BY rank DESC
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"q": tsquery, "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def search_vector(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
embedding: list[float],
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Semantic search on files.embedding."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score
|
||||
FROM files f
|
||||
WHERE f.tenant_id = :tid
|
||||
AND f.deleted_at IS NULL
|
||||
AND f.embedding IS NOT NULL
|
||||
ORDER BY f.embedding <=> cast(:emb AS vector)
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def get_embedding_text(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
entity_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
) -> str:
|
||||
"""Get text for embedding generation."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT name, content_text
|
||||
FROM files
|
||||
WHERE id = :eid AND tenant_id = :tid
|
||||
"""
|
||||
)
|
||||
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return ""
|
||||
name = row.get("name", "") or ""
|
||||
content = row.get("content_text", "") or ""
|
||||
return f"{name} {content[:5000]}"
|
||||
|
||||
def to_search_result(self, entity: object) -> dict[str, Any]:
|
||||
"""Convert file to search result dict."""
|
||||
if isinstance(entity, dict):
|
||||
name = entity.get("name", "")
|
||||
content = entity.get("content_text", "") or ""
|
||||
entity_id = str(entity.get("id", ""))
|
||||
else:
|
||||
name = getattr(entity, "name", "")
|
||||
content = getattr(entity, "content_text", "") or ""
|
||||
entity_id = str(getattr(entity, "id", ""))
|
||||
return {
|
||||
"entity_type": self.entity_type,
|
||||
"entity_id": entity_id,
|
||||
"title": name,
|
||||
"snippet": content[:200],
|
||||
"score": 0.0,
|
||||
"data": {},
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Mail search provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MailSearchProvider:
|
||||
"""Search provider for Mail entities."""
|
||||
|
||||
entity_type = "mail"
|
||||
|
||||
async def search_fts(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
tsquery: str,
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Full-text search on mails.body_tsv."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
|
||||
FROM mails m
|
||||
WHERE m.tenant_id = :tid
|
||||
AND m.deleted_at IS NULL
|
||||
AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q)
|
||||
ORDER BY rank DESC
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"q": tsquery, "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def search_vector(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
embedding: list[float],
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Semantic search on mails.embedding."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
|
||||
FROM mails m
|
||||
WHERE m.tenant_id = :tid
|
||||
AND m.deleted_at IS NULL
|
||||
AND m.embedding IS NOT NULL
|
||||
ORDER BY m.embedding <=> cast(:emb AS vector)
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def get_embedding_text(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
entity_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
) -> str:
|
||||
"""Get text for embedding generation."""
|
||||
sql = text(
|
||||
"""
|
||||
SELECT subject, body_text
|
||||
FROM mails
|
||||
WHERE id = :eid AND tenant_id = :tid
|
||||
"""
|
||||
)
|
||||
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
|
||||
row = result.mappings().first()
|
||||
if not row:
|
||||
return ""
|
||||
subject = row.get("subject", "") or ""
|
||||
body = row.get("body_text", "") or ""
|
||||
return f"{subject} {body[:5000]}"
|
||||
|
||||
def to_search_result(self, entity: object) -> dict[str, Any]:
|
||||
"""Convert mail to search result dict."""
|
||||
if isinstance(entity, dict):
|
||||
subject = entity.get("subject", "")
|
||||
body = entity.get("body_text", "") or ""
|
||||
entity_id = str(entity.get("id", ""))
|
||||
else:
|
||||
subject = getattr(entity, "subject", "")
|
||||
body = getattr(entity, "body_text", "") or ""
|
||||
entity_id = str(getattr(entity, "id", ""))
|
||||
return {
|
||||
"entity_type": self.entity_type,
|
||||
"entity_id": entity_id,
|
||||
"title": subject,
|
||||
"snippet": body[:200],
|
||||
"score": 0.0,
|
||||
"data": {},
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
"""KI query understanding and result aggregation via LiteLLM."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_LLM_MODEL = os.environ.get('SEARCH_LLM_MODEL', 'ollama/deepseek-v4')
|
||||
OLLAMA_API_KEY = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
|
||||
|
||||
QUERY_ANALYZE_SYSTEM = (
|
||||
"Du bist ein Query-Analyzer fuer ein CRM. "
|
||||
"Analysiere die Suchanfrage und gib JSON zurueck: "
|
||||
'{"normalized_query": str, "entities": {"person": str|null, "company": str|null, "topic": str|null}, '
|
||||
'"intent": str, "semantic_terms": [str], "suggested_filters": {}}'
|
||||
)
|
||||
|
||||
RESULT_AGGREGATE_SYSTEM = (
|
||||
"Du bist ein Result-Aggregator. Fasse Ergebnisse zusammen und generiere Facetten: "
|
||||
'{"summary": str, "facets": {"types": {}, "dates": {}, "people": []}, "suggestions": [str]}'
|
||||
)
|
||||
|
||||
|
||||
def _fallback_query_analysis(query: str) -> dict[str, Any]:
|
||||
return {
|
||||
"normalized_query": query,
|
||||
"entities": {},
|
||||
"intent": "search",
|
||||
"semantic_terms": [],
|
||||
"suggested_filters": {},
|
||||
}
|
||||
|
||||
|
||||
def _fallback_aggregate(results: list[dict], query: str) -> dict[str, Any]:
|
||||
return {
|
||||
"summary": f"{len(results)} Ergebnisse gefunden",
|
||||
"facets": {},
|
||||
"suggestions": [],
|
||||
}
|
||||
|
||||
|
||||
async def llm_analyze_query(query: str) -> dict[str, Any]:
|
||||
"""Analyze a search query using LLM for intent, entities, and semantic terms.
|
||||
|
||||
Falls back to a simple dict if LLM fails.
|
||||
"""
|
||||
try:
|
||||
response = await litellm.acompletion(
|
||||
model=DEFAULT_LLM_MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": QUERY_ANALYZE_SYSTEM},
|
||||
{"role": "user", "content": query},
|
||||
],
|
||||
temperature=0.1,
|
||||
max_tokens=500,
|
||||
response_format={"type": "json_object"},
|
||||
api_key=OLLAMA_API_KEY,
|
||||
)
|
||||
content = response.choices[0].message.content
|
||||
return json.loads(content)
|
||||
except Exception:
|
||||
logger.warning("LLM query analysis failed, using fallback")
|
||||
return _fallback_query_analysis(query)
|
||||
|
||||
|
||||
async def llm_aggregate_results(results: list[dict], query: str) -> dict[str, Any]:
|
||||
"""Aggregate search results using LLM for summary, facets, and suggestions.
|
||||
|
||||
Falls back to a simple dict if LLM fails.
|
||||
"""
|
||||
if not results:
|
||||
return _fallback_aggregate(results, query)
|
||||
try:
|
||||
# Truncate results to avoid token overflow
|
||||
compact = [
|
||||
{"entity_type": r.get("entity_type"), "title": r.get("title", "")[:100]}
|
||||
for r in results[:50]
|
||||
]
|
||||
user_msg = json.dumps({"query": query, "results": compact})
|
||||
response = await litellm.acompletion(
|
||||
model=DEFAULT_LLM_MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": RESULT_AGGREGATE_SYSTEM},
|
||||
{"role": "user", "content": user_msg},
|
||||
],
|
||||
temperature=0.1,
|
||||
max_tokens=1000,
|
||||
response_format={"type": "json_object"},
|
||||
api_key=OLLAMA_API_KEY,
|
||||
)
|
||||
content = response.choices[0].message.content
|
||||
return json.loads(content)
|
||||
except Exception:
|
||||
logger.warning("LLM result aggregation failed, using fallback")
|
||||
return _fallback_aggregate(results, query)
|
||||
@@ -0,0 +1,316 @@
|
||||
"""API routes for the Unified Search plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.db import get_db
|
||||
from app.core.jobs import enqueue_job
|
||||
from app.deps import get_current_user, require_permission
|
||||
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
|
||||
from app.plugins.builtins.unified_search.query_understanding import (
|
||||
llm_aggregate_results,
|
||||
llm_analyze_query,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.schemas import (
|
||||
ProviderResponse,
|
||||
ReindexRequest,
|
||||
SearchRequest,
|
||||
SearchResponse,
|
||||
SearchResult,
|
||||
SimilarRequest,
|
||||
SimilarResponse,
|
||||
SuggestRequest,
|
||||
SuggestResponse,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.search_engine import (
|
||||
autocomplete,
|
||||
find_similar_all_types,
|
||||
hybrid_search,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/v1/search", tags=["search"])
|
||||
|
||||
|
||||
# ─── Search ───
|
||||
|
||||
@router.post("", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def search(
|
||||
req: SearchRequest,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> SearchResponse:
|
||||
"""Perform hybrid search with KI query understanding."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
|
||||
# KI query understanding
|
||||
query_analysis = await llm_analyze_query(req.query)
|
||||
|
||||
# Hybrid search
|
||||
results = await hybrid_search(
|
||||
db=db,
|
||||
query_analysis=query_analysis,
|
||||
tenant_id=tenant_id,
|
||||
entity_types=req.entity_types,
|
||||
limit=req.limit,
|
||||
)
|
||||
|
||||
# KI result aggregation
|
||||
aggregation = await llm_aggregate_results(results, req.query)
|
||||
|
||||
search_results = [
|
||||
SearchResult(
|
||||
entity_type=r.get("entity_type", ""),
|
||||
entity_id=r.get("entity_id", ""),
|
||||
title=r.get("title", ""),
|
||||
snippet=r.get("snippet", ""),
|
||||
score=r.get("score", 0.0),
|
||||
data=r.get("data", {}),
|
||||
)
|
||||
for r in results
|
||||
]
|
||||
|
||||
return SearchResponse(
|
||||
query=req.query,
|
||||
normalized_query=query_analysis.get("normalized_query", req.query),
|
||||
results=search_results,
|
||||
facets=aggregation.get("facets", {}),
|
||||
summary=aggregation.get("summary", f"{len(results)} Ergebnisse"),
|
||||
suggestions=aggregation.get("suggestions", []),
|
||||
)
|
||||
|
||||
|
||||
# ─── Suggest / Autocomplete ───
|
||||
|
||||
@router.get("/suggest", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def suggest(
|
||||
q: str = Query(..., min_length=1, max_length=200),
|
||||
limit: int = Query(default=10, ge=1, le=50),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> SuggestResponse:
|
||||
"""Autocomplete suggestions using FTS prefix search."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
suggestions = await autocomplete(db, q, tenant_id, limit)
|
||||
return SuggestResponse(suggestions=suggestions)
|
||||
|
||||
|
||||
# ─── Similar ───
|
||||
|
||||
@router.post("/similar", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def find_similar(
|
||||
req: SimilarRequest,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> SimilarResponse:
|
||||
"""Find similar entities across all types based on embedding."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
try:
|
||||
entity_id = uuid.UUID(req.entity_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid entity_id")
|
||||
|
||||
similar = await find_similar_all_types(
|
||||
db=db,
|
||||
entity_type=req.entity_type,
|
||||
entity_id=entity_id,
|
||||
tenant_id=tenant_id,
|
||||
limit=req.limit,
|
||||
)
|
||||
|
||||
result_dict: dict[str, list[SearchResult]] = {}
|
||||
for etype, items in similar.items():
|
||||
result_dict[etype] = [
|
||||
SearchResult(
|
||||
entity_type=r.get("entity_type", etype),
|
||||
entity_id=r.get("entity_id", ""),
|
||||
title=r.get("title", ""),
|
||||
snippet=r.get("snippet", ""),
|
||||
score=r.get("score", 0.0),
|
||||
data=r.get("data", {}),
|
||||
)
|
||||
for r in items
|
||||
]
|
||||
|
||||
return SimilarResponse(similar=result_dict)
|
||||
|
||||
|
||||
# ─── Reindex ───
|
||||
|
||||
@router.post("/reindex", dependencies=[Depends(require_permission("search:admin"))])
|
||||
async def reindex(
|
||||
req: ReindexRequest,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Trigger reindexing of specified entity types."""
|
||||
entity_types = req.entity_types
|
||||
if not entity_types:
|
||||
entity_types = get_search_registry().get_entity_types()
|
||||
|
||||
job_ids: list[str] = []
|
||||
for etype in entity_types:
|
||||
job_id = await enqueue_job("reindex", etype)
|
||||
if job_id:
|
||||
job_ids.append(job_id)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"message": f"Reindexing {len(entity_types)} entity types",
|
||||
"entity_types": entity_types,
|
||||
"job_ids": job_ids,
|
||||
}
|
||||
|
||||
|
||||
# ─── Providers ───
|
||||
|
||||
@router.get("/providers", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def list_providers(
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> list[ProviderResponse]:
|
||||
"""List all active search providers."""
|
||||
registry = get_search_registry()
|
||||
providers = registry.get_all()
|
||||
return [
|
||||
ProviderResponse(
|
||||
entity_type=p.entity_type,
|
||||
plugin_name="unified_search",
|
||||
is_active=True,
|
||||
)
|
||||
for p in providers
|
||||
]
|
||||
|
||||
|
||||
@router.post("/providers/{entity_type}/toggle", dependencies=[Depends(require_permission("search:admin"))])
|
||||
async def toggle_provider(
|
||||
entity_type: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Toggle a search provider on/off."""
|
||||
registry = get_search_registry()
|
||||
provider = registry.get(entity_type)
|
||||
if provider is None:
|
||||
raise HTTPException(status_code=404, detail=f"Provider not found: {entity_type}")
|
||||
|
||||
# Toggle in DB
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT is_active FROM unified_search_providers
|
||||
WHERE tenant_id = :tid AND entity_type = :et
|
||||
"""
|
||||
),
|
||||
{"tid": tenant_id, "et": entity_type},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
|
||||
if row:
|
||||
new_status = not row["is_active"]
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE unified_search_providers SET is_active = :active
|
||||
WHERE tenant_id = :tid AND entity_type = :et
|
||||
"""
|
||||
),
|
||||
{"active": new_status, "tid": tenant_id, "et": entity_type},
|
||||
)
|
||||
else:
|
||||
new_status = False
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO unified_search_providers (tenant_id, entity_type, plugin_name, is_active)
|
||||
VALUES (:tid, :et, 'unified_search', false)
|
||||
"""
|
||||
),
|
||||
{"tid": tenant_id, "et": entity_type},
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
|
||||
if not new_status:
|
||||
registry.unregister(entity_type)
|
||||
else:
|
||||
# Re-register by clearing and re-running auto_register
|
||||
from app.plugins.builtins.unified_search.provider_registry import auto_register_providers
|
||||
await auto_register_providers(db)
|
||||
|
||||
return {
|
||||
"entity_type": entity_type,
|
||||
"is_active": new_status,
|
||||
}
|
||||
|
||||
|
||||
# ─── Stats ───
|
||||
|
||||
@router.get("/stats", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def search_stats(
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Get search index statistics."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
|
||||
stats: dict = {}
|
||||
tables = {
|
||||
"contacts": "embedding",
|
||||
"companies": "embedding",
|
||||
"mails": "embedding",
|
||||
"files": "embedding",
|
||||
"calendar_entries": "embedding",
|
||||
}
|
||||
|
||||
for table, col in tables.items():
|
||||
try:
|
||||
total_result = await db.execute(
|
||||
text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL"),
|
||||
{"tid": tenant_id},
|
||||
)
|
||||
total = total_result.scalar() or 0
|
||||
|
||||
indexed_result = await db.execute(
|
||||
text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL AND {col} IS NOT NULL"),
|
||||
{"tid": tenant_id},
|
||||
)
|
||||
indexed = indexed_result.scalar() or 0
|
||||
|
||||
stats[table] = {
|
||||
"total": total,
|
||||
"indexed": indexed,
|
||||
"pending": total - indexed,
|
||||
}
|
||||
except Exception:
|
||||
logger.exception("Stats query failed for %s", table)
|
||||
stats[table] = {"total": 0, "indexed": 0, "pending": 0}
|
||||
|
||||
# Last index log entries
|
||||
try:
|
||||
log_result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT entity_type, action, status, created_at
|
||||
FROM unified_search_index_log
|
||||
WHERE tenant_id = :tid
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 10
|
||||
"""
|
||||
),
|
||||
{"tid": tenant_id},
|
||||
)
|
||||
recent_logs = [dict(r) for r in log_result.mappings().all()]
|
||||
except Exception:
|
||||
recent_logs = []
|
||||
|
||||
stats["recent_logs"] = recent_logs
|
||||
return stats
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Pydantic schemas for the Unified Search plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
# ─── Search ───
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
query: str = Field(..., min_length=1, max_length=500)
|
||||
entity_types: list[str] | None = None
|
||||
limit: int = Field(default=20, ge=1, le=100)
|
||||
offset: int = Field(default=0, ge=0)
|
||||
|
||||
|
||||
class SearchResult(BaseModel):
|
||||
entity_type: str
|
||||
entity_id: str
|
||||
title: str
|
||||
snippet: str
|
||||
score: float
|
||||
data: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SearchResponse(BaseModel):
|
||||
query: str
|
||||
normalized_query: str
|
||||
results: list[SearchResult]
|
||||
facets: dict[str, Any]
|
||||
summary: str
|
||||
suggestions: list[str]
|
||||
|
||||
|
||||
# ─── Similar ───
|
||||
|
||||
class SimilarRequest(BaseModel):
|
||||
entity_type: str
|
||||
entity_id: str
|
||||
limit: int = Field(default=5, ge=1, le=50)
|
||||
|
||||
|
||||
class SimilarResponse(BaseModel):
|
||||
similar: dict[str, list[SearchResult]]
|
||||
|
||||
|
||||
# ─── Suggest ───
|
||||
|
||||
class SuggestRequest(BaseModel):
|
||||
q: str = Field(..., min_length=1, max_length=200)
|
||||
limit: int = Field(default=10, ge=1, le=50)
|
||||
|
||||
|
||||
class SuggestResponse(BaseModel):
|
||||
suggestions: list[str]
|
||||
|
||||
|
||||
# ─── Reindex ───
|
||||
|
||||
class ReindexRequest(BaseModel):
|
||||
entity_types: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ─── Provider ───
|
||||
|
||||
class ProviderResponse(BaseModel):
|
||||
entity_type: str
|
||||
plugin_name: str
|
||||
is_active: bool
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Hybrid search engine: FTS + pgvector with RRF fusion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.plugins.builtins.unified_search.embedding import generate_embedding
|
||||
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Entity type -> (table_name, tsv_column, embedding_column)
|
||||
SEARCHABLE_ENTITIES: dict[str, tuple[str, str, str]] = {
|
||||
"contact": ("contacts", "search_tsv", "embedding"),
|
||||
"company": ("companies", "search_tsv", "embedding"),
|
||||
"mail": ("mails", "body_tsv", "embedding"),
|
||||
"file": ("files", "content_tsv", "embedding"),
|
||||
"event": ("calendar_entries", "search_tsv", "embedding"),
|
||||
}
|
||||
|
||||
RRF_K = 60
|
||||
RRF_ALPHA = 0.5
|
||||
RRF_BETA = 0.5
|
||||
|
||||
|
||||
def rrf_fusion(
|
||||
fts_results: list[dict[str, Any]],
|
||||
vec_results: list[dict[str, Any]],
|
||||
entity_type: str,
|
||||
k: int = RRF_K,
|
||||
alpha: float = RRF_ALPHA,
|
||||
beta: float = RRF_BETA,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Reciprocal Rank Fusion of FTS and vector search results.
|
||||
|
||||
score = alpha * (1/(k+rank_fts)) + beta * (1/(k+rank_vec))
|
||||
"""
|
||||
fused: dict[str, dict[str, Any]] = {}
|
||||
|
||||
for rank, item in enumerate(fts_results):
|
||||
eid = str(item.get("id", ""))
|
||||
if not eid:
|
||||
continue
|
||||
rrf_score = alpha * (1.0 / (k + rank + 1))
|
||||
if eid not in fused:
|
||||
fused[eid] = {**item, "_score": 0.0, "_entity_type": entity_type}
|
||||
fused[eid]["_score"] += rrf_score
|
||||
|
||||
for rank, item in enumerate(vec_results):
|
||||
eid = str(item.get("id", ""))
|
||||
if not eid:
|
||||
continue
|
||||
rrf_score = beta * (1.0 / (k + rank + 1))
|
||||
if eid not in fused:
|
||||
fused[eid] = {**item, "_score": 0.0, "_entity_type": entity_type}
|
||||
fused[eid]["_score"] += rrf_score
|
||||
|
||||
return sorted(fused.values(), key=lambda x: x.get("_score", 0.0), reverse=True)
|
||||
|
||||
|
||||
async def hybrid_search(
|
||||
db: AsyncSession,
|
||||
query_analysis: dict[str, Any],
|
||||
tenant_id: uuid.UUID,
|
||||
entity_types: list[str] | None = None,
|
||||
limit: int = 20,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Perform hybrid search across all entity types.
|
||||
|
||||
For each entity: FTS search + vector search + RRF fusion.
|
||||
Merge all results, sort by fused score, return top limit.
|
||||
"""
|
||||
registry = get_search_registry()
|
||||
all_entity_types = entity_types or registry.get_entity_types()
|
||||
|
||||
normalized_query = query_analysis.get("normalized_query", "")
|
||||
semantic_terms = query_analysis.get("semantic_terms", [])
|
||||
|
||||
# Build tsquery string
|
||||
tsquery_parts = [normalized_query] + semantic_terms
|
||||
tsquery = " & ".join(
|
||||
part.strip().replace(" ", " & ")
|
||||
for part in tsquery_parts
|
||||
if part and part.strip()
|
||||
)
|
||||
if not tsquery:
|
||||
tsquery = normalized_query
|
||||
|
||||
# Generate query embedding
|
||||
query_text = normalized_query
|
||||
if semantic_terms:
|
||||
query_text = f"{normalized_query} {' '.join(semantic_terms)}"
|
||||
query_embedding = await generate_embedding(query_text)
|
||||
|
||||
all_results: list[dict[str, Any]] = []
|
||||
fetch_limit = limit * 2
|
||||
|
||||
for entity_type in all_entity_types:
|
||||
provider = registry.get(entity_type)
|
||||
if provider is None:
|
||||
continue
|
||||
|
||||
fts_results: list[dict[str, Any]] = []
|
||||
vec_results: list[dict[str, Any]] = []
|
||||
|
||||
try:
|
||||
fts_results = await provider.search_fts(db, tsquery, tenant_id, fetch_limit)
|
||||
except Exception:
|
||||
logger.exception("FTS search failed for %s", entity_type)
|
||||
|
||||
if query_embedding:
|
||||
try:
|
||||
vec_results = await provider.search_vector(db, query_embedding, tenant_id, fetch_limit)
|
||||
except Exception:
|
||||
logger.exception("Vector search failed for %s", entity_type)
|
||||
|
||||
fused = rrf_fusion(fts_results, vec_results, entity_type)
|
||||
|
||||
# Convert to search result format
|
||||
for item in fused:
|
||||
result = provider.to_search_result(item)
|
||||
result["score"] = item.get("_score", 0.0)
|
||||
all_results.append(result)
|
||||
|
||||
all_results.sort(key=lambda x: x.get("score", 0.0), reverse=True)
|
||||
return all_results[:limit]
|
||||
|
||||
|
||||
async def find_similar_all_types(
|
||||
db: AsyncSession,
|
||||
entity_type: str,
|
||||
entity_id: uuid.UUID,
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int = 5,
|
||||
) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Find similar entities across all types based on embedding.
|
||||
|
||||
Gets the embedding of the source entity, then searches all other tables.
|
||||
"""
|
||||
table_map = SEARCHABLE_ENTITIES
|
||||
source_info = table_map.get(entity_type)
|
||||
if not source_info:
|
||||
return {}
|
||||
|
||||
table_name, _, emb_col = source_info
|
||||
|
||||
# Get source embedding
|
||||
sql = text(
|
||||
f"SELECT {emb_col} AS embedding FROM {table_name} "
|
||||
f"WHERE id = :eid AND tenant_id = :tid"
|
||||
)
|
||||
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
|
||||
row = result.mappings().first()
|
||||
if not row or not row.get("embedding"):
|
||||
return {}
|
||||
|
||||
source_embedding_str = str(row["embedding"])
|
||||
|
||||
registry = get_search_registry()
|
||||
similar: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
for etype, (tbl, _, emb) in table_map.items():
|
||||
if etype == entity_type:
|
||||
continue
|
||||
provider = registry.get(etype)
|
||||
if provider is None:
|
||||
continue
|
||||
try:
|
||||
sql_sim = text(
|
||||
f"""
|
||||
SELECT *, 1 - ({emb} <=> cast(:emb_str AS vector)) AS score
|
||||
FROM {tbl}
|
||||
WHERE tenant_id = :tid
|
||||
AND deleted_at IS NULL
|
||||
AND {emb} IS NOT NULL
|
||||
ORDER BY {emb} <=> cast(:emb_str AS vector)
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
res = await db.execute(
|
||||
sql_sim,
|
||||
{"emb_str": source_embedding_str, "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = res.mappings().all()
|
||||
results = []
|
||||
for r in rows:
|
||||
sr = provider.to_search_result(dict(r))
|
||||
sr["score"] = float(r.get("score", 0.0))
|
||||
results.append(sr)
|
||||
similar[etype] = results
|
||||
except Exception:
|
||||
logger.exception("Similar search failed for %s", etype)
|
||||
similar[etype] = []
|
||||
|
||||
return similar
|
||||
|
||||
|
||||
async def autocomplete(
|
||||
db: AsyncSession,
|
||||
query: str,
|
||||
tenant_id: uuid.UUID,
|
||||
limit: int = 10,
|
||||
) -> list[str]:
|
||||
"""Autocomplete using FTS prefix search.
|
||||
|
||||
Uses to_tsquery with :* suffix for prefix matching.
|
||||
"""
|
||||
query_clean = query.strip().replace(" ", " & ")
|
||||
if not query_clean:
|
||||
return []
|
||||
tsquery = f"{query_clean}:*"
|
||||
|
||||
suggestions: list[str] = []
|
||||
registry = get_search_registry()
|
||||
|
||||
for entity_type in registry.get_entity_types():
|
||||
provider = registry.get(entity_type)
|
||||
if provider is None:
|
||||
continue
|
||||
entity_info = SEARCHABLE_ENTITIES.get(entity_type)
|
||||
if not entity_info:
|
||||
continue
|
||||
table_name, tsv_col, _ = entity_info
|
||||
try:
|
||||
sql = text(
|
||||
f"""
|
||||
SELECT * FROM {table_name}
|
||||
WHERE tenant_id = :tid
|
||||
AND deleted_at IS NULL
|
||||
AND {tsv_col} @@ to_tsquery('pg_catalog.german', :q)
|
||||
LIMIT :lim
|
||||
"""
|
||||
)
|
||||
result = await db.execute(
|
||||
sql,
|
||||
{"q": tsquery, "tid": tenant_id, "lim": limit},
|
||||
)
|
||||
rows = result.mappings().all()
|
||||
for r in rows:
|
||||
sr = provider.to_search_result(dict(r))
|
||||
title = sr.get("title", "")
|
||||
if title and title not in suggestions:
|
||||
suggestions.append(title)
|
||||
except Exception:
|
||||
logger.debug("Autocomplete failed for %s", entity_type)
|
||||
|
||||
return suggestions[:limit]
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Text extraction from various file formats for DMS indexing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_CHARS = 50_000
|
||||
|
||||
|
||||
async def extract_text_from_file(file_path: str, mime_type: str) -> str:
|
||||
"""Extract text content from a file based on its MIME type.
|
||||
|
||||
Supports:
|
||||
- PDF (via PyMuPDF/fitz)
|
||||
- DOCX (via python-docx)
|
||||
- XLSX (via openpyxl)
|
||||
- PPTX (via python-pptx)
|
||||
- text/* (via aiofiles)
|
||||
- Images: returns empty (OCR not implemented)
|
||||
|
||||
Args:
|
||||
file_path: Absolute path to the file.
|
||||
mime_type: MIME type of the file.
|
||||
|
||||
Returns:
|
||||
Extracted text, truncated to MAX_CHARS. Empty string on error.
|
||||
"""
|
||||
try:
|
||||
if mime_type == "application/pdf":
|
||||
return await _extract_pdf(file_path)
|
||||
elif mime_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
|
||||
return await _extract_docx(file_path)
|
||||
elif mime_type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
|
||||
return await _extract_xlsx(file_path)
|
||||
elif mime_type == "application/vnd.openxmlformats-officedocument.presentationml.presentation":
|
||||
return await _extract_pptx(file_path)
|
||||
elif mime_type.startswith("text/"):
|
||||
return await _extract_text(file_path)
|
||||
elif mime_type.startswith("image/"):
|
||||
return ""
|
||||
else:
|
||||
logger.debug("Unsupported mime_type=%s for text extraction", mime_type)
|
||||
return ""
|
||||
except Exception:
|
||||
logger.warning("Failed to extract text from %s (mime=%s)", file_path, mime_type)
|
||||
return ""
|
||||
|
||||
|
||||
def _truncate(text: str) -> str:
|
||||
"""Truncate text to MAX_CHARS with indicator."""
|
||||
if len(text) > MAX_CHARS:
|
||||
return text[:MAX_CHARS] + "... [truncated]"
|
||||
return text
|
||||
|
||||
|
||||
async def _extract_pdf(file_path: str) -> str:
|
||||
"""Extract text from PDF using PyMuPDF (fitz)."""
|
||||
import fitz # PyMuPDF
|
||||
|
||||
doc = fitz.open(file_path)
|
||||
parts: list[str] = []
|
||||
for page in doc:
|
||||
parts.append(page.get_text())
|
||||
doc.close()
|
||||
return _truncate("\n".join(parts))
|
||||
|
||||
|
||||
async def _extract_docx(file_path: str) -> str:
|
||||
"""Extract text from DOCX using python-docx."""
|
||||
from docx import Document
|
||||
|
||||
doc = Document(file_path)
|
||||
parts: list[str] = []
|
||||
for para in doc.paragraphs:
|
||||
if para.text.strip():
|
||||
parts.append(para.text)
|
||||
# Also extract table text
|
||||
for table in doc.tables:
|
||||
for row in table.rows:
|
||||
for cell in row.cells:
|
||||
if cell.text.strip():
|
||||
parts.append(cell.text)
|
||||
return _truncate("\n".join(parts))
|
||||
|
||||
|
||||
async def _extract_xlsx(file_path: str) -> str:
|
||||
"""Extract text from XLSX using openpyxl."""
|
||||
from openpyxl import load_workbook
|
||||
|
||||
wb = load_workbook(file_path, read_only=True, data_only=True)
|
||||
parts: list[str] = []
|
||||
for ws in wb.worksheets:
|
||||
for row in ws.iter_rows(values_only=True):
|
||||
row_text = " ".join(str(c) for c in row if c is not None)
|
||||
if row_text.strip():
|
||||
parts.append(row_text)
|
||||
wb.close()
|
||||
return _truncate("\n".join(parts))
|
||||
|
||||
|
||||
async def _extract_pptx(file_path: str) -> str:
|
||||
"""Extract text from PPTX using python-pptx."""
|
||||
from pptx import Presentation
|
||||
|
||||
prs = Presentation(file_path)
|
||||
parts: list[str] = []
|
||||
for slide in prs.slides:
|
||||
for shape in slide.shapes:
|
||||
if shape.has_text_frame:
|
||||
for para in shape.text_frame.paragraphs:
|
||||
text = para.text.strip()
|
||||
if text:
|
||||
parts.append(text)
|
||||
return _truncate("\n".join(parts))
|
||||
|
||||
|
||||
async def _extract_text(file_path: str) -> str:
|
||||
"""Read plain text file using aiofiles."""
|
||||
import aiofiles
|
||||
|
||||
async with aiofiles.open(file_path, mode="r", encoding="utf-8", errors="replace") as f:
|
||||
content = await f.read()
|
||||
return _truncate(content)
|
||||
@@ -0,0 +1,72 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { apiClient } from '@/api/client';
|
||||
|
||||
export interface Suggestion {
|
||||
id: string;
|
||||
entity_type: string;
|
||||
entity_id: string | null;
|
||||
suggestion_type: 'info' | 'warning' | 'action' | 'insight';
|
||||
title: string;
|
||||
content: string;
|
||||
confidence: number;
|
||||
actions: Array<{ method: string; path: string; body: any; description: string }>;
|
||||
created_at: string;
|
||||
is_dismissed: boolean;
|
||||
is_acted_upon: boolean;
|
||||
}
|
||||
|
||||
export function useSuggestions() {
|
||||
const [suggestions, setSuggestions] = useState<Suggestion[]>([]);
|
||||
const [connected, setConnected] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
// Initial load
|
||||
apiClient.get('/api/v1/ai-proactive/suggestions').then(r => {
|
||||
setSuggestions(r.data.items);
|
||||
}).catch(() => {});
|
||||
|
||||
// SSE Stream
|
||||
const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream');
|
||||
eventSource.onopen = () => setConnected(true);
|
||||
eventSource.onerror = () => setConnected(false);
|
||||
eventSource.onmessage = (e) => {
|
||||
try {
|
||||
const suggestion = JSON.parse(e.data);
|
||||
setSuggestions(prev => [suggestion, ...prev].slice(0, 50));
|
||||
} catch {}
|
||||
};
|
||||
|
||||
return () => eventSource.close();
|
||||
}, []);
|
||||
|
||||
const dismiss = useCallback(async (id: string) => {
|
||||
setSuggestions(prev => prev.filter(s => s.id !== id));
|
||||
await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/dismiss`).catch(() => {});
|
||||
}, []);
|
||||
|
||||
const act = useCallback(async (id: string, actionIndex: number) => {
|
||||
const result = await apiClient.post(`/api/v1/ai-proactive/suggestions/${id}/act`, {
|
||||
action_index: actionIndex
|
||||
});
|
||||
setSuggestions(prev => prev.map(s => s.id === id ? { ...s, is_acted_upon: true } : s));
|
||||
return result.data;
|
||||
}, []);
|
||||
|
||||
return { suggestions, connected, dismiss, act };
|
||||
}
|
||||
|
||||
export function useProactiveSettings() {
|
||||
const [settings, setSettings] = useState<any>(null);
|
||||
|
||||
useEffect(() => {
|
||||
apiClient.get('/api/v1/ai-proactive/settings').then(r => setSettings(r.data));
|
||||
}, []);
|
||||
|
||||
const update = useCallback(async (updates: any) => {
|
||||
const r = await apiClient.put('/api/v1/ai-proactive/settings', updates);
|
||||
setSettings(r.data);
|
||||
return r.data;
|
||||
}, []);
|
||||
|
||||
return { settings, update };
|
||||
}
|
||||
@@ -388,45 +388,9 @@ export function useGlobalSearch(query: string, entityTypes?: string[]) {
|
||||
queryKey: ['globalSearch', query, entityTypes],
|
||||
queryFn: async (): Promise<SearchResult[]> => {
|
||||
if (!query.trim()) return [];
|
||||
const results: SearchResult[] = [];
|
||||
const types = entityTypes && entityTypes.length > 0 ? entityTypes : ['company', 'contact'];
|
||||
const tasks: Promise<void>[] = [];
|
||||
if (types.includes('company')) {
|
||||
tasks.push(
|
||||
apiGet<PaginatedResponse<Company>>(`/companies?page=1&page_size=10&search=${encodeURIComponent(query)}`)
|
||||
.then((data) => {
|
||||
for (const item of data.items) {
|
||||
results.push({
|
||||
type: 'company',
|
||||
id: item.id,
|
||||
name: item.name,
|
||||
description: item.industry || item.email || '',
|
||||
url: `/companies/${item.id}`,
|
||||
});
|
||||
}
|
||||
})
|
||||
.catch(() => {})
|
||||
);
|
||||
}
|
||||
if (types.includes('contact')) {
|
||||
tasks.push(
|
||||
apiGet<PaginatedResponse<Contact>>(`/contacts?page=1&page_size=10&search=${encodeURIComponent(query)}`)
|
||||
.then((data) => {
|
||||
for (const item of data.items) {
|
||||
results.push({
|
||||
type: 'contact',
|
||||
id: item.id,
|
||||
name: `${item.first_name} ${item.last_name}`,
|
||||
description: item.email || item.phone || '',
|
||||
url: `/contacts/${item.id}`,
|
||||
});
|
||||
}
|
||||
})
|
||||
.catch(() => {})
|
||||
);
|
||||
}
|
||||
await Promise.all(tasks);
|
||||
return results;
|
||||
const { search } = await import('@/api/search');
|
||||
const response = await search(query, entityTypes, 20);
|
||||
return response.results || [];
|
||||
},
|
||||
enabled: query.trim().length > 0,
|
||||
staleTime: 30 * 1000,
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
import { apiClient } from './client';
|
||||
|
||||
export interface SearchResult {
|
||||
type: 'company' | 'contact' | 'mail' | 'file' | 'event';
|
||||
id: string;
|
||||
name: string;
|
||||
description?: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface SearchResponse {
|
||||
results: SearchResult[];
|
||||
facets?: Record<string, Array<{ value: string; count: number }>>;
|
||||
summary?: string;
|
||||
suggestions?: string[];
|
||||
}
|
||||
|
||||
export async function search(query: string, entityTypes?: string[], limit = 20): Promise<SearchResponse> {
|
||||
const r = await apiClient.post('/api/v1/search', {
|
||||
query,
|
||||
entity_types: entityTypes,
|
||||
limit,
|
||||
});
|
||||
return r.data;
|
||||
}
|
||||
|
||||
export async function searchSuggest(q: string): Promise<string[]> {
|
||||
const r = await apiClient.get('/api/v1/search/suggest', { params: { q } });
|
||||
return r.data.suggestions;
|
||||
}
|
||||
|
||||
export async function searchSimilar(entityType: string, entityId: string): Promise<SearchResult[]> {
|
||||
const r = await apiClient.post('/api/v1/search/similar', {
|
||||
entity_type: entityType,
|
||||
entity_id: entityId,
|
||||
});
|
||||
return r.data.similar;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { apiClient } from '@/api/client';
|
||||
|
||||
interface SuggestionBadgeProps {
|
||||
onClick: () => void;
|
||||
}
|
||||
|
||||
export function SuggestionBadge({ onClick }: SuggestionBadgeProps) {
|
||||
const [count, setCount] = useState(0);
|
||||
const [pulsing, setPulsing] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
// Initial count
|
||||
apiClient.get('/api/v1/ai-proactive/suggestions').then(r => {
|
||||
setCount(r.data.items.length);
|
||||
}).catch(() => {});
|
||||
|
||||
// SSE for real-time updates
|
||||
const eventSource = new EventSource('/api/v1/ai-proactive/suggestions/stream');
|
||||
eventSource.onmessage = () => {
|
||||
setCount(prev => prev + 1);
|
||||
setPulsing(true);
|
||||
setTimeout(() => setPulsing(false), 2000);
|
||||
};
|
||||
|
||||
return () => eventSource.close();
|
||||
}, []);
|
||||
|
||||
if (count === 0) {
|
||||
return (
|
||||
<button
|
||||
onClick={onClick}
|
||||
className="relative p-2 text-gray-500 hover:text-gray-700 transition-colors"
|
||||
title="KI Vorschläge"
|
||||
>
|
||||
🤖
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<button
|
||||
onClick={onClick}
|
||||
className={`relative p-2 text-gray-500 hover:text-gray-700 transition-colors ${
|
||||
pulsing ? 'animate-pulse' : ''
|
||||
}`}
|
||||
title={`${count} KI Vorschläge`}
|
||||
>
|
||||
🤖
|
||||
<span
|
||||
className={`absolute -top-0 -right-0 min-w-[18px] h-[18px] flex items-center justify-center text-[10px] font-bold text-white rounded-full ${
|
||||
pulsing ? 'bg-red-500 animate-bounce' : 'bg-blue-500'
|
||||
}`}
|
||||
>
|
||||
{count > 99 ? '99+' : count}
|
||||
</span>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
import { useState } from 'react';
|
||||
import type { Suggestion } from '@/api/aiProactive';
|
||||
|
||||
const typeConfig = {
|
||||
info: { icon: '💡', color: 'blue', bg: 'bg-blue-50', border: 'border-blue-200', text: 'text-blue-700', bar: 'bg-blue-500' },
|
||||
warning: { icon: '⚠️', color: 'orange', bg: 'bg-orange-50', border: 'border-orange-200', text: 'text-orange-700', bar: 'bg-orange-500' },
|
||||
action: { icon: '✅', color: 'green', bg: 'bg-green-50', border: 'border-green-200', text: 'text-green-700', bar: 'bg-green-500' },
|
||||
insight: { icon: '🔍', color: 'purple', bg: 'bg-purple-50', border: 'border-purple-200', text: 'text-purple-700', bar: 'bg-purple-500' },
|
||||
};
|
||||
|
||||
interface SuggestionCardProps {
|
||||
suggestion: Suggestion;
|
||||
onDismiss: (id: string) => void;
|
||||
onAct: (id: string, actionIndex: number) => void;
|
||||
}
|
||||
|
||||
export function SuggestionCard({ suggestion, onDismiss, onAct }: SuggestionCardProps) {
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const config = typeConfig[suggestion.suggestion_type] || typeConfig.info;
|
||||
const confidencePercent = Math.round(suggestion.confidence * 100);
|
||||
|
||||
return (
|
||||
<div
|
||||
className={`animate-slide-in ${config.bg} ${config.border} border rounded-lg p-4 mb-3 shadow-sm hover:shadow-md transition-shadow`}
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="flex items-start justify-between mb-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-lg">{config.icon}</span>
|
||||
<h3 className={`font-semibold text-sm ${config.text}`}>{suggestion.title}</h3>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => onDismiss(suggestion.id)}
|
||||
className="text-gray-400 hover:text-gray-600 transition-colors"
|
||||
title="Ignorieren"
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<p className="text-sm text-gray-600 mb-3 line-clamp-3">{suggestion.content}</p>
|
||||
|
||||
{/* Confidence Bar */}
|
||||
<div className="flex items-center gap-2 mb-3">
|
||||
<div className="flex-1 h-1.5 bg-gray-200 rounded-full overflow-hidden">
|
||||
<div
|
||||
className={`h-full ${config.bar} rounded-full transition-all duration-500`}
|
||||
style={{ width: `${confidencePercent}%` }}
|
||||
/>
|
||||
</div>
|
||||
<span className="text-xs text-gray-500 font-mono">{confidencePercent}%</span>
|
||||
</div>
|
||||
|
||||
{/* Actions */}
|
||||
{suggestion.actions && suggestion.actions.length > 0 && (
|
||||
<div className="space-y-2">
|
||||
{suggestion.actions.slice(0, expanded ? undefined : 2).map((action, idx) => (
|
||||
<button
|
||||
key={idx}
|
||||
onClick={() => onAct(suggestion.id, idx)}
|
||||
disabled={suggestion.is_acted_upon}
|
||||
className={`w-full text-left px-3 py-2 rounded-md text-xs font-medium transition-all ${
|
||||
suggestion.is_acted_upon
|
||||
? 'bg-gray-100 text-gray-400 cursor-not-allowed'
|
||||
: 'bg-white hover:bg-gray-50 text-gray-700 border border-gray-200 hover:border-gray-300'
|
||||
}`}
|
||||
>
|
||||
<span className="font-mono text-[10px] uppercase mr-1 px-1 py-0.5 bg-gray-100 rounded">
|
||||
{action.method}
|
||||
</span>
|
||||
{action.description}
|
||||
</button>
|
||||
))}
|
||||
{suggestion.actions.length > 2 && (
|
||||
<button
|
||||
onClick={() => setExpanded(!expanded)}
|
||||
className="text-xs text-gray-500 hover:text-gray-700"
|
||||
>
|
||||
{expanded ? 'Weniger' : `${suggestion.actions.length - 2} weitere Aktionen`}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Acted upon badge */}
|
||||
{suggestion.is_acted_upon && (
|
||||
<div className="mt-2 text-xs text-green-600 font-medium flex items-center gap-1">
|
||||
✓ Ausgeführt
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
import { useState } from 'react';
|
||||
import { useSuggestions } from '@/api/aiProactive';
|
||||
import { SuggestionCard } from '@/components/ai/SuggestionCard';
|
||||
|
||||
interface SuggestionSidebarProps {
|
||||
isOpen: boolean;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
export function SuggestionSidebar({ isOpen, onClose }: SuggestionSidebarProps) {
|
||||
const { suggestions, connected, dismiss, act } = useSuggestions();
|
||||
const [filter, setFilter] = useState<string>('all');
|
||||
|
||||
const filteredSuggestions = filter === 'all'
|
||||
? suggestions
|
||||
: suggestions.filter(s => s.suggestion_type === filter);
|
||||
|
||||
const filterOptions = [
|
||||
{ value: 'all', label: 'Alle' },
|
||||
{ value: 'info', label: 'Info' },
|
||||
{ value: 'warning', label: 'Warnung' },
|
||||
{ value: 'action', label: 'Aktion' },
|
||||
{ value: 'insight', label: 'Erkenntnis' },
|
||||
];
|
||||
|
||||
return (
|
||||
<>
|
||||
{/* Overlay */}
|
||||
{isOpen && (
|
||||
<div
|
||||
className="fixed inset-0 bg-black/20 z-40 lg:hidden"
|
||||
onClick={onClose}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Sidebar */}
|
||||
<aside
|
||||
className={`fixed right-0 top-0 h-full w-96 bg-white shadow-2xl z-50 transform transition-transform duration-300 flex flex-col ${
|
||||
isOpen ? 'translate-x-0' : 'translate-x-full'
|
||||
}`}
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="flex items-center justify-between p-4 border-b border-gray-200">
|
||||
<div className="flex items-center gap-2">
|
||||
<h2 className="font-semibold text-gray-800">KI Vorschläge</h2>
|
||||
{connected && (
|
||||
<span className="flex items-center gap-1 text-xs text-green-600">
|
||||
<span className="w-2 h-2 bg-green-500 rounded-full animate-pulse" />
|
||||
Live
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<button
|
||||
onClick={onClose}
|
||||
className="text-gray-400 hover:text-gray-600 transition-colors p-1"
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Filter */}
|
||||
<div className="flex gap-1 p-3 border-b border-gray-100 overflow-x-auto">
|
||||
{filterOptions.map(opt => (
|
||||
<button
|
||||
key={opt.value}
|
||||
onClick={() => setFilter(opt.value)}
|
||||
className={`px-3 py-1 rounded-full text-xs font-medium whitespace-nowrap transition-colors ${
|
||||
filter === opt.value
|
||||
? 'bg-blue-500 text-white'
|
||||
: 'bg-gray-100 text-gray-600 hover:bg-gray-200'
|
||||
}`}
|
||||
>
|
||||
{opt.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<div className="flex-1 overflow-y-auto p-4">
|
||||
{filteredSuggestions.length === 0 ? (
|
||||
<div className="flex flex-col items-center justify-center h-full text-gray-400">
|
||||
<div className="text-4xl mb-3">🤖</div>
|
||||
<p className="text-sm">Keine Vorschläge vorhanden</p>
|
||||
<p className="text-xs mt-1">Die KI analysiert deinen Kontext...</p>
|
||||
</div>
|
||||
) : (
|
||||
<div>
|
||||
{filteredSuggestions.map(suggestion => (
|
||||
<SuggestionCard
|
||||
key={suggestion.id}
|
||||
suggestion={suggestion}
|
||||
onDismiss={dismiss}
|
||||
onAct={act}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Footer */}
|
||||
<div className="p-3 border-t border-gray-200 text-center">
|
||||
<span className="text-xs text-gray-400">
|
||||
{suggestions.length} Vorschläge insgesamt
|
||||
</span>
|
||||
</div>
|
||||
</aside>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -4,11 +4,17 @@ import { Sidebar } from './Sidebar';
|
||||
import { TopBar } from './TopBar';
|
||||
import { PluginToolbar } from './PluginToolbar';
|
||||
import { AISidebar } from './AISidebar';
|
||||
import { SuggestionSidebar } from '@/components/ai/SuggestionSidebar';
|
||||
import { ToastContainer } from '@/components/ui/Toast';
|
||||
import { useUIStore } from '@/store/uiStore';
|
||||
import { useAIContext } from '@/hooks/useAIContext';
|
||||
|
||||
export function AppShell() {
|
||||
const location = useLocation();
|
||||
const { suggestionSidebarOpen, toggleSuggestionSidebar } = useUIStore();
|
||||
|
||||
// Track context for AI Proactive suggestions
|
||||
useAIContext();
|
||||
|
||||
// Hide AI sidebar on the AI Assistant page itself
|
||||
const showAISidebar = !location.pathname.startsWith('/ai-assistant');
|
||||
@@ -33,6 +39,11 @@ export function AppShell() {
|
||||
</div>
|
||||
{/* AI Sidebar — full height, right of TopBar and Toolbar */}
|
||||
{showAISidebar && <AISidebar />}
|
||||
{/* Suggestion Sidebar — overlays from right */}
|
||||
<SuggestionSidebar
|
||||
isOpen={suggestionSidebarOpen}
|
||||
onClose={() => toggleSuggestionSidebar()}
|
||||
/>
|
||||
<ToastContainer />
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -9,12 +9,13 @@ import { useLogout } from '@/api/hooks';
|
||||
import { Avatar } from '@/components/ui/Avatar';
|
||||
import { Badge } from '@/components/ui/Badge';
|
||||
import { SearchDropdown } from '@/components/shared/SearchDropdown';
|
||||
import { SuggestionBadge } from '@/components/ai/SuggestionBadge';
|
||||
|
||||
export function TopBar() {
|
||||
const { t } = useTranslation();
|
||||
const navigate = useNavigate();
|
||||
const { user } = useAuthStore();
|
||||
const { toggleSidebar, toggleAISidebar, aiSidebarCollapsed } = useUIStore();
|
||||
const { toggleSidebar, toggleAISidebar, toggleSuggestionSidebar, aiSidebarCollapsed } = useUIStore();
|
||||
const { currentTenant, availableTenants, switchTenant, isSwitching } = useTenant();
|
||||
const logoutMutation = useLogout();
|
||||
|
||||
@@ -141,6 +142,9 @@ export function TopBar() {
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
{/* AI Suggestions Badge */}
|
||||
<SuggestionBadge onClick={toggleSuggestionSidebar} />
|
||||
|
||||
{/* Notifications */}
|
||||
<div ref={notifRef} className="relative">
|
||||
<button
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { useLocation } from 'react-router-dom';
|
||||
import { apiClient } from '@/api/client';
|
||||
|
||||
export function useAIContext(entityType?: string, entityId?: string, entityData?: any) {
|
||||
const location = useLocation();
|
||||
const debounceRef = useRef<ReturnType<typeof setTimeout>>();
|
||||
|
||||
useEffect(() => {
|
||||
if (debounceRef.current) clearTimeout(debounceRef.current);
|
||||
debounceRef.current = setTimeout(() => {
|
||||
apiClient.post('/api/v1/ai-proactive/context', {
|
||||
page: location.pathname,
|
||||
entity_type: entityType,
|
||||
entity_id: entityId,
|
||||
entity_data: entityData,
|
||||
}).catch(() => {}); // Silent fail, don't bother user
|
||||
}, 500); // Debounce 500ms
|
||||
|
||||
return () => {
|
||||
if (debounceRef.current) clearTimeout(debounceRef.current);
|
||||
};
|
||||
}, [location.pathname, entityType, entityId, entityData]);
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { useProactiveSettings } from '@/api/aiProactive';
|
||||
|
||||
const categoryLabels: Record<string, string> = {
|
||||
mail: 'E-Mails',
|
||||
tasks: 'Aufgaben',
|
||||
contacts: 'Kontakte',
|
||||
companies: 'Firmen',
|
||||
insights: 'Erkenntnisse',
|
||||
};
|
||||
|
||||
const modelOptions = [
|
||||
{ value: 'ollama/deepseek-v4', label: 'DeepSeek V4 (empfohlen)' },
|
||||
{ value: 'ollama/deepseek-v4-pro', label: 'DeepSeek V4 Pro (präzise)' },
|
||||
{ value: 'ollama/llama3.2', label: 'Llama 3.2 (Alternative)' },
|
||||
{ value: 'ollama/gpt-4o-mini', label: 'GPT-4o mini via Ollama' },
|
||||
{ value: 'gpt-4o-mini', label: 'GPT-4o mini (Direct OpenAI)' },
|
||||
];
|
||||
|
||||
export function ProactiveAISettings() {
|
||||
const { settings, update } = useProactiveSettings();
|
||||
|
||||
if (!settings) {
|
||||
return (
|
||||
<div className="flex items-center justify-center p-8 text-gray-400">
|
||||
<div className="animate-pulse">Lade Einstellungen...</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const toggleCategory = async (cat: string) => {
|
||||
const current = settings.suggestion_categories || [];
|
||||
const newCats = current.includes(cat)
|
||||
? current.filter((c: string) => c !== cat)
|
||||
: [...current, cat];
|
||||
await update({ suggestion_categories: newCats });
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="max-w-2xl space-y-6">
|
||||
{/* Enable/Disable */}
|
||||
<div className="flex items-center justify-between p-4 bg-white rounded-lg border border-gray-200">
|
||||
<div>
|
||||
<h3 className="font-semibold text-gray-800">Proaktive KI</h3>
|
||||
<p className="text-sm text-gray-500">Aktiviert oder deaktiviert alle Vorschläge</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => update({ enabled: !settings.enabled })}
|
||||
className={`relative inline-flex h-6 w-11 items-center rounded-full transition-colors ${
|
||||
settings.enabled ? 'bg-blue-500' : 'bg-gray-300'
|
||||
}`}
|
||||
>
|
||||
<span
|
||||
className={`inline-block h-4 w-4 transform rounded-full bg-white transition-transform ${
|
||||
settings.enabled ? 'translate-x-6' : 'translate-x-1'
|
||||
}`}
|
||||
/>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Categories */}
|
||||
<div className="p-4 bg-white rounded-lg border border-gray-200">
|
||||
<h3 className="font-semibold text-gray-800 mb-3">Kategorien</h3>
|
||||
<p className="text-sm text-gray-500 mb-4">Für welche Bereiche sollen Vorschläge generiert werden?</p>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
{Object.entries(categoryLabels).map(([key, label]) => {
|
||||
const checked = settings.suggestion_categories?.includes(key);
|
||||
return (
|
||||
<label
|
||||
key={key}
|
||||
className={`flex items-center gap-2 p-3 rounded-lg border cursor-pointer transition-colors ${
|
||||
checked ? 'bg-blue-50 border-blue-300' : 'bg-gray-50 border-gray-200 hover:bg-gray-100'
|
||||
}`}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={checked || false}
|
||||
onChange={() => toggleCategory(key)}
|
||||
className="w-4 h-4 rounded text-blue-500 focus:ring-blue-400"
|
||||
/>
|
||||
<span className="text-sm text-gray-700">{label}</span>
|
||||
</label>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Confidence Threshold */}
|
||||
<div className="p-4 bg-white rounded-lg border border-gray-200">
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<h3 className="font-semibold text-gray-800">Confidence Schwellwert</h3>
|
||||
<span className="text-sm font-mono text-blue-600">
|
||||
{Math.round((settings.confidence_threshold || 0.5) * 100)}%
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-sm text-gray-500 mb-3">
|
||||
Nur Vorschläge mit höherer Confidence anzeigen
|
||||
</p>
|
||||
<input
|
||||
type="range"
|
||||
min="0"
|
||||
max="1"
|
||||
step="0.05"
|
||||
value={settings.confidence_threshold || 0.5}
|
||||
onChange={(e) => update({ confidence_threshold: parseFloat(e.target.value) })}
|
||||
className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500"
|
||||
/>
|
||||
<div className="flex justify-between text-xs text-gray-400 mt-1">
|
||||
<span>Alle</span>
|
||||
<span>Sehr sicher</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Rate Limit */}
|
||||
<div className="p-4 bg-white rounded-lg border border-gray-200">
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<h3 className="font-semibold text-gray-800">Rate Limit</h3>
|
||||
<span className="text-sm font-mono text-blue-600">
|
||||
{settings.rate_limit_seconds || 10}s
|
||||
</span>
|
||||
</div>
|
||||
<p className="text-sm text-gray-500 mb-3">
|
||||
Mindestabstand zwischen Vorschlägen
|
||||
</p>
|
||||
<input
|
||||
type="range"
|
||||
min="5"
|
||||
max="60"
|
||||
step="5"
|
||||
value={settings.rate_limit_seconds || 10}
|
||||
onChange={(e) => update({ rate_limit_seconds: parseInt(e.target.value) })}
|
||||
className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer accent-blue-500"
|
||||
/>
|
||||
<div className="flex justify-between text-xs text-gray-400 mt-1">
|
||||
<span>5s</span>
|
||||
<span>60s</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Model Selection */}
|
||||
<div className="p-4 bg-white rounded-lg border border-gray-200">
|
||||
<h3 className="font-semibold text-gray-800 mb-2">KI Modell (Ollama Cloud)</h3>
|
||||
<p className="text-sm text-gray-500 mb-3">Wähle das Modell für Vorschläge. DeepSeek V4 ist empfohlen für beste Performance/Kosten.</p>
|
||||
<select
|
||||
value={settings.model || 'ollama/deepseek-v4'}
|
||||
onChange={(e) => update({ model: e.target.value })}
|
||||
className="w-full px-3 py-2 rounded-lg border border-gray-200 bg-white text-sm text-gray-700 focus:ring-2 focus:ring-blue-400 focus:border-blue-400 outline-none"
|
||||
>
|
||||
{modelOptions.map(opt => (
|
||||
<option key={opt.value} value={opt.value}>
|
||||
{opt.label}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -17,6 +17,7 @@ export function SettingsPage() {
|
||||
{ to: '/settings/sequences', label: t('sequences.title'), icon: '\ud83d\udd22' },
|
||||
{ to: '/settings/mail', label: t('mail.settings'), icon: '\ud83d\udce7' },
|
||||
{ to: '/settings/ai', label: t('nav.aiAssistant'), icon: '\ud83e\udde0' },
|
||||
{ to: '/settings/ai-proactive', label: 'Proaktive KI', icon: '\ud83e\udd16' },
|
||||
{ to: '/settings/notifications', label: t('settings.notifications'), icon: '\ud83d\udd14' },
|
||||
];
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@ import { MailSettingsPage } from '@/pages/MailSettings';
|
||||
import { SettingsNotificationsPage } from '@/pages/SettingsNotifications';
|
||||
import { AIAssistantPage } from '@/pages/AIAssistant';
|
||||
import { AISettingsPage } from '@/pages/AISettings';
|
||||
import { ProactiveAISettings } from '@/pages/ProactiveAISettings';
|
||||
|
||||
const router = createBrowserRouter([
|
||||
{
|
||||
@@ -89,6 +90,7 @@ const router = createBrowserRouter([
|
||||
{ path: 'mail', element: <MailSettingsPage /> },
|
||||
{ path: 'notifications', element: <SettingsNotificationsPage /> },
|
||||
{ path: 'ai', element: <AISettingsPage /> },
|
||||
{ path: 'ai-proactive', element: <ProactiveAISettings /> },
|
||||
],
|
||||
},
|
||||
],
|
||||
|
||||
@@ -14,12 +14,14 @@ export interface UIState {
|
||||
theme: Theme;
|
||||
locale: Locale;
|
||||
sidebarOpen: boolean;
|
||||
suggestionSidebarOpen: boolean;
|
||||
aiSidebarCollapsed: boolean;
|
||||
toasts: Toast[];
|
||||
setTheme: (theme: Theme) => void;
|
||||
setLocale: (locale: Locale) => void;
|
||||
toggleSidebar: () => void;
|
||||
setSidebarOpen: (open: boolean) => void;
|
||||
toggleSuggestionSidebar: () => void;
|
||||
toggleAISidebar: () => void;
|
||||
setAISidebarCollapsed: (collapsed: boolean) => void;
|
||||
addToast: (toast: Omit<Toast, 'id'>) => void;
|
||||
@@ -33,6 +35,7 @@ export const useUIStore = create<UIState>((set) => ({
|
||||
theme: (localStorage.getItem('leocrm_theme') as Theme) || 'light',
|
||||
locale: (localStorage.getItem('leocrm_lang') as Locale) || 'de',
|
||||
sidebarOpen: true,
|
||||
suggestionSidebarOpen: false,
|
||||
aiSidebarCollapsed: true,
|
||||
toasts: [],
|
||||
setTheme: (theme) => {
|
||||
@@ -45,6 +48,7 @@ export const useUIStore = create<UIState>((set) => ({
|
||||
},
|
||||
toggleSidebar: () => set((s) => ({ sidebarOpen: !s.sidebarOpen })),
|
||||
setSidebarOpen: (open) => set({ sidebarOpen: open }),
|
||||
toggleSuggestionSidebar: () => set((s) => ({ suggestionSidebarOpen: !s.suggestionSidebarOpen })),
|
||||
toggleAISidebar: () => set((s) => ({ aiSidebarCollapsed: !s.aiSidebarCollapsed })),
|
||||
setAISidebarCollapsed: (collapsed) => set({ aiSidebarCollapsed: collapsed }),
|
||||
addToast: (toast) => {
|
||||
|
||||
@@ -46,3 +46,9 @@ openpyxl>=3.1
|
||||
prometheus-client>=0.20
|
||||
structlog>=24.0
|
||||
litellm
|
||||
|
||||
# AI / Search
|
||||
pgvector>=0.3.0
|
||||
PyMuPDF>=1.24.0
|
||||
python-docx>=1.1.0
|
||||
python-pptx>=0.6.23
|
||||
|
||||
@@ -0,0 +1,440 @@
|
||||
"""Tests for the AI Proactive plugin — Context API, Suggestions API,
|
||||
Dismiss API, Settings API, Stats API, and Proactive Engine unit tests.
|
||||
|
||||
Uses pytest + pytest-asyncio, httpx.AsyncClient, login_client,
|
||||
seed_tenant_and_users, ORIGIN_HEADER from conftest.py.
|
||||
Mocks litellm.acompletion and litellm.aembedding to avoid real API calls.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
||||
|
||||
# Import models at module level so Base.metadata.create_all includes their tables
|
||||
from app.plugins.builtins.ai_proactive.models import ( # noqa: F401
|
||||
ContextLog,
|
||||
ProactiveSettings,
|
||||
ProactiveSuggestion,
|
||||
)
|
||||
from app.plugins.builtins.ai_assistant.models import ( # noqa: F401
|
||||
AIProvider, AIModel, AIPreset, AIAgent, AIChatSession, AIChatMessage,
|
||||
AIChatFolder, AIChatAttachment,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.models import ( # noqa: F401
|
||||
SearchProviderRegistry as SearchProviderRegistryModel,
|
||||
SearchIndexLog,
|
||||
)
|
||||
from app.plugins.builtins.ai_proactive import AIProactivePlugin
|
||||
from app.plugins.builtins.ai_assistant import AIAssistantPlugin
|
||||
from app.plugins.builtins.unified_search import UnifiedSearchPlugin
|
||||
from app.core.db import close_engine, reset_engine_for_testing
|
||||
from app.core.service_container import get_container
|
||||
from app.main import create_app
|
||||
from app.plugins.registry import reset_registry_for_testing
|
||||
from app.services.plugin_service import reset_plugin_service_for_testing
|
||||
from tests.conftest import ORIGIN_HEADER, _get_sync_engine, login_client, seed_tenant_and_users
|
||||
|
||||
# ─── AI Proactive Fixtures ───
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
async def clean_ai_proactive_tables():
|
||||
"""Truncate ai_proactive tables before each test (not in conftest TRUNCATE list)."""
|
||||
sync_eng = _get_sync_engine()
|
||||
with sync_eng.connect() as conn:
|
||||
conn.execute(
|
||||
text(
|
||||
"TRUNCATE TABLE ai_proactive_suggestions, "
|
||||
"ai_proactive_context_log, ai_proactive_settings CASCADE;"
|
||||
)
|
||||
)
|
||||
conn.commit()
|
||||
sync_eng.dispose()
|
||||
yield
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def ai_proactive_app(engine: AsyncEngine, redis_client):
|
||||
"""FastAPI app with AI Proactive plugin registered, installed, and activated."""
|
||||
reset_engine_for_testing(engine)
|
||||
app = create_app()
|
||||
registry = reset_registry_for_testing()
|
||||
registry.initialize(engine, app)
|
||||
container = get_container()
|
||||
await container.initialize()
|
||||
registry.register_plugin(AIAssistantPlugin())
|
||||
registry.register_plugin(UnifiedSearchPlugin())
|
||||
registry.register_plugin(AIProactivePlugin())
|
||||
reset_plugin_service_for_testing(registry)
|
||||
|
||||
sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
|
||||
async with sf() as session:
|
||||
# Install dependencies first, then ai_proactive
|
||||
await registry.install(session, "ai_assistant")
|
||||
await registry.activate(session, "ai_assistant")
|
||||
await registry.install(session, "unified_search")
|
||||
await registry.activate(session, "unified_search")
|
||||
await registry.install(session, "ai_proactive")
|
||||
await registry.activate(session, "ai_proactive")
|
||||
await session.commit()
|
||||
|
||||
yield app
|
||||
await close_engine()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def ai_proactive_client(ai_proactive_app) -> AsyncClient:
|
||||
"""HTTP test client with ai_proactive plugin active."""
|
||||
transport = ASGITransport(app=ai_proactive_app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def ai_proactive_authed_client(
|
||||
ai_proactive_client: AsyncClient, db_session: AsyncSession
|
||||
) -> tuple[AsyncClient, dict]:
|
||||
"""Authenticated admin client with seeded data and ai_proactive plugin active."""
|
||||
seed = await seed_tenant_and_users(db_session)
|
||||
# Grant is_system_admin so require_permission passes for ai_proactive:* permissions
|
||||
from sqlalchemy import update
|
||||
from app.models.user import User
|
||||
await db_session.execute(
|
||||
update(User).where(User.id == seed["admin_a"].id).values(is_system_admin=True)
|
||||
)
|
||||
await db_session.commit()
|
||||
# Login and capture CSRF token
|
||||
login_resp = await ai_proactive_client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"email": "admin@tenanta.com", "password": "TestPass123!"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert login_resp.status_code == 200, f"Login failed: {login_resp.text}"
|
||||
csrf_token = login_resp.json().get("csrf_token", "")
|
||||
# Store CSRF token on client for use in requests
|
||||
ai_proactive_client.headers["X-CSRF-Token"] = csrf_token
|
||||
return ai_proactive_client, seed
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def mock_litellm():
|
||||
"""Mock litellm.acompletion and litellm.aembedding for all tests."""
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.choices = [MagicMock()]
|
||||
mock_resp.choices[0].message.content = json.dumps({
|
||||
"suggestion_type": "info",
|
||||
"title": "Test Suggestion",
|
||||
"content": "This is a test suggestion",
|
||||
"confidence": 0.8,
|
||||
"actions": [
|
||||
{
|
||||
"method": "GET",
|
||||
"path": "/api/v1/contacts",
|
||||
"body": {},
|
||||
"description": "View contacts",
|
||||
}
|
||||
],
|
||||
})
|
||||
|
||||
mock_emb_resp = MagicMock()
|
||||
mock_emb_resp.data = [{"embedding": [0.1] * 768}]
|
||||
|
||||
with (
|
||||
patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_resp),
|
||||
patch("litellm.aembedding", new_callable=AsyncMock, return_value=mock_emb_resp),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
# ─── 1. Context API (POST /api/v1/ai-proactive/context) ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_report_accepted(
|
||||
ai_proactive_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""Context-Report wird akzeptiert (200)."""
|
||||
client, _ = ai_proactive_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/ai-proactive/context",
|
||||
json={
|
||||
"page": "/contacts",
|
||||
"entity_type": "contact",
|
||||
"entity_id": str(uuid.uuid4()),
|
||||
},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "ok"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_unauthorized(ai_proactive_client: AsyncClient):
|
||||
"""Ohne Auth → 403 (CSRF middleware blocks before auth check on POST)."""
|
||||
resp = await ai_proactive_client.post(
|
||||
"/api/v1/ai-proactive/context",
|
||||
json={"page": "/contacts"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
# Without a session cookie, CSRF middleware returns 403 before auth is checked
|
||||
assert resp.status_code in (401, 403)
|
||||
|
||||
|
||||
# ─── 2. Suggestions API (GET /api/v1/ai-proactive/suggestions) ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_suggestions(
|
||||
ai_proactive_authed_client: tuple[AsyncClient, dict],
|
||||
db_session: AsyncSession,
|
||||
):
|
||||
"""Gibt Suggestions für User zurück."""
|
||||
client, seed = ai_proactive_authed_client
|
||||
suggestion = ProactiveSuggestion(
|
||||
tenant_id=seed["tenant_a"].id,
|
||||
user_id=seed["admin_a"].id,
|
||||
entity_type="contact",
|
||||
entity_id=uuid.uuid4(),
|
||||
suggestion_type="info",
|
||||
title="Test Suggestion",
|
||||
content="Test content",
|
||||
confidence=0.8,
|
||||
actions=[],
|
||||
context_snapshot={},
|
||||
)
|
||||
db_session.add(suggestion)
|
||||
await db_session.commit()
|
||||
|
||||
resp = await client.get(
|
||||
"/api/v1/ai-proactive/suggestions",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "items" in data
|
||||
assert data["total"] >= 1
|
||||
titles = [item["title"] for item in data["items"]]
|
||||
assert "Test Suggestion" in titles
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_suggestions_tenant_isolation(
|
||||
ai_proactive_authed_client: tuple[AsyncClient, dict],
|
||||
db_session: AsyncSession,
|
||||
):
|
||||
"""Nur eigene Tenant — Tenant B suggestions are not visible to Tenant A."""
|
||||
client, seed = ai_proactive_authed_client
|
||||
|
||||
# Suggestion for tenant B user
|
||||
suggestion_b = ProactiveSuggestion(
|
||||
tenant_id=seed["tenant_b"].id,
|
||||
user_id=seed["admin_b"].id,
|
||||
entity_type="contact",
|
||||
entity_id=uuid.uuid4(),
|
||||
suggestion_type="info",
|
||||
title="Tenant B Secret",
|
||||
content="Should not be visible to A",
|
||||
confidence=0.8,
|
||||
actions=[],
|
||||
context_snapshot={},
|
||||
)
|
||||
# Suggestion for tenant A user
|
||||
suggestion_a = ProactiveSuggestion(
|
||||
tenant_id=seed["tenant_a"].id,
|
||||
user_id=seed["admin_a"].id,
|
||||
entity_type="contact",
|
||||
entity_id=uuid.uuid4(),
|
||||
suggestion_type="info",
|
||||
title="Tenant A Visible",
|
||||
content="Should be visible",
|
||||
confidence=0.8,
|
||||
actions=[],
|
||||
context_snapshot={},
|
||||
)
|
||||
db_session.add_all([suggestion_b, suggestion_a])
|
||||
await db_session.commit()
|
||||
|
||||
resp = await client.get(
|
||||
"/api/v1/ai-proactive/suggestions",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
titles = [item["title"] for item in data["items"]]
|
||||
assert "Tenant A Visible" in titles
|
||||
assert "Tenant B Secret" not in titles
|
||||
|
||||
|
||||
# ─── 3. Dismiss API (POST /api/v1/ai-proactive/suggestions/{id}/dismiss) ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dismiss_suggestion(
|
||||
ai_proactive_authed_client: tuple[AsyncClient, dict],
|
||||
db_session: AsyncSession,
|
||||
):
|
||||
"""Suggestion wird dismissed."""
|
||||
client, seed = ai_proactive_authed_client
|
||||
suggestion = ProactiveSuggestion(
|
||||
tenant_id=seed["tenant_a"].id,
|
||||
user_id=seed["admin_a"].id,
|
||||
entity_type="contact",
|
||||
entity_id=uuid.uuid4(),
|
||||
suggestion_type="info",
|
||||
title="Dismiss Me",
|
||||
content="Please dismiss this",
|
||||
confidence=0.8,
|
||||
actions=[],
|
||||
context_snapshot={},
|
||||
)
|
||||
db_session.add(suggestion)
|
||||
await db_session.commit()
|
||||
|
||||
resp = await client.post(
|
||||
f"/api/v1/ai-proactive/suggestions/{suggestion.id}/dismiss",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["status"] == "ok"
|
||||
|
||||
# Verify it no longer appears in active suggestions
|
||||
resp2 = await client.get(
|
||||
"/api/v1/ai-proactive/suggestions",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
data = resp2.json()
|
||||
ids = [item["id"] for item in data["items"]]
|
||||
assert str(suggestion.id) not in ids
|
||||
|
||||
|
||||
# ─── 4. Settings API (GET/PUT /api/v1/ai-proactive/settings) ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_settings(
|
||||
ai_proactive_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""Settings werden zurückgegeben."""
|
||||
client, _ = ai_proactive_authed_client
|
||||
resp = await client.get(
|
||||
"/api/v1/ai-proactive/settings",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "enabled" in data
|
||||
assert "suggestion_categories" in data
|
||||
assert "confidence_threshold" in data
|
||||
assert "rate_limit_seconds" in data
|
||||
assert "model" in data
|
||||
assert "available_models" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_settings(
|
||||
ai_proactive_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""Settings können aktualisiert werden."""
|
||||
client, _ = ai_proactive_authed_client
|
||||
resp = await client.put(
|
||||
"/api/v1/ai-proactive/settings",
|
||||
json={"enabled": False, "confidence_threshold": 0.7},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["enabled"] is False
|
||||
assert data["confidence_threshold"] == 0.7
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_model(
|
||||
ai_proactive_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""Model kann geändert werden."""
|
||||
client, _ = ai_proactive_authed_client
|
||||
resp = await client.put(
|
||||
"/api/v1/ai-proactive/settings",
|
||||
json={"model": "ollama/llama3.2"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["model"] == "ollama/llama3.2"
|
||||
|
||||
|
||||
# ─── 5. Stats API (GET /api/v1/ai-proactive/stats) ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_stats(
|
||||
ai_proactive_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""Stats werden zurückgegeben."""
|
||||
client, _ = ai_proactive_authed_client
|
||||
resp = await client.get(
|
||||
"/api/v1/ai-proactive/stats",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "total_suggestions" in data
|
||||
assert "dismissed" in data
|
||||
assert "acted_upon" in data
|
||||
assert "active" in data
|
||||
assert "dismiss_rate" in data
|
||||
assert "act_rate" in data
|
||||
|
||||
|
||||
# ─── 6. Proactive Engine (Unit Tests with mocks) ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sse_push():
|
||||
"""Suggestion wird in Queue gepusht."""
|
||||
from app.plugins.builtins.ai_proactive.services import get_sse_queue, push_suggestion
|
||||
|
||||
user_id = f"test-sse-{uuid.uuid4()}"
|
||||
queue = get_sse_queue(user_id)
|
||||
|
||||
# Ensure queue is empty
|
||||
while not queue.empty():
|
||||
await queue.get()
|
||||
|
||||
suggestion = {"id": "test-1", "title": "Test SSE Push"}
|
||||
await push_suggestion(user_id, suggestion)
|
||||
|
||||
item = await asyncio.wait_for(queue.get(), timeout=1.0)
|
||||
assert item["id"] == "test-1"
|
||||
assert item["title"] == "Test SSE Push"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_limiting(redis_client):
|
||||
"""Rate-Limit funktioniert."""
|
||||
from app.plugins.builtins.ai_proactive.services import is_rate_limited
|
||||
|
||||
tenant_id = uuid.uuid4()
|
||||
user_id = uuid.uuid4()
|
||||
rate_limit_seconds = 10
|
||||
|
||||
with patch(
|
||||
"app.plugins.builtins.ai_proactive.services.get_cache",
|
||||
return_value=redis_client,
|
||||
):
|
||||
# First call should not be rate limited
|
||||
limited = await is_rate_limited(tenant_id, user_id, rate_limit_seconds)
|
||||
assert limited is False
|
||||
|
||||
# Second call should be rate limited
|
||||
limited = await is_rate_limited(tenant_id, user_id, rate_limit_seconds)
|
||||
assert limited is True
|
||||
@@ -0,0 +1,552 @@
|
||||
"""Tests for the Unified Search plugin — covers ACs 1-12 and unit tests.
|
||||
|
||||
Tests hybrid search infrastructure: API endpoints, RRF fusion, autocomplete,
|
||||
text extraction, and provider registry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
||||
|
||||
from app.core.db import close_engine, reset_engine_for_testing
|
||||
from app.core.service_container import get_container
|
||||
from app.main import create_app
|
||||
from app.plugins.builtins.unified_search import UnifiedSearchPlugin
|
||||
from app.plugins.builtins.unified_search.provider_registry import (
|
||||
SearchProviderRegistry,
|
||||
get_search_registry,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.search_engine import rrf_fusion
|
||||
from app.plugins.builtins.unified_search.text_extraction import extract_text_from_file
|
||||
from app.plugins.registry import reset_registry_for_testing
|
||||
from app.services.plugin_service import reset_plugin_service_for_testing
|
||||
from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users
|
||||
|
||||
# ─── Unified Search Fixtures ───
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def search_app(engine: AsyncEngine, redis_client):
|
||||
"""FastAPI app with UnifiedSearch plugin registered, installed, and activated."""
|
||||
reset_engine_for_testing(engine)
|
||||
app = create_app()
|
||||
registry = reset_registry_for_testing()
|
||||
registry.initialize(engine, app)
|
||||
container = get_container()
|
||||
await container.initialize()
|
||||
registry.register_plugin(UnifiedSearchPlugin())
|
||||
reset_plugin_service_for_testing(registry)
|
||||
|
||||
sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
|
||||
async with sf() as session:
|
||||
await registry.install(session, "unified_search")
|
||||
await registry.activate(session, "unified_search")
|
||||
await session.commit()
|
||||
|
||||
yield app
|
||||
await close_engine()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def search_client(search_app) -> AsyncClient:
|
||||
"""HTTP test client with unified search plugin active."""
|
||||
transport = ASGITransport(app=search_app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def search_authed_client(
|
||||
search_client: AsyncClient, db_session: AsyncSession
|
||||
) -> tuple[AsyncClient, dict]:
|
||||
"""Authenticated admin client with seeded data and unified search plugin active."""
|
||||
seed = await seed_tenant_and_users(db_session)
|
||||
await login_client(search_client, "admin@tenanta.com")
|
||||
return search_client, seed
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def mock_external_calls():
|
||||
"""Mock all external API calls (LiteLLM, job queue) for all tests."""
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.choices = [MagicMock()]
|
||||
mock_resp.choices[0].message.content = json.dumps({
|
||||
"normalized_query": "test",
|
||||
"entities": {},
|
||||
"intent": "search",
|
||||
"semantic_terms": [],
|
||||
"suggested_filters": {},
|
||||
})
|
||||
|
||||
mock_emb_resp = MagicMock()
|
||||
mock_emb_resp.data = [{"embedding": [0.1] * 768}]
|
||||
|
||||
with (
|
||||
patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_resp),
|
||||
patch("litellm.aembedding", new_callable=AsyncMock, return_value=mock_emb_resp),
|
||||
patch("app.core.jobs.enqueue_job", new_callable=AsyncMock, return_value="job-123"),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
# ─── AC1: POST /api/v1/search → 200 + results array ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
|
||||
async def test_ac1_search_returns_results(
|
||||
mock_hybrid_search,
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC1: POST /api/v1/search with query returns 200 + results array."""
|
||||
client, _ = search_authed_client
|
||||
mock_hybrid_search.return_value = [
|
||||
{
|
||||
"entity_type": "company",
|
||||
"entity_id": str(uuid.uuid4()),
|
||||
"title": "Company Alpha",
|
||||
"snippet": "IT company",
|
||||
"score": 0.95,
|
||||
"data": {"industry": "IT"},
|
||||
}
|
||||
]
|
||||
|
||||
resp = await client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": "Company"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "results" in data
|
||||
assert isinstance(data["results"], list)
|
||||
assert len(data["results"]) > 0
|
||||
assert data["results"][0]["entity_type"] == "company"
|
||||
assert data["results"][0]["title"] == "Company Alpha"
|
||||
assert "score" in data["results"][0]
|
||||
assert "query" in data
|
||||
assert "normalized_query" in data
|
||||
assert "summary" in data
|
||||
|
||||
|
||||
# ─── AC2: entity_types filter ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
|
||||
async def test_ac2_search_entity_types_filter(
|
||||
mock_hybrid_search,
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC2: Search with entity_types filters correctly."""
|
||||
client, _ = search_authed_client
|
||||
mock_hybrid_search.return_value = []
|
||||
|
||||
resp = await client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": "test", "entity_types": ["company"]},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
# Verify hybrid_search was called with entity_types=["company"]
|
||||
call_kwargs = mock_hybrid_search.call_args
|
||||
assert call_kwargs.kwargs.get("entity_types") == ["company"]
|
||||
|
||||
|
||||
# ─── AC3: No auth → 401 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac3_search_no_auth_401(search_client: AsyncClient):
|
||||
"""AC3: Search without auth returns 401."""
|
||||
resp = await search_client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": "test"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
# ─── AC4: Tenant isolation ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
|
||||
async def test_ac4_search_tenant_isolation(
|
||||
mock_hybrid_search,
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC4: Search only returns results from the user's tenant."""
|
||||
client, seed = search_authed_client
|
||||
mock_hybrid_search.return_value = []
|
||||
|
||||
resp = await client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": "test"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
# Verify hybrid_search was called with tenant A's ID
|
||||
call_args = mock_hybrid_search.call_args
|
||||
tenant_id_arg = call_args.kwargs.get("tenant_id")
|
||||
assert tenant_id_arg == seed["tenant_a"].id
|
||||
|
||||
|
||||
# ─── AC5: Empty query → 422 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac5_search_empty_query_422(search_authed_client: tuple[AsyncClient, dict]):
|
||||
"""AC5: Empty query returns 422 (min_length=1)."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": ""},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
# ─── AC6: GET /api/v1/search/suggest → suggestions ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac6_suggest_returns_suggestions(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC6: GET /api/v1/search/suggest returns 200 + suggestions list."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.get(
|
||||
"/api/v1/search/suggest",
|
||||
params={"q": "comp"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "suggestions" in data
|
||||
assert isinstance(data["suggestions"], list)
|
||||
|
||||
|
||||
# ─── AC7: Empty suggest query → 422 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac7_suggest_empty_query_422(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC7: GET /api/v1/search/suggest with empty query returns 422 (min_length=1)."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.get(
|
||||
"/api/v1/search/suggest",
|
||||
params={"q": ""},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
# ─── AC8: POST /api/v1/search/similar → similar results ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac8_similar_returns_results(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC8: POST /api/v1/search/similar returns 200 + similar dict."""
|
||||
client, seed = search_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/search/similar",
|
||||
json={
|
||||
"entity_type": "company",
|
||||
"entity_id": str(seed["company_a"].id),
|
||||
"limit": 5,
|
||||
},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "similar" in data
|
||||
assert isinstance(data["similar"], dict)
|
||||
|
||||
|
||||
# ─── AC9: Similar tenant isolation ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac9_similar_tenant_isolation(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC9: Similar search respects tenant isolation."""
|
||||
client, seed = search_authed_client
|
||||
# Use company from tenant B
|
||||
resp = await client.post(
|
||||
"/api/v1/search/similar",
|
||||
json={
|
||||
"entity_type": "company",
|
||||
"entity_id": str(seed["company_b"].id),
|
||||
"limit": 5,
|
||||
},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
# Should return 200 but with empty similar (tenant B company not accessible from tenant A)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert isinstance(data["similar"], dict)
|
||||
# Company B should not be found in tenant A's context
|
||||
for etype, items in data["similar"].items():
|
||||
for item in items:
|
||||
assert item.get("entity_id") != str(seed["company_b"].id)
|
||||
|
||||
|
||||
# ─── AC10: GET /api/v1/search/providers → provider list ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac10_list_providers(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC10: GET /api/v1/search/providers returns 200 + active provider list."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.get(
|
||||
"/api/v1/search/providers",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert isinstance(data, list)
|
||||
# After activation, providers should be registered
|
||||
if len(data) > 0:
|
||||
assert "entity_type" in data[0]
|
||||
assert "plugin_name" in data[0]
|
||||
assert "is_active" in data[0]
|
||||
assert data[0]["is_active"] is True
|
||||
|
||||
|
||||
# ─── AC11: POST /api/v1/search/reindex as admin → 200 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac11_reindex_admin_200(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC11: Admin can trigger reindex (200)."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/search/reindex",
|
||||
json={"entity_types": ["company"]},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["status"] == "ok"
|
||||
assert "entity_types" in data
|
||||
|
||||
|
||||
# ─── AC12: POST /api/v1/search/reindex as viewer → 403 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac12_reindex_viewer_403(
|
||||
search_client: AsyncClient, db_session: AsyncSession
|
||||
):
|
||||
"""AC12: Non-admin (viewer) gets 403 on reindex."""
|
||||
await seed_tenant_and_users(db_session)
|
||||
await login_client(search_client, "viewer@tenanta.com")
|
||||
|
||||
resp = await search_client.post(
|
||||
"/api/v1/search/reindex",
|
||||
json={"entity_types": ["company"]},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
# ─── Unit Tests: RRF Fusion ───
|
||||
|
||||
|
||||
def test_rrf_fusion_combines_fts_and_vector_scores():
|
||||
"""RRF fusion combines FTS + Vector scores correctly."""
|
||||
fts_results = [
|
||||
{"id": "1", "title": "Result A", "score": 0.9},
|
||||
{"id": "2", "title": "Result B", "score": 0.8},
|
||||
]
|
||||
vec_results = [
|
||||
{"id": "2", "title": "Result B", "score": 0.95},
|
||||
{"id": "3", "title": "Result C", "score": 0.7},
|
||||
]
|
||||
|
||||
fused = rrf_fusion(fts_results, vec_results, "company")
|
||||
|
||||
# All 3 unique IDs should be present
|
||||
ids = [str(r.get("id", "")) for r in fused]
|
||||
assert "1" in ids
|
||||
assert "2" in ids
|
||||
assert "3" in ids
|
||||
|
||||
# Results should be sorted by fused score descending
|
||||
assert fused[0]["_score"] >= fused[-1]["_score"]
|
||||
|
||||
# ID 2 appears in both FTS and vector, so should have highest score
|
||||
assert str(fused[0].get("id", "")) == "2"
|
||||
|
||||
|
||||
def test_rrf_fusion_empty_inputs():
|
||||
"""RRF fusion with empty inputs returns empty list."""
|
||||
fused = rrf_fusion([], [], "company")
|
||||
assert fused == []
|
||||
|
||||
|
||||
def test_rrf_fusion_fts_only():
|
||||
"""RRF fusion with only FTS results returns them sorted by score."""
|
||||
fts_results = [
|
||||
{"id": "1", "title": "A"},
|
||||
{"id": "2", "title": "B"},
|
||||
]
|
||||
fused = rrf_fusion(fts_results, [], "company")
|
||||
assert len(fused) == 2
|
||||
# First result (rank 0) should have higher score
|
||||
assert fused[0]["_score"] > fused[1]["_score"]
|
||||
|
||||
|
||||
# ─── Unit Tests: Text Extraction ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_text_from_text_file(tmp_path):
|
||||
"""Extract text from a plain text file."""
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("Hello World\nThis is a test.", encoding="utf-8")
|
||||
|
||||
result = await extract_text_from_file(str(test_file), "text/plain")
|
||||
assert "Hello World" in result
|
||||
assert "test" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_text_from_pdf_mocked(tmp_path):
|
||||
"""Extract text from PDF with mocked PyMuPDF."""
|
||||
test_file = tmp_path / "test.pdf"
|
||||
test_file.write_bytes(b"%PDF-1.4 fake")
|
||||
|
||||
mock_page = MagicMock()
|
||||
mock_page.get_text.return_value = "PDF content text"
|
||||
mock_doc = MagicMock()
|
||||
mock_doc.__iter__ = MagicMock(return_value=iter([mock_page]))
|
||||
mock_doc.close = MagicMock()
|
||||
|
||||
with patch("fitz.open", return_value=mock_doc):
|
||||
result = await extract_text_from_file(str(test_file), "application/pdf")
|
||||
assert "PDF content text" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_text_unsupported_mime():
|
||||
"""Unsupported MIME type returns empty string."""
|
||||
result = await extract_text_from_file("/tmp/nonexistent", "application/octet-stream")
|
||||
assert result == ""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_text_image_returns_empty(tmp_path):
|
||||
"""Image MIME type returns empty string (OCR not implemented)."""
|
||||
test_file = tmp_path / "image.png"
|
||||
test_file.write_bytes(b"\x89PNG\r\n\x1a\n")
|
||||
result = await extract_text_from_file(str(test_file), "image/png")
|
||||
assert result == ""
|
||||
|
||||
|
||||
# ─── Unit Tests: Provider Registry ───
|
||||
|
||||
|
||||
def test_provider_register_and_unregister():
|
||||
"""Provider can be registered and unregistered."""
|
||||
registry = SearchProviderRegistry()
|
||||
|
||||
# Create a mock provider
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.entity_type = "test_entity"
|
||||
|
||||
# Register
|
||||
registry.register(mock_provider)
|
||||
assert registry.get("test_entity") is mock_provider
|
||||
|
||||
# Unregister
|
||||
registry.unregister("test_entity")
|
||||
assert registry.get("test_entity") is None
|
||||
|
||||
|
||||
def test_provider_get_all():
|
||||
"""get_all returns all registered providers."""
|
||||
registry = SearchProviderRegistry()
|
||||
|
||||
p1 = MagicMock()
|
||||
p1.entity_type = "contact"
|
||||
p2 = MagicMock()
|
||||
p2.entity_type = "company"
|
||||
|
||||
registry.register(p1)
|
||||
registry.register(p2)
|
||||
|
||||
all_providers = registry.get_all()
|
||||
assert len(all_providers) == 2
|
||||
entity_types = [p.entity_type for p in all_providers]
|
||||
assert "contact" in entity_types
|
||||
assert "company" in entity_types
|
||||
|
||||
|
||||
def test_provider_clear():
|
||||
"""clear removes all providers."""
|
||||
registry = SearchProviderRegistry()
|
||||
p = MagicMock()
|
||||
p.entity_type = "test"
|
||||
registry.register(p)
|
||||
assert len(registry.get_all()) == 1
|
||||
|
||||
registry.clear()
|
||||
assert len(registry.get_all()) == 0
|
||||
|
||||
|
||||
def test_provider_get_entity_types():
|
||||
"""get_entity_types returns list of registered entity types."""
|
||||
registry = SearchProviderRegistry()
|
||||
p1 = MagicMock()
|
||||
p1.entity_type = "contact"
|
||||
p2 = MagicMock()
|
||||
p2.entity_type = "company"
|
||||
registry.register(p1)
|
||||
registry.register(p2)
|
||||
|
||||
types = registry.get_entity_types()
|
||||
assert "contact" in types
|
||||
assert "company" in types
|
||||
assert len(types) == 2
|
||||
|
||||
|
||||
# ─── Unit Tests: Autocomplete ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_autocomplete_empty_query_returns_empty(db_session: AsyncSession):
|
||||
"""Autocomplete with empty query returns empty list."""
|
||||
from app.plugins.builtins.unified_search.search_engine import autocomplete
|
||||
|
||||
result = await autocomplete(db_session, "", uuid.uuid4(), 10)
|
||||
assert result == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_autocomplete_returns_titles(db_session: AsyncSession):
|
||||
"""Autocomplete returns titles from FTS prefix search."""
|
||||
from app.plugins.builtins.unified_search.search_engine import autocomplete
|
||||
|
||||
# With no providers registered and no data, should return empty list
|
||||
result = await autocomplete(db_session, "test", uuid.uuid4(), 10)
|
||||
assert isinstance(result, list)
|
||||
assert len(result) <= 10
|
||||
Reference in New Issue
Block a user