feat: unified_search + ai_proactive plugins with Ollama Cloud DeepSeek V4
- unified_search: Hybride Suche (PostgreSQL FTS + pgvector + RRF Fusion) - 5 Search Providers (Contact, Company, Mail, File, Event) - KI Query Understanding (Fuzzy, Facetten via LiteLLM) - DMS Text-Extraction (PDF, DOCX, XLSX, PPTX) - Embedding Pipeline (ollama/nomic-embed-text, 768 Dim) - Background Jobs für Indexierung - Plugin-basierte Provider Registry - ai_proactive: Proaktiver KI-Agent - Context-Tracking (Frontend → Backend → Event Bus) - Proactive Engine mit LLM Suggestion-Generierung - SSE Real-time Push an Frontend - 6 AI Tools für Tool Registry - Rate-Limiting + User Settings - Deep Analysis Background Jobs - Frontend Integration: - useAIContext Hook, SuggestionSidebar, SuggestionBadge - ProactiveAISettings Page, Search API Client - Globale Suche auf neue API umgestellt - Tests: test_unified_search.py + test_ai_proactive.py (alle bestanden) - Config: Ollama Cloud DeepSeek V4 als Default, konfigurierbar - Dependencies: PyMuPDF, python-docx, python-pptx, pgvector - Bugfixes: notification type_key length, migration IF NOT EXISTS
This commit is contained in:
Agent Zero
@@ -0,0 +1,316 @@
|
||||
"""API routes for the Unified Search plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.db import get_db
|
||||
from app.core.jobs import enqueue_job
|
||||
from app.deps import get_current_user, require_permission
|
||||
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
|
||||
from app.plugins.builtins.unified_search.query_understanding import (
|
||||
llm_aggregate_results,
|
||||
llm_analyze_query,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.schemas import (
|
||||
ProviderResponse,
|
||||
ReindexRequest,
|
||||
SearchRequest,
|
||||
SearchResponse,
|
||||
SearchResult,
|
||||
SimilarRequest,
|
||||
SimilarResponse,
|
||||
SuggestRequest,
|
||||
SuggestResponse,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.search_engine import (
|
||||
autocomplete,
|
||||
find_similar_all_types,
|
||||
hybrid_search,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/v1/search", tags=["search"])
|
||||
|
||||
|
||||
# ─── Search ───
|
||||
|
||||
@router.post("", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def search(
|
||||
req: SearchRequest,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> SearchResponse:
|
||||
"""Perform hybrid search with KI query understanding."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
|
||||
# KI query understanding
|
||||
query_analysis = await llm_analyze_query(req.query)
|
||||
|
||||
# Hybrid search
|
||||
results = await hybrid_search(
|
||||
db=db,
|
||||
query_analysis=query_analysis,
|
||||
tenant_id=tenant_id,
|
||||
entity_types=req.entity_types,
|
||||
limit=req.limit,
|
||||
)
|
||||
|
||||
# KI result aggregation
|
||||
aggregation = await llm_aggregate_results(results, req.query)
|
||||
|
||||
search_results = [
|
||||
SearchResult(
|
||||
entity_type=r.get("entity_type", ""),
|
||||
entity_id=r.get("entity_id", ""),
|
||||
title=r.get("title", ""),
|
||||
snippet=r.get("snippet", ""),
|
||||
score=r.get("score", 0.0),
|
||||
data=r.get("data", {}),
|
||||
)
|
||||
for r in results
|
||||
]
|
||||
|
||||
return SearchResponse(
|
||||
query=req.query,
|
||||
normalized_query=query_analysis.get("normalized_query", req.query),
|
||||
results=search_results,
|
||||
facets=aggregation.get("facets", {}),
|
||||
summary=aggregation.get("summary", f"{len(results)} Ergebnisse"),
|
||||
suggestions=aggregation.get("suggestions", []),
|
||||
)
|
||||
|
||||
|
||||
# ─── Suggest / Autocomplete ───
|
||||
|
||||
@router.get("/suggest", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def suggest(
|
||||
q: str = Query(..., min_length=1, max_length=200),
|
||||
limit: int = Query(default=10, ge=1, le=50),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> SuggestResponse:
|
||||
"""Autocomplete suggestions using FTS prefix search."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
suggestions = await autocomplete(db, q, tenant_id, limit)
|
||||
return SuggestResponse(suggestions=suggestions)
|
||||
|
||||
|
||||
# ─── Similar ───
|
||||
|
||||
@router.post("/similar", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def find_similar(
|
||||
req: SimilarRequest,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> SimilarResponse:
|
||||
"""Find similar entities across all types based on embedding."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
try:
|
||||
entity_id = uuid.UUID(req.entity_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid entity_id")
|
||||
|
||||
similar = await find_similar_all_types(
|
||||
db=db,
|
||||
entity_type=req.entity_type,
|
||||
entity_id=entity_id,
|
||||
tenant_id=tenant_id,
|
||||
limit=req.limit,
|
||||
)
|
||||
|
||||
result_dict: dict[str, list[SearchResult]] = {}
|
||||
for etype, items in similar.items():
|
||||
result_dict[etype] = [
|
||||
SearchResult(
|
||||
entity_type=r.get("entity_type", etype),
|
||||
entity_id=r.get("entity_id", ""),
|
||||
title=r.get("title", ""),
|
||||
snippet=r.get("snippet", ""),
|
||||
score=r.get("score", 0.0),
|
||||
data=r.get("data", {}),
|
||||
)
|
||||
for r in items
|
||||
]
|
||||
|
||||
return SimilarResponse(similar=result_dict)
|
||||
|
||||
|
||||
# ─── Reindex ───
|
||||
|
||||
@router.post("/reindex", dependencies=[Depends(require_permission("search:admin"))])
|
||||
async def reindex(
|
||||
req: ReindexRequest,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Trigger reindexing of specified entity types."""
|
||||
entity_types = req.entity_types
|
||||
if not entity_types:
|
||||
entity_types = get_search_registry().get_entity_types()
|
||||
|
||||
job_ids: list[str] = []
|
||||
for etype in entity_types:
|
||||
job_id = await enqueue_job("reindex", etype)
|
||||
if job_id:
|
||||
job_ids.append(job_id)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"message": f"Reindexing {len(entity_types)} entity types",
|
||||
"entity_types": entity_types,
|
||||
"job_ids": job_ids,
|
||||
}
|
||||
|
||||
|
||||
# ─── Providers ───
|
||||
|
||||
@router.get("/providers", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def list_providers(
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> list[ProviderResponse]:
|
||||
"""List all active search providers."""
|
||||
registry = get_search_registry()
|
||||
providers = registry.get_all()
|
||||
return [
|
||||
ProviderResponse(
|
||||
entity_type=p.entity_type,
|
||||
plugin_name="unified_search",
|
||||
is_active=True,
|
||||
)
|
||||
for p in providers
|
||||
]
|
||||
|
||||
|
||||
@router.post("/providers/{entity_type}/toggle", dependencies=[Depends(require_permission("search:admin"))])
|
||||
async def toggle_provider(
|
||||
entity_type: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Toggle a search provider on/off."""
|
||||
registry = get_search_registry()
|
||||
provider = registry.get(entity_type)
|
||||
if provider is None:
|
||||
raise HTTPException(status_code=404, detail=f"Provider not found: {entity_type}")
|
||||
|
||||
# Toggle in DB
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT is_active FROM unified_search_providers
|
||||
WHERE tenant_id = :tid AND entity_type = :et
|
||||
"""
|
||||
),
|
||||
{"tid": tenant_id, "et": entity_type},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
|
||||
if row:
|
||||
new_status = not row["is_active"]
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE unified_search_providers SET is_active = :active
|
||||
WHERE tenant_id = :tid AND entity_type = :et
|
||||
"""
|
||||
),
|
||||
{"active": new_status, "tid": tenant_id, "et": entity_type},
|
||||
)
|
||||
else:
|
||||
new_status = False
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO unified_search_providers (tenant_id, entity_type, plugin_name, is_active)
|
||||
VALUES (:tid, :et, 'unified_search', false)
|
||||
"""
|
||||
),
|
||||
{"tid": tenant_id, "et": entity_type},
|
||||
)
|
||||
|
||||
await db.commit()
|
||||
|
||||
if not new_status:
|
||||
registry.unregister(entity_type)
|
||||
else:
|
||||
# Re-register by clearing and re-running auto_register
|
||||
from app.plugins.builtins.unified_search.provider_registry import auto_register_providers
|
||||
await auto_register_providers(db)
|
||||
|
||||
return {
|
||||
"entity_type": entity_type,
|
||||
"is_active": new_status,
|
||||
}
|
||||
|
||||
|
||||
# ─── Stats ───
|
||||
|
||||
@router.get("/stats", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def search_stats(
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Get search index statistics."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
|
||||
stats: dict = {}
|
||||
tables = {
|
||||
"contacts": "embedding",
|
||||
"companies": "embedding",
|
||||
"mails": "embedding",
|
||||
"files": "embedding",
|
||||
"calendar_entries": "embedding",
|
||||
}
|
||||
|
||||
for table, col in tables.items():
|
||||
try:
|
||||
total_result = await db.execute(
|
||||
text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL"),
|
||||
{"tid": tenant_id},
|
||||
)
|
||||
total = total_result.scalar() or 0
|
||||
|
||||
indexed_result = await db.execute(
|
||||
text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL AND {col} IS NOT NULL"),
|
||||
{"tid": tenant_id},
|
||||
)
|
||||
indexed = indexed_result.scalar() or 0
|
||||
|
||||
stats[table] = {
|
||||
"total": total,
|
||||
"indexed": indexed,
|
||||
"pending": total - indexed,
|
||||
}
|
||||
except Exception:
|
||||
logger.exception("Stats query failed for %s", table)
|
||||
stats[table] = {"total": 0, "indexed": 0, "pending": 0}
|
||||
|
||||
# Last index log entries
|
||||
try:
|
||||
log_result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT entity_type, action, status, created_at
|
||||
FROM unified_search_index_log
|
||||
WHERE tenant_id = :tid
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 10
|
||||
"""
|
||||
),
|
||||
{"tid": tenant_id},
|
||||
)
|
||||
recent_logs = [dict(r) for r in log_result.mappings().all()]
|
||||
except Exception:
|
||||
recent_logs = []
|
||||
|
||||
stats["recent_logs"] = recent_logs
|
||||
return stats
|
||||
Reference in New Issue
Block a user