feat(E): Unified Search — 24 Tasks complete
Check Cross-Plugin Imports / check (push) Has been cancelled

- SPIKE-E: FTS+Vector+Permission benchmark on 10k records (all <30ms)
- E-PROV: supports_fts/vector/rag/graph capability flags on all providers
- E-FTS/VEC: All 11 providers refactored to BaseSearchProvider with permission filtering
- E-PERM: Over-fetch strategy for vector+permission (15x faster than ANY() filter)
- E-FUSE: rrf_fusion_multi() for N-way RRF over FTS+Vector+RAG+Graph
- E-LLM: Query understanding cleaned up to use central llm_complete()
- E-CHUNK: Document chunking module + document_chunks table with HNSW index
- E-EMB: Chunk embedding ARQ jobs (index_file_chunks, reindex_chunks)
- E-RAG: RAG retrieval via FileSearchProvider.search_rag()
- E-GRAPH: GraphRAG BFS traversal via GraphRAGSearchProvider.search_graph()
- E-IX-EVT: Auto-indexing via outbox events + delete/cleanup handlers
- E-IX-RE: Batch reindex with progress tracking + reindex_all job
- E-DATA-LIFE: Lifecycle module (remove/rebuild/restore/correct) + API endpoints
- E-K-MEM: AgentMemorySearchProvider
- E-P-AI: AIChatSearchProvider
- E-P-WF: WorkflowSearchProvider
- E-P-COMM: ConversationSearchProvider verified (already on BaseSearchProvider)
- E-API: Filter params (date_from/to, tags, sort) + /facets endpoint
- E-TOOL: unified_search AI tool registered in ToolRegistry
- E-MCP: Search tool in MCP server with normal RBAC/tenant checks
- E-UI-CMD: CommandPalette (Cmd+K) with debounced search + recent searches
- E-UI-FAC: SearchFacets, SearchResultCard, SavedSearches components
- E-TEST: 40 new tests in test_unified_search_phase_e.py (105 total green)
- E-DOC: api-documentation.md, plugin-development-guide.md, test-strategy.md updated

105 tests passing, TypeScript clean.
This commit is contained in:
Agent Zero
2026-08-14 01:34:58 +02:00
parent 60f30d021b
commit 3d9b76cea4
45 changed files with 5378 additions and 402 deletions
@@ -2,7 +2,6 @@
from __future__ import annotations
import os
import json
import logging
import uuid
@@ -13,8 +12,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
DEFAULT_LLM_MODEL = os.environ.get('SEARCH_LLM_MODEL', 'ollama/deepseek-v4-flash')
QUERY_ANALYZE_SYSTEM = (
"Du bist ein Query-Analyzer fuer ein CRM. "
"Analysiere die Suchanfrage und gib JSON zurueck: "
@@ -27,33 +24,9 @@ RESULT_AGGREGATE_SYSTEM = (
'{"summary": str, "facets": {"types": {}, "dates": {}, "people": []}, "suggestions": [str]}'
)
async def _get_api_credentials(
db: AsyncSession | None, tenant_id: uuid.UUID | None
) -> tuple[str | None, str | None, str | None]:
"""Get API key, base_url and provider_type from the default AI provider in DB.
Falls back to API_KEY_OLLAMA_CLOUD env var.
Returns (api_key, base_url, provider_type).
"""
if db and tenant_id:
try:
from app.plugins.builtins.ai_assistant.contracts import get_default_provider
provider = await get_default_provider(db, tenant_id)
if provider and provider.api_key:
return provider.api_key, provider.base_url, provider.provider_type
except Exception:
logger.debug("Failed to get provider from DB, falling back to env")
env_key = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
return (env_key if env_key else None), None, None
def _build_model(model: str, provider_type: str | None) -> str:
"""Build litellm model string with provider prefix."""
if provider_type:
model_parts = model.split("/", 1)
return f"{provider_type}/{model_parts[-1]}"
return model
# Sensible default model used when no provider-specific model is configured.
# llm_complete() resolves credentials/model prefix from the central config.
DEFAULT_LLM_MODEL = "ollama/deepseek-v4-flash"
def _fallback_query_analysis(query: str) -> dict[str, Any]:
@@ -74,6 +47,16 @@ def _fallback_aggregate(results: list[dict], query: str) -> dict[str, Any]:
}
def _parse_json_content(content: str) -> dict[str, Any]:
"""Parse LLM JSON response, stripping markdown code fences if present."""
content = content.strip()
if content.startswith("```"):
content = content.split("\n", 1)[-1] if "\n" in content else content[3:]
if content.endswith("```"):
content = content[:-3].strip()
return json.loads(content)
async def llm_analyze_query(
query: str,
db: AsyncSession | None = None,
@@ -84,11 +67,8 @@ async def llm_analyze_query(
Falls back to a simple dict if LLM fails.
"""
try:
api_key, api_base, provider_type = await _get_api_credentials(db, tenant_id)
model = _build_model(DEFAULT_LLM_MODEL, provider_type)
result = await llm_complete(
model=model,
model=DEFAULT_LLM_MODEL,
messages=[
{"role": "system", "content": QUERY_ANALYZE_SYSTEM},
{"role": "user", "content": query},
@@ -96,17 +76,10 @@ async def llm_analyze_query(
temperature=0.1,
max_tokens=500,
response_format={"type": "json_object"},
api_key=api_key,
api_base=api_base,
db=db,
tenant_id=tenant_id,
)
content = result["content"]
# Strip markdown code fences if present
content = content.strip()
if content.startswith("```"):
content = content.split("\n", 1)[-1] if "\n" in content else content[3:]
if content.endswith("```"):
content = content[:-3].strip()
return json.loads(content)
return _parse_json_content(result["content"])
except Exception:
logger.warning("LLM query analysis failed, using fallback", exc_info=True)
return _fallback_query_analysis(query)
@@ -125,9 +98,6 @@ async def llm_aggregate_results(
if not results:
return _fallback_aggregate(results, query)
try:
api_key, api_base, provider_type = await _get_api_credentials(db, tenant_id)
model = _build_model(DEFAULT_LLM_MODEL, provider_type)
# Truncate results to avoid token overflow
compact = [
{"entity_type": r.get("entity_type"), "title": r.get("title", "")[:100]}
@@ -136,7 +106,7 @@ async def llm_aggregate_results(
user_msg = json.dumps({"query": query, "results": compact})
result = await llm_complete(
model=model,
model=DEFAULT_LLM_MODEL,
messages=[
{"role": "system", "content": RESULT_AGGREGATE_SYSTEM},
{"role": "user", "content": user_msg},
@@ -144,17 +114,10 @@ async def llm_aggregate_results(
temperature=0.1,
max_tokens=1000,
response_format={"type": "json_object"},
api_key=api_key,
api_base=api_base,
db=db,
tenant_id=tenant_id,
)
content = result["content"]
# Strip markdown code fences if present
content = content.strip()
if content.startswith("```"):
content = content.split("\n", 1)[-1] if "\n" in content else content[3:]
if content.endswith("```"):
content = content[:-3].strip()
return json.loads(content)
return _parse_json_content(result["content"])
except Exception:
logger.warning("LLM result aggregation failed, using fallback", exc_info=True)
return _fallback_aggregate(results, query)