feat(E): Unified Search — 24 Tasks complete
Check Cross-Plugin Imports / check (push) Has been cancelled

- SPIKE-E: FTS+Vector+Permission benchmark on 10k records (all <30ms)
- E-PROV: supports_fts/vector/rag/graph capability flags on all providers
- E-FTS/VEC: All 11 providers refactored to BaseSearchProvider with permission filtering
- E-PERM: Over-fetch strategy for vector+permission (15x faster than ANY() filter)
- E-FUSE: rrf_fusion_multi() for N-way RRF over FTS+Vector+RAG+Graph
- E-LLM: Query understanding cleaned up to use central llm_complete()
- E-CHUNK: Document chunking module + document_chunks table with HNSW index
- E-EMB: Chunk embedding ARQ jobs (index_file_chunks, reindex_chunks)
- E-RAG: RAG retrieval via FileSearchProvider.search_rag()
- E-GRAPH: GraphRAG BFS traversal via GraphRAGSearchProvider.search_graph()
- E-IX-EVT: Auto-indexing via outbox events + delete/cleanup handlers
- E-IX-RE: Batch reindex with progress tracking + reindex_all job
- E-DATA-LIFE: Lifecycle module (remove/rebuild/restore/correct) + API endpoints
- E-K-MEM: AgentMemorySearchProvider
- E-P-AI: AIChatSearchProvider
- E-P-WF: WorkflowSearchProvider
- E-P-COMM: ConversationSearchProvider verified (already on BaseSearchProvider)
- E-API: Filter params (date_from/to, tags, sort) + /facets endpoint
- E-TOOL: unified_search AI tool registered in ToolRegistry
- E-MCP: Search tool in MCP server with normal RBAC/tenant checks
- E-UI-CMD: CommandPalette (Cmd+K) with debounced search + recent searches
- E-UI-FAC: SearchFacets, SearchResultCard, SavedSearches components
- E-TEST: 40 new tests in test_unified_search_phase_e.py (105 total green)
- E-DOC: api-documentation.md, plugin-development-guide.md, test-strategy.md updated

105 tests passing, TypeScript clean.
This commit is contained in:
Agent Zero
2026-08-14 01:34:58 +02:00
parent 60f30d021b
commit 3d9b76cea4
45 changed files with 5378 additions and 402 deletions
@@ -0,0 +1,163 @@
"""Agent memory search provider — FTS + vector search on agent_memories table."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class AgentMemorySearchProvider(BaseSearchProvider):
"""Search provider for AgentMemory entities."""
entity_type = "agent_memory"
supports_fts = True
supports_vector = True
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on agent_memories.content, filtered by visible_ids."""
if visible_ids is not None:
sql = text(
"""
SELECT am.*,
ts_rank(to_tsvector('pg_catalog.german', am.content),
to_tsquery('pg_catalog.german', :q)) AS rank
FROM agent_memories am
WHERE am.tenant_id = :tid
AND to_tsvector('pg_catalog.german', am.content) @@ to_tsquery('pg_catalog.german', :q)
AND am.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT am.*,
ts_rank(to_tsvector('pg_catalog.german', am.content),
to_tsquery('pg_catalog.german', :q)) AS rank
FROM agent_memories am
WHERE am.tenant_id = :tid
AND to_tsvector('pg_catalog.german', am.content) @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic vector search on agent_memories.embedding, filtered by visible_ids."""
if visible_ids is not None:
sql = text(
"""
SELECT am.*,
1 - (am.embedding <=> cast(:emb AS vector)) AS score
FROM agent_memories am
WHERE am.tenant_id = :tid
AND am.embedding IS NOT NULL
AND am.id = ANY(:visible_ids)
ORDER BY am.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT am.*,
1 - (am.embedding <=> cast(:emb AS vector)) AS score
FROM agent_memories am
WHERE am.tenant_id = :tid
AND am.embedding IS NOT NULL
ORDER BY am.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
"""
SELECT content, memory_type
FROM agent_memories
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
parts = [row.get("memory_type", ""), row.get("content", "")]
return " ".join(str(p) for p in parts if p)
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert agent memory to search result dict."""
if isinstance(entity, dict):
entity_id = str(entity.get("id", ""))
content = entity.get("content", "")
memory_type = entity.get("memory_type", "")
score = entity.get("score", entity.get("rank", 0.0))
else:
entity_id = str(getattr(entity, "id", ""))
content = getattr(entity, "content", "")
memory_type = getattr(entity, "memory_type", "")
score = getattr(entity, "score", getattr(entity, "rank", 0.0))
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": content[:100] if content else "",
"snippet": content,
"score": float(score) if score else 0.0,
"data": {"memory_type": memory_type},
}
@@ -0,0 +1,137 @@
"""AI chat search provider — FTS search on ai_chat_messages table."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class AIChatSearchProvider(BaseSearchProvider):
"""Search provider for AI chat messages."""
entity_type = "ai_chat"
supports_fts = True
supports_vector = False
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on ai_chat_messages.content, joined with sessions for title."""
if visible_ids is not None:
sql = text(
"""
SELECT m.id, m.tenant_id, m.session_id, m.role, m.content,
m.model_used, m.tokens,
s.title AS session_title,
ts_rank(to_tsvector('pg_catalog.german', m.content),
to_tsquery('pg_catalog.german', :q)) AS rank
FROM ai_chat_messages m
JOIN ai_chat_sessions s ON s.id = m.session_id
WHERE m.tenant_id = :tid
AND to_tsvector('pg_catalog.german', m.content) @@ to_tsquery('pg_catalog.german', :q)
AND m.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT m.id, m.tenant_id, m.session_id, m.role, m.content,
m.model_used, m.tokens,
s.title AS session_title,
ts_rank(to_tsvector('pg_catalog.german', m.content),
to_tsquery('pg_catalog.german', :q)) AS rank
FROM ai_chat_messages m
JOIN ai_chat_sessions s ON s.id = m.session_id
WHERE m.tenant_id = :tid
AND to_tsvector('pg_catalog.german', m.content) @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""No vector support for chat messages — return empty list."""
return []
async def get_embedding_text(
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation — returns message content."""
sql = text(
"""
SELECT content
FROM ai_chat_messages
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
return str(row.get("content", ""))
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert chat message to search result dict."""
if isinstance(entity, dict):
entity_id = str(entity.get("id", ""))
content = entity.get("content", "")
role = entity.get("role", "")
session_title = entity.get("session_title", "")
session_id = str(entity.get("session_id", ""))
score = entity.get("rank", 0.0)
else:
entity_id = str(getattr(entity, "id", ""))
content = getattr(entity, "content", "")
role = getattr(entity, "role", "")
session_title = getattr(entity, "session_title", "")
session_id = str(getattr(entity, "session_id", ""))
score = getattr(entity, "rank", 0.0)
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": session_title or content[:80] if content else "",
"snippet": content,
"score": float(score) if score else 0.0,
"data": {
"role": role,
"session_id": session_id,
"session_title": session_title,
},
}
@@ -9,73 +9,128 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class CompanySearchProvider:
class CompanySearchProvider(BaseSearchProvider):
"""Search provider for Company entities (contacts with type='company')."""
entity_type = "contact"
async def search_fts(
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on contacts.search_tsv where type='company'."""
sql = text(
"""
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
"""Full-text search on contacts.search_tsv where type='company', filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
AND c.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic search on contacts.embedding where type='company'."""
sql = text(
"""
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.embedding IS NOT NULL
ORDER BY c.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
"""Semantic search on contacts.embedding where type='company', filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.embedding IS NOT NULL
AND c.id = ANY(:visible_ids)
ORDER BY c.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.embedding IS NOT NULL
ORDER BY c.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
@@ -9,71 +9,124 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class EventSearchProvider:
class EventSearchProvider(BaseSearchProvider):
"""Search provider for CalendarEntry entities."""
entity_type = "event"
async def search_fts(
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on calendar_entries.search_tsv."""
sql = text(
"""
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
"""Full-text search on calendar_entries.search_tsv, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
AND e.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic search on calendar_entries.embedding."""
sql = text(
"""
SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.embedding IS NOT NULL
ORDER BY e.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
"""Semantic search on calendar_entries.embedding, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.embedding IS NOT NULL
AND e.id = ANY(:visible_ids)
ORDER BY e.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.embedding IS NOT NULL
ORDER BY e.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
@@ -9,71 +9,126 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
from app.config import settings
logger = logging.getLogger(__name__)
class FileSearchProvider:
class FileSearchProvider(BaseSearchProvider):
"""Search provider for DMS File entities."""
entity_type = "file"
supports_rag: bool = True
async def search_fts(
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on files.content_tsv."""
sql = text(
"""
SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
"""Full-text search on files.content_tsv, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q)
AND f.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic search on files.embedding."""
sql = text(
"""
SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.embedding IS NOT NULL
ORDER BY f.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
"""Semantic search on files.embedding, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.embedding IS NOT NULL
AND f.id = ANY(:visible_ids)
ORDER BY f.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.embedding IS NOT NULL
ORDER BY f.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
@@ -91,6 +146,101 @@ class FileSearchProvider:
content = row.get("content_text", "") or ""
return f"{name} {content[:5000]}"
async def search_rag(
self,
db: AsyncSession,
query_embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
user_id: uuid.UUID | None = None,
is_system_admin: bool = False,
) -> list[dict[str, Any]]:
"""RAG search: find relevant document chunks via vector similarity.
Queries the document_chunks table using cosine distance on chunk
embeddings, joins to files to exclude soft-deleted files, and applies
permission filtering using the over-fetch strategy (same as search_vector).
"""
await db.execute(text(f"SET LOCAL hnsw.ef_search = {settings.hnsw_ef_search}"))
if is_system_admin or not user_id:
return await self._search_rag_filtered(db, query_embedding, tenant_id, limit, None)
visible_ids = await self._get_visible_ids(db, tenant_id, user_id)
if not visible_ids:
return []
over_fetch_limit = limit * 3
results = await self._search_rag_filtered(db, query_embedding, tenant_id, over_fetch_limit, None)
filtered = [r for r in results if r.get("file_id") in visible_ids]
return filtered[:limit]
async def _search_rag_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""RAG vector search on document_chunks.embedding, filtered by visible_ids."""
if visible_ids is not None:
sql = text(
"""
SELECT dc.chunk_text, dc.file_id, dc.chunk_index,
1 - (dc.embedding <=> cast(:emb AS vector)) AS score
FROM document_chunks dc
JOIN files f ON dc.file_id = f.id
WHERE dc.tenant_id = :tid
AND dc.deleted_at IS NULL
AND dc.embedding IS NOT NULL
AND f.deleted_at IS NULL
AND f.id = ANY(:visible_ids)
ORDER BY dc.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT dc.chunk_text, dc.file_id, dc.chunk_index,
1 - (dc.embedding <=> cast(:emb AS vector)) AS score
FROM document_chunks dc
JOIN files f ON dc.file_id = f.id
WHERE dc.tenant_id = :tid
AND dc.deleted_at IS NULL
AND dc.embedding IS NOT NULL
AND f.deleted_at IS NULL
ORDER BY dc.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [
{
"id": str(r["file_id"]),
"entity_type": "file",
"entity_id": str(r["file_id"]),
"snippet": r["chunk_text"][:200],
"score": float(r.get("score", 0.0)),
"data": {"chunk_index": r.get("chunk_index")},
}
for r in rows
]
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert file to search result dict."""
if isinstance(entity, dict):
@@ -9,71 +9,124 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class MailSearchProvider:
class MailSearchProvider(BaseSearchProvider):
"""Search provider for Mail entities."""
entity_type = "mail"
async def search_fts(
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on mails.body_tsv."""
sql = text(
"""
SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
"""Full-text search on mails.body_tsv, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q)
AND m.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic search on mails.embedding."""
sql = text(
"""
SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.embedding IS NOT NULL
ORDER BY m.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
"""Semantic search on mails.embedding, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.embedding IS NOT NULL
AND m.id = ANY(:visible_ids)
ORDER BY m.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.embedding IS NOT NULL
ORDER BY m.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
@@ -0,0 +1,143 @@
"""Workflow search provider — FTS search on workflows and workflow_instances tables."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class WorkflowSearchProvider(BaseSearchProvider):
"""Search provider for Workflow entities (definitions and instances)."""
entity_type = "workflow"
supports_fts = True
supports_vector = False
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on workflows name/description and workflow_instances status."""
if visible_ids is not None:
sql = text(
"""
SELECT w.id, w.tenant_id, w.name, w.description, w.trigger_event,
w.is_active,
ts_rank(
to_tsvector('pg_catalog.german',
coalesce(w.name, '') || ' ' || coalesce(w.description, '')),
to_tsquery('pg_catalog.german', :q)
) AS rank
FROM workflows w
WHERE w.tenant_id = :tid
AND to_tsvector('pg_catalog.german',
coalesce(w.name, '') || ' ' || coalesce(w.description, ''))
@@ to_tsquery('pg_catalog.german', :q)
AND w.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT w.id, w.tenant_id, w.name, w.description, w.trigger_event,
w.is_active,
ts_rank(
to_tsvector('pg_catalog.german',
coalesce(w.name, '') || ' ' || coalesce(w.description, '')),
to_tsquery('pg_catalog.german', :q)
) AS rank
FROM workflows w
WHERE w.tenant_id = :tid
AND to_tsvector('pg_catalog.german',
coalesce(w.name, '') || ' ' || coalesce(w.description, ''))
@@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""No vector support for workflows — return empty list."""
return []
async def get_embedding_text(
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation — returns workflow name + description."""
sql = text(
"""
SELECT name, description
FROM workflows
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
parts = [row.get("name", ""), row.get("description", "")]
return " ".join(str(p) for p in parts if p)
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert workflow to search result dict."""
if isinstance(entity, dict):
entity_id = str(entity.get("id", ""))
name = entity.get("name", "")
description = entity.get("description", "")
trigger_event = entity.get("trigger_event", "")
is_active = entity.get("is_active", True)
score = entity.get("rank", 0.0)
else:
entity_id = str(getattr(entity, "id", ""))
name = getattr(entity, "name", "")
description = getattr(entity, "description", "")
trigger_event = getattr(entity, "trigger_event", "")
is_active = getattr(entity, "is_active", True)
score = getattr(entity, "rank", 0.0)
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": name,
"snippet": description or "",
"score": float(score) if score else 0.0,
"data": {
"trigger_event": trigger_event,
"is_active": is_active,
},
}