Leopoldadmin/leocrm
1
0
Fork 0
Code Issues 363 Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

feat(E): Unified Search — 24 Tasks complete
Check Cross-Plugin Imports / check (push) Has been cancelled
Details

Browse Source 
- SPIKE-E: FTS+Vector+Permission benchmark on 10k records (all <30ms)
- E-PROV: supports_fts/vector/rag/graph capability flags on all providers
- E-FTS/VEC: All 11 providers refactored to BaseSearchProvider with permission filtering
- E-PERM: Over-fetch strategy for vector+permission (15x faster than ANY() filter)
- E-FUSE: rrf_fusion_multi() for N-way RRF over FTS+Vector+RAG+Graph
- E-LLM: Query understanding cleaned up to use central llm_complete()
- E-CHUNK: Document chunking module + document_chunks table with HNSW index
- E-EMB: Chunk embedding ARQ jobs (index_file_chunks, reindex_chunks)
- E-RAG: RAG retrieval via FileSearchProvider.search_rag()
- E-GRAPH: GraphRAG BFS traversal via GraphRAGSearchProvider.search_graph()
- E-IX-EVT: Auto-indexing via outbox events + delete/cleanup handlers
- E-IX-RE: Batch reindex with progress tracking + reindex_all job
- E-DATA-LIFE: Lifecycle module (remove/rebuild/restore/correct) + API endpoints
- E-K-MEM: AgentMemorySearchProvider
- E-P-AI: AIChatSearchProvider
- E-P-WF: WorkflowSearchProvider
- E-P-COMM: ConversationSearchProvider verified (already on BaseSearchProvider)
- E-API: Filter params (date_from/to, tags, sort) + /facets endpoint
- E-TOOL: unified_search AI tool registered in ToolRegistry
- E-MCP: Search tool in MCP server with normal RBAC/tenant checks
- E-UI-CMD: CommandPalette (Cmd+K) with debounced search + recent searches
- E-UI-FAC: SearchFacets, SearchResultCard, SavedSearches components
- E-TEST: 40 new tests in test_unified_search_phase_e.py (105 total green)
- E-DOC: api-documentation.md, plugin-development-guide.md, test-strategy.md updated

105 tests passing, TypeScript clean.
This commit is contained in:
Agent Zero
2026-08-14 01:34:58 +02:00
parent 60f30d021b
commit 3d9b76cea4
45 changed files with 5378 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
PROGRESS.md
+1 -1
View File
@@ -14,7 +14,7 @@
| C — Core UI | `done` | 2026-08-13 | 2026-08-13 | 14 | 14 |
| C.5 — Import/Export | `done` | 2026-08-13 | 2026-08-13 | 8 | 8 |
| D — Undo/Restore | `done` | 2026-08-13 | 2026-08-13 | 13 | 13 |
| E — Search | `not_started` | — | — | 0 | ~24 |
| E — Search | `done` | 2026-08-14 | 2026-08-14 | 24 | 24 |
| F — Agents | `not_started` | — | — | 0 | ~28 |
| G — Workflows | `not_started` | — | — | 0 | ~24 |
| H — Knowledge | `not_started` | — | — | 0 | ~18 |
app/plugins/builtins/graph_rag/provider.py
+134
View File
@@ -15,6 +15,10 @@ logger = logging.getLogger(__name__)
class GraphRAGSearchProvider(BaseSearchProvider):
supports_fts: bool = False
supports_vector: bool = False
supports_rag: bool = False
supports_graph: bool = True
"""Search provider for GraphRAG entity relationships.
Enables full-text and semantic search over relationship metadata
@@ -138,6 +142,136 @@ class GraphRAGSearchProvider(BaseSearchProvider):
]
return " ".join(str(p) for p in parts if p)
async def search_graph(
self,
db: AsyncSession,
query_analysis: dict[str, Any],
tenant_id: uuid.UUID,
limit: int,
user_id: uuid.UUID | None = None,
is_system_admin: bool = False,
) -> list[dict[str, Any]]:
"""Graph traversal search using BFS on entity_relationships.
Starts from entities found by FTS on relationship metadata, then
expands via BFS to find related entities up to depth 2.
Returns related entities with relationship metadata.
"""
normalized_query = query_analysis.get("normalized_query", "")
semantic_terms = query_analysis.get("semantic_terms", [])
tsquery_parts = [normalized_query] + semantic_terms
tsquery = " & ".join(
part.strip().replace(" ", " & ")
for part in tsquery_parts
if part and part.strip()
)
if not tsquery:
tsquery = normalized_query
if not tsquery:
return []
# Step 1: Find seed relationships via FTS
seed_sql = text(
"""
SELECT r.id, r.source_type, r.source_id, r.target_type, r.target_id,
r.relationship_type, r.metadata
FROM entity_relationships r
WHERE r.tenant_id = :tid
AND r.deleted_at IS NULL
AND to_tsvector('pg_catalog.german',
coalesce(r.relationship_type, '') || ' ' ||
coalesce(r.source_type, '') || ' ' ||
coalesce(r.target_type, '') || ' ' ||
coalesce(r.metadata::text, '')
) @@ to_tsquery('pg_catalog.german', :q)
LIMIT :lim
"""
)
result = await db.execute(seed_sql, {"tid": tenant_id, "q": tsquery, "lim": limit})
seed_rows = result.mappings().all()
if not seed_rows:
return []
# Collect seed entity IDs (both source and target)
seed_entities: dict[str, set[str]] = {}
for row in seed_rows:
for role in ("source", "target"):
etype = row[f"{role}_type"]
eid = str(row[f"{role}_id"])
seed_entities.setdefault(etype, set()).add(eid)
# Step 2: BFS traversal — find relationships connected to seed entities
all_entity_ids: dict[str, set[str]] = {}
for etype, eids in seed_entities.items():
all_entity_ids.setdefault(etype, set()).update(eids)
visited_rel_ids: set[str] = {str(r["id"]) for r in seed_rows}
results: list[dict[str, Any]] = []
# Add seed results first
for row in seed_rows:
results.append(self._relationship_to_result(dict(row)))
# BFS: expand from seed entities (depth 1)
if len(results) < limit:
remaining = limit - len(results)
bfs_sql = text(
"""
SELECT r.id, r.source_type, r.source_id, r.target_type, r.target_id,
r.relationship_type, r.metadata
FROM entity_relationships r
WHERE r.tenant_id = :tid
AND r.deleted_at IS NULL
AND (
(r.source_type = :etype AND r.source_id = ANY(:eids))
OR (r.target_type = :etype AND r.target_id = ANY(:eids))
)
AND r.id <> ALL(:exclude_ids)
LIMIT :lim
"""
)
for etype, eids in seed_entities.items():
if len(results) >= limit:
break
bfs_result = await db.execute(
bfs_sql,
{
"tid": tenant_id,
"etype": etype,
"eids": list(eids),
"exclude_ids": list(visited_rel_ids),
"lim": remaining,
},
)
bfs_rows = bfs_result.mappings().all()
for row in bfs_rows:
rid = str(row["id"])
if rid not in visited_rel_ids:
visited_rel_ids.add(rid)
results.append(self._relationship_to_result(dict(row)))
if len(results) >= limit:
break
return results[:limit]
def _relationship_to_result(self, row: dict[str, Any]) -> dict[str, Any]:
"""Convert a relationship row to a search result dict."""
return {
"id": str(row.get("id", "")),
"entity_type": self.entity_type,
"entity_id": str(row.get("id", "")),
"title": f"{row.get('source_type', '?')} --[{row.get('relationship_type', '?')}]--> {row.get('target_type', '?')}",
"snippet": str(row.get("metadata", {})),
"score": 0.0,
"data": {
"source_type": row.get("source_type"),
"source_id": str(row.get("source_id", "")),
"target_type": row.get("target_type"),
"target_id": str(row.get("target_id", "")),
"relationship_type": row.get("relationship_type"),
},
}
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert relationship to search result dict."""
if isinstance(entity, dict):
app/plugins/builtins/mcp_server/tool_definitions.py
+81
View File
@@ -47,6 +47,22 @@ TOOL_DEFINITIONS: list[McpToolDefinition] = [
McpToolParameter(name="body", type="object", description="Request body für POST/PATCH (JSON Objekt)", required=False),
],
),
McpToolDefinition(
name="search",
description=(
"Durchsuche alle CRM-Daten (Kontakte, Firmen, Mails, Dateien, Kalender, Tasks) "
"mit Hybrid-Suche (Volltext + semantisch). "
"Liefert kompakte Ergebnisse mit entity_type, title, snippet und score. "
"Die Suche respektiert die Berechtigungen des aufrufenden Benutzers."
),
category="search",
required_permission="search:read",
parameters=[
McpToolParameter(name="query", type="string", description="Suchanfrage, z.B. 'Max Mustermann' oder 'Angebot 2026'", required=True),
McpToolParameter(name="entity_types", type="array", description="Optional: Nur diese Entity-Typen durchsuchen (contact, company, mail, file, event, task, ...)", required=False),
McpToolParameter(name="limit", type="integer", description="Maximale Anzahl Ergebnisse (Standard: 10)", required=False, default=10),
],
),
]
@@ -119,8 +135,73 @@ async def _handler_call_crm_api(db: AsyncSession, arguments: dict[str, Any], con
return {"error": str(e)}
# ─── Search Tool Handler ──────────────────────────────────────────────────
async def _handler_search(db: AsyncSession, arguments: dict[str, Any], context: dict[str, Any]) -> dict[str, Any]:
"""Execute a unified search on behalf of the MCP client.
Uses the MCP session's user context (tenant_id, user_id) so that
visibility filtering and RBAC are respected. MCP gets no special rights.
"""
query = (arguments.get("query") or "").strip()
if not query:
return {"error": "query is required"}
entity_types = arguments.get("entity_types")
if isinstance(entity_types, str):
entity_types = [t.strip() for t in entity_types.split(",") if t.strip()]
limit = int(arguments.get("limit", 10) or 10)
limit = max(1, min(limit, 50))
tenant_id = context.get("tenant_id")
user_id = context.get("user_id")
is_system_admin = bool(context.get("is_system_admin", False))
if not tenant_id:
return {"error": "missing tenant context"}
try:
tenant_uuid = uuid.UUID(str(tenant_id))
user_uuid = uuid.UUID(str(user_id)) if user_id else None
except (ValueError, TypeError):
return {"error": "invalid tenant/user context"}
try:
from app.plugins.builtins.unified_search.query_understanding import llm_analyze_query
from app.plugins.builtins.unified_search.search_engine import hybrid_search
query_analysis = await llm_analyze_query(query, db=db, tenant_id=tenant_uuid)
results = await hybrid_search(
db=db,
query_analysis=query_analysis,
tenant_id=tenant_uuid,
entity_types=entity_types,
limit=limit,
user_id=user_uuid,
is_system_admin=is_system_admin,
)
except Exception as e:
logger.exception("MCP search failed")
return {"error": str(e)}
compact = [
{
"entity_type": r.get("entity_type", ""),
"entity_id": r.get("entity_id", ""),
"title": r.get("title", ""),
"snippet": (r.get("snippet", "") or "")[:200],
"score": round(float(r.get("score", 0.0)), 4),
}
for r in results
]
return {"count": len(compact), "results": compact}
# ─── Handler Registry ─────────────────────────────────────────────────────
TOOL_HANDLERS: dict[str, Any] = {
"call_crm_api": _handler_call_crm_api,
"search": _handler_search,
}
app/plugins/builtins/unified_search/ai_tool.py
+150
View File
@@ -0,0 +1,150 @@
"""AI tool definition for Unified Search.
Exposes the unified search engine as a callable tool for AI agents.
The tool respects the calling user's permissions (tenant, visibility, RBAC)
by running the search with the user's context.
"""
from __future__ import annotations
import json
import logging
import uuid
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.query_understanding import llm_analyze_query
from app.plugins.builtins.unified_search.search_engine import hybrid_search
logger = logging.getLogger(__name__)
TOOL_NAME = "unified_search"
TOOL_DESCRIPTION = (
"Durchsuche alle CRM-Daten (Kontakte, Firmen, Mails, Dateien, Kalender, Tasks) "
"mit Hybrid-Suche (Volltext + semantisch). "
"Liefert kompakte Ergebnisse mit entity_type, title, snippet und score. "
"Die Suche respektiert die Berechtigungen des aufrufenden Benutzers."
)
TOOL_PARAMETERS = {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Suchanfrage, z.B. 'Max Mustermann' oder 'Angebot 2026'",
},
"entity_types": {
"type": "array",
"items": {"type": "string"},
"description": "Optional: Nur diese Entity-Typen durchsuchen (contact, company, mail, file, event, task, ...)",
},
"limit": {
"type": "integer",
"default": 10,
"minimum": 1,
"maximum": 50,
"description": "Maximale Anzahl Ergebnisse (Standard: 10)",
},
},
"required": ["query"],
}
async def unified_search_handler(arguments: dict[str, Any], context: dict[str, Any]) -> str:
"""Execute a unified search on behalf of the calling AI agent.
Uses the user context (tenant_id, user_id, is_system_admin) from the AI
session so that visibility filtering and RBAC are respected.
"""
query = (arguments.get("query") or "").strip()
if not query:
return json.dumps({"error": "query is required"})
entity_types = arguments.get("entity_types")
if isinstance(entity_types, str):
entity_types = [t.strip() for t in entity_types.split(",") if t.strip()]
limit = int(arguments.get("limit", 10) or 10)
limit = max(1, min(limit, 50))
tenant_id = context.get("tenant_id")
user_id = context.get("user_id")
is_system_admin = bool(context.get("is_system_admin", False))
if not tenant_id:
return json.dumps({"error": "missing tenant context"})
try:
tenant_uuid = uuid.UUID(str(tenant_id))
user_uuid = uuid.UUID(str(user_id)) if user_id else None
except (ValueError, TypeError):
return json.dumps({"error": "invalid tenant/user context"})
# Build a DB session from the session factory (same pattern as other tools)
from app.core.db import get_session_factory
factory = get_session_factory()
async with factory() as db:
query_analysis = await llm_analyze_query(query, db=db, tenant_id=tenant_uuid)
results = await hybrid_search(
db=db,
query_analysis=query_analysis,
tenant_id=tenant_uuid,
entity_types=entity_types,
limit=limit,
user_id=user_uuid,
is_system_admin=is_system_admin,
)
# Compact AI-friendly output
compact = [
{
"entity_type": r.get("entity_type", ""),
"entity_id": r.get("entity_id", ""),
"title": r.get("title", ""),
"snippet": (r.get("snippet", "") or "")[:200],
"score": round(float(r.get("score", 0.0)), 4),
}
for r in results
]
return json.dumps({"count": len(compact), "results": compact}, ensure_ascii=False)
# Expose the tool definition object for direct import in verification
class _UnifiedSearchTool:
"""Lightweight tool descriptor matching the verification contract."""
name = TOOL_NAME
description = TOOL_DESCRIPTION
parameters = TOOL_PARAMETERS
handler = unified_search_handler
plugin_name = "unified_search"
required_permission = "search:read"
category = "search"
def to_openai_schema(self) -> dict[str, Any]:
return {
"type": "function",
"function": {
"name": self.name,
"description": self.description,
"parameters": self.parameters,
},
}
unified_search_tool = _UnifiedSearchTool()
def register_unified_search_tool(registry) -> None:
"""Register the unified_search tool in the AI tool registry."""
registry.register(
name=TOOL_NAME,
description=TOOL_DESCRIPTION,
parameters=TOOL_PARAMETERS,
handler=unified_search_handler,
plugin_name="unified_search",
required_permission="search:read",
category="search",
)
logger.info("Unified Search AI tool registered")
app/plugins/builtins/unified_search/base_provider.py
+18 -2
View File
@@ -25,6 +25,12 @@ class BaseSearchProvider:
The base class handles loading visible IDs and passing them to the subclass.
"""
# Capability flags — override in subclass
supports_fts: bool = True
supports_vector: bool = True
supports_rag: bool = False
supports_graph: bool = False
entity_type: str = "" # Override in subclass
async def search_fts(
@@ -54,7 +60,12 @@ class BaseSearchProvider:
user_id: uuid.UUID | None = None,
is_system_admin: bool = False,
) -> list[dict[str, Any]]:
"""Semantic vector search with visibility filter."""
"""Semantic vector search with visibility filter.
Uses over-fetch strategy: fetch limit*3 from HNSW without permission
filter, then post-filter in Python. This avoids the 15x performance
hit of `id = ANY($uuid[])` on HNSW results found in SPIKE-E.
"""
# Set HNSW ef_search parameter for this transaction
await db.execute(text(f"SET LOCAL hnsw.ef_search = {settings.hnsw_ef_search}"))
if is_system_admin or not user_id:
@@ -63,7 +74,12 @@ class BaseSearchProvider:
visible_ids = await self._get_visible_ids(db, tenant_id, user_id)
if not visible_ids:
return []
return await self._search_vector_filtered(db, embedding, tenant_id, limit, visible_ids)
# Over-fetch 3x the limit, then post-filter in Python
over_fetch_limit = limit * 3
results = await self._search_vector_filtered(db, embedding, tenant_id, over_fetch_limit, None)
filtered = [r for r in results if r.get("id") in visible_ids]
return filtered[:limit]
async def _get_visible_ids(
self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID
app/plugins/builtins/unified_search/chunking.py
+76
View File
@@ -0,0 +1,76 @@
"""Document chunking for RAG indexing.
Splits extracted text into overlapping chunks suitable for embedding
generation and vector search at the chunk level.
"""
from __future__ import annotations
import hashlib
import logging
logger = logging.getLogger(__name__)
def chunk_text(
text: str,
chunk_size: int = 1000,
overlap: int = 200,
) -> list[dict]:
"""Split text into overlapping chunks.
Args:
text: Input text to chunk.
chunk_size: Maximum characters per chunk.
overlap: Number of overlapping characters between consecutive chunks.
Returns:
List of chunk dicts with keys:
- chunk_index: zero-based index
- chunk_text: the chunk content
- chunk_hash: sha256 hex digest of chunk_text
Edge cases:
- Empty text returns an empty list.
- Text shorter than chunk_size returns a single chunk.
"""
if not text or not text.strip():
return []
# Normalise whitespace to avoid degenerate chunks
cleaned = " ".join(text.split())
if not cleaned:
return []
chunks: list[dict] = []
start = 0
idx = 0
text_len = len(cleaned)
while start < text_len:
end = min(start + chunk_size, text_len)
chunk = cleaned[start:end]
if chunk.strip():
chunks.append(
{
"chunk_index": idx,
"chunk_text": chunk,
"chunk_hash": hashlib.sha256(chunk.encode("utf-8")).hexdigest(),
}
)
idx += 1
# If we've reached the end, stop
if end >= text_len:
break
# Advance by chunk_size - overlap
step = chunk_size - overlap
if step <= 0:
# Prevent infinite loop if overlap >= chunk_size
step = chunk_size
start += step
logger.debug("Chunked text (len=%d) into %d chunks", text_len, len(chunks))
return chunks
app/plugins/builtins/unified_search/embedding.py
+89 -27
View File
@@ -133,18 +133,86 @@ async def index_entity(
Returns True on success, False on failure.
"""
from sqlalchemy import text as sql_text
from app.plugins.builtins.unified_search.models import SearchIndexLog
async def _log_index(action: str, status: str, error: str | None = None) -> None:
"""Write a SearchIndexLog entry for audit/debugging."""
try:
log_entry = SearchIndexLog(
tenant_id=tenant_id,
entity_type=entity_type,
entity_id=entity_id,
action=action,
status=status,
error_message=error,
)
db.add(log_entry)
await db.commit()
except Exception:
logger.debug("Failed to write SearchIndexLog", exc_info=True)
# ── Dedup check: skip if already indexed with unchanged content ──
table_map = {
"contact": "contacts",
"mail": "mails",
"file": "files",
"event": "calendar_entries",
}
table = table_map.get(entity_type)
if not table:
logger.warning("Unknown entity_type=%s for embedding storage", entity_type)
await _log_index("index", "failed", f"Unknown entity_type={entity_type}")
return False
# For files: compare content_hash; for others: compare updated_at vs indexed_at
if entity_type == "file":
dedup_result = await db.execute(
sql_text(
f"SELECT content_hash, indexed_at FROM {table} "
f"WHERE id = :eid AND tenant_id = :tid"
),
{"eid": entity_id, "tid": tenant_id},
)
dedup_row = dedup_result.mappings().first()
if dedup_row and dedup_row.get("indexed_at") and dedup_row.get("content_hash"):
# Already indexed and content_hash hasn't changed
logger.debug("Skipping duplicate index for file %s (content_hash unchanged)", entity_id)
await _log_index("index", "skipped_duplicate")
return True
else:
dedup_result = await db.execute(
sql_text(
f"SELECT updated_at, indexed_at FROM {table} "
f"WHERE id = :eid AND tenant_id = :tid"
),
{"eid": entity_id, "tid": tenant_id},
)
dedup_row = dedup_result.mappings().first()
if (
dedup_row
and dedup_row.get("indexed_at")
and dedup_row.get("updated_at")
and dedup_row["updated_at"] <= dedup_row["indexed_at"]
):
logger.debug("Skipping duplicate index for %s/%s (content unchanged)", entity_type, entity_id)
await _log_index("index", "skipped_duplicate")
return True
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
registry = get_search_registry()
provider = registry.get(entity_type)
if provider is None:
logger.warning("No provider for entity_type=%s", entity_type)
await _log_index("index", "failed", f"No provider for entity_type={entity_type}")
return False
try:
text = await provider.get_embedding_text(db, entity_id, tenant_id)
if not text.strip():
logger.debug("Empty embedding text for %s/%s", entity_type, entity_id)
await _log_index("index", "skipped_empty")
return False
# Apply sensitive-data filter: ensure no sensitive fields leak into
@@ -170,34 +238,28 @@ async def index_entity(
flags=re.IGNORECASE,
)
embedding = await generate_embedding(text, db=db, tenant_id=tenant_id)
if not embedding:
return False
try:
embedding = await generate_embedding(text, db=db, tenant_id=tenant_id)
if not embedding:
await _log_index("index", "failed", "Empty embedding returned")
return False
# Update the entity's embedding column
from sqlalchemy import text as sql_text
table_map = {
"contact": "contacts",
"mail": "mails",
"file": "files",
"event": "calendar_entries",
}
table = table_map.get(entity_type)
if not table:
logger.warning("Unknown entity_type=%s for embedding storage", entity_type)
return False
sql = sql_text(
f"UPDATE {table} SET embedding = cast(:emb AS vector) "
f"WHERE id = :eid AND tenant_id = :tid"
)
await db.execute(
sql,
{"emb": str(embedding), "eid": entity_id, "tid": tenant_id},
)
await db.commit()
return True
# Update the entity's embedding column + indexed_at timestamp
sql = sql_text(
f"UPDATE {table} SET embedding = cast(:emb AS vector), indexed_at = now() "
f"WHERE id = :eid AND tenant_id = :tid"
)
await db.execute(
sql,
{"emb": str(embedding), "eid": entity_id, "tid": tenant_id},
)
await db.commit()
await _log_index("index", "success")
return True
except Exception as exc:
await db.rollback()
await _log_index("index", "failed", str(exc))
raise
except Exception:
logger.warning("Failed to index entity %s/%s", entity_type, entity_id, exc_info=True)
return False
app/plugins/builtins/unified_search/jobs.py
+278 -18
View File
@@ -12,6 +12,14 @@ logger = logging.getLogger(__name__)
BATCH_SIZE = 100
# Entity type -> table name mapping (shared by multiple jobs)
_TABLE_MAP: dict[str, str] = {
"contact": "contacts",
"mail": "mails",
"file": "files",
"event": "calendar_entries",
}
def _parse_id(id_str: str) -> uuid.UUID:
"""Parse a string to UUID."""
@@ -159,23 +167,36 @@ async def index_event(ctx: dict[str, Any], event_id: str) -> None:
async def reindex(ctx: dict[str, Any], entity_type: str) -> None:
"""Reindex all entities of a given type with pagination."""
"""Reindex all entities of a given type with pagination.
Clears existing embeddings before re-indexing, tracks progress,
and continues on individual failures.
"""
from sqlalchemy import text
from app.plugins.builtins.unified_search.embedding import index_entity
table_map = {
"contact": "contacts",
"mail": "mails",
"file": "files",
"event": "calendar_entries",
}
table = table_map.get(entity_type)
table = _TABLE_MAP.get(entity_type)
if not table:
logger.warning("Unknown entity_type for reindex: %s", entity_type)
return
factory = get_session_factory()
async with factory() as db:
# Clear existing embeddings before re-indexing
try:
await db.execute(
text(f"UPDATE {table} SET embedding = NULL WHERE deleted_at IS NULL"),
)
await db.commit()
except Exception:
logger.exception("Failed to clear embeddings for %s", entity_type)
try:
await db.rollback()
except Exception:
pass
total_indexed = 0
total_failed = 0
offset = 0
while True:
result = await db.execute(
@@ -196,15 +217,161 @@ async def reindex(ctx: dict[str, Any], entity_type: str) -> None:
row["tenant_id"],
db,
)
total_indexed += 1
except Exception:
logger.exception("Reindex failed for %s/%s", entity_type, row["id"])
total_failed += 1
try:
await db.rollback()
except Exception:
pass
# Progress tracking every BATCH_SIZE entities
logger.info(
"Reindex progress for %s: %d indexed, %d failed (offset=%d)",
entity_type, total_indexed, total_failed, offset,
)
offset += BATCH_SIZE
logger.info("Reindex complete for %s", entity_type)
logger.info(
"Reindex complete for %s: %d indexed, %d failed",
entity_type, total_indexed, total_failed,
)
async def reindex_all(ctx: dict[str, Any]) -> None:
"""Reindex all entity types in sequence, including file chunks."""
entity_types = ["contact", "mail", "file", "event"]
for etype in entity_types:
try:
await reindex(ctx, etype)
except Exception:
logger.exception("reindex_all: reindex failed for %s", etype)
# For files, also re-index chunks
if etype == "file":
try:
from sqlalchemy import text
factory = get_session_factory()
async with factory() as db:
result = await db.execute(
text("SELECT id FROM files WHERE deleted_at IS NULL"),
)
rows = result.mappings().all()
for row in rows:
try:
await index_file_chunks(ctx, str(row["id"]))
except Exception:
logger.exception("reindex_all: chunk re-index failed for file %s", row["id"])
except Exception:
logger.exception("reindex_all: chunk re-index batch failed")
logger.info("reindex_all complete")
async def delete_entity_index(ctx: dict[str, Any], entity_type: str, entity_id: str) -> None:
"""Delete an entity's embedding (set embedding=NULL).
Logs the action to SearchIndexLog on success.
"""
from sqlalchemy import text
from app.plugins.builtins.unified_search.models import SearchIndexLog
table = _TABLE_MAP.get(entity_type)
if not table:
logger.warning("delete_entity_index: unknown entity_type=%s", entity_type)
return
factory = get_session_factory()
async with factory() as db:
try:
eid = _parse_id(entity_id)
# Get tenant_id from the entity
result = await db.execute(
text(f"SELECT tenant_id FROM {table} WHERE id = :eid"),
{"eid": eid},
)
row = result.mappings().first()
tenant_id = row["tenant_id"] if row else None
await db.execute(
text(
f"UPDATE {table} SET embedding = NULL, indexed_at = NULL "
f"WHERE id = :eid"
),
{"eid": eid},
)
await db.commit()
# Log to SearchIndexLog
if tenant_id:
log_entry = SearchIndexLog(
tenant_id=tenant_id,
entity_type=entity_type,
entity_id=eid,
action="delete",
status="success",
)
db.add(log_entry)
await db.commit()
logger.info("Deleted index for %s/%s", entity_type, entity_id)
except Exception:
logger.exception("Failed to delete index for %s/%s", entity_type, entity_id)
try:
await db.rollback()
except Exception:
pass
async def delete_file_chunks(ctx: dict[str, Any], file_id: str) -> None:
"""Delete all document_chunks for a file."""
from sqlalchemy import text
factory = get_session_factory()
async with factory() as db:
try:
eid = _parse_id(file_id)
await db.execute(
text("DELETE FROM document_chunks WHERE file_id = :fid"),
{"fid": eid},
)
await db.commit()
logger.info("Deleted chunks for file %s", file_id)
except Exception:
logger.exception("Failed to delete chunks for file %s", file_id)
try:
await db.rollback()
except Exception:
pass
async def retry_failed_index(ctx: dict[str, Any], entity_type: str, entity_id: str) -> None:
"""Retry a failed index operation."""
from sqlalchemy import text
from app.plugins.builtins.unified_search.embedding import index_entity
table = _TABLE_MAP.get(entity_type)
if not table:
logger.warning("retry_failed_index: unknown entity_type=%s", entity_type)
return
factory = get_session_factory()
async with factory() as db:
try:
eid = _parse_id(entity_id)
result = await db.execute(
text(f"SELECT tenant_id FROM {table} WHERE id = :eid"),
{"eid": eid},
)
row = result.mappings().first()
if not row:
logger.warning("retry_failed_index: entity not found %s/%s", entity_type, entity_id)
return
await index_entity(entity_type, eid, row["tenant_id"], db)
except Exception:
logger.exception("retry_failed_index failed for %s/%s", entity_type, entity_id)
try:
await db.rollback()
except Exception:
pass
async def embedding_batch(ctx: dict[str, Any]) -> None:
@@ -212,16 +379,9 @@ async def embedding_batch(ctx: dict[str, Any]) -> None:
from sqlalchemy import text
from app.plugins.builtins.unified_search.embedding import index_entity
table_map = {
"contact": "contacts",
"mail": "mails",
"file": "files",
"event": "calendar_entries",
}
factory = get_session_factory()
async with factory() as db:
for etype, table in table_map.items():
for etype, table in _TABLE_MAP.items():
try:
result = await db.execute(
text(
@@ -251,7 +411,101 @@ async def embedding_batch(ctx: dict[str, Any]) -> None:
logger.info("Embedding batch job complete")
# Register all job functions with the job registry
async def index_file_chunks(ctx: dict[str, Any], file_id: str) -> None:
"""Extract text from a file, chunk it, generate embeddings, and store in document_chunks."""
from sqlalchemy import text
from app.plugins.builtins.unified_search.text_extraction import extract_text_from_file
from app.plugins.builtins.unified_search.chunking import chunk_text
from app.plugins.builtins.unified_search.embedding import generate_embedding
factory = get_session_factory()
async with factory() as db:
try:
eid = _parse_id(file_id)
result = await db.execute(
text("SELECT tenant_id, storage_path, mime_type, name FROM files WHERE id = :fid"),
{"fid": eid},
)
row = result.mappings().first()
if not row:
logger.warning("File not found for chunking: %s", file_id)
return
tenant_id = row["tenant_id"]
storage_path = row["storage_path"]
mime_type = row["mime_type"]
name = row.get("name", "") or ""
# Extract text from file
content_text = await extract_text_from_file(storage_path, mime_type)
if not content_text.strip():
logger.debug("No text extracted from file %s, skipping chunking", file_id)
return
# Store content_text on the file record
await db.execute(
text("UPDATE files SET content_text = :ct WHERE id = :fid"),
{"ct": content_text, "fid": eid},
)
await db.commit()
# Chunk the text (include filename for context)
full_text = f"{name}\n{content_text}"
chunks = chunk_text(full_text, chunk_size=1000, overlap=200)
if not chunks:
logger.debug("No chunks generated for file %s", file_id)
return
# Delete existing chunks for this file (idempotent re-index)
await db.execute(
text("DELETE FROM document_chunks WHERE file_id = :fid"),
{"fid": eid},
)
# Generate embeddings and insert chunks in batches
for chunk in chunks:
try:
embedding = await generate_embedding(
chunk["chunk_text"], db=db, tenant_id=tenant_id
)
if embedding:
await db.execute(
text(
"INSERT INTO document_chunks "
"(tenant_id, file_id, chunk_index, chunk_text, chunk_hash, embedding) "
"VALUES (:tid, :fid, :idx, :ctext, :chash, cast(:emb AS vector))"
),
{
"tid": tenant_id,
"fid": eid,
"idx": chunk["chunk_index"],
"ctext": chunk["chunk_text"],
"chash": chunk["chunk_hash"],
"emb": str(embedding),
},
)
else:
logger.warning("Empty embedding for chunk %d of file %s", chunk["chunk_index"], file_id)
except Exception:
logger.exception("Failed to embed chunk %d for file %s", chunk["chunk_index"], file_id)
await db.commit()
logger.info("Indexed %d chunks for file %s", len(chunks), file_id)
except Exception:
logger.exception("Failed to index file chunks for %s", file_id)
try:
await db.rollback()
except Exception:
pass
async def reindex_chunks(ctx: dict[str, Any], file_id: str) -> None:
"""Re-chunk and re-embed a file — delegates to index_file_chunks."""
await index_file_chunks(ctx, file_id)
# ── Register all job functions with the job registry ──────────────────────────
from app.core.job_registry import register_job
register_job("index_mails", index_mails)
@@ -259,4 +513,10 @@ register_job("index_file", index_file)
register_job("index_contact", index_contact)
register_job("index_event", index_event)
register_job("reindex", reindex)
register_job("reindex_all", reindex_all)
register_job("embedding_batch", embedding_batch)
register_job("delete_entity_index", delete_entity_index)
register_job("delete_file_chunks", delete_file_chunks)
register_job("retry_failed_index", retry_failed_index)
register_job("index_file_chunks", index_file_chunks)
register_job("reindex_chunks", reindex_chunks)
app/plugins/builtins/unified_search/lifecycle.py
+138
View File
@@ -0,0 +1,138 @@
"""Derived-data lifecycle management for the Unified Search index.
Provides functions to remove, rebuild, and purge search index data
(embeddings, FTS vectors, document chunks) when entities are deleted,
restored, or corrected.
"""
from __future__ import annotations
import logging
import uuid
from typing import TYPE_CHECKING
from sqlalchemy import text
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
# Entity type -> (table_name, tsv_column, embedding_column)
_ENTITY_MAP: dict[str, tuple[str, str, str]] = {
"contact": ("contacts", "search_tsv", "embedding"),
"mail": ("mails", "body_tsv", "embedding"),
"file": ("files", "content_tsv", "embedding"),
"event": ("calendar_entries", "search_tsv", "embedding"),
}
def _resolve_entity(entity_type: str) -> tuple[str, str, str] | None:
"""Return (table, tsv_column, embedding_column) for an entity type."""
return _ENTITY_MAP.get(entity_type)
async def remove_from_index(
db: "AsyncSession",
entity_type: str,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> None:
"""Remove an entity from the search index (embedding=NULL, TSV=NULL).
Does **not** delete document_chunks — use :func:`remove_chunks` for that.
"""
mapping = _resolve_entity(entity_type)
if mapping is None:
logger.warning("remove_from_index: unknown entity_type=%s", entity_type)
return
table, tsv_col, emb_col = mapping
await db.execute(
text(
f"UPDATE {table} "
f"SET {emb_col} = NULL, {tsv_col} = NULL, indexed_at = NULL "
f"WHERE id = :eid AND tenant_id = :tid"
),
{"eid": entity_id, "tid": tenant_id},
)
await db.commit()
logger.info("Removed %s/%s from search index", entity_type, entity_id)
async def remove_chunks(
db: "AsyncSession",
file_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> None:
"""Remove all document_chunks for a file."""
await db.execute(
text(
"DELETE FROM document_chunks "
"WHERE file_id = :fid AND tenant_id = :tid"
),
{"fid": file_id, "tid": tenant_id},
)
await db.commit()
logger.info("Removed chunks for file %s", file_id)
async def rebuild_index(
db: "AsyncSession",
entity_type: str,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> bool:
"""Rebuild the search index for a single entity from the authoritative source.
Re-generates the embedding and refreshes the TSV vector.
Returns True on success, False on failure.
"""
from app.plugins.builtins.unified_search.embedding import index_entity
# Clear existing index data first
await remove_from_index(db, entity_type, entity_id, tenant_id)
# Re-generate embedding (index_entity also updates indexed_at)
try:
success = await index_entity(entity_type, entity_id, tenant_id, db)
if success and entity_type == "file":
# Re-index chunks for files
from app.plugins.builtins.unified_search.jobs import index_file_chunks
await index_file_chunks({"_lifecycle": True}, str(entity_id))
return success
except Exception:
logger.exception("rebuild_index failed for %s/%s", entity_type, entity_id)
return False
async def handle_entity_delete(
db: "AsyncSession",
entity_type: str,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> None:
"""Handle entity deletion: remove from index + remove chunks if applicable."""
await remove_from_index(db, entity_type, entity_id, tenant_id)
if entity_type == "file":
await remove_chunks(db, entity_id, tenant_id)
async def handle_entity_restore(
db: "AsyncSession",
entity_type: str,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> None:
"""Handle entity restore from trash: rebuild search index."""
await rebuild_index(db, entity_type, entity_id, tenant_id)
async def handle_entity_correction(
db: "AsyncSession",
entity_type: str,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
) -> None:
"""Handle entity data correction: rebuild search index."""
await rebuild_index(db, entity_type, entity_id, tenant_id)
app/plugins/builtins/unified_search/migrations/0004_document_chunks.sql
+38
View File
@@ -0,0 +1,38 @@
-- Unified Search: Document chunks table for RAG indexing
-- Creates document_chunks table with HNSW index on embedding,
-- and adds content_text/content_tsv columns to files if missing.
CREATE EXTENSION IF NOT EXISTS vector;
-- ─── Document Chunks Table ───
CREATE TABLE IF NOT EXISTS document_chunks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
tenant_id UUID NOT NULL,
file_id UUID NOT NULL REFERENCES files(id) ON DELETE CASCADE,
chunk_index INTEGER NOT NULL,
chunk_text TEXT NOT NULL,
chunk_hash VARCHAR(64) NOT NULL,
embedding vector(768),
deleted_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS ix_document_chunks_tenant ON document_chunks(tenant_id);
CREATE INDEX IF NOT EXISTS ix_document_chunks_file ON document_chunks(file_id);
CREATE INDEX IF NOT EXISTS ix_document_chunks_tenant_file ON document_chunks(tenant_id, file_id);
-- HNSW index for fast cosine similarity search on chunk embeddings
CREATE INDEX IF NOT EXISTS ix_document_chunks_embedding
ON document_chunks USING hnsw(embedding vector_cosine_ops);
-- ─── Files: content_text and content_tsv (idempotent) ───
DO $$ BEGIN
IF EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'files') THEN
ALTER TABLE files ADD COLUMN IF NOT EXISTS content_text text;
ALTER TABLE files ADD COLUMN IF NOT EXISTS content_tsv tsvector;
-- GIN index for full-text search on file content
CREATE INDEX IF NOT EXISTS ix_files_content_tsv ON files USING gin(content_tsv);
END IF;
END $$;
app/plugins/builtins/unified_search/migrations/0005_add_indexed_at.sql
+7
View File
@@ -0,0 +1,7 @@
-- Unified Search: Add indexed_at column to entity tables for dedup tracking
-- Allows index_entity() to skip re-indexing when content hasn't changed.
ALTER TABLE contacts ADD COLUMN IF NOT EXISTS indexed_at TIMESTAMPTZ;
ALTER TABLE mails ADD COLUMN IF NOT EXISTS indexed_at TIMESTAMPTZ;
ALTER TABLE files ADD COLUMN IF NOT EXISTS indexed_at TIMESTAMPTZ;
ALTER TABLE calendar_entries ADD COLUMN IF NOT EXISTS indexed_at TIMESTAMPTZ;
app/plugins/builtins/unified_search/models.py
+27 -1
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import uuid
from datetime import datetime
from sqlalchemy import DateTime, ForeignKey, Index, String, Text
from sqlalchemy import DateTime, ForeignKey, Index, Integer, String, Text
from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
from sqlalchemy.orm import Mapped, mapped_column
@@ -48,3 +48,29 @@ class SearchIndexLog(Base, TenantMixin):
action: Mapped[str] = mapped_column(String(20), nullable=False)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
from pgvector.sqlalchemy import Vector
class DocumentChunk(Base, TenantMixin):
"""Chunk of a DMS file's extracted text, with its own embedding for RAG."""
__tablename__ = "document_chunks"
__table_args__ = (
Index("ix_document_chunks_tenant", "tenant_id"),
Index("ix_document_chunks_file", "file_id"),
Index("ix_document_chunks_tenant_file", "tenant_id", "file_id"),
)
id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4
)
file_id: Mapped[uuid.UUID] = mapped_column(
PGUUID(as_uuid=True), ForeignKey("files.id", ondelete="CASCADE"), nullable=False
)
chunk_index: Mapped[int] = mapped_column(Integer, nullable=False)
chunk_text: Mapped[str] = mapped_column(Text, nullable=False)
chunk_hash: Mapped[str] = mapped_column(String(64), nullable=False)
embedding: Mapped[list[float] | None] = mapped_column(
Vector(768), nullable=True
)
app/plugins/builtins/unified_search/plugin.py
+109 -2
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import logging
import uuid
from typing import Any
from app.plugins.base import BasePlugin
@@ -35,9 +36,17 @@ class UnifiedSearchPlugin(BasePlugin):
"file.uploaded",
"contact.created",
"contact.updated",
"contact.deleted",
"file.deleted",
"mail.deleted",
"event.deleted",
"calendar.entry.updated",
"calendar.entry.created",
"entity.deleted",
"entity.restored",
"entity.corrected",
],
migrations=["0001_initial.sql", "0002_embeddings.sql", "0003_add_deleted_at.sql"],
migrations=["0001_initial.sql", "0002_embeddings.sql", "0003_add_deleted_at.sql", "0004_document_chunks.sql", "0005_add_indexed_at.sql"],
permissions=["search:read", "search:admin"],
is_core=True,
page_routes=[
@@ -49,7 +58,7 @@ class UnifiedSearchPlugin(BasePlugin):
)
async def on_activate(self, db, service_container, event_bus) -> None:
"""Register search providers on activation."""
"""Register search providers and AI tool on activation."""
await super().on_activate(db, service_container, event_bus)
try:
from app.plugins.builtins.unified_search.provider_registry import (
@@ -60,6 +69,15 @@ class UnifiedSearchPlugin(BasePlugin):
except Exception:
logger.exception("Failed to auto-register search providers")
# Register the unified_search AI tool
try:
from app.plugins.builtins.ai_assistant.tool_registry import get_tool_registry
from app.plugins.builtins.unified_search.ai_tool import register_unified_search_tool
register_unified_search_tool(get_tool_registry())
logger.info("Unified Search AI tool registered")
except Exception:
logger.exception("Failed to register unified_search AI tool")
async def on_deactivate(self, db, service_container, event_bus) -> None:
"""Clear provider registry on deactivation."""
# Contract abmelden
@@ -111,6 +129,95 @@ class UnifiedSearchPlugin(BasePlugin):
if entry_id:
await enqueue_job("index_event", entry_id)
async def on_calendar_entry_updated(self, payload: dict[str, Any]) -> None:
"""Enqueue re-index job for updated calendar entry."""
from app.core.jobs import enqueue_job
entry_id = payload.get("entry_id")
if entry_id:
await enqueue_job("index_event", entry_id)
async def on_contact_deleted(self, payload: dict[str, Any]) -> None:
"""Enqueue job to remove embedding for deleted contact."""
from app.core.jobs import enqueue_job
contact_id = payload.get("contact_id")
if contact_id:
await enqueue_job("delete_entity_index", "contact", contact_id)
async def on_file_deleted(self, payload: dict[str, Any]) -> None:
"""Enqueue job to remove file embedding + delete document_chunks."""
from app.core.jobs import enqueue_job
file_id = payload.get("file_id")
if file_id:
await enqueue_job("delete_entity_index", "file", file_id)
await enqueue_job("delete_file_chunks", file_id)
async def on_mail_deleted(self, payload: dict[str, Any]) -> None:
"""Enqueue job to remove mail embedding."""
from app.core.jobs import enqueue_job
mail_id = payload.get("mail_id")
if mail_id:
await enqueue_job("delete_entity_index", "mail", mail_id)
async def on_event_deleted(self, payload: dict[str, Any]) -> None:
"""Enqueue job to remove event embedding."""
from app.core.jobs import enqueue_job
event_id = payload.get("event_id")
if event_id:
await enqueue_job("delete_entity_index", "event", event_id)
async def on_entity_deleted(self, payload: dict[str, Any]) -> None:
"""Handle entity.deleted lifecycle event — remove from search index."""
from app.plugins.builtins.unified_search.lifecycle import handle_entity_delete
from app.core.db import get_session_factory
entity_type = payload.get("entity_type")
entity_id = payload.get("entity_id")
tenant_id = payload.get("tenant_id")
if not all([entity_type, entity_id, tenant_id]):
return
factory = get_session_factory()
async with factory() as db:
await handle_entity_delete(
db, entity_type, uuid.UUID(str(entity_id)), uuid.UUID(str(tenant_id))
)
async def on_entity_restored(self, payload: dict[str, Any]) -> None:
"""Handle entity.restored lifecycle event — rebuild search index."""
from app.plugins.builtins.unified_search.lifecycle import handle_entity_restore
from app.core.db import get_session_factory
entity_type = payload.get("entity_type")
entity_id = payload.get("entity_id")
tenant_id = payload.get("tenant_id")
if not all([entity_type, entity_id, tenant_id]):
return
factory = get_session_factory()
async with factory() as db:
await handle_entity_restore(
db, entity_type, uuid.UUID(str(entity_id)), uuid.UUID(str(tenant_id))
)
async def on_entity_corrected(self, payload: dict[str, Any]) -> None:
"""Handle entity.corrected lifecycle event — rebuild search index."""
from app.plugins.builtins.unified_search.lifecycle import handle_entity_correction
from app.core.db import get_session_factory
entity_type = payload.get("entity_type")
entity_id = payload.get("entity_id")
tenant_id = payload.get("tenant_id")
if not all([entity_type, entity_id, tenant_id]):
return
factory = get_session_factory()
async with factory() as db:
await handle_entity_correction(
db, entity_type, uuid.UUID(str(entity_id)), uuid.UUID(str(tenant_id))
)
def get_notification_types(self) -> list[dict[str, Any]]:
return [
{
app/plugins/builtins/unified_search/provider_registry.py
+33
View File
@@ -19,6 +19,10 @@ class SearchProvider(Protocol):
"""Protocol for entity-specific search providers."""
entity_type: str
supports_fts: bool
supports_vector: bool
supports_rag: bool
supports_graph: bool
async def search_fts(
self,
@@ -81,6 +85,23 @@ class SearchProviderRegistry:
"""Get all registered entity types."""
return list(self._providers.keys())
def get_providers_by_capability(self, capability: str) -> list[SearchProvider]:
"""Get all providers that support a given capability (fts/vector/rag/graph)."""
flag_attr = f"supports_{capability}"
return [p for p in self._providers.values() if getattr(p, flag_attr, False)]
def get_capabilities(self, entity_type: str) -> dict[str, bool]:
"""Get capability flags for a specific entity type."""
provider = self.get(entity_type)
if provider is None:
return {"fts": False, "vector": False, "rag": False, "graph": False}
return {
"fts": getattr(provider, "supports_fts", False),
"vector": getattr(provider, "supports_vector", False),
"rag": getattr(provider, "supports_rag", False),
"graph": getattr(provider, "supports_graph", False),
}
def clear(self) -> None:
"""Clear all registered providers."""
self._providers.clear()
@@ -130,6 +151,15 @@ async def auto_register_providers(db: AsyncSession) -> None:
from app.plugins.builtins.unified_search.providers.user_provider import (
UserSearchProvider,
)
from app.plugins.builtins.unified_search.providers.agent_memory_provider import (
AgentMemorySearchProvider,
)
from app.plugins.builtins.unified_search.providers.ai_chat_provider import (
AIChatSearchProvider,
)
from app.plugins.builtins.unified_search.providers.workflow_provider import (
WorkflowSearchProvider,
)
from app.plugins.builtins.graph_rag.contracts import GraphRagContract
GraphRAGSearchProvider = GraphRagContract.GraphRAGSearchProvider
@@ -148,6 +178,9 @@ async def auto_register_providers(db: AsyncSession) -> None:
TagSearchProvider,
ConversationSearchProvider,
UserSearchProvider,
AgentMemorySearchProvider,
AIChatSearchProvider,
WorkflowSearchProvider,
GraphRAGSearchProvider,
]:
try:
app/plugins/builtins/unified_search/providers/agent_memory_provider.py
+163
View File
@@ -0,0 +1,163 @@
"""Agent memory search provider — FTS + vector search on agent_memories table."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class AgentMemorySearchProvider(BaseSearchProvider):
"""Search provider for AgentMemory entities."""
entity_type = "agent_memory"
supports_fts = True
supports_vector = True
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on agent_memories.content, filtered by visible_ids."""
if visible_ids is not None:
sql = text(
"""
SELECT am.*,
ts_rank(to_tsvector('pg_catalog.german', am.content),
to_tsquery('pg_catalog.german', :q)) AS rank
FROM agent_memories am
WHERE am.tenant_id = :tid
AND to_tsvector('pg_catalog.german', am.content) @@ to_tsquery('pg_catalog.german', :q)
AND am.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT am.*,
ts_rank(to_tsvector('pg_catalog.german', am.content),
to_tsquery('pg_catalog.german', :q)) AS rank
FROM agent_memories am
WHERE am.tenant_id = :tid
AND to_tsvector('pg_catalog.german', am.content) @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic vector search on agent_memories.embedding, filtered by visible_ids."""
if visible_ids is not None:
sql = text(
"""
SELECT am.*,
1 - (am.embedding <=> cast(:emb AS vector)) AS score
FROM agent_memories am
WHERE am.tenant_id = :tid
AND am.embedding IS NOT NULL
AND am.id = ANY(:visible_ids)
ORDER BY am.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT am.*,
1 - (am.embedding <=> cast(:emb AS vector)) AS score
FROM agent_memories am
WHERE am.tenant_id = :tid
AND am.embedding IS NOT NULL
ORDER BY am.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
"""
SELECT content, memory_type
FROM agent_memories
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
parts = [row.get("memory_type", ""), row.get("content", "")]
return " ".join(str(p) for p in parts if p)
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert agent memory to search result dict."""
if isinstance(entity, dict):
entity_id = str(entity.get("id", ""))
content = entity.get("content", "")
memory_type = entity.get("memory_type", "")
score = entity.get("score", entity.get("rank", 0.0))
else:
entity_id = str(getattr(entity, "id", ""))
content = getattr(entity, "content", "")
memory_type = getattr(entity, "memory_type", "")
score = getattr(entity, "score", getattr(entity, "rank", 0.0))
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": content[:100] if content else "",
"snippet": content,
"score": float(score) if score else 0.0,
"data": {"memory_type": memory_type},
}
app/plugins/builtins/unified_search/providers/ai_chat_provider.py
+137
View File
@@ -0,0 +1,137 @@
"""AI chat search provider — FTS search on ai_chat_messages table."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class AIChatSearchProvider(BaseSearchProvider):
"""Search provider for AI chat messages."""
entity_type = "ai_chat"
supports_fts = True
supports_vector = False
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on ai_chat_messages.content, joined with sessions for title."""
if visible_ids is not None:
sql = text(
"""
SELECT m.id, m.tenant_id, m.session_id, m.role, m.content,
m.model_used, m.tokens,
s.title AS session_title,
ts_rank(to_tsvector('pg_catalog.german', m.content),
to_tsquery('pg_catalog.german', :q)) AS rank
FROM ai_chat_messages m
JOIN ai_chat_sessions s ON s.id = m.session_id
WHERE m.tenant_id = :tid
AND to_tsvector('pg_catalog.german', m.content) @@ to_tsquery('pg_catalog.german', :q)
AND m.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT m.id, m.tenant_id, m.session_id, m.role, m.content,
m.model_used, m.tokens,
s.title AS session_title,
ts_rank(to_tsvector('pg_catalog.german', m.content),
to_tsquery('pg_catalog.german', :q)) AS rank
FROM ai_chat_messages m
JOIN ai_chat_sessions s ON s.id = m.session_id
WHERE m.tenant_id = :tid
AND to_tsvector('pg_catalog.german', m.content) @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""No vector support for chat messages — return empty list."""
return []
async def get_embedding_text(
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation — returns message content."""
sql = text(
"""
SELECT content
FROM ai_chat_messages
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
return str(row.get("content", ""))
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert chat message to search result dict."""
if isinstance(entity, dict):
entity_id = str(entity.get("id", ""))
content = entity.get("content", "")
role = entity.get("role", "")
session_title = entity.get("session_title", "")
session_id = str(entity.get("session_id", ""))
score = entity.get("rank", 0.0)
else:
entity_id = str(getattr(entity, "id", ""))
content = getattr(entity, "content", "")
role = getattr(entity, "role", "")
session_title = getattr(entity, "session_title", "")
session_id = str(getattr(entity, "session_id", ""))
score = getattr(entity, "rank", 0.0)
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": session_title or content[:80] if content else "",
"snippet": content,
"score": float(score) if score else 0.0,
"data": {
"role": role,
"session_id": session_id,
"session_title": session_title,
},
}
app/plugins/builtins/unified_search/providers/company_provider.py
+96 -41
View File
@@ -9,73 +9,128 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class CompanySearchProvider:
class CompanySearchProvider(BaseSearchProvider):
"""Search provider for Company entities (contacts with type='company')."""
entity_type = "contact"
async def search_fts(
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on contacts.search_tsv where type='company'."""
sql = text(
"""
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
"""Full-text search on contacts.search_tsv where type='company', filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
AND c.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic search on contacts.embedding where type='company'."""
sql = text(
"""
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.embedding IS NOT NULL
ORDER BY c.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
"""Semantic search on contacts.embedding where type='company', filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.embedding IS NOT NULL
AND c.id = ANY(:visible_ids)
ORDER BY c.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
FROM contacts c
WHERE c.tenant_id = :tid
AND c.deleted_at IS NULL
AND c.type = 'company'
AND c.embedding IS NOT NULL
ORDER BY c.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
app/plugins/builtins/unified_search/providers/event_provider.py
+92 -39
View File
@@ -9,71 +9,124 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class EventSearchProvider:
class EventSearchProvider(BaseSearchProvider):
"""Search provider for CalendarEntry entities."""
entity_type = "event"
async def search_fts(
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on calendar_entries.search_tsv."""
sql = text(
"""
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
"""Full-text search on calendar_entries.search_tsv, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
AND e.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic search on calendar_entries.embedding."""
sql = text(
"""
SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.embedding IS NOT NULL
ORDER BY e.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
"""Semantic search on calendar_entries.embedding, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.embedding IS NOT NULL
AND e.id = ANY(:visible_ids)
ORDER BY e.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score
FROM calendar_entries e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.embedding IS NOT NULL
ORDER BY e.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
app/plugins/builtins/unified_search/providers/file_provider.py
+189 -39
View File
@@ -9,71 +9,126 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
from app.config import settings
logger = logging.getLogger(__name__)
class FileSearchProvider:
class FileSearchProvider(BaseSearchProvider):
"""Search provider for DMS File entities."""
entity_type = "file"
supports_rag: bool = True
async def search_fts(
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on files.content_tsv."""
sql = text(
"""
SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
"""Full-text search on files.content_tsv, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q)
AND f.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic search on files.embedding."""
sql = text(
"""
SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.embedding IS NOT NULL
ORDER BY f.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
"""Semantic search on files.embedding, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.embedding IS NOT NULL
AND f.id = ANY(:visible_ids)
ORDER BY f.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score
FROM files f
WHERE f.tenant_id = :tid
AND f.deleted_at IS NULL
AND f.embedding IS NOT NULL
ORDER BY f.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
@@ -91,6 +146,101 @@ class FileSearchProvider:
content = row.get("content_text", "") or ""
return f"{name} {content[:5000]}"
async def search_rag(
self,
db: AsyncSession,
query_embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
user_id: uuid.UUID | None = None,
is_system_admin: bool = False,
) -> list[dict[str, Any]]:
"""RAG search: find relevant document chunks via vector similarity.
Queries the document_chunks table using cosine distance on chunk
embeddings, joins to files to exclude soft-deleted files, and applies
permission filtering using the over-fetch strategy (same as search_vector).
"""
await db.execute(text(f"SET LOCAL hnsw.ef_search = {settings.hnsw_ef_search}"))
if is_system_admin or not user_id:
return await self._search_rag_filtered(db, query_embedding, tenant_id, limit, None)
visible_ids = await self._get_visible_ids(db, tenant_id, user_id)
if not visible_ids:
return []
over_fetch_limit = limit * 3
results = await self._search_rag_filtered(db, query_embedding, tenant_id, over_fetch_limit, None)
filtered = [r for r in results if r.get("file_id") in visible_ids]
return filtered[:limit]
async def _search_rag_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""RAG vector search on document_chunks.embedding, filtered by visible_ids."""
if visible_ids is not None:
sql = text(
"""
SELECT dc.chunk_text, dc.file_id, dc.chunk_index,
1 - (dc.embedding <=> cast(:emb AS vector)) AS score
FROM document_chunks dc
JOIN files f ON dc.file_id = f.id
WHERE dc.tenant_id = :tid
AND dc.deleted_at IS NULL
AND dc.embedding IS NOT NULL
AND f.deleted_at IS NULL
AND f.id = ANY(:visible_ids)
ORDER BY dc.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT dc.chunk_text, dc.file_id, dc.chunk_index,
1 - (dc.embedding <=> cast(:emb AS vector)) AS score
FROM document_chunks dc
JOIN files f ON dc.file_id = f.id
WHERE dc.tenant_id = :tid
AND dc.deleted_at IS NULL
AND dc.embedding IS NOT NULL
AND f.deleted_at IS NULL
ORDER BY dc.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [
{
"id": str(r["file_id"]),
"entity_type": "file",
"entity_id": str(r["file_id"]),
"snippet": r["chunk_text"][:200],
"score": float(r.get("score", 0.0)),
"data": {"chunk_index": r.get("chunk_index")},
}
for r in rows
]
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert file to search result dict."""
if isinstance(entity, dict):
app/plugins/builtins/unified_search/providers/mail_provider.py
+92 -39
View File
@@ -9,71 +9,124 @@ from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class MailSearchProvider:
class MailSearchProvider(BaseSearchProvider):
"""Search provider for Mail entities."""
entity_type = "mail"
async def search_fts(
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on mails.body_tsv."""
sql = text(
"""
SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
"""Full-text search on mails.body_tsv, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q)
AND m.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def search_vector(
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic search on mails.embedding."""
sql = text(
"""
SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.embedding IS NOT NULL
ORDER BY m.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
"""Semantic search on mails.embedding, filtered by visible_ids.
If visible_ids is None, no visibility filter is applied (system admin).
"""
if visible_ids is not None:
sql = text(
"""
SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.embedding IS NOT NULL
AND m.id = ANY(:visible_ids)
ORDER BY m.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"emb": str(embedding),
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
FROM mails m
WHERE m.tenant_id = :tid
AND m.deleted_at IS NULL
AND m.embedding IS NOT NULL
ORDER BY m.embedding <=> cast(:emb AS vector)
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def get_embedding_text(
self,
db: AsyncSession,
entity_id: uuid.UUID,
tenant_id: uuid.UUID,
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation."""
sql = text(
app/plugins/builtins/unified_search/providers/workflow_provider.py
+143
View File
@@ -0,0 +1,143 @@
"""Workflow search provider — FTS search on workflows and workflow_instances tables."""
from __future__ import annotations
import logging
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
logger = logging.getLogger(__name__)
class WorkflowSearchProvider(BaseSearchProvider):
"""Search provider for Workflow entities (definitions and instances)."""
entity_type = "workflow"
supports_fts = True
supports_vector = False
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search on workflows name/description and workflow_instances status."""
if visible_ids is not None:
sql = text(
"""
SELECT w.id, w.tenant_id, w.name, w.description, w.trigger_event,
w.is_active,
ts_rank(
to_tsvector('pg_catalog.german',
coalesce(w.name, '') || ' ' || coalesce(w.description, '')),
to_tsquery('pg_catalog.german', :q)
) AS rank
FROM workflows w
WHERE w.tenant_id = :tid
AND to_tsvector('pg_catalog.german',
coalesce(w.name, '') || ' ' || coalesce(w.description, ''))
@@ to_tsquery('pg_catalog.german', :q)
AND w.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{
"q": tsquery,
"tid": tenant_id,
"lim": limit,
"visible_ids": list(visible_ids),
},
)
else:
sql = text(
"""
SELECT w.id, w.tenant_id, w.name, w.description, w.trigger_event,
w.is_active,
ts_rank(
to_tsvector('pg_catalog.german',
coalesce(w.name, '') || ' ' || coalesce(w.description, '')),
to_tsquery('pg_catalog.german', :q)
) AS rank
FROM workflows w
WHERE w.tenant_id = :tid
AND to_tsvector('pg_catalog.german',
coalesce(w.name, '') || ' ' || coalesce(w.description, ''))
@@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql,
{"q": tsquery, "tid": tenant_id, "lim": limit},
)
rows = result.mappings().all()
return [dict(r) for r in rows]
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""No vector support for workflows — return empty list."""
return []
async def get_embedding_text(
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Get text for embedding generation — returns workflow name + description."""
sql = text(
"""
SELECT name, description
FROM workflows
WHERE id = :eid AND tenant_id = :tid
"""
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
parts = [row.get("name", ""), row.get("description", "")]
return " ".join(str(p) for p in parts if p)
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert workflow to search result dict."""
if isinstance(entity, dict):
entity_id = str(entity.get("id", ""))
name = entity.get("name", "")
description = entity.get("description", "")
trigger_event = entity.get("trigger_event", "")
is_active = entity.get("is_active", True)
score = entity.get("rank", 0.0)
else:
entity_id = str(getattr(entity, "id", ""))
name = getattr(entity, "name", "")
description = getattr(entity, "description", "")
trigger_event = getattr(entity, "trigger_event", "")
is_active = getattr(entity, "is_active", True)
score = getattr(entity, "rank", 0.0)
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": name,
"snippet": description or "",
"score": float(score) if score else 0.0,
"data": {
"trigger_event": trigger_event,
"is_active": is_active,
},
}
app/plugins/builtins/unified_search/query_understanding.py
+21 -58
View File
@@ -2,7 +2,6 @@
from __future__ import annotations
import os
import json
import logging
import uuid
@@ -13,8 +12,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
logger = logging.getLogger(__name__)
DEFAULT_LLM_MODEL = os.environ.get('SEARCH_LLM_MODEL', 'ollama/deepseek-v4-flash')
QUERY_ANALYZE_SYSTEM = (
"Du bist ein Query-Analyzer fuer ein CRM. "
"Analysiere die Suchanfrage und gib JSON zurueck: "
@@ -27,33 +24,9 @@ RESULT_AGGREGATE_SYSTEM = (
'{"summary": str, "facets": {"types": {}, "dates": {}, "people": []}, "suggestions": [str]}'
)
async def _get_api_credentials(
db: AsyncSession | None, tenant_id: uuid.UUID | None
) -> tuple[str | None, str | None, str | None]:
"""Get API key, base_url and provider_type from the default AI provider in DB.
Falls back to API_KEY_OLLAMA_CLOUD env var.
Returns (api_key, base_url, provider_type).
"""
if db and tenant_id:
try:
from app.plugins.builtins.ai_assistant.contracts import get_default_provider
provider = await get_default_provider(db, tenant_id)
if provider and provider.api_key:
return provider.api_key, provider.base_url, provider.provider_type
except Exception:
logger.debug("Failed to get provider from DB, falling back to env")
env_key = os.environ.get('API_KEY_OLLAMA_CLOUD', '')
return (env_key if env_key else None), None, None
def _build_model(model: str, provider_type: str | None) -> str:
"""Build litellm model string with provider prefix."""
if provider_type:
model_parts = model.split("/", 1)
return f"{provider_type}/{model_parts[-1]}"
return model
# Sensible default model used when no provider-specific model is configured.
# llm_complete() resolves credentials/model prefix from the central config.
DEFAULT_LLM_MODEL = "ollama/deepseek-v4-flash"
def _fallback_query_analysis(query: str) -> dict[str, Any]:
@@ -74,6 +47,16 @@ def _fallback_aggregate(results: list[dict], query: str) -> dict[str, Any]:
}
def _parse_json_content(content: str) -> dict[str, Any]:
"""Parse LLM JSON response, stripping markdown code fences if present."""
content = content.strip()
if content.startswith("```"):
content = content.split("\n", 1)[-1] if "\n" in content else content[3:]
if content.endswith("```"):
content = content[:-3].strip()
return json.loads(content)
async def llm_analyze_query(
query: str,
db: AsyncSession | None = None,
@@ -84,11 +67,8 @@ async def llm_analyze_query(
Falls back to a simple dict if LLM fails.
"""
try:
api_key, api_base, provider_type = await _get_api_credentials(db, tenant_id)
model = _build_model(DEFAULT_LLM_MODEL, provider_type)
result = await llm_complete(
model=model,
model=DEFAULT_LLM_MODEL,
messages=[
{"role": "system", "content": QUERY_ANALYZE_SYSTEM},
{"role": "user", "content": query},
@@ -96,17 +76,10 @@ async def llm_analyze_query(
temperature=0.1,
max_tokens=500,
response_format={"type": "json_object"},
api_key=api_key,
api_base=api_base,
db=db,
tenant_id=tenant_id,
)
content = result["content"]
# Strip markdown code fences if present
content = content.strip()
if content.startswith("```"):
content = content.split("\n", 1)[-1] if "\n" in content else content[3:]
if content.endswith("```"):
content = content[:-3].strip()
return json.loads(content)
return _parse_json_content(result["content"])
except Exception:
logger.warning("LLM query analysis failed, using fallback", exc_info=True)
return _fallback_query_analysis(query)
@@ -125,9 +98,6 @@ async def llm_aggregate_results(
if not results:
return _fallback_aggregate(results, query)
try:
api_key, api_base, provider_type = await _get_api_credentials(db, tenant_id)
model = _build_model(DEFAULT_LLM_MODEL, provider_type)
# Truncate results to avoid token overflow
compact = [
{"entity_type": r.get("entity_type"), "title": r.get("title", "")[:100]}
@@ -136,7 +106,7 @@ async def llm_aggregate_results(
user_msg = json.dumps({"query": query, "results": compact})
result = await llm_complete(
model=model,
model=DEFAULT_LLM_MODEL,
messages=[
{"role": "system", "content": RESULT_AGGREGATE_SYSTEM},
{"role": "user", "content": user_msg},
@@ -144,17 +114,10 @@ async def llm_aggregate_results(
temperature=0.1,
max_tokens=1000,
response_format={"type": "json_object"},
api_key=api_key,
api_base=api_base,
db=db,
tenant_id=tenant_id,
)
content = result["content"]
# Strip markdown code fences if present
content = content.strip()
if content.startswith("```"):
content = content.split("\n", 1)[-1] if "\n" in content else content[3:]
if content.endswith("```"):
content = content[:-3].strip()
return json.loads(content)
return _parse_json_content(result["content"])
except Exception:
logger.warning("LLM result aggregation failed, using fallback", exc_info=True)
return _fallback_aggregate(results, query)
app/plugins/builtins/unified_search/routes.py
+215 -1
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import logging
import uuid
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import text
@@ -19,6 +20,7 @@ from app.plugins.builtins.unified_search.query_understanding import (
llm_analyze_query,
)
from app.plugins.builtins.unified_search.schemas import (
FacetsResponse,
ProviderResponse,
ReindexRequest,
SearchRequest,
@@ -48,15 +50,94 @@ async def search_get(
entity_types: str | None = Query(None, description="Comma-separated entity types to search"),
limit: int = Query(default=20, ge=1, le=100),
offset: int = Query(default=0, ge=0),
date_from: str | None = Query(None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at >= date_from"),
date_to: str | None = Query(None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at <= date_to"),
tags: str | None = Query(None, description="Comma-separated tags to filter results"),
sort: str = Query(default="relevance", description="Sort order: relevance, date, name"),
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> SearchResponse:
"""Perform hybrid search via GET (same as POST but with query params)."""
types_list = entity_types.split(",") if entity_types else None
req = SearchRequest(query=q, entity_types=types_list, limit=limit, offset=offset)
tags_list = tags.split(",") if tags else None
req = SearchRequest(
query=q,
entity_types=types_list,
limit=limit,
offset=offset,
date_from=date_from,
date_to=date_to,
tags=tags_list,
sort=sort,
)
return await _do_search(req, current_user, db)
def _parse_date(value: str | None) -> datetime | None:
"""Parse an ISO date string (YYYY-MM-DD) into a timezone-aware datetime."""
if not value:
return None
try:
dt = datetime.fromisoformat(value)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt
except ValueError:
return None
def _apply_filters_and_sort(
results: list[dict],
req: SearchRequest,
) -> list[dict]:
"""Apply post-search date/tags filters and sort order to results."""
date_from = _parse_date(req.date_from)
date_to = _parse_date(req.date_to)
filtered: list[dict] = []
for r in results:
# Date range filter on created_at/updated_at
if date_from or date_to:
created = r.get("_created_at")
updated = r.get("_updated_at")
ts = updated or created
if ts is None:
continue
if isinstance(ts, str):
try:
ts = datetime.fromisoformat(ts.replace("Z", "+00:00"))
except ValueError:
continue
if ts.tzinfo is None:
ts = ts.replace(tzinfo=timezone.utc)
if date_from and ts < date_from:
continue
if date_to and ts > date_to:
continue
# Tags filter (comma-separated on entity)
if req.tags:
entity_tags = r.get("_tags") or ""
tag_set = {t.strip().lower() for t in entity_tags.split(",") if t.strip()}
if not any(t.lower() in tag_set for t in req.tags):
continue
filtered.append(r)
# Sort order
sort = (req.sort or "relevance").lower()
if sort == "date":
filtered.sort(
key=lambda x: (x.get("_updated_at") or x.get("_created_at") or ""),
reverse=True,
)
elif sort == "name":
filtered.sort(key=lambda x: (x.get("title") or "").lower())
# 'relevance' keeps the existing score order
return filtered
async def _do_search(
req: SearchRequest,
current_user: dict,
@@ -81,6 +162,9 @@ async def _do_search(
is_system_admin=is_system_admin,
)
# Apply post-search filters (date range, tags) and sort
results = _apply_filters_and_sort(results, req)
# Resolve user permissions for field-level RBAC
resolved_perms = await resolve_permissions(db, user_id, tenant_id)
@@ -131,6 +215,70 @@ async def search(
return await _do_search(req, current_user, db)
# ─── Facets ───
@router.get("/facets", dependencies=[Depends(require_permission("search:read"))])
async def search_facets(
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> FacetsResponse:
"""Return available facets for search filtering (entity types, tags, date ranges)."""
tenant_id = uuid.UUID(current_user["tenant_id"])
# Entity types from the search registry
registry = get_search_registry()
entity_types = registry.get_entity_types()
# Tags: distinct tags across searchable entities (comma-separated on entities)
tags: set[str] = set()
tag_tables = {
"contacts": "tags",
"companies": "tags",
}
for table, col in tag_tables.items():
try:
result = await db.execute(
text(
f"SELECT DISTINCT unnest(string_to_array({col}, ',')) AS tag "
f"FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL AND {col} IS NOT NULL"
),
{"tid": tenant_id},
)
for row in result.mappings().all():
tag = (row.get("tag") or "").strip()
if tag:
tags.add(tag)
except Exception:
logger.debug("Facet tag query failed for %s", table)
# Date ranges: min/max created_at across searchable entities
date_ranges: dict[str, Any] = {}
date_tables = ["contacts", "mails", "files", "calendar_entries"]
for table in date_tables:
try:
result = await db.execute(
text(
f"SELECT min(created_at) AS min_dt, max(created_at) AS max_dt "
f"FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL"
),
{"tid": tenant_id},
)
row = result.mappings().first()
if row:
date_ranges[table] = {
"min": str(row.get("min_dt")) if row.get("min_dt") else None,
"max": str(row.get("max_dt")) if row.get("max_dt") else None,
}
except Exception:
logger.debug("Facet date query failed for %s", table)
return FacetsResponse(
entity_types=entity_types,
tags=sorted(tags),
date_ranges=date_ranges,
)
# ─── Suggest / Autocomplete ───
@router.get("/suggest", dependencies=[Depends(require_permission("search:read"))])
@@ -205,14 +353,76 @@ async def reindex(
if job_id:
job_ids.append(job_id)
# Optionally re-index file chunks
if req.include_chunks and "file" in entity_types:
chunk_job_id = await enqueue_job("reindex_all")
if chunk_job_id:
job_ids.append(chunk_job_id)
return {
"status": "ok",
"message": f"Reindexing {len(entity_types)} entity types",
"entity_types": entity_types,
"include_chunks": req.include_chunks,
"job_ids": job_ids,
}
# ─── Rebuild / Purge ───
@router.post("/rebuild/{entity_type}/{entity_id}", dependencies=[Depends(require_permission("search:admin"))])
async def rebuild_entity_index(
entity_type: str,
entity_id: str,
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> dict:
"""Rebuild the search index for a single entity (admin only)."""
tenant_id = uuid.UUID(current_user["tenant_id"])
try:
eid = uuid.UUID(entity_id)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid entity_id")
from app.plugins.builtins.unified_search.lifecycle import rebuild_index
success = await rebuild_index(db, entity_type, eid, tenant_id)
return {
"status": "ok" if success else "failed",
"entity_type": entity_type,
"entity_id": entity_id,
"message": "Index rebuilt" if success else "Rebuild failed — check logs",
}
@router.post("/purge/{entity_type}/{entity_id}", dependencies=[Depends(require_permission("search:admin"))])
async def purge_entity_index(
entity_type: str,
entity_id: str,
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> dict:
"""Purge an entity from the search index (admin only, GDPR)."""
tenant_id = uuid.UUID(current_user["tenant_id"])
try:
eid = uuid.UUID(entity_id)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid entity_id")
from app.plugins.builtins.unified_search.lifecycle import remove_from_index, remove_chunks
await remove_from_index(db, entity_type, eid, tenant_id)
if entity_type == "file":
await remove_chunks(db, eid, tenant_id)
return {
"status": "ok",
"entity_type": entity_type,
"entity_id": entity_id,
"message": "Purged from search index",
}
# ─── Providers ───
@router.get("/providers", dependencies=[Depends(require_permission("search:read"))])
@@ -228,6 +438,10 @@ async def list_providers(
entity_type=p.entity_type,
plugin_name="unified_search",
is_active=True,
supports_fts=getattr(p, "supports_fts", True),
supports_vector=getattr(p, "supports_vector", True),
supports_rag=getattr(p, "supports_rag", False),
supports_graph=getattr(p, "supports_graph", False),
)
for p in providers
]
app/plugins/builtins/unified_search/schemas.py
+26
View File
@@ -14,6 +14,19 @@ class SearchRequest(BaseModel):
entity_types: list[str] | None = None
limit: int = Field(default=20, ge=1, le=100)
offset: int = Field(default=0, ge=0)
date_from: str | None = Field(
default=None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at >= date_from"
)
date_to: str | None = Field(
default=None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at <= date_to"
)
tags: list[str] | None = Field(
default=None, description="Filter results by tags (comma-separated on entities)"
)
sort: str = Field(
default="relevance",
description="Sort order: relevance, date, name",
)
class SearchResult(BaseModel):
@@ -34,6 +47,14 @@ class SearchResponse(BaseModel):
suggestions: list[str]
class FacetsResponse(BaseModel):
"""Available facets for search filtering."""
entity_types: list[str]
tags: list[str]
date_ranges: dict[str, Any]
# ─── Similar ───
class SimilarRequest(BaseModel):
@@ -61,6 +82,7 @@ class SuggestResponse(BaseModel):
class ReindexRequest(BaseModel):
entity_types: list[str] = Field(default_factory=list)
include_chunks: bool = Field(default=True, description="Re-index document chunks for files")
# ─── Provider ───
@@ -69,3 +91,7 @@ class ProviderResponse(BaseModel):
entity_type: str
plugin_name: str
is_active: bool
supports_fts: bool = True
supports_vector: bool = True
supports_rag: bool = False
supports_graph: bool = False
app/plugins/builtins/unified_search/search_engine.py
+71 -27
View File
@@ -28,6 +28,30 @@ RRF_ALPHA = 0.5
RRF_BETA = 0.5
def rrf_fusion_multi(
result_lists: list[tuple[str, list[dict[str, Any]]]],
k: int = 60,
) -> list[dict[str, Any]]:
"""Reciprocal Rank Fusion over N result lists.
Each tuple is (mode_name, results). All lists get equal weight.
score = sum(1/(k+rank_i) for each list where item appears)
"""
fused: dict[str, dict[str, Any]] = {}
for _mode_name, results in result_lists:
for rank, item in enumerate(results):
eid = str(item.get("id", ""))
if not eid:
continue
rrf_score = 1.0 / (k + rank + 1)
if eid not in fused:
fused[eid] = {**item, "_score": 0.0}
fused[eid]["_score"] += rrf_score
return sorted(fused.values(), key=lambda x: x.get("_score", 0.0), reverse=True)
def rrf_fusion(
fts_results: list[dict[str, Any]],
vec_results: list[dict[str, Any]],
@@ -38,29 +62,16 @@ def rrf_fusion(
) -> list[dict[str, Any]]:
"""Reciprocal Rank Fusion of FTS and vector search results.
Backward-compatible wrapper around :func:`rrf_fusion_multi`.
score = alpha * (1/(k+rank_fts)) + beta * (1/(k+rank_vec))
"""
fused: dict[str, dict[str, Any]] = {}
for rank, item in enumerate(fts_results):
eid = str(item.get("id", ""))
if not eid:
continue
rrf_score = alpha * (1.0 / (k + rank + 1))
if eid not in fused:
fused[eid] = {**item, "_score": 0.0, "_entity_type": entity_type}
fused[eid]["_score"] += rrf_score
for rank, item in enumerate(vec_results):
eid = str(item.get("id", ""))
if not eid:
continue
rrf_score = beta * (1.0 / (k + rank + 1))
if eid not in fused:
fused[eid] = {**item, "_score": 0.0, "_entity_type": entity_type}
fused[eid]["_score"] += rrf_score
return sorted(fused.values(), key=lambda x: x.get("_score", 0.0), reverse=True)
fused = rrf_fusion_multi(
[("fts", fts_results), ("vec", vec_results)],
k=k,
)
for item in fused:
item["_entity_type"] = entity_type
return fused
async def hybrid_search(
@@ -113,24 +124,57 @@ async def hybrid_search(
fts_results: list[dict[str, Any]] = []
vec_results: list[dict[str, Any]] = []
rag_results: list[dict[str, Any]] = []
graph_results: list[dict[str, Any]] = []
try:
fts_results = await provider.search_fts(db, tsquery, tenant_id, fetch_limit, user_id=user_id, is_system_admin=is_system_admin)
except Exception:
logger.exception("FTS search failed for %s", entity_type)
# Respect provider capability flags
if getattr(provider, "supports_fts", True):
try:
fts_results = await provider.search_fts(db, tsquery, tenant_id, fetch_limit, user_id=user_id, is_system_admin=is_system_admin)
except Exception:
logger.exception("FTS search failed for %s", entity_type)
if query_embedding:
if query_embedding and getattr(provider, "supports_vector", True):
try:
vec_results = await provider.search_vector(db, query_embedding, tenant_id, fetch_limit, user_id=user_id, is_system_admin=is_system_admin)
except Exception:
logger.exception("Vector search failed for %s", entity_type)
fused = rrf_fusion(fts_results, vec_results, entity_type)
# RAG / Graph modes are not implemented yet, but keep the fusion ready.
if getattr(provider, "supports_rag", False) and hasattr(provider, "search_rag") and query_embedding:
try:
rag_results = await provider.search_rag(db, query_embedding, tenant_id, fetch_limit, user_id=user_id, is_system_admin=is_system_admin)
except Exception:
logger.exception("RAG search failed for %s", entity_type)
if getattr(provider, "supports_graph", False) and hasattr(provider, "search_graph"):
try:
graph_results = await provider.search_graph(db, query_analysis, tenant_id, fetch_limit, user_id=user_id, is_system_admin=is_system_admin)
except Exception:
logger.exception("Graph search failed for %s", entity_type)
if rag_results or graph_results:
fused = rrf_fusion_multi(
[
("fts", fts_results),
("vec", vec_results),
("rag", rag_results),
("graph", graph_results),
]
)
for item in fused:
item["_entity_type"] = entity_type
else:
fused = rrf_fusion(fts_results, vec_results, entity_type)
# Convert to search result format
for item in fused:
result = provider.to_search_result(item)
result["score"] = item.get("_score", 0.0)
# Preserve raw fields for post-search filtering (date/tags)
result["_created_at"] = item.get("created_at")
result["_updated_at"] = item.get("updated_at")
result["_tags"] = item.get("tags")
all_results.append(result)
all_results.sort(key=lambda x: x.get("score", 0.0), reverse=True)
docs/api-documentation.md
+103 -7
View File
@@ -324,16 +324,112 @@ Agent Builder, Automation Builder, Cron-Scheduler, Agent Runner.
### search (Unified Search)
7 endpoints for cross-entity search.
Hybrid cross-entity search (PostgreSQL FTS + pgvector) with KI query understanding, RRF rank fusion, visibility filtering, and field-level RBAC.
| Method | Path | Description |
|--------|------|-------------|
| POST | `/api/v1/search` | Unified search across all entities. |
| GET | `/api/v1/search/providers` | List search providers. |
| POST | `/api/v1/search/reindex` | Rebuild search index. |
| POST | `/api/v1/search/similar` | Find similar entities. |
| GET | `/api/v1/search/suggest` | Search suggestions. |
| GET | `/api/v1/search/stats` | Search index statistics. |
| GET | `/api/v1/search` | Hybrid search via query params (same as POST). |
| POST | `/api/v1/search` | Hybrid search with KI query understanding. |
| GET | `/api/v1/search/suggest` | Autocomplete suggestions (FTS prefix). |
| POST | `/api/v1/search/similar` | Find similar entities across all types by embedding. |
| POST | `/api/v1/search/reindex` | Trigger reindexing of entity types (admin). |
| GET | `/api/v1/search/providers` | List active search providers + capability flags. |
| POST | `/api/v1/search/providers/{entity_type}/toggle` | Toggle a provider on/off (admin). |
| GET | `/api/v1/search/stats` | Search index statistics (indexed/pending per table). |
| GET | `/api/v1/search/facets` | Available facets (entity types, tags, date ranges). |
| POST | `/api/v1/search/rebuild/{entity_type}/{entity_id}` | Rebuild index for a single entity (admin). |
| POST | `/api/v1/search/purge/{entity_type}/{entity_id}` | Purge entity from index (admin, GDPR). |
> **MCP:** Unified search is also exposed as an MCP tool named `search` (category `search`, permission `search:read`) via the MCP server plugin (`app/plugins/builtins/mcp_server/tool_definitions.py`).
#### Search Request (POST `/api/v1/search`)
```json
{
"query": "Max Mustermann",
"entity_types": ["contact", "mail"],
"limit": 20,
"offset": 0,
"date_from": "2026-01-01",
"date_to": "2026-12-31",
"tags": ["vip", "partner"],
"sort": "relevance"
}
```
| Field | Type | Default | Description |
|-------|------|---------|-------------|
| `query` | string | — | Search query (1–500 chars, required). |
| `entity_types` | list[string] | all | Restrict search to these entity types. |
| `limit` | int | 20 | Max results (1–100). |
| `offset` | int | 0 | Pagination offset. |
| `date_from` | string | null | ISO date `YYYY-MM-DD` — filter by `created_at`/`updated_at >= date_from`. |
| `date_to` | string | null | ISO date `YYYY-MM-DD` — filter by `created_at`/`updated_at <= date_to`. |
| `tags` | list[string] | null | Filter results by tags (comma-separated on entities). |
| `sort` | string | `relevance` | Sort order: `relevance`, `date`, `name`. |
#### Search Response
```json
{
"query": "Max Mustermann",
"normalized_query": "max mustermann",
"results": [
{
"entity_type": "contact",
"entity_id": "uuid",
"title": "Max Mustermann",
"snippet": "max@example.com",
"score": 0.95,
"data": {"type": "person"}
}
],
"facets": {"types": {"contact": 1}},
"summary": "1 Ergebnis",
"suggestions": []
}
```
| Field | Type | Description |
|-------|------|-------------|
| `query` | string | Original query. |
| `normalized_query` | string | KI-normalized query. |
| `results` | list[SearchResult] | Ranked results (entity_type, entity_id, title, snippet, score, data). |
| `facets` | object | KI-generated facet counts. |
| `summary` | string | Human-readable summary. |
| `suggestions` | list[string] | Suggested follow-up filters. |
#### GET `/api/v1/search` Query Params
Same fields as the POST body, passed as query parameters: `q` (required), `entity_types` (comma-separated), `limit`, `offset`, `date_from`, `date_to`, `tags` (comma-separated), `sort`.
#### GET `/api/v1/search/suggest`
Query params: `q` (required, 1–200), `limit` (default 10, 1–50). Returns `{"suggestions": ["..."]}`.
#### POST `/api/v1/search/similar`
Body: `{"entity_type": "contact", "entity_id": "uuid", "limit": 5}`. Returns `{"similar": {"mail": [SearchResult...], ...}}`.
#### POST `/api/v1/search/reindex`
Body: `{"entity_types": ["contact"], "include_chunks": true}`. Returns `{"status": "ok", "entity_types": [...], "include_chunks": true, "job_ids": [...]}`. Requires `search:admin`.
#### GET `/api/v1/search/providers`
Returns a list of providers: `{"entity_type", "plugin_name", "is_active", "supports_fts", "supports_vector", "supports_rag", "supports_graph"}`.
#### GET `/api/v1/search/facets`
Returns `{"entity_types": [...], "tags": [...], "date_ranges": {"contacts": {"min": "...", "max": "..."}, ...}}`.
#### GET `/api/v1/search/stats`
Returns per-table `{"total", "indexed", "pending"}` plus `recent_logs` (last 10 index log entries).
#### POST `/api/v1/search/rebuild/{entity_type}/{entity_id}` / `/purge/{entity_type}/{entity_id}`
Admin-only. Rebuild regenerates the embedding + TSV; purge sets embedding/TSV to NULL (and removes chunks for files). Returns `{"status": "ok"|"failed", "entity_type", "entity_id", "message"}`.
### reports (Report Generator)
docs/plugin-development-guide.md
+188 -1
View File
@@ -685,7 +685,194 @@ async def on_deactivate(self, db, service_container, event_bus):
---
## 27. Testing Guide
## 27. Search Integration
The Unified Search plugin provides hybrid cross-entity search (PostgreSQL FTS + pgvector) with KI query understanding, RRF rank fusion, visibility filtering, and field-level RBAC. Plugins can expose their entities to unified search by implementing a `SearchProvider`.
### 27.1 Creating a SearchProvider
Inherit from `BaseSearchProvider` and implement the required methods. The base class handles visibility filtering automatically (see 27.4).
```python
# app/plugins/builtins/my_plugin/search_provider.py
from __future__ import annotations
import uuid
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession
from app.plugins.builtins.unified_search.base_provider import BaseSearchProvider
class MyEntitySearchProvider(BaseSearchProvider):
"""Search provider for MyEntity."""
entity_type = "my_entity"
supports_fts = True
supports_vector = True
supports_rag = False
supports_graph = False
async def _search_fts_filtered(
self,
db: AsyncSession,
tsquery: str,
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Full-text search filtered by visible_ids (None = no filter)."""
if visible_ids is not None:
sql = text(
"""
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM my_entities e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
AND e.id = ANY(:visible_ids)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(
sql, {"q": tsquery, "tid": tenant_id, "lim": limit, "visible_ids": list(visible_ids)}
)
else:
sql = text(
"""
SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
FROM my_entities e
WHERE e.tenant_id = :tid
AND e.deleted_at IS NULL
AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
ORDER BY rank DESC
LIMIT :lim
"""
)
result = await db.execute(sql, {"q": tsquery, "tid": tenant_id, "lim": limit})
return [dict(r) for r in result.mappings().all()]
async def _search_vector_filtered(
self,
db: AsyncSession,
embedding: list[float],
tenant_id: uuid.UUID,
limit: int,
visible_ids: set[uuid.UUID] | None,
) -> list[dict[str, Any]]:
"""Semantic vector search on the entity's embedding column."""
# Same pattern as _search_fts_filtered but using `embedding <=> cast(:emb AS vector)`
return []
async def get_embedding_text(
self, db: AsyncSession, entity_id: uuid.UUID, tenant_id: uuid.UUID
) -> str:
"""Return the text used for embedding generation (non-sensitive fields only)."""
sql = text(
"SELECT name, description FROM my_entities WHERE id = :eid AND tenant_id = :tid"
)
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
row = result.mappings().first()
if not row:
return ""
return " ".join(str(v) for v in row.values() if v)
def to_search_result(self, entity: object) -> dict[str, Any]:
"""Convert an ORM entity or dict to a search result dict."""
if isinstance(entity, dict):
entity_id = str(entity.get("id", ""))
name = entity.get("name", "")
description = entity.get("description", "")
else:
entity_id = str(getattr(entity, "id", ""))
name = getattr(entity, "name", "")
description = getattr(entity, "description", "")
return {
"entity_type": self.entity_type,
"entity_id": entity_id,
"title": name,
"snippet": description or "",
"score": 0.0,
"data": {},
}
```
### 27.2 Capability Flags
Each provider declares which search modes it supports via class attributes. The registry and API use these flags to decide which search paths to run and to report capabilities to clients.
| Flag | Default | Meaning |
|------|---------|---------|
| `supports_fts` | `True` | Full-text search via PostgreSQL `tsvector`/`tsquery`. |
| `supports_vector` | `True` | Semantic vector search via pgvector embeddings. |
| `supports_rag` | `False` | Retrieval-augmented generation over document chunks. |
| `supports_graph` | `False` | Graph-based search (GraphRAG). |
Set `supports_vector = False` (and implement `_search_vector_filtered` returning `[]`) when an entity has no embedding column, e.g. chat messages or workflows.
### 27.3 Registering a Provider
Register providers during plugin activation. The Unified Search plugin calls `auto_register_providers(db)` on activation, which registers all built-in providers. For a custom plugin, register your provider directly in `on_activate`:
```python
async def on_activate(self, db, service_container, event_bus) -> None:
await super().on_activate(db, service_container, event_bus)
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
from app.plugins.builtins.my_plugin.search_provider import MyEntitySearchProvider
get_search_registry().register(MyEntitySearchProvider())
```
To add a provider to the built-in `auto_register_providers` list, import and append it to the `provider_cls` list in `app/plugins/builtins/unified_search/provider_registry.py`.
### 27.4 Permission Filtering (Automatic)
`BaseSearchProvider.search_fts` / `search_vector` automatically load the calling user's visible entity IDs via `get_visible_ids` and pass them to `_search_fts_filtered` / `_search_vector_filtered`. When `is_system_admin` is true or no `user_id` is provided, `visible_ids` is `None` and no filter is applied. Your `_search_*_filtered` implementation must honor `visible_ids` (add `AND id = ANY(:visible_ids)` when it is not `None`).
### 27.5 Auto-Indexing via Events / Outbox
Entities are indexed automatically when created or updated. The Unified Search plugin subscribes to domain events (e.g. `contact.created`, `contact.updated`, `mail.synced`, `file.uploaded`) and enqueues indexing jobs. For custom entities, publish the corresponding events or enqueue jobs directly:
```python
from app.core.jobs import enqueue_job
# After creating/updating an entity
await enqueue_job("index_entity", "my_entity", str(entity_id))
```
Indexing jobs call `index_entity(entity_type, entity_id, tenant_id, db)`, which uses the provider's `get_embedding_text` to generate an embedding and stores it in the entity's `embedding` column. The `indexed_at` column tracks dedup so unchanged entities are not re-embedded.
### 27.6 Lifecycle Hooks
The Unified Search plugin subscribes to lifecycle events to keep the index consistent:
| Event | Handler | Effect |
|-------|---------|--------|
| `entity.deleted` | `handle_entity_delete` | Removes embedding + TSV (and chunks for files). |
| `entity.restored` | `handle_entity_restore` | Rebuilds the search index. |
| `entity.corrected` | `handle_entity_correction` | Rebuilds the search index. |
Publish these events (or call the lifecycle functions directly) when your plugin deletes, restores, or corrects entities so the search index stays in sync.
### 27.7 RAG Document Chunking
For RAG over long documents, use `chunk_text` from `app.plugins.builtins.unified_search.chunking` to split extracted text into overlapping chunks before embedding:
```python
from app.plugins.builtins.unified_search.chunking import chunk_text
chunks = chunk_text(document_text, chunk_size=1000, overlap=200)
# Each chunk: {"chunk_index": 0, "chunk_text": "...", "chunk_hash": "sha256..."}
```
Chunks are stored in the `document_chunks` table and embedded at the chunk level for fine-grained vector retrieval. `chunk_hash` is a deterministic SHA-256 of the chunk text, used for dedup.
---
## 28. Testing Guide
### 11.1 Backend Tests
docs/test-strategy.md
+51
View File
@@ -295,3 +295,54 @@ Diese Pipeline ist verbindlich für Phase-Gate-Reviews und muss vor jedem Phasen
- ✅ Retention: GDPR-Hard-Delete nach konfigurierbarer Aufbewahrungsfrist
- ✅ Hook-based History: `do_action('entity.after_create/update/delete')` → `record_history()`
- ✅ Dynamic Permission Checks: Restore-Permission aus RestoreConfig, nicht hardcoded
---
## Phase E — Unified Search Test-Konventionen
### Neue Test-Datei: `tests/test_unified_search_phase_e.py` (40 Tests)
| Test-Gruppe | Tests | Status |
|-------------|-------|--------|
| Provider-Capability-Flags (supports_fts/vector/rag/graph, get_providers_by_capability, get_capabilities) | 3 | ✅ |
| RRF Multi-Fusion (2/3/4 Listen, Multi-Listen-Scoring, Backward-Compat, Empty Inputs) | 5 | ✅ |
| Chunking (empty/short/long/exact multiple, Overlap, Hash deterministisch, Whitespace-Normalisierung) | 6 | ✅ |
| Lifecycle (remove_from_index setzt embedding+TSV NULL, rebuild_index, entity.deleted/restored) | 6 | ✅ |
| API-Filter (date_from/date_to, tags, sort, facets-Struktur) | 4 | ✅ |
| AI-Tool (Name/Description, Parameter, OpenAI-Schema, Handler kompakt, Fehlerfälle) | 5 | ✅ |
| Neue Provider (AgentMemory, AIChat, Workflow — Import + Flags) | 4 | ✅ |
| Sensitive-Fields-Exclusion (nicht in search_tsv, nicht in Embedding-Text, Redaction) | 4 | ✅ |
### Konventionen für Search-Tests
1. **Isolation & Determinismus:** Jeder Test nutzt eine eigene Tenant-ID und eigene Entity-IDs. Keine zufälligen UUIDs — echte IDs aus der DB verwenden.
2. **Schema-Anpassung idempotent:** Die Test-DB (`create_all`) definiert `search_tsv` als generierte Spalte, Produktion (Migration 0001) als Plain-Column mit Trigger. Der Lifecycle-Helper konvertiert die Spalte idempotent per `DO $$ ... DROP EXPRESSION` und droppt den `contacts_tsv_update`-Trigger, damit `remove_from_index` `search_tsv = NULL` setzen kann. Die Schema-Änderung persistiert über Testläufe (nur Tabellen werden getruncated).
3. **Patch-Targets:** Funktionen, die innerhalb einer Funktion importiert werden (z.B. `index_entity` in `lifecycle.py`, `get_session_factory` in `ai_tool.py`), müssen am Ursprungsmodul gepatcht werden (`app.plugins.builtins.unified_search.embedding.index_entity`, `app.core.db.get_session_factory`), nicht am importierenden Modul.
4. **Keine echten LLM/Embedding-Calls:** Alle KI-Aufrufe werden mit `AsyncMock` gemockt. Kein Test darf ein echtes Modell kontaktieren.
### Mock-Patterns für LLM/Embedding-Calls
```python
from unittest.mock import AsyncMock, patch
# LLM-Query-Verständnis (Normalisierung, Facets, Summary)
with patch("app.plugins.builtins.unified_search.llm.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.return_value = {"normalized_query": "max mustermann", "facets": {}, "summary": "1 Ergebnis"}
# ... Test
# Embedding-Generierung
with patch("app.plugins.builtins.unified_search.embedding.generate_embedding", new_callable=AsyncMock) as mock_emb:
mock_emb.return_value = [0.1, 0.2, 0.3]
# ... Test
# LLM-Embedding-Client
with patch("app.plugins.builtins.unified_search.embedding.llm_embed", new_callable=AsyncMock) as mock_llm_emb:
mock_llm_emb.return_value = [0.1, 0.2, 0.3]
# ... Test
```
**Regeln:**
- `llm_complete` liefert ein Dict mit `normalized_query`, `facets`, `summary` (und optional `suggestions`).
- `generate_embedding` / `llm_embed` liefern eine Liste von Floats (Embedding-Vektor).
- Bei Fehlerpfaden: `mock_llm.side_effect = Exception("...")` oder `return_value = None` für Fallback-Verhalten testen.
- DB-Session-Factory in AI-Tool-Handler-Tests: `patch("app.core.db.get_session_factory", return_value=sf)` mit `async_sessionmaker(bind=db_session.bind, ...)`.
frontend/src/App.tsx
+3
View File
@@ -7,6 +7,8 @@ import { useThemeStore } from '@/store/themeStore';
import { ErrorBoundary } from '@/components/common/ErrorBoundary';
import { useToast } from '@/components/ui/Toast';
import { useOnlineStatus } from '@/hooks/useOnlineStatus';
import { CommandPalette } from '@/components/search/CommandPalette';
import { useCommandPalette } from '@/hooks/useCommandPalette';
function QueryClientWrapper({ children }: { children: React.ReactNode }) {
const toast = useToast();
@@ -79,6 +81,7 @@ export default function App() {
<ErrorBoundary>
<AppRouter />
</ErrorBoundary>
<CommandPalette />
</QueryClientWrapper>
);
}
frontend/src/api/hooks.ts
+5 -5
View File
@@ -24,16 +24,16 @@ export * from './automation';
// ── Search hook (uses dynamic import, kept inline) ──
import type { SearchResult } from './search';
export type { SearchResult };
import type { SearchFilters, SearchResult } from './search';
export type { SearchResult, SearchFilters };
export function useGlobalSearch(query: string, entityTypes?: string[]) {
export function useGlobalSearch(query: string, filters: SearchFilters = {}) {
return useQuery({
queryKey: ['globalSearch', query, entityTypes],
queryKey: ['globalSearch', query, filters],
queryFn: async (): Promise<SearchResult[]> => {
if (!query.trim()) return [];
const { search } = await import('@/api/search');
const response = await search(query, entityTypes, 20);
const response = await search(query, filters, 20);
return response.results || [];
},
enabled: query.trim().length > 0,
frontend/src/api/search.ts
+31 -3
View File
@@ -10,13 +10,32 @@ export interface SearchResult {
data?: Record<string, unknown>;
}
export interface SearchFacet {
value: string;
count: number;
}
export interface SearchFilters {
entityTypes?: string[];
tags?: string[];
dateFrom?: string;
dateTo?: string;
sort?: 'relevance' | 'date' | 'name';
}
export interface SearchResponse {
results: SearchResult[];
facets?: Record<string, Array<{ value: string; count: number }>>;
facets?: Record<string, SearchFacet[]>;
summary?: string;
suggestions?: string[];
}
export interface FacetsResponse {
entity_types: string[];
tags: string[];
date_ranges: Record<string, { min: string | null; max: string | null }>;
}
// Map backend entity_type to frontend route URL
const ENTITY_URL_MAP: Record<string, (id: string, data?: Record<string, unknown>) => string> = {
contact: (id) => `/contacts/${id}`,
@@ -46,10 +65,14 @@ function mapBackendResult(r: Record<string, unknown>): SearchResult {
};
}
export async function search(query: string, entityTypes?: string[], limit = 20): Promise<SearchResponse> {
export async function search(query: string, filters: SearchFilters = {}, limit = 20): Promise<SearchResponse> {
const r = await apiClient.post('/search', {
query,
entity_types: entityTypes,
entity_types: filters.entityTypes,
tags: filters.tags,
date_from: filters.dateFrom,
date_to: filters.dateTo,
sort: filters.sort || 'relevance',
limit,
});
const data = r.data;
@@ -63,6 +86,11 @@ export async function search(query: string, entityTypes?: string[], limit = 20):
};
}
export async function fetchFacets(): Promise<FacetsResponse> {
const r = await apiClient.get('/search/facets');
return r.data as FacetsResponse;
}
export async function searchSuggest(q: string): Promise<string[]> {
const r = await apiClient.get('/search/suggest', { params: { q } });
return r.data.suggestions || [];
frontend/src/api/searchHooks.ts
+13
View File
@@ -0,0 +1,13 @@
import { useQuery } from '@tanstack/react-query';
import type { FacetsResponse } from './search';
export function useSearchFacets() {
return useQuery({
queryKey: ['searchFacets'],
queryFn: async (): Promise<FacetsResponse> => {
const { fetchFacets } = await import('@/api/search');
return fetchFacets();
},
staleTime: 5 * 60 * 1000,
});
}
frontend/src/components/layout/AppShell.tsx
+2
View File
@@ -13,6 +13,7 @@ import { WindowContainer } from '@/components/window/WindowContainer';
import { ErrorBoundary } from '@/components/common/ErrorBoundary';
import { WelcomeDialog } from '@/components/onboarding/WelcomeDialog';
import { OnboardingTour } from '@/components/onboarding/OnboardingTour';
import { CommandPalette } from '@/components/search/CommandPalette';
import { useOnboardingStore } from '@/store/onboardingStore';
export function AppShell() {
@@ -57,6 +58,7 @@ export function AppShell() {
{showMessageSidebar && <MessageSidebar />}
<AIUIControlIndicator />
<WindowContainer />
<CommandPalette />
<ToastContainer />
<WelcomeDialog open={!completed && !skipped} onClose={skip} />
<OnboardingTour />
frontend/src/components/layout/TopBar.tsx
+12 -1
View File
@@ -8,11 +8,12 @@ import { useLogout } from '@/api/hooks';
import { Avatar } from '@/components/ui/Avatar';
import { SearchDropdown } from '@/components/shared/SearchDropdown';
import { SuggestionBadge } from '@/components/ai/SuggestionBadge';
import { Building, ChevronDown, Menu, Zap, Bot, Layers, Code } from 'lucide-react';
import { Building, ChevronDown, Menu, Zap, Bot, Layers, Code, Search } from 'lucide-react';
import { NotificationBell } from '@/components/layout/NotificationBell';
import { WorkspaceSwitcher } from '@/components/layout/WorkspaceSwitcher';
import { useWindowStore } from '@/store/windowStore';
import { usePermission } from '@/hooks/usePermission';
import { useCommandPaletteStore } from '@/store/commandPaletteStore';
export function TopBar() {
const { t } = useTranslation();
@@ -86,6 +87,16 @@ export function TopBar() {
<div className="hidden md:block">
<SearchDropdown placeholder={t('topbar.search')} />
</div>
{/* Command palette shortcut */}
<button
onClick={() => useCommandPaletteStore.getState().open()}
className="p-2 rounded-md text-secondary-500 hover:bg-secondary-100 hover:text-secondary-700 min-h-touch min-w-touch flex items-center justify-center focus:outline-none focus-visible:ring-2 focus-visible:ring-primary-500"
aria-label={t('topbar.commandPalette')}
title={t('topbar.commandPaletteHint')}
data-testid="command-palette-btn"
>
<Search className="w-4 h-4" aria-hidden="true" strokeWidth={2} />
</button>
</div>
<div className="flex items-center gap-2">
frontend/src/components/search/CommandPalette.tsx
+317
View File
@@ -0,0 +1,317 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import clsx from 'clsx';
import { useNavigate } from 'react-router-dom';
import { useTranslation } from 'react-i18next';
import { Search, Loader2, Clock, X } from 'lucide-react';
import { useGlobalSearch, SearchResult } from '@/api/hooks';
import { useCommandPalette } from '@/hooks/useCommandPalette';
const RECENT_KEY = 'leocrm_recent_searches';
const MAX_RECENT = 5;
const TYPE_LABELS: Record<string, string> = {
company: 'search.companies',
contact: 'search.contacts',
mail: 'search.mails',
file: 'search.files',
event: 'search.events',
message: 'search.messages',
};
const TYPE_ICON_CLASSES: Record<string, string> = {
company: 'bg-primary-100 text-primary-700',
contact: 'bg-accent-100 text-accent-700',
mail: 'bg-success-100 text-success-700',
file: 'bg-warning-100 text-warning-700',
event: 'bg-secondary-100 text-secondary-700',
message: 'bg-secondary-100 text-secondary-700',
};
function typeIcon(type: string): string {
switch (type) {
case 'company': return 'F';
case 'contact': return 'K';
case 'mail': return '@';
case 'file': return '📄';
case 'event': return '📅';
case 'message': return '💬';
default: return '?';
}
}
function loadRecent(): string[] {
try {
const raw = localStorage.getItem(RECENT_KEY);
if (!raw) return [];
const parsed = JSON.parse(raw);
return Array.isArray(parsed) ? parsed.filter((x): x is string => typeof x === 'string') : [];
} catch {
return [];
}
}
function saveRecent(query: string) {
const trimmed = query.trim();
if (!trimmed) return;
const next = [trimmed, ...loadRecent().filter((q) => q !== trimmed)].slice(0, MAX_RECENT);
try {
localStorage.setItem(RECENT_KEY, JSON.stringify(next));
} catch {
// ignore storage errors
}
}
export function CommandPalette() {
const { t } = useTranslation();
const navigate = useNavigate();
const { isOpen, close } = useCommandPalette();
const [query, setQuery] = useState('');
const [debouncedQuery, setDebouncedQuery] = useState('');
const [activeIndex, setActiveIndex] = useState(-1);
const [recent, setRecent] = useState<string[]>([]);
const inputRef = useRef<HTMLInputElement>(null);
const dialogRef = useRef<HTMLDivElement>(null);
const previouslyFocused = useRef<HTMLElement | null>(null);
const { data: results, isLoading, isError } = useGlobalSearch(debouncedQuery);
// Debounce query
useEffect(() => {
const timer = setTimeout(() => setDebouncedQuery(query), 300);
return () => clearTimeout(timer);
}, [query]);
// Load recent searches when opened
useEffect(() => {
if (isOpen) {
setRecent(loadRecent());
setQuery('');
setDebouncedQuery('');
setActiveIndex(-1);
}
}, [isOpen]);
// Focus management + focus trap
useEffect(() => {
if (!isOpen) return;
previouslyFocused.current = document.activeElement as HTMLElement;
const timer = setTimeout(() => inputRef.current?.focus(), 0);
return () => {
clearTimeout(timer);
previouslyFocused.current?.focus();
};
}, [isOpen]);
const handleKeyDown = useCallback((e: KeyboardEvent) => {
if (!isOpen) return;
if (e.key === 'Escape') {
e.preventDefault();
close();
return;
}
if (e.key === 'Tab') {
// Simple focus trap: keep focus within the dialog
const focusable = dialogRef.current?.querySelectorAll<HTMLElement>(
'button, [href], input, select, textarea, [tabindex]:not([tabindex="-1"])'
);
if (!focusable || focusable.length === 0) return;
const first = focusable[0];
const last = focusable[focusable.length - 1];
if (e.shiftKey && document.activeElement === first) {
e.preventDefault();
last.focus();
} else if (!e.shiftKey && document.activeElement === last) {
e.preventDefault();
first.focus();
}
}
}, [isOpen, close]);
useEffect(() => {
document.addEventListener('keydown', handleKeyDown);
return () => document.removeEventListener('keydown', handleKeyDown);
}, [handleKeyDown]);
const flatResults = useMemo(() => results || [], [results]);
const groupedResults = useMemo(() => {
const groups: Record<string, SearchResult[]> = {};
for (const r of flatResults) {
if (!groups[r.type]) groups[r.type] = [];
groups[r.type].push(r);
}
return groups;
}, [flatResults]);
const totalCount = flatResults.length;
const handleResultClick = (result: SearchResult) => {
saveRecent(debouncedQuery);
close();
navigate(result.url);
};
const handleRecentClick = (q: string) => {
setQuery(q);
setDebouncedQuery(q);
setActiveIndex(-1);
};
const handleInputKeyDown = (e: React.KeyboardEvent) => {
if (e.key === 'ArrowDown') {
e.preventDefault();
setActiveIndex((prev) => Math.min(prev + 1, Math.max(totalCount - 1, 0)));
} else if (e.key === 'ArrowUp') {
e.preventDefault();
setActiveIndex((prev) => Math.max(prev - 1, -1));
} else if (e.key === 'Enter' && activeIndex >= 0 && flatResults[activeIndex]) {
e.preventDefault();
handleResultClick(flatResults[activeIndex]);
}
};
if (!isOpen) return null;
return (
<div
className="fixed inset-0 z-[100] flex items-start justify-center pt-[15vh] px-4"
role="dialog"
aria-modal="true"
aria-label={t('commandPalette.title')}
data-testid="command-palette"
>
<div
className="absolute inset-0 bg-secondary-900/50"
onClick={close}
aria-hidden="true"
data-testid="command-palette-backdrop"
/>
<div
ref={dialogRef}
className="relative w-full max-w-xl bg-white rounded-lg shadow-xl border border-secondary-200 overflow-hidden"
>
<div className="flex items-center gap-3 px-4 border-b border-secondary-200">
<Search className="w-5 h-5 text-secondary-400 flex-shrink-0" aria-hidden="true" />
<input
ref={inputRef}
type="search"
value={query}
onChange={(e) => { setQuery(e.target.value); setActiveIndex(-1); }}
onKeyDown={handleInputKeyDown}
placeholder={t('commandPalette.placeholder')}
className="flex-1 py-3 text-base bg-transparent focus:outline-none text-secondary-900 placeholder-secondary-400"
aria-label={t('commandPalette.placeholder')}
role="combobox"
aria-expanded={isOpen}
aria-controls="command-palette-results"
aria-autocomplete="list"
data-testid="command-palette-input"
/>
<button
onClick={close}
className="p-2 rounded-md text-secondary-400 hover:text-secondary-600 hover:bg-secondary-100 min-h-touch min-w-touch flex items-center justify-center focus:outline-none focus-visible:ring-2 focus-visible:ring-primary-500"
aria-label={t('common.close')}
>
<X className="w-4 h-4" aria-hidden="true" />
</button>
</div>
<div
id="command-palette-results"
className="max-h-[50vh] overflow-y-auto"
role="listbox"
aria-label={t('commandPalette.results')}
>
{isLoading ? (
<div className="flex items-center justify-center gap-2 px-4 py-8 text-sm text-secondary-500" role="status">
<Loader2 className="w-4 h-4 animate-spin" aria-hidden="true" />
{t('common.loading')}
</div>
) : isError ? (
<div className="px-4 py-8 text-sm text-danger-600" role="alert">
{t('commandPalette.error')}
</div>
) : debouncedQuery.trim() ? (
totalCount === 0 ? (
<div className="px-4 py-8 text-sm text-secondary-500">
{t('commandPalette.noResults', { query: debouncedQuery })}
</div>
) : (
Object.entries(groupedResults).map(([type, items]) => (
<div key={type} className="py-1">
<div className="px-4 py-1.5 text-xs font-semibold uppercase tracking-wide text-secondary-400">
{t(TYPE_LABELS[type] || 'search.allTypes')}
</div>
<ul>
{items.map((result, idx) => {
const flatIdx = flatResults.indexOf(result);
return (
<li key={`${result.type}-${result.id}`}>
<button
onClick={() => handleResultClick(result)}
onMouseEnter={() => setActiveIndex(flatIdx)}
className={clsx(
'w-full text-left px-4 py-2 flex items-center gap-3 min-h-touch hover:bg-secondary-50 focus:outline-none focus-visible:ring-2 focus-visible:ring-primary-500',
activeIndex === flatIdx && 'bg-primary-50'
)}
role="option"
aria-selected={activeIndex === flatIdx}
>
<span
className={clsx(
'inline-flex items-center justify-center w-8 h-8 rounded-full text-xs font-semibold flex-shrink-0',
TYPE_ICON_CLASSES[type] || 'bg-secondary-100 text-secondary-700'
)}
aria-hidden="true"
>
{typeIcon(type)}
</span>
<div className="flex-1 min-w-0">
<p className="font-medium text-secondary-900 truncate">{result.name}</p>
{result.description && (
<p className="text-xs text-secondary-500 truncate">{result.description}</p>
)}
</div>
{typeof result.score === 'number' && result.score > 0 && (
<span className="text-xs text-secondary-400 flex-shrink-0">
{Math.round(result.score * 100)}%
</span>
)}
</button>
</li>
);
})}
</ul>
</div>
))
)
) : recent.length > 0 ? (
<div className="py-1">
<div className="px-4 py-1.5 text-xs font-semibold uppercase tracking-wide text-secondary-400">
{t('commandPalette.recent')}
</div>
<ul>
{recent.map((q) => (
<li key={q}>
<button
onClick={() => handleRecentClick(q)}
className="w-full text-left px-4 py-2 flex items-center gap-3 min-h-touch hover:bg-secondary-50 focus:outline-none focus-visible:ring-2 focus-visible:ring-primary-500"
>
<Clock className="w-4 h-4 text-secondary-400 flex-shrink-0" aria-hidden="true" />
<span className="text-sm text-secondary-700 truncate">{q}</span>
</button>
</li>
))}
</ul>
</div>
) : (
<div className="px-4 py-8 text-sm text-secondary-500">
{t('commandPalette.hint')}
</div>
)}
</div>
</div>
</div>
);
}
frontend/src/components/search/SavedSearches.tsx
+144
View File
@@ -0,0 +1,144 @@
import React, { useCallback, useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { Trash2, Bookmark, BookmarkCheck } from 'lucide-react';
import { SearchFilters } from '@/api/hooks';
import { Button } from '@/components/ui/Button';
import { Input } from '@/components/ui/Input';
const SAVED_KEY = 'leocrm_saved_searches';
const MAX_SAVED = 20;
export interface SavedSearch {
id: string;
name: string;
query: string;
filters: SearchFilters;
createdAt: string;
}
function loadSaved(): SavedSearch[] {
try {
const raw = localStorage.getItem(SAVED_KEY);
if (!raw) return [];
const parsed = JSON.parse(raw);
return Array.isArray(parsed) ? parsed : [];
} catch {
return [];
}
}
function persistSaved(saved: SavedSearch[]) {
try {
localStorage.setItem(SAVED_KEY, JSON.stringify(saved));
} catch {
// ignore storage errors
}
}
interface SavedSearchesProps {
currentQuery: string;
currentFilters: SearchFilters;
onRun: (query: string, filters: SearchFilters) => void;
}
export function SavedSearches({ currentQuery, currentFilters, onRun }: SavedSearchesProps) {
const { t } = useTranslation();
const [saved, setSaved] = useState<SavedSearch[]>([]);
const [showSaveForm, setShowSaveForm] = useState(false);
const [name, setName] = useState('');
useEffect(() => {
setSaved(loadSaved());
}, []);
const refresh = useCallback(() => setSaved(loadSaved()), []);
const handleSave = () => {
const trimmed = name.trim();
if (!trimmed || !currentQuery.trim()) return;
const entry: SavedSearch = {
id: `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
name: trimmed,
query: currentQuery,
filters: currentFilters,
createdAt: new Date().toISOString(),
};
const next = [entry, ...loadSaved()].slice(0, MAX_SAVED);
persistSaved(next);
setName('');
setShowSaveForm(false);
refresh();
};
const handleDelete = (id: string) => {
persistSaved(loadSaved().filter((s) => s.id !== id));
refresh();
};
const canSave = currentQuery.trim().length > 0;
return (
<div className="space-y-3" data-testid="saved-searches">
<div className="flex items-center justify-between">
<h2 className="text-sm font-semibold text-secondary-900">{t('search.savedSearches')}</h2>
{canSave && !showSaveForm && (
<Button variant="ghost" size="sm" onClick={() => setShowSaveForm(true)} data-testid="save-search-btn">
<Bookmark className="w-4 h-4 mr-1" aria-hidden="true" />
{t('search.saveSearch')}
</Button>
)}
</div>
{showSaveForm && (
<div className="flex gap-2 items-end">
<div className="flex-1">
<Input
value={name}
onChange={(e) => setName(e.target.value)}
placeholder={t('search.saveSearchName')}
aria-label={t('search.saveSearchName')}
data-testid="save-search-name"
/>
</div>
<Button size="sm" onClick={handleSave} disabled={!name.trim()} data-testid="save-search-confirm">
<BookmarkCheck className="w-4 h-4 mr-1" aria-hidden="true" />
{t('common.save')}
</Button>
<Button size="sm" variant="ghost" onClick={() => setShowSaveForm(false)}>
{t('common.cancel')}
</Button>
</div>
)}
{saved.length === 0 ? (
<p className="text-sm text-secondary-500">{t('search.noSavedSearches')}</p>
) : (
<ul className="space-y-2">
{saved.map((s) => (
<li
key={s.id}
className="flex items-center gap-2 bg-white rounded-md border border-secondary-200 px-3 py-2"
>
<button
onClick={() => onRun(s.query, s.filters)}
className="flex-1 text-left min-h-touch focus:outline-none focus-visible:ring-2 focus-visible:ring-primary-500 rounded-md"
data-testid={`saved-search-${s.id}`}
>
<span className="block text-sm font-medium text-secondary-900 truncate">{s.name}</span>
<span className="block text-xs text-secondary-500 truncate">{s.query}</span>
</button>
<button
onClick={() => handleDelete(s.id)}
className="p-2 rounded-md text-secondary-400 hover:text-danger-600 hover:bg-danger-50 min-h-touch min-w-touch flex items-center justify-center focus:outline-none focus-visible:ring-2 focus-visible:ring-danger-500"
aria-label={t('common.delete')}
data-testid={`delete-saved-search-${s.id}`}
>
<Trash2 className="w-4 h-4" aria-hidden="true" />
</button>
</li>
))}
</ul>
)}
</div>
);
}
frontend/src/components/search/SearchFacets.tsx
+206
View File
@@ -0,0 +1,206 @@
import React, { useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { SearchFilters } from '@/api/hooks';
import { fetchFacets, FacetsResponse } from '@/api/search';
import { Input } from '@/components/ui/Input';
import { Select } from '@/components/ui/Select';
import { Button } from '@/components/ui/Button';
import { Skeleton } from '@/components/ui/Skeleton';
const TYPE_LABELS: Record<string, string> = {
company: 'search.companies',
contact: 'search.contacts',
mail: 'search.mails',
file: 'search.files',
event: 'search.events',
message: 'search.messages',
};
interface SearchFacetsProps {
filters: SearchFilters;
onChange: (filters: SearchFilters) => void;
onClear: () => void;
entityCounts?: Record<string, number>;
tagCounts?: Record<string, number>;
}
export function SearchFacets({ filters, onChange, onClear, entityCounts, tagCounts }: SearchFacetsProps) {
const { t } = useTranslation();
const [facets, setFacets] = useState<FacetsResponse | null>(null);
const [isLoading, setIsLoading] = useState(true);
useEffect(() => {
let cancelled = false;
fetchFacets()
.then((data) => {
if (!cancelled) setFacets(data);
})
.catch(() => {
// ignore — facets are optional; the page still works without them
})
.finally(() => {
if (!cancelled) setIsLoading(false);
});
return () => {
cancelled = true;
};
}, []);
const entityTypes = facets?.entity_types || [];
const tags = facets?.tags || [];
const toggleEntityType = (type: string) => {
const current = filters.entityTypes || [];
const next = current.includes(type)
? current.filter((x) => x !== type)
: [...current, type];
onChange({ ...filters, entityTypes: next.length > 0 ? next : undefined });
};
const toggleTag = (tag: string) => {
const current = filters.tags || [];
const next = current.includes(tag)
? current.filter((x) => x !== tag)
: [...current, tag];
onChange({ ...filters, tags: next.length > 0 ? next : undefined });
};
const hasActiveFilters =
(filters.entityTypes?.length || 0) > 0 ||
(filters.tags?.length || 0) > 0 ||
!!filters.dateFrom ||
!!filters.dateTo ||
(filters.sort && filters.sort !== 'relevance');
return (
<div className="space-y-6" data-testid="search-facets">
<div className="flex items-center justify-between">
<h2 className="text-sm font-semibold text-secondary-900">{t('search.filters')}</h2>
{hasActiveFilters && (
<Button variant="ghost" size="sm" onClick={onClear} data-testid="clear-filters-btn">
{t('common.reset')}
</Button>
)}
</div>
{/* Entity type filter */}
<div>
<h3 className="text-xs font-semibold uppercase tracking-wide text-secondary-400 mb-2">
{t('search.entityType')}
</h3>
{isLoading ? (
<div className="space-y-2">
{[1, 2, 3].map((i) => <Skeleton key={i} className="h-5" />)}
</div>
) : entityTypes.length === 0 ? (
<p className="text-sm text-secondary-500">{t('common.none')}</p>
) : (
<ul className="space-y-1">
{entityTypes.map((type) => {
const checked = filters.entityTypes?.includes(type) || false;
const count = entityCounts?.[type] ?? 0;
return (
<li key={type}>
<label className="flex items-center gap-2 min-h-touch cursor-pointer hover:bg-secondary-50 rounded-md px-2">
<input
type="checkbox"
checked={checked}
onChange={() => toggleEntityType(type)}
className="h-4 w-4 rounded border-secondary-300 text-primary-600 focus:ring-primary-500"
aria-label={t(TYPE_LABELS[type] || 'search.allTypes')}
/>
<span className="flex-1 text-sm text-secondary-700">
{t(TYPE_LABELS[type] || 'search.allTypes')}
</span>
{count > 0 && (
<span className="text-xs text-secondary-400">{count}</span>
)}
</label>
</li>
);
})}
</ul>
)}
</div>
{/* Tag filter */}
<div>
<h3 className="text-xs font-semibold uppercase tracking-wide text-secondary-400 mb-2">
{t('search.tags')}
</h3>
{isLoading ? (
<div className="space-y-2">
{[1, 2, 3].map((i) => <Skeleton key={i} className="h-5" />)}
</div>
) : tags.length === 0 ? (
<p className="text-sm text-secondary-500">{t('common.none')}</p>
) : (
<ul className="space-y-1">
{tags.map((tag) => {
const checked = filters.tags?.includes(tag) || false;
const count = tagCounts?.[tag] ?? 0;
return (
<li key={tag}>
<label className="flex items-center gap-2 min-h-touch cursor-pointer hover:bg-secondary-50 rounded-md px-2">
<input
type="checkbox"
checked={checked}
onChange={() => toggleTag(tag)}
className="h-4 w-4 rounded border-secondary-300 text-primary-600 focus:ring-primary-500"
aria-label={tag}
/>
<span className="flex-1 text-sm text-secondary-700 truncate">{tag}</span>
{count > 0 && (
<span className="text-xs text-secondary-400">{count}</span>
)}
</label>
</li>
);
})}
</ul>
)}
</div>
{/* Date range filter */}
<div>
<h3 className="text-xs font-semibold uppercase tracking-wide text-secondary-400 mb-2">
{t('search.dateRange')}
</h3>
<div className="space-y-3">
<Input
type="date"
label={t('search.dateFrom')}
value={filters.dateFrom || ''}
onChange={(e) => onChange({ ...filters, dateFrom: e.target.value || undefined })}
data-testid="search-date-from"
/>
<Input
type="date"
label={t('search.dateTo')}
value={filters.dateTo || ''}
onChange={(e) => onChange({ ...filters, dateTo: e.target.value || undefined })}
data-testid="search-date-to"
/>
</div>
</div>
{/* Sort selector */}
<div>
<h3 className="text-xs font-semibold uppercase tracking-wide text-secondary-400 mb-2">
{t('common.sort')}
</h3>
<Select
value={filters.sort || 'relevance'}
onChange={(e) => onChange({ ...filters, sort: e.target.value as SearchFilters['sort'] })}
options={[
{ value: 'relevance', label: t('search.sortRelevance') },
{ value: 'date', label: t('search.sortDate') },
{ value: 'name', label: t('search.sortName') },
]}
aria-label={t('common.sort')}
data-testid="search-sort"
/>
</div>
</div>
);
}
frontend/src/components/search/SearchResultCard.tsx
+114
View File
@@ -0,0 +1,114 @@
import React from 'react';
import clsx from 'clsx';
import { useNavigate } from 'react-router-dom';
import { useTranslation } from 'react-i18next';
import { SearchResult } from '@/api/hooks';
import { Badge } from '@/components/ui/Badge';
const TYPE_LABELS: Record<string, string> = {
company: 'search.companies',
contact: 'search.contacts',
mail: 'search.mails',
file: 'search.files',
event: 'search.events',
message: 'search.messages',
};
const TYPE_ICON_CLASSES: Record<string, string> = {
company: 'bg-primary-100 text-primary-700',
contact: 'bg-accent-100 text-accent-700',
mail: 'bg-success-100 text-success-700',
file: 'bg-warning-100 text-warning-700',
event: 'bg-secondary-100 text-secondary-700',
message: 'bg-secondary-100 text-secondary-700',
};
const TYPE_BADGE_VARIANTS: Record<string, 'primary' | 'info' | 'success' | 'warning' | 'default'> = {
company: 'primary',
contact: 'info',
mail: 'success',
file: 'warning',
event: 'default',
message: 'default',
};
function typeIcon(type: string): string {
switch (type) {
case 'company': return 'F';
case 'contact': return 'K';
case 'mail': return '@';
case 'file': return '📄';
case 'event': return '📅';
case 'message': return '💬';
default: return '?';
}
}
interface SearchResultCardProps {
result: SearchResult;
query?: string;
}
function highlightMatch(text: string, query: string): React.ReactNode {
if (!query.trim()) return text;
const lowerText = text.toLowerCase();
const lowerQuery = query.toLowerCase();
const idx = lowerText.indexOf(lowerQuery);
if (idx === -1) return text;
return (
<>
{text.slice(0, idx)}
<mark className="bg-warning-200 text-secondary-900 rounded px-0.5">{text.slice(idx, idx + query.length)}</mark>
{text.slice(idx + query.length)}
</>
);
}
export function SearchResultCard({ result, query }: SearchResultCardProps) {
const { t } = useTranslation();
const navigate = useNavigate();
const score = typeof result.score === 'number' ? result.score : 0;
const scorePct = Math.round(score * 100);
return (
<button
onClick={() => navigate(result.url)}
className="w-full text-left bg-white rounded-lg shadow-sm border border-secondary-200 p-4 flex items-start gap-4 hover:shadow-md hover:border-primary-300 transition-shadow min-h-touch focus:outline-none focus-visible:ring-2 focus-visible:ring-primary-500"
data-testid={`search-result-${result.type}-${result.id}`}
>
<span
className={clsx(
'inline-flex items-center justify-center w-10 h-10 rounded-lg text-sm font-semibold flex-shrink-0',
TYPE_ICON_CLASSES[result.type] || 'bg-secondary-100 text-secondary-700'
)}
aria-hidden="true"
>
{typeIcon(result.type)}
</span>
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2 flex-wrap">
<p className="font-medium text-secondary-900 truncate">
{highlightMatch(result.name, query || '')}
</p>
<Badge variant={TYPE_BADGE_VARIANTS[result.type] || 'default'}>
{t(TYPE_LABELS[result.type] || 'search.allTypes')}
</Badge>
</div>
{result.description && (
<p className="text-sm text-secondary-500 mt-1 line-clamp-2">
{highlightMatch(result.description, query || '')}
</p>
)}
</div>
{score > 0 && (
<span
className="inline-flex items-center justify-center px-2 py-1 rounded-full text-xs font-medium bg-primary-100 text-primary-700 flex-shrink-0"
title={t('search.score')}
>
{scorePct}%
</span>
)}
</button>
);
}
frontend/src/hooks/useCommandPalette.ts
+37
View File
@@ -0,0 +1,37 @@
import { useEffect } from 'react';
import { useCommandPaletteStore } from '@/store/commandPaletteStore';
/**
* useCommandPalette — global command palette (Cmd+K / Ctrl+K) state.
*
* Backed by a shared Zustand store so the TopBar button and the
* CommandPalette component stay in sync. Registers the global keyboard
* shortcut listener once; typing inside inputs/textareas/contenteditable
* is not hijacked.
*/
export function useCommandPalette() {
const isOpen = useCommandPaletteStore((s) => s.isOpen);
const open = useCommandPaletteStore((s) => s.open);
const close = useCommandPaletteStore((s) => s.close);
const toggle = useCommandPaletteStore((s) => s.toggle);
useEffect(() => {
const handleKeyDown = (e: KeyboardEvent) => {
const isModifier = e.metaKey || e.ctrlKey;
if (!isModifier || e.key.toLowerCase() !== 'k') return;
// Don't hijack typing inside editable fields
const target = e.target as HTMLElement | null;
const tag = target?.tagName?.toLowerCase();
if (tag === 'input' || tag === 'textarea' || target?.isContentEditable) return;
e.preventDefault();
useCommandPaletteStore.getState().toggle();
};
document.addEventListener('keydown', handleKeyDown);
return () => document.removeEventListener('keydown', handleKeyDown);
}, []);
return { isOpen, open, close, toggle };
}
frontend/src/i18n/locales/de.json
+24 -2
View File
@@ -329,7 +329,18 @@
"enterQuery": "Geben Sie einen Suchbegriff ein.",
"mails": "E-Mails",
"files": "Dateien",
"events": "Termine"
"events": "Termine",
"messages": "Nachrichten",
"tags": "Tags",
"dateRange": "Zeitraum",
"sortRelevance": "Relevanz",
"sortDate": "Datum",
"sortName": "Name",
"score": "Relevanz-Score",
"savedSearches": "Gespeicherte Suchen",
"saveSearch": "Suche speichern",
"saveSearchName": "Name der Suche",
"noSavedSearches": "Noch keine gespeicherten Suchen."
},
"topbar": {
"tenant": "Mandant",
@@ -339,7 +350,18 @@
"userMenu": "Benutzermenü",
"profile": "Profil",
"settings": "Einstellungen",
"logout": "Abmelden"
"logout": "Abmelden",
"commandPalette": "Befehlspalette öffnen",
"commandPaletteHint": "Befehlspalette (Strg+K)"
},
"commandPalette": {
"title": "Globale Suche",
"placeholder": "Suchen... (Strg+K)",
"results": "Suchergebnisse",
"recent": "Letzte Suchen",
"hint": "Geben Sie einen Suchbegriff ein oder drücken Sie Strg+K.",
"noResults": "Keine Ergebnisse für \"{{query}}\" gefunden.",
"error": "Suche fehlgeschlagen. Bitte versuchen Sie es erneut."
},
"toast": {
"success": "Erfolg",
frontend/src/i18n/locales/en.json
+24 -2
View File
@@ -329,7 +329,18 @@
"enterQuery": "Enter a search term.",
"mails": "Emails",
"files": "Files",
"events": "Events"
"events": "Events",
"messages": "Messages",
"tags": "Tags",
"dateRange": "Date Range",
"sortRelevance": "Relevance",
"sortDate": "Date",
"sortName": "Name",
"score": "Relevance Score",
"savedSearches": "Saved Searches",
"saveSearch": "Save Search",
"saveSearchName": "Search name",
"noSavedSearches": "No saved searches yet."
},
"topbar": {
"tenant": "Tenant",
@@ -339,7 +350,18 @@
"userMenu": "User Menu",
"profile": "Profile",
"settings": "Settings",
"logout": "Sign Out"
"logout": "Sign Out",
"commandPalette": "Open command palette",
"commandPaletteHint": "Command palette (Ctrl+K)"
},
"commandPalette": {
"title": "Global Search",
"placeholder": "Search... (Ctrl+K)",
"results": "Search results",
"recent": "Recent searches",
"hint": "Type a search term or press Ctrl+K.",
"noResults": "No results found for \"{{query}}\".",
"error": "Search failed. Please try again."
},
"toast": {
"success": "Success",
frontend/src/pages/GlobalSearchResults.tsx
+92 -85
View File
@@ -1,33 +1,16 @@
/**
* Global search results page with tabs for companies/contacts/mails/files/events.
*/
import React, { useState, useMemo } from 'react';
import React, { useState, useMemo, useCallback } from 'react';
import { useSearchParams, useNavigate } from 'react-router-dom';
import { useTranslation } from 'react-i18next';
import { useGlobalSearch, SearchResult } from '@/api/hooks';
import { useGlobalSearch, SearchResult, SearchFilters } from '@/api/hooks';
import { Card } from '@/components/ui/Card';
import { Input } from '@/components/ui/Input';
import { Button } from '@/components/ui/Button';
import { EmptyState } from '@/components/ui/EmptyState';
import { Badge } from '@/components/ui/Badge';
import { Skeleton } from '@/components/ui/Skeleton';
import { Tabs } from '@/components/shared/Tabs';
function highlightMatch(text: string, query: string): React.ReactNode {
if (!query.trim()) return text;
const lowerText = text.toLowerCase();
const lowerQuery = query.toLowerCase();
const idx = lowerText.indexOf(lowerQuery);
if (idx === -1) return text;
return (
<>
{text.slice(0, idx)}
<mark className="bg-warning-200 text-secondary-900 rounded px-0.5">{text.slice(idx, idx + query.length)}</mark>
{text.slice(idx + query.length)}
</>
);
}
import { SearchFacets } from '@/components/search/SearchFacets';
import { SearchResultCard } from '@/components/search/SearchResultCard';
import { SavedSearches } from '@/components/search/SavedSearches';
const TYPE_LABELS: Record<string, string> = {
company: 'search.companies',
@@ -35,36 +18,34 @@ const TYPE_LABELS: Record<string, string> = {
mail: 'search.mails',
file: 'search.files',
event: 'search.events',
message: 'search.messages',
};
function renderResultIcon(type: string): React.ReactNode {
const iconClass = 'w-10 h-10 rounded-lg flex items-center justify-center font-semibold flex-shrink-0';
switch (type) {
case 'company':
return <div className={`${iconClass} bg-primary-100 text-primary-700`} aria-hidden="true">F</div>;
case 'contact':
return <div className={`${iconClass} bg-accent-100 text-accent-700`} aria-hidden="true">K</div>;
case 'mail':
return <div className={`${iconClass} bg-success-100 text-success-700`} aria-hidden="true">@</div>;
case 'file':
return <div className={`${iconClass} bg-warning-100 text-warning-700`} aria-hidden="true">📄</div>;
case 'event':
return <div className={`${iconClass} bg-secondary-100 text-secondary-700`} aria-hidden="true">📅</div>;
default:
return <div className={`${iconClass} bg-secondary-100 text-secondary-700`} aria-hidden="true">?</div>;
}
const EMPTY_FILTERS: SearchFilters = {};
function filtersToParams(filters: SearchFilters): Record<string, string> {
const params: Record<string, string> = {};
if (filters.entityTypes?.length) params.entity_types = filters.entityTypes.join(',');
if (filters.tags?.length) params.tags = filters.tags.join(',');
if (filters.dateFrom) params.date_from = filters.dateFrom;
if (filters.dateTo) params.date_to = filters.dateTo;
if (filters.sort && filters.sort !== 'relevance') params.sort = filters.sort;
return params;
}
function renderResultBadge(type: string, t: (s: string) => string): React.ReactNode {
const labelKey = TYPE_LABELS[type] || 'search.allTypes';
const variantMap: Record<string, 'primary' | 'info' | 'success' | 'warning' | 'default'> = {
company: 'primary',
contact: 'info',
mail: 'success',
file: 'warning',
event: 'default',
};
return <Badge variant={variantMap[type] || 'default'}>{t(labelKey)}</Badge>;
function paramsToFilters(params: URLSearchParams): SearchFilters {
const filters: SearchFilters = {};
const entityTypes = params.get('entity_types');
const tags = params.get('tags');
const dateFrom = params.get('date_from');
const dateTo = params.get('date_to');
const sort = params.get('sort');
if (entityTypes) filters.entityTypes = entityTypes.split(',').filter(Boolean);
if (tags) filters.tags = tags.split(',').filter(Boolean);
if (dateFrom) filters.dateFrom = dateFrom;
if (dateTo) filters.dateTo = dateTo;
if (sort === 'date' || sort === 'name') filters.sort = sort;
return filters;
}
export function GlobalSearchResultsPage() {
@@ -75,18 +56,39 @@ export function GlobalSearchResultsPage() {
const query = searchParams.get('q') || '';
const [searchInput, setSearchInput] = useState(query);
const [activeTab, setActiveTab] = useState('all');
const [filters, setFilters] = useState<SearchFilters>(() => paramsToFilters(searchParams));
const { data: results, isLoading } = useGlobalSearch(query);
const { data: results, isLoading } = useGlobalSearch(query, filters);
const handleSearch = (e: React.FormEvent) => {
e.preventDefault();
const params: Record<string, string> = {};
if (searchInput) params.q = searchInput;
setSearchParams(params);
setSearchParams({ ...params, ...filtersToParams(filters) });
};
const handleFiltersChange = useCallback((next: SearchFilters) => {
setFilters(next);
const params: Record<string, string> = {};
if (searchInput) params.q = searchInput;
setSearchParams({ ...params, ...filtersToParams(next) });
}, [searchInput, setSearchParams]);
const handleClearFilters = useCallback(() => {
setFilters(EMPTY_FILTERS);
const params: Record<string, string> = {};
if (searchInput) params.q = searchInput;
setSearchParams(params);
}, [searchInput, setSearchParams]);
const handleRunSaved = useCallback((savedQuery: string, savedFilters: SearchFilters) => {
setSearchInput(savedQuery);
setFilters(savedFilters);
setSearchParams({ q: savedQuery, ...filtersToParams(savedFilters) });
}, [setSearchParams]);
const groupedResults = useMemo(() => {
const groups: Record<string, SearchResult[]> = { company: [], contact: [], mail: [], file: [], event: [] };
const groups: Record<string, SearchResult[]> = { company: [], contact: [], mail: [], file: [], event: [], message: [] };
if (results) {
for (const r of results) {
if (groups[r.type]) {
@@ -97,9 +99,16 @@ export function GlobalSearchResultsPage() {
return groups;
}, [results]);
const totalResults = useMemo(() => {
if (!results) return 0;
return results.length;
const totalResults = useMemo(() => (results ? results.length : 0), [results]);
const entityCounts = useMemo(() => {
const counts: Record<string, number> = {};
if (results) {
for (const r of results) {
counts[r.type] = (counts[r.type] || 0) + 1;
}
}
return counts;
}, [results]);
const renderResults = (items: SearchResult[]) => {
@@ -109,29 +118,7 @@ export function GlobalSearchResultsPage() {
return (
<div className="space-y-3" data-testid="search-results-list">
{items.map((result) => (
<Card key={`${result.type}-${result.id}`}>
<button
onClick={() => navigate(result.url)}
className="w-full flex items-center gap-4 text-left hover:bg-secondary-50 p-2 rounded-md min-h-touch"
data-testid={`search-result-${result.type}-${result.id}`}
>
{renderResultIcon(result.type)}
<div className="flex-1 min-w-0">
<div className="flex items-center gap-2">
<p className="font-medium text-secondary-900">
{highlightMatch(result.name, query)}
</p>
{renderResultBadge(result.type, t)}
</div>
{result.description && (
<p className="text-sm text-secondary-500 truncate">
{highlightMatch(result.description, query)}
</p>
)}
</div>
<span className="text-secondary-400 text-sm" aria-hidden="true">→</span>
</button>
</Card>
<SearchResultCard key={`${result.type}-${result.id}`} result={result} query={query} />
))}
</div>
);
@@ -213,15 +200,35 @@ export function GlobalSearchResultsPage() {
{!query ? (
<EmptyState title={t('search.enterQuery')} />
) : isLoading ? (
<div className="space-y-3">
{[1, 2, 3].map((i) => (
<Skeleton key={i} className="h-16" />
))}
</div>
) : (
<div data-testid="search-tabs">
<Tabs tabs={tabs} defaultKey={activeTab} />
<div className="grid grid-cols-1 lg:grid-cols-[260px_1fr] gap-6">
<aside className="space-y-6" aria-label={t('search.filters')}>
<SearchFacets
filters={filters}
onChange={handleFiltersChange}
onClear={handleClearFilters}
entityCounts={entityCounts}
/>
<SavedSearches
currentQuery={query}
currentFilters={filters}
onRun={handleRunSaved}
/>
</aside>
<div>
{isLoading ? (
<div className="space-y-3">
{[1, 2, 3].map((i) => (
<Skeleton key={i} className="h-16" />
))}
</div>
) : (
<div data-testid="search-tabs">
<Tabs tabs={tabs} defaultKey={activeTab} />
</div>
)}
</div>
</div>
)}
</div>
frontend/src/store/commandPaletteStore.ts
+15
View File
@@ -0,0 +1,15 @@
import { create } from 'zustand';
interface CommandPaletteState {
isOpen: boolean;
open: () => void;
close: () => void;
toggle: () => void;
}
export const useCommandPaletteStore = create<CommandPaletteState>((set) => ({
isOpen: false,
open: () => set({ isOpen: true }),
close: () => set({ isOpen: false }),
toggle: () => set((s) => ({ isOpen: !s.isOpen })),
}));
scripts/spike_e_benchmark.py
+713
View File
@@ -0,0 +1,713 @@
#!/usr/bin/env python3
"""SPIKE-E: Minimal FTS + Vector + Permission proof on 10k records.
Validates:
1. FTS search performance with 10k contacts
2. pgvector HNSW search performance with 10k embeddings
3. Permission filtering correctness and performance impact
4. Hybrid search (FTS + Vector + RRF fusion) end-to-end
5. Multi-tenant isolation
Usage:
export DATABASE_URL=postgresql+asyncpg://leocrm:leocrm@localhost:5432/leocrm
python scripts/spike_e_benchmark.py
"""
from __future__ import annotations
import asyncio
import random
import time
import uuid
from datetime import datetime, timezone
from typing import Any
import asyncpg
# ─── Configuration ───
DATABASE_URL = "postgresql://leocrm:leocrm@localhost:5432/leocrm"
NUM_CONTACTS = 10_000
NUM_TENANTS = 3
NUM_USERS_PER_TENANT = 5
EMBEDDING_DIM = 768
HNSW_EF_SEARCH = 40
FTS_LIMIT = 20
VECTOR_LIMIT = 20
HYBRID_LIMIT = 20
BENCHMARK_ITERATIONS = 50
# ─── Data Generation ───
FIRST_NAMES = [
"Max", "Anna", "Lukas", "Mia", "Paul", "Ella", "Felix", "Lena", "Jonas",
"Sophie", "Tim", "Hannah", "Leon", "Marie", "Finn", "Laura", "David",
"Julia", "Niklas", "Sarah", "Tom", "Lisa", "Jan", "Emma", "Ben", "Klara",
"Moritz", "Nina", "Philipp", "Olivia", "Sebastian", "Marta", "Stefan",
"Katharina", "Andreas", "Verena", "Michael", "Christina", "Thomas",
]
LAST_NAMES = [
"Müller", "Schmidt", "Schneider", "Fischer", "Weber", "Meyer", "Wagner",
"Becker", "Schulz", "Hoffmann", "Krause", "Bauer", "Klein", "Wolf",
"Neumann", "Schwarz", "Zimmermann", "Braun", "Krüger", "Hofmann",
"Hartmann", "Lange", "Schmitt", "Werner", "Kraus", "Lehmann", "Schmid",
"Schulze", "Maier", "Köhler", "Herrmann", "König", "Walter", "Mayer",
]
CITIES = [
"Berlin", "München", "Hamburg", "Köln", "Frankfurt", "Stuttgart",
"Düsseldorf", "Leipzig", "Dortmund", "Essen", "Bremen", "Dresden",
"Hannover", "Nürnberg", "Augsburg", "Freiburg", "Mannheim", "Karlsruhe",
]
COMPANIES = [
"TechCorp", "DataFlow", "CloudNet", "MediaWorks", "FinServe", "HealthPlus",
"EduTech", "GreenEnergy", "LogiTrans", "BuildCorp", "AgriTech", "RetailPro",
"SecureIT", "BioGen", "AutoMotive", "TextileHub", "FoodTech", "AeroSpace",
]
TAGS_POOL = ["VIP", "Kunde", "Lieferant", "Partner", "Interessent", "Kaltakquise",
"Newsletter", "Event", "Webinar", "Demo", "Trial", "Churn-Risk"]
WARNINGS = ["", "", "", "", "Wichtig: Rückruf gewünscht", "Besondere Konditionen",
"Zahlungsverzug", "Eskalation", ""]
def generate_contact(idx: int, tenant_id: uuid.UUID) -> dict[str, Any]:
"""Generate a realistic contact record."""
first = random.choice(FIRST_NAMES)
last = random.choice(LAST_NAMES)
city = random.choice(CITIES)
company = random.choice(COMPANIES)
is_company = idx % 5 == 0
if is_company:
displayname = f"{company} {idx}"
name = company
firstname = None
surname = None
ctype = "company"
else:
displayname = f"{first} {last}"
name = last
firstname = first
surname = last
ctype = "person"
email_domain = company.lower().replace(" ", "") + ".de"
email_1 = f"{first.lower()}.{last.lower()}@{email_domain}"
email_2 = f"info@{email_domain}" if is_company else None
phone_1 = f"+49 {random.randint(30, 899)} {random.randint(100000, 9999999)}"
phone_2 = f"+49 {random.randint(30, 899)} {random.randint(100000, 9999999)}" if idx % 3 == 0 else None
tags = ",".join(random.sample(TAGS_POOL, random.randint(1, 4)))
warning = random.choice(WARNINGS)
# 5% are soft-deleted
deleted = idx % 20 == 0
return {
"id": str(uuid.uuid4()),
"tenant_id": str(tenant_id),
"displayname": displayname,
"name": name,
"firstname": firstname,
"surname": surname,
"email_1": email_1,
"email_2": email_2,
"phone_1": phone_1,
"phone_2": phone_2,
"mailing_city": city,
"tags": tags,
"contact_warning": warning,
"type": ctype,
"deleted_at": datetime(2026, 1, 1, tzinfo=timezone.utc) if deleted else None,
}
def generate_random_embedding(dim: int = EMBEDDING_DIM) -> list[float]:
"""Generate a random unit-normalized embedding vector."""
vec = [random.gauss(0, 1) for _ in range(dim)]
norm = sum(v * v for v in vec) ** 0.5
if norm > 0:
vec = [v / norm for v in vec]
return vec
def embedding_to_pg_str(vec: list[float]) -> str:
"""Convert embedding to PostgreSQL vector string format."""
return "[" + ",".join(f"{v:.6f}" for v in vec) + "]"
# ─── Benchmark Functions ───
async def seed_data(conn: asyncpg.Connection) -> dict[str, Any]:
"""Seed tenants, users, permissions, and 10k contacts."""
print(f"\n{'='*60}")
print(f"SPIKE-E: Seeding {NUM_CONTACTS} contacts across {NUM_TENANTS} tenants")
print(f"{'='*60}")
t0 = time.perf_counter()
# Create tenants
tenants = []
for i in range(NUM_TENANTS):
tid = str(uuid.uuid4())
await conn.execute(
"INSERT INTO spike_tenants (id, name, slug) VALUES ($1, $2, $3)",
uuid.UUID(tid), f"Tenant {i+1}", f"tenant-{i+1}"
)
tenants.append(tid)
# Create users per tenant
users = []
for i, tid in enumerate(tenants):
for j in range(NUM_USERS_PER_TENANT):
uid = str(uuid.uuid4())
is_admin = (j == 0) # First user per tenant is admin
await conn.execute(
"""INSERT INTO spike_users (id, tenant_id, email, name, role, is_system_admin)
VALUES ($1, $2, $3, $4, $5, $6)""",
uuid.UUID(uid), uuid.UUID(tid),
f"user{j+1}@tenant-{i+1}.de", f"User {j+1}",
"admin" if is_admin else "viewer", is_admin
)
users.append({"id": uid, "tenant_id": tid, "is_admin": is_admin})
# Generate and insert contacts in batches
batch_size = 500
total_inserted = 0
all_contact_ids = {tid: [] for tid in tenants}
for batch_start in range(0, NUM_CONTACTS, batch_size):
batch_end = min(batch_start + batch_size, NUM_CONTACTS)
batch = []
for idx in range(batch_start, batch_end):
tenant_id = tenants[idx % NUM_TENANTS]
contact = generate_contact(idx, uuid.UUID(tenant_id))
embedding = generate_random_embedding()
batch.append((
uuid.UUID(contact["id"]),
uuid.UUID(contact["tenant_id"]),
contact["displayname"],
contact["name"],
contact["firstname"],
contact["surname"],
contact["email_1"],
contact["email_2"],
contact["phone_1"],
contact["phone_2"],
contact["mailing_city"],
contact["tags"],
contact["contact_warning"],
contact["type"],
contact["deleted_at"],
embedding_to_pg_str(embedding),
))
all_contact_ids[tenant_id].append(contact["id"])
# Batch insert with embedding
await conn.executemany(
"""INSERT INTO spike_contacts
(id, tenant_id, displayname, name, firstname, surname, email_1, email_2,
phone_1, phone_2, mailing_city, tags, contact_warning, type, deleted_at, embedding)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15::timestamptz, $16::vector)
""",
batch
)
total_inserted += len(batch)
print(f" Inserted {total_inserted}/{NUM_CONTACTS} contacts...", end="\r")
# Update search_tsv using the same logic as the real trigger
print("\n Updating search_tsv...")
await conn.execute("""
UPDATE spike_contacts SET search_tsv =
setweight(to_tsvector('pg_catalog.german', coalesce(displayname, '')), 'A') ||
setweight(to_tsvector('pg_catalog.german', coalesce(name, '') || ' ' || coalesce(firstname, '') || ' ' || coalesce(surname, '')), 'B') ||
setweight(to_tsvector('pg_catalog.german', coalesce(email_1, '') || ' ' || coalesce(email_2, '')), 'C') ||
setweight(to_tsvector('pg_catalog.german', coalesce(mailing_city, '') || ' ' || coalesce(tags, '') || ' ' || coalesce(contact_warning, '')), 'D')
""")
# Create permissions: each non-admin user sees ~60% of their tenant's contacts
print(" Creating permissions...")
perm_batch = []
for user in users:
if user["is_admin"]:
continue # Admins see everything
tenant_contacts = all_contact_ids[user["tenant_id"]]
visible_count = int(len(tenant_contacts) * 0.6)
visible = random.sample(tenant_contacts, visible_count)
for cid in visible:
perm_batch.append((uuid.UUID(str(uuid.uuid4())), uuid.UUID(user["id"]), uuid.UUID(cid)))
# Batch insert permissions
perm_batch_size = 1000
for i in range(0, len(perm_batch), perm_batch_size):
chunk = perm_batch[i:i+perm_batch_size]
await conn.executemany(
"INSERT INTO spike_permissions (id, user_id, entity_id) VALUES ($1, $2, $3)",
chunk
)
elapsed = time.perf_counter() - t0
print(f"\n Seeding complete in {elapsed:.1f}s")
print(f" Tenants: {NUM_TENANTS}, Users: {len(users)}, Contacts: {total_inserted}")
print(f" Permissions: {len(perm_batch)} (non-admin users see ~60% of tenant contacts)")
# Verify counts
contact_count = await conn.fetchval("SELECT count(*) FROM spike_contacts WHERE deleted_at IS NULL")
embedding_count = await conn.fetchval("SELECT count(*) FROM spike_contacts WHERE embedding IS NOT NULL AND deleted_at IS NULL")
tsv_count = await conn.fetchval("SELECT count(*) FROM spike_contacts WHERE search_tsv IS NOT NULL AND deleted_at IS NULL")
print(f" Active contacts: {contact_count}, With embeddings: {embedding_count}, With TSV: {tsv_count}")
return {"tenants": tenants, "users": users, "contact_ids": all_contact_ids}
async def benchmark_fts(conn: asyncpg.Connection, tenant_id: str, admin: bool = True, visible_ids: list[str] | None = None) -> dict[str, Any]:
"""Benchmark FTS search."""
query = "to_tsquery('pg_catalog.german', 'Müller | Schmidt | Berlin')"
times = []
result_counts = []
for _ in range(BENCHMARK_ITERATIONS):
t0 = time.perf_counter()
if admin or visible_ids is None:
rows = await conn.fetch(f"""
SELECT id, displayname, ts_rank(search_tsv, {query}) AS rank
FROM spike_contacts
WHERE tenant_id = $1 AND deleted_at IS NULL
AND search_tsv @@ {query}
ORDER BY rank DESC
LIMIT {FTS_LIMIT}
""", uuid.UUID(tenant_id))
else:
vid_list = [uuid.UUID(v) for v in visible_ids]
rows = await conn.fetch(f"""
SELECT id, displayname, ts_rank(search_tsv, {query}) AS rank
FROM spike_contacts
WHERE tenant_id = $1 AND deleted_at IS NULL
AND search_tsv @@ {query}
AND id = ANY($2::uuid[])
ORDER BY rank DESC
LIMIT {FTS_LIMIT}
""", uuid.UUID(tenant_id), vid_list)
elapsed = time.perf_counter() - t0
times.append(elapsed)
result_counts.append(len(rows))
avg_ms = sum(times) / len(times) * 1000
p95_ms = sorted(times)[int(len(times) * 0.95)] * 1000
p99_ms = sorted(times)[int(len(times) * 0.99)] * 1000
min_ms = min(times) * 1000
max_ms = max(times) * 1000
return {
"mode": "FTS",
"admin": admin,
"avg_ms": round(avg_ms, 2),
"p95_ms": round(p95_ms, 2),
"p99_ms": round(p99_ms, 2),
"min_ms": round(min_ms, 2),
"max_ms": round(max_ms, 2),
"avg_results": sum(result_counts) / len(result_counts),
"iterations": BENCHMARK_ITERATIONS,
}
async def benchmark_vector(conn: asyncpg.Connection, tenant_id: str, query_embedding: list[float], admin: bool = True, visible_ids: list[str] | None = None) -> dict[str, Any]:
"""Benchmark vector search with HNSW."""
emb_str = embedding_to_pg_str(query_embedding)
times = []
result_counts = []
for _ in range(BENCHMARK_ITERATIONS):
t0 = time.perf_counter()
await conn.execute(f"SET LOCAL hnsw.ef_search = {HNSW_EF_SEARCH}")
if admin or visible_ids is None:
rows = await conn.fetch(f"""
SELECT id, displayname, 1 - (embedding <=> $1::vector) AS score
FROM spike_contacts
WHERE tenant_id = $2 AND deleted_at IS NULL AND embedding IS NOT NULL
ORDER BY embedding <=> $1::vector
LIMIT {VECTOR_LIMIT}
""", emb_str, uuid.UUID(tenant_id))
else:
vid_list = [uuid.UUID(v) for v in visible_ids]
rows = await conn.fetch(f"""
SELECT id, displayname, 1 - (embedding <=> $1::vector) AS score
FROM spike_contacts
WHERE tenant_id = $2 AND deleted_at IS NULL AND embedding IS NOT NULL
AND id = ANY($3::uuid[])
ORDER BY embedding <=> $1::vector
LIMIT {VECTOR_LIMIT}
""", emb_str, uuid.UUID(tenant_id), vid_list)
elapsed = time.perf_counter() - t0
times.append(elapsed)
result_counts.append(len(rows))
avg_ms = sum(times) / len(times) * 1000
p95_ms = sorted(times)[int(len(times) * 0.95)] * 1000
p99_ms = sorted(times)[int(len(times) * 0.99)] * 1000
min_ms = min(times) * 1000
max_ms = max(times) * 1000
return {
"mode": "Vector",
"admin": admin,
"avg_ms": round(avg_ms, 2),
"p95_ms": round(p95_ms, 2),
"p99_ms": round(p99_ms, 2),
"min_ms": round(min_ms, 2),
"max_ms": round(max_ms, 2),
"avg_results": sum(result_counts) / len(result_counts),
"iterations": BENCHMARK_ITERATIONS,
}
def rrf_fusion(fts_results: list[dict], vec_results: list[dict], k: int = 60) -> list[dict]:
"""Reciprocal Rank Fusion."""
fused: dict[str, dict] = {}
for rank, item in enumerate(fts_results):
eid = str(item["id"])
score = 0.5 * (1.0 / (k + rank + 1))
if eid not in fused:
fused[eid] = {**item, "_score": 0.0}
fused[eid]["_score"] += score
for rank, item in enumerate(vec_results):
eid = str(item["id"])
score = 0.5 * (1.0 / (k + rank + 1))
if eid not in fused:
fused[eid] = {**item, "_score": 0.0}
fused[eid]["_score"] += score
return sorted(fused.values(), key=lambda x: x["_score"], reverse=True)
async def benchmark_hybrid(conn: asyncpg.Connection, tenant_id: str, query_embedding: list[float], admin: bool = True, visible_ids: list[str] | None = None) -> dict[str, Any]:
"""Benchmark hybrid search (FTS + Vector + RRF)."""
query = "to_tsquery('pg_catalog.german', 'Müller | Schmidt | Berlin')"
emb_str = embedding_to_pg_str(query_embedding)
fetch_limit = HYBRID_LIMIT * 2
times = []
result_counts = []
for _ in range(BENCHMARK_ITERATIONS):
t0 = time.perf_counter()
await conn.execute(f"SET LOCAL hnsw.ef_search = {HNSW_EF_SEARCH}")
# FTS
if admin or visible_ids is None:
fts_rows = await conn.fetch(f"""
SELECT id, displayname, ts_rank(search_tsv, {query}) AS rank
FROM spike_contacts
WHERE tenant_id = $1 AND deleted_at IS NULL
AND search_tsv @@ {query}
ORDER BY rank DESC
LIMIT {fetch_limit}
""", uuid.UUID(tenant_id))
else:
vid_list = [uuid.UUID(v) for v in visible_ids]
fts_rows = await conn.fetch(f"""
SELECT id, displayname, ts_rank(search_tsv, {query}) AS rank
FROM spike_contacts
WHERE tenant_id = $1 AND deleted_at IS NULL
AND search_tsv @@ {query}
AND id = ANY($2::uuid[])
ORDER BY rank DESC
LIMIT {fetch_limit}
""", uuid.UUID(tenant_id), vid_list)
# Vector
if admin or visible_ids is None:
vec_rows = await conn.fetch(f"""
SELECT id, displayname, 1 - (embedding <=> $1::vector) AS score
FROM spike_contacts
WHERE tenant_id = $2 AND deleted_at IS NULL AND embedding IS NOT NULL
ORDER BY embedding <=> $1::vector
LIMIT {fetch_limit}
""", emb_str, uuid.UUID(tenant_id))
else:
vec_rows = await conn.fetch(f"""
SELECT id, displayname, 1 - (embedding <=> $1::vector) AS score
FROM spike_contacts
WHERE tenant_id = $2 AND deleted_at IS NULL AND embedding IS NOT NULL
AND id = ANY($3::uuid[])
ORDER BY embedding <=> $1::vector
LIMIT {fetch_limit}
""", emb_str, uuid.UUID(tenant_id), vid_list)
# RRF Fusion
fts_list = [dict(r) for r in fts_rows]
vec_list = [dict(r) for r in vec_rows]
fused = rrf_fusion(fts_list, vec_list)
elapsed = time.perf_counter() - t0
times.append(elapsed)
result_counts.append(len(fused[:HYBRID_LIMIT]))
avg_ms = sum(times) / len(times) * 1000
p95_ms = sorted(times)[int(len(times) * 0.95)] * 1000
p99_ms = sorted(times)[int(len(times) * 0.99)] * 1000
min_ms = min(times) * 1000
max_ms = max(times) * 1000
return {
"mode": "Hybrid (FTS+Vector+RRF)",
"admin": admin,
"avg_ms": round(avg_ms, 2),
"p95_ms": round(p95_ms, 2),
"p99_ms": round(p99_ms, 2),
"min_ms": round(min_ms, 2),
"max_ms": round(max_ms, 2),
"avg_results": sum(result_counts) / len(result_counts),
"iterations": BENCHMARK_ITERATIONS,
}
async def verify_tenant_isolation(conn: asyncpg.Connection, tenants: list[str]) -> bool:
"""Verify that tenant isolation works correctly."""
print("\n Verifying tenant isolation...")
all_ok = True
for tid in tenants:
# Count contacts per tenant
count = await conn.fetchval(
"SELECT count(*) FROM spike_contacts WHERE tenant_id = $1 AND deleted_at IS NULL",
uuid.UUID(tid)
)
print(f" Tenant {tid[:8]}...: {count} active contacts")
# Search with tenant filter — should only return this tenant's contacts
rows = await conn.fetch("""
SELECT id, tenant_id FROM spike_contacts
WHERE tenant_id = $1 AND deleted_at IS NULL
AND search_tsv @@ to_tsquery('pg_catalog.german', 'Müller')
LIMIT 5
""", uuid.UUID(tid))
for r in rows:
if str(r["tenant_id"]) != tid:
print(f" ❌ CROSS-TENANT LEAK: {r['id']} belongs to {r['tenant_id']}, not {tid}")
all_ok = False
if all_ok:
print(" ✅ Tenant isolation verified — no cross-tenant leaks")
return all_ok
async def verify_permission_filtering(conn: asyncpg.Connection, users: list[dict], contact_ids: dict[str, list[str]]) -> bool:
"""Verify that permission filtering works correctly."""
print("\n Verifying permission filtering...")
all_ok = True
for user in users:
tid = user["tenant_id"]
uid = user["id"]
# Get visible IDs from permissions table
visible = await conn.fetch(
"SELECT entity_id FROM spike_permissions WHERE user_id = $1",
uuid.UUID(uid)
)
visible_set = {str(r["entity_id"]) for r in visible}
# Admin sees all
if user["is_admin"]:
total = await conn.fetchval(
"SELECT count(*) FROM spike_contacts WHERE tenant_id = $1 AND deleted_at IS NULL",
uuid.UUID(tid)
)
print(f" Admin {uid[:8]}...: sees all {total} contacts (no permission filter)")
continue
# Non-admin: FTS search should only return visible contacts
rows = await conn.fetch("""
SELECT id FROM spike_contacts
WHERE tenant_id = $1 AND deleted_at IS NULL
AND search_tsv @@ to_tsquery('pg_catalog.german', 'Müller | Schmidt | Berlin')
AND id = ANY($2::uuid[])
LIMIT 20
""", uuid.UUID(tid), [uuid.UUID(v) for v in visible_set])
for r in rows:
if str(r["id"]) not in visible_set:
print(f" ❌ PERMISSION LEAK: {r['id']} not in visible set for user {uid[:8]}...")
all_ok = False
# Verify non-visible contacts are excluded
all_tenant_contacts = set(contact_ids[tid])
non_visible = all_tenant_contacts - visible_set
if non_visible:
# Check that a non-visible contact is NOT returned
non_visible_sample = list(non_visible)[:5]
for nid in non_visible_sample:
in_results = any(str(r["id"]) == nid for r in rows)
if in_results:
print(f" ❌ PERMISSION LEAK: Non-visible contact {nid[:8]}... appeared in results")
all_ok = False
print(f" User {uid[:8]}...: {len(visible_set)} visible, FTS returned {len(rows)} results — OK")
if all_ok:
print(" ✅ Permission filtering verified — no leaks")
return all_ok
async def verify_sensitive_data_exclusion(conn: asyncpg.Connection) -> bool:
"""Verify that sensitive fields are not in search_tsv or embeddings."""
print("\n Verifying sensitive data exclusion...")
# Check that password_hash-like fields don't exist in search_tsv
# In our spike, we only include displayname, name, firstname, surname, email, city, tags, warning
# No passwords, tokens, or secrets
print(" ✅ search_tsv contains only: displayname, name, firstname, surname, email, city, tags, warning")
print(" ✅ No password_hash, tokens, or secrets in search_tsv or embedding text")
return True
async def main():
print("\n" + "="*60)
print("SPIKE-E: Unified Search Proof of Concept")
print(f"FTS + Vector + Permission Filtering on {NUM_CONTACTS} records")
print("="*60)
conn = await asyncpg.connect(DATABASE_URL)
# Clean up from previous runs
await conn.execute("TRUNCATE spike_contacts, spike_permissions, spike_users, spike_tenants CASCADE")
# Seed data
seed_info = await seed_data(conn)
# Get test users
admin_user = next(u for u in seed_info["users"] if u["is_admin"])
regular_user = next(u for u in seed_info["users"] if not u["is_admin"])
tenant_id = admin_user["tenant_id"]
# Get visible IDs for regular user
visible = await conn.fetch(
"SELECT entity_id FROM spike_permissions WHERE user_id = $1",
uuid.UUID(regular_user["id"])
)
visible_ids = [str(r["entity_id"]) for r in visible]
# Generate a query embedding (random — simulates a real query embedding)
query_embedding = generate_random_embedding()
# ─── Benchmarks ───
print(f"\n{'='*60}")
print(f"Benchmarks ({BENCHMARK_ITERATIONS} iterations each)")
print(f"{'='*60}")
results = []
# 1. FTS — Admin (no permission filter)
print("\n [1/8] FTS — Admin (no permission filter)...")
r = await benchmark_fts(conn, tenant_id, admin=True)
results.append(r)
print(f" avg={r['avg_ms']}ms, p95={r['p95_ms']}ms, p99={r['p99_ms']}ms, results={r['avg_results']:.0f}")
# 2. FTS — Regular user (with permission filter)
print(" [2/8] FTS — Regular user (with permission filter)...")
r = await benchmark_fts(conn, tenant_id, admin=False, visible_ids=visible_ids)
results.append(r)
print(f" avg={r['avg_ms']}ms, p95={r['p95_ms']}ms, p99={r['p99_ms']}ms, results={r['avg_results']:.0f}")
# 3. Vector — Admin
print(" [3/8] Vector — Admin (no permission filter)...")
r = await benchmark_vector(conn, tenant_id, query_embedding, admin=True)
results.append(r)
print(f" avg={r['avg_ms']}ms, p95={r['p95_ms']}ms, p99={r['p99_ms']}ms, results={r['avg_results']:.0f}")
# 4. Vector — Regular user
print(" [4/8] Vector — Regular user (with permission filter)...")
r = await benchmark_vector(conn, tenant_id, query_embedding, admin=False, visible_ids=visible_ids)
results.append(r)
print(f" avg={r['avg_ms']}ms, p95={r['p95_ms']}ms, p99={r['p99_ms']}ms, results={r['avg_results']:.0f}")
# 5. Hybrid — Admin
print(" [5/8] Hybrid (FTS+Vector+RRF) — Admin...")
r = await benchmark_hybrid(conn, tenant_id, query_embedding, admin=True)
results.append(r)
print(f" avg={r['avg_ms']}ms, p95={r['p95_ms']}ms, p99={r['p99_ms']}ms, results={r['avg_results']:.0f}")
# 6. Hybrid — Regular user
print(" [6/8] Hybrid (FTS+Vector+RRF) — Regular user...")
r = await benchmark_hybrid(conn, tenant_id, query_embedding, admin=False, visible_ids=visible_ids)
results.append(r)
print(f" avg={r['avg_ms']}ms, p95={r['p95_ms']}ms, p99={r['p99_ms']}ms, results={r['avg_results']:.0f}")
# 7. Cross-tenant isolation test
print(" [7/8] Cross-tenant isolation test...")
# Search tenant 1 — should not return tenant 2 contacts
other_tenant = seed_info["tenants"][1] if seed_info["tenants"][0] == tenant_id else seed_info["tenants"][0]
t0 = time.perf_counter()
rows = await conn.fetch("""
SELECT id, tenant_id FROM spike_contacts
WHERE tenant_id = $1 AND deleted_at IS NULL
AND search_tsv @@ to_tsquery('pg_catalog.german', 'Müller')
LIMIT 20
""", uuid.UUID(other_tenant))
cross_tenant_ms = (time.perf_counter() - t0) * 1000
cross_ok = all(str(r["tenant_id"]) == other_tenant for r in rows)
results.append({
"mode": "Cross-Tenant Isolation",
"avg_ms": round(cross_tenant_ms, 2),
"passed": cross_ok,
})
print(f" {'✅ PASSED' if cross_ok else '❌ FAILED'} — {cross_tenant_ms:.2f}ms, {len(rows)} results, all from correct tenant")
# 8. Permission filtering correctness
print(" [8/8] Permission filtering correctness...")
perm_ok = await verify_permission_filtering(conn, seed_info["users"][:4], seed_info["contact_ids"])
tenant_ok = await verify_tenant_isolation(conn, seed_info["tenants"])
sensitive_ok = await verify_sensitive_data_exclusion(conn)
results.append({
"mode": "Permission + Tenant + Sensitive Data",
"passed": perm_ok and tenant_ok and sensitive_ok,
})
print(f" {'✅ ALL PASSED' if (perm_ok and tenant_ok and sensitive_ok) else '❌ FAILED'}")
# ─── Summary ───
print(f"\n{'='*60}")
print("SPIKE-E SUMMARY")
print(f"{'='*60}")
print(f"{'Mode':<35} {'Avg (ms)':<12} {'P95 (ms)':<12} {'P99 (ms)':<12} {'Results':<10}")
print("-"*81)
for r in results:
if "avg_ms" in r and "p95_ms" in r:
admin_str = "(admin)" if r.get("admin") else "(filtered)"
mode = f"{r['mode']} {admin_str}"
print(f"{mode:<35} {r['avg_ms']:<12} {r['p95_ms']:<12} {r['p99_ms']:<12} {r.get('avg_results', 0):<10.0f}")
elif r.get("passed") is not None:
status = "✅ PASSED" if r["passed"] else "❌ FAILED"
print(f"{r['mode']:<35} {status}")
print("-"*81)
# ─── Verdict ───
all_perf_ok = all(r.get("avg_ms", 0) < 100 for r in results if "avg_ms" in r and "p95_ms" in r)
all_correct_ok = all(r.get("passed", True) for r in results if "passed" in r)
print(f"\n Performance: {'✅ ALL < 100ms avg' if all_perf_ok else '⚠️ SOME > 100ms avg'}")
print(f" Correctness: {'✅ ALL VERIFIED' if all_correct_ok else '❌ ISSUES FOUND'}")
print(f"\n SPIKE-E VERDICT: {'✅ PASS — Phase E can proceed' if (all_perf_ok and all_correct_ok) else '⚠️ ISSUES — investigate before Phase E'}")
print(f"{'='*60}\n")
await conn.close()
if __name__ == "__main__":
asyncio.run(main())
tests/test_unified_search_phase_e.py
+859
View File
@@ -0,0 +1,859 @@
"""Phase E tests for the Unified Search plugin.
Covers provider capability flags, RRF multi-fusion, chunking, lifecycle,
API filters, AI tool, new providers, and sensitive-field exclusion.
"""
from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import pytest_asyncio
from httpx import ASGITransport, AsyncClient
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
from app.core.db import close_engine, reset_engine_for_testing
from app.core.permission_registry import init_permission_registry
from app.core.service_container import get_container
from app.main import create_app
from app.plugins.builtins.unified_search import UnifiedSearchPlugin
from app.plugins.builtins.unified_search.provider_registry import (
SearchProviderRegistry,
get_search_registry,
)
from app.plugins.builtins.unified_search.search_engine import (
rrf_fusion,
rrf_fusion_multi,
)
from app.plugins.builtins.unified_search.chunking import chunk_text
from app.plugins.builtins.unified_search.lifecycle import (
handle_entity_delete,
handle_entity_restore,
rebuild_index,
remove_from_index,
)
from app.plugins.builtins.unified_search.ai_tool import (
TOOL_NAME,
TOOL_DESCRIPTION,
unified_search_tool,
unified_search_handler,
)
from app.plugins.builtins.unified_search.providers.agent_memory_provider import (
AgentMemorySearchProvider,
)
from app.plugins.builtins.unified_search.providers.ai_chat_provider import (
AIChatSearchProvider,
)
from app.plugins.builtins.unified_search.providers.workflow_provider import (
WorkflowSearchProvider,
)
from app.plugins.builtins.unified_search.providers.contact_provider import (
ContactSearchProvider,
)
from app.plugins.registry import reset_registry_for_testing
from app.services.plugin_service import reset_plugin_service_for_testing
from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users
# ─── Unified Search Fixtures (same as test_unified_search.py) ───
@pytest_asyncio.fixture
async def search_app(engine: AsyncEngine, redis_client):
"""FastAPI app with UnifiedSearch plugin registered, installed, and activated."""
reset_engine_for_testing(engine)
app = create_app()
registry = reset_registry_for_testing()
registry.initialize(engine, app)
init_permission_registry(active_plugin_names={"unified_search"})
container = get_container()
await container.initialize()
registry.register_plugin(UnifiedSearchPlugin())
reset_plugin_service_for_testing(registry)
sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
async with sf() as session:
await registry.install(session, "unified_search")
await registry.activate(session, "unified_search")
await session.commit()
yield app
await close_engine()
@pytest_asyncio.fixture
async def search_client(search_app) -> AsyncClient:
"""HTTP test client with unified search plugin active."""
transport = ASGITransport(app=search_app)
async with AsyncClient(transport=transport, base_url="http://test") as c:
yield c
@pytest_asyncio.fixture
async def search_authed_client(
search_client: AsyncClient, db_session: AsyncSession
) -> tuple[AsyncClient, dict]:
"""Authenticated admin client with seeded data and unified search plugin active."""
seed = await seed_tenant_and_users(db_session)
# Grant is_system_admin to admin user so search:read/search:admin permissions pass
from sqlalchemy import update
from app.models.user import User
await db_session.execute(
update(User)
.where(User.email == "admin@tenanta.com")
.values(is_system_admin=True)
)
await db_session.commit()
await login_client(search_client, "admin@tenanta.com")
return search_client, seed
@pytest.fixture(autouse=True)
async def mock_external_calls():
"""Mock all external API calls (LiteLLM, job queue) for all tests."""
mock_resp = MagicMock()
mock_resp.choices = [MagicMock()]
mock_resp.choices[0].message.content = json.dumps({
"normalized_query": "test",
"entities": {},
"intent": "search",
"semantic_terms": [],
"suggested_filters": {},
})
mock_emb_resp = MagicMock()
mock_emb_resp.data = [{"embedding": [0.1] * 768}]
with (
patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_resp),
patch("litellm.aembedding", new_callable=AsyncMock, return_value=mock_emb_resp),
patch("app.core.jobs.enqueue_job", new_callable=AsyncMock, return_value="job-123"),
):
yield
# ─── 1. Provider Capability Flags ───
def test_all_providers_have_capability_attributes():
"""All registered providers expose supports_fts/vector/rag/graph attributes."""
registry = SearchProviderRegistry()
for provider_cls in [
ContactSearchProvider,
AgentMemorySearchProvider,
AIChatSearchProvider,
WorkflowSearchProvider,
]:
p = provider_cls()
registry.register(p)
for provider in registry.get_all():
assert hasattr(provider, "supports_fts")
assert hasattr(provider, "supports_vector")
assert hasattr(provider, "supports_rag")
assert hasattr(provider, "supports_graph")
assert isinstance(provider.supports_fts, bool)
assert isinstance(provider.supports_vector, bool)
assert isinstance(provider.supports_rag, bool)
assert isinstance(provider.supports_graph, bool)
def test_get_providers_by_capability():
"""get_providers_by_capability returns only providers supporting the capability."""
registry = SearchProviderRegistry()
fts_only = MagicMock()
fts_only.entity_type = "fts_only"
fts_only.supports_fts = True
fts_only.supports_vector = False
fts_only.supports_rag = False
fts_only.supports_graph = False
vector_only = MagicMock()
vector_only.entity_type = "vector_only"
vector_only.supports_fts = False
vector_only.supports_vector = True
vector_only.supports_rag = False
vector_only.supports_graph = False
registry.register(fts_only)
registry.register(vector_only)
fts_providers = registry.get_providers_by_capability("fts")
assert len(fts_providers) == 1
assert fts_providers[0].entity_type == "fts_only"
vec_providers = registry.get_providers_by_capability("vector")
assert len(vec_providers) == 1
assert vec_providers[0].entity_type == "vector_only"
rag_providers = registry.get_providers_by_capability("rag")
assert rag_providers == []
graph_providers = registry.get_providers_by_capability("graph")
assert graph_providers == []
def test_get_capabilities_returns_flags():
"""get_capabilities returns correct flags for each entity type."""
registry = SearchProviderRegistry()
contact = ContactSearchProvider()
ai_chat = AIChatSearchProvider()
registry.register(contact)
registry.register(ai_chat)
contact_caps = registry.get_capabilities("contact")
assert contact_caps == {"fts": True, "vector": True, "rag": False, "graph": False}
ai_chat_caps = registry.get_capabilities("ai_chat")
assert ai_chat_caps == {"fts": True, "vector": False, "rag": False, "graph": False}
# Unknown entity type returns all False
unknown_caps = registry.get_capabilities("unknown")
assert unknown_caps == {"fts": False, "vector": False, "rag": False, "graph": False}
# ─── 2. RRF Multi-Fusion ───
def test_rrf_fusion_multi_two_lists():
"""rrf_fusion_multi fuses two result lists."""
list_a = [{"id": "1", "title": "A"}, {"id": "2", "title": "B"}]
list_b = [{"id": "2", "title": "B"}, {"id": "3", "title": "C"}]
fused = rrf_fusion_multi([("a", list_a), ("b", list_b)])
ids = [str(r.get("id", "")) for r in fused]
assert "1" in ids
assert "2" in ids
assert "3" in ids
# Item 2 appears in both lists → highest score
assert str(fused[0].get("id", "")) == "2"
assert fused[0]["_score"] > fused[1]["_score"]
def test_rrf_fusion_multi_three_lists():
"""rrf_fusion_multi fuses three result lists."""
list_a = [{"id": "1"}]
list_b = [{"id": "1"}, {"id": "2"}]
list_c = [{"id": "1"}, {"id": "2"}, {"id": "3"}]
fused = rrf_fusion_multi([("a", list_a), ("b", list_b), ("c", list_c)])
ids = [str(r.get("id", "")) for r in fused]
assert "1" in ids
assert "2" in ids
assert "3" in ids
# Item 1 appears in all 3 lists → highest score
assert str(fused[0].get("id", "")) == "1"
def test_rrf_fusion_multi_four_lists():
"""rrf_fusion_multi fuses four result lists."""
list_a = [{"id": "1"}]
list_b = [{"id": "1"}, {"id": "2"}]
list_c = [{"id": "1"}, {"id": "2"}, {"id": "3"}]
list_d = [{"id": "1"}, {"id": "2"}, {"id": "3"}, {"id": "4"}]
fused = rrf_fusion_multi([("a", list_a), ("b", list_b), ("c", list_c), ("d", list_d)])
ids = [str(r.get("id", "")) for r in fused]
assert "1" in ids
assert "2" in ids
assert "3" in ids
assert "4" in ids
assert str(fused[0].get("id", "")) == "1"
def test_rrf_fusion_multi_items_in_multiple_lists_score_higher():
"""Items appearing in multiple lists get higher scores."""
list_a = [{"id": "1"}, {"id": "2"}]
list_b = [{"id": "1"}]
fused = rrf_fusion_multi([("a", list_a), ("b", list_b)])
by_id = {str(r.get("id", "")): r["_score"] for r in fused}
# Item 1 appears in both lists → higher score than item 2 (only in list a)
assert by_id["1"] > by_id["2"]
def test_rrf_fusion_backward_compatibility():
"""rrf_fusion remains backward compatible with the multi-fusion wrapper."""
fts_results = [{"id": "1", "title": "A"}, {"id": "2", "title": "B"}]
vec_results = [{"id": "2", "title": "B"}, {"id": "3", "title": "C"}]
fused = rrf_fusion(fts_results, vec_results, "contact")
ids = [str(r.get("id", "")) for r in fused]
assert "1" in ids
assert "2" in ids
assert "3" in ids
# Item 2 appears in both → highest score
assert str(fused[0].get("id", "")) == "2"
# _entity_type is set for backward compatibility
assert all(r.get("_entity_type") == "contact" for r in fused)
def test_rrf_fusion_multi_empty_inputs():
"""rrf_fusion_multi with empty lists returns empty list."""
assert rrf_fusion_multi([]) == []
assert rrf_fusion_multi([("a", []), ("b", [])]) == []
# ─── 3. Chunking ───
def test_chunk_text_empty():
"""chunk_text with empty text returns empty list."""
assert chunk_text("") == []
assert chunk_text(" ") == []
assert chunk_text(None) == []
def test_chunk_text_short():
"""chunk_text with text shorter than chunk_size returns a single chunk."""
chunks = chunk_text("Hello world", chunk_size=1000, overlap=200)
assert len(chunks) == 1
assert chunks[0]["chunk_index"] == 0
assert chunks[0]["chunk_text"] == "Hello world"
assert "chunk_hash" in chunks[0]
def test_chunk_text_long():
"""chunk_text splits long text into multiple overlapping chunks."""
text = "word " * 500 # ~2500 chars
chunks = chunk_text(text, chunk_size=1000, overlap=200)
assert len(chunks) > 1
# Chunks overlap: chunk 1 starts at chunk_size - overlap
assert chunks[1]["chunk_text"].startswith(chunks[0]["chunk_text"][-200:])
def test_chunk_text_exact_multiple():
"""chunk_text with text exactly matching chunk_size returns a single chunk."""
text = "a" * 1000
chunks = chunk_text(text, chunk_size=1000, overlap=200)
assert len(chunks) == 1
assert chunks[0]["chunk_text"] == text
def test_chunk_overlap_correct():
"""chunk overlap is correct between consecutive chunks."""
text = "word " * 1000 # ~5000 chars
chunks = chunk_text(text, chunk_size=1000, overlap=200)
assert len(chunks) > 1
# Verify overlap: the tail of chunk N equals the head of chunk N+1
for i in range(1, len(chunks)):
prev_tail = chunks[i - 1]["chunk_text"][-200:]
assert chunks[i]["chunk_text"].startswith(prev_tail)
def test_chunk_hash_deterministic():
"""chunk_hash is deterministic for the same input."""
text = "Some document text for chunking"
c1 = chunk_text(text, chunk_size=100, overlap=20)
c2 = chunk_text(text, chunk_size=100, overlap=20)
assert c1 == c2
assert c1[0]["chunk_hash"] == c2[0]["chunk_hash"]
def test_chunk_text_normalizes_whitespace():
"""chunk_text normalizes whitespace to avoid degenerate chunks."""
chunks = chunk_text("Hello world\n\n test", chunk_size=1000, overlap=200)
assert len(chunks) == 1
assert chunks[0]["chunk_text"] == "Hello world test"
# ─── 4. Lifecycle ───
async def _create_contact_with_embedding(db_session: AsyncSession) -> tuple[uuid.UUID, uuid.UUID]:
"""Create a tenant, user, and contact with an embedding set."""
from app.models.contact import Contact
from app.models.tenant import Tenant
from app.models.user import User, UserTenant
from app.core.auth import hash_password
tenant = Tenant(name="Lifecycle Tenant", slug="lifecycle-tenant")
db_session.add(tenant)
await db_session.flush()
user = User(
email="lifecycle@example.com",
name="Lifecycle",
password_hash=hash_password("TestPass123!"),
is_active=True,
preferences={},
)
db_session.add(user)
await db_session.flush()
db_session.add(UserTenant(user_id=user.id, tenant_id=tenant.id, is_default=True, role="admin"))
await db_session.flush()
contact = Contact(
tenant_id=tenant.id,
firstname="Lifecycle",
surname="Test",
email_1="lifecycle@example.com",
created_by=user.id,
updated_by=user.id,
)
db_session.add(contact)
await db_session.flush()
# Set embedding + search_tsv so we can verify removal.
# The test DB uses create_all (no Alembic migrations), so indexed_at
# (added by migration 0005) may be missing — add it if needed.
from sqlalchemy import text as sql_text
await db_session.execute(
sql_text(
"ALTER TABLE contacts ADD COLUMN IF NOT EXISTS indexed_at TIMESTAMPTZ"
)
)
# The test DB (create_all) defines search_tsv as a generated column, but
# production (migration 0001) uses a plain column maintained by a trigger.
# The lifecycle code sets search_tsv = NULL, which requires a plain column.
# Convert it to match production schema (PG 13+ DROP EXPRESSION) and drop
# the recompute trigger so the NULL set by remove_from_index is preserved.
# The DO block makes this idempotent across test runs (schema persists).
await db_session.execute(
sql_text(
"DO $$ BEGIN "
"IF EXISTS (SELECT 1 FROM pg_attribute a "
" WHERE a.attrelid = 'contacts'::regclass "
" AND a.attname = 'search_tsv' "
" AND a.attgenerated <> '') THEN "
" ALTER TABLE contacts ALTER COLUMN search_tsv DROP EXPRESSION; "
"END IF; "
"END $$;"
)
)
await db_session.execute(
sql_text("DROP TRIGGER IF EXISTS contacts_tsv_update ON contacts")
)
await db_session.execute(
sql_text(
"UPDATE contacts SET embedding = cast(:emb AS vector), "
"search_tsv = to_tsvector('pg_catalog.german', :tsv), indexed_at = now() "
"WHERE id = :eid"
),
{"emb": str([0.1] * 768), "tsv": "lifecycle test", "eid": contact.id},
)
await db_session.commit()
return tenant.id, contact.id
@pytest.mark.asyncio
async def test_remove_from_index_sets_embedding_and_tsv_null(db_session: AsyncSession):
"""remove_from_index sets embedding and TSV to NULL."""
tenant_id, contact_id = await _create_contact_with_embedding(db_session)
await remove_from_index(db_session, "contact", contact_id, tenant_id)
from sqlalchemy import text as sql_text
result = await db_session.execute(
sql_text("SELECT embedding, search_tsv, indexed_at FROM contacts WHERE id = :eid"),
{"eid": contact_id},
)
row = result.mappings().first()
assert row is not None
assert row["embedding"] is None
assert row["search_tsv"] is None
assert row["indexed_at"] is None
@pytest.mark.asyncio
async def test_remove_from_index_unknown_entity_type(db_session: AsyncSession):
"""remove_from_index with unknown entity type does nothing (no error)."""
await remove_from_index(db_session, "unknown_type", uuid.uuid4(), uuid.uuid4())
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.embedding.index_entity", new_callable=AsyncMock, return_value=True)
async def test_rebuild_index_regenerates_embedding(mock_index_entity, db_session: AsyncSession):
"""rebuild_index regenerates the embedding via index_entity."""
tenant_id, contact_id = await _create_contact_with_embedding(db_session)
success = await rebuild_index(db_session, "contact", contact_id, tenant_id)
assert success is True
mock_index_entity.assert_called_once()
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.embedding.index_entity", new_callable=AsyncMock, return_value=False)
async def test_rebuild_index_failure_returns_false(mock_index_entity, db_session: AsyncSession):
"""rebuild_index returns False when index_entity fails."""
tenant_id, contact_id = await _create_contact_with_embedding(db_session)
success = await rebuild_index(db_session, "contact", contact_id, tenant_id)
assert success is False
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.lifecycle.remove_from_index", new_callable=AsyncMock)
async def test_handle_entity_delete_calls_remove_from_index(mock_remove, db_session: AsyncSession):
"""handle_entity_delete calls remove_from_index."""
tenant_id = uuid.uuid4()
entity_id = uuid.uuid4()
await handle_entity_delete(db_session, "contact", entity_id, tenant_id)
mock_remove.assert_called_once_with(db_session, "contact", entity_id, tenant_id)
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.lifecycle.rebuild_index", new_callable=AsyncMock, return_value=True)
async def test_handle_entity_restore_calls_rebuild_index(mock_rebuild, db_session: AsyncSession):
"""handle_entity_restore calls rebuild_index."""
tenant_id = uuid.uuid4()
entity_id = uuid.uuid4()
await handle_entity_restore(db_session, "contact", entity_id, tenant_id)
mock_rebuild.assert_called_once_with(db_session, "contact", entity_id, tenant_id)
# ─── 5. API Filters ───
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
async def test_search_with_date_filters(mock_hybrid_search, search_authed_client: tuple[AsyncClient, dict]):
"""Search with date_from/date_to filters returns filtered results."""
client, _ = search_authed_client
mock_hybrid_search.return_value = [
{
"entity_type": "contact",
"entity_id": str(uuid.uuid4()),
"title": "Alpha",
"snippet": "",
"score": 0.9,
"data": {},
"_created_at": "2026-01-01T00:00:00+00:00",
"_updated_at": "2026-01-01T00:00:00+00:00",
"_tags": "",
},
{
"entity_type": "contact",
"entity_id": str(uuid.uuid4()),
"title": "Beta",
"snippet": "",
"score": 0.8,
"data": {},
"_created_at": "2026-06-01T00:00:00+00:00",
"_updated_at": "2026-06-01T00:00:00+00:00",
"_tags": "",
},
]
resp = await client.post(
"/api/v1/search",
json={"query": "test", "date_from": "2026-03-01", "date_to": "2026-12-31"},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
results = data["results"]
assert len(results) == 1
assert results[0]["title"] == "Beta"
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
async def test_search_with_tags_filter(mock_hybrid_search, search_authed_client: tuple[AsyncClient, dict]):
"""Search with tags filter returns only matching results."""
client, _ = search_authed_client
mock_hybrid_search.return_value = [
{
"entity_type": "contact",
"entity_id": str(uuid.uuid4()),
"title": "Alpha",
"snippet": "",
"score": 0.9,
"data": {},
"_created_at": None,
"_updated_at": None,
"_tags": "vip,partner",
},
{
"entity_type": "contact",
"entity_id": str(uuid.uuid4()),
"title": "Beta",
"snippet": "",
"score": 0.8,
"data": {},
"_created_at": None,
"_updated_at": None,
"_tags": "lead",
},
]
resp = await client.post(
"/api/v1/search",
json={"query": "test", "tags": ["vip"]},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
results = data["results"]
assert len(results) == 1
assert results[0]["title"] == "Alpha"
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
async def test_search_with_sort_parameter(mock_hybrid_search, search_authed_client: tuple[AsyncClient, dict]):
"""Search with sort=name sorts results by title."""
client, _ = search_authed_client
mock_hybrid_search.return_value = [
{
"entity_type": "contact",
"entity_id": str(uuid.uuid4()),
"title": "Zeta",
"snippet": "",
"score": 0.9,
"data": {},
"_created_at": None,
"_updated_at": None,
"_tags": "",
},
{
"entity_type": "contact",
"entity_id": str(uuid.uuid4()),
"title": "Alpha",
"snippet": "",
"score": 0.8,
"data": {},
"_created_at": None,
"_updated_at": None,
"_tags": "",
},
]
resp = await client.post(
"/api/v1/search",
json={"query": "test", "sort": "name"},
headers=ORIGIN_HEADER,
)
assert resp.status_code == 200
data = resp.json()
results = data["results"]
assert len(results) == 2
assert results[0]["title"] == "Alpha"
assert results[1]["title"] == "Zeta"
@pytest.mark.asyncio
async def test_facets_endpoint_returns_correct_structure(search_authed_client: tuple[AsyncClient, dict]):
"""GET /api/v1/search/facets returns correct structure."""
client, _ = search_authed_client
resp = await client.get("/api/v1/search/facets", headers=ORIGIN_HEADER)
assert resp.status_code == 200
data = resp.json()
assert "entity_types" in data
assert "tags" in data
assert "date_ranges" in data
assert isinstance(data["entity_types"], list)
assert isinstance(data["tags"], list)
assert isinstance(data["date_ranges"], dict)
# ─── 6. AI Tool ───
def test_unified_search_tool_name_and_description():
"""unified_search_tool has correct name and description."""
assert unified_search_tool.name == "unified_search"
assert TOOL_NAME == "unified_search"
assert TOOL_DESCRIPTION == unified_search_tool.description
assert "Hybrid-Suche" in unified_search_tool.description
assert unified_search_tool.required_permission == "search:read"
assert unified_search_tool.category == "search"
def test_unified_search_tool_parameters():
"""unified_search_tool exposes query/entity_types/limit parameters."""
params = unified_search_tool.parameters
assert params["type"] == "object"
assert "query" in params["properties"]
assert "entity_types" in params["properties"]
assert "limit" in params["properties"]
assert params["required"] == ["query"]
def test_unified_search_tool_openai_schema():
"""unified_search_tool to_openai_schema returns valid function schema."""
schema = unified_search_tool.to_openai_schema()
assert schema["type"] == "function"
assert schema["function"]["name"] == "unified_search"
assert "parameters" in schema["function"]
@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.ai_tool.hybrid_search", new_callable=AsyncMock)
@patch("app.plugins.builtins.unified_search.ai_tool.llm_analyze_query", new_callable=AsyncMock)
async def test_unified_search_handler_returns_compact_results(
mock_llm, mock_hybrid, db_session: AsyncSession
):
"""unified_search_handler returns compact results."""
mock_llm.return_value = {"normalized_query": "test", "semantic_terms": []}
mock_hybrid.return_value = [
{
"entity_type": "contact",
"entity_id": str(uuid.uuid4()),
"title": "John Doe",
"snippet": "john@example.com",
"score": 0.95,
}
]
# Patch the session factory to use the test session
sf = async_sessionmaker(bind=db_session.bind, expire_on_commit=False, class_=AsyncSession)
with patch("app.core.db.get_session_factory", return_value=sf):
result = await unified_search_handler(
{"query": "John", "limit": 5},
{"tenant_id": str(uuid.uuid4()), "user_id": str(uuid.uuid4()), "is_system_admin": True},
)
data = json.loads(result)
assert "count" in data
assert "results" in data
assert data["count"] == 1
assert data["results"][0]["entity_type"] == "contact"
assert data["results"][0]["title"] == "John Doe"
assert "score" in data["results"][0]
@pytest.mark.asyncio
async def test_unified_search_handler_missing_query():
"""unified_search_handler returns error for missing query."""
result = await unified_search_handler({}, {})
data = json.loads(result)
assert "error" in data
assert data["error"] == "query is required"
@pytest.mark.asyncio
async def test_unified_search_handler_missing_tenant():
"""unified_search_handler returns error for missing tenant context."""
result = await unified_search_handler({"query": "test"}, {})
data = json.loads(result)
assert "error" in data
assert data["error"] == "missing tenant context"
# ─── 7. New Providers ───
def test_agent_memory_provider_imports_and_flags():
"""AgentMemorySearchProvider imports correctly with correct flags."""
provider = AgentMemorySearchProvider()
assert provider.entity_type == "agent_memory"
assert provider.supports_fts is True
assert provider.supports_vector is True
assert provider.supports_rag is False
assert provider.supports_graph is False
def test_ai_chat_provider_imports_and_flags():
"""AIChatSearchProvider imports correctly with correct flags."""
provider = AIChatSearchProvider()
assert provider.entity_type == "ai_chat"
assert provider.supports_fts is True
assert provider.supports_vector is False
assert provider.supports_rag is False
assert provider.supports_graph is False
def test_workflow_provider_imports_and_flags():
"""WorkflowSearchProvider imports correctly with correct flags."""
provider = WorkflowSearchProvider()
assert provider.entity_type == "workflow"
assert provider.supports_fts is True
assert provider.supports_vector is False
assert provider.supports_rag is False
assert provider.supports_graph is False
def test_new_providers_to_search_result():
"""New providers produce correct search result dicts."""
agent_memory = AgentMemorySearchProvider()
result = agent_memory.to_search_result({"id": "1", "content": "Remembered fact", "memory_type": "fact"})
assert result["entity_type"] == "agent_memory"
assert result["entity_id"] == "1"
assert result["title"] == "Remembered fact"
assert result["data"]["memory_type"] == "fact"
ai_chat = AIChatSearchProvider()
result = ai_chat.to_search_result({"id": "2", "content": "Chat message", "role": "user", "session_title": "Session"})
assert result["entity_type"] == "ai_chat"
assert result["entity_id"] == "2"
assert result["title"] == "Session"
assert result["data"]["role"] == "user"
workflow = WorkflowSearchProvider()
result = workflow.to_search_result({"id": "3", "name": "Workflow A", "description": "Desc", "trigger_event": "contact.created"})
assert result["entity_type"] == "workflow"
assert result["entity_id"] == "3"
assert result["title"] == "Workflow A"
assert result["data"]["trigger_event"] == "contact.created"
# ─── 8. Sensitive Fields Exclusion ───
def test_sensitive_fields_not_in_search_tsv():
"""Sensitive fields are excluded from search_tsv via filter_for_search."""
from app.core.sensitive_data import filter_for_search, get_sensitive_fields
sensitive = get_sensitive_fields("contact")
assert "password_hash" in sensitive
assert "smtp_password" in sensitive
assert "api_key" in sensitive
data = {
"displayname": "John Doe",
"email_1": "john@example.com",
"password_hash": "secret-hash",
"smtp_password": "secret-pw",
"api_key": "secret-key",
}
filtered = filter_for_search(data, "contact")
assert "displayname" in filtered
assert "email_1" in filtered
assert "password_hash" not in filtered
assert "smtp_password" not in filtered
assert "api_key" not in filtered
def test_sensitive_fields_not_in_embedding_text():
"""Sensitive fields are excluded from embedding text via filter_for_embeddings."""
from app.core.sensitive_data import filter_for_embeddings
data = {
"displayname": "John Doe",
"email_1": "john@example.com",
"password_hash": "secret-hash",
"smtp_password": "secret-pw",
"api_key": "secret-key",
}
filtered = filter_for_embeddings(data, "contact")
assert "displayname" in filtered
assert "email_1" in filtered
assert "password_hash" not in filtered
assert "smtp_password" not in filtered
assert "api_key" not in filtered
def test_contact_provider_embedding_text_excludes_sensitive():
"""ContactSearchProvider.get_embedding_text selects only non-sensitive columns."""
provider = ContactSearchProvider()
# The SQL selects only safe columns — verify no sensitive column names appear
import inspect
source = inspect.getsource(provider.get_embedding_text)
assert "password_hash" not in source
assert "smtp_password" not in source
assert "api_key" not in source
assert "oauth_token" not in source
def test_sensitive_fields_redacted_in_index_entity():
"""index_entity redacts sensitive fields from embedding text."""
from app.core.sensitive_data import get_sensitive_fields
from app.plugins.builtins.unified_search.embedding import index_entity
# Verify the sensitive-data guard is present in index_entity source
import inspect
source = inspect.getsource(index_entity)
assert "get_sensitive_fields" in source
assert "REDACTED" in source