Agent Zero
abbe7a18fc
- P0: hooks.py 3-tuple fix, trigger_dispatcher Contract, contacts/plugin unregister_actions_by_owner - P0: 5 test files — check_permission mocks removed, hardcoded DB credential → env var - P1: attachment_service DmsFile via Contract helper, restore_registry/history_hooks dedup - P1: mail/plugin restore unregister, mcp_client datetime.now(UTC), saved_views/filters patterns - P1: ProtectedRoute fail-closed, 13 test assertion fixes (bcrypt, DB-URLs, SECRET_KEYs) - P2: deprecated notifications → post_system_message (3 files), forgejo Base, report_generator lazy import - P2: webhooks permissions, deps.py/roles.py plugin perms removed, import_export default - P2: address/tags/entity_links patterns removed, worker.py Contract-Umgehungen fixed - P2: 28 frontend TODOs (hardcoded constants, deprecated notification API) - P3: dead code, duplicates, deprecated imports, private attr, __import__ inline - P3: 8 frontend TODOs (LucideIcons, inline styles, XSS, i18n) - ruff: 838 → 0 (612 auto-fix + 246 manual + 27 F821 regression fix) - F821: 30 → 0 (AutomationDefinition, DmsFile, user_id, Path, Any, String) - Contract-Umgehungen: 2 neue gefunden (worker.py:169, worker.py:280) und gefixt
314 lines
11 KiB
Python
314 lines
11 KiB
Python
"""Search provider for unified_search integration — FTS + vector search for messages."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import uuid
|
|
from typing import Any
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy import text as sql_text
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.plugins.builtins.kommunikation.models import (
|
|
CommConversation,
|
|
CommMessage,
|
|
CommParticipant,
|
|
)
|
|
from app.plugins.builtins.unified_search.contracts import get_contract as get_search_contract
|
|
|
|
_search = get_search_contract()
|
|
generate_embedding = _search.generate_embedding
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class CommSearchProvider:
|
|
"""Provider for unified_search — searches conversations and messages."""
|
|
|
|
entity_type = "message"
|
|
|
|
async def search(
|
|
self,
|
|
db: AsyncSession,
|
|
tenant_id: uuid.UUID,
|
|
user_id: uuid.UUID,
|
|
query: str,
|
|
limit: int = 20,
|
|
) -> list[dict[str, Any]]:
|
|
"""Search in messages and conversations the user has access to."""
|
|
results: list[dict[str, Any]] = []
|
|
|
|
# Get user's conversation IDs
|
|
conv_result = await db.execute(
|
|
select(CommParticipant.conversation_id).where(
|
|
CommParticipant.participant_id == user_id,
|
|
CommParticipant.participant_type == "user",
|
|
CommParticipant.left_at.is_(None),
|
|
CommParticipant.tenant_id == tenant_id,
|
|
)
|
|
)
|
|
conv_ids = [row[0] for row in conv_result.fetchall()]
|
|
|
|
if not conv_ids:
|
|
return results
|
|
|
|
# Search in messages
|
|
msg_result = await db.execute(
|
|
select(CommMessage, CommConversation.title)
|
|
.join(CommConversation, CommConversation.id == CommMessage.conversation_id)
|
|
.where(
|
|
CommMessage.conversation_id.in_(conv_ids),
|
|
CommMessage.tenant_id == tenant_id,
|
|
CommMessage.deleted_at.is_(None),
|
|
CommMessage.content.ilike(f"%{query}%"),
|
|
)
|
|
.order_by(CommMessage.created_at.desc())
|
|
.limit(limit)
|
|
)
|
|
|
|
for msg, conv_title in msg_result.fetchall():
|
|
# Build snippet around match
|
|
content = msg.content or ""
|
|
idx = content.lower().find(query.lower())
|
|
if idx >= 0:
|
|
start = max(0, idx - 30)
|
|
end = min(len(content), idx + len(query) + 30)
|
|
snippet = ("..." if start > 0 else "") + content[start:end] + ("..." if end < len(content) else "")
|
|
else:
|
|
snippet = content[:100]
|
|
|
|
results.append({
|
|
"type": "message",
|
|
"id": str(msg.id),
|
|
"conversation_id": str(msg.conversation_id),
|
|
"conversation_title": conv_title,
|
|
"content": msg.content,
|
|
"sender_type": msg.sender_type,
|
|
"created_at": msg.created_at.isoformat() if msg.created_at else None,
|
|
"snippet": snippet,
|
|
})
|
|
|
|
# Search in conversation titles
|
|
conv_title_result = await db.execute(
|
|
select(CommConversation).where(
|
|
CommConversation.id.in_(conv_ids),
|
|
CommConversation.tenant_id == tenant_id,
|
|
CommConversation.deleted_at.is_(None),
|
|
CommConversation.title.ilike(f"%{query}%"),
|
|
)
|
|
.limit(limit)
|
|
)
|
|
|
|
for conv in conv_title_result.scalars().all():
|
|
results.append({
|
|
"type": "conversation",
|
|
"id": str(conv.id),
|
|
"title": conv.title,
|
|
"last_msg_at": conv.last_msg_at.isoformat() if conv.last_msg_at else None,
|
|
})
|
|
|
|
return results
|
|
|
|
async def search_fts(
|
|
self,
|
|
db: AsyncSession,
|
|
tsquery: str,
|
|
tenant_id: uuid.UUID,
|
|
limit: int,
|
|
) -> list[dict[str, Any]]:
|
|
"""Full-text search on comm_messages.search_tsv."""
|
|
sql = sql_text(
|
|
"""
|
|
SELECT m.*, ts_rank(m.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
|
|
FROM comm_messages m
|
|
WHERE m.tenant_id = :tid
|
|
AND m.deleted_at IS NULL
|
|
AND m.search_tsv @@ to_tsquery('pg_catalog.german', :q)
|
|
ORDER BY rank DESC
|
|
LIMIT :lim
|
|
"""
|
|
)
|
|
result = await db.execute(
|
|
sql,
|
|
{"q": tsquery, "tid": tenant_id, "lim": limit},
|
|
)
|
|
rows = result.mappings().all()
|
|
return [dict(r) for r in rows]
|
|
|
|
async def search_vector(
|
|
self,
|
|
db: AsyncSession,
|
|
embedding: list[float],
|
|
tenant_id: uuid.UUID,
|
|
limit: int,
|
|
) -> list[dict[str, Any]]:
|
|
"""Semantic vector search on comm_messages.embedding."""
|
|
sql = sql_text(
|
|
"""
|
|
SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
|
|
FROM comm_messages m
|
|
WHERE m.tenant_id = :tid
|
|
AND m.deleted_at IS NULL
|
|
AND m.embedding IS NOT NULL
|
|
ORDER BY m.embedding <=> cast(:emb AS vector)
|
|
LIMIT :lim
|
|
"""
|
|
)
|
|
result = await db.execute(
|
|
sql,
|
|
{"emb": str(embedding), "tid": tenant_id, "lim": limit},
|
|
)
|
|
rows = result.mappings().all()
|
|
return [dict(r) for r in rows]
|
|
|
|
async def get_embedding_text(
|
|
self,
|
|
db: AsyncSession,
|
|
entity_id: uuid.UUID,
|
|
tenant_id: uuid.UUID,
|
|
) -> str:
|
|
"""Get message content for embedding generation."""
|
|
sql = sql_text(
|
|
"""
|
|
SELECT content FROM comm_messages
|
|
WHERE id = :eid AND tenant_id = :tid AND deleted_at IS NULL
|
|
"""
|
|
)
|
|
result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
|
|
row = result.mappings().first()
|
|
if row is None:
|
|
return ""
|
|
return row["content"] or ""
|
|
|
|
def to_search_result(self, entity: object) -> dict:
|
|
"""Convert a comm_messages row to a search result dict."""
|
|
row = dict(entity) if isinstance(entity, dict) else {}
|
|
return {
|
|
"type": "message",
|
|
"id": str(row.get("id", "")),
|
|
"conversation_id": str(row.get("conversation_id", "")),
|
|
"content": row.get("content", ""),
|
|
"sender_type": row.get("sender_type", ""),
|
|
"created_at": str(row.get("created_at", "")) if row.get("created_at") else None,
|
|
}
|
|
|
|
async def index_message(self, message: dict[str, Any]) -> None:
|
|
"""Index a single message for search — updates search_tsv and embedding.
|
|
|
|
Called after a message is created or updated. The message dict must contain
|
|
at least: id, tenant_id, content.
|
|
"""
|
|
from app.core.db import async_session_factory
|
|
|
|
msg_id = message.get("id")
|
|
tenant_id = message.get("tenant_id")
|
|
content = message.get("content", "")
|
|
|
|
if not msg_id or not tenant_id:
|
|
logger.warning("index_message: missing id or tenant_id")
|
|
return
|
|
|
|
async with async_session_factory() as db:
|
|
try:
|
|
# Update search_tsv using PostgreSQL to_tsvector
|
|
sql_update_tsv = sql_text(
|
|
"""
|
|
UPDATE comm_messages
|
|
SET search_tsv = to_tsvector('pg_catalog.german', :content)
|
|
WHERE id = :mid AND tenant_id = :tid
|
|
"""
|
|
)
|
|
await db.execute(
|
|
sql_update_tsv,
|
|
{"content": content, "mid": msg_id, "tid": tenant_id},
|
|
)
|
|
|
|
# Generate and store embedding
|
|
if content.strip():
|
|
embedding = await generate_embedding(
|
|
content, db=db, tenant_id=tenant_id
|
|
)
|
|
if embedding:
|
|
sql_update_emb = sql_text(
|
|
"""
|
|
UPDATE comm_messages
|
|
SET embedding = cast(:emb AS vector)
|
|
WHERE id = :mid AND tenant_id = :tid
|
|
"""
|
|
)
|
|
await db.execute(
|
|
sql_update_emb,
|
|
{"emb": str(embedding), "mid": msg_id, "tid": tenant_id},
|
|
)
|
|
|
|
await db.commit()
|
|
except Exception:
|
|
await db.rollback()
|
|
logger.exception("Failed to index message %s", msg_id)
|
|
|
|
async def reindex_all(self, db: AsyncSession, tenant_id: uuid.UUID) -> None:
|
|
"""Full reindex of all messages for a tenant.
|
|
|
|
Iterates over all non-deleted messages and updates search_tsv and embedding.
|
|
"""
|
|
sql_select = sql_text(
|
|
"""
|
|
SELECT id, content FROM comm_messages
|
|
WHERE tenant_id = :tid AND deleted_at IS NULL
|
|
ORDER BY created_at ASC
|
|
"""
|
|
)
|
|
result = await db.execute(sql_select, {"tid": tenant_id})
|
|
rows = result.mappings().all()
|
|
|
|
total = len(rows)
|
|
logger.info("Reindexing %d messages for tenant %s", total, tenant_id)
|
|
|
|
for i, row in enumerate(rows):
|
|
msg_id = row["id"]
|
|
content = row["content"] or ""
|
|
|
|
try:
|
|
# Update search_tsv
|
|
sql_update_tsv = sql_text(
|
|
"""
|
|
UPDATE comm_messages
|
|
SET search_tsv = to_tsvector('pg_catalog.german', :content)
|
|
WHERE id = :mid AND tenant_id = :tid
|
|
"""
|
|
)
|
|
await db.execute(
|
|
sql_update_tsv,
|
|
{"content": content, "mid": msg_id, "tid": tenant_id},
|
|
)
|
|
|
|
# Generate and store embedding
|
|
if content.strip():
|
|
embedding = await generate_embedding(
|
|
content, db=db, tenant_id=tenant_id
|
|
)
|
|
if embedding:
|
|
sql_update_emb = sql_text(
|
|
"""
|
|
UPDATE comm_messages
|
|
SET embedding = cast(:emb AS vector)
|
|
WHERE id = :mid AND tenant_id = :tid
|
|
"""
|
|
)
|
|
await db.execute(
|
|
sql_update_emb,
|
|
{"emb": str(embedding), "mid": msg_id, "tid": tenant_id},
|
|
)
|
|
|
|
if (i + 1) % 50 == 0:
|
|
await db.commit()
|
|
logger.debug("Reindexed %d/%d messages", i + 1, total)
|
|
|
|
except Exception:
|
|
logger.exception("Failed to reindex message %s", msg_id)
|
|
continue
|
|
|
|
await db.commit()
|
|
logger.info("Reindexing complete for tenant %s: %d messages", tenant_id, total)
|