feat(H): H-SRC/H-CITE — knowledge source adapter (wiki/dms/mail/communication), evidence references with deep-links and workstream blocks, 23 tests passing
This commit is contained in:
Agent Zero
@@ -0,0 +1,259 @@
|
||||
"""Knowledge source adapter — connects DMS, Wiki, Mail, Communication to the
|
||||
existing SearchProvider/RAG pipeline (H-SRC).
|
||||
|
||||
Originalquelle bleibt authoritative; Permissions/Sensitive Fields gelten
|
||||
durchgängig. No second universal knowledge store — uses existing
|
||||
unified_search infrastructure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ─── Source type registry ────────────────────────────────────────────────────
|
||||
|
||||
_SOURCE_TYPES: dict[str, dict[str, Any]] = {
|
||||
"wiki": {
|
||||
"display_name": "Wiki Articles",
|
||||
"model_path": "app.plugins.builtins.wiki.models.WikiArticle",
|
||||
"text_field": "content",
|
||||
"title_field": "title",
|
||||
"id_field": "id",
|
||||
"status_filter": {"status": "published"},
|
||||
},
|
||||
"dms": {
|
||||
"display_name": "DMS Documents",
|
||||
"model_path": "app.plugins.builtins.dms.models.DmsFile",
|
||||
"text_field": "extracted_text",
|
||||
"title_field": "filename",
|
||||
"id_field": "id",
|
||||
"status_filter": {},
|
||||
},
|
||||
"mail": {
|
||||
"display_name": "Mail Messages",
|
||||
"model_path": "app.plugins.builtins.mail.models.MailMessage",
|
||||
"text_field": "body_text",
|
||||
"title_field": "subject",
|
||||
"id_field": "id",
|
||||
"status_filter": {},
|
||||
},
|
||||
"communication": {
|
||||
"display_name": "Communication Messages",
|
||||
"model_path": "app.plugins.builtins.kommunikation.models.CommMessage",
|
||||
"text_field": "content",
|
||||
"title_field": "content",
|
||||
"id_field": "id",
|
||||
"status_filter": {},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_available_sources() -> list[dict[str, str]]:
|
||||
"""List all available knowledge source types."""
|
||||
return [
|
||||
{"type": k, "display_name": v["display_name"]}
|
||||
for k, v in _SOURCE_TYPES.items()
|
||||
]
|
||||
|
||||
|
||||
def get_source_config(source_type: str) -> dict[str, Any] | None:
|
||||
"""Get configuration for a knowledge source type."""
|
||||
return _SOURCE_TYPES.get(source_type)
|
||||
|
||||
|
||||
async def fetch_source_content(
|
||||
db: AsyncSession,
|
||||
tenant_id: uuid.UUID,
|
||||
source_type: str,
|
||||
entity_id: uuid.UUID,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Fetch content from a knowledge source for RAG indexing.
|
||||
|
||||
Returns a dict with:
|
||||
- ``title``: Title for the content
|
||||
- ``text``: Text content for embedding
|
||||
- ``source_type``: The source type
|
||||
- ``source_id``: The entity ID
|
||||
- ``source_url``: Deep link to the original content
|
||||
- ``metadata``: Additional metadata
|
||||
"""
|
||||
config = get_source_config(source_type)
|
||||
if config is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Dynamic import of the model
|
||||
import importlib
|
||||
module_path, class_name = config["model_path"].rsplit(".", 1)
|
||||
module = importlib.import_module(module_path)
|
||||
model = getattr(module, class_name)
|
||||
|
||||
# Fetch the entity
|
||||
query = select(model).where(
|
||||
model.id == entity_id,
|
||||
model.tenant_id == tenant_id,
|
||||
)
|
||||
if hasattr(model, "deleted_at"):
|
||||
query = query.where(model.deleted_at.is_(None))
|
||||
|
||||
# Apply status filter
|
||||
for field, value in config.get("status_filter", {}).items():
|
||||
if hasattr(model, field):
|
||||
query = query.where(getattr(model, field) == value)
|
||||
|
||||
result = await db.execute(query)
|
||||
entity = result.scalar_one_or_none()
|
||||
if entity is None:
|
||||
return None
|
||||
|
||||
# Extract text and title
|
||||
text = getattr(entity, config["text_field"], "") or ""
|
||||
title = getattr(entity, config["title_field"], "") or ""
|
||||
|
||||
# Build source URL (deep link)
|
||||
source_url = _build_source_url(source_type, entity_id)
|
||||
|
||||
# Build metadata
|
||||
metadata = {
|
||||
"source_type": source_type,
|
||||
"source_id": str(entity_id),
|
||||
"tenant_id": str(tenant_id),
|
||||
}
|
||||
if hasattr(entity, "owner_id"):
|
||||
metadata["owner_id"] = str(entity.owner_id) if entity.owner_id else None
|
||||
if hasattr(entity, "tags"):
|
||||
metadata["tags"] = entity.tags or []
|
||||
if hasattr(entity, "category_id"):
|
||||
metadata["category_id"] = str(entity.category_id) if entity.category_id else None
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"text": text,
|
||||
"source_type": source_type,
|
||||
"source_id": str(entity_id),
|
||||
"source_url": source_url,
|
||||
"metadata": metadata,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning("Failed to fetch %s content %s: %s", source_type, entity_id, e)
|
||||
return None
|
||||
|
||||
|
||||
def _build_source_url(source_type: str, entity_id: uuid.UUID) -> str:
|
||||
"""Build a deep-link URL to the original content."""
|
||||
url_map = {
|
||||
"wiki": f"/wiki/articles/{entity_id}",
|
||||
"dms": f"/dms/files/{entity_id}",
|
||||
"mail": f"/mail/messages/{entity_id}",
|
||||
"communication": f"/communication/messages/{entity_id}",
|
||||
}
|
||||
return url_map.get(source_type, f"/{source_type}/{entity_id}")
|
||||
|
||||
|
||||
# ─── Evidence/Source References (H-CITE) ─────────────────────────────────────
|
||||
|
||||
|
||||
class EvidenceReference:
|
||||
"""Structured source reference for RAG/Knowledge results (H-CITE).
|
||||
|
||||
Provides deep-links and cards to original documents, mails, messages,
|
||||
or business objects. Agents can display these in the workstream.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
source_type: str,
|
||||
source_id: str,
|
||||
title: str,
|
||||
url: str,
|
||||
snippet: str = "",
|
||||
confidence: float = 0.0,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
):
|
||||
self.source_type = source_type
|
||||
self.source_id = source_id
|
||||
self.title = title
|
||||
self.url = url
|
||||
self.snippet = snippet
|
||||
self.confidence = confidence
|
||||
self.metadata = metadata or {}
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Serialize to dict for API responses and workstream blocks."""
|
||||
return {
|
||||
"source_type": self.source_type,
|
||||
"source_id": self.source_id,
|
||||
"title": self.title,
|
||||
"url": self.url,
|
||||
"snippet": self.snippet,
|
||||
"confidence": self.confidence,
|
||||
"metadata": self.metadata,
|
||||
}
|
||||
|
||||
def to_workstream_block(self) -> dict[str, Any]:
|
||||
"""Convert to a typed workstream block for display in Communication."""
|
||||
return {
|
||||
"type": "evidence_card",
|
||||
"source_type": self.source_type,
|
||||
"source_id": self.source_id,
|
||||
"title": self.title,
|
||||
"url": self.url,
|
||||
"snippet": self.snippet[:200],
|
||||
"confidence": self.confidence,
|
||||
}
|
||||
|
||||
|
||||
def build_evidence_references(
|
||||
search_results: list[dict[str, Any]],
|
||||
max_results: int = 5,
|
||||
) -> list[EvidenceReference]:
|
||||
"""Build evidence references from search/RAG results.
|
||||
|
||||
Args:
|
||||
search_results: Raw search results with source_type, source_id, title, etc.
|
||||
max_results: Maximum number of references to return.
|
||||
|
||||
Returns:
|
||||
List of EvidenceReference objects sorted by confidence.
|
||||
"""
|
||||
refs: list[EvidenceReference] = []
|
||||
for result in search_results[:max_results]:
|
||||
source_type = result.get("source_type", "unknown")
|
||||
source_id = result.get("source_id", "")
|
||||
title = result.get("title", "")
|
||||
url = result.get("source_url") or _build_source_url(
|
||||
source_type, uuid.UUID(source_id) if source_id else uuid.uuid4()
|
||||
)
|
||||
snippet = result.get("snippet", "") or result.get("text", "")[:200]
|
||||
confidence = result.get("score", 0.0)
|
||||
|
||||
refs.append(EvidenceReference(
|
||||
source_type=source_type,
|
||||
source_id=source_id,
|
||||
title=title,
|
||||
url=url,
|
||||
snippet=snippet,
|
||||
confidence=confidence,
|
||||
metadata=result.get("metadata", {}),
|
||||
))
|
||||
|
||||
# Sort by confidence descending
|
||||
refs.sort(key=lambda r: r.confidence, reverse=True)
|
||||
return refs
|
||||
|
||||
|
||||
__all__ = [
|
||||
"get_available_sources",
|
||||
"get_source_config",
|
||||
"fetch_source_content",
|
||||
"EvidenceReference",
|
||||
"build_evidence_references",
|
||||
]
|
||||
@@ -169,3 +169,109 @@ class TestWikiServices:
|
||||
assert _slugify("Hello World") == "hello-world"
|
||||
assert _slugify("Überblick") == "ueberblick"
|
||||
assert _slugify("Test Article 123") == "test-article-123"
|
||||
|
||||
|
||||
# ─── H-SRC: Knowledge Source Adapter ─────────────────────────────────────────
|
||||
|
||||
|
||||
class TestKnowledgeSourceAdapter:
|
||||
"""Test the knowledge source adapter (H-SRC)."""
|
||||
|
||||
def test_available_sources(self):
|
||||
"""All 4 knowledge source types are registered."""
|
||||
from app.ai.knowledge_sources import get_available_sources
|
||||
|
||||
sources = get_available_sources()
|
||||
types = {s["type"] for s in sources}
|
||||
assert types == {"wiki", "dms", "mail", "communication"}
|
||||
|
||||
def test_get_source_config_wiki(self):
|
||||
"""Wiki source config is correct."""
|
||||
from app.ai.knowledge_sources import get_source_config
|
||||
|
||||
config = get_source_config("wiki")
|
||||
assert config is not None
|
||||
assert config["text_field"] == "content"
|
||||
assert config["title_field"] == "title"
|
||||
assert config["status_filter"] == {"status": "published"}
|
||||
|
||||
def test_get_source_config_unknown(self):
|
||||
"""Unknown source type returns None."""
|
||||
from app.ai.knowledge_sources import get_source_config
|
||||
|
||||
assert get_source_config("nonexistent") is None
|
||||
|
||||
|
||||
# ─── H-CITE: Evidence References ─────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestEvidenceReferences:
|
||||
"""Test the evidence reference builder (H-CITE)."""
|
||||
|
||||
def test_evidence_reference_to_dict(self):
|
||||
"""EvidenceReference serializes to dict correctly."""
|
||||
from app.ai.knowledge_sources import EvidenceReference
|
||||
|
||||
ref = EvidenceReference(
|
||||
source_type="wiki",
|
||||
source_id="abc-123",
|
||||
title="Test Article",
|
||||
url="/wiki/articles/abc-123",
|
||||
snippet="This is a test snippet",
|
||||
confidence=0.95,
|
||||
)
|
||||
d = ref.to_dict()
|
||||
assert d["source_type"] == "wiki"
|
||||
assert d["source_id"] == "abc-123"
|
||||
assert d["title"] == "Test Article"
|
||||
assert d["confidence"] == 0.95
|
||||
|
||||
def test_evidence_reference_to_workstream_block(self):
|
||||
"""EvidenceReference converts to workstream block."""
|
||||
from app.ai.knowledge_sources import EvidenceReference
|
||||
|
||||
ref = EvidenceReference(
|
||||
source_type="dms",
|
||||
source_id="file-123",
|
||||
title="Document.pdf",
|
||||
url="/dms/files/file-123",
|
||||
snippet="Important content here",
|
||||
)
|
||||
block = ref.to_workstream_block()
|
||||
assert block["type"] == "evidence_card"
|
||||
assert block["source_type"] == "dms"
|
||||
assert block["title"] == "Document.pdf"
|
||||
|
||||
def test_build_evidence_references_from_results(self):
|
||||
"""build_evidence_references creates refs from search results."""
|
||||
from app.ai.knowledge_sources import build_evidence_references
|
||||
|
||||
results = [
|
||||
{"source_type": "wiki", "source_id": str(uuid.uuid4()), "title": "Article 1", "score": 0.9, "snippet": "Snippet 1"},
|
||||
{"source_type": "dms", "source_id": str(uuid.uuid4()), "title": "Doc 2", "score": 0.7, "snippet": "Snippet 2"},
|
||||
{"source_type": "mail", "source_id": str(uuid.uuid4()), "title": "Mail 3", "score": 0.95, "snippet": "Snippet 3"},
|
||||
]
|
||||
refs = build_evidence_references(results)
|
||||
assert len(refs) == 3
|
||||
# Sorted by confidence descending
|
||||
assert refs[0].confidence == 0.95
|
||||
assert refs[1].confidence == 0.9
|
||||
assert refs[2].confidence == 0.7
|
||||
|
||||
def test_build_evidence_references_max_results(self):
|
||||
"""build_evidence_references respects max_results."""
|
||||
from app.ai.knowledge_sources import build_evidence_references
|
||||
|
||||
results = [
|
||||
{"source_type": "wiki", "source_id": str(uuid.uuid4()), "title": f"Article {i}", "score": 0.5}
|
||||
for i in range(10)
|
||||
]
|
||||
refs = build_evidence_references(results, max_results=3)
|
||||
assert len(refs) == 3
|
||||
|
||||
def test_build_evidence_references_empty(self):
|
||||
"""build_evidence_references handles empty results."""
|
||||
from app.ai.knowledge_sources import build_evidence_references
|
||||
|
||||
refs = build_evidence_references([])
|
||||
assert refs == []
|
||||
|
||||
Reference in New Issue
Block a user