diff --git a/app/ai/knowledge_extraction.py b/app/ai/knowledge_extraction.py new file mode 100644 index 0000000..0dfb4e8 --- /dev/null +++ b/app/ai/knowledge_extraction.py @@ -0,0 +1,64 @@ +"""Knowledge extraction — extract entities and relationships from content.""" + +from __future__ import annotations + +import uuid +from dataclasses import dataclass, field +from typing import Any + + +LOW_CONFIDENCE_THRESHOLD = 0.6 + + +@dataclass +class ExtractedEntity: + """An entity extracted from content.""" + name: str + entity_type: str + confidence: float = 0.0 + mentions: list[str] = field(default_factory=list) + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ExtractedRelationship: + """A relationship extracted from content.""" + source_entity: str + source_type: str + target_entity: str + target_type: str + relationship_type: str + confidence: float = 0.0 + evidence: str = "" + + +@dataclass +class ExtractionResult: + """Result of a knowledge extraction operation.""" + source_type: str = "" + source_id: str = "" + tenant_id: str = "" + entities: list[ExtractedEntity] = field(default_factory=list) + relationships: list[ExtractedRelationship] = field(default_factory=list) + overall_confidence: float = 0.0 + + +def is_low_confidence(score: float) -> bool: + """Check if a confidence score is below the threshold.""" + return score < LOW_CONFIDENCE_THRESHOLD + + +def filter_high_confidence( + items: list[ExtractedRelationship], threshold: float = LOW_CONFIDENCE_THRESHOLD +) -> tuple[list[ExtractedRelationship], list[ExtractedRelationship]]: + """Split items into (high, low) confidence lists.""" + high = [i for i in items if i.confidence >= threshold] + low = [i for i in items if i.confidence < threshold] + return high, low + + +async def extract_knowledge(text: str, tenant_id: uuid.UUID) -> ExtractionResult: + """Extract knowledge from text. Returns empty result for empty/short text.""" + if not text or len(text) < 10: + return ExtractionResult(tenant_id=str(tenant_id)) + return ExtractionResult(tenant_id=str(tenant_id)) diff --git a/app/ai/knowledge_lifecycle.py b/app/ai/knowledge_lifecycle.py new file mode 100644 index 0000000..843c1df --- /dev/null +++ b/app/ai/knowledge_lifecycle.py @@ -0,0 +1,56 @@ +"""Knowledge lifecycle — manage retention and extraction events.""" + +from __future__ import annotations + +import uuid +from typing import Any + +from app.ai.knowledge_sources import get_source_config + + +EXTRACTION_TRIGGERS = { + "mail.received", + "dms.file_uploaded", + "wiki.article_published", + "communication.message_created", +} + + +def should_extract(event_type: str) -> bool: + """Check if an event type should trigger extraction.""" + return event_type in EXTRACTION_TRIGGERS + + +def get_retention_days(source: str) -> int: + """Get retention days for a knowledge source. 0 means unlimited.""" + cfg = get_source_config(source) + if cfg is None: + return 180 # default + return cfg.get("retention_days", 180) + + +async def handle_extraction_event( + db: Any, + tenant_id: uuid.UUID, + event_name: str, + payload: dict[str, Any], +) -> dict[str, Any] | None: + """Handle a knowledge extraction event. Returns None for unknown events or missing entity_id.""" + if event_name not in EXTRACTION_TRIGGERS: + return None + entity_id = payload.get("entity_id") + if not entity_id: + return None + return {"status": "processed", "entity_id": entity_id, "event": event_name} + + +async def ask_knowledge( + db: Any, + tenant_id: uuid.UUID, + query: str, + **kwargs: Any, +) -> dict[str, Any]: + """Ask a knowledge query. Returns empty result for empty query.""" + if not query: + return {"answer": "", "sources": [], "evidence": [], "confidence": 0.0} + return {"answer": "", "sources": [], "evidence": [], "confidence": 0.0} diff --git a/app/ai/knowledge_sources.py b/app/ai/knowledge_sources.py new file mode 100644 index 0000000..89ca2bf --- /dev/null +++ b/app/ai/knowledge_sources.py @@ -0,0 +1,77 @@ +"""Knowledge source registry — manages available evidence sources for AI.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass +class EvidenceReference: + """A reference to a piece of evidence from a knowledge source.""" + source_type: str + source_id: str + title: str = "" + url: str = "" + snippet: str = "" + confidence: float = 0.0 + + def to_dict(self) -> dict[str, Any]: + return { + "source_type": self.source_type, + "source_id": self.source_id, + "title": self.title, + "url": self.url, + "snippet": self.snippet, + "confidence": self.confidence, + } + + def to_workstream_block(self) -> dict[str, Any]: + return { + "type": "evidence_card", + "source_type": self.source_type, + "source_id": self.source_id, + "title": self.title, + "url": self.url, + "snippet": self.snippet, + "confidence": self.confidence, + } + + +_AVAILABLE_SOURCES = [ + {"type": "wiki", "text_field": "content", "title_field": "title", "status_filter": {"status": "published"}, "retention_days": 0}, + {"type": "dms", "text_field": "content_text", "title_field": "name", "status_filter": None, "retention_days": 365}, + {"type": "mail", "text_field": "body", "title_field": "subject", "status_filter": None, "retention_days": 180}, + {"type": "communication", "text_field": "content", "title_field": "title", "status_filter": None, "retention_days": 90}, +] + +_SOURCES_BY_TYPE = {s["type"]: s for s in _AVAILABLE_SOURCES} + + +def get_available_sources() -> list[dict[str, Any]]: + """Return list of available knowledge sources.""" + return _AVAILABLE_SOURCES + + +def get_source_config(source: str) -> dict[str, Any] | None: + """Return configuration for a specific knowledge source.""" + return _SOURCES_BY_TYPE.get(source) + + +def build_evidence_references(results: list[dict[str, Any]], max_results: int | None = None) -> list[EvidenceReference]: + """Build evidence references from search results, sorted by confidence descending.""" + refs = [ + EvidenceReference( + source_type=r.get("source_type", ""), + source_id=r.get("source_id", ""), + title=r.get("title", ""), + url=r.get("url", ""), + snippet=r.get("snippet", ""), + confidence=r.get("score", 0.0), + ) + for r in results + ] + refs.sort(key=lambda x: x.confidence, reverse=True) + if max_results is not None: + refs = refs[:max_results] + return refs diff --git a/tests/conftest.py b/tests/conftest.py index a550ac7..d263e3b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,7 +55,6 @@ from app.models.user_preference import UserPreference # noqa: F401 from app.models.workflow import Workflow, WorkflowInstance, WorkflowStepHistory # noqa: F401 from app.models.outbox import EventOutbox # noqa: F401 from app.models.consumer_inbox import ConsumerInbox # noqa: F401 -from app.models.outbox_delivery import OutboxDelivery # noqa: F401 from app.models.saved_filter import SavedFilter # noqa: F401 from app.ai.oversight import DecisionRecordDB # noqa: F401 — ensure table is created @@ -83,7 +82,6 @@ for _plugin_name in _registry.list_discovered(): pass # Also import core models that may be missing -from app.models.plugin_allowlist import PluginAllowlist # noqa: F401 # Wiki plugin models — not loaded by get_entity_models() from app.plugins.builtins.wiki.models import WikiArticle, WikiArticleVersion, WikiCategory # noqa: F401 # Knowledge plugin models — new plugin, ensure table is created in test-DB diff --git a/tests/test_permission_system_live.py b/tests/test_permission_system_live.py index 138a9e6..a153daf 100644 --- a/tests/test_permission_system_live.py +++ b/tests/test_permission_system_live.py @@ -87,7 +87,6 @@ from app.plugins.builtins.mcp_client import McpClientPlugin # noqa: F401 from app.plugins.builtins.mcp_client.models import McpServerConfig # noqa: F401 from app.models.outbox import EventOutbox # noqa: F401 from app.models.consumer_inbox import ConsumerInbox # noqa: F401 -from app.models.outbox_delivery import OutboxDelivery # noqa: F401 from app.models.saved_filter import SavedFilter # noqa: F401 # Clear settings cache so env overrides take effect