feat: unified_search + ai_proactive plugins with Ollama Cloud DeepSeek V4

- unified_search: Hybride Suche (PostgreSQL FTS + pgvector + RRF Fusion) - 5 Search Providers (Contact, Company, Mail, File, Event) - KI Query Understanding (Fuzzy, Facetten via LiteLLM) - DMS Text-Extraction (PDF, DOCX, XLSX, PPTX) - Embedding Pipeline (ollama/nomic-embed-text, 768 Dim) - Background Jobs für Indexierung - Plugin-basierte Provider Registry - ai_proactive: Proaktiver KI-Agent - Context-Tracking (Frontend → Backend → Event Bus) - Proactive Engine mit LLM Suggestion-Generierung - SSE Real-time Push an Frontend - 6 AI Tools für Tool Registry - Rate-Limiting + User Settings - Deep Analysis Background Jobs - Frontend Integration: - useAIContext Hook, SuggestionSidebar, SuggestionBadge - ProactiveAISettings Page, Search API Client - Globale Suche auf neue API umgestellt - Tests: test_unified_search.py + test_ai_proactive.py (alle bestanden) - Config: Ollama Cloud DeepSeek V4 als Default, konfigurierbar - Dependencies: PyMuPDF, python-docx, python-pptx, pgvector - Bugfixes: notification type_key length, migration IF NOT EXISTS
2026-07-18 11:21:51 +02:00
parent 4c9295de21
commit 8cebb4f4e9
52 changed files with 6493 additions and 41 deletions
@@ -0,0 +1,3 @@
+"""Search providers for unified search."""
+
+from __future__ import annotations
@@ -0,0 +1,114 @@
+"""Company search provider."""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+
+class CompanySearchProvider:
+    """Search provider for Company entities."""
+
+    entity_type = "company"
+
+    async def search_fts(
+        self,
+        db: AsyncSession,
+        tsquery: str,
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Full-text search on companies.search_tsv."""
+        sql = text(
+            """
+            SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
+            FROM companies c
+            WHERE c.tenant_id = :tid
+              AND c.deleted_at IS NULL
+              AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
+            ORDER BY rank DESC
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"q": tsquery, "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def search_vector(
+        self,
+        db: AsyncSession,
+        embedding: list[float],
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Semantic search on companies.embedding."""
+        sql = text(
+            """
+            SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
+            FROM companies c
+            WHERE c.tenant_id = :tid
+              AND c.deleted_at IS NULL
+              AND c.embedding IS NOT NULL
+            ORDER BY c.embedding <=> cast(:emb AS vector)
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"emb": str(embedding), "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def get_embedding_text(
+        self,
+        db: AsyncSession,
+        entity_id: uuid.UUID,
+        tenant_id: uuid.UUID,
+    ) -> str:
+        """Get text for embedding generation."""
+        sql = text(
+            """
+            SELECT name, description, industry
+            FROM companies
+            WHERE id = :eid AND tenant_id = :tid
+            """
+        )
+        result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
+        row = result.mappings().first()
+        if not row:
+            return ""
+        parts = [
+            row.get("name", ""),
+            row.get("description", ""),
+            row.get("industry", ""),
+        ]
+        return " ".join(str(p) for p in parts if p)
+
+    def to_search_result(self, entity: object) -> dict[str, Any]:
+        """Convert company to search result dict."""
+        if isinstance(entity, dict):
+            name = entity.get("name", "")
+            description = entity.get("description", "") or ""
+            entity_id = str(entity.get("id", ""))
+        else:
+            name = getattr(entity, "name", "")
+            description = getattr(entity, "description", "") or ""
+            entity_id = str(getattr(entity, "id", ""))
+        return {
+            "entity_type": self.entity_type,
+            "entity_id": entity_id,
+            "title": name,
+            "snippet": description[:200],
+            "score": 0.0,
+            "data": {},
+        }
@@ -0,0 +1,121 @@
+"""Contact search provider."""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.contact import Contact
+
+logger = logging.getLogger(__name__)
+
+
+class ContactSearchProvider:
+    """Search provider for Contact entities."""
+
+    entity_type = "contact"
+
+    async def search_fts(
+        self,
+        db: AsyncSession,
+        tsquery: str,
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Full-text search on contacts.search_tsv."""
+        sql = text(
+            """
+            SELECT c.*, ts_rank(c.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
+            FROM contacts c
+            WHERE c.tenant_id = :tid
+              AND c.deleted_at IS NULL
+              AND c.search_tsv @@ to_tsquery('pg_catalog.german', :q)
+            ORDER BY rank DESC
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"q": tsquery, "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def search_vector(
+        self,
+        db: AsyncSession,
+        embedding: list[float],
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Semantic search on contacts.embedding."""
+        sql = text(
+            """
+            SELECT c.*, 1 - (c.embedding <=> cast(:emb AS vector)) AS score
+            FROM contacts c
+            WHERE c.tenant_id = :tid
+              AND c.deleted_at IS NULL
+              AND c.embedding IS NOT NULL
+            ORDER BY c.embedding <=> cast(:emb AS vector)
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"emb": str(embedding), "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def get_embedding_text(
+        self,
+        db: AsyncSession,
+        entity_id: uuid.UUID,
+        tenant_id: uuid.UUID,
+    ) -> str:
+        """Get text for embedding generation."""
+        sql = text(
+            """
+            SELECT first_name, last_name, email, phone, mobile, notes
+            FROM contacts
+            WHERE id = :eid AND tenant_id = :tid
+            """
+        )
+        result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
+        row = result.mappings().first()
+        if not row:
+            return ""
+        parts = [
+            row.get("first_name", ""),
+            row.get("last_name", ""),
+            row.get("email", ""),
+            row.get("phone", ""),
+            row.get("mobile", ""),
+            row.get("notes", ""),
+        ]
+        return " ".join(str(p) for p in parts if p)
+
+    def to_search_result(self, entity: object) -> dict[str, Any]:
+        """Convert contact to search result dict."""
+        if isinstance(entity, dict):
+            first = entity.get("first_name", "")
+            last = entity.get("last_name", "")
+            email = entity.get("email", "")
+            entity_id = str(entity.get("id", ""))
+        else:
+            first = getattr(entity, "first_name", "")
+            last = getattr(entity, "last_name", "")
+            email = getattr(entity, "email", "")
+            entity_id = str(getattr(entity, "id", ""))
+        return {
+            "entity_type": self.entity_type,
+            "entity_id": entity_id,
+            "title": f"{first} {last}".strip(),
+            "snippet": email or "",
+            "score": 0.0,
+            "data": {},
+        }
@@ -0,0 +1,114 @@
+"""Calendar event search provider."""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+
+class EventSearchProvider:
+    """Search provider for CalendarEntry entities."""
+
+    entity_type = "event"
+
+    async def search_fts(
+        self,
+        db: AsyncSession,
+        tsquery: str,
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Full-text search on calendar_entries.search_tsv."""
+        sql = text(
+            """
+            SELECT e.*, ts_rank(e.search_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
+            FROM calendar_entries e
+            WHERE e.tenant_id = :tid
+              AND e.deleted_at IS NULL
+              AND e.search_tsv @@ to_tsquery('pg_catalog.german', :q)
+            ORDER BY rank DESC
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"q": tsquery, "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def search_vector(
+        self,
+        db: AsyncSession,
+        embedding: list[float],
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Semantic search on calendar_entries.embedding."""
+        sql = text(
+            """
+            SELECT e.*, 1 - (e.embedding <=> cast(:emb AS vector)) AS score
+            FROM calendar_entries e
+            WHERE e.tenant_id = :tid
+              AND e.deleted_at IS NULL
+              AND e.embedding IS NOT NULL
+            ORDER BY e.embedding <=> cast(:emb AS vector)
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"emb": str(embedding), "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def get_embedding_text(
+        self,
+        db: AsyncSession,
+        entity_id: uuid.UUID,
+        tenant_id: uuid.UUID,
+    ) -> str:
+        """Get text for embedding generation."""
+        sql = text(
+            """
+            SELECT title, description, location
+            FROM calendar_entries
+            WHERE id = :eid AND tenant_id = :tid
+            """
+        )
+        result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
+        row = result.mappings().first()
+        if not row:
+            return ""
+        parts = [
+            row.get("title", ""),
+            row.get("description", ""),
+            row.get("location", ""),
+        ]
+        return " ".join(str(p) for p in parts if p)
+
+    def to_search_result(self, entity: object) -> dict[str, Any]:
+        """Convert calendar entry to search result dict."""
+        if isinstance(entity, dict):
+            title = entity.get("title", "")
+            description = entity.get("description", "") or ""
+            entity_id = str(entity.get("id", ""))
+        else:
+            title = getattr(entity, "title", "")
+            description = getattr(entity, "description", "") or ""
+            entity_id = str(getattr(entity, "id", ""))
+        return {
+            "entity_type": self.entity_type,
+            "entity_id": entity_id,
+            "title": title,
+            "snippet": description[:200],
+            "score": 0.0,
+            "data": {},
+        }
@@ -0,0 +1,111 @@
+"""DMS File search provider."""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+
+class FileSearchProvider:
+    """Search provider for DMS File entities."""
+
+    entity_type = "file"
+
+    async def search_fts(
+        self,
+        db: AsyncSession,
+        tsquery: str,
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Full-text search on files.content_tsv."""
+        sql = text(
+            """
+            SELECT f.*, ts_rank(f.content_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
+            FROM files f
+            WHERE f.tenant_id = :tid
+              AND f.deleted_at IS NULL
+              AND f.content_tsv @@ to_tsquery('pg_catalog.german', :q)
+            ORDER BY rank DESC
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"q": tsquery, "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def search_vector(
+        self,
+        db: AsyncSession,
+        embedding: list[float],
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Semantic search on files.embedding."""
+        sql = text(
+            """
+            SELECT f.*, 1 - (f.embedding <=> cast(:emb AS vector)) AS score
+            FROM files f
+            WHERE f.tenant_id = :tid
+              AND f.deleted_at IS NULL
+              AND f.embedding IS NOT NULL
+            ORDER BY f.embedding <=> cast(:emb AS vector)
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"emb": str(embedding), "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def get_embedding_text(
+        self,
+        db: AsyncSession,
+        entity_id: uuid.UUID,
+        tenant_id: uuid.UUID,
+    ) -> str:
+        """Get text for embedding generation."""
+        sql = text(
+            """
+            SELECT name, content_text
+            FROM files
+            WHERE id = :eid AND tenant_id = :tid
+            """
+        )
+        result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
+        row = result.mappings().first()
+        if not row:
+            return ""
+        name = row.get("name", "") or ""
+        content = row.get("content_text", "") or ""
+        return f"{name} {content[:5000]}"
+
+    def to_search_result(self, entity: object) -> dict[str, Any]:
+        """Convert file to search result dict."""
+        if isinstance(entity, dict):
+            name = entity.get("name", "")
+            content = entity.get("content_text", "") or ""
+            entity_id = str(entity.get("id", ""))
+        else:
+            name = getattr(entity, "name", "")
+            content = getattr(entity, "content_text", "") or ""
+            entity_id = str(getattr(entity, "id", ""))
+        return {
+            "entity_type": self.entity_type,
+            "entity_id": entity_id,
+            "title": name,
+            "snippet": content[:200],
+            "score": 0.0,
+            "data": {},
+        }
@@ -0,0 +1,111 @@
+"""Mail search provider."""
+
+from __future__ import annotations
+
+import logging
+import uuid
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+logger = logging.getLogger(__name__)
+
+
+class MailSearchProvider:
+    """Search provider for Mail entities."""
+
+    entity_type = "mail"
+
+    async def search_fts(
+        self,
+        db: AsyncSession,
+        tsquery: str,
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Full-text search on mails.body_tsv."""
+        sql = text(
+            """
+            SELECT m.*, ts_rank(m.body_tsv, to_tsquery('pg_catalog.german', :q)) AS rank
+            FROM mails m
+            WHERE m.tenant_id = :tid
+              AND m.deleted_at IS NULL
+              AND m.body_tsv @@ to_tsquery('pg_catalog.german', :q)
+            ORDER BY rank DESC
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"q": tsquery, "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def search_vector(
+        self,
+        db: AsyncSession,
+        embedding: list[float],
+        tenant_id: uuid.UUID,
+        limit: int,
+    ) -> list[dict[str, Any]]:
+        """Semantic search on mails.embedding."""
+        sql = text(
+            """
+            SELECT m.*, 1 - (m.embedding <=> cast(:emb AS vector)) AS score
+            FROM mails m
+            WHERE m.tenant_id = :tid
+              AND m.deleted_at IS NULL
+              AND m.embedding IS NOT NULL
+            ORDER BY m.embedding <=> cast(:emb AS vector)
+            LIMIT :lim
+            """
+        )
+        result = await db.execute(
+            sql,
+            {"emb": str(embedding), "tid": tenant_id, "lim": limit},
+        )
+        rows = result.mappings().all()
+        return [dict(r) for r in rows]
+
+    async def get_embedding_text(
+        self,
+        db: AsyncSession,
+        entity_id: uuid.UUID,
+        tenant_id: uuid.UUID,
+    ) -> str:
+        """Get text for embedding generation."""
+        sql = text(
+            """
+            SELECT subject, body_text
+            FROM mails
+            WHERE id = :eid AND tenant_id = :tid
+            """
+        )
+        result = await db.execute(sql, {"eid": entity_id, "tid": tenant_id})
+        row = result.mappings().first()
+        if not row:
+            return ""
+        subject = row.get("subject", "") or ""
+        body = row.get("body_text", "") or ""
+        return f"{subject} {body[:5000]}"
+
+    def to_search_result(self, entity: object) -> dict[str, Any]:
+        """Convert mail to search result dict."""
+        if isinstance(entity, dict):
+            subject = entity.get("subject", "")
+            body = entity.get("body_text", "") or ""
+            entity_id = str(entity.get("id", ""))
+        else:
+            subject = getattr(entity, "subject", "")
+            body = getattr(entity, "body_text", "") or ""
+            entity_id = str(getattr(entity, "id", ""))
+        return {
+            "entity_type": self.entity_type,
+            "entity_id": entity_id,
+            "title": subject,
+            "snippet": body[:200],
+            "score": 0.0,
+            "data": {},
+        }