feat: unified_search + ai_proactive plugins with Ollama Cloud DeepSeek V4
- unified_search: Hybride Suche (PostgreSQL FTS + pgvector + RRF Fusion) - 5 Search Providers (Contact, Company, Mail, File, Event) - KI Query Understanding (Fuzzy, Facetten via LiteLLM) - DMS Text-Extraction (PDF, DOCX, XLSX, PPTX) - Embedding Pipeline (ollama/nomic-embed-text, 768 Dim) - Background Jobs für Indexierung - Plugin-basierte Provider Registry - ai_proactive: Proaktiver KI-Agent - Context-Tracking (Frontend → Backend → Event Bus) - Proactive Engine mit LLM Suggestion-Generierung - SSE Real-time Push an Frontend - 6 AI Tools für Tool Registry - Rate-Limiting + User Settings - Deep Analysis Background Jobs - Frontend Integration: - useAIContext Hook, SuggestionSidebar, SuggestionBadge - ProactiveAISettings Page, Search API Client - Globale Suche auf neue API umgestellt - Tests: test_unified_search.py + test_ai_proactive.py (alle bestanden) - Config: Ollama Cloud DeepSeek V4 als Default, konfigurierbar - Dependencies: PyMuPDF, python-docx, python-pptx, pgvector - Bugfixes: notification type_key length, migration IF NOT EXISTS
This commit is contained in:
Agent Zero
@@ -0,0 +1,552 @@
|
||||
"""Tests for the Unified Search plugin — covers ACs 1-12 and unit tests.
|
||||
|
||||
Tests hybrid search infrastructure: API endpoints, RRF fusion, autocomplete,
|
||||
text extraction, and provider registry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
||||
|
||||
from app.core.db import close_engine, reset_engine_for_testing
|
||||
from app.core.service_container import get_container
|
||||
from app.main import create_app
|
||||
from app.plugins.builtins.unified_search import UnifiedSearchPlugin
|
||||
from app.plugins.builtins.unified_search.provider_registry import (
|
||||
SearchProviderRegistry,
|
||||
get_search_registry,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.search_engine import rrf_fusion
|
||||
from app.plugins.builtins.unified_search.text_extraction import extract_text_from_file
|
||||
from app.plugins.registry import reset_registry_for_testing
|
||||
from app.services.plugin_service import reset_plugin_service_for_testing
|
||||
from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users
|
||||
|
||||
# ─── Unified Search Fixtures ───
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def search_app(engine: AsyncEngine, redis_client):
|
||||
"""FastAPI app with UnifiedSearch plugin registered, installed, and activated."""
|
||||
reset_engine_for_testing(engine)
|
||||
app = create_app()
|
||||
registry = reset_registry_for_testing()
|
||||
registry.initialize(engine, app)
|
||||
container = get_container()
|
||||
await container.initialize()
|
||||
registry.register_plugin(UnifiedSearchPlugin())
|
||||
reset_plugin_service_for_testing(registry)
|
||||
|
||||
sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
|
||||
async with sf() as session:
|
||||
await registry.install(session, "unified_search")
|
||||
await registry.activate(session, "unified_search")
|
||||
await session.commit()
|
||||
|
||||
yield app
|
||||
await close_engine()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def search_client(search_app) -> AsyncClient:
|
||||
"""HTTP test client with unified search plugin active."""
|
||||
transport = ASGITransport(app=search_app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def search_authed_client(
|
||||
search_client: AsyncClient, db_session: AsyncSession
|
||||
) -> tuple[AsyncClient, dict]:
|
||||
"""Authenticated admin client with seeded data and unified search plugin active."""
|
||||
seed = await seed_tenant_and_users(db_session)
|
||||
await login_client(search_client, "admin@tenanta.com")
|
||||
return search_client, seed
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def mock_external_calls():
|
||||
"""Mock all external API calls (LiteLLM, job queue) for all tests."""
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.choices = [MagicMock()]
|
||||
mock_resp.choices[0].message.content = json.dumps({
|
||||
"normalized_query": "test",
|
||||
"entities": {},
|
||||
"intent": "search",
|
||||
"semantic_terms": [],
|
||||
"suggested_filters": {},
|
||||
})
|
||||
|
||||
mock_emb_resp = MagicMock()
|
||||
mock_emb_resp.data = [{"embedding": [0.1] * 768}]
|
||||
|
||||
with (
|
||||
patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_resp),
|
||||
patch("litellm.aembedding", new_callable=AsyncMock, return_value=mock_emb_resp),
|
||||
patch("app.core.jobs.enqueue_job", new_callable=AsyncMock, return_value="job-123"),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
# ─── AC1: POST /api/v1/search → 200 + results array ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
|
||||
async def test_ac1_search_returns_results(
|
||||
mock_hybrid_search,
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC1: POST /api/v1/search with query returns 200 + results array."""
|
||||
client, _ = search_authed_client
|
||||
mock_hybrid_search.return_value = [
|
||||
{
|
||||
"entity_type": "company",
|
||||
"entity_id": str(uuid.uuid4()),
|
||||
"title": "Company Alpha",
|
||||
"snippet": "IT company",
|
||||
"score": 0.95,
|
||||
"data": {"industry": "IT"},
|
||||
}
|
||||
]
|
||||
|
||||
resp = await client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": "Company"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "results" in data
|
||||
assert isinstance(data["results"], list)
|
||||
assert len(data["results"]) > 0
|
||||
assert data["results"][0]["entity_type"] == "company"
|
||||
assert data["results"][0]["title"] == "Company Alpha"
|
||||
assert "score" in data["results"][0]
|
||||
assert "query" in data
|
||||
assert "normalized_query" in data
|
||||
assert "summary" in data
|
||||
|
||||
|
||||
# ─── AC2: entity_types filter ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
|
||||
async def test_ac2_search_entity_types_filter(
|
||||
mock_hybrid_search,
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC2: Search with entity_types filters correctly."""
|
||||
client, _ = search_authed_client
|
||||
mock_hybrid_search.return_value = []
|
||||
|
||||
resp = await client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": "test", "entity_types": ["company"]},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
# Verify hybrid_search was called with entity_types=["company"]
|
||||
call_kwargs = mock_hybrid_search.call_args
|
||||
assert call_kwargs.kwargs.get("entity_types") == ["company"]
|
||||
|
||||
|
||||
# ─── AC3: No auth → 401 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac3_search_no_auth_401(search_client: AsyncClient):
|
||||
"""AC3: Search without auth returns 401."""
|
||||
resp = await search_client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": "test"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
# ─── AC4: Tenant isolation ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
|
||||
async def test_ac4_search_tenant_isolation(
|
||||
mock_hybrid_search,
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC4: Search only returns results from the user's tenant."""
|
||||
client, seed = search_authed_client
|
||||
mock_hybrid_search.return_value = []
|
||||
|
||||
resp = await client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": "test"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
# Verify hybrid_search was called with tenant A's ID
|
||||
call_args = mock_hybrid_search.call_args
|
||||
tenant_id_arg = call_args.kwargs.get("tenant_id")
|
||||
assert tenant_id_arg == seed["tenant_a"].id
|
||||
|
||||
|
||||
# ─── AC5: Empty query → 422 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac5_search_empty_query_422(search_authed_client: tuple[AsyncClient, dict]):
|
||||
"""AC5: Empty query returns 422 (min_length=1)."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/search",
|
||||
json={"query": ""},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
# ─── AC6: GET /api/v1/search/suggest → suggestions ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac6_suggest_returns_suggestions(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC6: GET /api/v1/search/suggest returns 200 + suggestions list."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.get(
|
||||
"/api/v1/search/suggest",
|
||||
params={"q": "comp"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "suggestions" in data
|
||||
assert isinstance(data["suggestions"], list)
|
||||
|
||||
|
||||
# ─── AC7: Empty suggest query → 422 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac7_suggest_empty_query_422(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC7: GET /api/v1/search/suggest with empty query returns 422 (min_length=1)."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.get(
|
||||
"/api/v1/search/suggest",
|
||||
params={"q": ""},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
# ─── AC8: POST /api/v1/search/similar → similar results ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac8_similar_returns_results(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC8: POST /api/v1/search/similar returns 200 + similar dict."""
|
||||
client, seed = search_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/search/similar",
|
||||
json={
|
||||
"entity_type": "company",
|
||||
"entity_id": str(seed["company_a"].id),
|
||||
"limit": 5,
|
||||
},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert "similar" in data
|
||||
assert isinstance(data["similar"], dict)
|
||||
|
||||
|
||||
# ─── AC9: Similar tenant isolation ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac9_similar_tenant_isolation(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC9: Similar search respects tenant isolation."""
|
||||
client, seed = search_authed_client
|
||||
# Use company from tenant B
|
||||
resp = await client.post(
|
||||
"/api/v1/search/similar",
|
||||
json={
|
||||
"entity_type": "company",
|
||||
"entity_id": str(seed["company_b"].id),
|
||||
"limit": 5,
|
||||
},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
# Should return 200 but with empty similar (tenant B company not accessible from tenant A)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert isinstance(data["similar"], dict)
|
||||
# Company B should not be found in tenant A's context
|
||||
for etype, items in data["similar"].items():
|
||||
for item in items:
|
||||
assert item.get("entity_id") != str(seed["company_b"].id)
|
||||
|
||||
|
||||
# ─── AC10: GET /api/v1/search/providers → provider list ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac10_list_providers(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC10: GET /api/v1/search/providers returns 200 + active provider list."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.get(
|
||||
"/api/v1/search/providers",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert isinstance(data, list)
|
||||
# After activation, providers should be registered
|
||||
if len(data) > 0:
|
||||
assert "entity_type" in data[0]
|
||||
assert "plugin_name" in data[0]
|
||||
assert "is_active" in data[0]
|
||||
assert data[0]["is_active"] is True
|
||||
|
||||
|
||||
# ─── AC11: POST /api/v1/search/reindex as admin → 200 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac11_reindex_admin_200(
|
||||
search_authed_client: tuple[AsyncClient, dict],
|
||||
):
|
||||
"""AC11: Admin can trigger reindex (200)."""
|
||||
client, _ = search_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/search/reindex",
|
||||
json={"entity_types": ["company"]},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["status"] == "ok"
|
||||
assert "entity_types" in data
|
||||
|
||||
|
||||
# ─── AC12: POST /api/v1/search/reindex as viewer → 403 ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ac12_reindex_viewer_403(
|
||||
search_client: AsyncClient, db_session: AsyncSession
|
||||
):
|
||||
"""AC12: Non-admin (viewer) gets 403 on reindex."""
|
||||
await seed_tenant_and_users(db_session)
|
||||
await login_client(search_client, "viewer@tenanta.com")
|
||||
|
||||
resp = await search_client.post(
|
||||
"/api/v1/search/reindex",
|
||||
json={"entity_types": ["company"]},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 403
|
||||
|
||||
|
||||
# ─── Unit Tests: RRF Fusion ───
|
||||
|
||||
|
||||
def test_rrf_fusion_combines_fts_and_vector_scores():
|
||||
"""RRF fusion combines FTS + Vector scores correctly."""
|
||||
fts_results = [
|
||||
{"id": "1", "title": "Result A", "score": 0.9},
|
||||
{"id": "2", "title": "Result B", "score": 0.8},
|
||||
]
|
||||
vec_results = [
|
||||
{"id": "2", "title": "Result B", "score": 0.95},
|
||||
{"id": "3", "title": "Result C", "score": 0.7},
|
||||
]
|
||||
|
||||
fused = rrf_fusion(fts_results, vec_results, "company")
|
||||
|
||||
# All 3 unique IDs should be present
|
||||
ids = [str(r.get("id", "")) for r in fused]
|
||||
assert "1" in ids
|
||||
assert "2" in ids
|
||||
assert "3" in ids
|
||||
|
||||
# Results should be sorted by fused score descending
|
||||
assert fused[0]["_score"] >= fused[-1]["_score"]
|
||||
|
||||
# ID 2 appears in both FTS and vector, so should have highest score
|
||||
assert str(fused[0].get("id", "")) == "2"
|
||||
|
||||
|
||||
def test_rrf_fusion_empty_inputs():
|
||||
"""RRF fusion with empty inputs returns empty list."""
|
||||
fused = rrf_fusion([], [], "company")
|
||||
assert fused == []
|
||||
|
||||
|
||||
def test_rrf_fusion_fts_only():
|
||||
"""RRF fusion with only FTS results returns them sorted by score."""
|
||||
fts_results = [
|
||||
{"id": "1", "title": "A"},
|
||||
{"id": "2", "title": "B"},
|
||||
]
|
||||
fused = rrf_fusion(fts_results, [], "company")
|
||||
assert len(fused) == 2
|
||||
# First result (rank 0) should have higher score
|
||||
assert fused[0]["_score"] > fused[1]["_score"]
|
||||
|
||||
|
||||
# ─── Unit Tests: Text Extraction ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_text_from_text_file(tmp_path):
|
||||
"""Extract text from a plain text file."""
|
||||
test_file = tmp_path / "test.txt"
|
||||
test_file.write_text("Hello World\nThis is a test.", encoding="utf-8")
|
||||
|
||||
result = await extract_text_from_file(str(test_file), "text/plain")
|
||||
assert "Hello World" in result
|
||||
assert "test" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_text_from_pdf_mocked(tmp_path):
|
||||
"""Extract text from PDF with mocked PyMuPDF."""
|
||||
test_file = tmp_path / "test.pdf"
|
||||
test_file.write_bytes(b"%PDF-1.4 fake")
|
||||
|
||||
mock_page = MagicMock()
|
||||
mock_page.get_text.return_value = "PDF content text"
|
||||
mock_doc = MagicMock()
|
||||
mock_doc.__iter__ = MagicMock(return_value=iter([mock_page]))
|
||||
mock_doc.close = MagicMock()
|
||||
|
||||
with patch("fitz.open", return_value=mock_doc):
|
||||
result = await extract_text_from_file(str(test_file), "application/pdf")
|
||||
assert "PDF content text" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_text_unsupported_mime():
|
||||
"""Unsupported MIME type returns empty string."""
|
||||
result = await extract_text_from_file("/tmp/nonexistent", "application/octet-stream")
|
||||
assert result == ""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_text_image_returns_empty(tmp_path):
|
||||
"""Image MIME type returns empty string (OCR not implemented)."""
|
||||
test_file = tmp_path / "image.png"
|
||||
test_file.write_bytes(b"\x89PNG\r\n\x1a\n")
|
||||
result = await extract_text_from_file(str(test_file), "image/png")
|
||||
assert result == ""
|
||||
|
||||
|
||||
# ─── Unit Tests: Provider Registry ───
|
||||
|
||||
|
||||
def test_provider_register_and_unregister():
|
||||
"""Provider can be registered and unregistered."""
|
||||
registry = SearchProviderRegistry()
|
||||
|
||||
# Create a mock provider
|
||||
mock_provider = MagicMock()
|
||||
mock_provider.entity_type = "test_entity"
|
||||
|
||||
# Register
|
||||
registry.register(mock_provider)
|
||||
assert registry.get("test_entity") is mock_provider
|
||||
|
||||
# Unregister
|
||||
registry.unregister("test_entity")
|
||||
assert registry.get("test_entity") is None
|
||||
|
||||
|
||||
def test_provider_get_all():
|
||||
"""get_all returns all registered providers."""
|
||||
registry = SearchProviderRegistry()
|
||||
|
||||
p1 = MagicMock()
|
||||
p1.entity_type = "contact"
|
||||
p2 = MagicMock()
|
||||
p2.entity_type = "company"
|
||||
|
||||
registry.register(p1)
|
||||
registry.register(p2)
|
||||
|
||||
all_providers = registry.get_all()
|
||||
assert len(all_providers) == 2
|
||||
entity_types = [p.entity_type for p in all_providers]
|
||||
assert "contact" in entity_types
|
||||
assert "company" in entity_types
|
||||
|
||||
|
||||
def test_provider_clear():
|
||||
"""clear removes all providers."""
|
||||
registry = SearchProviderRegistry()
|
||||
p = MagicMock()
|
||||
p.entity_type = "test"
|
||||
registry.register(p)
|
||||
assert len(registry.get_all()) == 1
|
||||
|
||||
registry.clear()
|
||||
assert len(registry.get_all()) == 0
|
||||
|
||||
|
||||
def test_provider_get_entity_types():
|
||||
"""get_entity_types returns list of registered entity types."""
|
||||
registry = SearchProviderRegistry()
|
||||
p1 = MagicMock()
|
||||
p1.entity_type = "contact"
|
||||
p2 = MagicMock()
|
||||
p2.entity_type = "company"
|
||||
registry.register(p1)
|
||||
registry.register(p2)
|
||||
|
||||
types = registry.get_entity_types()
|
||||
assert "contact" in types
|
||||
assert "company" in types
|
||||
assert len(types) == 2
|
||||
|
||||
|
||||
# ─── Unit Tests: Autocomplete ───
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_autocomplete_empty_query_returns_empty(db_session: AsyncSession):
|
||||
"""Autocomplete with empty query returns empty list."""
|
||||
from app.plugins.builtins.unified_search.search_engine import autocomplete
|
||||
|
||||
result = await autocomplete(db_session, "", uuid.uuid4(), 10)
|
||||
assert result == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_autocomplete_returns_titles(db_session: AsyncSession):
|
||||
"""Autocomplete returns titles from FTS prefix search."""
|
||||
from app.plugins.builtins.unified_search.search_engine import autocomplete
|
||||
|
||||
# With no providers registered and no data, should return empty list
|
||||
result = await autocomplete(db_session, "test", uuid.uuid4(), 10)
|
||||
assert isinstance(result, list)
|
||||
assert len(result) <= 10
|
||||
Reference in New Issue
Block a user