leocrm/tests/test_unified_search.py

"""Tests for the Unified Search plugin — covers ACs 1-12 and unit tests.

Tests hybrid search infrastructure: API endpoints, RRF fusion, autocomplete,
text extraction, and provider registry.
"""

from __future__ import annotations

import json
import uuid
from unittest.mock import AsyncMock, MagicMock, patch

import pytest
import pytest_asyncio
from httpx import ASGITransport, AsyncClient
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker

from app.core.db import close_engine, reset_engine_for_testing
from app.core.service_container import get_container
from app.main import create_app
from app.plugins.builtins.unified_search import UnifiedSearchPlugin
from app.plugins.builtins.unified_search.provider_registry import (
    SearchProviderRegistry,
    get_search_registry,
)
from app.plugins.builtins.unified_search.search_engine import rrf_fusion
from app.plugins.builtins.unified_search.text_extraction import extract_text_from_file
from app.plugins.registry import reset_registry_for_testing
from app.services.plugin_service import reset_plugin_service_for_testing
from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users

# ─── Unified Search Fixtures ───


@pytest_asyncio.fixture
async def search_app(engine: AsyncEngine, redis_client):
    """FastAPI app with UnifiedSearch plugin registered, installed, and activated."""
    reset_engine_for_testing(engine)
    app = create_app()
    registry = reset_registry_for_testing()
    registry.initialize(engine, app)
    container = get_container()
    await container.initialize()
    registry.register_plugin(UnifiedSearchPlugin())
    reset_plugin_service_for_testing(registry)

    sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
    async with sf() as session:
        await registry.install(session, "unified_search")
        await registry.activate(session, "unified_search")
        await session.commit()

    yield app
    await close_engine()


@pytest_asyncio.fixture
async def search_client(search_app) -> AsyncClient:
    """HTTP test client with unified search plugin active."""
    transport = ASGITransport(app=search_app)
    async with AsyncClient(transport=transport, base_url="http://test") as c:
        yield c


@pytest_asyncio.fixture
async def search_authed_client(
    search_client: AsyncClient, db_session: AsyncSession
) -> tuple[AsyncClient, dict]:
    """Authenticated admin client with seeded data and unified search plugin active."""
    seed = await seed_tenant_and_users(db_session)
    # Grant is_system_admin to admin user so search:read/search:admin permissions pass
    from sqlalchemy import update
    from app.models.user import User
    await db_session.execute(
        update(User)
        .where(User.email == "admin@tenanta.com")
        .values(is_system_admin=True)
    )
    await db_session.commit()
    # Login and capture CSRF token
    login_resp = await search_client.post(
        "/api/v1/auth/login",
        json={"email": "admin@tenanta.com", "password": "TestPass123!"},
        headers=ORIGIN_HEADER,
    )
    assert login_resp.status_code == 200, f"Login failed: {login_resp.text}"
    csrf_token = login_resp.json().get("csrf_token", "")
    search_client.headers["X-CSRF-Token"] = csrf_token
    return search_client, seed


@pytest.fixture(autouse=True)
async def mock_external_calls():
    """Mock all external API calls (LiteLLM, job queue) for all tests."""
    mock_resp = MagicMock()
    mock_resp.choices = [MagicMock()]
    mock_resp.choices[0].message.content = json.dumps({
        "normalized_query": "test",
        "entities": {},
        "intent": "search",
        "semantic_terms": [],
        "suggested_filters": {},
    })

    mock_emb_resp = MagicMock()
    mock_emb_resp.data = [{"embedding": [0.1] * 768}]

    with (
        patch("litellm.acompletion", new_callable=AsyncMock, return_value=mock_resp),
        patch("litellm.aembedding", new_callable=AsyncMock, return_value=mock_emb_resp),
        patch("app.core.jobs.enqueue_job", new_callable=AsyncMock, return_value="job-123"),
    ):
        yield


# ─── AC1: POST /api/v1/search → 200 + results array ───


@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
async def test_ac1_search_returns_results(
    mock_hybrid_search,
    search_authed_client: tuple[AsyncClient, dict],
):
    """AC1: POST /api/v1/search with query returns 200 + results array."""
    client, _ = search_authed_client
    mock_hybrid_search.return_value = [
        {
            "entity_type": "company",
            "entity_id": str(uuid.uuid4()),
            "title": "Company Alpha",
            "snippet": "IT company",
            "score": 0.95,
            "data": {"industry": "IT"},
        }
    ]

    resp = await client.post(
        "/api/v1/search",
        json={"query": "Company"},
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 200
    data = resp.json()
    assert "results" in data
    assert isinstance(data["results"], list)
    assert len(data["results"]) > 0
    assert data["results"][0]["entity_type"] == "company"
    assert data["results"][0]["title"] == "Company Alpha"
    assert "score" in data["results"][0]
    assert "query" in data
    assert "normalized_query" in data
    assert "summary" in data


# ─── AC2: entity_types filter ───


@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
async def test_ac2_search_entity_types_filter(
    mock_hybrid_search,
    search_authed_client: tuple[AsyncClient, dict],
):
    """AC2: Search with entity_types filters correctly."""
    client, _ = search_authed_client
    mock_hybrid_search.return_value = []

    resp = await client.post(
        "/api/v1/search",
        json={"query": "test", "entity_types": ["company"]},
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 200
    # Verify hybrid_search was called with entity_types=["company"]
    call_kwargs = mock_hybrid_search.call_args
    assert call_kwargs.kwargs.get("entity_types") == ["company"]


# ─── AC3: No auth → 401 ───


@pytest.mark.asyncio
async def test_ac3_search_no_auth_401(search_client: AsyncClient):
    """AC3: Search without auth returns 401 or 403 (CSRF may block first)."""
    resp = await search_client.post(
        "/api/v1/search",
        json={"query": "test"},
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code in (401, 403)


# ─── AC4: Tenant isolation ───


@pytest.mark.asyncio
@patch("app.plugins.builtins.unified_search.routes.hybrid_search", new_callable=AsyncMock)
async def test_ac4_search_tenant_isolation(
    mock_hybrid_search,
    search_authed_client: tuple[AsyncClient, dict],
):
    """AC4: Search only returns results from the user's tenant."""
    client, seed = search_authed_client
    mock_hybrid_search.return_value = []

    resp = await client.post(
        "/api/v1/search",
        json={"query": "test"},
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 200

    # Verify hybrid_search was called with tenant A's ID
    call_args = mock_hybrid_search.call_args
    tenant_id_arg = call_args.kwargs.get("tenant_id")
    assert tenant_id_arg == seed["tenant_a"].id


# ─── AC5: Empty query → 422 ───


@pytest.mark.asyncio
async def test_ac5_search_empty_query_422(search_authed_client: tuple[AsyncClient, dict]):
    """AC5: Empty query returns 422 (min_length=1)."""
    client, _ = search_authed_client
    resp = await client.post(
        "/api/v1/search",
        json={"query": ""},
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 422


# ─── AC6: GET /api/v1/search/suggest → suggestions ───


@pytest.mark.asyncio
async def test_ac6_suggest_returns_suggestions(
    search_authed_client: tuple[AsyncClient, dict],
):
    """AC6: GET /api/v1/search/suggest returns 200 + suggestions list."""
    client, _ = search_authed_client
    resp = await client.get(
        "/api/v1/search/suggest",
        params={"q": "comp"},
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 200
    data = resp.json()
    assert "suggestions" in data
    assert isinstance(data["suggestions"], list)


# ─── AC7: Empty suggest query → 422 ───


@pytest.mark.asyncio
async def test_ac7_suggest_empty_query_422(
    search_authed_client: tuple[AsyncClient, dict],
):
    """AC7: GET /api/v1/search/suggest with empty query returns 422 (min_length=1)."""
    client, _ = search_authed_client
    resp = await client.get(
        "/api/v1/search/suggest",
        params={"q": ""},
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 422


# ─── AC8: POST /api/v1/search/similar → similar results ───


@pytest.mark.asyncio
async def test_ac8_similar_returns_results(
    search_authed_client: tuple[AsyncClient, dict],
):
    """AC8: POST /api/v1/search/similar returns 200 + similar dict."""
    client, seed = search_authed_client
    resp = await client.post(
        "/api/v1/search/similar",
        json={
            "entity_type": "company",
            "entity_id": str(seed["company_a"].id),
            "limit": 5,
        },
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 200
    data = resp.json()
    assert "similar" in data
    assert isinstance(data["similar"], dict)


# ─── AC9: Similar tenant isolation ───


@pytest.mark.asyncio
async def test_ac9_similar_tenant_isolation(
    search_authed_client: tuple[AsyncClient, dict],
):
    """AC9: Similar search respects tenant isolation."""
    client, seed = search_authed_client
    # Use company from tenant B
    resp = await client.post(
        "/api/v1/search/similar",
        json={
            "entity_type": "company",
            "entity_id": str(seed["company_b"].id),
            "limit": 5,
        },
        headers=ORIGIN_HEADER,
    )
    # Should return 200 but with empty similar (tenant B company not accessible from tenant A)
    assert resp.status_code == 200
    data = resp.json()
    assert isinstance(data["similar"], dict)
    # Company B should not be found in tenant A's context
    for etype, items in data["similar"].items():
        for item in items:
            assert item.get("entity_id") != str(seed["company_b"].id)


# ─── AC10: GET /api/v1/search/providers → provider list ───


@pytest.mark.asyncio
async def test_ac10_list_providers(
    search_authed_client: tuple[AsyncClient, dict],
):
    """AC10: GET /api/v1/search/providers returns 200 + active provider list."""
    client, _ = search_authed_client
    resp = await client.get(
        "/api/v1/search/providers",
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 200
    data = resp.json()
    assert isinstance(data, list)
    # After activation, providers should be registered
    if len(data) > 0:
        assert "entity_type" in data[0]
        assert "plugin_name" in data[0]
        assert "is_active" in data[0]
        assert data[0]["is_active"] is True


# ─── AC11: POST /api/v1/search/reindex as admin → 200 ───


@pytest.mark.asyncio
async def test_ac11_reindex_admin_200(
    search_authed_client: tuple[AsyncClient, dict],
):
    """AC11: Admin can trigger reindex (200)."""
    client, _ = search_authed_client
    resp = await client.post(
        "/api/v1/search/reindex",
        json={"entity_types": ["company"]},
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 200
    data = resp.json()
    assert data["status"] == "ok"
    assert "entity_types" in data


# ─── AC12: POST /api/v1/search/reindex as viewer → 403 ───


@pytest.mark.asyncio
async def test_ac12_reindex_viewer_403(
    search_client: AsyncClient, db_session: AsyncSession
):
    """AC12: Non-admin (viewer) gets 403 on reindex."""
    await seed_tenant_and_users(db_session)
    await login_client(search_client, "viewer@tenanta.com")

    resp = await search_client.post(
        "/api/v1/search/reindex",
        json={"entity_types": ["company"]},
        headers=ORIGIN_HEADER,
    )
    assert resp.status_code == 403


# ─── Unit Tests: RRF Fusion ───


def test_rrf_fusion_combines_fts_and_vector_scores():
    """RRF fusion combines FTS + Vector scores correctly."""
    fts_results = [
        {"id": "1", "title": "Result A", "score": 0.9},
        {"id": "2", "title": "Result B", "score": 0.8},
    ]
    vec_results = [
        {"id": "2", "title": "Result B", "score": 0.95},
        {"id": "3", "title": "Result C", "score": 0.7},
    ]

    fused = rrf_fusion(fts_results, vec_results, "company")

    # All 3 unique IDs should be present
    ids = [str(r.get("id", "")) for r in fused]
    assert "1" in ids
    assert "2" in ids
    assert "3" in ids

    # Results should be sorted by fused score descending
    assert fused[0]["_score"] >= fused[-1]["_score"]

    # ID 2 appears in both FTS and vector, so should have highest score
    assert str(fused[0].get("id", "")) == "2"


def test_rrf_fusion_empty_inputs():
    """RRF fusion with empty inputs returns empty list."""
    fused = rrf_fusion([], [], "company")
    assert fused == []


def test_rrf_fusion_fts_only():
    """RRF fusion with only FTS results returns them sorted by score."""
    fts_results = [
        {"id": "1", "title": "A"},
        {"id": "2", "title": "B"},
    ]
    fused = rrf_fusion(fts_results, [], "company")
    assert len(fused) == 2
    # First result (rank 0) should have higher score
    assert fused[0]["_score"] > fused[1]["_score"]


# ─── Unit Tests: Text Extraction ───


@pytest.mark.asyncio
async def test_extract_text_from_text_file(tmp_path):
    """Extract text from a plain text file."""
    test_file = tmp_path / "test.txt"
    test_file.write_text("Hello World\nThis is a test.", encoding="utf-8")

    result = await extract_text_from_file(str(test_file), "text/plain")
    assert "Hello World" in result
    assert "test" in result


@pytest.mark.asyncio
async def test_extract_text_from_pdf_mocked(tmp_path):
    """Extract text from PDF with mocked PyMuPDF."""
    test_file = tmp_path / "test.pdf"
    test_file.write_bytes(b"%PDF-1.4 fake")

    mock_page = MagicMock()
    mock_page.get_text.return_value = "PDF content text"
    mock_doc = MagicMock()
    mock_doc.__iter__ = MagicMock(return_value=iter([mock_page]))
    mock_doc.close = MagicMock()

    with patch("fitz.open", return_value=mock_doc):
        result = await extract_text_from_file(str(test_file), "application/pdf")
    assert "PDF content text" in result


@pytest.mark.asyncio
async def test_extract_text_unsupported_mime():
    """Unsupported MIME type returns empty string."""
    result = await extract_text_from_file("/tmp/nonexistent", "application/octet-stream")
    assert result == ""


@pytest.mark.asyncio
async def test_extract_text_image_returns_empty(tmp_path):
    """Image MIME type returns empty string (OCR not implemented)."""
    test_file = tmp_path / "image.png"
    test_file.write_bytes(b"\x89PNG\r\n\x1a\n")
    result = await extract_text_from_file(str(test_file), "image/png")
    assert result == ""


# ─── Unit Tests: Provider Registry ───


def test_provider_register_and_unregister():
    """Provider can be registered and unregistered."""
    registry = SearchProviderRegistry()

    # Create a mock provider
    mock_provider = MagicMock()
    mock_provider.entity_type = "test_entity"

    # Register
    registry.register(mock_provider)
    assert registry.get("test_entity") is mock_provider

    # Unregister
    registry.unregister("test_entity")
    assert registry.get("test_entity") is None


def test_provider_get_all():
    """get_all returns all registered providers."""
    registry = SearchProviderRegistry()

    p1 = MagicMock()
    p1.entity_type = "contact"
    p2 = MagicMock()
    p2.entity_type = "company"

    registry.register(p1)
    registry.register(p2)

    all_providers = registry.get_all()
    assert len(all_providers) == 2
    entity_types = [p.entity_type for p in all_providers]
    assert "contact" in entity_types
    assert "company" in entity_types


def test_provider_clear():
    """clear removes all providers."""
    registry = SearchProviderRegistry()
    p = MagicMock()
    p.entity_type = "test"
    registry.register(p)
    assert len(registry.get_all()) == 1

    registry.clear()
    assert len(registry.get_all()) == 0


def test_provider_get_entity_types():
    """get_entity_types returns list of registered entity types."""
    registry = SearchProviderRegistry()
    p1 = MagicMock()
    p1.entity_type = "contact"
    p2 = MagicMock()
    p2.entity_type = "company"
    registry.register(p1)
    registry.register(p2)

    types = registry.get_entity_types()
    assert "contact" in types
    assert "company" in types
    assert len(types) == 2


# ─── Unit Tests: Autocomplete ───


@pytest.mark.asyncio
async def test_autocomplete_empty_query_returns_empty(db_session: AsyncSession):
    """Autocomplete with empty query returns empty list."""
    from app.plugins.builtins.unified_search.search_engine import autocomplete

    result = await autocomplete(db_session, "", uuid.uuid4(), 10)
    assert result == []


@pytest.mark.asyncio
async def test_autocomplete_returns_titles(db_session: AsyncSession):
    """Autocomplete returns titles from FTS prefix search."""
    from app.plugins.builtins.unified_search.search_engine import autocomplete

    # With no providers registered and no data, should return empty list
    result = await autocomplete(db_session, "test", uuid.uuid4(), 10)
    assert isinstance(result, list)
    assert len(result) <= 10