leocrm/app/plugins/builtins/unified_search/routes.py

"""API routes for the Unified Search plugin."""

from __future__ import annotations

import logging
import uuid
from datetime import UTC, datetime
from typing import Any

from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.db import get_db
from app.core.jobs import enqueue_job
from app.core.permissions import filter_fields_by_permission, resolve_permissions
from app.deps import get_current_user, require_permission
from app.plugins.builtins.unified_search.provider_registry import get_search_registry
from app.plugins.builtins.unified_search.query_understanding import (
    llm_aggregate_results,
    llm_analyze_query,
)
from app.plugins.builtins.unified_search.schemas import (
    FacetsResponse,
    ProviderResponse,
    ReindexRequest,
    SearchRequest,
    SearchResponse,
    SearchResult,
    SimilarRequest,
    SimilarResponse,
    SuggestResponse,
)
from app.plugins.builtins.unified_search.search_engine import (
    autocomplete,
    find_similar_all_types,
    hybrid_search,
)

logger = logging.getLogger(__name__)

router = APIRouter(prefix="/api/v1/search", tags=["search"])


# ─── Search ───

@router.get("", dependencies=[Depends(require_permission("search:read"))])
async def search_get(
    q: str = Query(..., min_length=1, max_length=500, description="Search query"),
    entity_types: str | None = Query(None, description="Comma-separated entity types to search"),
    limit: int = Query(default=20, ge=1, le=100),
    offset: int = Query(default=0, ge=0),
    date_from: str | None = Query(None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at >= date_from"),
    date_to: str | None = Query(None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at <= date_to"),
    tags: str | None = Query(None, description="Comma-separated tags to filter results"),
    sort: str = Query(default="relevance", description="Sort order: relevance, date, name"),
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> SearchResponse:
    """Perform hybrid search via GET (same as POST but with query params)."""
    types_list = entity_types.split(",") if entity_types else None
    tags_list = tags.split(",") if tags else None
    req = SearchRequest(
        query=q,
        entity_types=types_list,
        limit=limit,
        offset=offset,
        date_from=date_from,
        date_to=date_to,
        tags=tags_list,
        sort=sort,
    )
    return await _do_search(req, current_user, db)


def _parse_date(value: str | None) -> datetime | None:
    """Parse an ISO date string (YYYY-MM-DD) into a timezone-aware datetime."""
    if not value:
        return None
    try:
        dt = datetime.fromisoformat(value)
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=UTC)
        return dt
    except ValueError:
        return None


def _apply_filters_and_sort(
    results: list[dict],
    req: SearchRequest,
) -> list[dict]:
    """Apply post-search date/tags filters and sort order to results."""
    date_from = _parse_date(req.date_from)
    date_to = _parse_date(req.date_to)

    filtered: list[dict] = []
    for r in results:
        # Date range filter on created_at/updated_at
        if date_from or date_to:
            created = r.get("_created_at")
            updated = r.get("_updated_at")
            ts = updated or created
            if ts is None:
                continue
            if isinstance(ts, str):
                try:
                    ts = datetime.fromisoformat(ts.replace("Z", "+00:00"))
                except ValueError:
                    continue
            if ts.tzinfo is None:
                ts = ts.replace(tzinfo=UTC)
            if date_from and ts < date_from:
                continue
            if date_to and ts > date_to:
                continue

        # Tags filter (comma-separated on entity)
        if req.tags:
            entity_tags = r.get("_tags") or ""
            tag_set = {t.strip().lower() for t in entity_tags.split(",") if t.strip()}
            if not any(t.lower() in tag_set for t in req.tags):
                continue

        filtered.append(r)

    # Sort order
    sort = (req.sort or "relevance").lower()
    if sort == "date":
        filtered.sort(
            key=lambda x: (x.get("_updated_at") or x.get("_created_at") or ""),
            reverse=True,
        )
    elif sort == "name":
        filtered.sort(key=lambda x: (x.get("title") or "").lower())
    # 'relevance' keeps the existing score order

    return filtered


async def _do_search(
    req: SearchRequest,
    current_user: dict,
    db: AsyncSession,
) -> SearchResponse:
    """Shared search logic used by both GET and POST endpoints."""
    tenant_id = uuid.UUID(current_user["tenant_id"])
    user_id = uuid.UUID(current_user["user_id"])
    is_system_admin = current_user.get("is_system_admin", False)

    # KI query understanding (skip if use_ai=False for performance)
    use_ai = getattr(req, "use_ai", True)
    if use_ai:
        query_analysis = await llm_analyze_query(req.query, db=db, tenant_id=tenant_id)
    else:
        from app.plugins.builtins.unified_search.query_understanding import _fallback_query_analysis
        query_analysis = _fallback_query_analysis(req.query)

    # Hybrid search with visibility filtering
    results = await hybrid_search(
        db=db,
        query_analysis=query_analysis,
        tenant_id=tenant_id,
        entity_types=req.entity_types,
        limit=req.limit,
        user_id=user_id,
        is_system_admin=is_system_admin,
    )

    # Apply post-search filters (date range, tags) and sort
    results = _apply_filters_and_sort(results, req)

    # Resolve user permissions for field-level RBAC
    resolved_perms = await resolve_permissions(db, user_id, tenant_id)

    # Map entity_type to module name for field-level permissions
    _entity_to_module = {
        "contact": "contacts",
        "mail": "mail",
        "file": "dms",
        "event": "calendar",
    }

    # KI result aggregation (skip if use_ai=False for performance)
    if use_ai:
        aggregation = await llm_aggregate_results(results, req.query, db=db, tenant_id=tenant_id)
    else:
        from app.plugins.builtins.unified_search.query_understanding import _fallback_aggregate
        aggregation = _fallback_aggregate(results, req.query)

    search_results = [
        SearchResult(
            entity_type=r.get("entity_type", ""),
            entity_id=r.get("entity_id", ""),
            title=r.get("title", ""),
            snippet=r.get("snippet", ""),
            score=r.get("score", 0.0),
            data=filter_fields_by_permission(
                r.get("data", {}),
                resolved_perms,
                _entity_to_module.get(r.get("entity_type", ""), r.get("entity_type", "")),
            ),
        )
        for r in results
    ]

    return SearchResponse(
        query=req.query,
        normalized_query=query_analysis.get("normalized_query", req.query),
        results=search_results,
        facets=aggregation.get("facets", {}),
        summary=aggregation.get("summary", f"{len(results)} Ergebnisse"),
        suggestions=aggregation.get("suggestions", []),
    )


@router.post("", dependencies=[Depends(require_permission("search:read"))])
async def search(
    req: SearchRequest,
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> SearchResponse:
    """Perform hybrid search with KI query understanding."""
    return await _do_search(req, current_user, db)


# ─── Facets ───

@router.get("/facets", dependencies=[Depends(require_permission("search:read"))])
async def search_facets(
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> FacetsResponse:
    """Return available facets for search filtering (entity types, tags, date ranges)."""
    tenant_id = uuid.UUID(current_user["tenant_id"])

    # Entity types from the search registry
    registry = get_search_registry()
    entity_types = registry.get_entity_types()

    # Tags: distinct tags across searchable entities (comma-separated on entities)
    tags: set[str] = set()
    tag_tables = {
        "contacts": "tags",
        "companies": "tags",
    }
    for table, col in tag_tables.items():
        try:
            result = await db.execute(
                text(
                    f"SELECT DISTINCT unnest(string_to_array({col}, ',')) AS tag "
                    f"FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL AND {col} IS NOT NULL"
                ),
                {"tid": tenant_id},
            )
            for row in result.mappings().all():
                tag = (row.get("tag") or "").strip()
                if tag:
                    tags.add(tag)
        except Exception:
            logger.debug("Facet tag query failed for %s", table)

    # Date ranges: min/max created_at across searchable entities
    date_ranges: dict[str, Any] = {}
    date_tables = ["contacts", "mails", "files", "calendar_entries"]
    for table in date_tables:
        try:
            result = await db.execute(
                text(
                    f"SELECT min(created_at) AS min_dt, max(created_at) AS max_dt "
                    f"FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL"
                ),
                {"tid": tenant_id},
            )
            row = result.mappings().first()
            if row:
                date_ranges[table] = {
                    "min": str(row.get("min_dt")) if row.get("min_dt") else None,
                    "max": str(row.get("max_dt")) if row.get("max_dt") else None,
                }
        except Exception:
            logger.debug("Facet date query failed for %s", table)

    return FacetsResponse(
        entity_types=entity_types,
        tags=sorted(tags),
        date_ranges=date_ranges,
    )


# ─── Suggest / Autocomplete ───

@router.get("/suggest", dependencies=[Depends(require_permission("search:read"))])
async def suggest(
    q: str = Query(..., min_length=1, max_length=200),
    limit: int = Query(default=10, ge=1, le=50),
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> SuggestResponse:
    """Autocomplete suggestions using FTS prefix search."""
    tenant_id = uuid.UUID(current_user["tenant_id"])
    suggestions = await autocomplete(db, q, tenant_id, limit)
    return SuggestResponse(suggestions=suggestions)


# ─── Similar ───

@router.post("/similar", dependencies=[Depends(require_permission("search:read"))])
async def find_similar(
    req: SimilarRequest,
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> SimilarResponse:
    """Find similar entities across all types based on embedding."""
    tenant_id = uuid.UUID(current_user["tenant_id"])
    try:
        entity_id = uuid.UUID(req.entity_id)
    except ValueError:
        raise HTTPException(status_code=400, detail="Invalid entity_id") from None

    similar = await find_similar_all_types(
        db=db,
        entity_type=req.entity_type,
        entity_id=entity_id,
        tenant_id=tenant_id,
        limit=req.limit,
    )

    result_dict: dict[str, list[SearchResult]] = {}
    for etype, items in similar.items():
        result_dict[etype] = [
            SearchResult(
                entity_type=r.get("entity_type", etype),
                entity_id=r.get("entity_id", ""),
                title=r.get("title", ""),
                snippet=r.get("snippet", ""),
                score=r.get("score", 0.0),
                data=r.get("data", {}),
            )
            for r in items
        ]

    return SimilarResponse(similar=result_dict)


# ─── Reindex ───

@router.post("/reindex", dependencies=[Depends(require_permission("search:admin"))])
async def reindex(
    req: ReindexRequest,
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> dict:
    """Trigger reindexing of specified entity types."""
    entity_types = req.entity_types
    if not entity_types:
        entity_types = get_search_registry().get_entity_types()

    job_ids: list[str] = []
    for etype in entity_types:
        job_id = await enqueue_job("reindex", etype)
        if job_id:
            job_ids.append(job_id)

    # Optionally re-index file chunks
    if req.include_chunks and "file" in entity_types:
        chunk_job_id = await enqueue_job("reindex_all")
        if chunk_job_id:
            job_ids.append(chunk_job_id)

    return {
        "status": "ok",
        "message": f"Reindexing {len(entity_types)} entity types",
        "entity_types": entity_types,
        "include_chunks": req.include_chunks,
        "job_ids": job_ids,
    }


# ─── Rebuild / Purge ───

@router.post("/rebuild/{entity_type}/{entity_id}", dependencies=[Depends(require_permission("search:admin"))])
async def rebuild_entity_index(
    entity_type: str,
    entity_id: str,
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> dict:
    """Rebuild the search index for a single entity (admin only)."""
    tenant_id = uuid.UUID(current_user["tenant_id"])
    try:
        eid = uuid.UUID(entity_id)
    except ValueError:
        raise HTTPException(status_code=400, detail="Invalid entity_id") from None

    from app.plugins.builtins.unified_search.lifecycle import rebuild_index

    success = await rebuild_index(db, entity_type, eid, tenant_id)
    return {
        "status": "ok" if success else "failed",
        "entity_type": entity_type,
        "entity_id": entity_id,
        "message": "Index rebuilt" if success else "Rebuild failed — check logs",
    }


@router.post("/purge/{entity_type}/{entity_id}", dependencies=[Depends(require_permission("search:admin"))])
async def purge_entity_index(
    entity_type: str,
    entity_id: str,
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> dict:
    """Purge an entity from the search index (admin only, GDPR)."""
    tenant_id = uuid.UUID(current_user["tenant_id"])
    try:
        eid = uuid.UUID(entity_id)
    except ValueError:
        raise HTTPException(status_code=400, detail="Invalid entity_id") from None

    from app.plugins.builtins.unified_search.lifecycle import remove_chunks, remove_from_index

    await remove_from_index(db, entity_type, eid, tenant_id)
    if entity_type == "file":
        await remove_chunks(db, eid, tenant_id)

    return {
        "status": "ok",
        "entity_type": entity_type,
        "entity_id": entity_id,
        "message": "Purged from search index",
    }


# ─── Providers ───

@router.get("/providers", dependencies=[Depends(require_permission("search:read"))])
async def list_providers(
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> list[ProviderResponse]:
    """List all active search providers."""
    registry = get_search_registry()
    providers = registry.get_all()
    return [
        ProviderResponse(
            entity_type=p.entity_type,
            plugin_name="unified_search",
            is_active=True,
            supports_fts=getattr(p, "supports_fts", True),
            supports_vector=getattr(p, "supports_vector", True),
            supports_rag=getattr(p, "supports_rag", False),
            supports_graph=getattr(p, "supports_graph", False),
        )
        for p in providers
    ]


@router.post("/providers/{entity_type}/toggle", dependencies=[Depends(require_permission("search:admin"))])
async def toggle_provider(
    entity_type: str,
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> dict:
    """Toggle a search provider on/off."""
    registry = get_search_registry()
    provider = registry.get(entity_type)
    if provider is None:
        raise HTTPException(status_code=404, detail=f"Provider not found: {entity_type}")

    # Toggle in DB
    tenant_id = uuid.UUID(current_user["tenant_id"])
    result = await db.execute(
        text(
            """
            SELECT is_active FROM unified_search_providers
            WHERE tenant_id = :tid AND entity_type = :et
            """
        ),
        {"tid": tenant_id, "et": entity_type},
    )
    row = result.mappings().first()

    if row:
        new_status = not row["is_active"]
        await db.execute(
            text(
                """
                UPDATE unified_search_providers SET is_active = :active
                WHERE tenant_id = :tid AND entity_type = :et
                """
            ),
            {"active": new_status, "tid": tenant_id, "et": entity_type},
        )
    else:
        new_status = False
        await db.execute(
            text(
                """
                INSERT INTO unified_search_providers (tenant_id, entity_type, plugin_name, is_active)
                VALUES (:tid, :et, 'unified_search', false)
                """
            ),
            {"tid": tenant_id, "et": entity_type},
        )

    await db.commit()

    if not new_status:
        registry.unregister(entity_type)
    else:
        # Re-register by clearing and re-running auto_register
        from app.plugins.builtins.unified_search.provider_registry import auto_register_providers
        await auto_register_providers(db)

    return {
        "entity_type": entity_type,
        "is_active": new_status,
    }


# ─── Stats ───

@router.get("/stats", dependencies=[Depends(require_permission("search:read"))])
async def search_stats(
    current_user: dict = Depends(get_current_user),
    db: AsyncSession = Depends(get_db),
) -> dict:
    """Get search index statistics."""
    tenant_id = uuid.UUID(current_user["tenant_id"])

    stats: dict = {}
    tables = {
        "contacts": "embedding",
        "mails": "embedding",
        "files": "embedding",
        "calendar_entries": "embedding",
    }

    for table, col in tables.items():
        try:
            total_result = await db.execute(
                text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL"),
                {"tid": tenant_id},
            )
            total = total_result.scalar() or 0

            indexed_result = await db.execute(
                text(f"SELECT count(*) AS cnt FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL AND {col} IS NOT NULL"),
                {"tid": tenant_id},
            )
            indexed = indexed_result.scalar() or 0

            stats[table] = {
                "total": total,
                "indexed": indexed,
                "pending": total - indexed,
            }
        except Exception:
            logger.exception("Stats query failed for %s", table)
            stats[table] = {"total": 0, "indexed": 0, "pending": 0}

    # Last index log entries
    try:
        log_result = await db.execute(
            text(
                """
                SELECT entity_type, action, status, created_at
                FROM unified_search_index_log
                WHERE tenant_id = :tid
                ORDER BY created_at DESC
                LIMIT 10
                """
            ),
            {"tid": tenant_id},
        )
        recent_logs = [dict(r) for r in log_result.mappings().all()]
    except Exception:
        recent_logs = []

    stats["recent_logs"] = recent_logs
    return stats