feat(E): Unified Search — 24 Tasks complete
Check Cross-Plugin Imports / check (push) Has been cancelled
Check Cross-Plugin Imports / check (push) Has been cancelled
- SPIKE-E: FTS+Vector+Permission benchmark on 10k records (all <30ms) - E-PROV: supports_fts/vector/rag/graph capability flags on all providers - E-FTS/VEC: All 11 providers refactored to BaseSearchProvider with permission filtering - E-PERM: Over-fetch strategy for vector+permission (15x faster than ANY() filter) - E-FUSE: rrf_fusion_multi() for N-way RRF over FTS+Vector+RAG+Graph - E-LLM: Query understanding cleaned up to use central llm_complete() - E-CHUNK: Document chunking module + document_chunks table with HNSW index - E-EMB: Chunk embedding ARQ jobs (index_file_chunks, reindex_chunks) - E-RAG: RAG retrieval via FileSearchProvider.search_rag() - E-GRAPH: GraphRAG BFS traversal via GraphRAGSearchProvider.search_graph() - E-IX-EVT: Auto-indexing via outbox events + delete/cleanup handlers - E-IX-RE: Batch reindex with progress tracking + reindex_all job - E-DATA-LIFE: Lifecycle module (remove/rebuild/restore/correct) + API endpoints - E-K-MEM: AgentMemorySearchProvider - E-P-AI: AIChatSearchProvider - E-P-WF: WorkflowSearchProvider - E-P-COMM: ConversationSearchProvider verified (already on BaseSearchProvider) - E-API: Filter params (date_from/to, tags, sort) + /facets endpoint - E-TOOL: unified_search AI tool registered in ToolRegistry - E-MCP: Search tool in MCP server with normal RBAC/tenant checks - E-UI-CMD: CommandPalette (Cmd+K) with debounced search + recent searches - E-UI-FAC: SearchFacets, SearchResultCard, SavedSearches components - E-TEST: 40 new tests in test_unified_search_phase_e.py (105 total green) - E-DOC: api-documentation.md, plugin-development-guide.md, test-strategy.md updated 105 tests passing, TypeScript clean.
This commit is contained in:
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import text
|
||||
@@ -19,6 +20,7 @@ from app.plugins.builtins.unified_search.query_understanding import (
|
||||
llm_analyze_query,
|
||||
)
|
||||
from app.plugins.builtins.unified_search.schemas import (
|
||||
FacetsResponse,
|
||||
ProviderResponse,
|
||||
ReindexRequest,
|
||||
SearchRequest,
|
||||
@@ -48,15 +50,94 @@ async def search_get(
|
||||
entity_types: str | None = Query(None, description="Comma-separated entity types to search"),
|
||||
limit: int = Query(default=20, ge=1, le=100),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
date_from: str | None = Query(None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at >= date_from"),
|
||||
date_to: str | None = Query(None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at <= date_to"),
|
||||
tags: str | None = Query(None, description="Comma-separated tags to filter results"),
|
||||
sort: str = Query(default="relevance", description="Sort order: relevance, date, name"),
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> SearchResponse:
|
||||
"""Perform hybrid search via GET (same as POST but with query params)."""
|
||||
types_list = entity_types.split(",") if entity_types else None
|
||||
req = SearchRequest(query=q, entity_types=types_list, limit=limit, offset=offset)
|
||||
tags_list = tags.split(",") if tags else None
|
||||
req = SearchRequest(
|
||||
query=q,
|
||||
entity_types=types_list,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
date_from=date_from,
|
||||
date_to=date_to,
|
||||
tags=tags_list,
|
||||
sort=sort,
|
||||
)
|
||||
return await _do_search(req, current_user, db)
|
||||
|
||||
|
||||
def _parse_date(value: str | None) -> datetime | None:
|
||||
"""Parse an ISO date string (YYYY-MM-DD) into a timezone-aware datetime."""
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.fromisoformat(value)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _apply_filters_and_sort(
|
||||
results: list[dict],
|
||||
req: SearchRequest,
|
||||
) -> list[dict]:
|
||||
"""Apply post-search date/tags filters and sort order to results."""
|
||||
date_from = _parse_date(req.date_from)
|
||||
date_to = _parse_date(req.date_to)
|
||||
|
||||
filtered: list[dict] = []
|
||||
for r in results:
|
||||
# Date range filter on created_at/updated_at
|
||||
if date_from or date_to:
|
||||
created = r.get("_created_at")
|
||||
updated = r.get("_updated_at")
|
||||
ts = updated or created
|
||||
if ts is None:
|
||||
continue
|
||||
if isinstance(ts, str):
|
||||
try:
|
||||
ts = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
continue
|
||||
if ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=timezone.utc)
|
||||
if date_from and ts < date_from:
|
||||
continue
|
||||
if date_to and ts > date_to:
|
||||
continue
|
||||
|
||||
# Tags filter (comma-separated on entity)
|
||||
if req.tags:
|
||||
entity_tags = r.get("_tags") or ""
|
||||
tag_set = {t.strip().lower() for t in entity_tags.split(",") if t.strip()}
|
||||
if not any(t.lower() in tag_set for t in req.tags):
|
||||
continue
|
||||
|
||||
filtered.append(r)
|
||||
|
||||
# Sort order
|
||||
sort = (req.sort or "relevance").lower()
|
||||
if sort == "date":
|
||||
filtered.sort(
|
||||
key=lambda x: (x.get("_updated_at") or x.get("_created_at") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
elif sort == "name":
|
||||
filtered.sort(key=lambda x: (x.get("title") or "").lower())
|
||||
# 'relevance' keeps the existing score order
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
async def _do_search(
|
||||
req: SearchRequest,
|
||||
current_user: dict,
|
||||
@@ -81,6 +162,9 @@ async def _do_search(
|
||||
is_system_admin=is_system_admin,
|
||||
)
|
||||
|
||||
# Apply post-search filters (date range, tags) and sort
|
||||
results = _apply_filters_and_sort(results, req)
|
||||
|
||||
# Resolve user permissions for field-level RBAC
|
||||
resolved_perms = await resolve_permissions(db, user_id, tenant_id)
|
||||
|
||||
@@ -131,6 +215,70 @@ async def search(
|
||||
return await _do_search(req, current_user, db)
|
||||
|
||||
|
||||
# ─── Facets ───
|
||||
|
||||
@router.get("/facets", dependencies=[Depends(require_permission("search:read"))])
|
||||
async def search_facets(
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> FacetsResponse:
|
||||
"""Return available facets for search filtering (entity types, tags, date ranges)."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
|
||||
# Entity types from the search registry
|
||||
registry = get_search_registry()
|
||||
entity_types = registry.get_entity_types()
|
||||
|
||||
# Tags: distinct tags across searchable entities (comma-separated on entities)
|
||||
tags: set[str] = set()
|
||||
tag_tables = {
|
||||
"contacts": "tags",
|
||||
"companies": "tags",
|
||||
}
|
||||
for table, col in tag_tables.items():
|
||||
try:
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"SELECT DISTINCT unnest(string_to_array({col}, ',')) AS tag "
|
||||
f"FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL AND {col} IS NOT NULL"
|
||||
),
|
||||
{"tid": tenant_id},
|
||||
)
|
||||
for row in result.mappings().all():
|
||||
tag = (row.get("tag") or "").strip()
|
||||
if tag:
|
||||
tags.add(tag)
|
||||
except Exception:
|
||||
logger.debug("Facet tag query failed for %s", table)
|
||||
|
||||
# Date ranges: min/max created_at across searchable entities
|
||||
date_ranges: dict[str, Any] = {}
|
||||
date_tables = ["contacts", "mails", "files", "calendar_entries"]
|
||||
for table in date_tables:
|
||||
try:
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"SELECT min(created_at) AS min_dt, max(created_at) AS max_dt "
|
||||
f"FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL"
|
||||
),
|
||||
{"tid": tenant_id},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
if row:
|
||||
date_ranges[table] = {
|
||||
"min": str(row.get("min_dt")) if row.get("min_dt") else None,
|
||||
"max": str(row.get("max_dt")) if row.get("max_dt") else None,
|
||||
}
|
||||
except Exception:
|
||||
logger.debug("Facet date query failed for %s", table)
|
||||
|
||||
return FacetsResponse(
|
||||
entity_types=entity_types,
|
||||
tags=sorted(tags),
|
||||
date_ranges=date_ranges,
|
||||
)
|
||||
|
||||
|
||||
# ─── Suggest / Autocomplete ───
|
||||
|
||||
@router.get("/suggest", dependencies=[Depends(require_permission("search:read"))])
|
||||
@@ -205,14 +353,76 @@ async def reindex(
|
||||
if job_id:
|
||||
job_ids.append(job_id)
|
||||
|
||||
# Optionally re-index file chunks
|
||||
if req.include_chunks and "file" in entity_types:
|
||||
chunk_job_id = await enqueue_job("reindex_all")
|
||||
if chunk_job_id:
|
||||
job_ids.append(chunk_job_id)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"message": f"Reindexing {len(entity_types)} entity types",
|
||||
"entity_types": entity_types,
|
||||
"include_chunks": req.include_chunks,
|
||||
"job_ids": job_ids,
|
||||
}
|
||||
|
||||
|
||||
# ─── Rebuild / Purge ───
|
||||
|
||||
@router.post("/rebuild/{entity_type}/{entity_id}", dependencies=[Depends(require_permission("search:admin"))])
|
||||
async def rebuild_entity_index(
|
||||
entity_type: str,
|
||||
entity_id: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Rebuild the search index for a single entity (admin only)."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
try:
|
||||
eid = uuid.UUID(entity_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid entity_id")
|
||||
|
||||
from app.plugins.builtins.unified_search.lifecycle import rebuild_index
|
||||
|
||||
success = await rebuild_index(db, entity_type, eid, tenant_id)
|
||||
return {
|
||||
"status": "ok" if success else "failed",
|
||||
"entity_type": entity_type,
|
||||
"entity_id": entity_id,
|
||||
"message": "Index rebuilt" if success else "Rebuild failed — check logs",
|
||||
}
|
||||
|
||||
|
||||
@router.post("/purge/{entity_type}/{entity_id}", dependencies=[Depends(require_permission("search:admin"))])
|
||||
async def purge_entity_index(
|
||||
entity_type: str,
|
||||
entity_id: str,
|
||||
current_user: dict = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Purge an entity from the search index (admin only, GDPR)."""
|
||||
tenant_id = uuid.UUID(current_user["tenant_id"])
|
||||
try:
|
||||
eid = uuid.UUID(entity_id)
|
||||
except ValueError:
|
||||
raise HTTPException(status_code=400, detail="Invalid entity_id")
|
||||
|
||||
from app.plugins.builtins.unified_search.lifecycle import remove_from_index, remove_chunks
|
||||
|
||||
await remove_from_index(db, entity_type, eid, tenant_id)
|
||||
if entity_type == "file":
|
||||
await remove_chunks(db, eid, tenant_id)
|
||||
|
||||
return {
|
||||
"status": "ok",
|
||||
"entity_type": entity_type,
|
||||
"entity_id": entity_id,
|
||||
"message": "Purged from search index",
|
||||
}
|
||||
|
||||
|
||||
# ─── Providers ───
|
||||
|
||||
@router.get("/providers", dependencies=[Depends(require_permission("search:read"))])
|
||||
@@ -228,6 +438,10 @@ async def list_providers(
|
||||
entity_type=p.entity_type,
|
||||
plugin_name="unified_search",
|
||||
is_active=True,
|
||||
supports_fts=getattr(p, "supports_fts", True),
|
||||
supports_vector=getattr(p, "supports_vector", True),
|
||||
supports_rag=getattr(p, "supports_rag", False),
|
||||
supports_graph=getattr(p, "supports_graph", False),
|
||||
)
|
||||
for p in providers
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user