feat(E): Unified Search — 24 Tasks complete
Check Cross-Plugin Imports / check (push) Has been cancelled

- SPIKE-E: FTS+Vector+Permission benchmark on 10k records (all <30ms)
- E-PROV: supports_fts/vector/rag/graph capability flags on all providers
- E-FTS/VEC: All 11 providers refactored to BaseSearchProvider with permission filtering
- E-PERM: Over-fetch strategy for vector+permission (15x faster than ANY() filter)
- E-FUSE: rrf_fusion_multi() for N-way RRF over FTS+Vector+RAG+Graph
- E-LLM: Query understanding cleaned up to use central llm_complete()
- E-CHUNK: Document chunking module + document_chunks table with HNSW index
- E-EMB: Chunk embedding ARQ jobs (index_file_chunks, reindex_chunks)
- E-RAG: RAG retrieval via FileSearchProvider.search_rag()
- E-GRAPH: GraphRAG BFS traversal via GraphRAGSearchProvider.search_graph()
- E-IX-EVT: Auto-indexing via outbox events + delete/cleanup handlers
- E-IX-RE: Batch reindex with progress tracking + reindex_all job
- E-DATA-LIFE: Lifecycle module (remove/rebuild/restore/correct) + API endpoints
- E-K-MEM: AgentMemorySearchProvider
- E-P-AI: AIChatSearchProvider
- E-P-WF: WorkflowSearchProvider
- E-P-COMM: ConversationSearchProvider verified (already on BaseSearchProvider)
- E-API: Filter params (date_from/to, tags, sort) + /facets endpoint
- E-TOOL: unified_search AI tool registered in ToolRegistry
- E-MCP: Search tool in MCP server with normal RBAC/tenant checks
- E-UI-CMD: CommandPalette (Cmd+K) with debounced search + recent searches
- E-UI-FAC: SearchFacets, SearchResultCard, SavedSearches components
- E-TEST: 40 new tests in test_unified_search_phase_e.py (105 total green)
- E-DOC: api-documentation.md, plugin-development-guide.md, test-strategy.md updated

105 tests passing, TypeScript clean.
This commit is contained in:
Agent Zero
2026-08-14 01:34:58 +02:00
parent 60f30d021b
commit 3d9b76cea4
45 changed files with 5378 additions and 402 deletions
+215 -1
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import logging
import uuid
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import text
@@ -19,6 +20,7 @@ from app.plugins.builtins.unified_search.query_understanding import (
llm_analyze_query,
)
from app.plugins.builtins.unified_search.schemas import (
FacetsResponse,
ProviderResponse,
ReindexRequest,
SearchRequest,
@@ -48,15 +50,94 @@ async def search_get(
entity_types: str | None = Query(None, description="Comma-separated entity types to search"),
limit: int = Query(default=20, ge=1, le=100),
offset: int = Query(default=0, ge=0),
date_from: str | None = Query(None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at >= date_from"),
date_to: str | None = Query(None, description="ISO date (YYYY-MM-DD) — filter by created_at/updated_at <= date_to"),
tags: str | None = Query(None, description="Comma-separated tags to filter results"),
sort: str = Query(default="relevance", description="Sort order: relevance, date, name"),
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> SearchResponse:
"""Perform hybrid search via GET (same as POST but with query params)."""
types_list = entity_types.split(",") if entity_types else None
req = SearchRequest(query=q, entity_types=types_list, limit=limit, offset=offset)
tags_list = tags.split(",") if tags else None
req = SearchRequest(
query=q,
entity_types=types_list,
limit=limit,
offset=offset,
date_from=date_from,
date_to=date_to,
tags=tags_list,
sort=sort,
)
return await _do_search(req, current_user, db)
def _parse_date(value: str | None) -> datetime | None:
"""Parse an ISO date string (YYYY-MM-DD) into a timezone-aware datetime."""
if not value:
return None
try:
dt = datetime.fromisoformat(value)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt
except ValueError:
return None
def _apply_filters_and_sort(
results: list[dict],
req: SearchRequest,
) -> list[dict]:
"""Apply post-search date/tags filters and sort order to results."""
date_from = _parse_date(req.date_from)
date_to = _parse_date(req.date_to)
filtered: list[dict] = []
for r in results:
# Date range filter on created_at/updated_at
if date_from or date_to:
created = r.get("_created_at")
updated = r.get("_updated_at")
ts = updated or created
if ts is None:
continue
if isinstance(ts, str):
try:
ts = datetime.fromisoformat(ts.replace("Z", "+00:00"))
except ValueError:
continue
if ts.tzinfo is None:
ts = ts.replace(tzinfo=timezone.utc)
if date_from and ts < date_from:
continue
if date_to and ts > date_to:
continue
# Tags filter (comma-separated on entity)
if req.tags:
entity_tags = r.get("_tags") or ""
tag_set = {t.strip().lower() for t in entity_tags.split(",") if t.strip()}
if not any(t.lower() in tag_set for t in req.tags):
continue
filtered.append(r)
# Sort order
sort = (req.sort or "relevance").lower()
if sort == "date":
filtered.sort(
key=lambda x: (x.get("_updated_at") or x.get("_created_at") or ""),
reverse=True,
)
elif sort == "name":
filtered.sort(key=lambda x: (x.get("title") or "").lower())
# 'relevance' keeps the existing score order
return filtered
async def _do_search(
req: SearchRequest,
current_user: dict,
@@ -81,6 +162,9 @@ async def _do_search(
is_system_admin=is_system_admin,
)
# Apply post-search filters (date range, tags) and sort
results = _apply_filters_and_sort(results, req)
# Resolve user permissions for field-level RBAC
resolved_perms = await resolve_permissions(db, user_id, tenant_id)
@@ -131,6 +215,70 @@ async def search(
return await _do_search(req, current_user, db)
# ─── Facets ───
@router.get("/facets", dependencies=[Depends(require_permission("search:read"))])
async def search_facets(
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> FacetsResponse:
"""Return available facets for search filtering (entity types, tags, date ranges)."""
tenant_id = uuid.UUID(current_user["tenant_id"])
# Entity types from the search registry
registry = get_search_registry()
entity_types = registry.get_entity_types()
# Tags: distinct tags across searchable entities (comma-separated on entities)
tags: set[str] = set()
tag_tables = {
"contacts": "tags",
"companies": "tags",
}
for table, col in tag_tables.items():
try:
result = await db.execute(
text(
f"SELECT DISTINCT unnest(string_to_array({col}, ',')) AS tag "
f"FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL AND {col} IS NOT NULL"
),
{"tid": tenant_id},
)
for row in result.mappings().all():
tag = (row.get("tag") or "").strip()
if tag:
tags.add(tag)
except Exception:
logger.debug("Facet tag query failed for %s", table)
# Date ranges: min/max created_at across searchable entities
date_ranges: dict[str, Any] = {}
date_tables = ["contacts", "mails", "files", "calendar_entries"]
for table in date_tables:
try:
result = await db.execute(
text(
f"SELECT min(created_at) AS min_dt, max(created_at) AS max_dt "
f"FROM {table} WHERE tenant_id = :tid AND deleted_at IS NULL"
),
{"tid": tenant_id},
)
row = result.mappings().first()
if row:
date_ranges[table] = {
"min": str(row.get("min_dt")) if row.get("min_dt") else None,
"max": str(row.get("max_dt")) if row.get("max_dt") else None,
}
except Exception:
logger.debug("Facet date query failed for %s", table)
return FacetsResponse(
entity_types=entity_types,
tags=sorted(tags),
date_ranges=date_ranges,
)
# ─── Suggest / Autocomplete ───
@router.get("/suggest", dependencies=[Depends(require_permission("search:read"))])
@@ -205,14 +353,76 @@ async def reindex(
if job_id:
job_ids.append(job_id)
# Optionally re-index file chunks
if req.include_chunks and "file" in entity_types:
chunk_job_id = await enqueue_job("reindex_all")
if chunk_job_id:
job_ids.append(chunk_job_id)
return {
"status": "ok",
"message": f"Reindexing {len(entity_types)} entity types",
"entity_types": entity_types,
"include_chunks": req.include_chunks,
"job_ids": job_ids,
}
# ─── Rebuild / Purge ───
@router.post("/rebuild/{entity_type}/{entity_id}", dependencies=[Depends(require_permission("search:admin"))])
async def rebuild_entity_index(
entity_type: str,
entity_id: str,
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> dict:
"""Rebuild the search index for a single entity (admin only)."""
tenant_id = uuid.UUID(current_user["tenant_id"])
try:
eid = uuid.UUID(entity_id)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid entity_id")
from app.plugins.builtins.unified_search.lifecycle import rebuild_index
success = await rebuild_index(db, entity_type, eid, tenant_id)
return {
"status": "ok" if success else "failed",
"entity_type": entity_type,
"entity_id": entity_id,
"message": "Index rebuilt" if success else "Rebuild failed — check logs",
}
@router.post("/purge/{entity_type}/{entity_id}", dependencies=[Depends(require_permission("search:admin"))])
async def purge_entity_index(
entity_type: str,
entity_id: str,
current_user: dict = Depends(get_current_user),
db: AsyncSession = Depends(get_db),
) -> dict:
"""Purge an entity from the search index (admin only, GDPR)."""
tenant_id = uuid.UUID(current_user["tenant_id"])
try:
eid = uuid.UUID(entity_id)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid entity_id")
from app.plugins.builtins.unified_search.lifecycle import remove_from_index, remove_chunks
await remove_from_index(db, entity_type, eid, tenant_id)
if entity_type == "file":
await remove_chunks(db, eid, tenant_id)
return {
"status": "ok",
"entity_type": entity_type,
"entity_id": entity_id,
"message": "Purged from search index",
}
# ─── Providers ───
@router.get("/providers", dependencies=[Depends(require_permission("search:read"))])
@@ -228,6 +438,10 @@ async def list_providers(
entity_type=p.entity_type,
plugin_name="unified_search",
is_active=True,
supports_fts=getattr(p, "supports_fts", True),
supports_vector=getattr(p, "supports_vector", True),
supports_rag=getattr(p, "supports_rag", False),
supports_graph=getattr(p, "supports_graph", False),
)
for p in providers
]