Agent Zero
3d9b76cea4
Check Cross-Plugin Imports / check (push) Has been cancelled
- SPIKE-E: FTS+Vector+Permission benchmark on 10k records (all <30ms) - E-PROV: supports_fts/vector/rag/graph capability flags on all providers - E-FTS/VEC: All 11 providers refactored to BaseSearchProvider with permission filtering - E-PERM: Over-fetch strategy for vector+permission (15x faster than ANY() filter) - E-FUSE: rrf_fusion_multi() for N-way RRF over FTS+Vector+RAG+Graph - E-LLM: Query understanding cleaned up to use central llm_complete() - E-CHUNK: Document chunking module + document_chunks table with HNSW index - E-EMB: Chunk embedding ARQ jobs (index_file_chunks, reindex_chunks) - E-RAG: RAG retrieval via FileSearchProvider.search_rag() - E-GRAPH: GraphRAG BFS traversal via GraphRAGSearchProvider.search_graph() - E-IX-EVT: Auto-indexing via outbox events + delete/cleanup handlers - E-IX-RE: Batch reindex with progress tracking + reindex_all job - E-DATA-LIFE: Lifecycle module (remove/rebuild/restore/correct) + API endpoints - E-K-MEM: AgentMemorySearchProvider - E-P-AI: AIChatSearchProvider - E-P-WF: WorkflowSearchProvider - E-P-COMM: ConversationSearchProvider verified (already on BaseSearchProvider) - E-API: Filter params (date_from/to, tags, sort) + /facets endpoint - E-TOOL: unified_search AI tool registered in ToolRegistry - E-MCP: Search tool in MCP server with normal RBAC/tenant checks - E-UI-CMD: CommandPalette (Cmd+K) with debounced search + recent searches - E-UI-FAC: SearchFacets, SearchResultCard, SavedSearches components - E-TEST: 40 new tests in test_unified_search_phase_e.py (105 total green) - E-DOC: api-documentation.md, plugin-development-guide.md, test-strategy.md updated 105 tests passing, TypeScript clean.
77 lines
1.9 KiB
Python
77 lines
1.9 KiB
Python
"""Document chunking for RAG indexing.
|
|
|
|
Splits extracted text into overlapping chunks suitable for embedding
|
|
generation and vector search at the chunk level.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def chunk_text(
|
|
text: str,
|
|
chunk_size: int = 1000,
|
|
overlap: int = 200,
|
|
) -> list[dict]:
|
|
"""Split text into overlapping chunks.
|
|
|
|
Args:
|
|
text: Input text to chunk.
|
|
chunk_size: Maximum characters per chunk.
|
|
overlap: Number of overlapping characters between consecutive chunks.
|
|
|
|
Returns:
|
|
List of chunk dicts with keys:
|
|
- chunk_index: zero-based index
|
|
- chunk_text: the chunk content
|
|
- chunk_hash: sha256 hex digest of chunk_text
|
|
|
|
Edge cases:
|
|
- Empty text returns an empty list.
|
|
- Text shorter than chunk_size returns a single chunk.
|
|
"""
|
|
if not text or not text.strip():
|
|
return []
|
|
|
|
# Normalise whitespace to avoid degenerate chunks
|
|
cleaned = " ".join(text.split())
|
|
if not cleaned:
|
|
return []
|
|
|
|
chunks: list[dict] = []
|
|
start = 0
|
|
idx = 0
|
|
text_len = len(cleaned)
|
|
|
|
while start < text_len:
|
|
end = min(start + chunk_size, text_len)
|
|
chunk = cleaned[start:end]
|
|
|
|
if chunk.strip():
|
|
chunks.append(
|
|
{
|
|
"chunk_index": idx,
|
|
"chunk_text": chunk,
|
|
"chunk_hash": hashlib.sha256(chunk.encode("utf-8")).hexdigest(),
|
|
}
|
|
)
|
|
idx += 1
|
|
|
|
# If we've reached the end, stop
|
|
if end >= text_len:
|
|
break
|
|
|
|
# Advance by chunk_size - overlap
|
|
step = chunk_size - overlap
|
|
if step <= 0:
|
|
# Prevent infinite loop if overlap >= chunk_size
|
|
step = chunk_size
|
|
start += step
|
|
|
|
logger.debug("Chunked text (len=%d) into %d chunks", text_len, len(chunks))
|
|
return chunks
|