From 211242a807bc2894f295c5c32d3a8e2a26314dc3 Mon Sep 17 00:00:00 2001 From: Agent Zero Date: Thu, 13 Aug 2026 16:28:55 +0200 Subject: [PATCH] =?UTF-8?q?feat(B-VEC):=20pgvector=20HNSW=20Optimierung=20?= =?UTF-8?q?=E2=80=94=20ef=5Fconstruction=3D128,=20m=3D16,=20ef=5Fsearch=3D?= =?UTF-8?q?40?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B-VEC: Migration 0118 — HNSW-Indizes mit optimierten Parametern (ef_construction=128, m=16) - 5 Tabellen: contacts, mails, files, calendar_entries, tags - config.py: hnsw_ef_construction, hnsw_m, hnsw_ef_search, vector_index_type Settings - base_provider.py + search_engine.py: SET LOCAL hnsw.ef_search vor Vector-Queries B-VEC-IVF: IVFFlat als Alternative dokumentiert (vector_index_type Setting) B-VEC-BATCH: Batch-Embedding verifiziert (generate_embeddings_batch nutzt llm_embed()) B-VEC-TEST: Performance-Tests auf Coolify-Instanz verschoben (benötigt 10k+ Datensätze) --- alembic/versions/0118_optimize_hnsw_params.py | 88 +++++++++++++++++++ app/config.py | 6 ++ .../builtins/unified_search/base_provider.py | 4 + .../builtins/unified_search/search_engine.py | 4 + 4 files changed, 102 insertions(+) create mode 100644 alembic/versions/0118_optimize_hnsw_params.py diff --git a/alembic/versions/0118_optimize_hnsw_params.py b/alembic/versions/0118_optimize_hnsw_params.py new file mode 100644 index 0000000..b75e2c8 --- /dev/null +++ b/alembic/versions/0118_optimize_hnsw_params.py @@ -0,0 +1,88 @@ +"""Optimize HNSW index parameters for better vector search recall. + +Recreates existing HNSW indices with tuned parameters: +- ef_construction=128 (default 64, higher = better index quality, slower build) +- m=16 (default 16, higher = more memory, better recall) + +IVFFlat Alternative (B-VEC-IVF): +----------------------------- +comm_messages uses IVFFlat with lists=100 (migration 0035). +Rule of thumb for IVFFlat: lists = sqrt(rows) + ~10k rows → lists ≈ 100 + ~50k rows → lists ≈ 224 + ~100k rows → lists ≈ 316 +IVFFlat builds faster but HNSW has better recall. +To switch: DROP INDEX + CREATE INDEX ... USING hnsw (embedding vector_cosine_ops) + WITH (ef_construction=128, m=16) +Config: vector_index_type setting in app/config.py (default 'hnsw', alternative 'ivfflat'). + +Revision ID: 0118 +Revises: 0117 +""" + +from alembic import op +import sqlalchemy as sa + +revision = "0118" +down_revision = "0117" +branch_labels = None +depends_on = None + +# Optimized HNSW parameters +EF_CONSTRUCTION = 128 +M = 16 + +# Tables with HNSW indices (from migration 0104) +# Format: (table_name, index_name) +HNSW_TABLES = [ + ("contacts", "ix_contacts_embedding"), + ("mails", "ix_mails_embedding"), + ("files", "ix_files_embedding"), + ("calendar_entries", "ix_calendar_entries_embedding"), + ("tags", "ix_tags_embedding"), +] + + +def upgrade() -> None: + conn = op.get_bind() + + for table_name, index_name in HNSW_TABLES: + # Check if table exists + table_exists = conn.execute(sa.text( + "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = :t)" + ), {"t": table_name}).scalar() + + if not table_exists: + continue + + # Drop existing HNSW index (regardless of parameters) + op.execute(f"DROP INDEX IF EXISTS {index_name}") + + # Recreate with optimized parameters + op.execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " + f"ON {table_name} USING hnsw (embedding vector_cosine_ops) " + f"WITH (ef_construction={EF_CONSTRUCTION}, m={M})" + ) + + +def downgrade() -> None: + """Recreate HNSW indices with default parameters (no WITH clause).""" + conn = op.get_bind() + + for table_name, index_name in HNSW_TABLES: + table_exists = conn.execute(sa.text( + "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = :t)" + ), {"t": table_name}).scalar() + + if not table_exists: + continue + + # Drop optimized index + op.execute(f"DROP INDEX IF EXISTS {index_name}") + + # Recreate with default parameters (no WITH clause = pgvector defaults) + op.execute( + f"CREATE INDEX IF NOT EXISTS {index_name} " + f"ON {table_name} USING hnsw (embedding vector_cosine_ops)" + ) diff --git a/app/config.py b/app/config.py index 9ba6ef3..b75a6af 100644 --- a/app/config.py +++ b/app/config.py @@ -79,6 +79,12 @@ class Settings(BaseSettings): # Marketplace marketplace_server_url: str = "" + # pgvector / HNSW + hnsw_ef_construction: int = 128 + hnsw_m: int = 16 + hnsw_ef_search: int = 40 + vector_index_type: Literal["hnsw", "ivfflat"] = "hnsw" + # Rate Limiting rate_limit_login_max: int = 5 rate_limit_login_window: int = 900 # 15 min diff --git a/app/plugins/builtins/unified_search/base_provider.py b/app/plugins/builtins/unified_search/base_provider.py index ff530c1..ab05e2e 100644 --- a/app/plugins/builtins/unified_search/base_provider.py +++ b/app/plugins/builtins/unified_search/base_provider.py @@ -13,6 +13,8 @@ from typing import Any from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession +from app.config import settings + logger = logging.getLogger(__name__) @@ -53,6 +55,8 @@ class BaseSearchProvider: is_system_admin: bool = False, ) -> list[dict[str, Any]]: """Semantic vector search with visibility filter.""" + # Set HNSW ef_search parameter for this transaction + await db.execute(text(f"SET LOCAL hnsw.ef_search = {settings.hnsw_ef_search}")) if is_system_admin or not user_id: return await self._search_vector_filtered(db, embedding, tenant_id, limit, None) diff --git a/app/plugins/builtins/unified_search/search_engine.py b/app/plugins/builtins/unified_search/search_engine.py index 1f7448d..1efa293 100644 --- a/app/plugins/builtins/unified_search/search_engine.py +++ b/app/plugins/builtins/unified_search/search_engine.py @@ -9,6 +9,7 @@ from typing import Any from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession +from app.config import settings from app.plugins.builtins.unified_search.embedding import generate_embedding from app.plugins.builtins.unified_search.provider_registry import get_search_registry @@ -164,6 +165,9 @@ async def find_similar_all_types( source_embedding_str = str(row["embedding"]) + # Set HNSW ef_search parameter for this transaction + await db.execute(text(f"SET LOCAL hnsw.ef_search = {settings.hnsw_ef_search}")) + registry = get_search_registry() similar: dict[str, list[dict[str, Any]]] = {}