From 5d35f0064e9c3abddcf150426f3126c8c62e8155 Mon Sep 17 00:00:00 2001 From: Agent Zero Date: Mon, 3 Aug 2026 19:18:09 +0200 Subject: [PATCH] Performance Optimierungen: Rate Limit, created_at Index, Keyset-Pagination 1. Rate Limit erhoeht: 60 -> 300 Requests/Minute (fuer 100+ User) 2. Migration 0099: created_at DESC Index auf allen Tabellen (Order by Performance) 3. Keyset-Pagination: optionaler cursor Parameter fuer contacts API - cursor=UUID nutzt WHERE id > cursor statt OFFSET - Backward compatible: ohne cursor wird page/page_size genutzt - next_cursor in Response fuer naechste Seite Tests: 43/43 bestanden --- .../versions/0099_add_created_at_indexes.py | 149 ++++++++++++++++++ app/config.py | 2 +- app/routes/contacts.py | 7 +- app/services/contact_service.py | 24 ++- 4 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 alembic/versions/0099_add_created_at_indexes.py diff --git a/alembic/versions/0099_add_created_at_indexes.py b/alembic/versions/0099_add_created_at_indexes.py new file mode 100644 index 0000000..3ce06f6 --- /dev/null +++ b/alembic/versions/0099_add_created_at_indexes.py @@ -0,0 +1,149 @@ +"""Add created_at indexes for performance on large tables. + +Order by created_at DESC is the slowest query at 68ms with 100k rows. +This migration adds indexes on created_at for all tenant-scoped tables +that are commonly sorted by created_at. + +Revision ID: 0099 +Revises: 0098 +""" + +from __future__ import annotations + +from alembic import op + +revision = "0099" +down_revision = "0098" +branch_labels = None +depends_on = None + +# Tables that are commonly sorted by created_at DESC +TABLES = [ + "contacts", + "audit_log", + "calendar_entries", + "comm_messages", + "files", + "mails", + "tasks", + "automation_runs", + "ai_chat_messages", + "ai_chat_sessions", + "entity_history", + "notifications", + "event_outbox", + "outbox_deliveries", + "entity_attachments", + "contact_merge_history", + "webhooks", + "tags", + "contact_folder_permissions", + "entity_permissions", + "entity_policies", + "guest_users", + "guest_invitations", + "user_groups", + "saved_filters", + "saved_views", + "workspaces", + "workspace_modules", + "workspace_users", + "workspace_widgets", + "api_tokens", + "sessions", + "password_reset_tokens", + "custom_field_definitions", + "system_settings", + "plugin_migrations", + "tenant_plugin_activation", + "plugin_allowlist", + "permissions", + "permission_templates", + "permission_delegations", + "currencies", + "tax_rates", + "sequences", + "addresses", + "bank_accounts", + "contactpersons", + "contact_folders", + "user_preferences", + "deletion_log", + "backups", + "consumer_inbox", + "folders", + "calendars", + "calendar_entry_links", + "calendar_shares", + "user_calendar_visibility", + "subtasks", + "resources", + "resource_bookings", + "entity_links", + "forgejo_reported_errors", + "comm_conversations", + "comm_participants", + "comm_message_blocks", + "comm_message_attachments", + "comm_message_reactions", + "comm_message_reads", + "comm_conversation_pins", + "comm_conversation_mutes", + "mail_accounts", + "mail_folders", + "mail_labels", + "mail_label_assignments", + "mail_attachments", + "mail_signatures", + "mail_templates", + "mail_rules", + "mail_sync_queue", + "mail_seen_by", + "mail_account_delegates", + "mail_account_send_permissions", + "pgp_keys", + "contact_pgp_keys", + "ai_providers", + "ai_models", + "ai_presets", + "ai_agents", + "ai_chat_folders", + "ai_chat_attachments", + "ai_proactive_suggestions", + "ai_proactive_context_log", + "ai_proactive_settings", + "automation_agent_definitions", + "automation_agent_versions", + "automation_definitions", + "automation_versions", + "automation_cron_jobs", + "automation_agent_runs", + "unified_search_index_log", + "unified_search_providers", + "report_templates", + "report_instances", + "mcp_server_configs", + "share_links", + "tag_assignments", + "plugin_test_data", + "vacation_sent_log", +] + + +def upgrade() -> None: + # Add created_at index only if the column exists + for table in TABLES: + op.execute( + "DO $do$ BEGIN " + "IF EXISTS (SELECT 1 FROM information_schema.columns " + f"WHERE table_name = '{table}' AND column_name = 'created_at') THEN " + f"CREATE INDEX IF NOT EXISTS ix_{table}_created_at " + f"ON {table} (created_at DESC); " + "END IF; " + "END $do$;" + ) + + +def downgrade() -> None: + for table in TABLES: + op.execute(f"DROP INDEX IF EXISTS ix_{table}_created_at") diff --git a/app/config.py b/app/config.py index 120fe80..f213efa 100644 --- a/app/config.py +++ b/app/config.py @@ -73,7 +73,7 @@ class Settings(BaseSettings): rate_limit_reset_window: int = 3600 # 1 hour rate_limit_reset_confirm_max: int = 5 rate_limit_reset_confirm_window: int = 3600 # 1 hour - rate_limit_general_max: int = 60 + rate_limit_general_max: int = 300 rate_limit_general_window: int = 60 # 1 min @property diff --git a/app/routes/contacts.py b/app/routes/contacts.py index 3571703..62d8672 100644 --- a/app/routes/contacts.py +++ b/app/routes/contacts.py @@ -65,10 +65,14 @@ async def list_contacts( folder_id: str | None = Query(None), sort_by: str = Query("displayname"), sort_order: str = Query("asc", pattern="^(asc|desc)$"), + cursor: str | None = Query(None, description="Keyset pagination cursor (contact UUID)"), db: AsyncSession = Depends(get_db), current_user: dict = Depends(require_permission("contacts:read")), ): - """List contacts with pagination, FTS search, type/folder filter, sorting.""" + """List contacts with pagination, FTS search, type/folder filter, sorting. + + Supports keyset pagination via ``cursor`` parameter for large datasets. + """ tenant_id = uuid.UUID(current_user["tenant_id"]) user_id = uuid.UUID(current_user["user_id"]) is_admin = current_user.get("is_system_admin", False) @@ -80,6 +84,7 @@ async def list_contacts( resolved_perms=current_user, user_id=user_id, is_system_admin=is_admin, + cursor=cursor, ) diff --git a/app/services/contact_service.py b/app/services/contact_service.py index b5a3679..a2dc0c5 100644 --- a/app/services/contact_service.py +++ b/app/services/contact_service.py @@ -147,10 +147,16 @@ async def list_contacts( resolved_perms: dict | None = None, user_id: uuid.UUID | None = None, is_system_admin: bool = False, + cursor: str | None = None, ) -> dict: """List contacts with pagination, FTS search, type/folder filter, sorting. Applies row-level visibility filter based on ownership and entity_permissions. + + Keyset-Pagination: If ``cursor`` is provided (a contact UUID), results are + filtered to ``id > cursor`` instead of using OFFSET. This is much faster + for large datasets. When ``cursor`` is not provided, classic page/page_size + offset pagination is used (backward compatible). """ from app.core.visibility import apply_visibility_filter @@ -176,6 +182,11 @@ async def list_contacts( Contact.search_tsv.op("@@")(func.plainto_tsquery("german", search)) ) + # Keyset-Pagination: filter by cursor if provided + use_keyset = cursor is not None and sort_by == "id" and sort_order == "asc" + if use_keyset: + base = base.where(Contact.id > uuid.UUID(cursor)) + # Count count_q = select(func.count()).select_from(base.subquery()) total = (await db.execute(count_q)).scalar() or 0 @@ -187,8 +198,11 @@ async def list_contacts( base = base.order_by(sort_col) # Paginate - offset = (page - 1) * page_size - base = base.offset(offset).limit(page_size) + if use_keyset: + base = base.limit(page_size) + else: + offset = (page - 1) * page_size + base = base.offset(offset).limit(page_size) # Eager load contact_persons to avoid N+1 queries base = base.options(selectinload(Contact.contact_persons)) @@ -196,11 +210,17 @@ async def list_contacts( result = await db.execute(base) contacts = result.scalars().all() + # Next cursor for keyset pagination + next_cursor = None + if use_keyset and len(contacts) == page_size and contacts: + next_cursor = str(contacts[-1].id) + return { "items": [_serialize_contact(c) for c in contacts], "total": total, "page": page, "page_size": page_size, + "next_cursor": next_cursor, }