Phase 4: KI-UI-Steuerung — AI agent UI control via WebSocket

- New ai_ui_control plugin: WS endpoint /ws/ai-ui-control, REST API (POST /command, GET /command/{id}/status, GET /online-users) - UI-Command-Protocol: 6 command types (navigate, filter, open_contact, modal, tab, settings) with Pydantic schemas - WebSocket manager: per-user connections, command delivery, feedback storage, stale cleanup - Frontend useAIUIControl hook: WS client with auto-reconnect, command dispatch, feedback sending - aiUIControlStore: Zustand store for command state, active modal/tab, pending filter/settings - AIUIControlIndicator: visual KI indication (Bot icon, toast, pulse animation) - ContactDetail integration: syncs activeTab and personModalOpen from AI control store - AppShell integration: useAIUIControl hook + AIUIControlIndicator - i18n keys for DE/EN - 18 Vitest tests: command protocol, store actions, feedback, visual indication - TSC: 0 new errors (only 2 pre-existing Dms.tsx errors)
2026-07-23 20:13:39 +02:00
parent 5dc6f29ac1
commit 903d649a0f
15 changed files with 1582 additions and 5 deletions
@@ -0,0 +1 @@
+"""AI UI Control plugin — lets AI agents control the frontend UI via WebSocket."""
@@ -0,0 +1,59 @@
+"""AI UI Control plugin — WebSocket-based UI control for AI agents.
+
+Phase 4: KI-UI-Steuerung
+Enables AI agents to control the frontend UI: navigate, filter, open contacts,
+manage modals, switch tabs, and change settings. Commands flow:
+AI agent → REST API → WebSocket → Frontend → executes → WebSocket feedback → REST poll
+"""
+
+from __future__ import annotations
+
+import logging
+
+from app.plugins.base import BasePlugin
+from app.plugins.manifest import PluginManifest, PluginRouteDef
+
+logger = logging.getLogger(__name__)
+
+
+class AIUIControlPlugin(BasePlugin):
+    """AI UI Control plugin: lets AI agents control the frontend UI."""
+
+    manifest = PluginManifest(
+        name="ai_ui_control",
+        version="1.0.0",
+        display_name="KI UI-Steuerung",
+        description=(
+            "Enables AI agents to control the frontend UI via WebSocket. "
+            "Supports navigation, filtering, contact opening, modals, tabs, and settings."
+        ),
+        dependencies=["permissions"],
+        routes=[
+            PluginRouteDef(
+                path="/api/v1/ai-ui-control",
+                module="app.plugins.builtins.ai_ui_control.routes",
+                router_attr="router",
+            ),
+        ],
+        events=[],
+        migrations=[],
+        permissions=[
+            "ai_ui_control:read",
+            "ai_ui_control:write",
+        ],
+        is_core=True,
+    )
+
+    async def on_install(self, db, service_container) -> None:
+        """Register the WebSocket manager in the service container."""
+        from app.plugins.builtins.ai_ui_control.websocket_manager import AIUIControlWSManager
+
+        ws_manager = AIUIControlWSManager()
+        service_container.register("ai_ui_control_ws", ws_manager)
+        logger.info("AI UI Control WebSocket manager registered")
+
+    async def on_uninstall(self, db, service_container) -> None:
+        """Clean up the WebSocket manager."""
+        if service_container.has("ai_ui_control_ws"):
+            service_container.remove("ai_ui_control_ws")
+            logger.info("AI UI Control WebSocket manager removed")
@@ -0,0 +1,256 @@
+"""REST and WebSocket routes for AI UI Control.
+
+Task 4.2: WebSocket-Endpoint für KI-UI-Steuerung
+Backend WebSocket /ws/ai-ui-control. Authenticated via session cookie.
+AI agent sends commands via REST, frontend receives via WebSocket.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import uuid
+
+from fastapi import APIRouter, WebSocket, WebSocketDisconnect, Depends, HTTPException, Request
+from fastapi.responses import JSONResponse
+
+from app.plugins.builtins.ai_ui_control.schemas import (
+    UICommand,
+    UICommandCreate,
+    UICommandFeedback,
+    UICommandResponse,
+    UICommandStatus,
+    UICommandStatusResponse,
+    UICommandType,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+# ─── REST endpoints (for AI agents) ───
+
+@router.post("/command", response_model=UICommandResponse)
+async def send_ui_command(
+    request: Request,
+    body: UICommandCreate,
+):
+    """Send a UI command to the frontend.
+
+    Called by AI agents to control the UI. The command is forwarded to
+    the frontend via WebSocket. If the user is not online, returns pending status.
+
+    Authentication: requires valid session (same-user commands only).
+    """
+    from app.config import get_settings
+    from app.core.auth import get_session_data, get_redis
+    from app.core.service_container import get_container
+
+    settings = get_settings()
+    session_id = request.cookies.get(settings.session_cookie_name)
+    if not session_id:
+        raise HTTPException(status_code=401, detail="Not authenticated")
+
+    redis = get_redis()
+    session_data = await get_session_data(redis, session_id)
+    if session_data is None:
+        raise HTTPException(status_code=401, detail="Session expired")
+
+    user_id = session_data["user_id"]
+
+    container = get_container()
+    if not container.has("ai_ui_control_ws"):
+        raise HTTPException(status_code=503, detail="AI UI Control not available")
+
+    ws_manager = container.get("ai_ui_control_ws")
+
+    command_id = str(uuid.uuid4())
+    command_dict = {
+        "command_id": command_id,
+        "action": body.action.value,
+        "path": body.path,
+        "entity": body.entity,
+        "filter": body.filter,
+        "contact_id": body.contact_id,
+        "modal": body.modal,
+        "tab": body.tab,
+        "section": body.section,
+        "key": body.key,
+        "value": body.value,
+        "description": body.description,
+    }
+
+    delivered_id = await ws_manager.send_command(user_id, command_dict)
+
+    if delivered_id:
+        return UICommandResponse(
+            command_id=delivered_id,
+            status=UICommandStatus.delivered,
+            action=body.action,
+            message="Command delivered to frontend",
+        )
+    else:
+        # User not online — store as pending
+        ws_manager._feedback[command_id] = {
+            "command_id": command_id,
+            "status": "pending",
+            "action": body.action.value,
+            "message": "User not online, command pending",
+        }
+        return UICommandResponse(
+            command_id=command_id,
+            status=UICommandStatus.pending,
+            action=body.action,
+            message="User not online, command pending",
+        )
+
+
+@router.get("/command/{command_id}/status", response_model=UICommandStatusResponse)
+async def get_command_status(
+    request: Request,
+    command_id: str,
+):
+    """Poll the status of a previously sent command.
+
+    AI agents call this to check if the frontend has executed the command.
+    """
+    from app.config import get_settings
+    from app.core.auth import get_session_data, get_redis
+    from app.core.service_container import get_container
+
+    settings = get_settings()
+    session_id = request.cookies.get(settings.session_cookie_name)
+    if not session_id:
+        raise HTTPException(status_code=401, detail="Not authenticated")
+
+    redis = get_redis()
+    session_data = await get_session_data(redis, session_id)
+    if session_data is None:
+        raise HTTPException(status_code=401, detail="Session expired")
+
+    container = get_container()
+    if not container.has("ai_ui_control_ws"):
+        raise HTTPException(status_code=503, detail="AI UI Control not available")
+
+    ws_manager = container.get("ai_ui_control_ws")
+    feedback = ws_manager.get_feedback(command_id)
+
+    if feedback is None:
+        raise HTTPException(status_code=404, detail="Command not found")
+
+    status = UICommandStatus(feedback.get("status", "pending"))
+    action = None
+    if feedback.get("action"):
+        try:
+            action = UICommandType(feedback["action"])
+        except ValueError:
+            pass
+
+    fb_model = None
+    if status in (UICommandStatus.success, UICommandStatus.failed):
+        fb_model = UICommandFeedback(
+            command_id=feedback.get("command_id", command_id),
+            status=status,
+            action=action,
+            current_path=feedback.get("current_path"),
+            current_tab=feedback.get("current_tab"),
+            message=feedback.get("message"),
+            error=feedback.get("error"),
+            data=feedback.get("data"),
+        )
+
+    return UICommandStatusResponse(
+        command_id=command_id,
+        status=status,
+        action=action,
+        feedback=fb_model,
+    )
+
+
+@router.get("/online-users")
+async def get_online_users(request: Request):
+    """Check which users are currently online (have active frontend WS connections)."""
+    from app.config import get_settings
+    from app.core.auth import get_session_data, get_redis
+    from app.core.service_container import get_container
+
+    settings = get_settings()
+    session_id = request.cookies.get(settings.session_cookie_name)
+    if not session_id:
+        raise HTTPException(status_code=401, detail="Not authenticated")
+
+    redis = get_redis()
+    session_data = await get_session_data(redis, session_id)
+    if session_data is None:
+        raise HTTPException(status_code=401, detail="Session expired")
+
+    container = get_container()
+    if not container.has("ai_ui_control_ws"):
+        raise HTTPException(status_code=503, detail="AI UI Control not available")
+
+    ws_manager = container.get("ai_ui_control_ws")
+    return {"online_users": ws_manager.get_online_users()}
+
+
+# ─── WebSocket endpoint (for frontend) ───
+
+@router.websocket("/ws")
+async def ai_ui_control_ws(websocket: WebSocket):
+    """WebSocket endpoint for AI UI control.
+
+    Frontend connects here to receive UI commands from AI agents.
+    Frontend sends feedback back through this WebSocket after executing commands.
+
+    Authentication: via session cookie (same pattern as kommunikation plugin).
+    """
+    from app.config import get_settings
+    from app.core.auth import get_session_data, get_redis
+    from app.core.service_container import get_container
+
+    settings = get_settings()
+    session_id = websocket.cookies.get(settings.session_cookie_name)
+    if not session_id:
+        await websocket.close(code=4001, reason="Not authenticated")
+        return
+
+    redis = get_redis()
+    session_data = await get_session_data(redis, session_id)
+    if session_data is None:
+        await websocket.close(code=4001, reason="Session expired")
+        return
+
+    user_id = session_data["user_id"]
+    tenant_id = session_data["tenant_id"]
+
+    container = get_container()
+    if not container.has("ai_ui_control_ws"):
+        await websocket.close(code=4003, reason="AI UI Control not available")
+        return
+
+    ws_manager = container.get("ai_ui_control_ws")
+    await ws_manager.connect(websocket, user_id)
+
+    try:
+        while True:
+            data = await websocket.receive_text()
+            msg = json.loads(data)
+            msg_type = msg.get("type")
+
+            if msg_type == "ping":
+                await websocket.send_text(json.dumps({"type": "pong"}))
+            elif msg_type == "feedback":
+                # Frontend sends feedback after executing a command
+                ws_manager.store_feedback(msg)
+            elif msg_type == "status":
+                # Frontend requests current state info
+                await websocket.send_text(json.dumps({
+                    "type": "status",
+                    "online": True,
+                    "user_id": user_id,
+                }))
+    except WebSocketDisconnect:
+        await ws_manager.disconnect(websocket, user_id)
+    except Exception:
+        logger.exception("AI UI Control WebSocket error")
+        await ws_manager.disconnect(websocket, user_id)
@@ -0,0 +1,108 @@
+"""Pydantic schemas for AI UI Control commands and feedback."""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class UICommandType(str, Enum):
+    """Supported UI command types."""
+    navigate = "navigate"
+    filter = "filter"
+    open_contact = "open_contact"
+    modal = "modal"
+    tab = "tab"
+    settings = "settings"
+
+
+class UICommandStatus(str, Enum):
+    """Status of a UI command execution."""
+    pending = "pending"
+    delivered = "delivered"
+    success = "success"
+    failed = "failed"
+    timeout = "timeout"
+
+
+class UICommand(BaseModel):
+    """A UI command sent by an AI agent to control the frontend.
+
+    Task 4.1: UI-Command-Protokoll
+    JSON protocol for UI commands with action types:
+    - navigate: {action: 'navigate', path: '/contacts/123'}
+    - filter: {action: 'filter', entity: 'contacts', filter: {type: 'company'}}
+    - open_contact: {action: 'open_contact', id: '...'}
+    - modal: {action: 'modal', modal: 'edit', contactId: '...'}
+    - tab: {action: 'tab', tab: 'emails', contactId: '...'}
+    - settings: {action: 'settings', section: 'ai', key: 'model', value: 'gpt-4'}
+    """
+    command_id: str = Field(..., description="Unique command ID")
+    action: UICommandType = Field(..., description="Command type")
+    # navigate
+    path: str | None = Field(None, description="Target path for navigate command")
+    # filter
+    entity: str | None = Field(None, description="Entity type for filter command")
+    filter: dict[str, Any] | None = Field(None, description="Filter criteria")
+    # open_contact
+    contact_id: str | None = Field(None, description="Contact ID for open_contact command")
+    # modal
+    modal: str | None = Field(None, description="Modal type: 'edit', 'create', 'delete', 'close'")
+    # tab
+    tab: str | None = Field(None, description="Tab name: 'emails', 'files', 'calendar', etc.")
+    # settings
+    section: str | None = Field(None, description="Settings section")
+    key: str | None = Field(None, description="Setting key")
+    value: Any | None = Field(None, description="Setting value")
+    # metadata
+    timestamp: str | None = Field(None, description="Command timestamp ISO format")
+    description: str | None = Field(None, description="Human-readable description of the command")
+
+
+class UICommandCreate(BaseModel):
+    """Request body for creating a UI command (sent by AI agent)."""
+    action: UICommandType = Field(..., description="Command type")
+    path: str | None = None
+    entity: str | None = None
+    filter: dict[str, Any] | None = None
+    contact_id: str | None = None
+    modal: str | None = None
+    tab: str | None = None
+    section: str | None = None
+    key: str | None = None
+    value: Any | None = None
+    description: str | None = None
+
+
+class UICommandFeedback(BaseModel):
+    """Feedback from frontend after executing a command.
+
+    Task 4.10: UI-Action-Feedback an KI
+    Frontend sends confirmation back: {action: 'navigate', status: 'success', current_path: '/contacts/123'}
+    """
+    command_id: str = Field(..., description="ID of the command being acknowledged")
+    status: UICommandStatus = Field(..., description="Execution status")
+    action: UICommandType | None = None
+    current_path: str | None = Field(None, description="Current URL path after command")
+    current_tab: str | None = Field(None, description="Current active tab")
+    message: str | None = Field(None, description="Optional message")
+    error: str | None = Field(None, description="Error message if failed")
+    data: dict[str, Any] | None = Field(None, description="Additional response data")
+
+
+class UICommandResponse(BaseModel):
+    """Response after creating a command."""
+    command_id: str
+    status: UICommandStatus
+    action: UICommandType
+    message: str | None = None
+
+
+class UICommandStatusResponse(BaseModel):
+    """Status response for polling command execution result."""
+    command_id: str
+    status: UICommandStatus
+    action: UICommandType | None = None
+    feedback: UICommandFeedback | None = None
@@ -0,0 +1,129 @@
+"""WebSocket connection manager for AI UI Control.
+
+Manages WebSocket connections from frontend clients. When an AI agent sends
+a UI command via REST API, the command is forwarded to the frontend via
+these WebSocket connections. The frontend executes the command and sends
+feedback back through the same WebSocket.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+import uuid
+from typing import Any
+
+from fastapi import WebSocket
+
+logger = logging.getLogger(__name__)
+
+
+class AIUIControlWSManager:
+    """Manages WebSocket connections for AI UI control.
+
+    Connections are per-user: each authenticated user can have one or more
+    frontend tabs connected. Commands are delivered to all tabs of the target
+    user. Feedback from any tab is accepted and stored.
+    """
+
+    def __init__(self) -> None:
+        # user_id → list of WebSocket connections
+        self._connections: dict[str, list[WebSocket]] = {}
+        # command_id → feedback dict (stored when frontend responds)
+        self._feedback: dict[str, dict[str, Any]] = {}
+        # command_id → timestamp when delivered (for timeout tracking)
+        self._delivered_at: dict[str, float] = {}
+        # command_id → user_id (to route feedback)
+        self._command_user: dict[str, str] = {}
+
+    async def connect(self, websocket: WebSocket, user_id: str) -> None:
+        """Accept and register a new WebSocket connection."""
+        await websocket.accept()
+        if user_id not in self._connections:
+            self._connections[user_id] = []
+        self._connections[user_id].append(websocket)
+        logger.debug(f"AI UI Control WS connected: user={user_id}, total={len(self._connections[user_id])}")
+
+    async def disconnect(self, websocket: WebSocket, user_id: str) -> None:
+        """Remove a WebSocket connection."""
+        conns = self._connections.get(user_id, [])
+        if websocket in conns:
+            conns.remove(websocket)
+        if not conns:
+            self._connections.pop(user_id, None)
+        logger.debug(f"AI UI Control WS disconnected: user={user_id}, remaining={len(conns)}")
+
+    async def send_command(self, user_id: str, command: dict[str, Any]) -> str | None:
+        """Send a UI command to all frontend connections of a user.
+
+        Returns the command_id if delivered, None if user has no active connections.
+        """
+        command_id = command.get("command_id") or str(uuid.uuid4())
+        command["command_id"] = command_id
+        command.setdefault("timestamp", time.time())
+
+        conns = self._connections.get(user_id, [])
+        if not conns:
+            logger.debug(f"AI UI Control: no active connections for user {user_id}")
+            return None
+
+        self._command_user[command_id] = user_id
+        self._delivered_at[command_id] = time.time()
+
+        text = json.dumps(command, default=str)
+        delivered = False
+        for ws in conns:
+            try:
+                await ws.send_text(text)
+                delivered = True
+            except Exception:
+                logger.warning(f"Failed to send command to user {user_id}, removing connection")
+                await self.disconnect(ws, user_id)
+
+        if delivered:
+            # Initialize as pending feedback
+            self._feedback.setdefault(command_id, {
+                "command_id": command_id,
+                "status": "delivered",
+                "action": command.get("action"),
+            })
+            return command_id
+        return None
+
+    def store_feedback(self, feedback: dict[str, Any]) -> None:
+        """Store feedback from frontend after command execution."""
+        command_id = feedback.get("command_id")
+        if command_id:
+            self._feedback[command_id] = feedback
+            logger.debug(f"AI UI Control: feedback stored for command {command_id}: {feedback.get('status')}")
+
+    def get_feedback(self, command_id: str) -> dict[str, Any] | None:
+        """Get stored feedback for a command."""
+        return self._feedback.get(command_id)
+
+    def is_user_online(self, user_id: str) -> bool:
+        """Check if a user has any active frontend connections."""
+        return user_id in self._connections and len(self._connections[user_id]) > 0
+
+    def get_online_users(self) -> list[str]:
+        """Get list of currently connected user IDs."""
+        return list(self._connections.keys())
+
+    def cleanup_stale(self, timeout_seconds: int = 60) -> None:
+        """Remove stale command tracking entries older than timeout."""
+        now = time.time()
+        stale_ids = [
+            cid for cid, ts in self._delivered_at.items()
+            if now - ts > timeout_seconds
+        ]
+        for cid in stale_ids:
+            if cid not in self._feedback or self._feedback[cid].get("status") == "delivered":
+                self._feedback[cid] = {
+                    "command_id": cid,
+                    "status": "timeout",
+                    "action": None,
+                    "message": "Command timed out waiting for frontend response",
+                }
+            self._delivered_at.pop(cid, None)
+            self._command_user.pop(cid, None)
				`@@ -0,0 +1 @@`
				`"""AI UI Control plugin — lets AI agents control the frontend UI via WebSocket."""`