Files
leocrm/tests/test_agent_loop.py
T

781 lines
28 KiB
Python
Raw Normal View History

"""Tests for the ReAct agent loop (app/ai/agent_loop.py).
All tests mock llm_complete and tool_registry — no real LLM or DB needed.
"""
from __future__ import annotations
import asyncio
import json
import uuid
from dataclasses import dataclass
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.ai.agent_loop import ReActResult, ReActStep, run_react_loop
# ──────────────────────────────────────────────────────────────────────────
# Test helpers
# ──────────────────────────────────────────────────────────────────────────
@dataclass
class MockAgentDefinition:
"""Minimal stand-in for AgentDefinition used by run_react_loop."""
id: uuid.UUID
llm_model: str = "gpt-4o"
system_prompt: str = "You are a helpful assistant."
api_key: str | None = None
api_base: str | None = None
provider: str | None = None
max_tokens: int = 1000
def _make_tool_call(
call_id: str = "call_1",
name: str = "search",
arguments: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Build a mock LiteLLM tool-call object."""
args_str = json.dumps(arguments or {})
function_mock = MagicMock()
function_mock.name = name
function_mock.arguments = args_str
tc = MagicMock()
tc.id = call_id
tc.function = function_mock
return tc
def _make_llm_response(
content: str = "",
tool_calls: list[Any] | None = None,
cost_usd: float = 0.001,
) -> dict[str, Any]:
"""Build a mock llm_complete() return value."""
message = MagicMock()
message.content = content
message.tool_calls = tool_calls or None
choice = MagicMock()
choice.message = message
raw_response = MagicMock()
raw_response.choices = [choice]
return {
"content": content,
"usage": {"total_tokens": 100},
"cost_usd": cost_usd,
"model": "gpt-4o",
"raw_response": raw_response,
}
def _make_tool_registry(tools: dict[str, AsyncMock] | None = None) -> MagicMock:
"""Build a mock ToolRegistry."""
registry = MagicMock()
_tools = tools or {}
def _get(name: str) -> Any:
if name in _tools:
tool = MagicMock()
tool.handler = _tools[name]
return tool
return None
registry.get = _get
return registry
@pytest.fixture
def agent_def() -> MockAgentDefinition:
return MockAgentDefinition(id=uuid.uuid4())
@pytest.fixture
def tenant_id() -> uuid.UUID:
return uuid.uuid4()
@pytest.fixture
def user_id() -> uuid.UUID:
return uuid.uuid4()
@pytest.fixture
def mock_db() -> AsyncMock:
return AsyncMock()
@pytest.fixture(autouse=True)
def _mock_hooks():
"""Patch do_action so the loop doesn't try to fire real hooks."""
with patch("app.core.hooks.do_action", new_callable=AsyncMock):
yield
# ──────────────────────────────────────────────────────────────────────────
# Tests
# ──────────────────────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_basic_react_loop_no_tools(agent_def, tenant_id, user_id, mock_db):
"""Single-step loop: LLM returns final response, no tool calls."""
registry = _make_tool_registry()
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.return_value = _make_llm_response(
content="Hello! How can I help you?",
cost_usd=0.002,
)
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Hi"}],
tools=[],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
)
assert result.status == "completed"
assert result.final_content == "Hello! How can I help you?"
assert result.steps_taken == 1
assert len(result.steps) == 1
assert result.steps[0].action is None # No tool call
assert result.steps[0].thought == "Hello! How can I help you?"
assert result.total_cost_usd == pytest.approx(0.002)
mock_llm.assert_awaited_once()
@pytest.mark.asyncio
async def test_multi_step_loop_with_tool_calls(agent_def, tenant_id, user_id, mock_db):
"""Multi-step loop: LLM calls a tool, gets result, then gives final answer."""
search_handler = AsyncMock(return_value="Found 3 contacts: Alice, Bob, Charlie")
registry = _make_tool_registry({"search": search_handler})
# Step 1: LLM calls search tool
# Step 2: LLM gives final answer
responses = [
_make_llm_response(
content="I'll search for contacts.",
tool_calls=[_make_tool_call(name="search", arguments={"query": "contacts"})],
cost_usd=0.003,
),
_make_llm_response(
content="I found 3 contacts: Alice, Bob, and Charlie.",
cost_usd=0.004,
),
]
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = responses
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Find contacts"}],
tools=[{"type": "function", "function": {"name": "search"}}],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
)
assert result.status == "completed"
assert result.steps_taken == 2
assert len(result.steps) == 2
assert result.steps[0].action == "search"
assert result.steps[0].action_input == {"query": "contacts"}
assert result.steps[0].observation == "Found 3 contacts: Alice, Bob, Charlie"
assert result.steps[1].action is None # Final response
assert result.final_content == "I found 3 contacts: Alice, Bob, and Charlie."
assert result.total_cost_usd == pytest.approx(0.007)
assert mock_llm.await_count == 2
search_handler.assert_awaited_once()
@pytest.mark.asyncio
async def test_max_steps_limit(agent_def, tenant_id, user_id, mock_db):
"""Loop stops gracefully when max_steps is reached."""
search_handler = AsyncMock(return_value="result")
registry = _make_tool_registry({"search": search_handler})
# Every response has a tool call — never gives final answer
tool_call = _make_tool_call(name="search", arguments={})
response = _make_llm_response(
content="Searching...",
tool_calls=[tool_call],
cost_usd=0.001,
)
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.return_value = response
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Keep searching"}],
tools=[{"type": "function", "function": {"name": "search"}}],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
max_steps=3,
)
assert result.status == "stopped_max_steps"
assert result.steps_taken == 3
assert len(result.steps) == 3
assert "max_steps" in (result.error or "")
assert mock_llm.await_count == 3
@pytest.mark.asyncio
async def test_timeout_graceful_stop(agent_def, tenant_id, user_id, mock_db):
"""Loop stops gracefully when timeout is exceeded."""
registry = _make_tool_registry()
# Simulate slow LLM responses that eventually exceed timeout
async def slow_llm(**kwargs: Any) -> dict[str, Any]:
await asyncio.sleep(0.5)
return _make_llm_response(content="thinking...", tool_calls=None, cost_usd=0.001)
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = slow_llm
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Hi"}],
tools=[],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
timeout_seconds=0, # Immediate timeout on first check
)
assert result.status == "stopped_timeout"
assert "timeout" in (result.error or "").lower()
@pytest.mark.asyncio
async def test_transient_error_retry(agent_def, tenant_id, user_id, mock_db):
"""Transient errors are retried, then succeed."""
registry = _make_tool_registry()
# First call raises transient error, second succeeds
responses: list[Any] = [
Exception("rate limit exceeded"),
_make_llm_response(content="Success after retry", cost_usd=0.002),
]
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = responses
with patch("app.ai.agent_loop.asyncio.sleep", new_callable=AsyncMock):
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Hi"}],
tools=[],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
)
assert result.status == "completed"
assert result.final_content == "Success after retry"
assert mock_llm.await_count == 2 # First failed, second succeeded
@pytest.mark.asyncio
async def test_permanent_error_stops(agent_def, tenant_id, user_id, mock_db):
"""Permanent errors stop the loop immediately."""
registry = _make_tool_registry()
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = Exception("authentication error: invalid api key")
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Hi"}],
tools=[],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
)
assert result.status == "stopped_error"
assert "Permanent error" in (result.error or "")
assert mock_llm.await_count == 1 # No retry for permanent errors
@pytest.mark.asyncio
async def test_cost_accumulation(agent_def, tenant_id, user_id, mock_db):
"""Cost is accumulated across multiple LLM calls."""
search_handler = AsyncMock(return_value="result")
registry = _make_tool_registry({"search": search_handler})
responses = [
_make_llm_response(
content="Step 1",
tool_calls=[_make_tool_call(name="search")],
cost_usd=0.01,
),
_make_llm_response(
content="Step 2",
tool_calls=[_make_tool_call(name="search")],
cost_usd=0.02,
),
_make_llm_response(
content="Final answer",
cost_usd=0.03,
),
]
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = responses
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Search twice"}],
tools=[{"type": "function", "function": {"name": "search"}}],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
)
assert result.status == "completed"
assert result.total_cost_usd == pytest.approx(0.06) # 0.01 + 0.02 + 0.03
assert result.steps_taken == 3
@pytest.mark.asyncio
async def test_step_persistence_via_callback(agent_def, tenant_id, user_id, mock_db):
"""Steps are passed to on_step callback for persistence."""
registry = _make_tool_registry()
collected_steps: list[ReActStep] = []
async def on_step(step: ReActStep) -> None:
collected_steps.append(step)
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.return_value = _make_llm_response(content="Done", cost_usd=0.001)
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Hi"}],
tools=[],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
on_step=on_step,
)
assert len(collected_steps) == 1
assert collected_steps[0].step_number == 1
assert collected_steps[0].thought == "Done"
assert result.status == "completed"
@pytest.mark.asyncio
async def test_tool_not_found(agent_def, tenant_id, user_id, mock_db):
"""When a tool is not found, the observation contains an error message."""
registry = _make_tool_registry() # No tools registered
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = [
_make_llm_response(
content="Calling unknown tool",
tool_calls=[_make_tool_call(name="nonexistent", arguments={})],
cost_usd=0.001,
),
_make_llm_response(content="OK", cost_usd=0.001),
]
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Call unknown tool"}],
tools=[{"type": "function", "function": {"name": "nonexistent"}}],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
)
assert result.status == "completed"
assert result.steps[0].action == "nonexistent"
assert "not found" in (result.steps[0].observation or "").lower()
def _make_guard_registry(tools: dict[str, Any]) -> MagicMock:
"""Build a mock ToolRegistry with REAL attribute values.
Unlike ``_make_tool_registry`` (whose MagicMock tools auto-create any
attribute), this sets ``required_permission`` explicitly so the F01
guard behaves like production AITool objects.
"""
registry = MagicMock()
def _get(name: str) -> Any:
return tools.get(name)
registry.get = _get
return registry
def _guard_tool(name: str, handler: AsyncMock, required_permission: str | None = None) -> MagicMock:
t = MagicMock()
t.name = name
t.handler = handler
t.required_permission = required_permission
return t
# ──────────────────────────────────────────────────────────────────────────
# F01 (Astra P0): allowlist + required_permission enforced at execution time
# ──────────────────────────────────────────────────────────────────────────
@pytest.mark.asyncio
async def test_f01_registered_but_not_offered_tool_rejected(agent_def, tenant_id, user_id, mock_db):
"""F01 acceptance: a tool registered in the registry but NOT offered
to the LLM is rejected — the handler must never run (stays null)."""
handler = AsyncMock(return_value="secret audit data")
registry = _make_guard_registry({
"audit_restricted": _guard_tool("audit_restricted", handler, required_permission="system:admin"),
})
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = [
_make_llm_response(
content="calling restricted tool",
tool_calls=[_make_tool_call(name="audit_restricted")],
cost_usd=0.001,
),
_make_llm_response(content="understood", cost_usd=0.001),
]
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "run audit_restricted"}],
tools=[{"type": "function", "function": {"name": "audit_allowed"}}], # only audit_allowed offered
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
user_permissions={"permissions": ["system:admin"], "denied": [], "is_system_admin": True},
)
assert result.status == "completed"
handler.assert_not_awaited() # core F01 acceptance: handler stays null
assert "not available" in (result.steps[0].observation or "")
@pytest.mark.asyncio
async def test_f01_offered_tool_user_lacks_permission_rejected(agent_def, tenant_id, user_id, mock_db):
"""Offered tool with required_permission: user WITHOUT the permission
is rejected even though the tool was offered to the LLM."""
handler = AsyncMock(return_value="mail sent")
registry = _make_guard_registry({
"send_mail": _guard_tool("send_mail", handler, required_permission="mail:write"),
})
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = [
_make_llm_response(
content="sending",
tool_calls=[_make_tool_call(name="send_mail")],
cost_usd=0.001,
),
_make_llm_response(content="ok", cost_usd=0.001),
]
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "send a mail"}],
tools=[{"type": "function", "function": {"name": "send_mail"}}],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
user_permissions={"permissions": ["mail:read"], "denied": [], "is_system_admin": False},
)
assert result.status == "completed"
handler.assert_not_awaited()
assert "Permission 'mail:write' required" in (result.steps[0].observation or "")
@pytest.mark.asyncio
async def test_f01_offered_tool_with_permission_executes(agent_def, tenant_id, user_id, mock_db):
"""Offered tool + user HAS the required permission → executes normally."""
handler = AsyncMock(return_value="mail sent")
registry = _make_guard_registry({
"send_mail": _guard_tool("send_mail", handler, required_permission="mail:write"),
})
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = [
_make_llm_response(
content="sending",
tool_calls=[_make_tool_call(name="send_mail")],
cost_usd=0.001,
),
_make_llm_response(content="done", cost_usd=0.001),
]
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "send a mail"}],
tools=[{"type": "function", "function": {"name": "send_mail"}}],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
user_permissions={"permissions": ["mail:write", "mail:read"], "denied": [], "is_system_admin": False},
)
assert result.status == "completed"
handler.assert_awaited_once()
assert result.steps[0].observation == "mail sent"
@pytest.mark.asyncio
async def test_f01_fail_closed_without_permission_context(agent_def, tenant_id, user_id, mock_db):
"""Tool with required_permission but NO permission context → rejected
(fail-closed: missing context is not implicit access)."""
handler = AsyncMock(return_value="boom")
registry = _make_guard_registry({
"dangerous": _guard_tool("dangerous", handler, required_permission="system:admin"),
})
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = [
_make_llm_response(
content="calling",
tool_calls=[_make_tool_call(name="dangerous")],
cost_usd=0.001,
),
_make_llm_response(content="ok", cost_usd=0.001),
]
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "call it"}],
tools=[{"type": "function", "function": {"name": "dangerous"}}],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
user_permissions=None, # no permission context at all
)
assert result.status == "completed"
handler.assert_not_awaited()
assert "Permission 'system:admin' required" in (result.steps[0].observation or "")
@pytest.mark.asyncio
async def test_f01_deny_list_session_shape_respected(agent_def, tenant_id, user_id, mock_db):
"""Session user contexts carry ``denied_permissions`` (not ``denied``).
An explicit deny must reject even when the permission is also granted."""
handler = AsyncMock(return_value="mail sent")
registry = _make_guard_registry({
"send_mail": _guard_tool("send_mail", handler, required_permission="mail:write"),
})
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = [
_make_llm_response(
content="sending",
tool_calls=[_make_tool_call(name="send_mail")],
cost_usd=0.001,
),
_make_llm_response(content="ok", cost_usd=0.001),
]
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "send"}],
tools=[{"type": "function", "function": {"name": "send_mail"}}],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
# session shape: granted AND denied via denied_permissions key
user_permissions={
"permissions": ["mail:write"],
"denied_permissions": ["mail:write"],
"is_system_admin": False,
},
)
assert result.status == "completed"
handler.assert_not_awaited()
assert "Permission 'mail:write' required" in (result.steps[0].observation or "")
@pytest.mark.asyncio
async def test_f01_permission_revoked_during_run_takes_effect_next_action(agent_def, tenant_id, user_id, mock_db):
"""F01 acceptance: revoking permissions DURING a run takes effect on
the next action — the guard reads the CURRENT context every call."""
handler = AsyncMock(return_value="mail sent")
registry = _make_guard_registry({
"send_mail": _guard_tool("send_mail", handler, required_permission="mail:write"),
})
perms: dict[str, Any] = {
"permissions": ["mail:write"],
"denied": [],
"is_system_admin": False,
}
call_count = 0
async def llm_with_revocation(**kwargs: Any) -> dict[str, Any]:
nonlocal call_count
call_count += 1
if call_count == 1:
return _make_llm_response(
content="first send",
tool_calls=[_make_tool_call(name="send_mail")],
cost_usd=0.001,
)
# Revoke the permission before the second LLM response
if call_count == 2:
perms["permissions"] = [] # permission revoked mid-run
return _make_llm_response(
content="second send",
tool_calls=[_make_tool_call(name="send_mail")],
cost_usd=0.001,
)
return _make_llm_response(content="finished", cost_usd=0.001)
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = llm_with_revocation
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "send twice"}],
tools=[{"type": "function", "function": {"name": "send_mail"}}],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
user_permissions=perms,
)
assert result.status == "completed"
handler.assert_awaited_once() # first call ran, second was rejected
assert result.steps[0].observation == "mail sent"
assert "Permission 'mail:write' required" in (result.steps[1].observation or "")
@pytest.mark.asyncio
async def test_f01_dry_run_still_guards_non_offered_tools(agent_def, tenant_id, user_id, mock_db):
"""Dry-run must not even SIMULATE a non-offered tool — the guard runs
before the dry-run path."""
handler = AsyncMock(return_value="x")
registry = _make_guard_registry({
"secret_tool": _guard_tool("secret_tool", handler),
})
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = [
_make_llm_response(
content="trying",
tool_calls=[_make_tool_call(name="secret_tool")],
cost_usd=0.001,
),
_make_llm_response(content="done", cost_usd=0.001),
]
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "try it"}],
tools=[], # nothing offered
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
dry_run=True,
)
assert result.status == "completed"
obs = result.steps[0].observation or ""
assert "not available" in obs
assert "would_execute" not in obs # guard fired, not the dry-run simulation
@pytest.mark.asyncio
async def test_multiple_tool_calls_per_step(agent_def, tenant_id, user_id, mock_db):
"""Multiple tool calls in a single LLM response are all executed."""
handler_a = AsyncMock(return_value="Result A")
handler_b = AsyncMock(return_value="Result B")
registry = _make_tool_registry({"tool_a": handler_a, "tool_b": handler_b})
with patch("app.ai.agent_loop.llm_complete", new_callable=AsyncMock) as mock_llm:
mock_llm.side_effect = [
_make_llm_response(
content="Calling two tools",
tool_calls=[
_make_tool_call(call_id="c1", name="tool_a", arguments={}),
_make_tool_call(call_id="c2", name="tool_b", arguments={}),
],
cost_usd=0.005,
),
_make_llm_response(content="Done with both", cost_usd=0.002),
]
result = await run_react_loop(
agent_definition=agent_def,
messages=[{"role": "user", "content": "Call two tools"}],
tools=[
{"type": "function", "function": {"name": "tool_a"}},
{"type": "function", "function": {"name": "tool_b"}},
],
tool_registry=registry,
db=mock_db,
tenant_id=tenant_id,
user_id=user_id,
)
assert result.status == "completed"
assert result.steps_taken == 2
# Step 1 has two tool calls → two step entries
assert len(result.steps) == 3 # 2 tool steps + 1 final step
handler_a.assert_awaited_once()
handler_b.assert_awaited_once()
@pytest.mark.asyncio
async def test_react_result_dataclass_fields():
"""ReActResult and ReActStep have correct default values."""
step = ReActStep(
step_number=1,
thought="test",
action=None,
action_input=None,
observation=None,
cost_usd=0.0,
timestamp="2025-01-01T00:00:00+00:00",
)
assert step.step_number == 1
assert step.thought == "test"
result = ReActResult(final_content="hello")
assert result.final_content == "hello"
assert result.steps == []
assert result.total_cost_usd == 0.0
assert result.steps_taken == 0
assert result.status == "completed"
assert result.error is None