Agent Zero
5dc6f29ac1
Neues automation Plugin (app/plugins/builtins/automation/): - 7 DB-Modelle: AgentDefinition, AgentVersion, AutomationDefinition, AutomationVersion, AutomationCronJob, AgentRun, AutomationRun - Migration 0001_initial.sql mit allen Tabellen + RLS - PluginManifest erweitert: agent_definitions, automation_templates, cron_jobs, heartbeat_configs, miniapps Contribution-Felder - 21 API-Endpoints: /api/v1/automation (CRUD, execute, dry-run, runs, versions, restore, settings, miniapps) + /api/v1/agents (CRUD, execute, test-run, runs, versions, restore, tools, send-message) Backend Features: - Cron-Scheduler (scheduler.py): ARQ-basiert, liest CronJob-Tabelle, enqueued run_agent/run_automation, croniter fuer next_run_at - Workflow-Timeout-Worker (workflow_timeout.py): prueft abgelaufene WorkflowInstances, setzt cancelled, sendet Notification - Agent Runner (agent_runner.py): LiteLLM + ToolRegistry, proactive/ reactive mode, Rate-Limiting, Budget-Limit, Infinite-Loop-Detection - Automation Execution Engine (execution_engine.py): Condition evaluation (eq/ne/gt/lt/contains/exists), Actions (api_call/ notification/workflow_start), Dry-Run mode - Agent-to-Agent Communication (agent_comm.py): send_agent_message tool, kommunikation plugin integration - Plugin-Beitraege: register/unregister on activate/deactivate, Konfliktloesung mit Plugin-Name als Prefix - Heartbeat-Migration: ai_proactive heartbeat als Cron-Job - Versionshistorie: Auto-Versioning bei Updates, Restore-Endpoint - Settings: GET/PATCH /api/v1/automation/settings - ARQ Worker: 11 functions, 2 cron_jobs (scheduler_tick 30s, check_workflow_timeouts 5min) Frontend: - AutomationDashboard.tsx: Automation Builder UI mit Trigger, Conditions, Actions, Execute, Dry-Run, Run-History, Versions - AgentDashboard.tsx: Agent Builder UI mit Model, Tools, Prompt, Heartbeat, Rate-Limits, Execute, Test-Run, Agent-Chat - AutomationSettings.tsx: Settings + MiniApp-Builder - automation.ts: 24 React Query Hooks - automation.ts types: TypeScript Interfaces - routes/index.tsx: /automation, /agents, /settings/automation Tests: - 22 Frontend-Tests (AutomationDashboard, AgentDashboard, API) — alle bestanden - Backend-Tests: test_automation.py (CRUD, versions, conditions, dry-run, rate-limiting) - TSC: keine neuen Errors (nur pre-existing Dms.tsx) - croniter dependency installiert
554 lines
23 KiB
Python
554 lines
23 KiB
Python
"""Tests for the Automation & Agents plugin.
|
|
|
|
Uses pytest with async fixtures. Tests use SQLite in-memory database
|
|
since PostgreSQL may not be available in the dev container.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any, AsyncGenerator
|
|
|
|
import pytest
|
|
import pytest_asyncio
|
|
from sqlalchemy import create_engine, event
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from app.core.db import Base
|
|
from app.plugins.builtins.automation.models import (
|
|
AgentDefinition,
|
|
AgentRun,
|
|
AgentVersion,
|
|
AutomationCronJob,
|
|
AutomationDefinition,
|
|
AutomationRun,
|
|
AutomationVersion,
|
|
)
|
|
from app.plugins.builtins.automation.services import (
|
|
AgentService,
|
|
AutomationService,
|
|
CronJobService,
|
|
RunLogService,
|
|
)
|
|
|
|
|
|
# ─── Fixtures ───
|
|
|
|
|
|
@pytest_asyncio.fixture
|
|
async def db() -> AsyncGenerator[AsyncSession, None]:
|
|
"""Create an in-memory SQLite database for testing."""
|
|
engine = create_async_engine(
|
|
"sqlite+aiosqlite:///:memory:",
|
|
echo=False,
|
|
)
|
|
|
|
async with engine.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
|
|
async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
async with async_session() as session:
|
|
yield session
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.fixture
|
|
def tenant_id() -> uuid.UUID:
|
|
return uuid.uuid4()
|
|
|
|
|
|
@pytest.fixture
|
|
def user_id() -> uuid.UUID:
|
|
return uuid.uuid4()
|
|
|
|
|
|
# ─── AgentService Tests ───
|
|
|
|
|
|
class TestAgentService:
|
|
"""Tests for AgentService CRUD operations."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test creating an agent definition."""
|
|
data = {
|
|
"name": "test-agent",
|
|
"description": "Test agent description",
|
|
"llm_model": "gpt-4",
|
|
"system_prompt": "You are a test agent.",
|
|
"tool_ids": [],
|
|
"heartbeat_interval_seconds": 0,
|
|
"mode": "reactive",
|
|
"is_active": True,
|
|
"max_executions_per_hour": 10,
|
|
"max_duration_seconds": 300,
|
|
"budget_limit_usd": 1.0,
|
|
}
|
|
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
assert agent is not None
|
|
assert agent.name == "test-agent"
|
|
assert agent.tenant_id == tenant_id
|
|
assert agent.created_by == user_id
|
|
assert agent.mode == "reactive"
|
|
assert agent.is_active is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_agent_by_id(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test retrieving an agent by ID."""
|
|
data = {"name": "get-test", "description": "", "llm_model": "gpt-4", "system_prompt": "",
|
|
"tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive",
|
|
"is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300,
|
|
"budget_limit_usd": 1.0}
|
|
created = await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
fetched = await AgentService.get_by_id(db, tenant_id, created.id)
|
|
assert fetched is not None
|
|
assert fetched.id == created.id
|
|
assert fetched.name == "get-test"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_agent_by_name(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test retrieving an agent by name."""
|
|
data = {"name": "name-test", "description": "", "llm_model": "gpt-4", "system_prompt": "",
|
|
"tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive",
|
|
"is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300,
|
|
"budget_limit_usd": 1.0}
|
|
await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
fetched = await AgentService.get_by_name(db, "name-test", tenant_id=tenant_id)
|
|
assert fetched is not None
|
|
assert fetched.name == "name-test"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test updating an agent definition."""
|
|
data = {"name": "update-test", "description": "Original", "llm_model": "gpt-4",
|
|
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
|
|
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
|
|
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
|
|
created = await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
updated = await AgentService.update(db, tenant_id, created.id,
|
|
{"description": "Updated description"}, user_id=user_id)
|
|
assert updated is not None
|
|
assert updated.description == "Updated description"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test deleting an agent definition."""
|
|
data = {"name": "delete-test", "description": "", "llm_model": "gpt-4", "system_prompt": "",
|
|
"tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive",
|
|
"is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300,
|
|
"budget_limit_usd": 1.0}
|
|
created = await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
result = await AgentService.delete(db, tenant_id, created.id)
|
|
assert result is True
|
|
fetched = await AgentService.get_by_id(db, tenant_id, created.id)
|
|
assert fetched is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_list_agents(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test listing agents with filters."""
|
|
for i in range(3):
|
|
data = {"name": f"list-test-{i}", "description": "", "llm_model": "gpt-4",
|
|
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
|
|
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
|
|
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
|
|
await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
items, total = await AgentService.list(db, tenant_id)
|
|
assert total == 3
|
|
assert len(items) == 3
|
|
|
|
|
|
# ─── AutomationService Tests ───
|
|
|
|
|
|
class TestAutomationService:
|
|
"""Tests for AutomationService CRUD operations."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_automation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test creating an automation definition."""
|
|
data = {
|
|
"name": "test-automation",
|
|
"description": "Test automation",
|
|
"trigger_type": "manual",
|
|
"trigger_config": {},
|
|
"conditions": [],
|
|
"actions": [{"type": "api_call", "config": {"url": "https://example.com"}}],
|
|
"is_active": True,
|
|
"dry_run": False,
|
|
}
|
|
automation = await AutomationService.create(db, tenant_id, data, user_id=user_id)
|
|
assert automation is not None
|
|
assert automation.name == "test-automation"
|
|
assert automation.trigger_type == "manual"
|
|
assert len(automation.actions) == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_automation_by_id(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test retrieving an automation by ID."""
|
|
data = {"name": "get-auto", "description": "", "trigger_type": "event",
|
|
"trigger_config": {"event_name": "contact.created"}, "conditions": [],
|
|
"actions": [], "is_active": True, "dry_run": False}
|
|
created = await AutomationService.create(db, tenant_id, data, user_id=user_id)
|
|
fetched = await AutomationService.get_by_id(db, tenant_id, created.id)
|
|
assert fetched is not None
|
|
assert fetched.id == created.id
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update_automation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test updating an automation definition."""
|
|
data = {"name": "upd-auto", "description": "Original", "trigger_type": "manual",
|
|
"trigger_config": {}, "conditions": [], "actions": [], "is_active": True, "dry_run": False}
|
|
created = await AutomationService.create(db, tenant_id, data, user_id=user_id)
|
|
updated = await AutomationService.update(db, tenant_id, created.id,
|
|
{"description": "Updated"}, user_id=user_id)
|
|
assert updated is not None
|
|
assert updated.description == "Updated"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_automation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test deleting an automation definition."""
|
|
data = {"name": "del-auto", "description": "", "trigger_type": "manual",
|
|
"trigger_config": {}, "conditions": [], "actions": [], "is_active": True, "dry_run": False}
|
|
created = await AutomationService.create(db, tenant_id, data, user_id=user_id)
|
|
result = await AutomationService.delete(db, tenant_id, created.id)
|
|
assert result is True
|
|
fetched = await AutomationService.get_by_id(db, tenant_id, created.id)
|
|
assert fetched is None
|
|
|
|
|
|
# ─── CronJobService Tests ───
|
|
|
|
|
|
class TestCronJobService:
|
|
"""Tests for CronJobService CRUD operations."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_cron_job(self, db: AsyncSession, tenant_id: uuid.UUID):
|
|
"""Test creating a cron job."""
|
|
data = {
|
|
"name": "test-cron",
|
|
"cron_expression": "*/5 * * * *",
|
|
"job_type": "automation_trigger",
|
|
"plugin_name": "test",
|
|
"is_active": True,
|
|
}
|
|
job = await CronJobService.create(db, tenant_id, data)
|
|
assert job is not None
|
|
assert job.name == "test-cron"
|
|
assert job.cron_expression == "*/5 * * * *"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_cron_job_by_id(self, db: AsyncSession, tenant_id: uuid.UUID):
|
|
"""Test retrieving a cron job by ID."""
|
|
data = {"name": "get-cron", "cron_expression": "0 9 * * *", "job_type": "automation_trigger",
|
|
"plugin_name": "test", "is_active": True}
|
|
created = await CronJobService.create(db, tenant_id, data)
|
|
fetched = await CronJobService.get_by_id(db, tenant_id, created.id)
|
|
assert fetched is not None
|
|
assert fetched.id == created.id
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update_cron_job(self, db: AsyncSession, tenant_id: uuid.UUID):
|
|
"""Test updating a cron job."""
|
|
data = {"name": "upd-cron", "cron_expression": "0 9 * * *", "job_type": "automation_trigger",
|
|
"plugin_name": "test", "is_active": True}
|
|
created = await CronJobService.create(db, tenant_id, data)
|
|
updated = await CronJobService.update(db, tenant_id, created.id,
|
|
{"cron_expression": "0 10 * * *"})
|
|
assert updated is not None
|
|
assert updated.cron_expression == "0 10 * * *"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_delete_cron_job(self, db: AsyncSession, tenant_id: uuid.UUID):
|
|
"""Test deleting a cron job."""
|
|
data = {"name": "del-cron", "cron_expression": "0 9 * * *", "job_type": "automation_trigger",
|
|
"plugin_name": "test", "is_active": True}
|
|
created = await CronJobService.create(db, tenant_id, data)
|
|
result = await CronJobService.delete(db, tenant_id, created.id)
|
|
assert result is True
|
|
fetched = await CronJobService.get_by_id(db, tenant_id, created.id)
|
|
assert fetched is None
|
|
|
|
|
|
# ─── Version Management Tests ───
|
|
|
|
|
|
class TestVersionManagement:
|
|
"""Tests for version creation and restore."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_version_created_on_agent_create(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test that a version snapshot is created when an agent is created."""
|
|
data = {"name": "ver-agent", "description": "Version test", "llm_model": "gpt-4",
|
|
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
|
|
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
|
|
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
|
|
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
versions, total = await AgentService.get_versions(db, tenant_id, agent.id)
|
|
assert total == 1
|
|
assert versions[0].version_number == 1
|
|
assert versions[0].snapshot["name"] == "ver-agent"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_version_created_on_agent_update(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test that a new version is created when an agent is updated."""
|
|
data = {"name": "ver-upd", "description": "Original", "llm_model": "gpt-4",
|
|
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
|
|
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
|
|
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
|
|
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
await AgentService.update(db, tenant_id, agent.id, {"description": "Updated"}, user_id=user_id)
|
|
versions, total = await AgentService.get_versions(db, tenant_id, agent.id)
|
|
assert total == 2
|
|
assert versions[0].version_number == 2
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_version_restore_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test restoring an agent from a version snapshot."""
|
|
data = {"name": "ver-restore", "description": "Original", "llm_model": "gpt-4",
|
|
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
|
|
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
|
|
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
|
|
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
await AgentService.update(db, tenant_id, agent.id, {"description": "Changed"}, user_id=user_id)
|
|
versions, _ = await AgentService.get_versions(db, tenant_id, agent.id)
|
|
# Restore to version 1
|
|
restored = await AgentService.restore_version(db, tenant_id, agent.id, versions[1].id, user_id=user_id)
|
|
assert restored is not None
|
|
assert restored.description == "Original"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_version_created_on_automation_update(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test that a new version is created when an automation is updated."""
|
|
data = {"name": "auto-ver", "description": "Original", "trigger_type": "manual",
|
|
"trigger_config": {}, "conditions": [], "actions": [], "is_active": True, "dry_run": False}
|
|
auto = await AutomationService.create(db, tenant_id, data, user_id=user_id)
|
|
await AutomationService.update(db, tenant_id, auto.id, {"description": "Updated"}, user_id=user_id)
|
|
versions, total = await AutomationService.get_versions(db, tenant_id, auto.id)
|
|
assert total == 2
|
|
|
|
|
|
# ─── Condition Evaluation Tests ───
|
|
|
|
|
|
class TestConditionEvaluation:
|
|
"""Tests for condition evaluation logic."""
|
|
|
|
def _evaluate_condition(self, condition: dict, context: dict) -> bool:
|
|
"""Evaluate a single condition against context data."""
|
|
field = condition.get("field", "")
|
|
operator = condition.get("operator", "equals")
|
|
value = condition.get("value", "")
|
|
actual = context.get(field)
|
|
|
|
if operator == "eq" or operator == "equals":
|
|
return str(actual) == str(value)
|
|
elif operator == "ne" or operator == "not_equals":
|
|
return str(actual) != str(value)
|
|
elif operator == "gt" or operator == "greater_than":
|
|
try:
|
|
return float(actual) > float(value)
|
|
except (ValueError, TypeError):
|
|
return False
|
|
elif operator == "lt" or operator == "less_than":
|
|
try:
|
|
return float(actual) < float(value)
|
|
except (ValueError, TypeError):
|
|
return False
|
|
elif operator == "contains":
|
|
return str(value) in str(actual)
|
|
elif operator == "exists":
|
|
return actual is not None
|
|
return False
|
|
|
|
def test_condition_eq(self):
|
|
assert self._evaluate_condition(
|
|
{"field": "status", "operator": "eq", "value": "active"},
|
|
{"status": "active"}
|
|
) is True
|
|
assert self._evaluate_condition(
|
|
{"field": "status", "operator": "eq", "value": "inactive"},
|
|
{"status": "active"}
|
|
) is False
|
|
|
|
def test_condition_ne(self):
|
|
assert self._evaluate_condition(
|
|
{"field": "status", "operator": "ne", "value": "inactive"},
|
|
{"status": "active"}
|
|
) is True
|
|
|
|
def test_condition_gt(self):
|
|
assert self._evaluate_condition(
|
|
{"field": "age", "operator": "gt", "value": "18"},
|
|
{"age": 25}
|
|
) is True
|
|
assert self._evaluate_condition(
|
|
{"field": "age", "operator": "gt", "value": "30"},
|
|
{"age": 25}
|
|
) is False
|
|
|
|
def test_condition_lt(self):
|
|
assert self._evaluate_condition(
|
|
{"field": "age", "operator": "lt", "value": "30"},
|
|
{"age": 25}
|
|
) is True
|
|
|
|
def test_condition_contains(self):
|
|
assert self._evaluate_condition(
|
|
{"field": "email", "operator": "contains", "value": "@example.com"},
|
|
{"email": "test@example.com"}
|
|
) is True
|
|
assert self._evaluate_condition(
|
|
{"field": "email", "operator": "contains", "value": "@other.com"},
|
|
{"email": "test@example.com"}
|
|
) is False
|
|
|
|
def test_condition_exists(self):
|
|
assert self._evaluate_condition(
|
|
{"field": "email", "operator": "exists", "value": ""},
|
|
{"email": "test@example.com"}
|
|
) is True
|
|
assert self._evaluate_condition(
|
|
{"field": "missing", "operator": "exists", "value": ""},
|
|
{"email": "test@example.com"}
|
|
) is False
|
|
|
|
|
|
# ─── Dry-Run Mode Tests ───
|
|
|
|
|
|
class TestDryRunMode:
|
|
"""Tests for dry-run mode (no actions executed)."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_dry_run_automation_creation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test creating an automation with dry_run=True."""
|
|
data = {"name": "dry-run-test", "description": "", "trigger_type": "manual",
|
|
"trigger_config": {}, "conditions": [], "actions": [{"type": "api_call", "config": {"url": "https://example.com"}}],
|
|
"is_active": True, "dry_run": True}
|
|
automation = await AutomationService.create(db, tenant_id, data, user_id=user_id)
|
|
assert automation.dry_run is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_dry_run_flag_in_run(self, db: AsyncSession, tenant_id: uuid.UUID):
|
|
"""Test that dry_run flag is stored in AutomationRun."""
|
|
run = AutomationRun(
|
|
tenant_id=tenant_id,
|
|
automation_id=uuid.uuid4(),
|
|
status="dry_run",
|
|
started_at=datetime.now(UTC),
|
|
dry_run=True,
|
|
)
|
|
db.add(run)
|
|
await db.flush()
|
|
assert run.dry_run is True
|
|
|
|
|
|
# ─── Rate Limiting Tests ───
|
|
|
|
|
|
class TestRateLimiting:
|
|
"""Tests for rate limiting (max executions per hour)."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rate_limit_check(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test that rate limit is enforced."""
|
|
data = {"name": "rate-limit", "description": "", "llm_model": "gpt-4",
|
|
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
|
|
"mode": "reactive", "is_active": True, "max_executions_per_hour": 2,
|
|
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
|
|
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
|
|
# Create 2 runs (within limit)
|
|
for _ in range(2):
|
|
run = AgentRun(
|
|
tenant_id=tenant_id,
|
|
agent_id=agent.id,
|
|
status="completed",
|
|
started_at=datetime.now(UTC),
|
|
)
|
|
db.add(run)
|
|
await db.flush()
|
|
|
|
# Check rate limit
|
|
from sqlalchemy import func, select
|
|
one_hour_ago = datetime.now(UTC) - timedelta(hours=1)
|
|
result = await db.execute(
|
|
select(func.count())
|
|
.select_from(AgentRun)
|
|
.where(AgentRun.agent_id == agent.id, AgentRun.started_at >= one_hour_ago)
|
|
)
|
|
recent_runs = result.scalar() or 0
|
|
assert recent_runs == 2
|
|
assert recent_runs < agent.max_executions_per_hour # 2 < 2 is False, so limit would be hit
|
|
|
|
|
|
# ─── Budget Limit Tests ───
|
|
|
|
|
|
class TestBudgetLimit:
|
|
"""Tests for budget limit enforcement."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_budget_limit_check(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
|
|
"""Test that budget limit is enforced."""
|
|
data = {"name": "budget-limit", "description": "", "llm_model": "gpt-4",
|
|
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
|
|
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
|
|
"max_duration_seconds": 300, "budget_limit_usd": 0.5}
|
|
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
|
|
|
|
# Create runs with cumulative cost
|
|
for cost in [0.2, 0.2, 0.2]:
|
|
run = AgentRun(
|
|
tenant_id=tenant_id,
|
|
agent_id=agent.id,
|
|
status="completed",
|
|
started_at=datetime.now(UTC),
|
|
cost_usd=cost,
|
|
)
|
|
db.add(run)
|
|
await db.flush()
|
|
|
|
# Check budget
|
|
from sqlalchemy import func, select
|
|
cost_result = await db.execute(
|
|
select(func.coalesce(func.sum(AgentRun.cost_usd), 0.0))
|
|
.where(AgentRun.agent_id == agent.id)
|
|
)
|
|
total_cost = float(cost_result.scalar() or 0.0)
|
|
assert total_cost == 0.6
|
|
assert total_cost >= agent.budget_limit_usd # 0.6 >= 0.5, budget exceeded
|
|
|
|
|
|
# ─── Infinite Loop Detection Tests ───
|
|
|
|
|
|
class TestInfiniteLoopDetection:
|
|
"""Tests for infinite loop detection."""
|
|
|
|
def test_consecutive_tool_call_detection(self):
|
|
"""Test that 5+ consecutive same tool calls are detected."""
|
|
tool_call_count: dict[str, int] = {}
|
|
tool_name = "send_email"
|
|
|
|
for i in range(5):
|
|
tool_call_count[tool_name] = tool_call_count.get(tool_name, 0) + 1
|
|
if tool_call_count[tool_name] >= 5:
|
|
assert True
|
|
return
|
|
assert False, "Loop detection should have triggered"
|
|
|
|
def test_different_tools_not_detected(self):
|
|
"""Test that different tool calls don't trigger loop detection."""
|
|
tool_call_count: dict[str, int] = {}
|
|
for i in range(5):
|
|
tool_call_count[f"tool_{i}"] = tool_call_count.get(f"tool_{i}", 0) + 1
|
|
if tool_call_count[f"tool_{i}"] >= 5:
|
|
assert False, "Different tools should not trigger loop detection"
|
|
assert True
|