"""Tests for the Automation & Agents plugin. Uses pytest with async fixtures. Tests use SQLite in-memory database since PostgreSQL may not be available in the dev container. """ from __future__ import annotations import uuid from datetime import UTC, datetime, timedelta from typing import Any, AsyncGenerator import pytest import pytest_asyncio from sqlalchemy import create_engine, event from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.orm import sessionmaker from app.core.db import Base from app.plugins.builtins.automation.models import ( AgentDefinition, AgentRun, AgentVersion, AutomationCronJob, AutomationDefinition, AutomationRun, AutomationVersion, ) from app.plugins.builtins.automation.services import ( AgentService, AutomationService, CronJobService, RunLogService, ) # ─── Fixtures ─── @pytest_asyncio.fixture async def db() -> AsyncGenerator[AsyncSession, None]: """Create an in-memory SQLite database for testing.""" engine = create_async_engine( "sqlite+aiosqlite:///:memory:", echo=False, ) async with engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) async with async_session() as session: yield session await engine.dispose() @pytest.fixture def tenant_id() -> uuid.UUID: return uuid.uuid4() @pytest.fixture def user_id() -> uuid.UUID: return uuid.uuid4() # ─── AgentService Tests ─── class TestAgentService: """Tests for AgentService CRUD operations.""" @pytest.mark.asyncio async def test_create_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test creating an agent definition.""" data = { "name": "test-agent", "description": "Test agent description", "llm_model": "gpt-4", "system_prompt": "You are a test agent.", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 1.0, } agent = await AgentService.create(db, tenant_id, data, user_id=user_id) assert agent is not None assert agent.name == "test-agent" assert agent.tenant_id == tenant_id assert agent.created_by == user_id assert agent.mode == "reactive" assert agent.is_active is True @pytest.mark.asyncio async def test_get_agent_by_id(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test retrieving an agent by ID.""" data = {"name": "get-test", "description": "", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 1.0} created = await AgentService.create(db, tenant_id, data, user_id=user_id) fetched = await AgentService.get_by_id(db, tenant_id, created.id) assert fetched is not None assert fetched.id == created.id assert fetched.name == "get-test" @pytest.mark.asyncio async def test_get_agent_by_name(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test retrieving an agent by name.""" data = {"name": "name-test", "description": "", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 1.0} await AgentService.create(db, tenant_id, data, user_id=user_id) fetched = await AgentService.get_by_name(db, "name-test", tenant_id=tenant_id) assert fetched is not None assert fetched.name == "name-test" @pytest.mark.asyncio async def test_update_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test updating an agent definition.""" data = {"name": "update-test", "description": "Original", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 1.0} created = await AgentService.create(db, tenant_id, data, user_id=user_id) updated = await AgentService.update(db, tenant_id, created.id, {"description": "Updated description"}, user_id=user_id) assert updated is not None assert updated.description == "Updated description" @pytest.mark.asyncio async def test_delete_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test deleting an agent definition.""" data = {"name": "delete-test", "description": "", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 1.0} created = await AgentService.create(db, tenant_id, data, user_id=user_id) result = await AgentService.delete(db, tenant_id, created.id) assert result is True fetched = await AgentService.get_by_id(db, tenant_id, created.id) assert fetched is None @pytest.mark.asyncio async def test_list_agents(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test listing agents with filters.""" for i in range(3): data = {"name": f"list-test-{i}", "description": "", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 1.0} await AgentService.create(db, tenant_id, data, user_id=user_id) items, total = await AgentService.list(db, tenant_id) assert total == 3 assert len(items) == 3 # ─── AutomationService Tests ─── class TestAutomationService: """Tests for AutomationService CRUD operations.""" @pytest.mark.asyncio async def test_create_automation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test creating an automation definition.""" data = { "name": "test-automation", "description": "Test automation", "trigger_type": "manual", "trigger_config": {}, "conditions": [], "actions": [{"type": "api_call", "config": {"url": "https://example.com"}}], "is_active": True, "dry_run": False, } automation = await AutomationService.create(db, tenant_id, data, user_id=user_id) assert automation is not None assert automation.name == "test-automation" assert automation.trigger_type == "manual" assert len(automation.actions) == 1 @pytest.mark.asyncio async def test_get_automation_by_id(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test retrieving an automation by ID.""" data = {"name": "get-auto", "description": "", "trigger_type": "event", "trigger_config": {"event_name": "contact.created"}, "conditions": [], "actions": [], "is_active": True, "dry_run": False} created = await AutomationService.create(db, tenant_id, data, user_id=user_id) fetched = await AutomationService.get_by_id(db, tenant_id, created.id) assert fetched is not None assert fetched.id == created.id @pytest.mark.asyncio async def test_update_automation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test updating an automation definition.""" data = {"name": "upd-auto", "description": "Original", "trigger_type": "manual", "trigger_config": {}, "conditions": [], "actions": [], "is_active": True, "dry_run": False} created = await AutomationService.create(db, tenant_id, data, user_id=user_id) updated = await AutomationService.update(db, tenant_id, created.id, {"description": "Updated"}, user_id=user_id) assert updated is not None assert updated.description == "Updated" @pytest.mark.asyncio async def test_delete_automation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test deleting an automation definition.""" data = {"name": "del-auto", "description": "", "trigger_type": "manual", "trigger_config": {}, "conditions": [], "actions": [], "is_active": True, "dry_run": False} created = await AutomationService.create(db, tenant_id, data, user_id=user_id) result = await AutomationService.delete(db, tenant_id, created.id) assert result is True fetched = await AutomationService.get_by_id(db, tenant_id, created.id) assert fetched is None # ─── CronJobService Tests ─── class TestCronJobService: """Tests for CronJobService CRUD operations.""" @pytest.mark.asyncio async def test_create_cron_job(self, db: AsyncSession, tenant_id: uuid.UUID): """Test creating a cron job.""" data = { "name": "test-cron", "cron_expression": "*/5 * * * *", "job_type": "automation_trigger", "plugin_name": "test", "is_active": True, } job = await CronJobService.create(db, tenant_id, data) assert job is not None assert job.name == "test-cron" assert job.cron_expression == "*/5 * * * *" @pytest.mark.asyncio async def test_get_cron_job_by_id(self, db: AsyncSession, tenant_id: uuid.UUID): """Test retrieving a cron job by ID.""" data = {"name": "get-cron", "cron_expression": "0 9 * * *", "job_type": "automation_trigger", "plugin_name": "test", "is_active": True} created = await CronJobService.create(db, tenant_id, data) fetched = await CronJobService.get_by_id(db, tenant_id, created.id) assert fetched is not None assert fetched.id == created.id @pytest.mark.asyncio async def test_update_cron_job(self, db: AsyncSession, tenant_id: uuid.UUID): """Test updating a cron job.""" data = {"name": "upd-cron", "cron_expression": "0 9 * * *", "job_type": "automation_trigger", "plugin_name": "test", "is_active": True} created = await CronJobService.create(db, tenant_id, data) updated = await CronJobService.update(db, tenant_id, created.id, {"cron_expression": "0 10 * * *"}) assert updated is not None assert updated.cron_expression == "0 10 * * *" @pytest.mark.asyncio async def test_delete_cron_job(self, db: AsyncSession, tenant_id: uuid.UUID): """Test deleting a cron job.""" data = {"name": "del-cron", "cron_expression": "0 9 * * *", "job_type": "automation_trigger", "plugin_name": "test", "is_active": True} created = await CronJobService.create(db, tenant_id, data) result = await CronJobService.delete(db, tenant_id, created.id) assert result is True fetched = await CronJobService.get_by_id(db, tenant_id, created.id) assert fetched is None # ─── Version Management Tests ─── class TestVersionManagement: """Tests for version creation and restore.""" @pytest.mark.asyncio async def test_version_created_on_agent_create(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test that a version snapshot is created when an agent is created.""" data = {"name": "ver-agent", "description": "Version test", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 1.0} agent = await AgentService.create(db, tenant_id, data, user_id=user_id) versions, total = await AgentService.get_versions(db, tenant_id, agent.id) assert total == 1 assert versions[0].version_number == 1 assert versions[0].snapshot["name"] == "ver-agent" @pytest.mark.asyncio async def test_version_created_on_agent_update(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test that a new version is created when an agent is updated.""" data = {"name": "ver-upd", "description": "Original", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 1.0} agent = await AgentService.create(db, tenant_id, data, user_id=user_id) await AgentService.update(db, tenant_id, agent.id, {"description": "Updated"}, user_id=user_id) versions, total = await AgentService.get_versions(db, tenant_id, agent.id) assert total == 2 assert versions[0].version_number == 2 @pytest.mark.asyncio async def test_version_restore_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test restoring an agent from a version snapshot.""" data = {"name": "ver-restore", "description": "Original", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 1.0} agent = await AgentService.create(db, tenant_id, data, user_id=user_id) await AgentService.update(db, tenant_id, agent.id, {"description": "Changed"}, user_id=user_id) versions, _ = await AgentService.get_versions(db, tenant_id, agent.id) # Restore to version 1 restored = await AgentService.restore_version(db, tenant_id, agent.id, versions[1].id, user_id=user_id) assert restored is not None assert restored.description == "Original" @pytest.mark.asyncio async def test_version_created_on_automation_update(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test that a new version is created when an automation is updated.""" data = {"name": "auto-ver", "description": "Original", "trigger_type": "manual", "trigger_config": {}, "conditions": [], "actions": [], "is_active": True, "dry_run": False} auto = await AutomationService.create(db, tenant_id, data, user_id=user_id) await AutomationService.update(db, tenant_id, auto.id, {"description": "Updated"}, user_id=user_id) versions, total = await AutomationService.get_versions(db, tenant_id, auto.id) assert total == 2 # ─── Condition Evaluation Tests ─── class TestConditionEvaluation: """Tests for condition evaluation logic.""" def _evaluate_condition(self, condition: dict, context: dict) -> bool: """Evaluate a single condition against context data.""" field = condition.get("field", "") operator = condition.get("operator", "equals") value = condition.get("value", "") actual = context.get(field) if operator == "eq" or operator == "equals": return str(actual) == str(value) elif operator == "ne" or operator == "not_equals": return str(actual) != str(value) elif operator == "gt" or operator == "greater_than": try: return float(actual) > float(value) except (ValueError, TypeError): return False elif operator == "lt" or operator == "less_than": try: return float(actual) < float(value) except (ValueError, TypeError): return False elif operator == "contains": return str(value) in str(actual) elif operator == "exists": return actual is not None return False def test_condition_eq(self): assert self._evaluate_condition( {"field": "status", "operator": "eq", "value": "active"}, {"status": "active"} ) is True assert self._evaluate_condition( {"field": "status", "operator": "eq", "value": "inactive"}, {"status": "active"} ) is False def test_condition_ne(self): assert self._evaluate_condition( {"field": "status", "operator": "ne", "value": "inactive"}, {"status": "active"} ) is True def test_condition_gt(self): assert self._evaluate_condition( {"field": "age", "operator": "gt", "value": "18"}, {"age": 25} ) is True assert self._evaluate_condition( {"field": "age", "operator": "gt", "value": "30"}, {"age": 25} ) is False def test_condition_lt(self): assert self._evaluate_condition( {"field": "age", "operator": "lt", "value": "30"}, {"age": 25} ) is True def test_condition_contains(self): assert self._evaluate_condition( {"field": "email", "operator": "contains", "value": "@example.com"}, {"email": "test@example.com"} ) is True assert self._evaluate_condition( {"field": "email", "operator": "contains", "value": "@other.com"}, {"email": "test@example.com"} ) is False def test_condition_exists(self): assert self._evaluate_condition( {"field": "email", "operator": "exists", "value": ""}, {"email": "test@example.com"} ) is True assert self._evaluate_condition( {"field": "missing", "operator": "exists", "value": ""}, {"email": "test@example.com"} ) is False # ─── Dry-Run Mode Tests ─── class TestDryRunMode: """Tests for dry-run mode (no actions executed).""" @pytest.mark.asyncio async def test_dry_run_automation_creation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test creating an automation with dry_run=True.""" data = {"name": "dry-run-test", "description": "", "trigger_type": "manual", "trigger_config": {}, "conditions": [], "actions": [{"type": "api_call", "config": {"url": "https://example.com"}}], "is_active": True, "dry_run": True} automation = await AutomationService.create(db, tenant_id, data, user_id=user_id) assert automation.dry_run is True @pytest.mark.asyncio async def test_dry_run_flag_in_run(self, db: AsyncSession, tenant_id: uuid.UUID): """Test that dry_run flag is stored in AutomationRun.""" run = AutomationRun( tenant_id=tenant_id, automation_id=uuid.uuid4(), status="dry_run", started_at=datetime.now(UTC), dry_run=True, ) db.add(run) await db.flush() assert run.dry_run is True # ─── Rate Limiting Tests ─── class TestRateLimiting: """Tests for rate limiting (max executions per hour).""" @pytest.mark.asyncio async def test_rate_limit_check(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test that rate limit is enforced.""" data = {"name": "rate-limit", "description": "", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 2, "max_duration_seconds": 300, "budget_limit_usd": 1.0} agent = await AgentService.create(db, tenant_id, data, user_id=user_id) # Create 2 runs (within limit) for _ in range(2): run = AgentRun( tenant_id=tenant_id, agent_id=agent.id, status="completed", started_at=datetime.now(UTC), ) db.add(run) await db.flush() # Check rate limit from sqlalchemy import func, select one_hour_ago = datetime.now(UTC) - timedelta(hours=1) result = await db.execute( select(func.count()) .select_from(AgentRun) .where(AgentRun.agent_id == agent.id, AgentRun.started_at >= one_hour_ago) ) recent_runs = result.scalar() or 0 assert recent_runs == 2 assert recent_runs < agent.max_executions_per_hour # 2 < 2 is False, so limit would be hit # ─── Budget Limit Tests ─── class TestBudgetLimit: """Tests for budget limit enforcement.""" @pytest.mark.asyncio async def test_budget_limit_check(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID): """Test that budget limit is enforced.""" data = {"name": "budget-limit", "description": "", "llm_model": "gpt-4", "system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive", "is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300, "budget_limit_usd": 0.5} agent = await AgentService.create(db, tenant_id, data, user_id=user_id) # Create runs with cumulative cost for cost in [0.2, 0.2, 0.2]: run = AgentRun( tenant_id=tenant_id, agent_id=agent.id, status="completed", started_at=datetime.now(UTC), cost_usd=cost, ) db.add(run) await db.flush() # Check budget from sqlalchemy import func, select cost_result = await db.execute( select(func.coalesce(func.sum(AgentRun.cost_usd), 0.0)) .where(AgentRun.agent_id == agent.id) ) total_cost = float(cost_result.scalar() or 0.0) assert total_cost == 0.6 assert total_cost >= agent.budget_limit_usd # 0.6 >= 0.5, budget exceeded # ─── Infinite Loop Detection Tests ─── class TestInfiniteLoopDetection: """Tests for infinite loop detection.""" def test_consecutive_tool_call_detection(self): """Test that 5+ consecutive same tool calls are detected.""" tool_call_count: dict[str, int] = {} tool_name = "send_email" for i in range(5): tool_call_count[tool_name] = tool_call_count.get(tool_name, 0) + 1 if tool_call_count[tool_name] >= 5: assert True return assert False, "Loop detection should have triggered" def test_different_tools_not_detected(self): """Test that different tool calls don't trigger loop detection.""" tool_call_count: dict[str, int] = {} for i in range(5): tool_call_count[f"tool_{i}"] = tool_call_count.get(f"tool_{i}", 0) + 1 if tool_call_count[f"tool_{i}"] >= 5: assert False, "Different tools should not trigger loop detection" assert True