Files
leocrm/app/plugins/builtins/automation/tests/test_automation.py
T
Agent Zero d89044d8f7
Check Cross-Plugin Imports / check (push) Has been cancelled
fix(d2): datetime.now(UTC) everywhere + SQLITE-001 automation tests on ephemeral postgres
2026-08-24 01:22:42 +02:00

628 lines
26 KiB
Python

"""Tests for the Automation & Agents plugin.
Uses pytest with async fixtures against an ephemeral PostgreSQL database
(SQLITE-001 fix) — matches the project convention and exercises the real
PGUUID/JSONB column types.
"""
from __future__ import annotations
import os
import uuid
from collections.abc import AsyncGenerator
from datetime import UTC, datetime, timedelta
import pytest
import pytest_asyncio
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from app.core.db import Base
import app.models # noqa: F401 — registers core models
import app.models.outbox # noqa: F401 — event_outbox is NOT re-exported by app.models
# Register ALL plugin models so create_all can resolve cross-plugin FKs
# (e.g. entity_attachments.dms_file_id -> files) — same pattern as
# scripts/sync_plugin_schema.py.
import importlib
import pkgutil
import app.plugins.builtins as _builtins_pkg
for _importer, _modname, _ispkg in pkgutil.iter_modules(_builtins_pkg.__path__):
if not _ispkg:
continue
try:
importlib.import_module(f"app.plugins.builtins.{_modname}.models")
except ImportError:
pass # plugin without models module
except Exception: # pragma: no cover - defensive
pass
from app.plugins.builtins.automation.models import (
AgentRun,
AutomationRun,
)
from app.plugins.builtins.automation.services import (
AgentService,
AutomationService,
CronJobService,
)
def _ephemeral_db_url() -> str:
"""Derive an ephemeral test DB URL from DATABASE_URL/.env.test."""
base_url = os.environ.get(
"DATABASE_URL",
"postgresql+asyncpg://leocrm_test:test123@localhost:5432/leocrm_test",
)
return f"{base_url.rsplit('/', 1)[0]}/automation_test_{uuid.uuid4().hex[:8]}"
# ─── Fixtures ───
@pytest_asyncio.fixture
async def db() -> AsyncGenerator[AsyncSession, None]:
"""Create an ephemeral PostgreSQL database for this test run."""
db_url = _ephemeral_db_url()
admin_url = db_url.rsplit("/", 1)[0] + "/postgres"
from sqlalchemy.ext.asyncio import create_async_engine as _cae
admin_engine = _cae(admin_url, isolation_level="AUTOCOMMIT")
async with admin_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_url.rsplit("/", 1)[1]}"'))
await admin_engine.dispose()
# Plugin models use the pgvector Vector type — enable the extension in
# the fresh database before create_all runs (must connect to the target
# DB itself; CREATE EXTENSION has no ON DATABASE clause).
ext_engine = _cae(db_url, isolation_level="AUTOCOMMIT")
async with ext_engine.connect() as conn:
await conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
await ext_engine.dispose()
engine = create_async_engine(db_url, echo=False)
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
try:
async with async_session() as session:
yield session
finally:
await engine.dispose()
admin_engine2 = _cae(admin_url, isolation_level="AUTOCOMMIT")
async with admin_engine2.connect() as conn:
await conn.execute(text(f'DROP DATABASE IF EXISTS "{db_url.rsplit("/", 1)[1]}"'))
await admin_engine2.dispose()
@pytest_asyncio.fixture
async def tenant_id(db: AsyncSession) -> uuid.UUID:
"""Create a real tenant row — PostgreSQL enforces FKs, unlike SQLite."""
from app.models.tenant import Tenant
tid = uuid.uuid4()
db.add(Tenant(id=tid, name="Test Org", slug=f"test-{tid.hex[:8]}"))
await db.commit()
return tid
@pytest_asyncio.fixture
async def user_id(db: AsyncSession, tenant_id: uuid.UUID) -> uuid.UUID:
"""Create a real user row belonging to the test tenant."""
from app.models.user import User
uid = uuid.uuid4()
db.add(
User(
id=uid,
email=f"test-{uid.hex[:8]}@example.com",
name="Test User",
password_hash="not-a-real-hash",
is_active=True,
)
)
await db.commit()
return uid
# ─── AgentService Tests ───
class TestAgentService:
"""Tests for AgentService CRUD operations."""
@pytest.mark.asyncio
async def test_create_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test creating an agent definition."""
data = {
"name": "test-agent",
"description": "Test agent description",
"llm_model": "gpt-4",
"system_prompt": "You are a test agent.",
"tool_ids": [],
"heartbeat_interval_seconds": 0,
"mode": "reactive",
"is_active": True,
"max_executions_per_hour": 10,
"max_duration_seconds": 300,
"budget_limit_usd": 1.0,
}
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
assert agent is not None
assert agent.name == "test-agent"
assert agent.tenant_id == tenant_id
assert agent.created_by == user_id
assert agent.mode == "reactive"
assert agent.is_active is True
@pytest.mark.asyncio
async def test_get_agent_by_id(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test retrieving an agent by ID."""
data = {"name": "get-test", "description": "", "llm_model": "gpt-4", "system_prompt": "",
"tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive",
"is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300,
"budget_limit_usd": 1.0}
created = await AgentService.create(db, tenant_id, data, user_id=user_id)
fetched = await AgentService.get_by_id(db, tenant_id, created.id)
assert fetched is not None
assert fetched.id == created.id
assert fetched.name == "get-test"
@pytest.mark.asyncio
async def test_get_agent_by_name(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test retrieving an agent by name."""
data = {"name": "name-test", "description": "", "llm_model": "gpt-4", "system_prompt": "",
"tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive",
"is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300,
"budget_limit_usd": 1.0}
await AgentService.create(db, tenant_id, data, user_id=user_id)
fetched = await AgentService.get_by_name(db, "name-test", tenant_id=tenant_id)
assert fetched is not None
assert fetched.name == "name-test"
@pytest.mark.asyncio
async def test_update_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test updating an agent definition."""
data = {"name": "update-test", "description": "Original", "llm_model": "gpt-4",
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
created = await AgentService.create(db, tenant_id, data, user_id=user_id)
updated = await AgentService.update(db, tenant_id, created.id,
{"description": "Updated description"}, user_id=user_id)
assert updated is not None
assert updated.description == "Updated description"
@pytest.mark.asyncio
async def test_delete_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test deleting an agent definition."""
data = {"name": "delete-test", "description": "", "llm_model": "gpt-4", "system_prompt": "",
"tool_ids": [], "heartbeat_interval_seconds": 0, "mode": "reactive",
"is_active": True, "max_executions_per_hour": 10, "max_duration_seconds": 300,
"budget_limit_usd": 1.0}
created = await AgentService.create(db, tenant_id, data, user_id=user_id)
result = await AgentService.delete(db, tenant_id, created.id)
assert result is True
fetched = await AgentService.get_by_id(db, tenant_id, created.id)
assert fetched is None
@pytest.mark.asyncio
async def test_list_agents(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test listing agents with filters."""
for i in range(3):
data = {"name": f"list-test-{i}", "description": "", "llm_model": "gpt-4",
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
await AgentService.create(db, tenant_id, data, user_id=user_id)
items, total = await AgentService.list(db, tenant_id)
assert total == 3
assert len(items) == 3
# ─── AutomationService Tests ───
class TestAutomationService:
"""Tests for AutomationService CRUD operations."""
@pytest.mark.asyncio
async def test_create_automation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test creating an automation definition."""
data = {
"name": "test-automation",
"description": "Test automation",
"trigger_type": "manual",
"trigger_config": {},
"conditions": [],
"actions": [{"type": "api_call", "config": {"url": "https://example.com"}}],
"is_active": True,
"dry_run": False,
}
automation = await AutomationService.create(db, tenant_id, data, user_id=user_id)
assert automation is not None
assert automation.name == "test-automation"
assert automation.trigger_type == "manual"
assert len(automation.actions) == 1
@pytest.mark.asyncio
async def test_get_automation_by_id(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test retrieving an automation by ID."""
data = {"name": "get-auto", "description": "", "trigger_type": "event",
"trigger_config": {"event_name": "contact.created"}, "conditions": [],
"actions": [], "is_active": True, "dry_run": False}
created = await AutomationService.create(db, tenant_id, data, user_id=user_id)
fetched = await AutomationService.get_by_id(db, tenant_id, created.id)
assert fetched is not None
assert fetched.id == created.id
@pytest.mark.asyncio
async def test_update_automation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test updating an automation definition."""
data = {"name": "upd-auto", "description": "Original", "trigger_type": "manual",
"trigger_config": {}, "conditions": [], "actions": [], "is_active": True, "dry_run": False}
created = await AutomationService.create(db, tenant_id, data, user_id=user_id)
updated = await AutomationService.update(db, tenant_id, created.id,
{"description": "Updated"}, user_id=user_id)
assert updated is not None
assert updated.description == "Updated"
@pytest.mark.asyncio
async def test_delete_automation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test deleting an automation definition."""
data = {"name": "del-auto", "description": "", "trigger_type": "manual",
"trigger_config": {}, "conditions": [], "actions": [], "is_active": True, "dry_run": False}
created = await AutomationService.create(db, tenant_id, data, user_id=user_id)
result = await AutomationService.delete(db, tenant_id, created.id)
assert result is True
fetched = await AutomationService.get_by_id(db, tenant_id, created.id)
assert fetched is None
# ─── CronJobService Tests ───
class TestCronJobService:
"""Tests for CronJobService CRUD operations."""
@pytest.mark.asyncio
async def test_create_cron_job(self, db: AsyncSession, tenant_id: uuid.UUID):
"""Test creating a cron job."""
data = {
"name": "test-cron",
"cron_expression": "*/5 * * * *",
"job_type": "automation_trigger",
"plugin_name": "test",
"is_active": True,
}
job = await CronJobService.create(db, tenant_id, data)
assert job is not None
assert job.name == "test-cron"
assert job.cron_expression == "*/5 * * * *"
@pytest.mark.asyncio
async def test_get_cron_job_by_id(self, db: AsyncSession, tenant_id: uuid.UUID):
"""Test retrieving a cron job by ID."""
data = {"name": "get-cron", "cron_expression": "0 9 * * *", "job_type": "automation_trigger",
"plugin_name": "test", "is_active": True}
created = await CronJobService.create(db, tenant_id, data)
fetched = await CronJobService.get_by_id(db, tenant_id, created.id)
assert fetched is not None
assert fetched.id == created.id
@pytest.mark.asyncio
async def test_update_cron_job(self, db: AsyncSession, tenant_id: uuid.UUID):
"""Test updating a cron job."""
data = {"name": "upd-cron", "cron_expression": "0 9 * * *", "job_type": "automation_trigger",
"plugin_name": "test", "is_active": True}
created = await CronJobService.create(db, tenant_id, data)
updated = await CronJobService.update(db, tenant_id, created.id,
{"cron_expression": "0 10 * * *"})
assert updated is not None
assert updated.cron_expression == "0 10 * * *"
@pytest.mark.asyncio
async def test_delete_cron_job(self, db: AsyncSession, tenant_id: uuid.UUID):
"""Test deleting a cron job."""
data = {"name": "del-cron", "cron_expression": "0 9 * * *", "job_type": "automation_trigger",
"plugin_name": "test", "is_active": True}
created = await CronJobService.create(db, tenant_id, data)
result = await CronJobService.delete(db, tenant_id, created.id)
assert result is True
fetched = await CronJobService.get_by_id(db, tenant_id, created.id)
assert fetched is None
# ─── Version Management Tests ───
class TestVersionManagement:
"""Tests for version creation and restore."""
@pytest.mark.asyncio
async def test_version_created_on_agent_create(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test that a version snapshot is created when an agent is created."""
data = {"name": "ver-agent", "description": "Version test", "llm_model": "gpt-4",
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
versions, total = await AgentService.get_versions(db, tenant_id, agent.id)
assert total == 1
assert versions[0].version_number == 1
assert versions[0].snapshot["name"] == "ver-agent"
@pytest.mark.asyncio
async def test_version_created_on_agent_update(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test that a new version is created when an agent is updated."""
data = {"name": "ver-upd", "description": "Original", "llm_model": "gpt-4",
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
await AgentService.update(db, tenant_id, agent.id, {"description": "Updated"}, user_id=user_id)
versions, total = await AgentService.get_versions(db, tenant_id, agent.id)
assert total == 2
assert versions[0].version_number == 2
@pytest.mark.asyncio
async def test_version_restore_agent(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test restoring an agent from a version snapshot."""
data = {"name": "ver-restore", "description": "Original", "llm_model": "gpt-4",
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
await AgentService.update(db, tenant_id, agent.id, {"description": "Changed"}, user_id=user_id)
versions, _ = await AgentService.get_versions(db, tenant_id, agent.id)
# Restore to version 1
restored = await AgentService.restore_version(db, tenant_id, agent.id, versions[1].id, user_id=user_id)
assert restored is not None
assert restored.description == "Original"
@pytest.mark.asyncio
async def test_version_created_on_automation_update(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test that a new version is created when an automation is updated."""
data = {"name": "auto-ver", "description": "Original", "trigger_type": "manual",
"trigger_config": {}, "conditions": [], "actions": [], "is_active": True, "dry_run": False}
auto = await AutomationService.create(db, tenant_id, data, user_id=user_id)
await AutomationService.update(db, tenant_id, auto.id, {"description": "Updated"}, user_id=user_id)
versions, total = await AutomationService.get_versions(db, tenant_id, auto.id)
assert total == 2
# ─── Condition Evaluation Tests ───
class TestConditionEvaluation:
"""Tests for condition evaluation logic."""
def _evaluate_condition(self, condition: dict, context: dict) -> bool:
"""Evaluate a single condition against context data."""
field = condition.get("field", "")
operator = condition.get("operator", "equals")
value = condition.get("value", "")
actual = context.get(field)
if operator == "eq" or operator == "equals":
return str(actual) == str(value)
elif operator == "ne" or operator == "not_equals":
return str(actual) != str(value)
elif operator == "gt" or operator == "greater_than":
try:
return float(actual) > float(value)
except (ValueError, TypeError):
return False
elif operator == "lt" or operator == "less_than":
try:
return float(actual) < float(value)
except (ValueError, TypeError):
return False
elif operator == "contains":
return str(value) in str(actual)
elif operator == "exists":
return actual is not None
return False
def test_condition_eq(self):
assert self._evaluate_condition(
{"field": "status", "operator": "eq", "value": "active"},
{"status": "active"}
) is True
assert self._evaluate_condition(
{"field": "status", "operator": "eq", "value": "inactive"},
{"status": "active"}
) is False
def test_condition_ne(self):
assert self._evaluate_condition(
{"field": "status", "operator": "ne", "value": "inactive"},
{"status": "active"}
) is True
def test_condition_gt(self):
assert self._evaluate_condition(
{"field": "age", "operator": "gt", "value": "18"},
{"age": 25}
) is True
assert self._evaluate_condition(
{"field": "age", "operator": "gt", "value": "30"},
{"age": 25}
) is False
def test_condition_lt(self):
assert self._evaluate_condition(
{"field": "age", "operator": "lt", "value": "30"},
{"age": 25}
) is True
def test_condition_contains(self):
assert self._evaluate_condition(
{"field": "email", "operator": "contains", "value": "@example.com"},
{"email": "test@example.com"}
) is True
assert self._evaluate_condition(
{"field": "email", "operator": "contains", "value": "@other.com"},
{"email": "test@example.com"}
) is False
def test_condition_exists(self):
assert self._evaluate_condition(
{"field": "email", "operator": "exists", "value": ""},
{"email": "test@example.com"}
) is True
assert self._evaluate_condition(
{"field": "missing", "operator": "exists", "value": ""},
{"email": "test@example.com"}
) is False
# ─── Dry-Run Mode Tests ───
class TestDryRunMode:
"""Tests for dry-run mode (no actions executed)."""
@pytest.mark.asyncio
async def test_dry_run_automation_creation(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test creating an automation with dry_run=True."""
data = {"name": "dry-run-test", "description": "", "trigger_type": "manual",
"trigger_config": {}, "conditions": [], "actions": [{"type": "api_call", "config": {"url": "https://example.com"}}],
"is_active": True, "dry_run": True}
automation = await AutomationService.create(db, tenant_id, data, user_id=user_id)
assert automation.dry_run is True
@pytest.mark.asyncio
async def test_dry_run_flag_in_run(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test that dry_run flag is stored in AutomationRun."""
# PostgreSQL enforces the FK to automations — create a real one first
data = {"name": "dry-run-flag", "description": "", "trigger_type": "manual",
"trigger_config": {}, "conditions": [], "actions": [],
"is_active": True, "dry_run": True}
automation = await AutomationService.create(db, tenant_id, data, user_id=user_id)
run = AutomationRun(
tenant_id=tenant_id,
automation_id=automation.id,
status="dry_run",
started_at=datetime.now(UTC),
dry_run=True,
)
db.add(run)
await db.flush()
assert run.dry_run is True
# ─── Rate Limiting Tests ───
class TestRateLimiting:
"""Tests for rate limiting (max executions per hour)."""
@pytest.mark.asyncio
async def test_rate_limit_check(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test that rate limit is enforced."""
data = {"name": "rate-limit", "description": "", "llm_model": "gpt-4",
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
"mode": "reactive", "is_active": True, "max_executions_per_hour": 2,
"max_duration_seconds": 300, "budget_limit_usd": 1.0}
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
# Create 2 runs (within limit)
for _ in range(2):
run = AgentRun(
tenant_id=tenant_id,
agent_id=agent.id,
status="completed",
started_at=datetime.now(UTC),
)
db.add(run)
await db.flush()
# Check rate limit
from sqlalchemy import func, select
one_hour_ago = datetime.now(UTC) - timedelta(hours=1)
result = await db.execute(
select(func.count())
.select_from(AgentRun)
.where(AgentRun.agent_id == agent.id, AgentRun.started_at >= one_hour_ago)
)
recent_runs = result.scalar() or 0
assert recent_runs == 2
# With max_executions_per_hour=2 and 2 runs in the window, the limit
# is reached — the next execution must be blocked.
assert recent_runs >= agent.max_executions_per_hour
# ─── Budget Limit Tests ───
class TestBudgetLimit:
"""Tests for budget limit enforcement."""
@pytest.mark.asyncio
async def test_budget_limit_check(self, db: AsyncSession, tenant_id: uuid.UUID, user_id: uuid.UUID):
"""Test that budget limit is enforced."""
data = {"name": "budget-limit", "description": "", "llm_model": "gpt-4",
"system_prompt": "", "tool_ids": [], "heartbeat_interval_seconds": 0,
"mode": "reactive", "is_active": True, "max_executions_per_hour": 10,
"max_duration_seconds": 300, "budget_limit_usd": 0.5}
agent = await AgentService.create(db, tenant_id, data, user_id=user_id)
# Create runs with cumulative cost
for cost in [0.2, 0.2, 0.2]:
run = AgentRun(
tenant_id=tenant_id,
agent_id=agent.id,
status="completed",
started_at=datetime.now(UTC),
cost_usd=cost,
)
db.add(run)
await db.flush()
# Check budget
from sqlalchemy import func, select
cost_result = await db.execute(
select(func.coalesce(func.sum(AgentRun.cost_usd), 0.0))
.where(AgentRun.agent_id == agent.id)
)
total_cost = float(cost_result.scalar() or 0.0)
# FLOAT column accumulates binary rounding (0.6000000000000001)
assert total_cost == pytest.approx(0.6)
assert total_cost >= agent.budget_limit_usd # 0.6 >= 0.5, budget exceeded
# ─── Infinite Loop Detection Tests ───
class TestInfiniteLoopDetection:
"""Tests for infinite loop detection."""
def test_consecutive_tool_call_detection(self):
"""Test that 5+ consecutive same tool calls are detected."""
tool_call_count: dict[str, int] = {}
tool_name = "send_email"
for _i in range(5):
tool_call_count[tool_name] = tool_call_count.get(tool_name, 0) + 1
if tool_call_count[tool_name] >= 5:
assert True
return
raise AssertionError("Loop detection should have triggered")
def test_different_tools_not_detected(self):
"""Test that different tool calls don't trigger loop detection."""
tool_call_count: dict[str, int] = {}
for i in range(5):
tool_call_count[f"tool_{i}"] = tool_call_count.get(f"tool_{i}", 0) + 1
if tool_call_count[f"tool_{i}"] >= 5:
raise AssertionError("Different tools should not trigger loop detection")
assert True