977 lines
38 KiB
Python
977 lines
38 KiB
Python
|
"""Phase J Self-Improvement Plugin Tests — integration tests with real DB.
|
||
|
|
|
||
|
|
Covers the controlled self-improvement loop:
|
||
|
|
- Signal collection from AgentRun / ProactiveSuggestion / AuditLog
|
||
|
|
- Pattern detection from grouped signals
|
||
|
|
- Proposal creation (draft) and evaluation (LLM mocked)
|
||
|
|
- Activation + rollback of agent config
|
||
|
|
- Tenant isolation
|
||
|
|
- API routes under /api/v1/improvement/
|
||
|
|
|
||
|
|
Uses the real PostgreSQL test DB (conftest fixtures) and mocks only the
|
||
|
|
LLM (llm_complete) for deterministic evaluation.
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import uuid
|
||
|
|
from datetime import UTC, datetime, timedelta
|
||
|
|
from unittest.mock import AsyncMock, patch
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
import pytest_asyncio
|
||
|
|
from httpx import ASGITransport, AsyncClient
|
||
|
|
from sqlalchemy import select, update
|
||
|
|
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
||
|
|
|
||
|
|
from app.core.db import close_engine, reset_engine_for_testing
|
||
|
|
from app.core.permission_registry import init_permission_registry
|
||
|
|
from app.core.service_container import get_container
|
||
|
|
from app.main import create_app
|
||
|
|
from app.models.audit import AuditLog
|
||
|
|
from app.models.tenant import Tenant
|
||
|
|
from app.models.user import User
|
||
|
|
from app.models.workflow import WorkflowInstance
|
||
|
|
from app.plugins.builtins.ai_proactive.models import ProactiveSuggestion
|
||
|
|
from app.plugins.builtins.automation.models import AgentDefinition, AgentRun, AgentRunStep, AgentVersion
|
||
|
|
from app.plugins.builtins.self_improvement.models import (
|
||
|
|
ImpactMeasurement,
|
||
|
|
ImprovementPattern,
|
||
|
|
ImprovementProposal,
|
||
|
|
ImprovementSignal,
|
||
|
|
)
|
||
|
|
from app.plugins.builtins.self_improvement.services import (
|
||
|
|
activate_proposal,
|
||
|
|
collect_signals,
|
||
|
|
create_proposal,
|
||
|
|
detect_patterns,
|
||
|
|
evaluate_proposal,
|
||
|
|
measure_impact,
|
||
|
|
request_approval,
|
||
|
|
rollback_proposal,
|
||
|
|
)
|
||
|
|
from app.plugins.registry import reset_registry_for_testing
|
||
|
|
from app.services.plugin_service import reset_plugin_service_for_testing
|
||
|
|
from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users
|
||
|
|
|
||
|
|
pytestmark = pytest.mark.asyncio
|
||
|
|
|
||
|
|
|
||
|
|
@pytest_asyncio.fixture(scope="session", autouse=True)
|
||
|
|
async def _ensure_improvement_tables(engine: AsyncEngine):
|
||
|
|
"""Create the self_improvement tables if they don't exist yet.
|
||
|
|
|
||
|
|
conftest.db_setup only runs Base.metadata.create_all when the test DB is
|
||
|
|
empty. Since the DB already has tables, new plugin tables (self_improvement)
|
||
|
|
are never created. This fixture runs create_all (idempotent, checkfirst=True)
|
||
|
|
so the improvement_* tables exist before any test runs.
|
||
|
|
"""
|
||
|
|
from app.core.db import Base
|
||
|
|
|
||
|
|
async with engine.begin() as conn:
|
||
|
|
await conn.run_sync(Base.metadata.create_all)
|
||
|
|
yield
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# Shared seed fixture (service-level tests)
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
@pytest_asyncio.fixture
|
||
|
|
async def seed(db_session):
|
||
|
|
"""Create tenant + admin user for service-level tests."""
|
||
|
|
tenant = Tenant(name="Test Tenant", slug="test-tenant")
|
||
|
|
db_session.add(tenant)
|
||
|
|
await db_session.flush()
|
||
|
|
admin = User(
|
||
|
|
email="admin@test.local",
|
||
|
|
name="Admin",
|
||
|
|
password_hash="$2b$12$placeholder",
|
||
|
|
is_active=True,
|
||
|
|
is_system_admin=True,
|
||
|
|
preferences={},
|
||
|
|
)
|
||
|
|
db_session.add(admin)
|
||
|
|
await db_session.flush()
|
||
|
|
return {"tenant": tenant, "admin": admin}
|
||
|
|
|
||
|
|
|
||
|
|
@pytest_asyncio.fixture(autouse=True)
|
||
|
|
async def _init_perms():
|
||
|
|
"""Ensure permission registry is initialized for every test."""
|
||
|
|
init_permission_registry(
|
||
|
|
active_plugin_names={
|
||
|
|
"permissions",
|
||
|
|
"automation",
|
||
|
|
"ai_proactive",
|
||
|
|
"self_improvement",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
yield
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# API fixtures (self_improvement plugin active)
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
@pytest_asyncio.fixture
|
||
|
|
async def improvement_app(engine: AsyncEngine, redis_client):
|
||
|
|
"""FastAPI app with self_improvement + dependencies registered and active."""
|
||
|
|
reset_engine_for_testing(engine)
|
||
|
|
app = create_app()
|
||
|
|
registry = reset_registry_for_testing()
|
||
|
|
registry.initialize(engine, app)
|
||
|
|
init_permission_registry(
|
||
|
|
active_plugin_names={
|
||
|
|
"permissions",
|
||
|
|
"automation",
|
||
|
|
"ai_assistant",
|
||
|
|
"unified_search",
|
||
|
|
"kommunikation",
|
||
|
|
"dms",
|
||
|
|
"ai_proactive",
|
||
|
|
"self_improvement",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
container = get_container()
|
||
|
|
await container.initialize()
|
||
|
|
|
||
|
|
from app.plugins.builtins.permissions.plugin import PermissionsPlugin
|
||
|
|
from app.plugins.builtins.automation.plugin import AutomationPlugin
|
||
|
|
from app.plugins.builtins.ai_assistant.plugin import AIAssistantPlugin
|
||
|
|
from app.plugins.builtins.unified_search.plugin import UnifiedSearchPlugin
|
||
|
|
from app.plugins.builtins.kommunikation.plugin import KommunikationPlugin
|
||
|
|
from app.plugins.builtins.dms.plugin import DmsPlugin
|
||
|
|
from app.plugins.builtins.ai_proactive.plugin import AIProactivePlugin
|
||
|
|
from app.plugins.builtins.self_improvement.plugin import SelfImprovementPlugin
|
||
|
|
|
||
|
|
registry.register_plugin(PermissionsPlugin())
|
||
|
|
registry.register_plugin(AutomationPlugin())
|
||
|
|
registry.register_plugin(AIAssistantPlugin())
|
||
|
|
registry.register_plugin(UnifiedSearchPlugin())
|
||
|
|
registry.register_plugin(KommunikationPlugin())
|
||
|
|
registry.register_plugin(DmsPlugin())
|
||
|
|
registry.register_plugin(AIProactivePlugin())
|
||
|
|
registry.register_plugin(SelfImprovementPlugin())
|
||
|
|
reset_plugin_service_for_testing(registry)
|
||
|
|
|
||
|
|
sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
|
||
|
|
async with sf() as session:
|
||
|
|
await registry.install(session, "permissions")
|
||
|
|
await registry.activate(session, "permissions")
|
||
|
|
await registry.install(session, "dms")
|
||
|
|
await registry.activate(session, "dms")
|
||
|
|
await registry.install(session, "kommunikation")
|
||
|
|
await registry.activate(session, "kommunikation")
|
||
|
|
await registry.install(session, "automation")
|
||
|
|
await registry.activate(session, "automation")
|
||
|
|
await registry.install(session, "unified_search")
|
||
|
|
await registry.activate(session, "unified_search")
|
||
|
|
await registry.install(session, "ai_assistant")
|
||
|
|
await registry.activate(session, "ai_assistant")
|
||
|
|
await registry.install(session, "ai_proactive")
|
||
|
|
await registry.activate(session, "ai_proactive")
|
||
|
|
await registry.install(session, "self_improvement")
|
||
|
|
await registry.activate(session, "self_improvement")
|
||
|
|
await session.commit()
|
||
|
|
|
||
|
|
yield app
|
||
|
|
await close_engine()
|
||
|
|
|
||
|
|
|
||
|
|
@pytest_asyncio.fixture
|
||
|
|
async def improvement_client(improvement_app) -> AsyncClient:
|
||
|
|
"""HTTP test client with self_improvement plugin active."""
|
||
|
|
transport = ASGITransport(app=improvement_app)
|
||
|
|
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
||
|
|
yield c
|
||
|
|
|
||
|
|
|
||
|
|
@pytest_asyncio.fixture
|
||
|
|
async def improvement_authed_client(
|
||
|
|
improvement_client: AsyncClient, db_session: AsyncSession
|
||
|
|
) -> tuple[AsyncClient, dict]:
|
||
|
|
"""Authenticated admin client with seeded data and self_improvement active."""
|
||
|
|
seed = await seed_tenant_and_users(db_session)
|
||
|
|
# Grant is_system_admin so require_permission(automation:*) passes
|
||
|
|
await db_session.execute(
|
||
|
|
update(User).where(User.id == seed["admin_a"].id).values(is_system_admin=True)
|
||
|
|
)
|
||
|
|
await db_session.commit()
|
||
|
|
await login_client(improvement_client, "admin@tenanta.com")
|
||
|
|
return improvement_client, seed
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# Helpers
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
def _mock_llm_eval(
|
||
|
|
content: str = (
|
||
|
|
'{"score": 0.8, "assessment": "Sound proposal", '
|
||
|
|
'"risks_identified": ["minor"], "recommendation": "approve", '
|
||
|
|
'"test_scenarios": ["run once"]}'
|
||
|
|
)
|
||
|
|
):
|
||
|
|
"""Return an AsyncMock for llm_complete returning a JSON evaluation."""
|
||
|
|
return AsyncMock(return_value={"content": content, "cost_usd": 0.001})
|
||
|
|
|
||
|
|
|
||
|
|
async def _create_agent(db, tenant_id, user_id, **overrides):
|
||
|
|
"""Create a real AgentDefinition in the DB and return it."""
|
||
|
|
agent = AgentDefinition(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
name=overrides.get("name", "Test Agent"),
|
||
|
|
description=overrides.get("description", "A test agent"),
|
||
|
|
system_prompt=overrides.get("system_prompt", "You are a helpful assistant."),
|
||
|
|
tool_ids=overrides.get("tool_ids", []),
|
||
|
|
temperature=overrides.get("temperature", 0.3),
|
||
|
|
max_tokens=overrides.get("max_tokens", 1000),
|
||
|
|
max_steps=overrides.get("max_steps", 20),
|
||
|
|
created_by=user_id,
|
||
|
|
)
|
||
|
|
db.add(agent)
|
||
|
|
await db.flush()
|
||
|
|
return agent
|
||
|
|
|
||
|
|
|
||
|
|
async def _create_failed_run(db, tenant_id, agent_id, status="failed"):
|
||
|
|
"""Create a real AgentRun with a failure status."""
|
||
|
|
run = AgentRun(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
agent_id=agent_id,
|
||
|
|
status=status,
|
||
|
|
started_at=datetime.now(UTC),
|
||
|
|
duration_seconds=5.0,
|
||
|
|
error="boom",
|
||
|
|
)
|
||
|
|
db.add(run)
|
||
|
|
await db.flush()
|
||
|
|
return run
|
||
|
|
|
||
|
|
|
||
|
|
async def _create_dismissed_suggestion(db, tenant_id, user_id):
|
||
|
|
"""Create a real dismissed ProactiveSuggestion."""
|
||
|
|
sug = ProactiveSuggestion(
|
||
|
|
tenant_id=tenant_id,
|
||
|
|
user_id=user_id,
|
||
|
|
entity_type="contact",
|
||
|
|
suggestion_type="follow_up",
|
||
|
|
title="Follow up",
|
||
|
|
content="Consider following up",
|
||
|
|
confidence=0.7,
|
||
|
|
is_dismissed=True,
|
||
|
|
)
|
||
|
|
db.add(sug)
|
||
|
|
await db.flush()
|
||
|
|
return sug
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# J-SIGNAL: Signal Collection
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
class TestSignalCollection:
|
||
|
|
async def test_signal_collection_from_agent_run(self, db_session, seed):
|
||
|
|
"""collect_signals creates ImprovementSignals from failed AgentRuns."""
|
||
|
|
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||
|
|
run = await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
result = await collect_signals(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["collected"] >= 1
|
||
|
|
kinds = {s["signal_kind"] for s in result["signals"]}
|
||
|
|
assert "failure" in kinds
|
||
|
|
|
||
|
|
rows = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImprovementSignal).where(
|
||
|
|
ImprovementSignal.tenant_id == seed["tenant"].id,
|
||
|
|
ImprovementSignal.source_type == "agent_run",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).scalars().all()
|
||
|
|
assert len(rows) >= 1
|
||
|
|
assert rows[0].signal_kind == "failure"
|
||
|
|
assert rows[0].severity == "error"
|
||
|
|
assert rows[0].source_ref_id == run.id
|
||
|
|
|
||
|
|
async def test_signal_collection_from_proactive_suggestion(self, db_session, seed):
|
||
|
|
"""collect_signals creates signals from dismissed ProactiveSuggestions."""
|
||
|
|
await _create_dismissed_suggestion(db_session, seed["tenant"].id, seed["admin"].id)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
result = await collect_signals(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||
|
|
)
|
||
|
|
|
||
|
|
kinds = {s["signal_kind"] for s in result["signals"]}
|
||
|
|
assert "dismissal" in kinds
|
||
|
|
|
||
|
|
rows = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImprovementSignal).where(
|
||
|
|
ImprovementSignal.tenant_id == seed["tenant"].id,
|
||
|
|
ImprovementSignal.source_type == "proactive_suggestion",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).scalars().all()
|
||
|
|
assert len(rows) >= 1
|
||
|
|
assert rows[0].signal_kind == "dismissal"
|
||
|
|
|
||
|
|
async def test_signal_collection_from_audit_log(self, db_session, seed):
|
||
|
|
"""collect_signals creates correction signals from repeated audit updates."""
|
||
|
|
for _ in range(3):
|
||
|
|
db_session.add(
|
||
|
|
AuditLog(
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
action="update",
|
||
|
|
entity_type="contact",
|
||
|
|
entity_id=uuid.uuid4(),
|
||
|
|
changes={"name": "x"},
|
||
|
|
)
|
||
|
|
)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
result = await collect_signals(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||
|
|
)
|
||
|
|
|
||
|
|
kinds = {s["signal_kind"] for s in result["signals"]}
|
||
|
|
assert "correction" in kinds
|
||
|
|
|
||
|
|
async def test_signal_collection_workflow(self, db_session, seed):
|
||
|
|
"""collect_signals creates signals from failed WorkflowInstances."""
|
||
|
|
from app.models.workflow import Workflow
|
||
|
|
|
||
|
|
wf = Workflow(
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
name="Test WF",
|
||
|
|
description="",
|
||
|
|
steps=[],
|
||
|
|
is_active=True,
|
||
|
|
created_by=seed["admin"].id,
|
||
|
|
)
|
||
|
|
db_session.add(wf)
|
||
|
|
await db_session.flush()
|
||
|
|
db_session.add(
|
||
|
|
WorkflowInstance(
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
workflow_id=wf.id,
|
||
|
|
status="failed",
|
||
|
|
initiated_by=seed["admin"].id,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
result = await collect_signals(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||
|
|
)
|
||
|
|
|
||
|
|
kinds = {s["signal_kind"] for s in result["signals"]}
|
||
|
|
assert "failure" in kinds
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# J-PATTERN: Pattern Detection
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
class TestPatternDetection:
|
||
|
|
async def test_detect_patterns_groups_signals(self, db_session, seed):
|
||
|
|
"""detect_patterns groups signals and creates an ImprovementPattern."""
|
||
|
|
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||
|
|
for _ in range(2):
|
||
|
|
await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
await collect_signals(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||
|
|
)
|
||
|
|
result = await detect_patterns(db=db_session, tenant_id=seed["tenant"].id, min_occurrences=2)
|
||
|
|
|
||
|
|
assert result["patterns_created"] >= 1
|
||
|
|
pattern = result["patterns"][0]
|
||
|
|
assert pattern["pattern_kind"] == "retry_bottleneck"
|
||
|
|
assert pattern["target_type"] == "agent"
|
||
|
|
assert pattern["occurrence_count"] >= 2
|
||
|
|
|
||
|
|
rows = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImprovementPattern).where(
|
||
|
|
ImprovementPattern.tenant_id == seed["tenant"].id
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).scalars().all()
|
||
|
|
assert len(rows) >= 1
|
||
|
|
assert rows[0].status == "detected"
|
||
|
|
|
||
|
|
async def test_detect_patterns_links_signals(self, db_session, seed):
|
||
|
|
"""detect_patterns links signals to the created pattern."""
|
||
|
|
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||
|
|
for _ in range(2):
|
||
|
|
await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
await collect_signals(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||
|
|
)
|
||
|
|
await detect_patterns(db=db_session, tenant_id=seed["tenant"].id, min_occurrences=2)
|
||
|
|
|
||
|
|
rows = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImprovementSignal).where(
|
||
|
|
ImprovementSignal.tenant_id == seed["tenant"].id,
|
||
|
|
ImprovementSignal.pattern_id.is_not(None),
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).scalars().all()
|
||
|
|
assert len(rows) >= 2
|
||
|
|
|
||
|
|
async def test_detect_patterns_no_signals(self, db_session, seed):
|
||
|
|
"""detect_patterns returns empty when no signals exist."""
|
||
|
|
result = await detect_patterns(db=db_session, tenant_id=seed["tenant"].id)
|
||
|
|
assert result["patterns_created"] == 0
|
||
|
|
assert result["patterns"] == []
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# J-PROP: Proposal Creation
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
class TestProposalCreation:
|
||
|
|
async def test_create_proposal_draft(self, db_session, seed):
|
||
|
|
"""create_proposal creates an ImprovementProposal in draft status."""
|
||
|
|
proposal = await create_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
title="Improve agent prompt",
|
||
|
|
description="Tune the system prompt",
|
||
|
|
target_type="agent",
|
||
|
|
target_name="Test Agent",
|
||
|
|
proposed_config={"temperature": 0.1},
|
||
|
|
rationale="Reduce failures",
|
||
|
|
expected_benefit="Fewer retries",
|
||
|
|
risk_assessment="Low",
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert proposal.status == "draft"
|
||
|
|
assert proposal.tenant_id == seed["tenant"].id
|
||
|
|
assert proposal.owner_id == seed["admin"].id
|
||
|
|
assert proposal.proposed_config == {"temperature": 0.1}
|
||
|
|
|
||
|
|
row = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImprovementProposal).where(ImprovementProposal.id == proposal.id)
|
||
|
|
)
|
||
|
|
).scalar_one()
|
||
|
|
assert row.status == "draft"
|
||
|
|
|
||
|
|
async def test_create_proposal_captures_previous_config(self, db_session, seed):
|
||
|
|
"""create_proposal captures previous agent config for rollback."""
|
||
|
|
agent = await _create_agent(
|
||
|
|
db_session, seed["tenant"].id, seed["admin"].id, temperature=0.3
|
||
|
|
)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
proposal = await create_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
title="Tune temp",
|
||
|
|
description="Lower temperature",
|
||
|
|
target_type="agent",
|
||
|
|
target_ref_id=agent.id,
|
||
|
|
target_name=agent.name,
|
||
|
|
proposed_config={"temperature": 0.1},
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert proposal.previous_config.get("temperature") == 0.3
|
||
|
|
assert proposal.previous_config.get("system_prompt") == agent.system_prompt
|
||
|
|
|
||
|
|
async def test_create_proposal_links_pattern(self, db_session, seed):
|
||
|
|
"""create_proposal links to a pattern and updates its status."""
|
||
|
|
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||
|
|
for _ in range(2):
|
||
|
|
await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||
|
|
await db_session.flush()
|
||
|
|
await collect_signals(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||
|
|
)
|
||
|
|
pat_result = await detect_patterns(db=db_session, tenant_id=seed["tenant"].id, min_occurrences=2)
|
||
|
|
pattern_id = uuid.UUID(pat_result["patterns"][0]["id"])
|
||
|
|
|
||
|
|
proposal = await create_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
pattern_id=pattern_id,
|
||
|
|
title="Fix agent",
|
||
|
|
description="Fix the agent",
|
||
|
|
target_type="agent",
|
||
|
|
target_ref_id=agent.id,
|
||
|
|
target_name=agent.name,
|
||
|
|
proposed_config={"max_steps": 30},
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert proposal.pattern_id == pattern_id
|
||
|
|
assert len(proposal.evidence_refs) >= 1
|
||
|
|
pattern = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImprovementPattern).where(ImprovementPattern.id == pattern_id)
|
||
|
|
)
|
||
|
|
).scalar_one()
|
||
|
|
assert pattern.status == "proposal_created"
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# J-EVAL: Evaluation
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
class TestProposalEvaluation:
|
||
|
|
async def test_evaluate_proposal_sets_evaluated(self, db_session, seed):
|
||
|
|
"""evaluate_proposal sets status to evaluated with mocked LLM."""
|
||
|
|
proposal = await create_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
title="Evaluate me",
|
||
|
|
description="Assess this change",
|
||
|
|
target_type="agent",
|
||
|
|
target_name="Test Agent",
|
||
|
|
proposed_config={"temperature": 0.1},
|
||
|
|
rationale="Reduce failures",
|
||
|
|
expected_benefit="Fewer retries",
|
||
|
|
risk_assessment="Low",
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
with patch(
|
||
|
|
"app.plugins.builtins.self_improvement.services.llm_complete",
|
||
|
|
new_callable=AsyncMock,
|
||
|
|
return_value={
|
||
|
|
"content": json.dumps(
|
||
|
|
{
|
||
|
|
"score": 0.8,
|
||
|
|
"assessment": "Sound",
|
||
|
|
"risks_identified": [],
|
||
|
|
"recommendation": "approve",
|
||
|
|
"test_scenarios": ["run"],
|
||
|
|
}
|
||
|
|
),
|
||
|
|
"cost_usd": 0.001,
|
||
|
|
},
|
||
|
|
):
|
||
|
|
result = await evaluate_proposal(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, proposal_id=proposal.id
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["status"] == "evaluated"
|
||
|
|
assert result["evaluation"]["score"] == 0.8
|
||
|
|
assert result["evaluation"]["recommendation"] == "approve"
|
||
|
|
|
||
|
|
await db_session.refresh(proposal)
|
||
|
|
assert proposal.status == "evaluated"
|
||
|
|
assert proposal.evaluated_at is not None
|
||
|
|
assert proposal.evaluation_result["score"] == 0.8
|
||
|
|
|
||
|
|
async def test_evaluate_proposal_nonexistent(self, db_session, seed):
|
||
|
|
"""evaluate_proposal returns error for nonexistent proposal."""
|
||
|
|
result = await evaluate_proposal(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, proposal_id=uuid.uuid4()
|
||
|
|
)
|
||
|
|
assert "error" in result
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# J-ACTIVATE: Activation + Rollback
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
class TestActivationAndRollback:
|
||
|
|
async def test_activate_proposal_applies_config(self, db_session, seed):
|
||
|
|
"""activate_proposal applies proposed config to the agent."""
|
||
|
|
agent = await _create_agent(
|
||
|
|
db_session, seed["tenant"].id, seed["admin"].id, temperature=0.3, max_steps=20
|
||
|
|
)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
proposal = await create_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
title="Tune agent",
|
||
|
|
description="Lower temperature",
|
||
|
|
target_type="agent",
|
||
|
|
target_ref_id=agent.id,
|
||
|
|
target_name=agent.name,
|
||
|
|
proposed_config={"temperature": 0.1, "max_steps": 30},
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
)
|
||
|
|
proposal.status = "approved"
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
result = await activate_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
proposal_id=proposal.id,
|
||
|
|
approved_by=seed["admin"].id,
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["status"] == "active"
|
||
|
|
assert result["applied"] is True
|
||
|
|
|
||
|
|
await db_session.refresh(agent)
|
||
|
|
assert agent.temperature == 0.1
|
||
|
|
assert agent.max_steps == 30
|
||
|
|
|
||
|
|
# A version snapshot should have been created
|
||
|
|
versions = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(AgentVersion).where(
|
||
|
|
AgentVersion.tenant_id == seed["tenant"].id,
|
||
|
|
AgentVersion.agent_id == agent.id,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).scalars().all()
|
||
|
|
assert len(versions) >= 1
|
||
|
|
assert versions[0].snapshot["temperature"] == 0.3
|
||
|
|
|
||
|
|
async def test_rollback_proposal_restores_config(self, db_session, seed):
|
||
|
|
"""rollback_proposal restores the previous agent config."""
|
||
|
|
agent = await _create_agent(
|
||
|
|
db_session, seed["tenant"].id, seed["admin"].id, temperature=0.3, max_steps=20
|
||
|
|
)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
proposal = await create_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
title="Tune agent",
|
||
|
|
description="Lower temperature",
|
||
|
|
target_type="agent",
|
||
|
|
target_ref_id=agent.id,
|
||
|
|
target_name=agent.name,
|
||
|
|
proposed_config={"temperature": 0.1},
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
)
|
||
|
|
proposal.status = "approved"
|
||
|
|
await db_session.flush()
|
||
|
|
await activate_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
proposal_id=proposal.id,
|
||
|
|
approved_by=seed["admin"].id,
|
||
|
|
)
|
||
|
|
await db_session.refresh(agent)
|
||
|
|
assert agent.temperature == 0.1
|
||
|
|
|
||
|
|
result = await rollback_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
proposal_id=proposal.id,
|
||
|
|
reason="Regression",
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["status"] == "rolled_back"
|
||
|
|
assert result["restored"] is True
|
||
|
|
|
||
|
|
await db_session.refresh(agent)
|
||
|
|
assert agent.temperature == 0.3
|
||
|
|
await db_session.refresh(proposal)
|
||
|
|
assert proposal.rollback_reason == "Regression"
|
||
|
|
|
||
|
|
async def test_rollback_requires_active(self, db_session, seed):
|
||
|
|
"""rollback_proposal rejects a non-active proposal."""
|
||
|
|
proposal = await create_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
title="Draft",
|
||
|
|
description="Not active",
|
||
|
|
target_type="agent",
|
||
|
|
target_name="x",
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
result = await rollback_proposal(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, proposal_id=proposal.id
|
||
|
|
)
|
||
|
|
assert "error" in result
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# J-MEASURE: Impact Measurement
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
class TestImpactMeasurement:
|
||
|
|
async def test_measure_impact_creates_measurement(self, db_session, seed):
|
||
|
|
"""measure_impact creates an ImpactMeasurement for an active proposal."""
|
||
|
|
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
proposal = await create_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
title="Measure",
|
||
|
|
description="Measure impact",
|
||
|
|
target_type="agent",
|
||
|
|
target_ref_id=agent.id,
|
||
|
|
target_name=agent.name,
|
||
|
|
proposed_config={"temperature": 0.1},
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
)
|
||
|
|
proposal.status = "approved"
|
||
|
|
await db_session.flush()
|
||
|
|
await activate_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
proposal_id=proposal.id,
|
||
|
|
approved_by=seed["admin"].id,
|
||
|
|
)
|
||
|
|
|
||
|
|
result = await measure_impact(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, proposal_id=proposal.id
|
||
|
|
)
|
||
|
|
|
||
|
|
assert "measurement_id" in result
|
||
|
|
assert "pre_metrics" in result
|
||
|
|
assert "post_metrics" in result
|
||
|
|
assert "delta" in result
|
||
|
|
|
||
|
|
rows = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImpactMeasurement).where(
|
||
|
|
ImpactMeasurement.tenant_id == seed["tenant"].id,
|
||
|
|
ImpactMeasurement.proposal_id == proposal.id,
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).scalars().all()
|
||
|
|
assert len(rows) == 1
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# J-ISOLATION: Tenant Isolation
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
class TestTenantIsolation:
|
||
|
|
async def test_signals_are_tenant_isolated(self, db_session, seed):
|
||
|
|
"""Signals created for one tenant are not visible to another."""
|
||
|
|
tenant_b = Tenant(name="Tenant B", slug="tenant-b")
|
||
|
|
db_session.add(tenant_b)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||
|
|
await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||
|
|
await db_session.flush()
|
||
|
|
await collect_signals(
|
||
|
|
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||
|
|
)
|
||
|
|
|
||
|
|
# Tenant B sees no signals
|
||
|
|
result_b = await collect_signals(
|
||
|
|
db=db_session, tenant_id=tenant_b.id, since=datetime.now(UTC) - timedelta(days=1)
|
||
|
|
)
|
||
|
|
assert result_b["collected"] == 0
|
||
|
|
|
||
|
|
rows_b = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImprovementSignal).where(ImprovementSignal.tenant_id == tenant_b.id)
|
||
|
|
)
|
||
|
|
).scalars().all()
|
||
|
|
assert len(rows_b) == 0
|
||
|
|
|
||
|
|
async def test_proposals_are_tenant_isolated(self, db_session, seed):
|
||
|
|
"""Proposals created for one tenant are not visible to another."""
|
||
|
|
tenant_b = Tenant(name="Tenant B", slug="tenant-b")
|
||
|
|
db_session.add(tenant_b)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
await create_proposal(
|
||
|
|
db=db_session,
|
||
|
|
tenant_id=seed["tenant"].id,
|
||
|
|
title="Tenant A proposal",
|
||
|
|
description="Only for A",
|
||
|
|
target_type="agent",
|
||
|
|
target_name="x",
|
||
|
|
user_id=seed["admin"].id,
|
||
|
|
)
|
||
|
|
await db_session.flush()
|
||
|
|
|
||
|
|
rows_b = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImprovementProposal).where(ImprovementProposal.tenant_id == tenant_b.id)
|
||
|
|
)
|
||
|
|
).scalars().all()
|
||
|
|
assert len(rows_b) == 0
|
||
|
|
|
||
|
|
rows_a = (
|
||
|
|
await db_session.execute(
|
||
|
|
select(ImprovementProposal).where(
|
||
|
|
ImprovementProposal.tenant_id == seed["tenant"].id
|
||
|
|
)
|
||
|
|
)
|
||
|
|
).scalars().all()
|
||
|
|
assert len(rows_a) == 1
|
||
|
|
|
||
|
|
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
# J-API: API Routes
|
||
|
|
# ──────────────────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
|
||
|
|
class TestApiRoutes:
|
||
|
|
async def test_api_signal_collect(self, improvement_authed_client):
|
||
|
|
"""POST /api/v1/improvement/signals/collect works."""
|
||
|
|
client, seed = improvement_authed_client
|
||
|
|
resp = await client.post(
|
||
|
|
"/api/v1/improvement/signals/collect",
|
||
|
|
json={"limit": 10},
|
||
|
|
headers=ORIGIN_HEADER,
|
||
|
|
)
|
||
|
|
assert resp.status_code == 200, resp.text
|
||
|
|
data = resp.json()
|
||
|
|
assert "collected" in data
|
||
|
|
assert "signals" in data
|
||
|
|
|
||
|
|
async def test_api_proposal_create(self, improvement_authed_client):
|
||
|
|
"""POST /api/v1/improvement/proposals works."""
|
||
|
|
client, seed = improvement_authed_client
|
||
|
|
resp = await client.post(
|
||
|
|
"/api/v1/improvement/proposals",
|
||
|
|
json={
|
||
|
|
"title": "API proposal",
|
||
|
|
"description": "Created via API",
|
||
|
|
"target_type": "agent",
|
||
|
|
"target_name": "Test Agent",
|
||
|
|
"proposed_config": {"temperature": 0.1},
|
||
|
|
},
|
||
|
|
headers=ORIGIN_HEADER,
|
||
|
|
)
|
||
|
|
assert resp.status_code == 200, resp.text
|
||
|
|
data = resp.json()
|
||
|
|
assert data["status"] == "draft"
|
||
|
|
assert data["title"] == "API proposal"
|
||
|
|
assert "id" in data
|
||
|
|
|
||
|
|
async def test_api_proposal_create_missing_fields(self, improvement_authed_client):
|
||
|
|
"""POST /api/v1/improvement/proposals rejects missing title/target_type."""
|
||
|
|
client, _ = improvement_authed_client
|
||
|
|
resp = await client.post(
|
||
|
|
"/api/v1/improvement/proposals",
|
||
|
|
json={"description": "no title"},
|
||
|
|
headers=ORIGIN_HEADER,
|
||
|
|
)
|
||
|
|
assert resp.status_code == 400, resp.text
|
||
|
|
|
||
|
|
async def test_api_proposal_list(self, improvement_authed_client):
|
||
|
|
"""GET /api/v1/improvement/proposals lists proposals."""
|
||
|
|
client, _ = improvement_authed_client
|
||
|
|
resp = await client.get("/api/v1/improvement/proposals", headers=ORIGIN_HEADER)
|
||
|
|
assert resp.status_code == 200, resp.text
|
||
|
|
data = resp.json()
|
||
|
|
assert "items" in data
|
||
|
|
assert "total" in data
|
||
|
|
|
||
|
|
async def test_api_proposal_evaluate(self, improvement_authed_client):
|
||
|
|
"""POST /api/v1/improvement/proposals/{id}/evaluate works with mocked LLM."""
|
||
|
|
client, seed = improvement_authed_client
|
||
|
|
create_resp = await client.post(
|
||
|
|
"/api/v1/improvement/proposals",
|
||
|
|
json={
|
||
|
|
"title": "Evaluate via API",
|
||
|
|
"description": "Assess",
|
||
|
|
"target_type": "agent",
|
||
|
|
"target_name": "Test Agent",
|
||
|
|
"proposed_config": {"temperature": 0.1},
|
||
|
|
},
|
||
|
|
headers=ORIGIN_HEADER,
|
||
|
|
)
|
||
|
|
assert create_resp.status_code == 200, create_resp.text
|
||
|
|
proposal_id = create_resp.json()["id"]
|
||
|
|
|
||
|
|
with patch(
|
||
|
|
"app.plugins.builtins.self_improvement.services.llm_complete",
|
||
|
|
new_callable=AsyncMock,
|
||
|
|
return_value={
|
||
|
|
"content": json.dumps(
|
||
|
|
{
|
||
|
|
"score": 0.7,
|
||
|
|
"assessment": "OK",
|
||
|
|
"risks_identified": [],
|
||
|
|
"recommendation": "approve",
|
||
|
|
"test_scenarios": [],
|
||
|
|
}
|
||
|
|
),
|
||
|
|
"cost_usd": 0.001,
|
||
|
|
},
|
||
|
|
):
|
||
|
|
resp = await client.post(
|
||
|
|
f"/api/v1/improvement/proposals/{proposal_id}/evaluate",
|
||
|
|
headers=ORIGIN_HEADER,
|
||
|
|
)
|
||
|
|
assert resp.status_code == 200, resp.text
|
||
|
|
assert resp.json()["status"] == "evaluated"
|
||
|
|
|
||
|
|
async def test_api_proposal_activate_and_rollback(self, improvement_authed_client, db_session):
|
||
|
|
"""POST activate + rollback via API works end-to-end."""
|
||
|
|
client, seed = improvement_authed_client
|
||
|
|
agent = await _create_agent(db_session, seed["tenant_a"].id, seed["admin_a"].id)
|
||
|
|
await db_session.commit()
|
||
|
|
|
||
|
|
create_resp = await client.post(
|
||
|
|
"/api/v1/improvement/proposals",
|
||
|
|
json={
|
||
|
|
"title": "Activate via API",
|
||
|
|
"description": "Apply config",
|
||
|
|
"target_type": "agent",
|
||
|
|
"target_ref_id": str(agent.id),
|
||
|
|
"target_name": agent.name,
|
||
|
|
"proposed_config": {"temperature": 0.1},
|
||
|
|
},
|
||
|
|
headers=ORIGIN_HEADER,
|
||
|
|
)
|
||
|
|
assert create_resp.status_code == 200, create_resp.text
|
||
|
|
proposal_id = create_resp.json()["id"]
|
||
|
|
|
||
|
|
# Set to approved directly (approval flow is covered by service tests)
|
||
|
|
await db_session.execute(
|
||
|
|
update(ImprovementProposal)
|
||
|
|
.where(ImprovementProposal.id == uuid.UUID(proposal_id))
|
||
|
|
.values(status="approved")
|
||
|
|
)
|
||
|
|
await db_session.commit()
|
||
|
|
|
||
|
|
act_resp = await client.post(
|
||
|
|
f"/api/v1/improvement/proposals/{proposal_id}/activate",
|
||
|
|
headers=ORIGIN_HEADER,
|
||
|
|
)
|
||
|
|
assert act_resp.status_code == 200, act_resp.text
|
||
|
|
assert act_resp.json()["status"] == "active"
|
||
|
|
|
||
|
|
await db_session.refresh(agent)
|
||
|
|
assert agent.temperature == 0.1
|
||
|
|
|
||
|
|
roll_resp = await client.post(
|
||
|
|
f"/api/v1/improvement/proposals/{proposal_id}/rollback",
|
||
|
|
json={"reason": "test"},
|
||
|
|
headers=ORIGIN_HEADER,
|
||
|
|
)
|
||
|
|
assert roll_resp.status_code == 200, roll_resp.text
|
||
|
|
assert roll_resp.json()["status"] == "rolled_back"
|
||
|
|
|
||
|
|
await db_session.refresh(agent)
|
||
|
|
assert agent.temperature == 0.3
|