feat(J): Phase J Self-Improvement Plugin — controlled improvement loop
Check Cross-Plugin Imports / check (push) Has been cancelled
Check Cross-Plugin Imports / check (push) Has been cancelled
- New self_improvement plugin: models, services, routes, plugin - 4 SQLAlchemy models: ImprovementSignal, ImprovementPattern, ImprovementProposal, ImpactMeasurement - Migration 0132: 4 tables with RLS - Services: collect_signals, detect_patterns, create_proposal, evaluate_proposal, request_approval, activate_proposal, rollback_proposal, measure_impact - 11 API routes under /api/v1/improvement/ - Frontend: improvement.ts API client, ImprovementPanel.tsx in AISidebar proactive tab - 24/24 integration tests pass - Fix: __init__.py imports Plugin class for discover_builtins() (self_improvement + knowledge) - Fix: automation plugin register_plugin_contributions skips when no tenant exists
This commit is contained in:
@@ -82,6 +82,13 @@ from app.models.plugin_allowlist import PluginAllowlist # noqa: F401
|
||||
from app.plugins.builtins.wiki.models import WikiArticle, WikiArticleVersion, WikiCategory # noqa: F401
|
||||
# Knowledge plugin models — new plugin, ensure table is created in test-DB
|
||||
from app.plugins.builtins.knowledge.models import KnowledgeExtraction # noqa: F401
|
||||
# Self-improvement plugin models — new plugin, ensure tables are created in test-DB
|
||||
from app.plugins.builtins.self_improvement.models import ( # noqa: F401
|
||||
ImprovementSignal,
|
||||
ImprovementPattern,
|
||||
ImprovementProposal,
|
||||
ImpactMeasurement,
|
||||
)
|
||||
|
||||
from app.plugins.registry import reset_registry_for_testing # noqa: F401
|
||||
from app.core.permission_registry import init_permission_registry # noqa: F401
|
||||
|
||||
@@ -0,0 +1,976 @@
|
||||
"""Phase J Self-Improvement Plugin Tests — integration tests with real DB.
|
||||
|
||||
Covers the controlled self-improvement loop:
|
||||
- Signal collection from AgentRun / ProactiveSuggestion / AuditLog
|
||||
- Pattern detection from grouped signals
|
||||
- Proposal creation (draft) and evaluation (LLM mocked)
|
||||
- Activation + rollback of agent config
|
||||
- Tenant isolation
|
||||
- API routes under /api/v1/improvement/
|
||||
|
||||
Uses the real PostgreSQL test DB (conftest fixtures) and mocks only the
|
||||
LLM (llm_complete) for deterministic evaluation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
|
||||
|
||||
from app.core.db import close_engine, reset_engine_for_testing
|
||||
from app.core.permission_registry import init_permission_registry
|
||||
from app.core.service_container import get_container
|
||||
from app.main import create_app
|
||||
from app.models.audit import AuditLog
|
||||
from app.models.tenant import Tenant
|
||||
from app.models.user import User
|
||||
from app.models.workflow import WorkflowInstance
|
||||
from app.plugins.builtins.ai_proactive.models import ProactiveSuggestion
|
||||
from app.plugins.builtins.automation.models import AgentDefinition, AgentRun, AgentRunStep, AgentVersion
|
||||
from app.plugins.builtins.self_improvement.models import (
|
||||
ImpactMeasurement,
|
||||
ImprovementPattern,
|
||||
ImprovementProposal,
|
||||
ImprovementSignal,
|
||||
)
|
||||
from app.plugins.builtins.self_improvement.services import (
|
||||
activate_proposal,
|
||||
collect_signals,
|
||||
create_proposal,
|
||||
detect_patterns,
|
||||
evaluate_proposal,
|
||||
measure_impact,
|
||||
request_approval,
|
||||
rollback_proposal,
|
||||
)
|
||||
from app.plugins.registry import reset_registry_for_testing
|
||||
from app.services.plugin_service import reset_plugin_service_for_testing
|
||||
from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope="session", autouse=True)
|
||||
async def _ensure_improvement_tables(engine: AsyncEngine):
|
||||
"""Create the self_improvement tables if they don't exist yet.
|
||||
|
||||
conftest.db_setup only runs Base.metadata.create_all when the test DB is
|
||||
empty. Since the DB already has tables, new plugin tables (self_improvement)
|
||||
are never created. This fixture runs create_all (idempotent, checkfirst=True)
|
||||
so the improvement_* tables exist before any test runs.
|
||||
"""
|
||||
from app.core.db import Base
|
||||
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
yield
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Shared seed fixture (service-level tests)
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def seed(db_session):
|
||||
"""Create tenant + admin user for service-level tests."""
|
||||
tenant = Tenant(name="Test Tenant", slug="test-tenant")
|
||||
db_session.add(tenant)
|
||||
await db_session.flush()
|
||||
admin = User(
|
||||
email="admin@test.local",
|
||||
name="Admin",
|
||||
password_hash="$2b$12$placeholder",
|
||||
is_active=True,
|
||||
is_system_admin=True,
|
||||
preferences={},
|
||||
)
|
||||
db_session.add(admin)
|
||||
await db_session.flush()
|
||||
return {"tenant": tenant, "admin": admin}
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
async def _init_perms():
|
||||
"""Ensure permission registry is initialized for every test."""
|
||||
init_permission_registry(
|
||||
active_plugin_names={
|
||||
"permissions",
|
||||
"automation",
|
||||
"ai_proactive",
|
||||
"self_improvement",
|
||||
}
|
||||
)
|
||||
yield
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# API fixtures (self_improvement plugin active)
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def improvement_app(engine: AsyncEngine, redis_client):
|
||||
"""FastAPI app with self_improvement + dependencies registered and active."""
|
||||
reset_engine_for_testing(engine)
|
||||
app = create_app()
|
||||
registry = reset_registry_for_testing()
|
||||
registry.initialize(engine, app)
|
||||
init_permission_registry(
|
||||
active_plugin_names={
|
||||
"permissions",
|
||||
"automation",
|
||||
"ai_assistant",
|
||||
"unified_search",
|
||||
"kommunikation",
|
||||
"dms",
|
||||
"ai_proactive",
|
||||
"self_improvement",
|
||||
}
|
||||
)
|
||||
container = get_container()
|
||||
await container.initialize()
|
||||
|
||||
from app.plugins.builtins.permissions.plugin import PermissionsPlugin
|
||||
from app.plugins.builtins.automation.plugin import AutomationPlugin
|
||||
from app.plugins.builtins.ai_assistant.plugin import AIAssistantPlugin
|
||||
from app.plugins.builtins.unified_search.plugin import UnifiedSearchPlugin
|
||||
from app.plugins.builtins.kommunikation.plugin import KommunikationPlugin
|
||||
from app.plugins.builtins.dms.plugin import DmsPlugin
|
||||
from app.plugins.builtins.ai_proactive.plugin import AIProactivePlugin
|
||||
from app.plugins.builtins.self_improvement.plugin import SelfImprovementPlugin
|
||||
|
||||
registry.register_plugin(PermissionsPlugin())
|
||||
registry.register_plugin(AutomationPlugin())
|
||||
registry.register_plugin(AIAssistantPlugin())
|
||||
registry.register_plugin(UnifiedSearchPlugin())
|
||||
registry.register_plugin(KommunikationPlugin())
|
||||
registry.register_plugin(DmsPlugin())
|
||||
registry.register_plugin(AIProactivePlugin())
|
||||
registry.register_plugin(SelfImprovementPlugin())
|
||||
reset_plugin_service_for_testing(registry)
|
||||
|
||||
sf = async_sessionmaker(bind=engine, expire_on_commit=False, class_=AsyncSession)
|
||||
async with sf() as session:
|
||||
await registry.install(session, "permissions")
|
||||
await registry.activate(session, "permissions")
|
||||
await registry.install(session, "dms")
|
||||
await registry.activate(session, "dms")
|
||||
await registry.install(session, "kommunikation")
|
||||
await registry.activate(session, "kommunikation")
|
||||
await registry.install(session, "automation")
|
||||
await registry.activate(session, "automation")
|
||||
await registry.install(session, "unified_search")
|
||||
await registry.activate(session, "unified_search")
|
||||
await registry.install(session, "ai_assistant")
|
||||
await registry.activate(session, "ai_assistant")
|
||||
await registry.install(session, "ai_proactive")
|
||||
await registry.activate(session, "ai_proactive")
|
||||
await registry.install(session, "self_improvement")
|
||||
await registry.activate(session, "self_improvement")
|
||||
await session.commit()
|
||||
|
||||
yield app
|
||||
await close_engine()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def improvement_client(improvement_app) -> AsyncClient:
|
||||
"""HTTP test client with self_improvement plugin active."""
|
||||
transport = ASGITransport(app=improvement_app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def improvement_authed_client(
|
||||
improvement_client: AsyncClient, db_session: AsyncSession
|
||||
) -> tuple[AsyncClient, dict]:
|
||||
"""Authenticated admin client with seeded data and self_improvement active."""
|
||||
seed = await seed_tenant_and_users(db_session)
|
||||
# Grant is_system_admin so require_permission(automation:*) passes
|
||||
await db_session.execute(
|
||||
update(User).where(User.id == seed["admin_a"].id).values(is_system_admin=True)
|
||||
)
|
||||
await db_session.commit()
|
||||
await login_client(improvement_client, "admin@tenanta.com")
|
||||
return improvement_client, seed
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# Helpers
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _mock_llm_eval(
|
||||
content: str = (
|
||||
'{"score": 0.8, "assessment": "Sound proposal", '
|
||||
'"risks_identified": ["minor"], "recommendation": "approve", '
|
||||
'"test_scenarios": ["run once"]}'
|
||||
)
|
||||
):
|
||||
"""Return an AsyncMock for llm_complete returning a JSON evaluation."""
|
||||
return AsyncMock(return_value={"content": content, "cost_usd": 0.001})
|
||||
|
||||
|
||||
async def _create_agent(db, tenant_id, user_id, **overrides):
|
||||
"""Create a real AgentDefinition in the DB and return it."""
|
||||
agent = AgentDefinition(
|
||||
tenant_id=tenant_id,
|
||||
name=overrides.get("name", "Test Agent"),
|
||||
description=overrides.get("description", "A test agent"),
|
||||
system_prompt=overrides.get("system_prompt", "You are a helpful assistant."),
|
||||
tool_ids=overrides.get("tool_ids", []),
|
||||
temperature=overrides.get("temperature", 0.3),
|
||||
max_tokens=overrides.get("max_tokens", 1000),
|
||||
max_steps=overrides.get("max_steps", 20),
|
||||
created_by=user_id,
|
||||
)
|
||||
db.add(agent)
|
||||
await db.flush()
|
||||
return agent
|
||||
|
||||
|
||||
async def _create_failed_run(db, tenant_id, agent_id, status="failed"):
|
||||
"""Create a real AgentRun with a failure status."""
|
||||
run = AgentRun(
|
||||
tenant_id=tenant_id,
|
||||
agent_id=agent_id,
|
||||
status=status,
|
||||
started_at=datetime.now(UTC),
|
||||
duration_seconds=5.0,
|
||||
error="boom",
|
||||
)
|
||||
db.add(run)
|
||||
await db.flush()
|
||||
return run
|
||||
|
||||
|
||||
async def _create_dismissed_suggestion(db, tenant_id, user_id):
|
||||
"""Create a real dismissed ProactiveSuggestion."""
|
||||
sug = ProactiveSuggestion(
|
||||
tenant_id=tenant_id,
|
||||
user_id=user_id,
|
||||
entity_type="contact",
|
||||
suggestion_type="follow_up",
|
||||
title="Follow up",
|
||||
content="Consider following up",
|
||||
confidence=0.7,
|
||||
is_dismissed=True,
|
||||
)
|
||||
db.add(sug)
|
||||
await db.flush()
|
||||
return sug
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# J-SIGNAL: Signal Collection
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestSignalCollection:
|
||||
async def test_signal_collection_from_agent_run(self, db_session, seed):
|
||||
"""collect_signals creates ImprovementSignals from failed AgentRuns."""
|
||||
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||||
run = await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||||
await db_session.flush()
|
||||
|
||||
result = await collect_signals(
|
||||
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||||
)
|
||||
|
||||
assert result["collected"] >= 1
|
||||
kinds = {s["signal_kind"] for s in result["signals"]}
|
||||
assert "failure" in kinds
|
||||
|
||||
rows = (
|
||||
await db_session.execute(
|
||||
select(ImprovementSignal).where(
|
||||
ImprovementSignal.tenant_id == seed["tenant"].id,
|
||||
ImprovementSignal.source_type == "agent_run",
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows) >= 1
|
||||
assert rows[0].signal_kind == "failure"
|
||||
assert rows[0].severity == "error"
|
||||
assert rows[0].source_ref_id == run.id
|
||||
|
||||
async def test_signal_collection_from_proactive_suggestion(self, db_session, seed):
|
||||
"""collect_signals creates signals from dismissed ProactiveSuggestions."""
|
||||
await _create_dismissed_suggestion(db_session, seed["tenant"].id, seed["admin"].id)
|
||||
await db_session.flush()
|
||||
|
||||
result = await collect_signals(
|
||||
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||||
)
|
||||
|
||||
kinds = {s["signal_kind"] for s in result["signals"]}
|
||||
assert "dismissal" in kinds
|
||||
|
||||
rows = (
|
||||
await db_session.execute(
|
||||
select(ImprovementSignal).where(
|
||||
ImprovementSignal.tenant_id == seed["tenant"].id,
|
||||
ImprovementSignal.source_type == "proactive_suggestion",
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows) >= 1
|
||||
assert rows[0].signal_kind == "dismissal"
|
||||
|
||||
async def test_signal_collection_from_audit_log(self, db_session, seed):
|
||||
"""collect_signals creates correction signals from repeated audit updates."""
|
||||
for _ in range(3):
|
||||
db_session.add(
|
||||
AuditLog(
|
||||
tenant_id=seed["tenant"].id,
|
||||
user_id=seed["admin"].id,
|
||||
action="update",
|
||||
entity_type="contact",
|
||||
entity_id=uuid.uuid4(),
|
||||
changes={"name": "x"},
|
||||
)
|
||||
)
|
||||
await db_session.flush()
|
||||
|
||||
result = await collect_signals(
|
||||
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||||
)
|
||||
|
||||
kinds = {s["signal_kind"] for s in result["signals"]}
|
||||
assert "correction" in kinds
|
||||
|
||||
async def test_signal_collection_workflow(self, db_session, seed):
|
||||
"""collect_signals creates signals from failed WorkflowInstances."""
|
||||
from app.models.workflow import Workflow
|
||||
|
||||
wf = Workflow(
|
||||
tenant_id=seed["tenant"].id,
|
||||
name="Test WF",
|
||||
description="",
|
||||
steps=[],
|
||||
is_active=True,
|
||||
created_by=seed["admin"].id,
|
||||
)
|
||||
db_session.add(wf)
|
||||
await db_session.flush()
|
||||
db_session.add(
|
||||
WorkflowInstance(
|
||||
tenant_id=seed["tenant"].id,
|
||||
workflow_id=wf.id,
|
||||
status="failed",
|
||||
initiated_by=seed["admin"].id,
|
||||
)
|
||||
)
|
||||
await db_session.flush()
|
||||
|
||||
result = await collect_signals(
|
||||
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||||
)
|
||||
|
||||
kinds = {s["signal_kind"] for s in result["signals"]}
|
||||
assert "failure" in kinds
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# J-PATTERN: Pattern Detection
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestPatternDetection:
|
||||
async def test_detect_patterns_groups_signals(self, db_session, seed):
|
||||
"""detect_patterns groups signals and creates an ImprovementPattern."""
|
||||
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||||
for _ in range(2):
|
||||
await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||||
await db_session.flush()
|
||||
|
||||
await collect_signals(
|
||||
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||||
)
|
||||
result = await detect_patterns(db=db_session, tenant_id=seed["tenant"].id, min_occurrences=2)
|
||||
|
||||
assert result["patterns_created"] >= 1
|
||||
pattern = result["patterns"][0]
|
||||
assert pattern["pattern_kind"] == "retry_bottleneck"
|
||||
assert pattern["target_type"] == "agent"
|
||||
assert pattern["occurrence_count"] >= 2
|
||||
|
||||
rows = (
|
||||
await db_session.execute(
|
||||
select(ImprovementPattern).where(
|
||||
ImprovementPattern.tenant_id == seed["tenant"].id
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows) >= 1
|
||||
assert rows[0].status == "detected"
|
||||
|
||||
async def test_detect_patterns_links_signals(self, db_session, seed):
|
||||
"""detect_patterns links signals to the created pattern."""
|
||||
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||||
for _ in range(2):
|
||||
await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||||
await db_session.flush()
|
||||
|
||||
await collect_signals(
|
||||
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||||
)
|
||||
await detect_patterns(db=db_session, tenant_id=seed["tenant"].id, min_occurrences=2)
|
||||
|
||||
rows = (
|
||||
await db_session.execute(
|
||||
select(ImprovementSignal).where(
|
||||
ImprovementSignal.tenant_id == seed["tenant"].id,
|
||||
ImprovementSignal.pattern_id.is_not(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows) >= 2
|
||||
|
||||
async def test_detect_patterns_no_signals(self, db_session, seed):
|
||||
"""detect_patterns returns empty when no signals exist."""
|
||||
result = await detect_patterns(db=db_session, tenant_id=seed["tenant"].id)
|
||||
assert result["patterns_created"] == 0
|
||||
assert result["patterns"] == []
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# J-PROP: Proposal Creation
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestProposalCreation:
|
||||
async def test_create_proposal_draft(self, db_session, seed):
|
||||
"""create_proposal creates an ImprovementProposal in draft status."""
|
||||
proposal = await create_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
title="Improve agent prompt",
|
||||
description="Tune the system prompt",
|
||||
target_type="agent",
|
||||
target_name="Test Agent",
|
||||
proposed_config={"temperature": 0.1},
|
||||
rationale="Reduce failures",
|
||||
expected_benefit="Fewer retries",
|
||||
risk_assessment="Low",
|
||||
user_id=seed["admin"].id,
|
||||
)
|
||||
|
||||
assert proposal.status == "draft"
|
||||
assert proposal.tenant_id == seed["tenant"].id
|
||||
assert proposal.owner_id == seed["admin"].id
|
||||
assert proposal.proposed_config == {"temperature": 0.1}
|
||||
|
||||
row = (
|
||||
await db_session.execute(
|
||||
select(ImprovementProposal).where(ImprovementProposal.id == proposal.id)
|
||||
)
|
||||
).scalar_one()
|
||||
assert row.status == "draft"
|
||||
|
||||
async def test_create_proposal_captures_previous_config(self, db_session, seed):
|
||||
"""create_proposal captures previous agent config for rollback."""
|
||||
agent = await _create_agent(
|
||||
db_session, seed["tenant"].id, seed["admin"].id, temperature=0.3
|
||||
)
|
||||
await db_session.flush()
|
||||
|
||||
proposal = await create_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
title="Tune temp",
|
||||
description="Lower temperature",
|
||||
target_type="agent",
|
||||
target_ref_id=agent.id,
|
||||
target_name=agent.name,
|
||||
proposed_config={"temperature": 0.1},
|
||||
user_id=seed["admin"].id,
|
||||
)
|
||||
|
||||
assert proposal.previous_config.get("temperature") == 0.3
|
||||
assert proposal.previous_config.get("system_prompt") == agent.system_prompt
|
||||
|
||||
async def test_create_proposal_links_pattern(self, db_session, seed):
|
||||
"""create_proposal links to a pattern and updates its status."""
|
||||
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||||
for _ in range(2):
|
||||
await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||||
await db_session.flush()
|
||||
await collect_signals(
|
||||
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||||
)
|
||||
pat_result = await detect_patterns(db=db_session, tenant_id=seed["tenant"].id, min_occurrences=2)
|
||||
pattern_id = uuid.UUID(pat_result["patterns"][0]["id"])
|
||||
|
||||
proposal = await create_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
pattern_id=pattern_id,
|
||||
title="Fix agent",
|
||||
description="Fix the agent",
|
||||
target_type="agent",
|
||||
target_ref_id=agent.id,
|
||||
target_name=agent.name,
|
||||
proposed_config={"max_steps": 30},
|
||||
user_id=seed["admin"].id,
|
||||
)
|
||||
|
||||
assert proposal.pattern_id == pattern_id
|
||||
assert len(proposal.evidence_refs) >= 1
|
||||
pattern = (
|
||||
await db_session.execute(
|
||||
select(ImprovementPattern).where(ImprovementPattern.id == pattern_id)
|
||||
)
|
||||
).scalar_one()
|
||||
assert pattern.status == "proposal_created"
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# J-EVAL: Evaluation
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestProposalEvaluation:
|
||||
async def test_evaluate_proposal_sets_evaluated(self, db_session, seed):
|
||||
"""evaluate_proposal sets status to evaluated with mocked LLM."""
|
||||
proposal = await create_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
title="Evaluate me",
|
||||
description="Assess this change",
|
||||
target_type="agent",
|
||||
target_name="Test Agent",
|
||||
proposed_config={"temperature": 0.1},
|
||||
rationale="Reduce failures",
|
||||
expected_benefit="Fewer retries",
|
||||
risk_assessment="Low",
|
||||
user_id=seed["admin"].id,
|
||||
)
|
||||
await db_session.flush()
|
||||
|
||||
with patch(
|
||||
"app.plugins.builtins.self_improvement.services.llm_complete",
|
||||
new_callable=AsyncMock,
|
||||
return_value={
|
||||
"content": json.dumps(
|
||||
{
|
||||
"score": 0.8,
|
||||
"assessment": "Sound",
|
||||
"risks_identified": [],
|
||||
"recommendation": "approve",
|
||||
"test_scenarios": ["run"],
|
||||
}
|
||||
),
|
||||
"cost_usd": 0.001,
|
||||
},
|
||||
):
|
||||
result = await evaluate_proposal(
|
||||
db=db_session, tenant_id=seed["tenant"].id, proposal_id=proposal.id
|
||||
)
|
||||
|
||||
assert result["status"] == "evaluated"
|
||||
assert result["evaluation"]["score"] == 0.8
|
||||
assert result["evaluation"]["recommendation"] == "approve"
|
||||
|
||||
await db_session.refresh(proposal)
|
||||
assert proposal.status == "evaluated"
|
||||
assert proposal.evaluated_at is not None
|
||||
assert proposal.evaluation_result["score"] == 0.8
|
||||
|
||||
async def test_evaluate_proposal_nonexistent(self, db_session, seed):
|
||||
"""evaluate_proposal returns error for nonexistent proposal."""
|
||||
result = await evaluate_proposal(
|
||||
db=db_session, tenant_id=seed["tenant"].id, proposal_id=uuid.uuid4()
|
||||
)
|
||||
assert "error" in result
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# J-ACTIVATE: Activation + Rollback
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestActivationAndRollback:
|
||||
async def test_activate_proposal_applies_config(self, db_session, seed):
|
||||
"""activate_proposal applies proposed config to the agent."""
|
||||
agent = await _create_agent(
|
||||
db_session, seed["tenant"].id, seed["admin"].id, temperature=0.3, max_steps=20
|
||||
)
|
||||
await db_session.flush()
|
||||
|
||||
proposal = await create_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
title="Tune agent",
|
||||
description="Lower temperature",
|
||||
target_type="agent",
|
||||
target_ref_id=agent.id,
|
||||
target_name=agent.name,
|
||||
proposed_config={"temperature": 0.1, "max_steps": 30},
|
||||
user_id=seed["admin"].id,
|
||||
)
|
||||
proposal.status = "approved"
|
||||
await db_session.flush()
|
||||
|
||||
result = await activate_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
proposal_id=proposal.id,
|
||||
approved_by=seed["admin"].id,
|
||||
)
|
||||
|
||||
assert result["status"] == "active"
|
||||
assert result["applied"] is True
|
||||
|
||||
await db_session.refresh(agent)
|
||||
assert agent.temperature == 0.1
|
||||
assert agent.max_steps == 30
|
||||
|
||||
# A version snapshot should have been created
|
||||
versions = (
|
||||
await db_session.execute(
|
||||
select(AgentVersion).where(
|
||||
AgentVersion.tenant_id == seed["tenant"].id,
|
||||
AgentVersion.agent_id == agent.id,
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(versions) >= 1
|
||||
assert versions[0].snapshot["temperature"] == 0.3
|
||||
|
||||
async def test_rollback_proposal_restores_config(self, db_session, seed):
|
||||
"""rollback_proposal restores the previous agent config."""
|
||||
agent = await _create_agent(
|
||||
db_session, seed["tenant"].id, seed["admin"].id, temperature=0.3, max_steps=20
|
||||
)
|
||||
await db_session.flush()
|
||||
|
||||
proposal = await create_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
title="Tune agent",
|
||||
description="Lower temperature",
|
||||
target_type="agent",
|
||||
target_ref_id=agent.id,
|
||||
target_name=agent.name,
|
||||
proposed_config={"temperature": 0.1},
|
||||
user_id=seed["admin"].id,
|
||||
)
|
||||
proposal.status = "approved"
|
||||
await db_session.flush()
|
||||
await activate_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
proposal_id=proposal.id,
|
||||
approved_by=seed["admin"].id,
|
||||
)
|
||||
await db_session.refresh(agent)
|
||||
assert agent.temperature == 0.1
|
||||
|
||||
result = await rollback_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
proposal_id=proposal.id,
|
||||
reason="Regression",
|
||||
)
|
||||
|
||||
assert result["status"] == "rolled_back"
|
||||
assert result["restored"] is True
|
||||
|
||||
await db_session.refresh(agent)
|
||||
assert agent.temperature == 0.3
|
||||
await db_session.refresh(proposal)
|
||||
assert proposal.rollback_reason == "Regression"
|
||||
|
||||
async def test_rollback_requires_active(self, db_session, seed):
|
||||
"""rollback_proposal rejects a non-active proposal."""
|
||||
proposal = await create_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
title="Draft",
|
||||
description="Not active",
|
||||
target_type="agent",
|
||||
target_name="x",
|
||||
user_id=seed["admin"].id,
|
||||
)
|
||||
await db_session.flush()
|
||||
|
||||
result = await rollback_proposal(
|
||||
db=db_session, tenant_id=seed["tenant"].id, proposal_id=proposal.id
|
||||
)
|
||||
assert "error" in result
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# J-MEASURE: Impact Measurement
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestImpactMeasurement:
|
||||
async def test_measure_impact_creates_measurement(self, db_session, seed):
|
||||
"""measure_impact creates an ImpactMeasurement for an active proposal."""
|
||||
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||||
await db_session.flush()
|
||||
|
||||
proposal = await create_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
title="Measure",
|
||||
description="Measure impact",
|
||||
target_type="agent",
|
||||
target_ref_id=agent.id,
|
||||
target_name=agent.name,
|
||||
proposed_config={"temperature": 0.1},
|
||||
user_id=seed["admin"].id,
|
||||
)
|
||||
proposal.status = "approved"
|
||||
await db_session.flush()
|
||||
await activate_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
proposal_id=proposal.id,
|
||||
approved_by=seed["admin"].id,
|
||||
)
|
||||
|
||||
result = await measure_impact(
|
||||
db=db_session, tenant_id=seed["tenant"].id, proposal_id=proposal.id
|
||||
)
|
||||
|
||||
assert "measurement_id" in result
|
||||
assert "pre_metrics" in result
|
||||
assert "post_metrics" in result
|
||||
assert "delta" in result
|
||||
|
||||
rows = (
|
||||
await db_session.execute(
|
||||
select(ImpactMeasurement).where(
|
||||
ImpactMeasurement.tenant_id == seed["tenant"].id,
|
||||
ImpactMeasurement.proposal_id == proposal.id,
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows) == 1
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# J-ISOLATION: Tenant Isolation
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestTenantIsolation:
|
||||
async def test_signals_are_tenant_isolated(self, db_session, seed):
|
||||
"""Signals created for one tenant are not visible to another."""
|
||||
tenant_b = Tenant(name="Tenant B", slug="tenant-b")
|
||||
db_session.add(tenant_b)
|
||||
await db_session.flush()
|
||||
|
||||
agent = await _create_agent(db_session, seed["tenant"].id, seed["admin"].id)
|
||||
await _create_failed_run(db_session, seed["tenant"].id, agent.id, status="failed")
|
||||
await db_session.flush()
|
||||
await collect_signals(
|
||||
db=db_session, tenant_id=seed["tenant"].id, since=datetime.now(UTC) - timedelta(days=1)
|
||||
)
|
||||
|
||||
# Tenant B sees no signals
|
||||
result_b = await collect_signals(
|
||||
db=db_session, tenant_id=tenant_b.id, since=datetime.now(UTC) - timedelta(days=1)
|
||||
)
|
||||
assert result_b["collected"] == 0
|
||||
|
||||
rows_b = (
|
||||
await db_session.execute(
|
||||
select(ImprovementSignal).where(ImprovementSignal.tenant_id == tenant_b.id)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows_b) == 0
|
||||
|
||||
async def test_proposals_are_tenant_isolated(self, db_session, seed):
|
||||
"""Proposals created for one tenant are not visible to another."""
|
||||
tenant_b = Tenant(name="Tenant B", slug="tenant-b")
|
||||
db_session.add(tenant_b)
|
||||
await db_session.flush()
|
||||
|
||||
await create_proposal(
|
||||
db=db_session,
|
||||
tenant_id=seed["tenant"].id,
|
||||
title="Tenant A proposal",
|
||||
description="Only for A",
|
||||
target_type="agent",
|
||||
target_name="x",
|
||||
user_id=seed["admin"].id,
|
||||
)
|
||||
await db_session.flush()
|
||||
|
||||
rows_b = (
|
||||
await db_session.execute(
|
||||
select(ImprovementProposal).where(ImprovementProposal.tenant_id == tenant_b.id)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows_b) == 0
|
||||
|
||||
rows_a = (
|
||||
await db_session.execute(
|
||||
select(ImprovementProposal).where(
|
||||
ImprovementProposal.tenant_id == seed["tenant"].id
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
assert len(rows_a) == 1
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
# J-API: API Routes
|
||||
# ──────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestApiRoutes:
|
||||
async def test_api_signal_collect(self, improvement_authed_client):
|
||||
"""POST /api/v1/improvement/signals/collect works."""
|
||||
client, seed = improvement_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/improvement/signals/collect",
|
||||
json={"limit": 10},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
assert "collected" in data
|
||||
assert "signals" in data
|
||||
|
||||
async def test_api_proposal_create(self, improvement_authed_client):
|
||||
"""POST /api/v1/improvement/proposals works."""
|
||||
client, seed = improvement_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/improvement/proposals",
|
||||
json={
|
||||
"title": "API proposal",
|
||||
"description": "Created via API",
|
||||
"target_type": "agent",
|
||||
"target_name": "Test Agent",
|
||||
"proposed_config": {"temperature": 0.1},
|
||||
},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
assert data["status"] == "draft"
|
||||
assert data["title"] == "API proposal"
|
||||
assert "id" in data
|
||||
|
||||
async def test_api_proposal_create_missing_fields(self, improvement_authed_client):
|
||||
"""POST /api/v1/improvement/proposals rejects missing title/target_type."""
|
||||
client, _ = improvement_authed_client
|
||||
resp = await client.post(
|
||||
"/api/v1/improvement/proposals",
|
||||
json={"description": "no title"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 400, resp.text
|
||||
|
||||
async def test_api_proposal_list(self, improvement_authed_client):
|
||||
"""GET /api/v1/improvement/proposals lists proposals."""
|
||||
client, _ = improvement_authed_client
|
||||
resp = await client.get("/api/v1/improvement/proposals", headers=ORIGIN_HEADER)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
assert "items" in data
|
||||
assert "total" in data
|
||||
|
||||
async def test_api_proposal_evaluate(self, improvement_authed_client):
|
||||
"""POST /api/v1/improvement/proposals/{id}/evaluate works with mocked LLM."""
|
||||
client, seed = improvement_authed_client
|
||||
create_resp = await client.post(
|
||||
"/api/v1/improvement/proposals",
|
||||
json={
|
||||
"title": "Evaluate via API",
|
||||
"description": "Assess",
|
||||
"target_type": "agent",
|
||||
"target_name": "Test Agent",
|
||||
"proposed_config": {"temperature": 0.1},
|
||||
},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert create_resp.status_code == 200, create_resp.text
|
||||
proposal_id = create_resp.json()["id"]
|
||||
|
||||
with patch(
|
||||
"app.plugins.builtins.self_improvement.services.llm_complete",
|
||||
new_callable=AsyncMock,
|
||||
return_value={
|
||||
"content": json.dumps(
|
||||
{
|
||||
"score": 0.7,
|
||||
"assessment": "OK",
|
||||
"risks_identified": [],
|
||||
"recommendation": "approve",
|
||||
"test_scenarios": [],
|
||||
}
|
||||
),
|
||||
"cost_usd": 0.001,
|
||||
},
|
||||
):
|
||||
resp = await client.post(
|
||||
f"/api/v1/improvement/proposals/{proposal_id}/evaluate",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["status"] == "evaluated"
|
||||
|
||||
async def test_api_proposal_activate_and_rollback(self, improvement_authed_client, db_session):
|
||||
"""POST activate + rollback via API works end-to-end."""
|
||||
client, seed = improvement_authed_client
|
||||
agent = await _create_agent(db_session, seed["tenant_a"].id, seed["admin_a"].id)
|
||||
await db_session.commit()
|
||||
|
||||
create_resp = await client.post(
|
||||
"/api/v1/improvement/proposals",
|
||||
json={
|
||||
"title": "Activate via API",
|
||||
"description": "Apply config",
|
||||
"target_type": "agent",
|
||||
"target_ref_id": str(agent.id),
|
||||
"target_name": agent.name,
|
||||
"proposed_config": {"temperature": 0.1},
|
||||
},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert create_resp.status_code == 200, create_resp.text
|
||||
proposal_id = create_resp.json()["id"]
|
||||
|
||||
# Set to approved directly (approval flow is covered by service tests)
|
||||
await db_session.execute(
|
||||
update(ImprovementProposal)
|
||||
.where(ImprovementProposal.id == uuid.UUID(proposal_id))
|
||||
.values(status="approved")
|
||||
)
|
||||
await db_session.commit()
|
||||
|
||||
act_resp = await client.post(
|
||||
f"/api/v1/improvement/proposals/{proposal_id}/activate",
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert act_resp.status_code == 200, act_resp.text
|
||||
assert act_resp.json()["status"] == "active"
|
||||
|
||||
await db_session.refresh(agent)
|
||||
assert agent.temperature == 0.1
|
||||
|
||||
roll_resp = await client.post(
|
||||
f"/api/v1/improvement/proposals/{proposal_id}/rollback",
|
||||
json={"reason": "test"},
|
||||
headers=ORIGIN_HEADER,
|
||||
)
|
||||
assert roll_resp.status_code == 200, roll_resp.text
|
||||
assert roll_resp.json()["status"] == "rolled_back"
|
||||
|
||||
await db_session.refresh(agent)
|
||||
assert agent.temperature == 0.3
|
||||
Reference in New Issue
Block a user