feat(J): Phase J Self-Improvement Plugin — controlled improvement loop

- New self_improvement plugin: models, services, routes, plugin
- 4 SQLAlchemy models: ImprovementSignal, ImprovementPattern, ImprovementProposal, ImpactMeasurement
- Migration 0132: 4 tables with RLS
- Services: collect_signals, detect_patterns, create_proposal, evaluate_proposal, request_approval, activate_proposal, rollback_proposal, measure_impact
- 11 API routes under /api/v1/improvement/
- Frontend: improvement.ts API client, ImprovementPanel.tsx in AISidebar proactive tab
- 24/24 integration tests pass
- Fix: __init__.py imports Plugin class for discover_builtins() (self_improvement + knowledge)
- Fix: automation plugin register_plugin_contributions skips when no tenant exists

app/plugins/builtins/self_improvement/__init__.py

@@ -0,0 +1,5 @@
+"""Self-improvement plugin — controlled improvement loop on real usage signals."""
+
+from app.plugins.builtins.self_improvement.plugin import SelfImprovementPlugin
+
+__all__ = ["SelfImprovementPlugin"]

app/plugins/builtins/self_improvement/models.py

@@ -0,0 +1,152 @@
+"""Self-improvement models — signals, patterns, proposals, impact measurements.
+
+All models are real SQLAlchemy models with TenantMixin (no dataclasses).
+Tables get RLS via migration.
+"""
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import (
+    DateTime,
+    Float,
+    ForeignKey,
+    Index,
+    Integer,
+    String,
+    Text,
+    func,
+)
+from sqlalchemy.dialects.postgresql import JSONB, UUID as PGUUID
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.core.db import Base, TenantMixin
+from app.models.owned_mixin import OwnedMixin
+
+# Constants
+SIGNAL_KINDS = ("failure", "retry", "dismissal", "correction", "observation", "handoff")
+SEVERITY_LEVELS = ("info", "warning", "error")
+PATTERN_KINDS = ("retry_bottleneck", "manual_correction", "suggestion_dismissal", "repetitive_handoff")
+PATTERN_STATUSES = ("detected", "proposal_created", "resolved", "ignored")
+PROPOSAL_STATUSES = (
+    "draft", "evaluating", "evaluated", "pending_approval",
+    "approved", "active", "rolled_back", "rejected", "expired",
+)
+PROPOSAL_TARGET_TYPES = ("agent", "skill", "trigger", "workflow", "miniapp_template", "plugin_patch")
+
+
+class ImprovementSignal(Base, TenantMixin):
+    """A single improvement signal collected from real system usage.
+
+    References source data (AgentRun, WorkflowInstance, ProactiveSuggestion,
+    AuditLog) by ID — stores no personal data copies.
+    """
+    __tablename__ = "improvement_signals"
+    __table_args__ = (
+        Index("ix_impr_signals_tenant_kind", "tenant_id", "signal_kind"),
+        Index("ix_impr_signals_tenant_source", "tenant_id", "source_type"),
+        Index("ix_impr_signals_tenant_pattern", "tenant_id", "pattern_id"),
+    )
+
+    id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    source_type: Mapped[str] = mapped_column(String(50), nullable=False)
+    source_ref_id: Mapped[uuid.UUID | None] = mapped_column(PGUUID(as_uuid=True), nullable=True)
+    source_metadata: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
+    summary: Mapped[str] = mapped_column(Text, nullable=False)
+    signal_kind: Mapped[str] = mapped_column(String(30), nullable=False)
+    severity: Mapped[str] = mapped_column(String(20), nullable=False, default="info")
+    confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.5)
+    pattern_id: Mapped[uuid.UUID | None] = mapped_column(
+        PGUUID(as_uuid=True), ForeignKey("improvement_patterns.id", ondelete="SET NULL"), nullable=True
+    )
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+
+
+class ImprovementPattern(Base, TenantMixin):
+    """A detected pattern from grouped improvement signals."""
+    __tablename__ = "improvement_patterns"
+    __table_args__ = (
+        Index("ix_impr_patterns_tenant_status", "tenant_id", "status"),
+        Index("ix_impr_patterns_tenant_target", "tenant_id", "target_type"),
+    )
+
+    id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    pattern_kind: Mapped[str] = mapped_column(String(40), nullable=False)
+    title: Mapped[str] = mapped_column(String(300), nullable=False)
+    description: Mapped[str] = mapped_column(Text, nullable=False, default="")
+    target_type: Mapped[str] = mapped_column(String(30), nullable=False)
+    target_name: Mapped[str] = mapped_column(String(200), nullable=False, default="unknown")
+    evidence_refs: Mapped[list] = mapped_column(JSONB, nullable=False, default=list)
+    occurrence_count: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.5)
+    status: Mapped[str] = mapped_column(String(20), nullable=False, default="detected")
+    proposed_action: Mapped[str] = mapped_column(Text, nullable=False, default="")
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now())
+
+
+class ImprovementProposal(Base, TenantMixin, OwnedMixin):
+    """A versioned improvement proposal with lifecycle.
+
+    Status flow:
+    draft -> evaluating -> evaluated -> pending_approval -> approved -> active
+                                                                     -> rolled_back
+                                  rejected <- pending_approval
+                                  expired <- pending_approval
+    """
+    __tablename__ = "improvement_proposals"
+    __table_args__ = (
+        Index("ix_impr_proposals_tenant_status", "tenant_id", "status"),
+        Index("ix_impr_proposals_tenant_target", "tenant_id", "target_type"),
+        Index("ix_impr_proposals_tenant_pattern", "tenant_id", "pattern_id"),
+    )
+
+    id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    pattern_id: Mapped[uuid.UUID | None] = mapped_column(
+        PGUUID(as_uuid=True), ForeignKey("improvement_patterns.id", ondelete="SET NULL"), nullable=True
+    )
+    title: Mapped[str] = mapped_column(String(300), nullable=False)
+    description: Mapped[str] = mapped_column(Text, nullable=False, default="")
+    target_type: Mapped[str] = mapped_column(String(30), nullable=False)
+    target_ref_id: Mapped[uuid.UUID | None] = mapped_column(PGUUID(as_uuid=True), nullable=True)
+    target_name: Mapped[str | None] = mapped_column(String(200), nullable=True)
+    version_number: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
+    proposed_config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
+    previous_config: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
+    evidence_refs: Mapped[list] = mapped_column(JSONB, nullable=False, default=list)
+    rationale: Mapped[str] = mapped_column(Text, nullable=False, default="")
+    expected_benefit: Mapped[str] = mapped_column(Text, nullable=False, default="")
+    risk_assessment: Mapped[str] = mapped_column(Text, nullable=False, default="")
+    evaluation_result: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
+    evaluated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    approval_request_id: Mapped[uuid.UUID | None] = mapped_column(PGUUID(as_uuid=True), nullable=True)
+    approved_by: Mapped[uuid.UUID | None] = mapped_column(
+        PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True
+    )
+    approved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    activated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    rolled_back_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    rollback_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
+    status: Mapped[str] = mapped_column(String(20), nullable=False, default="draft")
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now())
+
+
+class ImpactMeasurement(Base, TenantMixin):
+    """Pre/post impact measurement for an activated proposal."""
+    __tablename__ = "improvement_impact_measurements"
+    __table_args__ = (
+        Index("ix_impr_impact_tenant_proposal", "tenant_id", "proposal_id"),
+    )
+
+    id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    proposal_id: Mapped[uuid.UUID] = mapped_column(
+        PGUUID(as_uuid=True), ForeignKey("improvement_proposals.id", ondelete="CASCADE"), nullable=False
+    )
+    pre_metrics: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
+    post_metrics: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
+    delta: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
+    assessment: Mapped[str] = mapped_column(Text, nullable=False, default="")
+    is_positive: Mapped[str] = mapped_column(String(20), nullable=False, default="neutral")
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())

app/plugins/builtins/self_improvement/plugin.py

@@ -0,0 +1,46 @@
+"""Self-improvement plugin — controlled improvement loop.
+
+Builds on existing systems:
+- ai_proactive: ContextLog, ProactiveSuggestion (signals)
+- automation: AgentRun, AgentVersion (versioning + runs)
+- app.core.approval: ApprovalRequest (human approval)
+- app.ai.oversight: DecisionRecord (audit trail)
+- app.ai.llm_client: llm_complete (evaluation)
+- AuditLog, WorkflowInstance (usage data)
+"""
+from __future__ import annotations
+
+import logging
+
+from app.plugins.base import BasePlugin
+from app.plugins.manifest import PluginManifest, PluginRouteDef
+
+logger = logging.getLogger(__name__)
+
+
+class SelfImprovementPlugin(BasePlugin):
+    manifest = PluginManifest(
+        name="self_improvement",
+        version="1.0.0",
+        display_name="Self-Improvement",
+        description="Controlled self-improvement loop: signals, patterns, proposals, evaluation, approval, activation, rollback, impact measurement.",
+        dependencies=["permissions", "automation", "ai_proactive"],
+        routes=[
+            PluginRouteDef(
+                path="/api/v1/improvement",
+                module="app.plugins.builtins.self_improvement.routes",
+                router_attr="router",
+            ),
+        ],
+        permissions=["improvement:read", "improvement:write", "improvement:admin"],
+    )
+
+    async def on_activate(self, db, service_container, event_bus) -> None:
+        """Register self-improvement hooks on activation."""
+        await super().on_activate(db, service_container, event_bus)
+        logger.info("Self-improvement plugin activated")
+
+    async def on_deactivate(self, db, service_container, event_bus) -> None:
+        """Clean up on deactivation."""
+        await super().on_deactivate(db, service_container, event_bus)
+        logger.info("Self-improvement plugin deactivated")

app/plugins/builtins/self_improvement/routes.py

@@ -0,0 +1,299 @@
+"""Self-improvement plugin routes — signals, patterns, proposals, evaluation, approval, activation, impact."""
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.db import get_db
+from app.deps import require_permission
+from app.plugins.builtins.self_improvement.services import (
+    activate_proposal,
+    collect_signals,
+    create_proposal,
+    detect_patterns,
+    evaluate_proposal,
+    get_proposal_detail,
+    list_patterns,
+    list_proposals,
+    list_signals,
+    measure_impact,
+    request_approval,
+    rollback_proposal,
+)
+
+router = APIRouter(prefix="/api/v1/improvement", tags=["improvement"])
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# J-SIGNAL: Signal Collection
+# NOTE: /signals/collect must be defined before /signals to avoid route conflicts
+# ──────────────────────────────────────────────────────────────────────────
+
+@router.post("/signals/collect")
+async def collect(
+    body: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:read")),
+):
+    """Collect improvement signals from existing system data."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    since_str = body.get("since")
+    since = None
+    if since_str:
+        try:
+            since = datetime.fromisoformat(since_str)
+        except ValueError:
+            raise HTTPException(400, detail={"detail": "Invalid since format", "code": "invalid_date"}) from None
+    limit = min(body.get("limit", 100), 500)
+    result = await collect_signals(db=db, tenant_id=tenant_id, since=since, limit=limit)
+    await db.commit()
+    return result
+
+
+@router.get("/signals")
+async def signals(
+    page: int = Query(1, ge=1),
+    page_size: int = Query(20, ge=1, le=100),
+    source_type: str | None = Query(None),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:read")),
+):
+    """List improvement signals with pagination."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    return await list_signals(db=db, tenant_id=tenant_id, page=page, page_size=page_size, source_type=source_type)
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# J-PATTERN: Pattern Detection
+# ──────────────────────────────────────────────────────────────────────────
+
+@router.post("/patterns/detect")
+async def detect(
+    body: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:read")),
+):
+    """Detect recurring patterns from collected signals."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    min_occurrences = body.get("min_occurrences", 2)
+    result = await detect_patterns(db=db, tenant_id=tenant_id, min_occurrences=min_occurrences)
+    await db.commit()
+    return result
+
+
+@router.get("/patterns")
+async def patterns(
+    page: int = Query(1, ge=1),
+    page_size: int = Query(20, ge=1, le=100),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:read")),
+):
+    """List detected patterns with pagination."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    return await list_patterns(db=db, tenant_id=tenant_id, page=page, page_size=page_size)
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# J-PROP: Improvement Proposals
+# ──────────────────────────────────────────────────────────────────────────
+
+@router.post("/proposals")
+async def create(
+    body: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:write")),
+):
+    """Create a new improvement proposal."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    user_id = uuid.UUID(current_user["user_id"]) if current_user.get("user_id") else None
+
+    pattern_id = None
+    if body.get("pattern_id"):
+        try:
+            pattern_id = uuid.UUID(body["pattern_id"])
+        except ValueError:
+            raise HTTPException(400, detail={"detail": "Invalid pattern_id", "code": "invalid_id"}) from None
+
+    target_ref_id = None
+    if body.get("target_ref_id"):
+        try:
+            target_ref_id = uuid.UUID(body["target_ref_id"])
+        except ValueError:
+            raise HTTPException(400, detail={"detail": "Invalid target_ref_id", "code": "invalid_id"}) from None
+
+    if not body.get("title") or not body.get("target_type"):
+        raise HTTPException(400, detail={"detail": "title and target_type required", "code": "missing_fields"})
+
+    proposal = await create_proposal(
+        db=db, tenant_id=tenant_id,
+        pattern_id=pattern_id,
+        title=body["title"],
+        description=body.get("description", ""),
+        target_type=body["target_type"],
+        target_ref_id=target_ref_id,
+        target_name=body.get("target_name"),
+        proposed_config=body.get("proposed_config", {}),
+        rationale=body.get("rationale", ""),
+        expected_benefit=body.get("expected_benefit", ""),
+        risk_assessment=body.get("risk_assessment", ""),
+        user_id=user_id,
+    )
+    await db.commit()
+    return {"id": str(proposal.id), "status": proposal.status, "title": proposal.title}
+
+
+@router.get("/proposals")
+async def proposals(
+    page: int = Query(1, ge=1),
+    page_size: int = Query(20, ge=1, le=100),
+    status: str | None = Query(None),
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:read")),
+):
+    """List improvement proposals with pagination."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    return await list_proposals(db=db, tenant_id=tenant_id, page=page, page_size=page_size, status=status)
+
+
+@router.get("/proposals/{proposal_id}")
+async def proposal_detail(
+    proposal_id: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:read")),
+):
+    """Get full proposal detail."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    try:
+        pid = uuid.UUID(proposal_id)
+    except ValueError:
+        raise HTTPException(400, detail={"detail": "Invalid proposal_id", "code": "invalid_id"}) from None
+    result = await get_proposal_detail(db=db, tenant_id=tenant_id, proposal_id=pid)
+    if "error" in result:
+        raise HTTPException(404, detail={"detail": result["error"], "code": "not_found"})
+    return result
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# J-EVAL: Evaluation
+# ──────────────────────────────────────────────────────────────────────────
+
+@router.post("/proposals/{proposal_id}/evaluate")
+async def evaluate(
+    proposal_id: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:write")),
+):
+    """Evaluate a proposal via LLM-based dry-run assessment."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    try:
+        pid = uuid.UUID(proposal_id)
+    except ValueError:
+        raise HTTPException(400, detail={"detail": "Invalid proposal_id", "code": "invalid_id"}) from None
+    result = await evaluate_proposal(db=db, tenant_id=tenant_id, proposal_id=pid)
+    if "error" in result:
+        raise HTTPException(404, detail={"detail": result["error"], "code": "not_found"})
+    await db.commit()
+    return result
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# J-APPROVAL: Human Approval
+# ──────────────────────────────────────────────────────────────────────────
+
+@router.post("/proposals/{proposal_id}/request-approval")
+async def req_approval(
+    proposal_id: str,
+    body: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:write")),
+):
+    """Request human approval for a proposal."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    user_id = uuid.UUID(current_user["user_id"]) if current_user.get("user_id") else None
+    try:
+        pid = uuid.UUID(proposal_id)
+    except ValueError:
+        raise HTTPException(400, detail={"detail": "Invalid proposal_id", "code": "invalid_id"}) from None
+    approver_id = None
+    if body.get("approver_id"):
+        try:
+            approver_id = uuid.UUID(body["approver_id"])
+        except ValueError:
+            raise HTTPException(400, detail={"detail": "Invalid approver_id", "code": "invalid_id"}) from None
+    result = await request_approval(db=db, tenant_id=tenant_id, proposal_id=pid, requested_by=user_id, approver_id=approver_id)
+    if "error" in result:
+        raise HTTPException(400, detail={"detail": result["error"], "code": "invalid_state"})
+    await db.commit()
+    return result
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# J-ACTIVATE: Controlled Activation + Rollback
+# ──────────────────────────────────────────────────────────────────────────
+
+@router.post("/proposals/{proposal_id}/activate")
+async def activate(
+    proposal_id: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:admin")),
+):
+    """Activate an approved proposal."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    user_id = uuid.UUID(current_user["user_id"]) if current_user.get("user_id") else None
+    try:
+        pid = uuid.UUID(proposal_id)
+    except ValueError:
+        raise HTTPException(400, detail={"detail": "Invalid proposal_id", "code": "invalid_id"}) from None
+    result = await activate_proposal(db=db, tenant_id=tenant_id, proposal_id=pid, approved_by=user_id)
+    if "error" in result:
+        raise HTTPException(400, detail={"detail": result["error"], "code": "invalid_state"})
+    await db.commit()
+    return result
+
+
+@router.post("/proposals/{proposal_id}/rollback")
+async def rollback(
+    proposal_id: str,
+    body: dict,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:admin")),
+):
+    """Rollback an active proposal."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    try:
+        pid = uuid.UUID(proposal_id)
+    except ValueError:
+        raise HTTPException(400, detail={"detail": "Invalid proposal_id", "code": "invalid_id"}) from None
+    reason = body.get("reason", "")
+    result = await rollback_proposal(db=db, tenant_id=tenant_id, proposal_id=pid, reason=reason)
+    if "error" in result:
+        raise HTTPException(400, detail={"detail": result["error"], "code": "invalid_state"})
+    await db.commit()
+    return result
+
+
+# ──────────────────────────────────────────────────────────────────────────
+# J-MEASURE: Impact Measurement
+# ──────────────────────────────────────────────────────────────────────────
+
+@router.post("/proposals/{proposal_id}/measure")
+async def measure(
+    proposal_id: str,
+    db: AsyncSession = Depends(get_db),
+    current_user: dict = Depends(require_permission("automation:read")),
+):
+    """Measure pre/post impact of an activated proposal."""
+    tenant_id = uuid.UUID(current_user["tenant_id"])
+    try:
+        pid = uuid.UUID(proposal_id)
+    except ValueError:
+        raise HTTPException(400, detail={"detail": "Invalid proposal_id", "code": "invalid_id"}) from None
+    result = await measure_impact(db=db, tenant_id=tenant_id, proposal_id=pid)
+    if "error" in result:
+        raise HTTPException(400, detail={"detail": result["error"], "code": "invalid_state"})
+    await db.commit()
+    return result