Leopoldadmin/leocrm
1
0
Fork 0
Code Issues 363 Pull Requests Actions 1 Packages Projects Releases 1 Wiki Activity

feat(G): G-RUN/G-CTX/G-WAIT/G-HTTP/G-MAIL/G-CAL/G-DMS/G-SEARCH/G-AGENT/G-CRM/G-EVT/G-WEB — Durable WorkflowRun, 10 step handlers, resume/wait/lock/retry, SSRF protection, frontend step editor

Browse Source
This commit is contained in:
Agent Zero
2026-08-18 00:29:58 +02:00
parent 4ec2ac9eb5
commit db41e60042
8 changed files with 1783 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/workflows/engine.py
+287 -12
View File
@@ -1,13 +1,19 @@
"""Workflow execution engine — step processing, conditions, approvals.
"""Workflow execution engine — step processing, conditions, approvals, wait/resume.
Processes workflow instances by evaluating steps sequentially.
Supports step types: action, approval, notification, condition.
Integrates with the event bus for event-triggered workflows.
Supports step types: action, approval, notification, condition, wait,
http, mail, calendar, dms, search, agent, crm, event, webhook.
G-RUN: Durable/resumable with resume_at, step_state, lock_owner.
G-RETRY: Retry with backoff for failed steps.
G-IDEMP: Idempotency key for side-effect steps.
G-LOG: Execution log per step (input, output, duration, status).
"""
from __future__ import annotations
import logging
import time
import uuid
from datetime import UTC, datetime
from typing import Any
@@ -24,6 +30,7 @@ from app.services.workflow_service import (
create_instance,
find_workflows_for_event,
)
from app.workflows.step_handlers import StepResult, get_step_handler
logger = logging.getLogger(__name__)
@@ -31,11 +38,21 @@ logger = logging.getLogger(__name__)
class WorkflowEngine:
"""Processes workflow instances through their defined steps.
Step types:
Step types (built-in):
- action: Executes a configured action (e.g. create entity, send notification)
- approval: Pauses and waits for user approve/reject via API
- notification: Sends a notification to specified users
- condition: Evaluates a condition and branches accordingly
- wait: Pauses for a duration or until a specific time (G-WAIT)
- http: Sends an HTTP request with SSRF protection (G-HTTP)
- mail: Sends an email via the mail plugin (G-MAIL)
- calendar: Creates/updates/deletes calendar events (G-CAL)
- dms: Interacts with the document management system (G-DMS)
- search: Runs a unified search query (G-SEARCH)
- agent: Invokes an autonomous AI agent (G-AGENT)
- crm: Creates/updates/deletes contacts and companies (G-CRM)
- event: Publishes an event to the event bus (G-EVT)
- webhook: Sends an outgoing webhook (G-WEB)
"""
def __init__(self, db: AsyncSession, tenant_id: uuid.UUID):
@@ -45,8 +62,9 @@ class WorkflowEngine:
async def process_step(self, instance: WorkflowInstance) -> dict[str, Any]:
"""Process the current step of a workflow instance.
For action/notification/condition steps: executes and advances.
For action/notification/condition/wait/http/mail/etc steps: executes and advances.
For approval steps: sets status to in_progress and waits.
For wait steps: sets resume_at and pauses.
Returns the updated instance dict.
"""
wf_result = await self.db.execute(
@@ -76,7 +94,8 @@ class WorkflowEngine:
step = steps[instance.current_step_index]
step_type = step.get("type", "action")
# Log step entry
# Log step entry (G-LOG)
step_start_time = time.monotonic()
await _log_step_history(
self.db,
self.tenant_id,
@@ -84,28 +103,180 @@ class WorkflowEngine:
step_index=instance.current_step_index,
step_type=step_type,
action="processing",
details={"step_name": step.get("name")},
details={"step_name": step.get("name"), "config": step.get("config", {})},
)
# Approval steps pause and wait for user input
if step_type == "approval":
# Approval steps pause and wait for user input
if instance.status == "pending":
instance.status = "in_progress"
instance.resume_reason = "approval"
await self.db.flush()
return _instance_to_dict(instance)
elif step_type == "notification":
return await self._process_notification(instance, step)
# Try registered step handlers first (new G step types)
handler = get_step_handler(step_type)
if handler is not None:
return await self._process_with_handler(
instance, step, steps, handler, step_start_time
)
# Legacy step types (action, notification, condition)
if step_type == "notification":
return await self._process_notification(instance, step)
elif step_type == "condition":
return await self._process_condition(instance, step, steps)
elif step_type == "action":
return await self._process_action(instance, step, steps)
else:
return {"error": f"Unknown step type: {step_type}", "status_code": 400}
async def _process_with_handler(
self,
instance: WorkflowInstance,
step: dict,
steps: list,
handler: Any,
step_start_time: float,
) -> dict[str, Any]:
"""Execute a step using a registered step handler (G step types)."""
step_type = step.get("type", "action")
try:
result: StepResult = await handler(
self.db,
self.tenant_id,
instance,
step,
)
except Exception as e:
logger.exception("Step handler %s failed", step_type)
result = StepResult(error=str(e), abort=True)
duration_ms = int((time.monotonic() - step_start_time) * 1000)
# Handle abort
if result.abort:
instance.status = "failed"
instance.error_message = result.error or "Unknown error"
await _log_step_history(
self.db,
self.tenant_id,
instance.id,
step_index=instance.current_step_index,
step_type=step_type,
action="failed",
details={
"error": result.error,
"duration_ms": duration_ms,
"output": result.output,
},
)
await self.db.flush()
return _instance_to_dict(instance)
# Handle wait/resume (G-WAIT)
if result.wait_until is not None:
instance.status = "waiting"
instance.resume_at = result.wait_until
instance.resume_reason = result.wait_reason or "wait"
await _log_step_history(
self.db,
self.tenant_id,
instance.id,
step_index=instance.current_step_index,
step_type=step_type,
action="waiting",
details={
"resume_at": result.wait_until.isoformat(),
"reason": result.wait_reason,
"duration_ms": duration_ms,
},
)
await self.db.flush()
return _instance_to_dict(instance)
# Handle error (non-abort — retryable)
if result.error:
instance.retry_count += 1
if instance.retry_count >= instance.max_retries:
instance.status = "failed"
instance.error_message = result.error
await _log_step_history(
self.db,
self.tenant_id,
instance.id,
step_index=instance.current_step_index,
step_type=step_type,
action="failed",
details={
"error": result.error,
"duration_ms": duration_ms,
"retry_count": instance.retry_count,
},
)
await self.db.flush()
return _instance_to_dict(instance)
else:
# Retry: stay on same step, set resume_at with backoff
import asyncio
backoff = min(2 ** instance.retry_count, 60)
from datetime import timedelta
instance.resume_at = datetime.now(UTC) + timedelta(seconds=backoff)
instance.resume_reason = "retry"
instance.status = "waiting"
await _log_step_history(
self.db,
self.tenant_id,
instance.id,
step_index=instance.current_step_index,
step_type=step_type,
action="retry",
details={
"error": result.error,
"retry_count": instance.retry_count,
"resume_at": instance.resume_at.isoformat(),
"duration_ms": duration_ms,
},
)
await self.db.flush()
return _instance_to_dict(instance)
# Handle branch (next_index override)
if result.next_index is not None:
instance.current_step_index = result.next_index
instance.status = "in_progress"
elif result.advance:
# Advance to next step
next_idx = instance.current_step_index + 1
if next_idx >= len(steps):
instance.status = "completed"
instance.completed_at = datetime.now(UTC)
else:
instance.current_step_index = next_idx
instance.status = "in_progress"
# Store step output in step_state (G-CTX)
step_key = f"step_{instance.current_step_index}_output"
instance.step_state[step_key] = result.output
# Log success (G-LOG)
await _log_step_history(
self.db,
self.tenant_id,
instance.id,
step_index=instance.current_step_index,
step_type=step_type,
action="executed",
details={
"output": result.output,
"duration_ms": duration_ms,
},
)
await self.db.flush()
return _instance_to_dict(instance)
async def _process_action(
self, instance: WorkflowInstance, step: dict, steps: list
) -> dict[str, Any]:
@@ -259,6 +430,90 @@ class WorkflowEngine:
await self.db.flush()
return _instance_to_dict(instance)
async def resume(self, instance: WorkflowInstance) -> dict[str, Any]:
"""Resume a waiting workflow instance.
Called when resume_at has passed, an approval is decided,
or an event/webhook triggers a resume.
Clears resume_at/resume_reason and processes the current step.
"""
if instance.status != "waiting":
return _instance_to_dict(instance)
# Clear resume state
instance.resume_at = None
instance.resume_reason = None
instance.status = "in_progress"
# For wait steps, advance to next step after resume
wf_result = await self.db.execute(
select(Workflow).where(Workflow.id == instance.workflow_id)
)
workflow = wf_result.scalar_one_or_none()
if workflow is None:
return {"error": "Workflow not found", "status_code": 404}
steps = workflow.steps or []
if instance.current_step_index < len(steps):
step = steps[instance.current_step_index]
step_type = step.get("type", "action")
# If it was a wait step, advance to next
if step_type == "wait":
next_idx = instance.current_step_index + 1
if next_idx >= len(steps):
instance.status = "completed"
instance.completed_at = datetime.now(UTC)
else:
instance.current_step_index = next_idx
await self.db.flush()
# Process the next step
return await self.process_step(instance)
async def acquire_lock(
self, instance: WorkflowInstance, owner: str, ttl_seconds: int = 300
) -> bool:
"""Acquire a Redis lock for a workflow instance (G-RUN concurrency).
Prevents two workers from processing the same instance simultaneously.
"""
from app.core.redis import get_redis
import redis.asyncio as aioredis
try:
r = await get_redis()
lock_key = f"workflow_lock:{instance.id}"
acquired = await r.set(
lock_key,
owner,
nx=True,
ex=ttl_seconds,
)
if acquired:
instance.lock_owner = owner
instance.lock_expires_at = datetime.now(UTC) + timedelta(seconds=ttl_seconds)
await self.db.flush()
return True
return False
except Exception as e:
logger.warning("Failed to acquire workflow lock: %s", e)
return True # Fail open — allow processing without lock
async def release_lock(self, instance: WorkflowInstance) -> None:
"""Release the Redis lock for a workflow instance."""
from app.core.redis import get_redis
try:
r = await get_redis()
lock_key = f"workflow_lock:{instance.id}"
await r.delete(lock_key)
instance.lock_owner = None
instance.lock_expires_at = None
await self.db.flush()
except Exception as e:
logger.warning("Failed to release workflow lock: %s", e)
async def handle_event(
db: AsyncSession,
@@ -314,3 +569,23 @@ def register_workflow_event_handlers() -> None:
# Subscribe to ALL events via wildcard '*' — the handler dynamically
# queries for workflows whose trigger_event matches the published event.
event_bus.subscribe('*', _workflow_event_handler)
async def find_resumable_workflows(
db: AsyncSession,
tenant_id: uuid.UUID,
) -> list[WorkflowInstance]:
"""Find workflow instances that are waiting and their resume_at has passed.
Called by the ARQ cron job to resume waiting workflows.
"""
now = datetime.now(UTC)
result = await db.execute(
select(WorkflowInstance).where(
WorkflowInstance.tenant_id == tenant_id,
WorkflowInstance.status == "waiting",
WorkflowInstance.resume_at.is_not(None),
WorkflowInstance.resume_at <= now,
)
)
return list(result.scalars().all())