Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

feat(jobs): run workflow executions as a workflow.execute unified job (Phase 6)

Juan Denisc0cb0f088ea9
diff --git a/backend/app/__init__.py b/backend/app/__init__.py
index 59af5bfd..d96414f8 100644
--- a/backend/app/__init__.py
+++ b/backend/app/__init__.py
@@ -437,9 +437,11 @@ def create_app(config_name=None):
from app.jobs import start_job_system
from app.jobs.builtin_handlers import register_builtin_handlers, seed_builtin_schedules
register_builtin_handlers()
- # Register event-driven job handlers (deployment installs, etc.).
+ # Register event-driven job handlers (deployment installs, workflow runs).
from app.services.deployment_job_service import DeploymentJobService
DeploymentJobService.register_jobs()
+ from app.services.workflow_engine import WorkflowEngine
+ WorkflowEngine.register_jobs()
start_job_system(app, seed=seed_builtin_schedules)
# Request body size limit
diff --git a/backend/app/api/workflows.py b/backend/app/api/workflows.py
index 3ac8c2af..442c82fb 100644
--- a/backend/app/api/workflows.py
+++ b/backend/app/api/workflows.py
@@ -194,7 +194,7 @@ def execute_workflow(workflow_id):
data = request.get_json() or {}
context = data.get('context', {})
- execution_id = WorkflowEngine.execute_workflow(
+ execution_id = WorkflowEngine.enqueue_execution(
workflow_id=workflow_id,
trigger_type='manual',
context=context
@@ -337,7 +337,7 @@ def webhook_trigger(webhook_id):
context['query'] = dict(request.args)
try:
- execution_id = WorkflowEngine.execute_workflow(
+ execution_id = WorkflowEngine.enqueue_execution(
workflow_id=target_workflow.id,
trigger_type='webhook',
context=context
diff --git a/backend/app/jobs/builtin_handlers.py b/backend/app/jobs/builtin_handlers.py
index f4ae1406..51eae503 100644
--- a/backend/app/jobs/builtin_handlers.py
+++ b/backend/app/jobs/builtin_handlers.py
@@ -95,7 +95,7 @@ def check_workflow_schedules():
if seconds_since_last_run < 110:
continue
logger.info(f'Scheduled workflow triggered: {workflow.name} (ID: {workflow.id})')
- WorkflowEngine.execute_workflow(
+ WorkflowEngine.enqueue_execution(
workflow_id=workflow.id,
trigger_type='cron',
context={'scheduled_at': prev_run.isoformat()},
diff --git a/backend/app/services/workflow_engine.py b/backend/app/services/workflow_engine.py
index a07fe926..d7a3b0d6 100644
--- a/backend/app/services/workflow_engine.py
+++ b/backend/app/services/workflow_engine.py
@@ -34,6 +34,10 @@ MAX_RETRY_COUNT = 5
DEFAULT_RETRY_DELAY = 5 # seconds
MAX_OUTPUT_SIZE = 1024 * 512 # 512 KB
+# Unified job kind for asynchronous workflow executions
+# (see WorkflowEngine.enqueue_execution / register_jobs).
+WORKFLOW_JOB_KIND = 'workflow.execute'
+
def _parse_version(s: str) -> tuple:
"""Turn '3.11.4' / 'v20.10.0' / '8.2' into a comparable tuple of ints.
@@ -144,11 +148,37 @@ class WorkflowEngine:
@staticmethod
def execute_workflow(workflow_id: int, trigger_type: str = 'manual',
context: Dict[str, Any] = None) -> int:
- """
- Execute a workflow by ID.
+ """Create and SYNCHRONOUSLY run a workflow. Returns the execution id.
- Returns the ID of the created WorkflowExecution.
+ Kept for callers that want to block on the result (and tests). Async
+ callers (API/webhook/cron/event) use enqueue_execution instead.
"""
+ execution = WorkflowEngine._create_execution(workflow_id, trigger_type, context)
+ WorkflowEngine.run_execution(execution.id)
+ return execution.id
+
+ @staticmethod
+ def enqueue_execution(workflow_id: int, trigger_type: str = 'manual',
+ context: Dict[str, Any] = None) -> int:
+ """Create the execution row and run it asynchronously via the unified job
+ system (kind ``workflow.execute``). Returns the execution id immediately,
+ so triggers don't block on the DAG. Graph validation still happens here,
+ so an invalid workflow raises synchronously (callers can 400)."""
+ execution = WorkflowEngine._create_execution(workflow_id, trigger_type, context)
+ from app.jobs.service import JobService
+ JobService.enqueue(
+ WORKFLOW_JOB_KIND,
+ payload={'execution_id': execution.id},
+ max_attempts=1, # workflows aren't idempotent (deploy/script/notify)
+ owner_type='workflow',
+ owner_id=workflow_id,
+ )
+ return execution.id
+
+ @staticmethod
+ def _create_execution(workflow_id: int, trigger_type: str = 'manual',
+ context: Dict[str, Any] = None) -> WorkflowExecution:
+ """Validate the workflow graph and persist a new (running) execution row."""
workflow = Workflow.query.get(workflow_id)
if not workflow:
raise ValueError(f"Workflow {workflow_id} not found")
@@ -156,7 +186,7 @@ class WorkflowEngine:
nodes = json.loads(workflow.nodes) if workflow.nodes else []
edges = json.loads(workflow.edges) if workflow.edges else []
- # Validate graph before executing
+ # Validate graph before creating the execution
cycle_err = WorkflowEngine.validate_graph(nodes, edges)
if cycle_err:
raise CycleDetectedError(cycle_err)
@@ -173,6 +203,18 @@ class WorkflowEngine:
workflow.last_run_at = execution.started_at
db.session.commit()
+ return execution
+
+ @staticmethod
+ def run_execution(execution_id: int) -> int:
+ """Run an already-created execution to completion. The synchronous core
+ shared by execute_workflow and the workflow.execute job handler."""
+ execution = WorkflowExecution.query.get(execution_id)
+ if not execution:
+ raise ValueError(f"WorkflowExecution {execution_id} not found")
+ workflow = execution.workflow
+ nodes = json.loads(workflow.nodes) if workflow.nodes else []
+ edges = json.loads(workflow.edges) if workflow.edges else []
try:
WorkflowEngine._run_execution(execution.id, nodes, edges)
@@ -185,6 +227,28 @@ class WorkflowEngine:
return execution.id
+ @staticmethod
+ def _run_workflow_job(job):
+ """Unified-job handler for ``workflow.execute``. Runs the queued execution;
+ a failed run is raised so the unified job is marked failed too (the
+ WorkflowExecution row carries the detailed per-node status/logs)."""
+ execution_id = (job.get_payload() or {}).get('execution_id')
+ if not execution_id:
+ raise ValueError('workflow.execute job missing execution_id')
+ WorkflowEngine.run_execution(execution_id)
+ execution = WorkflowExecution.query.get(execution_id)
+ if execution and execution.status == 'failed':
+ raise RuntimeError(f'Workflow execution {execution_id} failed')
+ return {'execution_id': execution_id,
+ 'status': execution.status if execution else None}
+
+ @staticmethod
+ def register_jobs():
+ """Register the workflow.execute handler with the unified job registry.
+ Called once at app startup (see app/__init__.py)."""
+ from app.jobs import registry
+ registry.register(WORKFLOW_JOB_KIND, WorkflowEngine._run_workflow_job, replace=True)
+
# ------------------------------------------------------------------
# DAG Execution
# ------------------------------------------------------------------
@@ -925,7 +989,7 @@ class WorkflowEventBus:
'event_data': data,
'triggered_at': datetime.utcnow().isoformat()
}
- WorkflowEngine.execute_workflow(
+ WorkflowEngine.enqueue_execution(
workflow_id=workflow.id,
trigger_type='event',
context=context
diff --git a/backend/tests/test_jobs.py b/backend/tests/test_jobs.py
index 081edb75..a852c14a 100644
--- a/backend/tests/test_jobs.py
+++ b/backend/tests/test_jobs.py
@@ -299,3 +299,68 @@ class TestDeploymentInstallHandler:
assert unified.get_payload() == {'deployment_job_id': dep_id}
assert unified.max_attempts == 1
assert unified.correlation_id == dep.correlation_id
+
+
+class TestWorkflowExecutionHandler:
+ """Phase 6 — workflow executions run as a 'workflow.execute' unified job
+ instead of inline/threaded. Empty-node workflows run for real to success;
+ the failure path patches _run_execution to blow up."""
+
+ def _make_workflow(self, nodes='[]', edges='[]'):
+ from app.models.workflow import Workflow
+ wf = Workflow(name='wf-test', user_id=1, nodes=nodes, edges=edges,
+ is_active=True, trigger_type='manual')
+ db.session.add(wf)
+ db.session.commit()
+ return wf
+
+ def test_register_jobs_adds_handler(self, app):
+ from app.services.workflow_engine import WorkflowEngine, WORKFLOW_JOB_KIND
+ WorkflowEngine.register_jobs()
+ assert WORKFLOW_JOB_KIND == 'workflow.execute'
+ assert registry.is_registered('workflow.execute')
+
+ def test_enqueue_creates_running_row_and_job(self, app):
+ from app.services.workflow_engine import WorkflowEngine
+ from app.models.workflow import WorkflowExecution
+ wf = self._make_workflow()
+
+ execution_id = WorkflowEngine.enqueue_execution(wf.id, trigger_type='manual')
+
+ ex = WorkflowExecution.query.get(execution_id)
+ assert ex is not None and ex.status == 'running'
+ unified = Job.query.filter_by(kind='workflow.execute', owner_id=str(wf.id)).first()
+ assert unified is not None
+ assert unified.get_payload() == {'execution_id': execution_id}
+ assert unified.max_attempts == 1
+
+ def test_handler_runs_empty_workflow_to_success(self, app):
+ from app.services.workflow_engine import WorkflowEngine
+ from app.models.workflow import WorkflowExecution
+ WorkflowEngine.register_jobs()
+ wf = self._make_workflow(nodes='[]', edges='[]')
+
+ execution_id = WorkflowEngine.enqueue_execution(wf.id)
+ _drain_once()
+
+ assert WorkflowExecution.query.get(execution_id).status == 'success'
+ unified = Job.query.filter_by(kind='workflow.execute').first()
+ assert unified.status == Job.STATUS_SUCCEEDED
+
+ def test_handler_failure_marks_job_failed(self, app, monkeypatch):
+ from app.services.workflow_engine import WorkflowEngine
+ from app.models.workflow import WorkflowExecution
+
+ def boom(execution_id, nodes, edges):
+ raise RuntimeError('node blew up')
+
+ monkeypatch.setattr(WorkflowEngine, '_run_execution', staticmethod(boom))
+ WorkflowEngine.register_jobs()
+ wf = self._make_workflow(nodes='[{"id":"a","type":"trigger","data":{}}]', edges='[]')
+
+ execution_id = WorkflowEngine.enqueue_execution(wf.id)
+ _drain_once()
+
+ assert WorkflowExecution.query.get(execution_id).status == 'failed'
+ unified = Job.query.filter_by(kind='workflow.execute').first()
+ assert unified.status == Job.STATUS_FAILED