Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/executor): Make graph execution status transitions atomic and enforce state machine (#10863)

Zamil Majdy27fccdbf31a7
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.py
index cc9c797b22..f1d2607563 100644
--- a/autogpt_platform/backend/backend/data/execution.py
+++ b/autogpt_platform/backend/backend/data/execution.py
@@ -92,6 +92,31 @@ ExecutionStatus = AgentExecutionStatus
NodeInputMask = Mapping[str, JsonValue]
NodesInputMasks = Mapping[str, NodeInputMask]
+# dest: source
+VALID_STATUS_TRANSITIONS = {
+ ExecutionStatus.QUEUED: [
+ ExecutionStatus.INCOMPLETE,
+ ],
+ ExecutionStatus.RUNNING: [
+ ExecutionStatus.INCOMPLETE,
+ ExecutionStatus.QUEUED,
+ ExecutionStatus.TERMINATED, # For resuming halted execution
+ ],
+ ExecutionStatus.COMPLETED: [
+ ExecutionStatus.RUNNING,
+ ],
+ ExecutionStatus.FAILED: [
+ ExecutionStatus.INCOMPLETE,
+ ExecutionStatus.QUEUED,
+ ExecutionStatus.RUNNING,
+ ],
+ ExecutionStatus.TERMINATED: [
+ ExecutionStatus.INCOMPLETE,
+ ExecutionStatus.QUEUED,
+ ExecutionStatus.RUNNING,
+ ],
+}
+
class GraphExecutionMeta(BaseDbModel):
id: str # type: ignore # Override base class to make this required
@@ -580,7 +605,7 @@ async def create_graph_execution(
data={
"agentGraphId": graph_id,
"agentGraphVersion": graph_version,
- "executionStatus": ExecutionStatus.QUEUED,
+ "executionStatus": ExecutionStatus.INCOMPLETE,
"inputs": SafeJson(inputs),
"credentialInputs": (
SafeJson(credential_inputs) if credential_inputs else Json({})
@@ -727,6 +752,11 @@ async def update_graph_execution_stats(
status: ExecutionStatus | None = None,
stats: GraphExecutionStats | None = None,
) -> GraphExecution | None:
+ if not status and not stats:
+ raise ValueError(
+ f"Must provide either status or stats to update for execution {graph_exec_id}"
+ )
+
update_data: AgentGraphExecutionUpdateManyMutationInput = {}
if stats:
@@ -738,20 +768,25 @@ async def update_graph_execution_stats(
if status:
update_data["executionStatus"] = status
- updated_count = await AgentGraphExecution.prisma().update_many(
- where={
- "id": graph_exec_id,
- "OR": [
- {"executionStatus": ExecutionStatus.RUNNING},
- {"executionStatus": ExecutionStatus.QUEUED},
- # Terminated graph can be resumed.
- {"executionStatus": ExecutionStatus.TERMINATED},
- ],
- },
+ where_clause: AgentGraphExecutionWhereInput = {"id": graph_exec_id}
+
+ if status:
+ if allowed_from := VALID_STATUS_TRANSITIONS.get(status, []):
+ # Add OR clause to check if current status is one of the allowed source statuses
+ where_clause["AND"] = [
+ {"id": graph_exec_id},
+ {"OR": [{"executionStatus": s} for s in allowed_from]},
+ ]
+ else:
+ raise ValueError(
+ f"Status {status} cannot be set via update for execution {graph_exec_id}. "
+ f"This status can only be set at creation or is not a valid target status."
+ )
+
+ await AgentGraphExecution.prisma().update_many(
+ where=where_clause,
data=update_data,
)
- if updated_count == 0:
- return None
graph_exec = await AgentGraphExecution.prisma().find_unique_or_raise(
where={"id": graph_exec_id},
@@ -759,6 +794,7 @@ async def update_graph_execution_stats(
[*get_io_block_ids(), *get_webhook_block_ids()]
),
)
+
return GraphExecution.from_db(graph_exec)
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index 38377c1853..43696d7bce 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -605,7 +605,7 @@ class ExecutionProcessor:
)
return
- if exec_meta.status == ExecutionStatus.QUEUED:
+ if exec_meta.status in [ExecutionStatus.QUEUED, ExecutionStatus.INCOMPLETE]:
log_metadata.info(f"⚙️ Starting graph execution #{graph_exec.graph_exec_id}")
exec_meta.status = ExecutionStatus.RUNNING
send_execution_update(
diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.py
index b5b3cba200..7f72fe1dcc 100644
--- a/autogpt_platform/backend/backend/executor/utils.py
+++ b/autogpt_platform/backend/backend/executor/utils.py
@@ -914,29 +914,30 @@ async def add_graph_execution(
preset_id=preset_id,
)
- # Fetch user context for the graph execution
- user_context = await get_user_context(user_id)
-
- queue = await get_async_execution_queue()
graph_exec_entry = graph_exec.to_graph_execution_entry(
- user_context, compiled_nodes_input_masks
+ user_context=await get_user_context(user_id),
+ compiled_nodes_input_masks=compiled_nodes_input_masks,
)
-
logger.info(
f"Created graph execution #{graph_exec.id} for graph "
f"#{graph_id} with {len(starting_nodes_input)} starting nodes. "
f"Now publishing to execution queue."
)
- await queue.publish_message(
+ exec_queue = await get_async_execution_queue()
+ await exec_queue.publish_message(
routing_key=GRAPH_EXECUTION_ROUTING_KEY,
message=graph_exec_entry.model_dump_json(),
exchange=GRAPH_EXECUTION_EXCHANGE,
)
logger.info(f"Published execution {graph_exec.id} to RabbitMQ queue")
- bus = get_async_execution_event_bus()
- await bus.publish(graph_exec)
+ graph_exec.status = ExecutionStatus.QUEUED
+ await edb.update_graph_execution_stats(
+ graph_exec_id=graph_exec.id,
+ status=graph_exec.status,
+ )
+ await get_async_execution_event_bus().publish(graph_exec)
return graph_exec
except BaseException as e:
diff --git a/autogpt_platform/backend/backend/executor/utils_test.py b/autogpt_platform/backend/backend/executor/utils_test.py
index c36208c045..bcaa3289e0 100644
--- a/autogpt_platform/backend/backend/executor/utils_test.py
+++ b/autogpt_platform/backend/backend/executor/utils_test.py
@@ -316,6 +316,7 @@ async def test_add_graph_execution_is_repeatable(mocker: MockerFixture):
# Mock the graph execution object
mock_graph_exec = mocker.MagicMock(spec=GraphExecutionWithNodes)
mock_graph_exec.id = "execution-id-123"
+ mock_graph_exec.node_executions = [] # Add this to avoid AttributeError
mock_graph_exec.to_graph_execution_entry.return_value = mocker.MagicMock()
# Mock user context
@@ -346,6 +347,10 @@ async def test_add_graph_execution_is_repeatable(mocker: MockerFixture):
)
mock_prisma.is_connected.return_value = True
mock_edb.create_graph_execution = mocker.AsyncMock(return_value=mock_graph_exec)
+ mock_edb.update_graph_execution_stats = mocker.AsyncMock(
+ return_value=mock_graph_exec
+ )
+ mock_edb.update_node_execution_status_batch = mocker.AsyncMock()
mock_get_user_context.return_value = mock_user_context
mock_get_queue.return_value = mock_queue
mock_get_event_bus.return_value = mock_event_bus