AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/executor): Make graph execution status transitions atomic and enforce state machine (#10863)
Zamil Majdy27fccdbf31a7
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex cc9c797b22..f1d2607563 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -92,6 +92,31 @@ ExecutionStatus = AgentExecutionStatusNodeInputMask = Mapping[str, JsonValue]NodesInputMasks = Mapping[str, NodeInputMask]+# dest: source+VALID_STATUS_TRANSITIONS = {+ ExecutionStatus.QUEUED: [+ ExecutionStatus.INCOMPLETE,+ ],+ ExecutionStatus.RUNNING: [+ ExecutionStatus.INCOMPLETE,+ ExecutionStatus.QUEUED,+ ExecutionStatus.TERMINATED, # For resuming halted execution+ ],+ ExecutionStatus.COMPLETED: [+ ExecutionStatus.RUNNING,+ ],+ ExecutionStatus.FAILED: [+ ExecutionStatus.INCOMPLETE,+ ExecutionStatus.QUEUED,+ ExecutionStatus.RUNNING,+ ],+ ExecutionStatus.TERMINATED: [+ ExecutionStatus.INCOMPLETE,+ ExecutionStatus.QUEUED,+ ExecutionStatus.RUNNING,+ ],+}+class GraphExecutionMeta(BaseDbModel):id: str # type: ignore # Override base class to make this required@@ -580,7 +605,7 @@ async def create_graph_execution(data={"agentGraphId": graph_id,"agentGraphVersion": graph_version,- "executionStatus": ExecutionStatus.QUEUED,+ "executionStatus": ExecutionStatus.INCOMPLETE,"inputs": SafeJson(inputs),"credentialInputs": (SafeJson(credential_inputs) if credential_inputs else Json({})@@ -727,6 +752,11 @@ async def update_graph_execution_stats(status: ExecutionStatus | None = None,stats: GraphExecutionStats | None = None,) -> GraphExecution | None:+ if not status and not stats:+ raise ValueError(+ f"Must provide either status or stats to update for execution {graph_exec_id}"+ )+update_data: AgentGraphExecutionUpdateManyMutationInput = {}if stats:@@ -738,20 +768,25 @@ async def update_graph_execution_stats(if status:update_data["executionStatus"] = status- updated_count = await AgentGraphExecution.prisma().update_many(- where={- "id": graph_exec_id,- "OR": [- {"executionStatus": ExecutionStatus.RUNNING},- {"executionStatus": ExecutionStatus.QUEUED},- # Terminated graph can be resumed.- {"executionStatus": ExecutionStatus.TERMINATED},- ],- },+ where_clause: AgentGraphExecutionWhereInput = {"id": graph_exec_id}++ if status:+ if allowed_from := VALID_STATUS_TRANSITIONS.get(status, []):+ # Add OR clause to check if current status is one of the allowed source statuses+ where_clause["AND"] = [+ {"id": graph_exec_id},+ {"OR": [{"executionStatus": s} for s in allowed_from]},+ ]+ else:+ raise ValueError(+ f"Status {status} cannot be set via update for execution {graph_exec_id}. "+ f"This status can only be set at creation or is not a valid target status."+ )++ await AgentGraphExecution.prisma().update_many(+ where=where_clause,data=update_data,)- if updated_count == 0:- return Nonegraph_exec = await AgentGraphExecution.prisma().find_unique_or_raise(where={"id": graph_exec_id},@@ -759,6 +794,7 @@ async def update_graph_execution_stats([*get_io_block_ids(), *get_webhook_block_ids()]),)+return GraphExecution.from_db(graph_exec)diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 38377c1853..43696d7bce 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -605,7 +605,7 @@ class ExecutionProcessor:)return- if exec_meta.status == ExecutionStatus.QUEUED:+ if exec_meta.status in [ExecutionStatus.QUEUED, ExecutionStatus.INCOMPLETE]:log_metadata.info(f"⚙️ Starting graph execution #{graph_exec.graph_exec_id}")exec_meta.status = ExecutionStatus.RUNNINGsend_execution_update(diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex b5b3cba200..7f72fe1dcc 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -914,29 +914,30 @@ async def add_graph_execution(preset_id=preset_id,)- # Fetch user context for the graph execution- user_context = await get_user_context(user_id)-- queue = await get_async_execution_queue()graph_exec_entry = graph_exec.to_graph_execution_entry(- user_context, compiled_nodes_input_masks+ user_context=await get_user_context(user_id),+ compiled_nodes_input_masks=compiled_nodes_input_masks,)-logger.info(f"Created graph execution #{graph_exec.id} for graph "f"#{graph_id} with {len(starting_nodes_input)} starting nodes. "f"Now publishing to execution queue.")- await queue.publish_message(+ exec_queue = await get_async_execution_queue()+ await exec_queue.publish_message(routing_key=GRAPH_EXECUTION_ROUTING_KEY,message=graph_exec_entry.model_dump_json(),exchange=GRAPH_EXECUTION_EXCHANGE,)logger.info(f"Published execution {graph_exec.id} to RabbitMQ queue")- bus = get_async_execution_event_bus()- await bus.publish(graph_exec)+ graph_exec.status = ExecutionStatus.QUEUED+ await edb.update_graph_execution_stats(+ graph_exec_id=graph_exec.id,+ status=graph_exec.status,+ )+ await get_async_execution_event_bus().publish(graph_exec)return graph_execexcept BaseException as e:diff --git a/autogpt_platform/backend/backend/executor/utils_test.py b/autogpt_platform/backend/backend/executor/utils_test.pyindex c36208c045..bcaa3289e0 100644--- a/autogpt_platform/backend/backend/executor/utils_test.py+++ b/autogpt_platform/backend/backend/executor/utils_test.py@@ -316,6 +316,7 @@ async def test_add_graph_execution_is_repeatable(mocker: MockerFixture):# Mock the graph execution objectmock_graph_exec = mocker.MagicMock(spec=GraphExecutionWithNodes)mock_graph_exec.id = "execution-id-123"+ mock_graph_exec.node_executions = [] # Add this to avoid AttributeErrormock_graph_exec.to_graph_execution_entry.return_value = mocker.MagicMock()# Mock user context@@ -346,6 +347,10 @@ async def test_add_graph_execution_is_repeatable(mocker: MockerFixture):)mock_prisma.is_connected.return_value = Truemock_edb.create_graph_execution = mocker.AsyncMock(return_value=mock_graph_exec)+ mock_edb.update_graph_execution_stats = mocker.AsyncMock(+ return_value=mock_graph_exec+ )+ mock_edb.update_node_execution_status_batch = mocker.AsyncMock()mock_get_user_context.return_value = mock_user_contextmock_get_queue.return_value = mock_queuemock_get_event_bus.return_value = mock_event_bus