AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/executor): finish skipped nodes instead of leaving them QUEUED (#14921)
Nicholas Tindle077d7c7cccdc
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex 0700a673c3..5ac703794c 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -1363,6 +1363,13 @@ async def update_node_execution_status(if res := await AgentNodeExecution.prisma().find_unique(where={"id": node_exec_id}, include=EXECUTION_RESULT_INCLUDE):+ if res.executionStatus != status:+ # VALID_STATUS_TRANSITIONS rejected the write. Say so: callers get+ # the unchanged row back and can't tell the update didn't happen.+ logger.warning(+ f"Node execution #{node_exec_id} can't go from "+ f"{res.executionStatus} to {status}; it stays {res.executionStatus}"+ )return NodeExecutionResult.from_db(res)raise ValueError(f"Execution {node_exec_id} not found.")diff --git a/autogpt_platform/backend/backend/data/execution_node_status_test.py b/autogpt_platform/backend/backend/data/execution_node_status_test.pynew file mode 100644index 0000000000..019caa0673--- /dev/null+++ b/autogpt_platform/backend/backend/data/execution_node_status_test.py@@ -0,0 +1,47 @@+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.data.execution import ExecutionStatus, update_node_execution_status+++async def _write_status(+ current: ExecutionStatus, requested: ExecutionStatus+) -> MagicMock:+ """Write `requested` to a node execution that is `current`, with the update+ matching no row (VALID_STATUS_TRANSITIONS doesn't allow it), and return the+ module logger's mock."""+ row = MagicMock(executionStatus=current)+ with patch("backend.data.execution.AgentNodeExecution") as mock_node, patch(+ "backend.data.execution.NodeExecutionResult.from_db",+ side_effect=lambda db_row: db_row,+ ), patch("backend.data.execution.logger") as mock_logger:+ mock_node.prisma.return_value.update = AsyncMock(return_value=None)+ mock_node.prisma.return_value.find_unique = AsyncMock(return_value=row)+ result = await update_node_execution_status("node-exec-1", requested)+ assert result is row+ return mock_logger+++async def test_a_rejected_node_transition_is_logged():+ """A write that VALID_STATUS_TRANSITIONS rejects leaves the row as it was,+ and the caller gets that row back without knowing. Skipped nodes were+ written straight from QUEUED to COMPLETED this way and sat QUEUED unnoticed,+ so a rejection must at least be logged."""+ mock_logger = await _write_status(ExecutionStatus.QUEUED, ExecutionStatus.COMPLETED)++ mock_logger.warning.assert_called_once()+ message = mock_logger.warning.call_args.args[0]+ assert "node-exec-1" in message+ assert "QUEUED" in message and "COMPLETED" in message+++async def test_writing_the_status_a_node_already_has_is_not_logged():+ """Writing the status a node already has changes nothing and isn't a+ rejection worth a warning, e.g. a node resumed while RUNNING that passes+ through RUNNING again when it's skipped."""+ mock_logger = await _write_status(ExecutionStatus.RUNNING, ExecutionStatus.RUNNING)++ mock_logger.warning.assert_not_called()diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 00a3b26453..c065a93303 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -175,6 +175,12 @@ KNOWN_GRAPH_EXECUTION_ERRORS = (UserPaywalledError,)+# The statuses a node skipped for missing optional credentials goes through.+# It never runs, but COMPLETED can only be reached from RUNNING+# (VALID_STATUS_TRANSITIONS); a direct QUEUED -> COMPLETED write is rejected+# and leaves the node QUEUED after its run has finished.+SKIPPED_NODE_STATUSES = (ExecutionStatus.RUNNING, ExecutionStatus.COMPLETED)+def _propagate_node_failure(graph_stats: GraphExecutionStats,@@ -1241,11 +1247,12 @@ class ExecutionProcessor:)# Mark the node as completed without executing# No outputs will be produced, so downstream nodes won't trigger- update_node_execution_status(- db_client=db_client,- exec_id=queued_node_exec.node_exec_id,- status=ExecutionStatus.COMPLETED,- )+ for status in SKIPPED_NODE_STATUSES:+ update_node_execution_status(+ db_client=db_client,+ exec_id=queued_node_exec.node_exec_id,+ status=status,+ )continuelog_metadata.debug(diff --git a/autogpt_platform/backend/backend/executor/manager_skipped_node_test.py b/autogpt_platform/backend/backend/executor/manager_skipped_node_test.pynew file mode 100644index 0000000000..817b1a05e8--- /dev/null+++ b/autogpt_platform/backend/backend/executor/manager_skipped_node_test.py@@ -0,0 +1,28 @@+import pytest++from backend.data.execution import VALID_STATUS_TRANSITIONS, ExecutionStatus+from backend.executor.manager import SKIPPED_NODE_STATUSES+++ "start",+ [+ # The executor queues every node execution in one of these statuses.+ ExecutionStatus.QUEUED,+ ExecutionStatus.RUNNING,+ ExecutionStatus.TERMINATED,+ ExecutionStatus.REVIEW,+ ],+)+def test_a_skipped_node_reaches_completed(start: ExecutionStatus):+ """The executor completes a node skipped for missing optional credentials+ by writing SKIPPED_NODE_STATUSES in order. Each write must be one that+ VALID_STATUS_TRANSITIONS accepts, or a no-op because the node already has+ that status. Writing COMPLETED straight from QUEUED was rejected, so the+ node stayed QUEUED after its run had finished."""+ status = start+ for next_status in SKIPPED_NODE_STATUSES:+ assert next_status == status or status in VALID_STATUS_TRANSITIONS[next_status]+ status = next_status++ assert status == ExecutionStatus.COMPLETED