AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/executor): count graph runs by terminal status (#14310)
Bentlyde40a178c882
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex a1ec649ffb..946084ab0a 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -1213,10 +1213,16 @@ async def update_graph_execution_stats(f"This status can only be set at creation or is not a valid target status.")- await AgentGraphExecution.prisma().update_many(+ updated = await AgentGraphExecution.prisma().update_many(where=where_clause,data=update_data,)+ if status is not None and updated == 0:+ # The row exists but is not in a state this status may be reached+ # from (VALID_STATUS_TRANSITIONS), e.g. a second terminal write after+ # the run already finished. Nothing changed, so do not score it,+ # cascade its children, or hand back a row that suggests it did.+ return Noneif status in TERMINAL_GRAPH_EXECUTION_STATUSES:# Score the finished run for the Briefing while its stats are fresh.diff --git a/autogpt_platform/backend/backend/data/execution_cascade_test.py b/autogpt_platform/backend/backend/data/execution_cascade_test.pyindex 326a150df0..f83aa80feb 100644--- a/autogpt_platform/backend/backend/data/execution_cascade_test.py+++ b/autogpt_platform/backend/backend/data/execution_cascade_test.py@@ -156,3 +156,51 @@ async def test_cascade_records_terminal_status_in_node_error():assert captured.get("status") == ExecutionStatus.FAILEDassert "terminated" in captured.get("stats", {}).get("error", "").lower()+++async def test_rejected_terminal_transition_returns_none_and_does_nothing(+ mock_score_completed_run,+):+ """A terminal write that VALID_STATUS_TRANSITIONS filters out (update_many+ matches zero rows, e.g. the run already finished) must return None and must+ not score the run or cascade its children. Returning the unchanged row here+ made the executor count a completion that never happened."""+ with patch("backend.data.execution.AgentGraphExecution") as mock_graph, patch(+ "backend.data.execution.AgentNodeExecution"+ ) as mock_node:+ mock_graph.prisma.return_value.update_many = AsyncMock(return_value=0)+ mock_graph.prisma.return_value.find_unique_or_raise = AsyncMock()+ mock_node.prisma.return_value.update_many = AsyncMock()++ result = await update_graph_execution_stats(+ graph_exec_id="ge-1", status=ExecutionStatus.COMPLETED+ )++ assert result is None+ mock_score_completed_run.assert_not_awaited()+ mock_node.prisma.return_value.update_many.assert_not_awaited()+ mock_graph.prisma.return_value.find_unique_or_raise.assert_not_awaited()+++async def test_applied_terminal_transition_returns_row(mock_score_completed_run):+ """The one-row case is unchanged: score, cascade, return the fresh row."""+ with patch("backend.data.execution.AgentGraphExecution") as mock_graph, patch(+ "backend.data.execution.AgentNodeExecution"+ ) as mock_node, patch(+ "backend.data.execution.GraphExecution.from_db", return_value="row"+ ):+ mock_graph.prisma.return_value.update_many = AsyncMock(return_value=1)+ mock_graph.prisma.return_value.find_unique_or_raise = AsyncMock(+ return_value=MagicMock()+ )+ mock_node.prisma.return_value.update_many = AsyncMock()++ result = await update_graph_execution_stats(+ graph_exec_id="ge-1", status=ExecutionStatus.COMPLETED+ )++ assert result == "row"+ mock_score_completed_run.assert_awaited_once()+ mock_node.prisma.return_value.update_many.assert_awaited_once()diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 1879d15507..7134592466 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -51,6 +51,7 @@ from backend.integrations.codex.access import enforce_codex_accessfrom backend.integrations.credential_lease import CredentialLeasefrom backend.integrations.credentials_store import provider_matchesfrom backend.integrations.creds_manager import IntegrationCredentialsManager+from backend.monitoring.instrumentation import record_graph_run_completionfrom backend.util import jsonfrom backend.util.clients import (get_async_execution_event_bus,@@ -2114,6 +2115,17 @@ def update_node_execution_status(return exec_update+_TERMINAL_RUN_STATUSES = frozenset(+ {ExecutionStatus.COMPLETED, ExecutionStatus.FAILED, ExecutionStatus.TERMINATED}+)+++def _record_terminal_status(status: "ExecutionStatus | None") -> None:+ """Feed the run-outcome counter exactly once, at the terminal transition."""+ if status is not None and status in _TERMINAL_RUN_STATUSES:+ record_graph_run_completion(status.value)++async def async_update_graph_execution_state(db_client: "DatabaseManagerAsyncClient",graph_exec_id: str,@@ -2125,6 +2137,7 @@ async def async_update_graph_execution_state(graph_exec_id, status, stats)if graph_update:+ _record_terminal_status(status)await send_async_execution_update(graph_update)else:logger.error(f"Failed to update graph execution stats for {graph_exec_id}")@@ -2140,6 +2153,7 @@ def update_graph_execution_state("""Sets status and fetches+broadcasts the latest state of the graph execution"""graph_update = db_client.update_graph_execution_stats(graph_exec_id, status, stats)if graph_update:+ _record_terminal_status(status)send_execution_update(graph_update)else:logger.error(f"Failed to update graph execution stats for {graph_exec_id}")diff --git a/autogpt_platform/backend/backend/executor/manager_test.py b/autogpt_platform/backend/backend/executor/manager_test.pyindex 76093e5174..e30650f4fb 100644--- a/autogpt_platform/backend/backend/executor/manager_test.py+++ b/autogpt_platform/backend/backend/executor/manager_test.py@@ -570,3 +570,78 @@ async def test_store_listing_graph(server: SpinTestServer):await assert_sample_graph_executions(test_graph, alt_test_user, graph_exec_id)logger.info("Completed test_agent_execution")+++def _run_completions(status: str) -> float:+ from prometheus_client import REGISTRY++ return (+ REGISTRY.get_sample_value(+ "autogpt_graph_run_completions_total", {"status": status}+ )+ or 0.0+ )+++async def test_async_wrapper_counts_only_an_applied_terminal_transition():+ """Executor-side wrapper: a row back means the transition applied and is+ counted once; None back (VALID_STATUS_TRANSITIONS rejected it) is not+ counted and not broadcast."""+ from unittest.mock import MagicMock++ from backend.executor.manager import async_update_graph_execution_state++ db = MagicMock()+ before = _run_completions("COMPLETED")++ with patch(+ "backend.executor.manager.send_async_execution_update",+ new_callable=AsyncMock,+ ) as send:+ db.update_graph_execution_stats = AsyncMock(return_value=MagicMock())+ await async_update_graph_execution_state(+ db, "ge-1", status=execution.ExecutionStatus.COMPLETED+ )+ assert _run_completions("COMPLETED") == before + 1+ send.assert_awaited_once()++ send.reset_mock()+ db.update_graph_execution_stats = AsyncMock(return_value=None)+ await async_update_graph_execution_state(+ db, "ge-1", status=execution.ExecutionStatus.COMPLETED+ )+ assert _run_completions("COMPLETED") == before + 1+ send.assert_not_awaited()+++def test_sync_wrapper_counts_only_an_applied_terminal_transition():+ from unittest.mock import MagicMock++ from backend.executor.manager import update_graph_execution_state++ db = MagicMock()+ before = _run_completions("FAILED")++ with patch("backend.executor.manager.send_execution_update") as send:+ db.update_graph_execution_stats = MagicMock(return_value=MagicMock())+ update_graph_execution_state(+ db, "ge-1", status=execution.ExecutionStatus.FAILED+ )+ assert _run_completions("FAILED") == before + 1+ send.assert_called_once()++ send.reset_mock()+ db.update_graph_execution_stats = MagicMock(return_value=None)+ update_graph_execution_state(+ db, "ge-1", status=execution.ExecutionStatus.FAILED+ )+ assert _run_completions("FAILED") == before + 1+ send.assert_not_called()++ # Non-terminal transitions are never counted, applied or not.+ db.update_graph_execution_stats = MagicMock(return_value=MagicMock())+ update_graph_execution_state(+ db, "ge-1", status=execution.ExecutionStatus.RUNNING+ )+ assert _run_completions("FAILED") == before + 1diff --git a/autogpt_platform/backend/backend/monitoring/instrumentation.py b/autogpt_platform/backend/backend/monitoring/instrumentation.pyindex bd384b4ad2..9610d2b79e 100644--- a/autogpt_platform/backend/backend/monitoring/instrumentation.py+++ b/autogpt_platform/backend/backend/monitoring/instrumentation.py@@ -29,6 +29,16 @@ GRAPH_EXECUTIONS_BY_USER = Counter(labelnames=["status"], # Only status, user_id tracked separately when needed)+# Terminal outcome of a graph RUN, recorded by the executor when the run's+# final status is persisted. Distinct from GRAPH_EXECUTIONS above, which counts+# execute *requests* at the API and cannot see a run that starts fine and then+# fails ten nodes in. This is the series an "agents are failing" alert reads.+GRAPH_RUN_COMPLETIONS = Counter(+ "autogpt_graph_run_completions_total",+ "Graph runs reaching a terminal status, by status",+ labelnames=["status"], # COMPLETED / FAILED / TERMINATED — bounded+)+BLOCK_EXECUTIONS = Counter("autogpt_block_executions_total","Total number of block executions",@@ -230,6 +240,11 @@ def record_graph_execution(graph_id: str, status: str, user_id: str):GRAPH_EXECUTIONS_BY_USER.labels(status=status).inc()+def record_graph_run_completion(status: str):+ """Record a graph run reaching a terminal status (COMPLETED/FAILED/TERMINATED)."""+ GRAPH_RUN_COMPLETIONS.labels(status=status).inc()++def record_block_execution(block_type: str, status: str, duration: float):"""Record a block execution event with duration."""BLOCK_EXECUTIONS.labels(block_type=block_type, status=status).inc()diff --git a/autogpt_platform/backend/backend/monitoring/instrumentation_test.py b/autogpt_platform/backend/backend/monitoring/instrumentation_test.pynew file mode 100644index 0000000000..f7dfeb6817--- /dev/null+++ b/autogpt_platform/backend/backend/monitoring/instrumentation_test.py@@ -0,0 +1,22 @@+"""Tests for Prometheus instrumentation helpers."""+++def test_record_graph_run_completion_increments_by_status():+ from prometheus_client import REGISTRY++ from backend.monitoring.instrumentation import record_graph_run_completion++ def value(status: str) -> float:+ return (+ REGISTRY.get_sample_value(+ "autogpt_graph_run_completions_total", {"status": status}+ )+ or 0.0+ )++ before_ok, before_fail = value("COMPLETED"), value("FAILED")+ record_graph_run_completion("COMPLETED")+ record_graph_run_completion("FAILED")+ record_graph_run_completion("FAILED")+ assert value("COMPLETED") == before_ok + 1+ assert value("FAILED") == before_fail + 2