Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/executor): bound LLM calls and reap orphan node_execs (#13051)

Zamil Majdy27746ba1a4b2
diff --git a/autogpt_platform/backend/backend/blocks/_base.py b/autogpt_platform/backend/backend/blocks/_base.py
index e2238bcae5..ea992d4469 100644
--- a/autogpt_platform/backend/backend/blocks/_base.py
+++ b/autogpt_platform/backend/backend/blocks/_base.py
@@ -479,8 +479,17 @@ class BlockWebhookConfig(BlockManualWebhookConfig):
# --8<-- [end:BlockWebhookConfig]
+# Default wall-clock cap on a single block-run invocation. Leaf compute blocks
+# inherit this; coordination blocks (AgentExecutor, AutoPilot) override their
+# instance attribute to None to opt out. The executor consults
+# `block.execution_timeout_seconds` and only wraps `run` in `wait_for` when
+# the value is not None.
+DEFAULT_BLOCK_EXECUTION_TIMEOUT_SECONDS: int = 30 * 60
+
+
class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):
_optimized_description: ClassVar[str | None] = None
+ execution_timeout_seconds: int | None = DEFAULT_BLOCK_EXECUTION_TIMEOUT_SECONDS
def __init__(
self,
diff --git a/autogpt_platform/backend/backend/blocks/agent.py b/autogpt_platform/backend/backend/blocks/agent.py
index 67eba1aa9c..cd49fc5f55 100644
--- a/autogpt_platform/backend/backend/blocks/agent.py
+++ b/autogpt_platform/backend/backend/blocks/agent.py
@@ -22,6 +22,11 @@ _logger = logging.getLogger(__name__)
class AgentExecutorBlock(Block):
+ # Coordination block: waits on a child graph's full execution. The child
+ # has its own per-node wall-clock caps, so applying the parent's leaf-
+ # block cap here would false-positive on legitimately long sub-agent runs.
+ execution_timeout_seconds: int | None = None
+
class Input(BlockSchemaInput):
user_id: str = SchemaField(description="User ID")
graph_id: str = SchemaField(description="Graph ID")
diff --git a/autogpt_platform/backend/backend/blocks/autopilot.py b/autogpt_platform/backend/backend/blocks/autopilot.py
index ab044181d3..9c2ef42ba4 100644
--- a/autogpt_platform/backend/backend/blocks/autopilot.py
+++ b/autogpt_platform/backend/backend/blocks/autopilot.py
@@ -93,6 +93,13 @@ class AutoPilotBlock(Block):
autopilot) and scheduled autopilot execution via the agent executor.
"""
+ # Coordination block: an autopilot run is a multi-step agentic loop that
+ # can dispatch sub-agents (recursively) and orchestrate platform tools.
+ # Applying the leaf-block wall-clock cap would false-positive on
+ # legitimately long autopilot runs; rely on per-LLM-call timeouts and
+ # downstream sub-graph caps for bounding.
+ execution_timeout_seconds: int | None = None
+
class Input(BlockSchemaInput):
"""Input schema for the AutoPilot block."""
diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py
index 6d2d04c3ed..05bca5fa62 100644
--- a/autogpt_platform/backend/backend/blocks/llm.py
+++ b/autogpt_platform/backend/backend/blocks/llm.py
@@ -1,5 +1,6 @@
# This file contains a lot of prompt block strings that would trigger "line too long"
# flake8: noqa: E501
+import asyncio
import logging
import math
import re
@@ -56,6 +57,14 @@ fmt = TextFormatter(autoescape=False)
# HTTP status codes for user-caused errors that should not be reported to Sentry.
USER_ERROR_STATUS_CODES = (401, 403, 429)
+# Hard cap on a single provider HTTP request. Healthy non-streaming Responses /
+# Messages calls finish in seconds; anything past this is almost certainly a
+# stalled socket (server keeping connection alive but starving response bytes,
+# which the SDK's read-timeout doesn't reliably detect on its own). Lower than
+# the SDK defaults (typically 600s) so retries-on-timeout don't compound into
+# multi-hour worst cases when a block makes many sequential calls.
+LLM_REQUEST_TIMEOUT_SECONDS = 120
+
LLMProviderName = Literal[
ProviderName.AIML_API,
ProviderName.ANTHROPIC,
@@ -901,6 +910,41 @@ async def llm_call(
ollama_host: str = "localhost:11434",
parallel_tool_calls=None,
compress_prompt_to_fit: bool = True,
+) -> LLMResponse:
+ """Public LLM-call entry point. Wraps the provider dispatch in a hard timeout
+ so that no single request can park an executor thread indefinitely."""
+ try:
+ return await asyncio.wait_for(
+ _llm_call(
+ credentials=credentials,
+ llm_model=llm_model,
+ prompt=prompt,
+ max_tokens=max_tokens,
+ force_json_output=force_json_output,
+ tools=tools,
+ ollama_host=ollama_host,
+ parallel_tool_calls=parallel_tool_calls,
+ compress_prompt_to_fit=compress_prompt_to_fit,
+ ),
+ timeout=LLM_REQUEST_TIMEOUT_SECONDS,
+ )
+ except asyncio.TimeoutError as e:
+ raise TimeoutError(
+ f"LLM request to {llm_model.metadata.provider}/{llm_model.value} "
+ f"exceeded {LLM_REQUEST_TIMEOUT_SECONDS}s and was cancelled."
+ ) from e
+
+
+async def _llm_call(
+ credentials: APIKeyCredentials,
+ llm_model: LlmModel,
+ prompt: list[dict],
+ max_tokens: int | None,
+ force_json_output: bool = False,
+ tools: list[dict] | None = None,
+ ollama_host: str = "localhost:11434",
+ parallel_tool_calls=None,
+ compress_prompt_to_fit: bool = True,
) -> LLMResponse:
"""
Make a call to a language model.
@@ -978,6 +1022,7 @@ async def llm_call(
),
text=text_config, # type: ignore[arg-type]
store=False,
+ timeout=LLM_REQUEST_TIMEOUT_SECONDS,
)
raw_tool_calls = extract_responses_tool_calls(response)
@@ -1052,7 +1097,7 @@ async def llm_call(
# equivalent to not including the key at all — no `tools` field is
# sent to the API in that case.
tools=an_tools,
- timeout=600,
+ timeout=LLM_REQUEST_TIMEOUT_SECONDS,
)
if sysprompt.strip():
# Wrap the system prompt in a single cacheable text block.
@@ -1128,6 +1173,7 @@ async def llm_call(
messages=prompt, # type: ignore
response_format=response_format, # type: ignore
max_tokens=max_tokens,
+ timeout=LLM_REQUEST_TIMEOUT_SECONDS,
)
if not response.choices:
raise ValueError("Groq returned empty choices in response")
@@ -1149,7 +1195,9 @@ async def llm_call(
ollama_host, trusted_hostnames=[settings.config.ollama_host]
)
- client = ollama.AsyncClient(host=ollama_host)
+ client = ollama.AsyncClient(
+ host=ollama_host, timeout=LLM_REQUEST_TIMEOUT_SECONDS
+ )
sys_messages = [p["content"] for p in prompt if p["role"] == "system"]
usr_messages = [p["content"] for p in prompt if p["role"] != "system"]
response = await client.generate(
@@ -1197,6 +1245,7 @@ async def llm_call(
if force_json_output
else openai.omit
),
+ timeout=LLM_REQUEST_TIMEOUT_SECONDS,
)
if not response.choices:
@@ -1236,6 +1285,7 @@ async def llm_call(
max_tokens=max_tokens,
tools=tools_param, # type: ignore
parallel_tool_calls=parallel_tool_calls_param,
+ timeout=LLM_REQUEST_TIMEOUT_SECONDS,
)
if not response.choices:
@@ -1254,7 +1304,7 @@ async def llm_call(
reasoning=reasoning,
)
elif provider == "aiml_api":
- client = openai.OpenAI(
+ client = openai.AsyncOpenAI(
base_url="https://api.aimlapi.com/v2",
api_key=credentials.api_key.get_secret_value(),
default_headers={
@@ -1264,10 +1314,11 @@ async def llm_call(
},
)
- completion = client.chat.completions.create(
+ completion = await client.chat.completions.create(
model=llm_model.value,
messages=prompt, # type: ignore
max_tokens=max_tokens,
+ timeout=LLM_REQUEST_TIMEOUT_SECONDS,
)
if not completion.choices:
raise ValueError("AI/ML API returned empty choices in response")
@@ -1305,6 +1356,7 @@ async def llm_call(
max_tokens=max_tokens,
tools=tools_param, # type: ignore
parallel_tool_calls=parallel_tool_calls_param,
+ timeout=LLM_REQUEST_TIMEOUT_SECONDS,
)
if not response.choices:
@@ -1664,8 +1716,15 @@ class AIStructuredResponseGeneratorBlock(AIBlockBase):
logger.warning(f"Error calling LLM: {e}")
error_feedback_message = f"Error calling LLM: {e}"
break
- else:
- logger.exception(f"Error calling LLM: {e}")
+ if isinstance(e, TimeoutError):
+ # A request that hung once will most likely hang again on
+ # retry — the underlying issue (server-side starvation,
+ # network partition, etc.) doesn't clear on a fresh socket.
+ # Skip retries to avoid the N×timeout wait cascade.
+ logger.warning(f"LLM call timed out, not retrying: {e}")
+ error_feedback_message = f"Error calling LLM: {e}"
+ break
+ logger.exception(f"Error calling LLM: {e}")
if (
"maximum context length" in str(e).lower()
or "token limit" in str(e).lower()
diff --git a/autogpt_platform/backend/backend/blocks/test/test_llm.py b/autogpt_platform/backend/backend/blocks/test/test_llm.py
index 3acc8d16ed..01bfc97012 100644
--- a/autogpt_platform/backend/backend/blocks/test/test_llm.py
+++ b/autogpt_platform/backend/backend/blocks/test/test_llm.py
@@ -1,3 +1,4 @@
+import asyncio
from typing import Any, cast
from unittest.mock import AsyncMock, MagicMock, patch
@@ -1456,3 +1457,94 @@ class TestAnthropicCacheControl:
assert (
"system" not in captured_kwargs
), "whitespace-only sysprompt must be omitted to avoid Anthropic 400"
+
+
+class TestLLMRequestTimeout:
+ """Test that llm_call enforces a hard request timeout regardless of provider."""
+
+ @pytest.mark.asyncio
+ async def test_llm_call_times_out_when_provider_hangs(self, monkeypatch):
+ """A hanging provider call must not park the caller indefinitely."""
+ # Force a tiny timeout so the test runs quickly.
+ monkeypatch.setattr(llm, "LLM_REQUEST_TIMEOUT_SECONDS", 0.2)
+
+ async def hang_forever(*args, **kwargs):
+ await asyncio.sleep(60)
+
+ with patch("openai.AsyncOpenAI") as mock_openai:
+ mock_client = AsyncMock()
+ mock_openai.return_value = mock_client
+ mock_client.responses.create = AsyncMock(side_effect=hang_forever)
+
+ with pytest.raises(TimeoutError) as exc_info:
+ await llm.llm_call(
+ credentials=llm.TEST_CREDENTIALS,
+ llm_model=llm.DEFAULT_LLM_MODEL,
+ prompt=[{"role": "user", "content": "Hello"}],
+ max_tokens=100,
+ )
+
+ assert "0.2s" in str(exc_info.value) or "exceeded" in str(exc_info.value)
+
+ @pytest.mark.asyncio
+ async def test_llm_call_passes_timeout_to_openai_sdk(self):
+ """The OpenAI SDK call must receive a `timeout=` argument so it errors
+ out on its own without waiting for our outer wait_for to fire."""
+ captured_kwargs: dict[str, Any] = {}
+
+ mock_response = MagicMock()
+ mock_response.output_text = "ok"
+ mock_response.output = []
+ mock_response.usage = MagicMock(input_tokens=1, output_tokens=1)
+
+ async def capture(*args, **kwargs):
+ captured_kwargs.update(kwargs)
+ return mock_response
+
+ with patch("openai.AsyncOpenAI") as mock_openai:
+ mock_client = AsyncMock()
+ mock_openai.return_value = mock_client
+ mock_client.responses.create = AsyncMock(side_effect=capture)
+
+ await llm.llm_call(
+ credentials=llm.TEST_CREDENTIALS,
+ llm_model=llm.DEFAULT_LLM_MODEL,
+ prompt=[{"role": "user", "content": "Hello"}],
+ max_tokens=100,
+ )
+
+ assert captured_kwargs.get("timeout") == llm.LLM_REQUEST_TIMEOUT_SECONDS
+
+ @pytest.mark.asyncio
+ async def test_structured_block_does_not_retry_on_timeout(self, monkeypatch):
+ """A timed-out llm_call must not be retried — retrying a hung request
+ wastes another full timeout window per attempt."""
+ monkeypatch.setattr(llm, "LLM_REQUEST_TIMEOUT_SECONDS", 0.05)
+
+ call_count = {"n": 0}
+
+ async def always_timeout(*args, **kwargs):
+ call_count["n"] += 1
+ await asyncio.sleep(60)
+
+ with patch("openai.AsyncOpenAI") as mock_openai:
+ mock_client = AsyncMock()
+ mock_openai.return_value = mock_client
+ mock_client.responses.create = AsyncMock(side_effect=always_timeout)
+
+ block = llm.AIStructuredResponseGeneratorBlock()
+ input_data = llm.AIStructuredResponseGeneratorBlock.Input(
+ prompt="Hello",
+ expected_format={"key": "value"},
+ model=llm.DEFAULT_LLM_MODEL,
+ credentials=llm.TEST_CREDENTIALS_INPUT, # type: ignore
+ retry=5, # would be 5 attempts × timeout if retry kicked in
+ )
+
+ with pytest.raises(RuntimeError):
+ async for _ in block.run(input_data, credentials=llm.TEST_CREDENTIALS):
+ pass
+
+ assert (
+ call_count["n"] == 1
+ ), f"Expected exactly 1 call (no retry on timeout), got {call_count['n']}"
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.py
index 01c33504d7..846928781d 100644
--- a/autogpt_platform/backend/backend/data/execution.py
+++ b/autogpt_platform/backend/backend/data/execution.py
@@ -992,11 +992,33 @@ async def update_graph_execution_start_time(
return GraphExecution.from_db(res) if res else None
+TERMINAL_GRAPH_EXECUTION_STATUSES = (
+ ExecutionStatus.COMPLETED,
+ ExecutionStatus.FAILED,
+ ExecutionStatus.TERMINATED,
+)
+
+
async def update_graph_execution_stats(
graph_exec_id: str,
status: ExecutionStatus | None = None,
stats: GraphExecutionStats | None = None,
+ cascade_running_children: bool = True,
) -> GraphExecution | None:
+ """Update a graph_exec's status and/or stats.
+
+ When `status` transitions the row into a terminal state and
+ `cascade_running_children` is True (default), all of its child node
+ executions still in `RUNNING` are batch-updated to `FAILED`. This
+ keeps the invariant "if parent is terminal, no child is RUNNING"
+ in a single transaction-adjacent pair of writes, so callers don't
+ need to remember to clean up node_execs after marking a graph
+ terminal.
+
+ Set `cascade_running_children=False` only if you have a specific
+ reason to leave child rows untouched (e.g. resume flows or
+ speculative writes that will be reconciled separately).
+ """
if not status and not stats:
raise ValueError(
f"Must provide either status or stats to update for execution {graph_exec_id}"
@@ -1013,12 +1035,7 @@ async def update_graph_execution_stats(
if status:
update_data["executionStatus"] = status
# Set endedAt when execution reaches a terminal status
- terminal_statuses = [
- ExecutionStatus.COMPLETED,
- ExecutionStatus.FAILED,
- ExecutionStatus.TERMINATED,
- ]
- if status in terminal_statuses:
+ if status in TERMINAL_GRAPH_EXECUTION_STATUSES:
update_data["endedAt"] = datetime.now(tz=timezone.utc)
where_clause: AgentGraphExecutionWhereInput = {"id": graph_exec_id}
@@ -1041,6 +1058,23 @@ async def update_graph_execution_stats(
data=update_data,
)
+ if cascade_running_children and status in TERMINAL_GRAPH_EXECUTION_STATUSES:
+ # Sweep any child node_execs that are still RUNNING. Without this,
+ # an in-flight node task whose asyncio cancel didn't propagate
+ # (e.g. provider-SDK stuck in a write) would leave a ghost RUNNING
+ # row long after the graph itself was finalized.
+ await AgentNodeExecution.prisma().update_many(
+ where={
+ "agentGraphExecutionId": graph_exec_id,
+ "executionStatus": ExecutionStatus.RUNNING.value,
+ },
+ data=_get_update_status_data(
+ ExecutionStatus.FAILED,
+ None,
+ {"error": f"graph_execution_{status.value.lower()}"},
+ ),
+ )
+
graph_exec = await AgentGraphExecution.prisma().find_unique_or_raise(
where={"id": graph_exec_id},
include=graph_execution_include(
diff --git a/autogpt_platform/backend/backend/data/execution_cascade_test.py b/autogpt_platform/backend/backend/data/execution_cascade_test.py
new file mode 100644
index 0000000000..ee52c17866
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/execution_cascade_test.py
@@ -0,0 +1,149 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from backend.blocks._base import DEFAULT_BLOCK_EXECUTION_TIMEOUT_SECONDS
+from backend.blocks.agent import AgentExecutorBlock
+from backend.blocks.autopilot import AutoPilotBlock
+from backend.data.execution import ExecutionStatus, update_graph_execution_stats
+
+
+def test_default_block_has_execution_timeout():
+ """Leaf compute blocks inherit the default wall-clock cap."""
+
+ from backend.blocks.basic import StoreValueBlock
+
+ assert (
+ StoreValueBlock().execution_timeout_seconds
+ == DEFAULT_BLOCK_EXECUTION_TIMEOUT_SECONDS
+ )
+
+
+def test_coordination_blocks_opt_out_of_timeout():
+ """AgentExecutorBlock and AutoPilotBlock must not have a wall-clock cap;
+ they wait on sub-graphs that have their own per-node timeouts."""
+ assert AgentExecutorBlock().execution_timeout_seconds is None
+ assert AutoPilotBlock().execution_timeout_seconds is None
+
+
+ "terminal_status",
+ [
+ ExecutionStatus.FAILED,
+ ExecutionStatus.COMPLETED,
+ ExecutionStatus.TERMINATED,
+ ],
+)
+async def test_terminal_transition_cascades_running_children(terminal_status):
+ """When a graph_exec moves to a terminal state, its still-RUNNING child
+ node_execs must be batch-updated to FAILED in the same call so we never
+ leak the 'parent terminal, child running' invariant."""
+ with patch("backend.data.execution.AgentGraphExecution") as mock_graph, patch(
+ "backend.data.execution.AgentNodeExecution"
+ ) as mock_node:
+ mock_graph.prisma.return_value.update_many = AsyncMock()
+ mock_graph.prisma.return_value.find_unique_or_raise = AsyncMock(
+ return_value=MagicMock()
+ )
+ mock_node.prisma.return_value.update_many = AsyncMock()
+
+ with patch(
+ "backend.data.execution.GraphExecution.from_db",
+ return_value=MagicMock(),
+ ):
+ await update_graph_execution_stats(
+ graph_exec_id="ge-1",
+ status=terminal_status,
+ )
+
+ mock_node.prisma.return_value.update_many.assert_awaited_once()
+ where = mock_node.prisma.return_value.update_many.await_args.kwargs["where"]
+ assert where["agentGraphExecutionId"] == "ge-1"
+ assert where["executionStatus"] == ExecutionStatus.RUNNING.value
+
+
+async def test_non_terminal_transition_does_not_cascade():
+ """Mid-flight status changes (RUNNING/QUEUED/REVIEW) must leave child rows alone."""
+ with patch("backend.data.execution.AgentGraphExecution") as mock_graph, patch(
+ "backend.data.execution.AgentNodeExecution"
+ ) as mock_node:
+ mock_graph.prisma.return_value.update_many = AsyncMock()
+ mock_graph.prisma.return_value.find_unique_or_raise = AsyncMock(
+ return_value=MagicMock()
+ )
+ mock_node.prisma.return_value.update_many = AsyncMock()
+
+ with patch(
+ "backend.data.execution.GraphExecution.from_db",
+ return_value=MagicMock(),
+ ):
+ await update_graph_execution_stats(
+ graph_exec_id="ge-1",
+ status=ExecutionStatus.RUNNING,
+ )
+
+ mock_node.prisma.return_value.update_many.assert_not_awaited()
+
+
+async def test_cascade_can_be_disabled_explicitly():
+ """`cascade_running_children=False` is the escape hatch for callers that
+ need to mark a graph terminal without touching children (e.g. resume flows)."""
+ with patch("backend.data.execution.AgentGraphExecution") as mock_graph, patch(
+ "backend.data.execution.AgentNodeExecution"
+ ) as mock_node:
+ mock_graph.prisma.return_value.update_many = AsyncMock()
+ mock_graph.prisma.return_value.find_unique_or_raise = AsyncMock(
+ return_value=MagicMock()
+ )
+ mock_node.prisma.return_value.update_many = AsyncMock()
+
+ with patch(
+ "backend.data.execution.GraphExecution.from_db",
+ return_value=MagicMock(),
+ ):
+ await update_graph_execution_stats(
+ graph_exec_id="ge-1",
+ status=ExecutionStatus.FAILED,
+ cascade_running_children=False,
+ )
+
+ mock_node.prisma.return_value.update_many.assert_not_awaited()
+
+
+async def test_cascade_records_terminal_status_in_node_error():
+ """The child error stamp should reference which terminal status caused it
+ so we can tell deploy-time cancellations from billing failures from manual stops."""
+ captured: dict = {}
+
+ def fake_get_update_status_data(status, execution_data, stats):
+ captured["status"] = status
+ captured["stats"] = stats
+ return {"executionStatus": status}
+
+ with patch("backend.data.execution.AgentGraphExecution") as mock_graph, patch(
+ "backend.data.execution.AgentNodeExecution"
+ ) as mock_node, patch(
+ "backend.data.execution._get_update_status_data",
+ side_effect=fake_get_update_status_data,
+ ):
+ mock_graph.prisma.return_value.update_many = AsyncMock()
+ mock_graph.prisma.return_value.find_unique_or_raise = AsyncMock(
+ return_value=MagicMock()
+ )
+ mock_node.prisma.return_value.update_many = AsyncMock()
+
+ with patch(
+ "backend.data.execution.GraphExecution.from_db",
+ return_value=MagicMock(),
+ ):
+ await update_graph_execution_stats(
+ graph_exec_id="ge-1",
+ status=ExecutionStatus.TERMINATED,
+ )
+
+ assert captured.get("status") == ExecutionStatus.FAILED
+ assert "terminated" in captured.get("stats", {}).get("error", "").lower()
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index e9e8ad114d..dec45316f8 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -615,7 +615,15 @@ class ExecutionProcessor:
# blocks produce a zero delta; dynamic types (SECOND/ITEMS/COST_USD/
# TOKENS) settle their post-flight charge or refund here. Dry runs
# skip reconciliation so simulation never touches the user's wallet.
- if status == ExecutionStatus.COMPLETED:
+ # Reconcile on FAILED / TERMINATED too — partial work consumed real
+ # provider tokens, and the pre-flight charge should be refunded down
+ # to the actually-tracked usage rather than being absorbed wholesale
+ # by the user.
+ if status in (
+ ExecutionStatus.COMPLETED,
+ ExecutionStatus.FAILED,
+ ExecutionStatus.TERMINATED,
+ ):
await log_system_credential_cost(
node_exec=node_exec,
block=node.block,
@@ -706,14 +714,7 @@ class ExecutionProcessor:
)
)
- try:
- log_metadata.info(f"Start node execution {node_exec.node_exec_id}")
- await async_update_node_execution_status(
- db_client=db_client,
- exec_id=node_exec.node_exec_id,
- status=ExecutionStatus.RUNNING,
- )
-
+ async def _drive_execution() -> None:
async for output_name, output_data in execute_node(
node=node,
data=node_exec,
@@ -724,9 +725,39 @@ class ExecutionProcessor:
):
await persist_output(output_name, output_data)
+ try:
+ log_metadata.info(f"Start node execution {node_exec.node_exec_id}")
+ await async_update_node_execution_status(
+ db_client=db_client,
+ exec_id=node_exec.node_exec_id,
+ status=ExecutionStatus.RUNNING,
+ )
+
+ # Per-block wall-clock cap on `run`. Leaf compute blocks inherit
+ # the default cap; coordination blocks (AgentExecutor, AutoPilot)
+ # opt out by overriding `execution_timeout_seconds = None`. Their
+ # sub-graphs and inner LLM calls have their own bounds, so the
+ # outer cap would false-positive on legitimately long runs.
+ block_timeout = node.block.execution_timeout_seconds
+ if block_timeout is None:
+ await _drive_execution()
+ else:
+ await asyncio.wait_for(_drive_execution(), timeout=block_timeout)
+
log_metadata.info(f"Finished node execution {node_exec.node_exec_id}")
status = ExecutionStatus.COMPLETED
+ except asyncio.TimeoutError:
+ block_timeout = node.block.execution_timeout_seconds
+ stats.error = TimeoutError(
+ f"Node execution exceeded {block_timeout}s wall-clock cap"
+ )
+ log_metadata.warning(
+ f"Node execution {node_exec.node_exec_id} timed out after "
+ f"{block_timeout}s — marking FAILED"
+ )
+ status = ExecutionStatus.FAILED
+
except BaseException as e:
stats.error = e