Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/executor): charge orchestrator tool calls before they run (#15631)

qxyuan853ce0a20cecb5f
diff --git a/autogpt_platform/backend/backend/blocks/orchestrator.py b/autogpt_platform/backend/backend/blocks/orchestrator.py
index 330a499f67..108995862f 100644
--- a/autogpt_platform/backend/backend/blocks/orchestrator.py
+++ b/autogpt_platform/backend/backend/blocks/orchestrator.py
@@ -1173,6 +1173,10 @@ class OrchestratorBlock(Block):
block_id=target_node.block_id,
inputs=final_input_data or {},
execution_context=execution_params.execution_context,
+ # Nothing has been collected for this node yet. Without an explicit
+ # baseline, reconciliation re-derives the catalogue pre-flight and
+ # refunds it (minus actual usage) although it was never charged.
+ pre_flight_charge=0,
)
# Apply node input overrides (credential masks from Library/Otto).
@@ -1191,6 +1195,34 @@ class OrchestratorBlock(Block):
# Use the execution manager to execute the tool node
try:
+ # Charge the pre-flight cost BEFORE running the tool, as the main
+ # execution queue does. Tools spawned by the orchestrator bypass
+ # that queue, so we must charge here to avoid free tool execution.
+ # charge_node_usage pins node_exec_entry.pre_flight_charge to the
+ # amount spent, so the reconciliation inside on_node_execution
+ # settles against what was actually collected: a failed tool call
+ # is refunded down to its tracked usage, never below what was paid.
+ # Skipped for dry runs.
+ if not execution_params.execution_context.dry_run:
+ try:
+ await execution_processor.charge_node_usage(node_exec_entry)
+ except InsufficientBalanceError:
+ # IBE must propagate — see OrchestratorBlock class docstring.
+ logger.warning(
+ "Insufficient balance charging for tool node %s; "
+ "agent loop will be aborted",
+ sink_node_id,
+ )
+ raise
+ except Exception:
+ # Non-billing charge failures (DB outage, network, etc.)
+ # are logged, not surfaced as a tool error. The baseline
+ # holds only what was collected (0 if the spend never
+ # landed), so reconciliation never refunds unpaid credit.
+ logger.exception(
+ "Unexpected error charging for tool node %s", sink_node_id
+ )
+
# Get NodeExecutionProgress from the execution manager's running nodes
node_exec_progress = execution_processor.running_node_execution[
sink_node_id
@@ -1243,54 +1275,6 @@ class OrchestratorBlock(Block):
node_exec_future.set_exception(exec_err)
raise
- # Charge user credits AFTER successful tool execution. Tools
- # spawned by the orchestrator bypass the main execution queue
- # (where _charge_usage is called), so we must charge here to
- # avoid free tool execution. Charging post-completion (vs.
- # pre-execution) avoids billing users for failed tool calls.
- # Skipped for dry runs.
- #
- # `error is None` intentionally excludes both Exception and
- # BaseException subclasses (e.g. CancelledError) so cancelled
- # or terminated tool runs are not billed.
- #
- # Billing errors (including non-balance exceptions) are kept
- # in a separate try/except so they are never silently swallowed
- # by the generic tool-error handler below.
- if (
- not execution_params.execution_context.dry_run
- and tool_node_stats.error is None
- ):
- # Charge the sub-block for telemetry / wallet debit. The
- # return value is intentionally discarded: on_node_execution
- # above ran the sub-block against this graph's own
- # graph_stats_pair (manager.py:659-668), so its cost already
- # lands in graph_stats.cost on the sub-block's completion.
- # Re-merging here would double-count in telemetry / UI / audit.
- try:
- await execution_processor.charge_node_usage(node_exec_entry)
- except InsufficientBalanceError:
- # IBE must propagate — see OrchestratorBlock class docstring.
- # Log the billing failure here so the discarded tool result
- # is traceable before the loop aborts.
- logger.warning(
- "Insufficient balance charging for tool node %s after "
- "successful execution; agent loop will be aborted",
- sink_node_id,
- )
- raise
- except Exception:
- # Non-billing charge failures (DB outage, network, etc.)
- # must NOT propagate to the outer except handler because
- # the tool itself succeeded. Re-raising would mark the
- # tool as failed (_is_error=True), causing the LLM to
- # retry side-effectful operations. Log and continue.
- logger.exception(
- "Unexpected error charging for tool node %s; "
- "tool execution was successful",
- sink_node_id,
- )
-
# Get outputs from database after execution completes using database manager client
node_outputs = await db_client.get_execution_outputs_by_node_exec_id(
node_exec_result.node_exec_id
diff --git a/autogpt_platform/backend/backend/blocks/test/test_orchestrator.py b/autogpt_platform/backend/backend/blocks/test/test_orchestrator.py
index 2eb27012dc..d1b3d4f754 100644
--- a/autogpt_platform/backend/backend/blocks/test/test_orchestrator.py
+++ b/autogpt_platform/backend/backend/blocks/test/test_orchestrator.py
@@ -7,8 +7,13 @@ import pytest
from backend.api.model import CreateGraph
from backend.api.rest_api import AgentServer
+from backend.blocks._base import BlockCost, BlockCostType
+from backend.blocks.llm import AITextGeneratorBlock
+from backend.data.block_cost_config import BLOCK_COSTS
from backend.data.execution import ExecutionContext
-from backend.data.model import ProviderName, User
+from backend.data.model import NodeExecutionStats, ProviderName, User
+from backend.executor import billing
+from backend.executor import utils as executor_utils
from backend.usecases.sample import create_test_graph, create_test_user
from backend.util.test import SpinTestServer, wait_execution
@@ -922,10 +927,9 @@ async def test_orchestrator_agent_mode():
mock_execution_processor.on_node_execution = AsyncMock(
return_value=mock_node_stats
)
- # Mock charge_node_usage (called after successful tool execution).
+ # Mock charge_node_usage (called before the tool executes; pins the baseline).
# Returns (cost, remaining_balance). Must be AsyncMock because it is
# an async method and is directly awaited in _execute_single_tool_with_manager.
- # Use a non-zero cost so the merge_stats branch is exercised.
mock_execution_processor.charge_node_usage = AsyncMock(return_value=(10, 990))
# Mock the get_execution_outputs_by_node_exec_id method
@@ -974,7 +978,7 @@ async def test_orchestrator_agent_mode():
# Verify charge_node_usage was actually called for the successful
# tool execution — this guards against regressions where the
- # post-execution tool charging is accidentally removed.
+ # tool charging is accidentally removed.
assert mock_execution_processor.charge_node_usage.call_count == 1
@@ -1219,3 +1223,204 @@ async def test_orchestrator_agent_falls_back_to_graph_name():
assert result["type"] == "function"
assert result["function"]["name"] == "original_agent_name" # Graph name cleaned
assert result["function"]["_sink_node_id"] == "test-agent-node-id"
+
+
+async def _run_nested_tool(
+ *,
+ events: list[str],
+ seen_baseline: list,
+ charge_side_effect,
+ dry_run: bool = False,
+ tool_error: Exception | None = None,
+ settle=None,
+) -> None:
+ """Run ``_execute_single_tool_with_manager`` against a tool node whose
+ pre-flight is 60 credits, with the charge and the node run replaced by
+ fakes that record what the executor would see: ``events`` collects
+ "charge" / "run" in call order and ``seen_baseline`` the
+ ``pre_flight_charge`` each run was started with. ``settle``, if given, is
+ awaited with the entry after the run, where the executor reconciles.
+ """
+ from backend.blocks.orchestrator import ExecutionParams, OrchestratorBlock, ToolInfo
+
+ block = OrchestratorBlock()
+ mock_db_client = AsyncMock()
+ target_node = MagicMock()
+ target_node.input_default = {"model": "gpt-5.5-pro"}
+ target_node.block_id = "1f292d4a-41a4-4977-9684-7c8d560b9f91"
+ mock_db_client.get_node.return_value = target_node
+ node_exec_result = MagicMock()
+ node_exec_result.node_exec_id = "nested-exec-id"
+ mock_db_client.upsert_execution_input.return_value = (
+ node_exec_result,
+ {"model": "gpt-5.5-pro", "prompt": "x"},
+ )
+ mock_db_client.get_execution_outputs_by_node_exec_id.return_value = {}
+
+ async def charge(entry):
+ events.append("charge")
+ return await charge_side_effect(entry)
+
+ async def run_node(node_exec, **_kwargs):
+ events.append("run")
+ seen_baseline.append(node_exec.pre_flight_charge)
+ if settle:
+ await settle(node_exec)
+ stats = MagicMock()
+ stats.error = tool_error
+ return stats
+
+ processor = MagicMock()
+ processor.nodes_input_masks = None
+ processor.running_node_execution = defaultdict(MagicMock)
+ processor.execution_stats = MagicMock()
+ processor.execution_stats_lock = threading.Lock()
+ processor.charge_node_usage = AsyncMock(side_effect=charge)
+ processor.on_node_execution = AsyncMock(side_effect=run_node)
+
+ tool_info = ToolInfo(
+ tool_call=MagicMock(id="call_1"),
+ tool_name="write_text",
+ tool_def={"function": {"name": "write_text", "_sink_node_id": "sink"}},
+ input_data={"prompt": "x"},
+ field_mapping={},
+ )
+ params = ExecutionParams(
+ user_id="u",
+ graph_id="g",
+ node_id="n",
+ graph_version=1,
+ graph_exec_id="ge",
+ node_exec_id="ne",
+ execution_context=ExecutionContext(dry_run=dry_run),
+ )
+ with patch(
+ "backend.blocks.orchestrator.get_database_manager_async_client",
+ return_value=mock_db_client,
+ ):
+ await block._execute_single_tool_with_manager(tool_info, params, processor)
+
+
+async def test_nested_tool_reconciles_against_the_charge_actually_collected():
+ """A nested tool execution is charged before it runs, and reaches
+ reconciliation with a pre-flight baseline equal to what was charged.
+
+ Kills: charging the tool only after it runs (the old flow), where a call
+ that fails before any usage is tracked is reconciled against a pre-flight
+ that was never collected. A missing baseline is caught by the zero-baseline
+ and wallet tests below.
+ """
+
+ async def charge(entry):
+ entry.pre_flight_charge = 60
+ return 60, 40
+
+ events: list[str] = []
+ seen_baseline: list = []
+ await _run_nested_tool(
+ events=events,
+ seen_baseline=seen_baseline,
+ charge_side_effect=charge,
+ tool_error=RuntimeError("tool failed before any usage was tracked"),
+ )
+
+ assert events == ["charge", "run"]
+ assert seen_baseline == [60]
+
+
+async def test_nested_tool_baseline_is_zero_when_nothing_was_charged():
+ """If the charge does not happen (dry run) or fails for a reason other than
+ the balance, the tool runs with a baseline of 0, so reconciliation can
+ never refund a pre-flight that was not collected."""
+
+ async def broken_charge(_entry):
+ raise RuntimeError("database unavailable")
+
+ async def no_charge(_entry): # pragma: no cover - must not be called
+ raise AssertionError("dry runs are not charged")
+
+ events: list[str] = []
+ seen_baseline: list = []
+ await _run_nested_tool(
+ events=events, seen_baseline=seen_baseline, charge_side_effect=broken_charge
+ )
+ assert events == ["charge", "run"]
+ assert seen_baseline == [0]
+
+ events, seen_baseline = [], []
+ await _run_nested_tool(
+ events=events,
+ seen_baseline=seen_baseline,
+ charge_side_effect=no_charge,
+ dry_run=True,
+ )
+ assert events == ["run"]
+ assert seen_baseline == [0]
+
+
+async def test_nested_tool_is_not_run_when_the_balance_cannot_cover_it():
+ """The pre-flight is charged before the tool runs, so an insufficient
+ balance stops the agent loop instead of letting the tool run unpaid."""
+ from backend.util.exceptions import InsufficientBalanceError
+
+ async def refused(_entry):
+ raise InsufficientBalanceError(
+ user_id="u", message="not enough credits", balance=0, amount=60
+ )
+
+ events: list[str] = []
+ with pytest.raises(InsufficientBalanceError):
+ await _run_nested_tool(
+ events=events, seen_baseline=[], charge_side_effect=refused
+ )
+ assert events == ["charge"]
+
+
+async def test_a_failed_tool_call_with_no_usage_leaves_the_wallet_unchanged(
+ monkeypatch,
+):
+ """Through the real charge and reconciliation: a tool call that fails
+ before tracking any usage is charged its pre-flight, then refunded exactly
+ that. The old flow charged only after a successful run, so the refund
+ landed with no charge before it and credited the wallet 25."""
+ ledger: list[int] = []
+
+ def spend(*, cost: int, **_kwargs) -> int:
+ ledger.append(cost)
+ return 1000 - sum(ledger)
+
+ sync_db = MagicMock()
+ sync_db.spend_credits.side_effect = spend
+ async_db = MagicMock()
+ async_db.spend_credits = AsyncMock(side_effect=spend)
+ monkeypatch.setattr(billing, "get_db_client", lambda: sync_db)
+ monkeypatch.setattr(billing, "get_database_manager_async_client", lambda: async_db)
+ monkeypatch.setattr(billing, "get_block", lambda _id: AITextGeneratorBlock())
+ monkeypatch.setattr(billing, "handle_low_balance", lambda *_args: None)
+ monkeypatch.setattr(executor_utils, "get_preflight_estimate", lambda _id: 25)
+ monkeypatch.setitem(
+ BLOCK_COSTS,
+ AITextGeneratorBlock,
+ [BlockCost(cost_amount=150, cost_type=BlockCostType.COST_USD)],
+ )
+
+ async def reconcile(node_exec):
+ await billing.charge_reconciled_usage(
+ node_exec=node_exec,
+ stats=NodeExecutionStats(),
+ pre_flight_charge=node_exec.pre_flight_charge,
+ )
+
+ await _run_nested_tool(
+ events=[],
+ seen_baseline=[],
+ charge_side_effect=billing.charge_node_usage,
+ tool_error=RuntimeError("tool failed before any usage was tracked"),
+ settle=reconcile,
+ )
+
+ assert ledger == [25, -25]
diff --git a/autogpt_platform/backend/backend/blocks/test/test_orchestrator_dynamic_fields.py b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_dynamic_fields.py
index f2242ea527..f7100b7cfe 100644
--- a/autogpt_platform/backend/backend/blocks/test/test_orchestrator_dynamic_fields.py
+++ b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_dynamic_fields.py
@@ -641,7 +641,7 @@ async def test_validation_errors_dont_pollute_conversation():
mock_execution_processor.on_node_execution.return_value = (
mock_node_stats
)
- # Mock charge_node_usage (called after successful tool execution).
+ # Mock charge_node_usage (called before the tool executes).
# Must be AsyncMock because it is async and is awaited in
# _execute_single_tool_with_manager — a plain MagicMock would
# return a non-awaitable tuple and TypeError out, then be
diff --git a/autogpt_platform/backend/backend/blocks/test/test_orchestrator_responses_api.py b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_responses_api.py
index 3764c63568..b0a733fda6 100644
--- a/autogpt_platform/backend/backend/blocks/test/test_orchestrator_responses_api.py
+++ b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_responses_api.py
@@ -1037,7 +1037,7 @@ async def test_agent_mode_conversation_valid_for_responses_api():
ep.execution_stats_lock = threading.Lock()
ns = MagicMock(error=None)
ep.on_node_execution = AsyncMock(return_value=ns)
- # Mock charge_node_usage (called after successful tool execution).
+ # Mock charge_node_usage (called before the tool executes).
# Must be AsyncMock because it is async and is awaited in
# _execute_single_tool_with_manager — a plain MagicMock would return a
# non-awaitable tuple and TypeError out, then be silently swallowed by
diff --git a/autogpt_platform/backend/backend/executor/billing.py b/autogpt_platform/backend/backend/executor/billing.py
index c4d6fb23fb..c95da47081 100644
--- a/autogpt_platform/backend/backend/executor/billing.py
+++ b/autogpt_platform/backend/backend/executor/billing.py
@@ -386,10 +386,15 @@ async def charge_node_usage(node_exec: NodeExecutionEntry) -> tuple[int, int]:
(``increment_execution_count``). Nested tool executions are sub-steps
of a single block run from the user's perspective and should not push
them into higher per-execution cost tiers.
+
+ Pins ``node_exec.pre_flight_charge`` to the block cost actually spent,
+ so the post-flight reconciliation settles against what the wallet paid
+ (as the main queue does), not against a re-derived catalogue estimate.
"""
def _run():
- total_cost, remaining, _ = charge_usage(node_exec, 0)
+ total_cost, remaining, block_pre_flight = charge_usage(node_exec, 0)
+ node_exec.pre_flight_charge = block_pre_flight
if total_cost > 0:
handle_low_balance(
get_db_client(), node_exec.user_id, remaining, total_cost
diff --git a/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py b/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py
index d8af0bfe5a..9f94c9c546 100644
--- a/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py
+++ b/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py
@@ -423,3 +423,57 @@ async def test_pre_flight_charge_override_pins_reconciliation_baseline(
# right assertion: with stats provided, post-flight goes through the
# COST_USD post-flight branch which doesn't touch get_preflight_estimate.
assert get_preflight.call_count == 0
+
+
+async def test_charge_node_usage_pins_the_amount_actually_charged(
+ tmp_block_costs_override,
+):
+ """Nested tool calls are charged through ``charge_node_usage``; like the
+ main queue, it must record the block cost it spent on the entry so that
+ reconciliation settles against that amount."""
+ from backend.executor.billing import charge_node_usage
+
+ tmp_block_costs_override([BlockCost(cost_amount=7, cost_type=BlockCostType.RUN)])
+ exec_entry = _node_exec(SearchTheWebBlock().id)
+ assert exec_entry.pre_flight_charge is None
+
+ db_client = MagicMock()
+ db_client.spend_credits.return_value = 93
+ with (
+ patch("backend.executor.billing.get_db_client", return_value=db_client),
+ patch("backend.executor.billing.handle_low_balance"),
+ ):
+ cost, remaining = await charge_node_usage(exec_entry)
+
+ assert (cost, remaining) == (7, 93)
+ assert exec_entry.pre_flight_charge == 7
+ assert db_client.spend_credits.call_args.kwargs["cost"] == 7
+
+
+async def test_charge_node_usage_leaves_the_baseline_alone_when_refused(
+ tmp_block_costs_override,
+):
+ """A refused charge collects nothing, so the baseline the caller set
+ beforehand (0 for a nested tool call) must survive."""
+ from backend.executor.billing import charge_node_usage
+ from backend.util.exceptions import InsufficientBalanceError
+
+ tmp_block_costs_override(
+ [BlockCost(cost_amount=100, cost_type=BlockCostType.COST_USD)]
+ )
+ exec_entry = _node_exec(SearchTheWebBlock().id)
+ exec_entry.pre_flight_charge = 0
+
+ db_client = MagicMock()
+ db_client.get_credits.return_value = 0 # empty wallet
+ with (
+ patch("backend.executor.billing.get_db_client", return_value=db_client),
+ patch("backend.executor.billing.handle_low_balance"),
+ pytest.raises(InsufficientBalanceError),
+ ):
+ await charge_node_usage(exec_entry)
+
+ assert exec_entry.pre_flight_charge == 0
+ db_client.spend_credits.assert_not_called()
diff --git a/autogpt_platform/backend/test/blocks/test_orchestrator_credential_masks.py b/autogpt_platform/backend/test/blocks/test_orchestrator_credential_masks.py
index 39b1d28671..c0f79a5e8e 100644
--- a/autogpt_platform/backend/test/blocks/test_orchestrator_credential_masks.py
+++ b/autogpt_platform/backend/test/blocks/test_orchestrator_credential_masks.py
@@ -119,7 +119,7 @@ def _setup_mocks(sink_node_id, params, masks):
mock_processor.execution_stats = GraphExecutionStats()
mock_processor.execution_stats_lock = threading.Lock()
mock_processor.on_node_execution = AsyncMock(return_value=NodeExecutionStats())
- # charge_node_usage is awaited after a successful tool run; it must be an
+ # charge_node_usage is awaited before the tool runs; it must be an
# AsyncMock or the await raises TypeError mid-flight.
mock_processor.charge_node_usage = AsyncMock(return_value=None)