AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/executor): charge per LLM iteration and per tool call in OrchestratorBlock (#12735)
Zamil Majdy8a2e2365f77a
diff --git a/autogpt_platform/backend/backend/blocks/_base.py b/autogpt_platform/backend/backend/blocks/_base.pyindex 56986d15c4..2a26421c91 100644--- a/autogpt_platform/backend/backend/blocks/_base.py+++ b/autogpt_platform/backend/backend/blocks/_base.py@@ -25,6 +25,7 @@ from backend.data.model import (Credentials,CredentialsFieldInfo,CredentialsMetaInput,+ NodeExecutionStats,SchemaField,is_credentials_field_name,)@@ -43,7 +44,7 @@ logger = logging.getLogger(__name__)if TYPE_CHECKING:from backend.data.execution import ExecutionContext- from backend.data.model import ContributorDetails, NodeExecutionStats+ from backend.data.model import ContributorDetailsfrom ..data.graph import Link@@ -420,6 +421,19 @@ class BlockWebhookConfig(BlockManualWebhookConfig):class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):_optimized_description: ClassVar[str | None] = None+ def extra_runtime_cost(self, execution_stats: NodeExecutionStats) -> int:+ """Return extra runtime cost to charge after this block run completes.++ Called by the executor after a block finishes with COMPLETED status.+ The return value is the number of additional base-cost credits to+ charge beyond the single credit already collected by charge_usage+ at the start of execution. Defaults to 0 (no extra charges).++ Override in blocks (e.g. OrchestratorBlock) that make multiple LLM+ calls within one run and should be billed per call.+ """+ return 0+def __init__(self,id: str = "",@@ -455,8 +469,6 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):disabled: If the block is disabled, it will not be available for execution.static_output: Whether the output links of the block are static by default."""- from backend.data.model import NodeExecutionStats-self.id = idself.input_schema = input_schemaself.output_schema = output_schema@@ -474,7 +486,7 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):self.is_sensitive_action = is_sensitive_action# Read from ClassVar set by initialize_blocks()self.optimized_description: str | None = type(self)._optimized_description- self.execution_stats: "NodeExecutionStats" = NodeExecutionStats()+ self.execution_stats: NodeExecutionStats = NodeExecutionStats()if self.webhook_config:if isinstance(self.webhook_config, BlockWebhookConfig):@@ -554,7 +566,7 @@ class Block(ABC, Generic[BlockSchemaInputType, BlockSchemaOutputType]):return dataraise ValueError(f"{self.name} did not produce any output for {output}")- def merge_stats(self, stats: "NodeExecutionStats") -> "NodeExecutionStats":+ def merge_stats(self, stats: NodeExecutionStats) -> NodeExecutionStats:self.execution_stats += statsreturn self.execution_statsdiff --git a/autogpt_platform/backend/backend/blocks/orchestrator.py b/autogpt_platform/backend/backend/blocks/orchestrator.pyindex 6fbff643fb..b2a6df8481 100644--- a/autogpt_platform/backend/backend/blocks/orchestrator.py+++ b/autogpt_platform/backend/backend/blocks/orchestrator.py@@ -36,6 +36,7 @@ from backend.data.execution import ExecutionContextfrom backend.data.model import NodeExecutionStats, SchemaFieldfrom backend.util import jsonfrom backend.util.clients import get_database_manager_async_client+from backend.util.exceptions import InsufficientBalanceErrorfrom backend.util.prompt import MAIN_OBJECTIVE_PREFIXfrom backend.util.security import SENSITIVE_FIELD_NAMESfrom backend.util.tool_call_loop import (@@ -364,10 +365,31 @@ def _disambiguate_tool_names(tools: list[dict[str, Any]]) -> None:class OrchestratorBlock(Block):+ """A block that uses a language model to orchestrate tool calls.++ Supports both single-shot and iterative agent mode execution.++ **InsufficientBalanceError propagation contract**: ``InsufficientBalanceError``+ (IBE) must always re-raise through every ``except`` block in this class.+ Swallowing IBE would let the agent loop continue with unpaid work. Every+ exception handler that catches ``Exception`` includes an explicit IBE+ re-raise carve-out for this reason."""- A block that uses a language model to orchestrate tool calls, supporting both- single-shot and iterative agent mode execution.- """++ def extra_runtime_cost(self, execution_stats: NodeExecutionStats) -> int:+ """Charge one extra runtime cost per LLM call beyond the first.++ In agent mode each iteration makes one LLM call. The first is already+ covered by charge_usage(); this returns the number of additional+ credits so the executor can bill the remaining calls post-completion.++ SDK-mode exemption: when the block runs via _execute_tools_sdk_mode,+ the SDK manages its own conversation loop and only exposes aggregate+ usage. We hardcode llm_call_count=1 there (the SDK does not report a+ per-turn call count), so this method always returns 0 for SDK-mode+ executions. Per-iteration billing does not apply to SDK mode.+ """+ return max(0, execution_stats.llm_call_count - 1)# MCP server name used by the Claude Code SDK execution mode. Keep in sync# with _create_graph_mcp_server and the MCP_PREFIX derivation in _execute_tools_sdk_mode.@@ -1077,7 +1099,10 @@ class OrchestratorBlock(Block):input_data=input_value,)- assert node_exec_result is not None, "node_exec_result should not be None"+ if node_exec_result is None:+ raise RuntimeError(+ f"upsert_execution_input returned None for node {sink_node_id}"+ )# Create NodeExecutionEntry for execution managernode_exec_entry = NodeExecutionEntry(@@ -1112,15 +1137,86 @@ class OrchestratorBlock(Block):task=node_exec_future,)- # Execute the node directly since we're in the Orchestrator context- node_exec_future.set_result(- await execution_processor.on_node_execution(+ # Execute the node directly since we're in the Orchestrator context.+ # Wrap in try/except so the future is always resolved, even on+ # error — an unresolved Future would block anything awaiting it.+ #+ # on_node_execution is decorated with @async_error_logged(swallow=True),+ # which catches BaseException and returns None rather than raising.+ # Treat a None return as a failure: set_exception so the future+ # carries an error state rather than a None result, and return an+ # error response so the LLM knows the tool failed.+ try:+ tool_node_stats = await execution_processor.on_node_execution(node_exec=node_exec_entry,node_exec_progress=node_exec_progress,nodes_input_masks=None,graph_stats_pair=graph_stats_pair,)- )+ if tool_node_stats is None:+ nil_err = RuntimeError(+ f"on_node_execution returned None for node {sink_node_id} "+ "(error was swallowed by @async_error_logged)"+ )+ node_exec_future.set_exception(nil_err)+ resp = _create_tool_response(+ tool_call.id,+ "Tool execution returned no result",+ responses_api=responses_api,+ )+ resp["_is_error"] = True+ return resp+ node_exec_future.set_result(tool_node_stats)+ except Exception as exec_err:+ node_exec_future.set_exception(exec_err)+ raise++ # Charge user credits AFTER successful tool execution. Tools+ # spawned by the orchestrator bypass the main execution queue+ # (where _charge_usage is called), so we must charge here to+ # avoid free tool execution. Charging post-completion (vs.+ # pre-execution) avoids billing users for failed tool calls.+ # Skipped for dry runs.+ #+ # `error is None` intentionally excludes both Exception and+ # BaseException subclasses (e.g. CancelledError) so cancelled+ # or terminated tool runs are not billed.+ #+ # Billing errors (including non-balance exceptions) are kept+ # in a separate try/except so they are never silently swallowed+ # by the generic tool-error handler below.+ if (+ not execution_params.execution_context.dry_run+ and tool_node_stats.error is None+ ):+ try:+ tool_cost, _ = await execution_processor.charge_node_usage(+ node_exec_entry,+ )+ except InsufficientBalanceError:+ # IBE must propagate — see OrchestratorBlock class docstring.+ # Log the billing failure here so the discarded tool result+ # is traceable before the loop aborts.+ logger.warning(+ "Insufficient balance charging for tool node %s after "+ "successful execution; agent loop will be aborted",+ sink_node_id,+ )+ raise+ except Exception:+ # Non-billing charge failures (DB outage, network, etc.)+ # must NOT propagate to the outer except handler because+ # the tool itself succeeded. Re-raising would mark the+ # tool as failed (_is_error=True), causing the LLM to+ # retry side-effectful operations. Log and continue.+ logger.exception(+ "Unexpected error charging for tool node %s; "+ "tool execution was successful",+ sink_node_id,+ )+ tool_cost = 0+ if tool_cost > 0:+ self.merge_stats(NodeExecutionStats(extra_cost=tool_cost))# Get outputs from database after execution completes using database manager clientnode_outputs = await db_client.get_execution_outputs_by_node_exec_id(@@ -1133,18 +1229,26 @@ class OrchestratorBlock(Block):if node_outputselse "Tool executed successfully")- return _create_tool_response(+ resp = _create_tool_response(tool_call.id, tool_response_content, responses_api=responses_api)+ resp["_is_error"] = False+ return resp+ except InsufficientBalanceError:+ # IBE must propagate — see class docstring.+ raiseexcept Exception as e:- logger.warning("Tool execution with manager failed: %s", e)- # Return error response- return _create_tool_response(+ logger.warning("Tool execution with manager failed: %s", e, exc_info=True)+ # Return a generic error to the LLM — internal exception messages+ # may contain server paths, DB details, or infrastructure info.+ resp = _create_tool_response(tool_call.id,- f"Tool execution failed: {e}",+ "Tool execution failed due to an internal error",responses_api=responses_api,)+ resp["_is_error"] = True+ return respasync def _agent_mode_llm_caller(self,@@ -1244,13 +1348,16 @@ class OrchestratorBlock(Block):content = str(raw_content)else:content = "Tool executed successfully"- tool_failed = content.startswith("Tool execution failed:")+ tool_failed = result.get("_is_error", True)return ToolCallResult(tool_call_id=tool_call.id,tool_name=tool_call.name,content=content,is_error=tool_failed,)+ except InsufficientBalanceError:+ # IBE must propagate — see class docstring.+ raiseexcept Exception as e:logger.error("Tool execution failed: %s", e)return ToolCallResult(@@ -1370,9 +1477,13 @@ class OrchestratorBlock(Block):"arguments": tc.arguments,},)+ except InsufficientBalanceError:+ # IBE must propagate — see class docstring.+ raiseexcept Exception as e:- # Catch all errors (validation, network, API) so that the block- # surfaces them as user-visible output instead of crashing.+ # Catch all OTHER errors (validation, network, API) so that+ # the block surfaces them as user-visible output instead of+ # crashing.yield "error", str(e)return@@ -1450,11 +1561,14 @@ class OrchestratorBlock(Block):text = contentelse:text = json.dumps(content)- tool_failed = text.startswith("Tool execution failed:")+ tool_failed = result.get("_is_error", True)return {"content": [{"type": "text", "text": text}],"isError": tool_failed,}+ except InsufficientBalanceError:+ # IBE must propagate — see class docstring.+ raiseexcept Exception as e:logger.error("SDK tool execution failed: %s", e)return {@@ -1733,11 +1847,15 @@ class OrchestratorBlock(Block):await pending_taskexcept (asyncio.CancelledError, StopAsyncIteration):pass+ except InsufficientBalanceError:+ # IBE must propagate — see class docstring. The `finally`+ # block below still runs and records partial token usage.+ raiseexcept Exception as e:- # Surface SDK errors as user-visible output instead of crashing,- # consistent with _execute_tools_agent_mode error handling.- # Don't return yet — fall through to merge_stats below so- # partial token usage is always recorded.+ # Surface OTHER SDK errors as user-visible output instead+ # of crashing, consistent with _execute_tools_agent_mode+ # error handling. Don't return yet — fall through to+ # merge_stats below so partial token usage is always recorded.sdk_error = efinally:# Always record usage stats, even on error. The SDK may havediff --git a/autogpt_platform/backend/backend/blocks/test/test_orchestrator.py b/autogpt_platform/backend/backend/blocks/test/test_orchestrator.pyindex 55f137428f..2eb27012dc 100644--- a/autogpt_platform/backend/backend/blocks/test/test_orchestrator.py+++ b/autogpt_platform/backend/backend/blocks/test/test_orchestrator.py@@ -922,6 +922,11 @@ async def test_orchestrator_agent_mode():mock_execution_processor.on_node_execution = AsyncMock(return_value=mock_node_stats)+ # Mock charge_node_usage (called after successful tool execution).+ # Returns (cost, remaining_balance). Must be AsyncMock because it is+ # an async method and is directly awaited in _execute_single_tool_with_manager.+ # Use a non-zero cost so the merge_stats branch is exercised.+ mock_execution_processor.charge_node_usage = AsyncMock(return_value=(10, 990))# Mock the get_execution_outputs_by_node_exec_id methodmock_db_client.get_execution_outputs_by_node_exec_id.return_value = {@@ -967,6 +972,11 @@ async def test_orchestrator_agent_mode():# Verify tool was executed via execution processorassert mock_execution_processor.on_node_execution.call_count == 1+ # Verify charge_node_usage was actually called for the successful+ # tool execution — this guards against regressions where the+ # post-execution tool charging is accidentally removed.+ assert mock_execution_processor.charge_node_usage.call_count == 1+@pytest.mark.asyncioasync def test_orchestrator_traditional_mode_default():diff --git a/autogpt_platform/backend/backend/blocks/test/test_orchestrator_dynamic_fields.py b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_dynamic_fields.pyindex 1069fc8ad5..f2242ea527 100644--- a/autogpt_platform/backend/backend/blocks/test/test_orchestrator_dynamic_fields.py+++ b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_dynamic_fields.py@@ -641,6 +641,14 @@ async def test_validation_errors_dont_pollute_conversation():mock_execution_processor.on_node_execution.return_value = (mock_node_stats)+ # Mock charge_node_usage (called after successful tool execution).+ # Must be AsyncMock because it is async and is awaited in+ # _execute_single_tool_with_manager — a plain MagicMock would+ # return a non-awaitable tuple and TypeError out, then be+ # silently swallowed by the orchestrator's catch-all.+ mock_execution_processor.charge_node_usage = AsyncMock(+ return_value=(0, 0)+ )async for output_name, output_value in block.run(input_data,diff --git a/autogpt_platform/backend/backend/blocks/test/test_orchestrator_per_iteration_cost.py b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_per_iteration_cost.pynew file mode 100644index 0000000000..441bc08a42--- /dev/null+++ b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_per_iteration_cost.py@@ -0,0 +1,1020 @@+"""Tests for OrchestratorBlock per-iteration cost charging.++The OrchestratorBlock in agent mode makes multiple LLM calls in a single+node execution. The executor uses ``Block.extra_runtime_cost`` to detect+this and charge ``base_cost * (llm_call_count - 1)`` extra credits after+the block completes.+"""++import threading+from collections import defaultdict+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.blocks._base import Block+from backend.blocks.orchestrator import ExecutionParams, OrchestratorBlock+from backend.data.execution import ExecutionContext, ExecutionStatus+from backend.data.model import NodeExecutionStats+from backend.executor import billing, manager+from backend.util.exceptions import InsufficientBalanceError++# ── extra_runtime_cost hook ────────────────────────────────────────+++class _NoOpBlock(Block):+ """Minimal concrete Block subclass that does not override extra_runtime_cost."""++ def __init__(self):+ super().__init__(+ id="00000000-0000-0000-0000-000000000001", description="No-op test block"+ )++ def run(self, input_data, **kwargs): # type: ignore[override]+ yield "out", {}+++class TestExtraRuntimeCost:+ """OrchestratorBlock opts into per-LLM-call billing via extra_runtime_cost."""++ def test_orchestrator_returns_nonzero_for_multiple_calls(self):+ block = OrchestratorBlock()+ stats = NodeExecutionStats(llm_call_count=3)+ assert block.extra_runtime_cost(stats) == 2++ def test_orchestrator_returns_zero_for_single_call(self):+ block = OrchestratorBlock()+ stats = NodeExecutionStats(llm_call_count=1)+ assert block.extra_runtime_cost(stats) == 0++ def test_orchestrator_returns_zero_for_zero_calls(self):+ block = OrchestratorBlock()+ stats = NodeExecutionStats(llm_call_count=0)+ assert block.extra_runtime_cost(stats) == 0++ def test_default_block_returns_zero(self):+ """A block that does not override extra_runtime_cost returns 0."""+ block = _NoOpBlock()+ stats = NodeExecutionStats(llm_call_count=10)+ assert block.extra_runtime_cost(stats) == 0+++# ── charge_extra_runtime_cost math ───────────────────────────────────+++def fake_node_exec():+ node_exec = MagicMock()+ node_exec.user_id = "u"+ node_exec.graph_exec_id = "g"+ node_exec.graph_id = "g"+ node_exec.node_exec_id = "ne"+ node_exec.node_id = "n"+ node_exec.block_id = "b"+ node_exec.inputs = {}+ return node_exec+++def patched_processor(monkeypatch):+ """ExecutionProcessor with stubbed db client / block lookup helpers.++ Returns the processor and a list of credit amounts spent so tests can+ assert on what was charged.++ Note: ``ExecutionProcessor.__new__()`` bypasses ``__init__`` — if+ ``__init__`` gains required state in the future this fixture will need+ updating.+ """+ spent: list[int] = []++ class FakeDb:+ def spend_credits(self, *, user_id, cost, metadata):+ spent.append(cost)+ return 1000 # remaining balance++ fake_block = MagicMock()+ fake_block.name = "FakeBlock"++ monkeypatch.setattr(billing, "get_db_client", lambda: FakeDb())+ monkeypatch.setattr(billing, "get_block", lambda block_id: fake_block)+ monkeypatch.setattr(+ billing,+ "block_usage_cost",+ lambda block, input_data, **_kw: (10, {"model": "claude-sonnet-4-6"}),+ )++ proc = manager.ExecutionProcessor.__new__(manager.ExecutionProcessor)+ return proc, spent+++class TestChargeExtraRuntimeCost:+ @pytest.mark.asyncio+ async def test_zero_extra_iterations_charges_nothing(+ self, patched_processor, fake_node_exec+ ):+ proc, spent = patched_processor+ cost, balance = await proc.charge_extra_runtime_cost(+ fake_node_exec, extra_count=0+ )+ assert cost == 0+ assert balance == 0+ assert spent == []++ @pytest.mark.asyncio+ async def test_extra_iterations_multiplies_base_cost(+ self, patched_processor, fake_node_exec+ ):+ proc, spent = patched_processor+ cost, balance = await proc.charge_extra_runtime_cost(+ fake_node_exec, extra_count=4+ )+ assert cost == 40 # 4 × 10+ assert balance == 1000+ assert spent == [40]++ @pytest.mark.asyncio+ async def test_negative_extra_iterations_charges_nothing(+ self, patched_processor, fake_node_exec+ ):+ proc, spent = patched_processor+ cost, balance = await proc.charge_extra_runtime_cost(+ fake_node_exec, extra_count=-1+ )+ assert cost == 0+ assert balance == 0+ assert spent == []++ @pytest.mark.asyncio+ async def test_capped_at_max(self, monkeypatch, fake_node_exec):+ """Runaway llm_call_count is capped at _MAX_EXTRA_RUNTIME_COST."""++ spent: list[int] = []++ class FakeDb:+ def spend_credits(self, *, user_id, cost, metadata):+ spent.append(cost)+ return 1000++ fake_block = MagicMock()+ fake_block.name = "FakeBlock"++ monkeypatch.setattr(billing, "get_db_client", lambda: FakeDb())+ monkeypatch.setattr(billing, "get_block", lambda block_id: fake_block)+ monkeypatch.setattr(+ billing,+ "block_usage_cost",+ lambda block, input_data, **_kw: (10, {}),+ )++ proc = manager.ExecutionProcessor.__new__(manager.ExecutionProcessor)+ cap = billing._MAX_EXTRA_RUNTIME_COST+ cost, _ = await proc.charge_extra_runtime_cost(+ fake_node_exec, extra_count=cap * 100+ )+ # Charged at most cap × 10+ assert cost == cap * 10+ assert spent == [cap * 10]++ @pytest.mark.asyncio+ async def test_zero_base_cost_skips_charge(self, monkeypatch, fake_node_exec):++ spent: list[int] = []++ class FakeDb:+ def spend_credits(self, *, user_id, cost, metadata):+ spent.append(cost)+ return 0++ fake_block = MagicMock()+ fake_block.name = "FakeBlock"++ monkeypatch.setattr(billing, "get_db_client", lambda: FakeDb())+ monkeypatch.setattr(billing, "get_block", lambda block_id: fake_block)+ monkeypatch.setattr(+ billing, "block_usage_cost", lambda block, input_data, **_kw: (0, {})+ )++ proc = manager.ExecutionProcessor.__new__(manager.ExecutionProcessor)+ cost, balance = await proc.charge_extra_runtime_cost(+ fake_node_exec, extra_count=4+ )+ assert cost == 0+ assert balance == 0+ assert spent == []++ @pytest.mark.asyncio+ async def test_block_not_found_skips_charge(self, monkeypatch, fake_node_exec):++ spent: list[int] = []++ class FakeDb:+ def spend_credits(self, *, user_id, cost, metadata):+ spent.append(cost)+ return 0++ monkeypatch.setattr(billing, "get_db_client", lambda: FakeDb())+ monkeypatch.setattr(billing, "get_block", lambda block_id: None)+ monkeypatch.setattr(+ billing, "block_usage_cost", lambda block, input_data, **_kw: (10, {})+ )++ proc = manager.ExecutionProcessor.__new__(manager.ExecutionProcessor)+ cost, balance = await proc.charge_extra_runtime_cost(+ fake_node_exec, extra_count=3+ )+ assert cost == 0+ assert balance == 0+ assert spent == []++ @pytest.mark.asyncio+ async def test_propagates_insufficient_balance_error(+ self, monkeypatch, fake_node_exec+ ):+ """Out-of-credits errors must propagate, not be silently swallowed."""++ class FakeDb:+ def spend_credits(self, *, user_id, cost, metadata):+ raise InsufficientBalanceError(+ user_id=user_id,+ message="Insufficient balance",+ balance=0,+ amount=cost,+ )++ fake_block = MagicMock()+ fake_block.name = "FakeBlock"++ monkeypatch.setattr(billing, "get_db_client", lambda: FakeDb())+ monkeypatch.setattr(billing, "get_block", lambda block_id: fake_block)+ monkeypatch.setattr(+ billing, "block_usage_cost", lambda block, input_data, **_kw: (10, {})+ )++ proc = manager.ExecutionProcessor.__new__(manager.ExecutionProcessor)+ with pytest.raises(InsufficientBalanceError):+ await proc.charge_extra_runtime_cost(fake_node_exec, extra_count=4)+++# ── charge_node_usage ──────────────────────────────────────────────+++class TestChargeNodeUsage:+ """charge_node_usage delegates to billing.charge_usage with execution_count=0."""++ @pytest.mark.asyncio+ async def test_delegates_with_zero_execution_count(+ self, monkeypatch, fake_node_exec+ ):+ """Nested tool charges should NOT inflate the per-execution counter."""++ captured: dict = {}++ def fake_charge_usage(node_exec, execution_count):+ captured["execution_count"] = execution_count+ captured["node_exec"] = node_exec+ return (5, 100)++ def fake_handle_low_balance(+ db_client, user_id, current_balance, transaction_cost+ ):+ pass++ monkeypatch.setattr(billing, "charge_usage", fake_charge_usage)+ monkeypatch.setattr(billing, "handle_low_balance", fake_handle_low_balance)+ monkeypatch.setattr(billing, "get_db_client", lambda: MagicMock())++ proc = manager.ExecutionProcessor.__new__(manager.ExecutionProcessor)+ cost, balance = await proc.charge_node_usage(fake_node_exec)+ assert cost == 5+ assert balance == 100+ assert captured["execution_count"] == 0++ @pytest.mark.asyncio+ async def test_calls_handle_low_balance_when_cost_nonzero(+ self, monkeypatch, fake_node_exec+ ):+ """charge_node_usage should call handle_low_balance when total_cost > 0."""++ low_balance_calls: list[dict] = []++ def fake_charge_usage(node_exec, execution_count):+ return (10, 50)++ def fake_handle_low_balance(+ db_client, user_id, current_balance, transaction_cost+ ):+ low_balance_calls.append(+ {+ "user_id": user_id,+ "current_balance": current_balance,+ "transaction_cost": transaction_cost,+ }+ )++ monkeypatch.setattr(billing, "charge_usage", fake_charge_usage)+ monkeypatch.setattr(billing, "handle_low_balance", fake_handle_low_balance)+ monkeypatch.setattr(billing, "get_db_client", lambda: MagicMock())++ proc = manager.ExecutionProcessor.__new__(manager.ExecutionProcessor)+ cost, balance = await proc.charge_node_usage(fake_node_exec)+ assert cost == 10+ assert balance == 50+ assert len(low_balance_calls) == 1+ assert low_balance_calls[0]["user_id"] == "u"+ assert low_balance_calls[0]["current_balance"] == 50+ assert low_balance_calls[0]["transaction_cost"] == 10++ @pytest.mark.asyncio+ async def test_skips_handle_low_balance_when_cost_zero(+ self, monkeypatch, fake_node_exec+ ):+ """charge_node_usage should NOT call handle_low_balance when cost is 0."""++ low_balance_calls: list = []++ def fake_charge_usage(node_exec, execution_count):+ return (0, 200)++ def fake_handle_low_balance(+ db_client, user_id, current_balance, transaction_cost+ ):+ low_balance_calls.append(True)++ monkeypatch.setattr(billing, "charge_usage", fake_charge_usage)+ monkeypatch.setattr(billing, "handle_low_balance", fake_handle_low_balance)+ monkeypatch.setattr(billing, "get_db_client", lambda: MagicMock())++ proc = manager.ExecutionProcessor.__new__(manager.ExecutionProcessor)+ cost, balance = await proc.charge_node_usage(fake_node_exec)+ assert cost == 0+ assert low_balance_calls == []+++# ── on_node_execution charging gate ────────────────────────────────+++class _FakeNode:+ """Minimal stand-in for a ``Node`` object with a block attribute."""++ def __init__(self, extra_charges: int = 0, block_name: str = "FakeBlock"):+ self.block = MagicMock()+ self.block.name = block_name+ self.block.extra_runtime_cost = MagicMock(return_value=extra_charges)+++class _FakeExecContext:+ def __init__(self, dry_run: bool = False):+ self.dry_run = dry_run+++def _make_node_exec(dry_run: bool = False) -> MagicMock:+ """Build a NodeExecutionEntry-like mock for on_node_execution tests."""+ ne = MagicMock()+ ne.user_id = "u"+ ne.graph_id = "g"+ ne.graph_exec_id = "ge"+ ne.node_id = "n"+ ne.node_exec_id = "ne"+ ne.block_id = "b"+ ne.inputs = {}+ ne.execution_context = _FakeExecContext(dry_run=dry_run)+ return ne+++def gated_processor(monkeypatch):+ """ExecutionProcessor with on_node_execution's downstream calls stubbed.++ Lets tests flip the gate conditions (status, extra_runtime_cost result,+ llm_call_count, dry_run) and observe whether charge_extra_runtime_cost+ was called.+ """++ calls: dict[str, list] = {+ "charge_extra_runtime_cost": [],+ "handle_low_balance": [],+ "handle_insufficient_funds_notif": [],+ }++ # Stub node lookup + DB client so the wrapper doesn't touch real infra.+ fake_db = MagicMock()+ fake_db.get_node = AsyncMock(return_value=_FakeNode(extra_charges=2))+ monkeypatch.setattr(manager, "get_db_async_client", lambda: fake_db)+ monkeypatch.setattr(billing, "get_db_client", lambda: fake_db)+ # get_block is called by LogMetadata construction in on_node_execution.+ monkeypatch.setattr(+ manager,+ "get_block",+ lambda block_id: MagicMock(name="FakeBlock"),+ )+ # Persistence + cost logging are not under test here.+ monkeypatch.setattr(+ manager,+ "async_update_node_execution_status",+ AsyncMock(return_value=None),+ )+ monkeypatch.setattr(+ manager,+ "async_update_graph_execution_state",+ AsyncMock(return_value=None),+ )+ monkeypatch.setattr(+ manager,+ "log_system_credential_cost",+ AsyncMock(return_value=None),+ )++ proc = manager.ExecutionProcessor.__new__(manager.ExecutionProcessor)++ # Control the status returned by the inner execution call.+ inner_result = {"status": ExecutionStatus.COMPLETED, "llm_call_count": 3}++ async def fake_inner(+ self,+ *,+ node,+ node_exec,+ node_exec_progress,+ stats,+ db_client,+ log_metadata,+ nodes_input_masks=None,+ nodes_to_skip=None,+ ):+ stats.llm_call_count = inner_result["llm_call_count"]+ return MagicMock(wall_time=0.1, cpu_time=0.1), inner_result["status"]++ monkeypatch.setattr(+ manager.ExecutionProcessor,+ "_on_node_execution",+ fake_inner,+ )++ async def fake_charge_extra(node_exec, extra_count):+ calls["charge_extra_runtime_cost"].append(extra_count)+ return (extra_count * 10, 500)++ monkeypatch.setattr(billing, "charge_extra_runtime_cost", fake_charge_extra)++ def fake_low_balance(db_client, user_id, current_balance, transaction_cost):+ calls["handle_low_balance"].append(+ {+ "user_id": user_id,+ "current_balance": current_balance,+ "transaction_cost": transaction_cost,+ }+ )++ monkeypatch.setattr(billing, "handle_low_balance", fake_low_balance)++ def fake_notif(db_client, user_id, graph_id, e):+ calls["handle_insufficient_funds_notif"].append(+ {"user_id": user_id, "graph_id": graph_id, "error": e}+ )++ monkeypatch.setattr(billing, "handle_insufficient_funds_notif", fake_notif)++ return proc, calls, inner_result, fake_db, NodeExecutionStats+++async def test_on_node_execution_charges_extra_iterations_when_gate_passes(+ gated_processor,+):+ """COMPLETED + extra_runtime_cost > 0 + not dry_run → charged."""++ proc, calls, inner, fake_db, _ = gated_processor+ inner["status"] = ExecutionStatus.COMPLETED+ inner["llm_call_count"] = 3 # → extra_charges = 2+ fake_db.get_node = AsyncMock(return_value=_FakeNode(extra_charges=2))++ stats_pair = (+ MagicMock(+ node_count=0, nodes_cputime=0, nodes_walltime=0, cost=0, node_error_count=0+ ),+ threading.Lock(),+ )+ await proc.on_node_execution(+ node_exec=_make_node_exec(dry_run=False),+ node_exec_progress=MagicMock(),+ nodes_input_masks=None,+ graph_stats_pair=stats_pair,+ )+ assert calls["charge_extra_runtime_cost"] == [2]+ # handle_low_balance must be called with the remaining balance returned by+ # charge_extra_runtime_cost (500) so users are alerted when balance drops low.+ assert len(calls["handle_low_balance"]) == 1+++async def test_on_node_execution_skips_when_status_not_completed(gated_processor):++ proc, calls, inner, fake_db, _ = gated_processor+ inner["status"] = ExecutionStatus.FAILED+ inner["llm_call_count"] = 5+ fake_db.get_node = AsyncMock(return_value=_FakeNode(extra_charges=4))++ stats_pair = (+ MagicMock(+ node_count=0, nodes_cputime=0, nodes_walltime=0, cost=0, node_error_count=0+ ),+ threading.Lock(),+ )+ await proc.on_node_execution(+ node_exec=_make_node_exec(dry_run=False),+ node_exec_progress=MagicMock(),+ nodes_input_masks=None,+ graph_stats_pair=stats_pair,+ )+ assert calls["charge_extra_runtime_cost"] == []+++async def test_on_node_execution_skips_when_extra_charges_zero(gated_processor):++ proc, calls, inner, fake_db, _ = gated_processor+ inner["status"] = ExecutionStatus.COMPLETED+ inner["llm_call_count"] = 5+ # Block returns 0 extra charges (base class default)+ fake_db.get_node = AsyncMock(return_value=_FakeNode(extra_charges=0))++ stats_pair = (+ MagicMock(+ node_count=0, nodes_cputime=0, nodes_walltime=0, cost=0, node_error_count=0+ ),+ threading.Lock(),+ )+ await proc.on_node_execution(+ node_exec=_make_node_exec(dry_run=False),+ node_exec_progress=MagicMock(),+ nodes_input_masks=None,+ graph_stats_pair=stats_pair,+ )+ assert calls["charge_extra_runtime_cost"] == []+++async def test_on_node_execution_skips_when_dry_run(gated_processor):++ proc, calls, inner, fake_db, _ = gated_processor+ inner["status"] = ExecutionStatus.COMPLETED+ inner["llm_call_count"] = 5+ fake_db.get_node = AsyncMock(return_value=_FakeNode(extra_charges=4))++ stats_pair = (+ MagicMock(+ node_count=0, nodes_cputime=0, nodes_walltime=0, cost=0, node_error_count=0+ ),+ threading.Lock(),+ )+ await proc.on_node_execution(+ node_exec=_make_node_exec(dry_run=True),+ node_exec_progress=MagicMock(),+ nodes_input_masks=None,+ graph_stats_pair=stats_pair,+ )+ assert calls["charge_extra_runtime_cost"] == []+++async def test_on_node_execution_insufficient_balance_records_error_and_notifies(+ monkeypatch,+ gated_processor,+):+ """When extra-iteration charging fails with InsufficientBalanceError:++ - the run still reports COMPLETED (the work is already done)+ - execution_stats.error is NOT set (would flip node_error_count and+ leak balance amounts into persisted node_stats — see manager.py+ comment in the IBE handler)+ - _handle_insufficient_funds_notif is called so the user is notified+ - the structured ERROR log is the alerting hook+ """++ proc, calls, inner, fake_db, _ = gated_processor+ inner["status"] = ExecutionStatus.COMPLETED+ inner["llm_call_count"] = 4+ fake_db.get_node = AsyncMock(return_value=_FakeNode(extra_charges=3))++ async def raise_ibe(node_exec, extra_count):+ raise InsufficientBalanceError(+ user_id=node_exec.user_id,+ message="Insufficient balance",+ balance=0,+ amount=extra_count * 10,+ )++ monkeypatch.setattr(billing, "charge_extra_runtime_cost", raise_ibe)++ stats_pair = (+ MagicMock(+ node_count=0, nodes_cputime=0, nodes_walltime=0, cost=0, node_error_count=0+ ),+ threading.Lock(),+ )+ result_stats = await proc.on_node_execution(+ node_exec=_make_node_exec(dry_run=False),+ node_exec_progress=MagicMock(),+ nodes_input_masks=None,+ graph_stats_pair=stats_pair,+ )+ # error stays None — node ran to completion, only the post-hoc+ # charge failed. Setting .error would (a) flip node_error_count+++ # creating an "errored COMPLETED node" inconsistency, and (b) leak+ # balance amounts into persisted node_stats.+ assert result_stats.error is None+ # User notification fired.+ assert len(calls["handle_insufficient_funds_notif"]) == 1+ assert calls["handle_insufficient_funds_notif"][0]["user_id"] == "u"+++# ── Orchestrator _execute_single_tool_with_manager charging gates ──+++async def _run_tool_exec_with_stats(+ *,+ dry_run: bool,+ tool_stats_error,+ charge_node_usage_mock=None,+):+ """Invoke _execute_single_tool_with_manager against fully mocked deps+ and return (charge_call_count, merge_stats_calls).++ Used to prove the dry_run and error guards around charge_node_usage+ behave as documented, and that InsufficientBalanceError propagates.+ """+ block = OrchestratorBlock()++ # Mocked async DB client used inside orchestrator.+ mock_db_client = AsyncMock()+ mock_target_node = MagicMock()+ mock_target_node.block_id = "test-block-id"+ mock_target_node.input_default = {}+ mock_db_client.get_node.return_value = mock_target_node+ mock_node_exec_result = MagicMock()+ mock_node_exec_result.node_exec_id = "test-tool-exec-id"+ mock_db_client.upsert_execution_input.return_value = (+ mock_node_exec_result,+ {"query": "t"},+ )+ mock_db_client.get_execution_outputs_by_node_exec_id.return_value = {"result": "ok"}++ # ExecutionProcessor mock: on_node_execution returns supplied error.+ mock_processor = AsyncMock()+ mock_processor.running_node_execution = defaultdict(MagicMock)+ mock_processor.execution_stats = MagicMock()+ mock_processor.execution_stats_lock = threading.Lock()+ mock_node_stats = MagicMock()+ mock_node_stats.error = tool_stats_error+ mock_processor.on_node_execution = AsyncMock(return_value=mock_node_stats)+ mock_processor.charge_node_usage = charge_node_usage_mock or AsyncMock(+ return_value=(10, 990)+ )++ # Build a tool_info shaped like _build_tool_info_from_args output.+ tool_call = MagicMock()+ tool_call.id = "call-1"+ tool_call.name = "search_keywords"+ tool_call.arguments = '{"query":"t"}'+ tool_def = {+ "type": "function",+ "function": {+ "name": "search_keywords",+ "_sink_node_id": "test-sink-node-id",+ "_field_mapping": {},+ "parameters": {+ "properties": {"query": {"type": "string"}},+ "required": ["query"],+ },+ },+ }+ tool_info = OrchestratorBlock._build_tool_info_from_args(+ tool_call_id="call-1",+ tool_name="search_keywords",+ tool_args={"query": "t"},+ tool_def=tool_def,+ )++ exec_params = ExecutionParams(+ user_id="u",+ graph_id="g",+ node_id="n",+ graph_version=1,+ graph_exec_id="ge",+ node_exec_id="ne",+ execution_context=ExecutionContext(+ human_in_the_loop_safe_mode=False, dry_run=dry_run+ ),+ )++ with patch(+ "backend.blocks.orchestrator.get_database_manager_async_client",+ return_value=mock_db_client,+ ):+ try:+ await block._execute_single_tool_with_manager(+ tool_info, exec_params, mock_processor, responses_api=False+ )+ raised = None+ except Exception as e:+ raised = e++ return mock_processor.charge_node_usage, raised+++async def test_tool_execution_skips_charging_on_dry_run():+ """dry_run=True → charge_node_usage is NOT called."""+ charge_mock, raised = await _run_tool_exec_with_stats(+ dry_run=True, tool_stats_error=None+ )+ assert raised is None+ assert charge_mock.call_count == 0+++async def test_tool_execution_skips_charging_on_failed_tool():+ """tool_node_stats.error is an Exception → charge_node_usage NOT called."""+ charge_mock, raised = await _run_tool_exec_with_stats(+ dry_run=False, tool_stats_error=RuntimeError("tool blew up")+ )+ assert raised is None+ assert charge_mock.call_count == 0+++async def test_tool_execution_skips_charging_on_cancelled_tool():+ """Cancellation (BaseException subclass) → charge_node_usage NOT called.++ Guards the fix for sentry's BaseException concern: the old+ `isinstance(error, Exception)` check would have treated CancelledError+ as "no error" and billed the user for a terminated run.+ """+ import asyncio as _asyncio++ charge_mock, raised = await _run_tool_exec_with_stats(+ dry_run=False, tool_stats_error=_asyncio.CancelledError()+ )+ assert raised is None+ assert charge_mock.call_count == 0+++async def test_tool_execution_insufficient_balance_propagates():+ """InsufficientBalanceError from charge_node_usage must propagate out.++ If this leaked into a ToolCallResult the LLM loop would keep running+ with 'tool failed' errors and the user would get unpaid work.+ """+ raising_charge = AsyncMock(+ side_effect=InsufficientBalanceError(+ user_id="u", message="nope", balance=0, amount=10+ )+ )+ _, raised = await _run_tool_exec_with_stats(+ dry_run=False,+ tool_stats_error=None,+ charge_node_usage_mock=raising_charge,+ )+ assert isinstance(raised, InsufficientBalanceError)+++async def test_tool_execution_on_node_execution_returns_none_sets_is_error():+ """on_node_execution returning None (swallowed by @async_error_logged) must+ result in a tool response with _is_error=True so the LLM loop knows the+ tool failed and does not treat a silent error as a successful execution.+ """+ block = OrchestratorBlock()++ mock_db_client = AsyncMock()+ mock_target_node = MagicMock()+ mock_target_node.block_id = "test-block-id"+ mock_target_node.input_default = {}+ mock_db_client.get_node.return_value = mock_target_node+ mock_node_exec_result = MagicMock()+ mock_node_exec_result.node_exec_id = "test-tool-exec-id"+ mock_db_client.upsert_execution_input.return_value = (+ mock_node_exec_result,+ {"query": "t"},+ )++ mock_processor = AsyncMock()+ mock_processor.running_node_execution = defaultdict(MagicMock)+ mock_processor.execution_stats = MagicMock()+ mock_processor.execution_stats_lock = threading.Lock()+ # on_node_execution returns None — simulates @async_error_logged(swallow=True)+ # swallowing an internal error+ mock_processor.on_node_execution = AsyncMock(return_value=None)++ tool_call = MagicMock()+ tool_call.id = "call-none"+ tool_call.name = "search_keywords"+ tool_call.arguments = '{"query":"t"}'+ tool_def = {+ "type": "function",+ "function": {+ "name": "search_keywords",+ "_sink_node_id": "test-sink-node-id",+ "_field_mapping": {},+ "parameters": {+ "properties": {"query": {"type": "string"}},+ "required": ["query"],+ },+ },+ }+ tool_info = OrchestratorBlock._build_tool_info_from_args(+ tool_call_id="call-none",+ tool_name="search_keywords",+ tool_args={"query": "t"},+ tool_def=tool_def,+ )++ exec_params = ExecutionParams(+ user_id="u",+ graph_id="g",+ node_id="n",+ graph_version=1,+ graph_exec_id="ge",+ node_exec_id="ne",+ execution_context=ExecutionContext(+ human_in_the_loop_safe_mode=False, dry_run=False+ ),+ )++ with patch(+ "backend.blocks.orchestrator.get_database_manager_async_client",+ return_value=mock_db_client,+ ):+ resp = await block._execute_single_tool_with_manager(+ tool_info, exec_params, mock_processor, responses_api=False+ )++ assert resp.get("_is_error") is True+ # charge_node_usage must NOT be called for a failed tool execution+ mock_processor.charge_node_usage.assert_not_called()+++# ── on_node_execution FAILED + InsufficientBalanceError notification ──+++async def test_on_node_execution_failed_ibe_sends_notification(+ monkeypatch,+ gated_processor,+):+ """When status == FAILED and execution_stats.error is InsufficientBalanceError,+ _handle_insufficient_funds_notif must be called.++ This path fires when a nested tool charge inside the orchestrator raises+ InsufficientBalanceError, which propagates out of the block's run() generator+ and is caught by _on_node_execution's broad except, setting status=FAILED and+ execution_stats.error=IBE. on_node_execution's post-execution block then+ sends the user notification so they understand why the run stopped.+ """++ proc, calls, inner, fake_db, NodeExecutionStats = gated_processor+ ibe = InsufficientBalanceError(+ user_id="u",+ message="Insufficient balance",+ balance=0,+ amount=30,+ )++ # Simulate _on_node_execution returning FAILED with IBE in stats.error.+ async def fake_inner_failed(+ self,+ *,+ node,+ node_exec,+ node_exec_progress,+ stats,+ db_client,+ log_metadata,+ nodes_input_masks=None,+ nodes_to_skip=None,+ ):+ stats.error = ibe+ return MagicMock(wall_time=0.1, cpu_time=0.1), ExecutionStatus.FAILED++ monkeypatch.setattr(+ manager.ExecutionProcessor,+ "_on_node_execution",+ fake_inner_failed,+ )+ fake_db.get_node = AsyncMock(return_value=_FakeNode(extra_charges=0))++ stats_pair = (+ MagicMock(+ node_count=0, nodes_cputime=0, nodes_walltime=0, cost=0, node_error_count=0+ ),+ threading.Lock(),+ )+ await proc.on_node_execution(+ node_exec=_make_node_exec(dry_run=False),+ node_exec_progress=MagicMock(),+ nodes_input_masks=None,+ graph_stats_pair=stats_pair,+ )+ # The notification must have fired so the user knows why their run stopped.+ assert len(calls["handle_insufficient_funds_notif"]) == 1+ assert calls["handle_insufficient_funds_notif"][0]["user_id"] == "u"+ # charge_extra_runtime_cost must NOT be called — status is FAILED.+ assert calls["charge_extra_runtime_cost"] == []+++# ── Billing leak: non-IBE exception during extra-iteration charging ──+++async def test_on_node_execution_non_ibe_billing_failure_keeps_completed(+ monkeypatch,+ gated_processor,+):+ """When charge_extra_runtime_cost raises a non-IBE exception (e.g. DB outage):++ - execution_stats.error stays None (node ran to completion)+ - status stays COMPLETED (work already done)+ - the billing_leak error is logged but does not corrupt execution_stats+ """+ proc, calls, inner, fake_db, _ = gated_processor+ inner["status"] = ExecutionStatus.COMPLETED+ inner["llm_call_count"] = 4+ fake_db.get_node = AsyncMock(return_value=_FakeNode(extra_charges=3))++ async def raise_conn_error(node_exec, extra_count):+ raise ConnectionError("DB connection lost")++ monkeypatch.setattr(billing, "charge_extra_runtime_cost", raise_conn_error)++ stats_pair = (+ MagicMock(+ node_count=0,+ nodes_cputime=0,+ nodes_walltime=0,+ cost=0,+ node_error_count=0,+ ),+ threading.Lock(),+ )+ result_stats = await proc.on_node_execution(+ node_exec=_make_node_exec(dry_run=False),+ node_exec_progress=MagicMock(),+ nodes_input_masks=None,+ graph_stats_pair=stats_pair,+ )+ # error stays None — node completed, only billing failed.+ assert result_stats.error is None+ # No notification was sent (only IBE triggers notification).+ assert len(calls["handle_insufficient_funds_notif"]) == 0+++# ── _charge_usage with execution_count=0 ──+++class TestChargeUsageZeroExecutionCount:+ """Verify _charge_usage(node_exec, 0) does not invoke execution_usage_cost."""++ def test_execution_count_zero_skips_execution_tier(self, monkeypatch):+ """_charge_usage with execution_count=0 must not call execution_usage_cost."""+ execution_tier_called = []++ def fake_execution_usage_cost(count):+ execution_tier_called.append(count)+ return (100, count)++ spent: list[int] = []++ class FakeDb:+ def spend_credits(self, *, user_id, cost, metadata):+ spent.append(cost)+ return 500++ fake_block = MagicMock()+ fake_block.name = "FakeBlock"++ monkeypatch.setattr(billing, "get_db_client", lambda: FakeDb())+ monkeypatch.setattr(billing, "get_block", lambda block_id: fake_block)+ monkeypatch.setattr(+ billing,+ "block_usage_cost",+ lambda block, input_data, **_kw: (10, {}),+ )+ monkeypatch.setattr(billing, "execution_usage_cost", fake_execution_usage_cost)++ ne = MagicMock()+ ne.user_id = "u"+ ne.graph_exec_id = "ge"+ ne.graph_id = "g"+ ne.node_exec_id = "ne"+ ne.node_id = "n"+ ne.block_id = "b"+ ne.inputs = {}++ total_cost, remaining = billing.charge_usage(ne, 0)+ assert total_cost == 10 # block cost only+ assert remaining == 500+ assert spent == [10]+ # execution_usage_cost must NOT have been called+ assert execution_tier_called == []diff --git a/autogpt_platform/backend/backend/blocks/test/test_orchestrator_responses_api.py b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_responses_api.pyindex f9ec7676ba..ac78b6d35b 100644--- a/autogpt_platform/backend/backend/blocks/test/test_orchestrator_responses_api.py+++ b/autogpt_platform/backend/backend/blocks/test/test_orchestrator_responses_api.py@@ -956,6 +956,12 @@ async def test_agent_mode_conversation_valid_for_responses_api():ep.execution_stats_lock = threading.Lock()ns = MagicMock(error=None)ep.on_node_execution = AsyncMock(return_value=ns)+ # Mock charge_node_usage (called after successful tool execution).+ # Must be AsyncMock because it is async and is awaited in+ # _execute_single_tool_with_manager — a plain MagicMock would return a+ # non-awaitable tuple and TypeError out, then be silently swallowed by+ # the orchestrator's catch-all.+ ep.charge_node_usage = AsyncMock(return_value=(0, 0))with patch("backend.blocks.llm.llm_call", llm_mock), patch.object(block, "_create_tool_node_signatures", return_value=tool_sigsdiff --git a/autogpt_platform/backend/backend/data/credit.py b/autogpt_platform/backend/backend/data/credit.pyindex 0959c15d34..24b5aae80d 100644--- a/autogpt_platform/backend/backend/data/credit.py+++ b/autogpt_platform/backend/backend/data/credit.py@@ -349,7 +349,7 @@ class UserCreditBase(ABC):CreditTransactionType.GRANT,CreditTransactionType.TOP_UP,]:- from backend.executor.manager import (+ from backend.executor.billing import (clear_insufficient_funds_notifications,)@@ -554,7 +554,7 @@ class UserCreditBase(ABC):in [CreditTransactionType.GRANT, CreditTransactionType.TOP_UP]):# Lazy import to avoid circular dependency with executor.manager- from backend.executor.manager import (+ from backend.executor.billing import (clear_insufficient_funds_notifications,)diff --git a/autogpt_platform/backend/backend/data/model.py b/autogpt_platform/backend/backend/data/model.pyindex f0393133e6..09fdaa6cf8 100644--- a/autogpt_platform/backend/backend/data/model.py+++ b/autogpt_platform/backend/backend/data/model.py@@ -852,6 +852,7 @@ class NodeExecutionStats(BaseModel):output_token_count: int = 0cache_read_token_count: int = 0cache_creation_token_count: int = 0+ cost: int = 0extra_cost: int = 0extra_steps: int = 0provider_cost: float | None = Nonediff --git a/autogpt_platform/backend/backend/executor/billing.py b/autogpt_platform/backend/backend/executor/billing.pynew file mode 100644index 0000000000..24bdec2c5c--- /dev/null+++ b/autogpt_platform/backend/backend/executor/billing.py@@ -0,0 +1,509 @@+import asyncio+import logging+from typing import TYPE_CHECKING, Any, cast++from backend.blocks import get_block+from backend.blocks._base import Block+from backend.blocks.io import AgentOutputBlock+from backend.data import redis_client as redis+from backend.data.credit import UsageTransactionMetadata+from backend.data.execution import (+ ExecutionStatus,+ GraphExecutionEntry,+ NodeExecutionEntry,+)+from backend.data.graph import Node+from backend.data.model import GraphExecutionStats, NodeExecutionStats+from backend.data.notifications import (+ AgentRunData,+ LowBalanceData,+ NotificationEventModel,+ NotificationType,+ ZeroBalanceData,+)+from backend.notifications.notifications import queue_notification+from backend.util.clients import (+ get_database_manager_client,+ get_notification_manager_client,+)+from backend.util.exceptions import InsufficientBalanceError+from backend.util.logging import TruncatedLogger+from backend.util.metrics import DiscordChannel+from backend.util.settings import Settings++from .utils import LogMetadata, block_usage_cost, execution_usage_cost++if TYPE_CHECKING:+ from backend.data.db_manager import DatabaseManagerClient++_logger = logging.getLogger(__name__)+logger = TruncatedLogger(_logger, prefix="[Billing]")+settings = Settings()++# Redis key prefix for tracking insufficient funds Discord notifications.+# We only send one notification per user per agent until they top up credits.+INSUFFICIENT_FUNDS_NOTIFIED_PREFIX = "insufficient_funds_discord_notified"+# TTL for the notification flag (30 days) - acts as a fallback cleanup+INSUFFICIENT_FUNDS_NOTIFIED_TTL_SECONDS = 30 * 24 * 60 * 60++# Hard cap on the multiplier passed to charge_extra_runtime_cost to+# protect against a corrupted llm_call_count draining a user's balance.+# Real agent-mode runs are bounded by agent_mode_max_iterations (~50);+# 200 leaves headroom while preventing runaway charges.+_MAX_EXTRA_RUNTIME_COST = 200+++def get_db_client() -> "DatabaseManagerClient":+ return get_database_manager_client()+++async def clear_insufficient_funds_notifications(user_id: str) -> int:+ """+ Clear all insufficient funds notification flags for a user.++ This should be called when a user tops up their credits, allowing+ Discord notifications to be sent again if they run out of funds.++ Args:+ user_id: The user ID to clear notifications for.++ Returns:+ The number of keys that were deleted.+ """+ try:+ redis_client = await redis.get_redis_async()+ pattern = f"{INSUFFICIENT_FUNDS_NOTIFIED_PREFIX}:{user_id}:*"+ keys = [key async for key in redis_client.scan_iter(match=pattern)]+ if keys:+ return await redis_client.delete(*keys)+ return 0+ except Exception as e:+ logger.warning(+ f"Failed to clear insufficient funds notification flags for user "+ f"{user_id}: {e}"+ )+ return 0+++def resolve_block_cost(+ node_exec: NodeExecutionEntry,+) -> tuple["Block | None", int, dict[str, Any]]:+ """Look up the block and compute its base usage cost for an exec.++ Shared by charge_usage and charge_extra_runtime_cost so the+ (get_block, block_usage_cost) lookup lives in exactly one place.+ Returns ``(block, cost, matching_filter)``. ``block`` is ``None`` if+ the block id can't be resolved — callers should treat that as+ "nothing to charge".+ """+ block = get_block(node_exec.block_id)+ if not block:+ logger.error(f"Block {node_exec.block_id} not found.")+ return None, 0, {}+ cost, matching_filter = block_usage_cost(block=block, input_data=node_exec.inputs)+ return block, cost, matching_filter+++def charge_usage(+ node_exec: NodeExecutionEntry,+ execution_count: int,+) -> tuple[int, int]:+ total_cost = 0+ remaining_balance = 0+ db_client = get_db_client()+ block, cost, matching_filter = resolve_block_cost(node_exec)+ if not block:+ return total_cost, 0++ if cost > 0:+ remaining_balance = db_client.spend_credits(+ user_id=node_exec.user_id,+ cost=cost,+ metadata=UsageTransactionMetadata(+ graph_exec_id=node_exec.graph_exec_id,+ graph_id=node_exec.graph_id,+ node_exec_id=node_exec.node_exec_id,+ node_id=node_exec.node_id,+ block_id=node_exec.block_id,+ block=block.name,+ input=matching_filter,+ reason=f"Ran block {node_exec.block_id} {block.name}",+ ),+ )+ total_cost += cost++ # execution_count=0 is used by charge_node_usage for nested tool calls+ # which must not be pushed into higher execution-count tiers.+ # execution_usage_cost(0) would trigger a charge because 0 % threshold == 0,+ # so skip it entirely when execution_count is 0.+ cost, usage_count = (+ execution_usage_cost(execution_count) if execution_count > 0 else (0, 0)+ )+ if cost > 0:+ remaining_balance = db_client.spend_credits(+ user_id=node_exec.user_id,+ cost=cost,+ metadata=UsageTransactionMetadata(+ graph_exec_id=node_exec.graph_exec_id,+ graph_id=node_exec.graph_id,+ input={+ "execution_count": usage_count,+ "charge": "Execution Cost",+ },+ reason=f"Execution Cost for {usage_count} blocks of ex_id:{node_exec.graph_exec_id} g_id:{node_exec.graph_id}",+ ),+ )+ total_cost += cost++ return total_cost, remaining_balance+++def _charge_extra_runtime_cost_sync(+ node_exec: NodeExecutionEntry,+ capped_count: int,+) -> tuple[int, int]:+ """Synchronous implementation — runs in a thread-pool worker.++ Called only from charge_extra_runtime_cost. Do not call directly from+ async code.++ Note: ``resolve_block_cost`` is called again here (rather than reusing+ the result from ``charge_usage`` at the start of execution) because the+ two calls happen in separate thread-pool workers and sharing mutable+ state across workers would require locks. The block config is immutable+ during a run, so the repeated lookup is safe and produces the same cost;+ the only overhead is an extra registry lookup.+ """+ db_client = get_db_client()+ block, cost, matching_filter = resolve_block_cost(node_exec)+ if not block or cost <= 0:+ return 0, 0+ total_extra_cost = cost * capped_count+ remaining_balance = db_client.spend_credits(+ user_id=node_exec.user_id,+ cost=total_extra_cost,+ metadata=UsageTransactionMetadata(+ graph_exec_id=node_exec.graph_exec_id,+ graph_id=node_exec.graph_id,+ node_exec_id=node_exec.node_exec_id,+ node_id=node_exec.node_id,+ block_id=node_exec.block_id,+ block=block.name,+ input={+ **matching_filter,+ "extra_runtime_cost_count": capped_count,+ },+ reason=(+ f"Extra agent-mode iterations for {block.name} "+ f"({capped_count} additional LLM calls)"+ ),+ ),+ )+ return total_extra_cost, remaining_balance+++async def charge_extra_runtime_cost(+ node_exec: NodeExecutionEntry,+ extra_count: int,+) -> tuple[int, int]:+ """Charge a block extra runtime cost beyond the initial run.++ Used by agent-mode blocks (e.g. OrchestratorBlock) that make multiple+ LLM calls within a single node execution. The first iteration is already+ charged by charge_usage; this method charges *extra_count* additional+ copies of the block's base cost.++ Returns ``(total_extra_cost, remaining_balance)``. May raise+ ``InsufficientBalanceError`` if the user can't afford the charge.+ """+ if extra_count <= 0:+ return 0, 0+ # Cap to protect against a corrupted llm_call_count.+ capped = min(extra_count, _MAX_EXTRA_RUNTIME_COST)+ if extra_count > _MAX_EXTRA_RUNTIME_COST:+ logger.warning(+ f"extra_count {extra_count} exceeds cap {_MAX_EXTRA_RUNTIME_COST};"+ f" charging {_MAX_EXTRA_RUNTIME_COST} (llm_call_count may be corrupted)"+ )+ return await asyncio.to_thread(_charge_extra_runtime_cost_sync, node_exec, capped)+++async def charge_node_usage(node_exec: NodeExecutionEntry) -> tuple[int, int]:+ """Charge a single node execution to the user.++ Public async wrapper around charge_usage for blocks (e.g. the+ OrchestratorBlock) that spawn nested node executions outside the main+ queue and therefore need to charge them explicitly.++ Also handles low-balance notification so callers don't need to touch+ private functions directly.++ Note: this **does not** increment the global execution counter+ (``increment_execution_count``). Nested tool executions are sub-steps+ of a single block run from the user's perspective and should not push+ them into higher per-execution cost tiers.+ """++ def _run():+ total_cost, remaining = charge_usage(node_exec, 0)+ if total_cost > 0:+ handle_low_balance(+ get_db_client(), node_exec.user_id, remaining, total_cost+ )+ return total_cost, remaining++ return await asyncio.to_thread(_run)+++async def try_send_insufficient_funds_notif(+ user_id: str,+ graph_id: str,+ error: InsufficientBalanceError,+ log_metadata: LogMetadata,+) -> None:+ """Send an insufficient-funds notification, swallowing failures."""+ try:+ await asyncio.to_thread(+ handle_insufficient_funds_notif,+ get_db_client(),+ user_id,+ graph_id,+ error,+ )+ except Exception as notif_error: # pragma: no cover+ log_metadata.warning(+ f"Failed to send insufficient funds notification: {notif_error}"+ )+++async def handle_post_execution_billing(+ node: Node,+ node_exec: NodeExecutionEntry,+ execution_stats: NodeExecutionStats,+ status: ExecutionStatus,+ log_metadata: LogMetadata,+) -> None:+ """Charge extra runtime cost for blocks that opt into per-LLM-call billing.++ The first LLM call is already covered by charge_usage(); each additional+ call costs another base_cost. Skipped for dry runs and failed runs.++ InsufficientBalanceError here is a post-hoc billing leak: the work is+ already done but the user can no longer pay. The run stays COMPLETED and+ the error is logged with ``billing_leak: True`` for alerting.+ """+ extra_iterations = (+ cast(Block, node.block).extra_runtime_cost(execution_stats)+ if status == ExecutionStatus.COMPLETED+ and not node_exec.execution_context.dry_run+ else 0+ )+ if extra_iterations <= 0:+ return++ try:+ extra_cost, remaining_balance = await charge_extra_runtime_cost(+ node_exec,+ extra_iterations,+ )+ if extra_cost > 0:+ execution_stats.extra_cost += extra_cost+ await asyncio.to_thread(+ handle_low_balance,+ get_db_client(),+ node_exec.user_id,+ remaining_balance,+ extra_cost,+ )+ except InsufficientBalanceError as e:+ log_metadata.error(+ "billing_leak: insufficient balance after "+ f"{node.block.name} completed {extra_iterations} "+ f"extra iterations",+ extra={+ "billing_leak": True,+ "user_id": node_exec.user_id,+ "graph_id": node_exec.graph_id,+ "block_id": node_exec.block_id,+ "extra_runtime_cost_count": extra_iterations,+ "error": str(e),+ },+ )+ # Do NOT set execution_stats.error — the node ran to completion,+ # only the post-hoc charge failed. See class-level billing-leak+ # contract documentation.+ await try_send_insufficient_funds_notif(+ node_exec.user_id,+ node_exec.graph_id,+ e,+ log_metadata,+ )+ except Exception as e:+ log_metadata.error(+ f"billing_leak: failed to charge extra iterations for {node.block.name}",+ extra={+ "billing_leak": True,+ "user_id": node_exec.user_id,+ "graph_id": node_exec.graph_id,+ "block_id": node_exec.block_id,+ "extra_runtime_cost_count": extra_iterations,+ "error_type": type(e).__name__,+ "error": str(e),+ },+ exc_info=True,+ )+++def handle_agent_run_notif(+ db_client: "DatabaseManagerClient",+ graph_exec: GraphExecutionEntry,+ exec_stats: GraphExecutionStats,+) -> None:+ metadata = db_client.get_graph_metadata(+ graph_exec.graph_id, graph_exec.graph_version+ )+ outputs = db_client.get_node_executions(+ graph_exec.graph_exec_id,+ block_ids=[AgentOutputBlock().id],+ )++ named_outputs = [+ {+ key: value[0] if key == "name" else value+ for key, value in output.output_data.items()+ }+ for output in outputs+ ]++ queue_notification(+ NotificationEventModel(+ user_id=graph_exec.user_id,+ type=NotificationType.AGENT_RUN,+ data=AgentRunData(+ outputs=named_outputs,+ agent_name=metadata.name if metadata else "Unknown Agent",+ credits_used=exec_stats.cost,+ execution_time=exec_stats.walltime,+ graph_id=graph_exec.graph_id,+ node_count=exec_stats.node_count,+ ),+ )+ )+++def handle_insufficient_funds_notif(+ db_client: "DatabaseManagerClient",+ user_id: str,+ graph_id: str,+ e: InsufficientBalanceError,+) -> None:+ # Check if we've already sent a notification for this user+agent combo.+ # We only send one notification per user per agent until they top up credits.+ redis_key = f"{INSUFFICIENT_FUNDS_NOTIFIED_PREFIX}:{user_id}:{graph_id}"+ try:+ redis_client = redis.get_redis()+ # SET NX returns True only if the key was newly set (didn't exist)+ is_new_notification = redis_client.set(+ redis_key,+ "1",+ nx=True,+ ex=INSUFFICIENT_FUNDS_NOTIFIED_TTL_SECONDS,+ )+ if not is_new_notification:+ # Already notified for this user+agent, skip all notifications+ logger.debug(+ f"Skipping duplicate insufficient funds notification for "+ f"user={user_id}, graph={graph_id}"+ )+ return+ except Exception as redis_error:+ # If Redis fails, log and continue to send the notification+ # (better to occasionally duplicate than to never notify)+ logger.warning(+ f"Failed to check/set insufficient funds notification flag in Redis: "+ f"{redis_error}"+ )++ shortfall = abs(e.amount) - e.balance+ metadata = db_client.get_graph_metadata(graph_id)+ base_url = settings.config.frontend_base_url or settings.config.platform_base_url++ # Queue user email notification+ queue_notification(+ NotificationEventModel(+ user_id=user_id,+ type=NotificationType.ZERO_BALANCE,+ data=ZeroBalanceData(+ current_balance=e.balance,+ billing_page_link=f"{base_url}/profile/credits",+ shortfall=shortfall,+ agent_name=metadata.name if metadata else "Unknown Agent",+ ),+ )+ )++ # Send Discord system alert+ try:+ user_email = db_client.get_user_email_by_id(user_id)++ alert_message = (+ f"❌ **Insufficient Funds Alert**\n"+ f"User: {user_email or user_id}\n"+ f"Agent: {metadata.name if metadata else 'Unknown Agent'}\n"+ f"Current balance: ${e.balance / 100:.2f}\n"+ f"Attempted cost: ${abs(e.amount) / 100:.2f}\n"+ f"Shortfall: ${abs(shortfall) / 100:.2f}\n"+ f"[View User Details]({base_url}/admin/spending?search={user_email})"+ )++ get_notification_manager_client().discord_system_alert(+ alert_message, DiscordChannel.PRODUCT+ )+ except Exception as alert_error:+ logger.error(f"Failed to send insufficient funds Discord alert: {alert_error}")+++def handle_low_balance(+ db_client: "DatabaseManagerClient",+ user_id: str,+ current_balance: int,+ transaction_cost: int,+) -> None:+ """Check and handle low balance scenarios after a transaction"""+ LOW_BALANCE_THRESHOLD = settings.config.low_balance_threshold++ balance_before = current_balance + transaction_cost++ if (+ current_balance < LOW_BALANCE_THRESHOLD+ and balance_before >= LOW_BALANCE_THRESHOLD+ ):+ base_url = (+ settings.config.frontend_base_url or settings.config.platform_base_url+ )+ queue_notification(+ NotificationEventModel(+ user_id=user_id,+ type=NotificationType.LOW_BALANCE,+ data=LowBalanceData(+ current_balance=current_balance,+ billing_page_link=f"{base_url}/profile/credits",+ ),+ )+ )++ try:+ user_email = db_client.get_user_email_by_id(user_id)+ alert_message = (+ f"⚠️ **Low Balance Alert**\n"+ f"User: {user_email or user_id}\n"+ f"Balance dropped below ${LOW_BALANCE_THRESHOLD / 100:.2f}\n"+ f"Current balance: ${current_balance / 100:.2f}\n"+ f"Transaction cost: ${transaction_cost / 100:.2f}\n"+ f"[View User Details]({base_url}/admin/spending?search={user_email})"+ )+ get_notification_manager_client().discord_system_alert(+ alert_message, DiscordChannel.PRODUCT+ )+ except Exception as e:+ logger.warning(f"Failed to send low balance Discord alert: {e}")diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex bd718d168f..2af3ce784e 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -21,11 +21,9 @@ from sentry_sdk.api import get_current_scope as _sentry_get_current_scopefrom backend.blocks import get_blockfrom backend.blocks._base import BlockSchemafrom backend.blocks.agent import AgentExecutorBlock-from backend.blocks.io import AgentOutputBlockfrom backend.blocks.mcp.block import MCPToolBlockfrom backend.data import redis_client as redisfrom backend.data.block import BlockInput, BlockOutput, BlockOutputEntry-from backend.data.credit import UsageTransactionMetadatafrom backend.data.dynamic_fields import parse_execution_outputfrom backend.data.execution import (ExecutionContext,@@ -39,27 +37,18 @@ from backend.data.execution import ()from backend.data.graph import Link, Nodefrom backend.data.model import GraphExecutionStats, NodeExecutionStats-from backend.data.notifications import (- AgentRunData,- LowBalanceData,- NotificationEventModel,- NotificationType,- ZeroBalanceData,-)from backend.data.rabbitmq import SyncRabbitMQfrom backend.executor.cost_tracking import (drain_pending_cost_logs,log_system_credential_cost,)from backend.integrations.creds_manager import IntegrationCredentialsManager-from backend.notifications.notifications import queue_notificationfrom backend.util import jsonfrom backend.util.clients import (get_async_execution_event_bus,get_database_manager_async_client,get_database_manager_client,get_execution_event_bus,- get_notification_manager_client,)from backend.util.decorator import (async_error_logged,@@ -75,7 +64,6 @@ from backend.util.exceptions import ()from backend.util.file import clean_exec_filesfrom backend.util.logging import TruncatedLogger, configure_logging-from backend.util.metrics import DiscordChannelfrom backend.util.process import AppProcess, set_service_namefrom backend.util.retry import (continuous_retry,@@ -84,6 +72,7 @@ from backend.util.retry import ()from backend.util.settings import Settings+from . import billingfrom .activity_status_generator import generate_activity_status_for_executionfrom .automod.manager import automod_managerfrom .cluster_lock import ClusterLock@@ -98,9 +87,7 @@ from .utils import (ExecutionOutputEntry,LogMetadata,NodeExecutionProgress,- block_usage_cost,create_execution_queue_config,- execution_usage_cost,validate_exec,)@@ -126,40 +113,6 @@ utilization_gauge = Gauge("Ratio of active graph runs to max graph workers",)-# Redis key prefix for tracking insufficient funds Discord notifications.-# We only send one notification per user per agent until they top up credits.-INSUFFICIENT_FUNDS_NOTIFIED_PREFIX = "insufficient_funds_discord_notified"-# TTL for the notification flag (30 days) - acts as a fallback cleanup-INSUFFICIENT_FUNDS_NOTIFIED_TTL_SECONDS = 30 * 24 * 60 * 60---async def clear_insufficient_funds_notifications(user_id: str) -> int:- """- Clear all insufficient funds notification flags for a user.-- This should be called when a user tops up their credits, allowing- Discord notifications to be sent again if they run out of funds.-- Args:- user_id: The user ID to clear notifications for.-- Returns:- The number of keys that were deleted.- """- try:- redis_client = await redis.get_redis_async()- pattern = f"{INSUFFICIENT_FUNDS_NOTIFIED_PREFIX}:{user_id}:*"- keys = [key async for key in redis_client.scan_iter(match=pattern)]- if keys:- return await redis_client.delete(*keys)- return 0- except Exception as e:- logger.warning(- f"Failed to clear insufficient funds notification flags for user "- f"{user_id}: {e}"- )- return 0-# Thread-local storage for ExecutionProcessor instances_tls = threading.local()@@ -681,12 +634,16 @@ class ExecutionProcessor:execution_stats.walltime = timing_info.wall_timeexecution_stats.cputime = timing_info.cpu_time+ await billing.handle_post_execution_billing(+ node, node_exec, execution_stats, status, log_metadata+ )+graph_stats, graph_stats_lock = graph_stats_pairwith graph_stats_lock:graph_stats.node_count += 1 + execution_stats.extra_stepsgraph_stats.nodes_cputime += execution_stats.cputimegraph_stats.nodes_walltime += execution_stats.walltime- graph_stats.cost += execution_stats.extra_cost+ graph_stats.cost += execution_stats.cost + execution_stats.extra_costif isinstance(execution_stats.error, Exception):graph_stats.node_error_count += 1@@ -716,6 +673,18 @@ class ExecutionProcessor:db_client=db_client,)+ # If the node failed because a nested tool charge raised IBE,+ # send the user notification so they understand why the run stopped.+ if status == ExecutionStatus.FAILED and isinstance(+ execution_stats.error, InsufficientBalanceError+ ):+ await billing.try_send_insufficient_funds_notif(+ node_exec.user_id,+ node_exec.graph_id,+ execution_stats.error,+ log_metadata,+ )+return execution_stats@async_time_measured@@ -935,7 +904,7 @@ class ExecutionProcessor:)finally:# Communication handling- self._handle_agent_run_notif(db_client, graph_exec, exec_stats)+ billing.handle_agent_run_notif(db_client, graph_exec, exec_stats)update_graph_execution_state(db_client=db_client,@@ -944,57 +913,18 @@ class ExecutionProcessor:stats=exec_stats,)- def _charge_usage(+ async def charge_node_usage(self,node_exec: NodeExecutionEntry,- execution_count: int,) -> tuple[int, int]:- total_cost = 0- remaining_balance = 0- db_client = get_db_client()- block = get_block(node_exec.block_id)- if not block:- logger.error(f"Block {node_exec.block_id} not found.")- return total_cost, 0+ return await billing.charge_node_usage(node_exec)- cost, matching_filter = block_usage_cost(- block=block, input_data=node_exec.inputs- )- if cost > 0:- remaining_balance = db_client.spend_credits(- user_id=node_exec.user_id,- cost=cost,- metadata=UsageTransactionMetadata(- graph_exec_id=node_exec.graph_exec_id,- graph_id=node_exec.graph_id,- node_exec_id=node_exec.node_exec_id,- node_id=node_exec.node_id,- block_id=node_exec.block_id,- block=block.name,- input=matching_filter,- reason=f"Ran block {node_exec.block_id} {block.name}",- ),- )- total_cost += cost-- cost, usage_count = execution_usage_cost(execution_count)- if cost > 0:- remaining_balance = db_client.spend_credits(- user_id=node_exec.user_id,- cost=cost,- metadata=UsageTransactionMetadata(- graph_exec_id=node_exec.graph_exec_id,- graph_id=node_exec.graph_id,- input={- "execution_count": usage_count,- "charge": "Execution Cost",- },- reason=f"Execution Cost for {usage_count} blocks of ex_id:{node_exec.graph_exec_id} g_id:{node_exec.graph_id}",- ),- )- total_cost += cost-- return total_cost, remaining_balance+ async def charge_extra_runtime_cost(+ self,+ node_exec: NodeExecutionEntry,+ extra_count: int,+ ) -> tuple[int, int]:+ return await billing.charge_extra_runtime_cost(node_exec, extra_count)@time_measureddef _on_graph_execution(@@ -1106,7 +1036,7 @@ class ExecutionProcessor:# Charge usage (may raise) — skipped for dry runstry:if not graph_exec.execution_context.dry_run:- cost, remaining_balance = self._charge_usage(+ cost, remaining_balance = billing.charge_usage(node_exec=queued_node_exec,execution_count=increment_execution_count(graph_exec.user_id@@ -1115,7 +1045,7 @@ class ExecutionProcessor:with execution_stats_lock:execution_stats.cost += cost# Check if we crossed the low balance threshold- self._handle_low_balance(+ billing.handle_low_balance(db_client=db_client,user_id=graph_exec.user_id,current_balance=remaining_balance,@@ -1135,7 +1065,7 @@ class ExecutionProcessor:status=ExecutionStatus.FAILED,)- self._handle_insufficient_funds_notif(+ billing.handle_insufficient_funds_notif(db_client,graph_exec.user_id,graph_exec.graph_id,@@ -1397,165 +1327,6 @@ class ExecutionProcessor:):execution_queue.add(next_execution)- def _handle_agent_run_notif(- self,- db_client: "DatabaseManagerClient",- graph_exec: GraphExecutionEntry,- exec_stats: GraphExecutionStats,- ):- metadata = db_client.get_graph_metadata(- graph_exec.graph_id, graph_exec.graph_version- )- outputs = db_client.get_node_executions(- graph_exec.graph_exec_id,- block_ids=[AgentOutputBlock().id],- )-- named_outputs = [- {- key: value[0] if key == "name" else value- for key, value in output.output_data.items()- }- for output in outputs- ]-- queue_notification(- NotificationEventModel(- user_id=graph_exec.user_id,- type=NotificationType.AGENT_RUN,- data=AgentRunData(- outputs=named_outputs,- agent_name=metadata.name if metadata else "Unknown Agent",- credits_used=exec_stats.cost,- execution_time=exec_stats.walltime,- graph_id=graph_exec.graph_id,- node_count=exec_stats.node_count,- ),- )- )-- def _handle_insufficient_funds_notif(- self,- db_client: "DatabaseManagerClient",- user_id: str,- graph_id: str,- e: InsufficientBalanceError,- ):- # Check if we've already sent a notification for this user+agent combo.- # We only send one notification per user per agent until they top up credits.- redis_key = f"{INSUFFICIENT_FUNDS_NOTIFIED_PREFIX}:{user_id}:{graph_id}"- try:- redis_client = redis.get_redis()- # SET NX returns True only if the key was newly set (didn't exist)- is_new_notification = redis_client.set(- redis_key,- "1",- nx=True,- ex=INSUFFICIENT_FUNDS_NOTIFIED_TTL_SECONDS,- )- if not is_new_notification:- # Already notified for this user+agent, skip all notifications- logger.debug(- f"Skipping duplicate insufficient funds notification for "- f"user={user_id}, graph={graph_id}"- )- return- except Exception as redis_error:- # If Redis fails, log and continue to send the notification- # (better to occasionally duplicate than to never notify)- logger.warning(- f"Failed to check/set insufficient funds notification flag in Redis: "- f"{redis_error}"- )-- shortfall = abs(e.amount) - e.balance- metadata = db_client.get_graph_metadata(graph_id)- base_url = (- settings.config.frontend_base_url or settings.config.platform_base_url- )-- # Queue user email notification- queue_notification(- NotificationEventModel(- user_id=user_id,- type=NotificationType.ZERO_BALANCE,- data=ZeroBalanceData(- current_balance=e.balance,- billing_page_link=f"{base_url}/profile/credits",- shortfall=shortfall,- agent_name=metadata.name if metadata else "Unknown Agent",- ),- )- )-- # Send Discord system alert- try:- user_email = db_client.get_user_email_by_id(user_id)-- alert_message = (- f"❌ **Insufficient Funds Alert**\n"- f"User: {user_email or user_id}\n"- f"Agent: {metadata.name if metadata else 'Unknown Agent'}\n"- f"Current balance: ${e.balance / 100:.2f}\n"- f"Attempted cost: ${abs(e.amount) / 100:.2f}\n"- f"Shortfall: ${abs(shortfall) / 100:.2f}\n"- f"[View User Details]({base_url}/admin/spending?search={user_email})"- )-- get_notification_manager_client().discord_system_alert(- alert_message, DiscordChannel.PRODUCT- )- except Exception as alert_error:- logger.error(- f"Failed to send insufficient funds Discord alert: {alert_error}"- )-- def _handle_low_balance(- self,- db_client: "DatabaseManagerClient",- user_id: str,- current_balance: int,- transaction_cost: int,- ):- """Check and handle low balance scenarios after a transaction"""- LOW_BALANCE_THRESHOLD = settings.config.low_balance_threshold-- balance_before = current_balance + transaction_cost-- if (- current_balance < LOW_BALANCE_THRESHOLD- and balance_before >= LOW_BALANCE_THRESHOLD- ):- base_url = (- settings.config.frontend_base_url or settings.config.platform_base_url- )- queue_notification(- NotificationEventModel(- user_id=user_id,- type=NotificationType.LOW_BALANCE,- data=LowBalanceData(- current_balance=current_balance,- billing_page_link=f"{base_url}/profile/credits",- ),- )- )-- try:- user_email = db_client.get_user_email_by_id(user_id)- alert_message = (- f"⚠️ **Low Balance Alert**\n"- f"User: {user_email or user_id}\n"- f"Balance dropped below ${LOW_BALANCE_THRESHOLD / 100:.2f}\n"- f"Current balance: ${current_balance / 100:.2f}\n"- f"Transaction cost: ${transaction_cost / 100:.2f}\n"- f"[View User Details]({base_url}/admin/spending?search={user_email})"- )- get_notification_manager_client().discord_system_alert(- alert_message, DiscordChannel.PRODUCT- )- except Exception as e:- logger.warning(f"Failed to send low balance Discord alert: {e}")-class ExecutionManager(AppProcess):def __init__(self):diff --git a/autogpt_platform/backend/backend/executor/manager_insufficient_funds_test.py b/autogpt_platform/backend/backend/executor/manager_insufficient_funds_test.pyindex 276c9f4f7a..ddbb4e0e1c 100644--- a/autogpt_platform/backend/backend/executor/manager_insufficient_funds_test.py+++ b/autogpt_platform/backend/backend/executor/manager_insufficient_funds_test.py@@ -4,9 +4,9 @@ import pytestfrom prisma.enums import NotificationTypefrom backend.data.notifications import ZeroBalanceData-from backend.executor.manager import (+from backend.executor import billing+from backend.executor.billing import (INSUFFICIENT_FUNDS_NOTIFIED_PREFIX,- ExecutionProcessor,clear_insufficient_funds_notifications,)from backend.util.exceptions import InsufficientBalanceError@@ -25,7 +25,6 @@ async def test_handle_insufficient_funds_sends_discord_alert_first_time():"""Test that the first insufficient funds notification sends a Discord alert."""- execution_processor = ExecutionProcessor()user_id = "test-user-123"graph_id = "test-graph-456"error = InsufficientBalanceError(@@ -36,13 +35,13 @@ async def test_handle_insufficient_funds_sends_discord_alert_first_time()with patch(- "backend.executor.manager.queue_notification"+ "backend.executor.billing.queue_notification") as mock_queue_notif, patch(- "backend.executor.manager.get_notification_manager_client"+ "backend.executor.billing.get_notification_manager_client") as mock_get_client, patch(- "backend.executor.manager.settings"+ "backend.executor.billing.settings") as mock_settings, patch(- "backend.executor.manager.redis"+ "backend.executor.billing.redis") as mock_redis_module:# Setup mocks@@ -63,7 +62,7 @@ async def test_handle_insufficient_funds_sends_discord_alert_first_time(mock_db_client.get_user_email_by_id.return_value = "[email protected]"# Test the insufficient funds handler- execution_processor._handle_insufficient_funds_notif(+ billing.handle_insufficient_funds_notif(db_client=mock_db_client,user_id=user_id,graph_id=graph_id,@@ -99,7 +98,6 @@ async def test_handle_insufficient_funds_skips_duplicate_notifications():"""Test that duplicate insufficient funds notifications skip both email and Discord."""- execution_processor = ExecutionProcessor()user_id = "test-user-123"graph_id = "test-graph-456"error = InsufficientBalanceError(@@ -110,13 +108,13 @@ async def test_handle_insufficient_funds_skips_duplicate_notifications()with patch(- "backend.executor.manager.queue_notification"+ "backend.executor.billing.queue_notification") as mock_queue_notif, patch(- "backend.executor.manager.get_notification_manager_client"+ "backend.executor.billing.get_notification_manager_client") as mock_get_client, patch(- "backend.executor.manager.settings"+ "backend.executor.billing.settings") as mock_settings, patch(- "backend.executor.manager.redis"+ "backend.executor.billing.redis") as mock_redis_module:# Setup mocks@@ -134,7 +132,7 @@ async def test_handle_insufficient_funds_skips_duplicate_notifications(mock_db_client.get_graph_metadata.return_value = MagicMock(name="Test Agent")# Test the insufficient funds handler- execution_processor._handle_insufficient_funds_notif(+ billing.handle_insufficient_funds_notif(db_client=mock_db_client,user_id=user_id,graph_id=graph_id,@@ -154,7 +152,6 @@ async def test_handle_insufficient_funds_different_agents_get_separate_alerts():"""Test that different agents for the same user get separate Discord alerts."""- execution_processor = ExecutionProcessor()user_id = "test-user-123"graph_id_1 = "test-graph-111"graph_id_2 = "test-graph-222"@@ -166,12 +163,12 @@ async def test_handle_insufficient_funds_different_agents_get_separate_alerts(amount=-714,)- with patch("backend.executor.manager.queue_notification"), patch(- "backend.executor.manager.get_notification_manager_client"+ with patch("backend.executor.billing.queue_notification"), patch(+ "backend.executor.billing.get_notification_manager_client") as mock_get_client, patch(- "backend.executor.manager.settings"+ "backend.executor.billing.settings") as mock_settings, patch(- "backend.executor.manager.redis"+ "backend.executor.billing.redis") as mock_redis_module:mock_client = MagicMock()@@ -190,7 +187,7 @@ async def test_handle_insufficient_funds_different_agents_get_separate_alerts(mock_db_client.get_user_email_by_id.return_value = "[email protected]"# First agent notification- execution_processor._handle_insufficient_funds_notif(+ billing.handle_insufficient_funds_notif(db_client=mock_db_client,user_id=user_id,graph_id=graph_id_1,@@ -198,7 +195,7 @@ async def test_handle_insufficient_funds_different_agents_get_separate_alerts()# Second agent notification- execution_processor._handle_insufficient_funds_notif(+ billing.handle_insufficient_funds_notif(db_client=mock_db_client,user_id=user_id,graph_id=graph_id_2,@@ -227,7 +224,7 @@ async def test_clear_insufficient_funds_notifications(server: SpinTestServer):user_id = "test-user-123"- with patch("backend.executor.manager.redis") as mock_redis_module:+ with patch("backend.executor.billing.redis") as mock_redis_module:mock_redis_client = MagicMock()# get_redis_async is an async function, so we need AsyncMock for it@@ -263,7 +260,7 @@ async def test_clear_insufficient_funds_notifications_no_keys(server: SpinTestSeuser_id = "test-user-no-notifications"- with patch("backend.executor.manager.redis") as mock_redis_module:+ with patch("backend.executor.billing.redis") as mock_redis_module:mock_redis_client = MagicMock()# get_redis_async is an async function, so we need AsyncMock for it@@ -290,7 +287,7 @@ async def test_clear_insufficient_funds_notifications_handles_redis_error(user_id = "test-user-redis-error"- with patch("backend.executor.manager.redis") as mock_redis_module:+ with patch("backend.executor.billing.redis") as mock_redis_module:# Mock get_redis_async to raise an errormock_redis_module.get_redis_async = AsyncMock(@@ -310,7 +307,6 @@ async def test_handle_insufficient_funds_continues_on_redis_error():"""Test that both email and Discord notifications are still sent when Redis fails."""- execution_processor = ExecutionProcessor()user_id = "test-user-123"graph_id = "test-graph-456"error = InsufficientBalanceError(@@ -321,13 +317,13 @@ async def test_handle_insufficient_funds_continues_on_redis_error()with patch(- "backend.executor.manager.queue_notification"+ "backend.executor.billing.queue_notification") as mock_queue_notif, patch(- "backend.executor.manager.get_notification_manager_client"+ "backend.executor.billing.get_notification_manager_client") as mock_get_client, patch(- "backend.executor.manager.settings"+ "backend.executor.billing.settings") as mock_settings, patch(- "backend.executor.manager.redis"+ "backend.executor.billing.redis") as mock_redis_module:mock_client = MagicMock()@@ -346,7 +342,7 @@ async def test_handle_insufficient_funds_continues_on_redis_error(mock_db_client.get_user_email_by_id.return_value = "[email protected]"# Test the insufficient funds handler- execution_processor._handle_insufficient_funds_notif(+ billing.handle_insufficient_funds_notif(db_client=mock_db_client,user_id=user_id,graph_id=graph_id,@@ -370,7 +366,7 @@ async def test_add_transaction_clears_notifications_on_grant(server: SpinTestSeruser_id = "test-user-grant-clear"with patch("backend.data.credit.query_raw_with_schema") as mock_query, patch(- "backend.executor.manager.redis"+ "backend.executor.billing.redis") as mock_redis_module:# Mock the query to return a successful transaction@@ -412,7 +408,7 @@ async def test_add_transaction_clears_notifications_on_top_up(server: SpinTestSeuser_id = "test-user-topup-clear"with patch("backend.data.credit.query_raw_with_schema") as mock_query, patch(- "backend.executor.manager.redis"+ "backend.executor.billing.redis") as mock_redis_module:# Mock the query to return a successful transaction@@ -450,7 +446,7 @@ async def test_add_transaction_skips_clearing_for_inactive_transaction(user_id = "test-user-inactive"with patch("backend.data.credit.query_raw_with_schema") as mock_query, patch(- "backend.executor.manager.redis"+ "backend.executor.billing.redis") as mock_redis_module:# Mock the query to return a successful transaction@@ -486,7 +482,7 @@ async def test_add_transaction_skips_clearing_for_usage_transaction(user_id = "test-user-usage"with patch("backend.data.credit.query_raw_with_schema") as mock_query, patch(- "backend.executor.manager.redis"+ "backend.executor.billing.redis") as mock_redis_module:# Mock the query to return a successful transaction@@ -521,7 +517,7 @@ async def test_enable_transaction_clears_notifications(server: SpinTestServer):with patch("backend.data.credit.CreditTransaction") as mock_credit_tx, patch("backend.data.credit.query_raw_with_schema"- ) as mock_query, patch("backend.executor.manager.redis") as mock_redis_module:+ ) as mock_query, patch("backend.executor.billing.redis") as mock_redis_module:# Mock finding the pending transactionmock_transaction = MagicMock()diff --git a/autogpt_platform/backend/backend/executor/manager_low_balance_test.py b/autogpt_platform/backend/backend/executor/manager_low_balance_test.pyindex d51ffb2511..fe99379782 100644--- a/autogpt_platform/backend/backend/executor/manager_low_balance_test.py+++ b/autogpt_platform/backend/backend/executor/manager_low_balance_test.py@@ -4,26 +4,25 @@ import pytestfrom prisma.enums import NotificationTypefrom backend.data.notifications import LowBalanceData-from backend.executor.manager import ExecutionProcessor+from backend.executor import billingfrom backend.util.test import SpinTestServer@pytest.mark.asyncio(loop_scope="session")async def test_handle_low_balance_threshold_crossing(server: SpinTestServer):- """Test that _handle_low_balance triggers notification when crossing threshold."""+ """Test that handle_low_balance triggers notification when crossing threshold."""- execution_processor = ExecutionProcessor()user_id = "test-user-123"current_balance = 400 # $4 - below $5 thresholdtransaction_cost = 600 # $6 transaction# Mock dependencieswith patch(- "backend.executor.manager.queue_notification"+ "backend.executor.billing.queue_notification") as mock_queue_notif, patch(- "backend.executor.manager.get_notification_manager_client"+ "backend.executor.billing.get_notification_manager_client") as mock_get_client, patch(- "backend.executor.manager.settings"+ "backend.executor.billing.settings") as mock_settings:# Setup mocks@@ -37,7 +36,7 @@ async def test_handle_low_balance_threshold_crossing(server: SpinTestServer):mock_db_client.get_user_email_by_id.return_value = "[email protected]"# Test the low balance handler- execution_processor._handle_low_balance(+ billing.handle_low_balance(db_client=mock_db_client,user_id=user_id,current_balance=current_balance,@@ -69,7 +68,6 @@ async def test_handle_low_balance_no_notification_when_not_crossing():"""Test that no notification is sent when not crossing the threshold."""- execution_processor = ExecutionProcessor()user_id = "test-user-123"current_balance = 600 # $6 - above $5 thresholdtransaction_cost = (@@ -78,11 +76,11 @@ async def test_handle_low_balance_no_notification_when_not_crossing(# Mock dependencieswith patch(- "backend.executor.manager.queue_notification"+ "backend.executor.billing.queue_notification") as mock_queue_notif, patch(- "backend.executor.manager.get_notification_manager_client"+ "backend.executor.billing.get_notification_manager_client") as mock_get_client, patch(- "backend.executor.manager.settings"+ "backend.executor.billing.settings") as mock_settings:# Setup mocks@@ -94,7 +92,7 @@ async def test_handle_low_balance_no_notification_when_not_crossing(mock_db_client = MagicMock()# Test the low balance handler- execution_processor._handle_low_balance(+ billing.handle_low_balance(db_client=mock_db_client,user_id=user_id,current_balance=current_balance,@@ -112,7 +110,6 @@ async def test_handle_low_balance_no_duplicate_when_already_below():"""Test that no notification is sent when already below threshold."""- execution_processor = ExecutionProcessor()user_id = "test-user-123"current_balance = 300 # $3 - below $5 thresholdtransaction_cost = (@@ -121,11 +118,11 @@ async def test_handle_low_balance_no_duplicate_when_already_below(# Mock dependencieswith patch(- "backend.executor.manager.queue_notification"+ "backend.executor.billing.queue_notification") as mock_queue_notif, patch(- "backend.executor.manager.get_notification_manager_client"+ "backend.executor.billing.get_notification_manager_client") as mock_get_client, patch(- "backend.executor.manager.settings"+ "backend.executor.billing.settings") as mock_settings:# Setup mocks@@ -137,7 +134,7 @@ async def test_handle_low_balance_no_duplicate_when_already_below(mock_db_client = MagicMock()# Test the low balance handler- execution_processor._handle_low_balance(+ billing.handle_low_balance(db_client=mock_db_client,user_id=user_id,current_balance=current_balance,