Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): allow negative balance on post-flight reconciliation (#13043)

Zamil Majdy3c7133a5dd3d
diff --git a/autogpt_platform/backend/backend/data/credit.py b/autogpt_platform/backend/backend/data/credit.py
index bbaf4d8b54..2b99533f2f 100644
--- a/autogpt_platform/backend/backend/data/credit.py
+++ b/autogpt_platform/backend/backend/data/credit.py
@@ -144,6 +144,7 @@ class UserCreditBase(ABC):
user_id: str,
cost: int,
metadata: UsageTransactionMetadata,
+ fail_insufficient_credits: bool = True,
) -> int:
"""
Spend the credits for the user based on the cost.
@@ -152,6 +153,12 @@ class UserCreditBase(ABC):
user_id (str): The user ID.
cost (int): The cost to spend.
metadata (UsageTransactionMetadata): The metadata of the transaction.
+ fail_insufficient_credits (bool): When True (default) raise
+ InsufficientBalanceError if the wallet can't cover the spend.
+ When False the spend is recorded unconditionally and the
+ balance may go negative — used by post-flight reconciliation
+ so a delta charge that exceeds the wallet is captured as
+ debt instead of silently leaking.
Returns:
int: The remaining balance.
@@ -673,6 +680,7 @@ class UserCredit(UserCreditBase):
user_id: str,
cost: int,
metadata: UsageTransactionMetadata,
+ fail_insufficient_credits: bool = True,
) -> int:
if cost == 0:
return 0
@@ -682,6 +690,7 @@ class UserCredit(UserCreditBase):
amount=-cost,
transaction_type=CreditTransactionType.USAGE,
metadata=SafeJson(metadata.model_dump()),
+ fail_insufficient_credits=fail_insufficient_credits,
)
# Auto top-up if balance is below threshold.
diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.py
index 82b1bb59c9..1d297a89d0 100644
--- a/autogpt_platform/backend/backend/data/db_manager.py
+++ b/autogpt_platform/backend/backend/data/db_manager.py
@@ -147,10 +147,15 @@ R = TypeVar("R")
async def _spend_credits(
- user_id: str, cost: int, metadata: UsageTransactionMetadata
+ user_id: str,
+ cost: int,
+ metadata: UsageTransactionMetadata,
+ fail_insufficient_credits: bool = True,
) -> int:
user_credit_model = await get_user_credit_model(user_id)
- return await user_credit_model.spend_credits(user_id, cost, metadata)
+ return await user_credit_model.spend_credits(
+ user_id, cost, metadata, fail_insufficient_credits=fail_insufficient_credits
+ )
async def _get_credits(user_id: str) -> int:
diff --git a/autogpt_platform/backend/backend/executor/billing.py b/autogpt_platform/backend/backend/executor/billing.py
index 4c97886330..58e89533c5 100644
--- a/autogpt_platform/backend/backend/executor/billing.py
+++ b/autogpt_platform/backend/backend/executor/billing.py
@@ -193,15 +193,22 @@ async def charge_reconciled_usage(
"""Charge the dynamic portion of a block's cost from its execution stats.
Computes post-flight cost from the execution stats and settles the delta
- against the pre-flight estimate. Positive delta → charge the user; negative
- delta → refund the overcharge (happens when a TOKENS block's flat
- MODEL_COST floor exceeds the real token-metered cost). Zero delta is a
- no-op — common for RUN-only blocks and any balanced estimate.
+ against the pre-flight estimate. Positive delta → charge the user (allowed
+ to push the balance negative — see below); negative delta → refund the
+ overcharge (happens when a TOKENS block's flat MODEL_COST floor exceeds
+ the real token-metered cost). Zero delta is a no-op — common for RUN-only
+ blocks and any balanced estimate.
+
+ Negative-balance handling: positive deltas are recorded with
+ ``fail_insufficient_credits=False`` so a wallet that can't cover the
+ shortfall ends up in debt rather than producing a `billing_leak`. The
+ block already ran and the provider was already paid in real $$ — failing
+ the spend would just hide the cost. The debt is naturally cleared on the
+ next top-up (auto or manual), since top-ups simply add to the balance.
Called once per node execution AFTER the block has finished running and
- stats (walltime, tokens, provider_cost) are populated. Swallows its own
- InsufficientBalanceError because reconciliation must never poison the
- success path — log and move on; the shortfall is captured in telemetry.
+ stats (walltime, tokens, provider_cost) are populated. Swallows any
+ unexpected exception so reconciliation never poisons the success path.
"""
try:
db_client = get_database_manager_async_client()
@@ -221,6 +228,13 @@ async def charge_reconciled_usage(
# amount is positive (i.e. credits back to the wallet). We reuse the
# USAGE type so the refund is attributable to the same graph
# execution in credit history.
+ #
+ # `fail_insufficient_credits=False` is passed for every reconciliation
+ # spend. For positive deltas this bypasses the balance guard so the
+ # spend always lands — the wallet may go negative; that's the point,
+ # we'd rather record real debt than leak the cost. For negative deltas
+ # (refunds) the flag is moot: `amount = -cost > 0`, so the SQL guard's
+ # `$2 >= 0` short-circuit holds regardless.
remaining_balance = await db_client.spend_credits(
user_id=node_exec.user_id,
cost=delta,
@@ -237,6 +251,7 @@ async def charge_reconciled_usage(
f"actual={post_flight} credits, pre-flight={pre_flight}"
),
),
+ fail_insufficient_credits=False,
)
# Refunds can't push the balance below the threshold — skip.
if delta > 0:
@@ -251,38 +266,6 @@ async def charge_reconciled_usage(
delta,
)
return delta, remaining_balance
- except InsufficientBalanceError as e:
- # Billing leak: work is already done, but the user cannot pay. Emit a
- # structured ERROR so alerting can pick it up and ping the platform
- # channel so the leak is visible in real time.
- logger.error(
- "billing_leak: insufficient balance after post-flight reconciliation",
- extra={
- "billing_leak": True,
- "user_id": node_exec.user_id,
- "graph_id": node_exec.graph_id,
- "graph_exec_id": node_exec.graph_exec_id,
- "node_exec_id": node_exec.node_exec_id,
- "block_id": node_exec.block_id,
- "cost": abs(e.amount),
- "balance": e.balance,
- "error": str(e),
- },
- )
- try:
- await discord_send_alert(
- f"⚠️ billing_leak (post-flight reconciliation)\n"
- f"user={node_exec.user_id} graph={node_exec.graph_id} "
- f"exec={node_exec.graph_exec_id}\n"
- f"block={node_exec.block_id} needed={abs(e.amount)} "
- f"balance={e.balance}",
- DiscordChannel.PLATFORM,
- )
- except Exception:
- # Discord dispatch is best-effort; never let alert failure poison
- # the success path of a completed block.
- logger.exception("billing_leak discord alert dispatch failed")
- return 0, 0
except Exception:
logger.exception(
f"charge_reconciled_usage failed unexpectedly for block "
diff --git a/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py b/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py
index 58e3510fe8..4e0e27546b 100644
--- a/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py
+++ b/autogpt_platform/backend/backend/executor/billing_reconciliation_test.py
@@ -158,6 +158,41 @@ async def test_cost_usd_charges_post_flight_delta(tmp_block_costs_override):
assert lb_args[3] == 5
+async def test_positive_delta_passes_fail_insufficient_credits_false(
+ tmp_block_costs_override,
+):
+ """Reconciliation must call spend_credits with
+ fail_insufficient_credits=False on a positive delta — the wallet is
+ allowed to go negative so the platform records debt instead of leaking
+ the cost."""
+ tmp_block_costs_override(
+ [BlockCost(cost_amount=100, cost_type=BlockCostType.COST_USD)]
+ )
+ exec_entry = _node_exec(SearchTheWebBlock().id)
+ stats = NodeExecutionStats(provider_cost=0.05, provider_cost_type="cost_usd")
+
+ # Wallet went negative: the user had 1 credit, the delta is 5, the new
+ # balance is -4. spend_credits returns the post-spend balance.
+ db_client = _async_db_client(spend_credits_return=-4)
+ with (
+ patch(
+ "backend.executor.billing.get_database_manager_async_client",
+ return_value=db_client,
+ ),
+ patch("backend.executor.billing.get_db_client", return_value=MagicMock()),
+ patch("backend.executor.billing.handle_low_balance"),
+ ):
+ delta, remaining = await charge_reconciled_usage(exec_entry, stats)
+
+ assert delta == 5
+ assert remaining == -4
+ db_client.spend_credits.assert_awaited_once()
+ call_kwargs = db_client.spend_credits.await_args.kwargs
+ assert call_kwargs["cost"] == 5
+ assert call_kwargs["fail_insufficient_credits"] is False
+
+
@pytest.mark.asyncio
async def test_missing_block_returns_zero():
exec_entry = _node_exec("deadbeef-0000-0000-0000-000000000000")