Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/executor): don't re-charge REVIEW/RUNNING nodes on re-dispatch (#15327)

Nicholas Tindleef7d851b4d8a
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.py
index b5c0b551d5..58fea5c45e 100644
--- a/autogpt_platform/backend/backend/data/execution.py
+++ b/autogpt_platform/backend/backend/data/execution.py
@@ -1561,6 +1561,11 @@ class NodeExecutionEntry(BaseModel):
# reconcile, leaving the delta computed against a different number).
# Excluded from dumps — in-memory only, scoped to a single live execution.
pre_flight_charge: Optional[int] = Field(default=None, exclude=True)
+ # The status a REVIEW/RUNNING row had when a resumed or redelivered run
+ # dispatched it again. Its first dispatch was already charged, so this
+ # one must not be (#15272), and the status says what is still billed:
+ # see `billing.redispatch_reconcile_baseline`. Also exclude=True.
+ redispatched_from: Optional[ExecutionStatus] = Field(default=None, exclude=True)
# When the node was queued and by what: ``graph_start`` for a node queued
# before this executor picked the run up (an entry node, or one left
# queued by an interrupted run), ``node_output`` for one queued by an
diff --git a/autogpt_platform/backend/backend/executor/billing.py b/autogpt_platform/backend/backend/executor/billing.py
index 4bcc3bad08..d557252422 100644
--- a/autogpt_platform/backend/backend/executor/billing.py
+++ b/autogpt_platform/backend/backend/executor/billing.py
@@ -9,7 +9,7 @@ from backend.blocks import get_block
from backend.blocks._base import Block
from backend.data import redis_client as redis
from backend.data.credit import UsageTransactionMetadata
-from backend.data.execution import NodeExecutionEntry
+from backend.data.execution import ExecutionStatus, NodeExecutionEntry
from backend.data.expert_spend import add_weekly_spend, add_weekly_spend_sync
from backend.data.model import NodeExecutionStats
from backend.notifications.alert_causes import LowBalanceCause, ZeroBalanceCause
@@ -220,6 +220,32 @@ def charge_usage(
return total_cost, remaining_balance, block_pre_flight
+def redispatch_reconcile_baseline(node_exec: NodeExecutionEntry) -> int | None:
+ """What a re-dispatched node still has billed, for its reconciliation to
+ settle against. Its first dispatch was charged and this one is not (#15272).
+
+ - REVIEW: the run that parked the block returned without doing anything,
+ and ``on_node_execution`` reconciled it as a COMPLETED run with empty
+ stats. That settled the charge down to the block's price for empty
+ stats: the run price for per-run blocks, 0 for usage-priced ones. (A
+ SECOND-priced park would have settled to 1 credit, since a sub-second
+ run bills one unit; no block that can pause is SECOND-priced.) That
+ price is returned, so the resumed run settles to one run's cost.
+ - RUNNING: the pod died before reconciling, so the pre-flight charge still
+ stands. ``None`` makes reconciliation read the block's pre-flight price,
+ which equals that charge unless the price changed in between.
+ """
+ if node_exec.redispatched_from != ExecutionStatus.REVIEW:
+ return None
+ block = get_block(node_exec.block_id)
+ if not block:
+ return None
+ settled, _ = block_usage_cost(
+ block=block, input_data=node_exec.inputs, stats=NodeExecutionStats()
+ )
+ return settled
+
+
async def charge_reconciled_usage(
node_exec: NodeExecutionEntry,
stats: NodeExecutionStats,
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index 41f06306a2..5431a9904d 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -188,6 +188,11 @@ KNOWN_GRAPH_EXECUTION_ERRORS = (
# and leaves the node QUEUED after its run has finished.
SKIPPED_NODE_STATUSES = (ExecutionStatus.RUNNING, ExecutionStatus.COMPLETED)
+# Rows charged on their first dispatch: the charge precedes the RUNNING write,
+# and a block parks itself in REVIEW mid-run. A resume or a redelivery after a
+# pod died dispatches them without a second charge; gaps: #15327, Known limits.
+BILLED_REDISPATCH_STATUSES = (ExecutionStatus.REVIEW, ExecutionStatus.RUNNING)
+
def _propagate_node_failure(
graph_stats: GraphExecutionStats,
@@ -1236,6 +1241,13 @@ class ExecutionProcessor:
node_entry = node_exec.to_node_execution_entry(
graph_exec.execution_context
)
+ # A REVIEW/RUNNING row was charged when it was first
+ # dispatched, so it must not be charged again (#15272).
+ # A TERMINATED row is charged again: its interrupt settled the
+ # first charge, so a per-run block pays twice. Not reachable:
+ # no caller requeues a TERMINATED run.
+ if node_exec.status in BILLED_REDISPATCH_STATUSES:
+ node_entry.redispatched_from = node_exec.status
node_entry.queued_at = node_exec.queue_time
node_entry.queued_by = "graph_start"
execution_queue.add(node_entry)
@@ -1271,9 +1283,14 @@ class ExecutionProcessor:
f"for node {queued_node_exec.node_id}",
)
- # Charge usage (may raise) — skipped for dry runs
+ # Charge usage (may raise) — skipped for dry runs and for
+ # REVIEW/RUNNING re-dispatches that were already charged.
try:
- if not graph_exec.execution_context.dry_run:
+ if queued_node_exec.redispatched_from is not None:
+ queued_node_exec.pre_flight_charge = (
+ billing.redispatch_reconcile_baseline(queued_node_exec)
+ )
+ elif not graph_exec.execution_context.dry_run:
(
cost,
remaining_balance,
diff --git a/autogpt_platform/backend/backend/executor/manager_activity_status_test.py b/autogpt_platform/backend/backend/executor/manager_activity_status_test.py
index 1dc148821b..f842c2ff48 100644
--- a/autogpt_platform/backend/backend/executor/manager_activity_status_test.py
+++ b/autogpt_platform/backend/backend/executor/manager_activity_status_test.py
@@ -9,6 +9,7 @@ from backend.data.execution import (
ExecutionStatus,
GraphExecutionEntry,
GraphExecutionMeta,
+ NodeExecutionEntry,
)
from backend.data.model import GraphExecutionStats
from backend.executor.manager import (
@@ -329,3 +330,101 @@ def test_nested_credit_failure_marker_respects_cancellation_precedence(
assert status == expected_status
assert stats.failure_reason == ExecutionFailureReason.INSUFFICIENT_BALANCE
assert stats.error == "Organization has 0 credits but needs 25"
+
+
+def _queued_node(node_exec_id: str, status: ExecutionStatus):
+ entry = NodeExecutionEntry(
+ user_id="user-1",
+ graph_exec_id="exec-1",
+ graph_id="graph-1",
+ graph_version=1,
+ node_exec_id=node_exec_id,
+ node_id=f"node-{node_exec_id}",
+ block_id="block-1",
+ inputs={},
+ )
+ return SimpleNamespace(
+ status=status,
+ queue_time=None,
+ to_node_execution_entry=lambda execution_context: entry.model_copy(
+ update={"execution_context": execution_context}
+ ),
+ )
+
+
+def test_redispatched_review_and_running_nodes_are_not_charged_again():
+ """#15272: on resume the executor re-queues REVIEW/RUNNING node execs.
+ They were charged on first dispatch, so they are not charged again.
+ A TERMINATED row is: it was reconciled when it was interrupted, so a
+ re-dispatch is a new run."""
+ execution = GraphExecutionEntry(
+ user_id="user-1",
+ graph_exec_id="exec-1",
+ graph_id="graph-1",
+ graph_version=1,
+ )
+ db_client = MagicMock()
+ db_client.get_credits.return_value = 100
+ db_client.get_node_executions.return_value = [
+ _queued_node("review", ExecutionStatus.REVIEW),
+ _queued_node("running", ExecutionStatus.RUNNING),
+ _queued_node("terminated", ExecutionStatus.TERMINATED),
+ _queued_node("queued", ExecutionStatus.QUEUED),
+ ]
+ db_client.has_pending_reviews_for_graph_exec.return_value = False
+ processor = ExecutionProcessor()
+ processor._cleanup_graph_execution = MagicMock()
+ processor.node_evaluation_loop = MagicMock()
+ processor.node_execution_loop = MagicMock()
+ processor.on_node_execution = MagicMock()
+ completed_future = MagicMock()
+ completed_future.result.return_value = None
+
+ def complete_coroutine(coroutine, _loop):
+ # The stubbed on_node_execution returns None rather than a coroutine.
+ if coroutine is not None:
+ coroutine.close()
+ return completed_future
+
+ dispatched: list = []
+ processor.on_node_execution.side_effect = lambda node_exec, **_: dispatched.append(
+ node_exec
+ )
+ charge_usage = MagicMock(return_value=(7, 93, 7))
+ increment = MagicMock(return_value=1)
+
+ with (
+ patch("backend.executor.manager.get_db_client", return_value=db_client),
+ patch(
+ "backend.executor.manager.asyncio.run_coroutine_threadsafe",
+ side_effect=complete_coroutine,
+ ),
+ patch("backend.executor.manager.billing.charge_usage", charge_usage),
+ patch(
+ "backend.executor.manager.billing.redispatch_reconcile_baseline",
+ side_effect=lambda node_exec: {"review": 3, "running": None}[
+ node_exec.node_exec_id
+ ],
+ ),
+ patch("backend.executor.manager.billing.handle_low_balance"),
+ patch("backend.executor.manager.increment_execution_count", increment),
+ ):
+ processor._on_graph_execution(
+ graph_exec=execution,
+ cancel=threading.Event(),
+ log_metadata=MagicMock(),
+ execution_stats=GraphExecutionStats(),
+ cluster_lock=MagicMock(),
+ )
+
+ charged = [c.kwargs["node_exec"].node_exec_id for c in charge_usage.call_args_list]
+ assert charged == ["terminated", "queued"]
+ assert increment.call_count == 2
+ by_id = {n.node_exec_id: n for n in dispatched}
+ assert set(by_id) == {"review", "running", "terminated", "queued"}
+ # Reconciliation of a re-dispatched row settles against what its first
+ # dispatch left billed; charged rows settle against the new charge.
+ assert by_id["review"].pre_flight_charge == 3
+ assert by_id["running"].pre_flight_charge is None
+ assert by_id["terminated"].pre_flight_charge == 7
+ assert by_id["queued"].pre_flight_charge == 7
diff --git a/autogpt_platform/backend/backend/executor/manager_redispatch_billing_test.py b/autogpt_platform/backend/backend/executor/manager_redispatch_billing_test.py
new file mode 100644
index 0000000000..211422bdc4
--- /dev/null
+++ b/autogpt_platform/backend/backend/executor/manager_redispatch_billing_test.py
@@ -0,0 +1,284 @@
+"""A node execution is billed once however many times it is dispatched (#15272).
+
+These drive the real ``_on_graph_execution`` → ``charge_usage`` →
+``on_node_execution`` → ``charge_reconciled_usage`` chain, on real event loops,
+against a wallet that records every spend. Only the block run itself
+(``_on_node_execution``) is replaced, so each dispatch can park the block for
+review, run it, or die mid-run the way the real one does.
+"""
+
+import asyncio
+import threading
+from collections.abc import Iterator
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from backend.blocks._base import BlockCost, BlockCostType
+from backend.blocks.basic import StoreValueBlock
+from backend.blocks.helpers.review import HITLReviewHelper
+from backend.data.block_cost_config import BLOCK_COSTS
+from backend.data.execution import (
+ ExecutionContext,
+ ExecutionStatus,
+ GraphExecutionEntry,
+ NodeExecutionEntry,
+)
+from backend.data.graph import Node
+from backend.data.model import GraphExecutionStats, NodeExecutionStats
+from backend.executor.manager import ExecutionProcessor
+from backend.executor.utils import NodeExecutionProgress
+from backend.util.decorator import TimingInfo
+
+USER_ID = "user-1"
+GRAPH_EXEC_ID = "exec-1"
+PRE_FLIGHT_ESTIMATE = 10
+PER_RUN_PRICE = 2
+RUN_USD = 15.0 # a COST_USD block at 1 credit per dollar bills 15 for the run
+
+PER_RUN = [BlockCost(cost_amount=PER_RUN_PRICE, cost_type=BlockCostType.RUN)]
+USAGE_PRICED = [BlockCost(cost_amount=1, cost_type=BlockCostType.COST_USD)]
+
+
+class Wallet:
+ """Every credit spend, sync (pre-flight) and async (reconciliation)."""
+
+ def __init__(self) -> None:
+ self.spent: list[int] = []
+
+ @property
+ def total(self) -> int:
+ return sum(self.spent)
+
+ def spend(self, user_id: str, cost: int, metadata, **_) -> int:
+ self.spent.append(cost)
+ return 1_000 - self.total
+
+ async def spend_async(self, user_id: str, cost: int, metadata, **_) -> int:
+ return self.spend(user_id, cost, metadata)
+
+
+class NodeRow:
+ """The persisted node execution: the only state that survives a dispatch."""
+
+ def __init__(self) -> None:
+ self.status = ExecutionStatus.QUEUED
+ self.queue_time = None
+ self.entry = NodeExecutionEntry(
+ user_id=USER_ID,
+ graph_exec_id=GRAPH_EXEC_ID,
+ graph_id="graph-1",
+ graph_version=1,
+ node_exec_id="node-exec-1",
+ node_id="node-1",
+ block_id=StoreValueBlock().id,
+ inputs={"input": "post this"},
+ )
+
+ def to_node_execution_entry(self, execution_context: ExecutionContext):
+ return self.entry.model_copy(update={"execution_context": execution_context})
+
+
+def _parks_for_review(row: NodeRow):
+ """The block writes REVIEW and returns without output, so the run reports
+ COMPLETED (``_base.py`` ``_execute`` returns early on a pending review)."""
+
+ async def run(**_):
+ row.status = ExecutionStatus.REVIEW
+ return TimingInfo(cpu_time=0.0, wall_time=0.0), ExecutionStatus.COMPLETED
+
+ return run
+
+
+def _runs(row: NodeRow, *, approved: bool = True):
+ async def run(*, stats: NodeExecutionStats, **_):
+ row.status = ExecutionStatus.RUNNING
+ if not approved:
+ stats.error = ValueError("Block execution rejected by reviewer")
+ return TimingInfo(cpu_time=0.0, wall_time=0.0), ExecutionStatus.FAILED
+ stats.provider_cost = RUN_USD
+ stats.provider_cost_type = "cost_usd"
+ return TimingInfo(cpu_time=0.0, wall_time=0.0), ExecutionStatus.COMPLETED
+
+ return run
+
+
+def _dies_mid_run(row: NodeRow):
+ """The pod goes away after RUNNING is written, before reconciliation."""
+
+ async def run(**_):
+ row.status = ExecutionStatus.RUNNING
+ return TimingInfo(cpu_time=0.0, wall_time=0.0), RuntimeError("pod died")
+
+ return run
+
+
+def processor() -> Iterator[ExecutionProcessor]:
+ processor = ExecutionProcessor()
+ loops = [asyncio.new_event_loop() for _ in range(2)]
+ threads = [threading.Thread(target=loop.run_forever, daemon=True) for loop in loops]
+ for thread in threads:
+ thread.start()
+ processor.node_execution_loop, processor.node_evaluation_loop = loops
+ processor._cleanup_graph_execution = MagicMock()
+ yield processor
+ for loop, thread in zip(loops, threads):
+ loop.call_soon_threadsafe(loop.stop)
+ thread.join(timeout=5)
+ loop.close()
+
+
+def harness(processor: ExecutionProcessor):
+ wallet = Wallet()
+ row = NodeRow()
+ block = StoreValueBlock()
+
+ sync_db = MagicMock()
+ sync_db.spend_credits.side_effect = wallet.spend
+ sync_db.get_credits.return_value = 1_000
+ sync_db.get_node_executions.side_effect = lambda _id, statuses: (
+ [row] if row.status in statuses else []
+ )
+ sync_db.has_pending_reviews_for_graph_exec.side_effect = (
+ lambda _id: row.status == ExecutionStatus.REVIEW
+ )
+
+ async_db = MagicMock()
+ async_db.spend_credits = AsyncMock(side_effect=wallet.spend_async)
+ async_db.get_node = AsyncMock(return_value=SimpleNamespace(block=block))
+
+ original_costs = BLOCK_COSTS.get(StoreValueBlock)
+ with (
+ patch("backend.executor.manager.get_db_client", return_value=sync_db),
+ patch("backend.executor.billing.get_db_client", return_value=sync_db),
+ patch("backend.executor.manager.get_db_async_client", return_value=async_db),
+ patch(
+ "backend.executor.billing.get_database_manager_async_client",
+ return_value=async_db,
+ ),
+ patch(
+ "backend.executor.utils.get_preflight_estimate",
+ return_value=PRE_FLIGHT_ESTIMATE,
+ ),
+ patch("backend.executor.manager.increment_execution_count", return_value=1),
+ patch("backend.executor.billing.execution_usage_cost", return_value=(0, 0)),
+ patch("backend.executor.billing.handle_low_balance"),
+ patch("backend.executor.manager.automod_manager") as automod,
+ patch("backend.executor.manager.log_system_credential_cost", AsyncMock()),
+ patch(
+ "backend.executor.manager.activity_events.log_node_integration_activity",
+ AsyncMock(),
+ ),
+ patch("backend.executor.manager.async_update_node_execution_status"),
+ patch("backend.executor.manager.async_update_graph_execution_state"),
+ ):
+ automod.moderate_graph_execution_inputs = AsyncMock(return_value=None)
+ automod.moderate_graph_execution_outputs = AsyncMock(return_value=None)
+
+ def dispatch(block_run) -> ExecutionStatus:
+ processor._on_node_execution = block_run
+ _, status = processor._on_graph_execution(
+ graph_exec=GraphExecutionEntry(
+ user_id=USER_ID,
+ graph_exec_id=GRAPH_EXEC_ID,
+ graph_id="graph-1",
+ graph_version=1,
+ ),
+ cancel=threading.Event(),
+ log_metadata=MagicMock(),
+ execution_stats=GraphExecutionStats(),
+ cluster_lock=MagicMock(),
+ )
+ return status
+
+ def price(costs: list[BlockCost]) -> None:
+ BLOCK_COSTS[StoreValueBlock] = costs
+
+ yield SimpleNamespace(wallet=wallet, row=row, dispatch=dispatch, price=price)
+
+ if original_costs is None:
+ BLOCK_COSTS.pop(StoreValueBlock, None)
+ else:
+ BLOCK_COSTS[StoreValueBlock] = original_costs
+
+
+ "costs, approved, billed",
+ [
+ pytest.param(PER_RUN, True, PER_RUN_PRICE, id="per-run, approved"),
+ pytest.param(PER_RUN, False, PER_RUN_PRICE, id="per-run, rejected"),
+ pytest.param(USAGE_PRICED, True, int(RUN_USD), id="usage-priced, approved"),
+ pytest.param(USAGE_PRICED, False, 0, id="usage-priced, rejected"),
+ ],
+)
+def test_a_review_resume_bills_the_block_once(harness, costs, approved, billed):
+ harness.price(costs)
+
+ assert harness.dispatch(_parks_for_review(harness.row)) == ExecutionStatus.REVIEW
+ assert harness.row.status == ExecutionStatus.REVIEW
+
+ harness.dispatch(_runs(harness.row, approved=approved))
+
+ assert harness.wallet.total == billed, harness.wallet.spent
+
+
+ "costs, billed",
+ [
+ pytest.param(PER_RUN, PER_RUN_PRICE, id="per-run"),
+ pytest.param(USAGE_PRICED, int(RUN_USD), id="usage-priced"),
+ ],
+)
+def test_a_redelivery_after_a_pod_died_bills_the_block_once(harness, costs, billed):
+ harness.price(costs)
+
+ harness.dispatch(_dies_mid_run(harness.row))
+ assert harness.row.status == ExecutionStatus.RUNNING
+
+ harness.dispatch(_runs(harness.row))
+
+ assert harness.wallet.total == billed, harness.wallet.spent
+
+
+async def test_a_parked_block_reports_a_completed_run_with_nothing_to_bill():
+ """The premise ``_parks_for_review`` stubs, through the real block run: a
+ block that parks for review yields nothing and its run reports COMPLETED."""
+ block = StoreValueBlock()
+ block.is_irreversible_action = True
+ row = NodeRow()
+ processor = ExecutionProcessor()
+ processor.creds_manager = MagicMock()
+ progress = NodeExecutionProgress()
+ stats = NodeExecutionStats()
+
+ with (
+ patch("backend.data.graph.get_block", return_value=block),
+ patch("backend.executor.manager.async_update_node_execution_status"),
+ patch.object(
+ HITLReviewHelper, "handle_review_decision", AsyncMock(return_value=None)
+ ) as review,
+ ):
+ _, status = await processor._on_node_execution(
+ node=Node(id=row.entry.node_id, block_id=block.id),
+ node_exec=row.to_node_execution_entry(
+ ExecutionContext(
+ graph_exec_id=GRAPH_EXEC_ID, sensitive_action_safe_mode=True
+ )
+ ),
+ node_exec_progress=progress,
+ stats=stats,
+ db_client=MagicMock(
+ upsert_execution_output=AsyncMock(),
+ get_node_execution=AsyncMock(return_value=None),
+ ),
+ log_metadata=MagicMock(),
+ )
+
+ review.assert_awaited_once()
+ assert status == ExecutionStatus.COMPLETED
+ assert not progress.output
+ assert stats.error is None
+ assert stats.provider_cost is None