Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

dx(backend/copilot): test the billing audit's mischarge report, and say which relaunches keep the cost baseline (#15404)

Nicholas Tindlef7d24f805e11
diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/audit_test.py b/autogpt_platform/backend/backend/copilot/billing_e2e/audit_test.py
new file mode 100644
index 0000000000..87125f5e23
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/audit_test.py
@@ -0,0 +1,88 @@
+"""The audit on its own: a relaunched turn owes only the attempt that succeeded."""
+
+from backend.copilot.model import Usage
+from backend.data.platform_cost import PlatformCostEntry, usd_to_microdollars
+
+from .audit import _audit_turn
+from .invoice import ANTHROPIC_PRICES, SONNET, Call, scripted_usage
+from .ledger import CostLogRow, PlatformLedger, SpendCharge, TreeCharge
+from .scenarios import SCENARIOS
+
+_RELAUNCHED = next(s for s in SCENARIOS if s.id == "sdk-anthropic-dropped-after-tool")
+
+
+def _call(index: int, request: int) -> Call:
+ usage = scripted_usage(index, cached=True)
+ return Call(
+ turn=1,
+ request=request,
+ route="anthropic",
+ model=SONNET,
+ usage=usage,
+ cache_ttl="5m",
+ invoice_usd=ANTHROPIC_PRICES[SONNET].of(usage, "5m"),
+ )
+
+
+# The tool round's first call, then the dropped request 2, then the retry's call.
+_FAILED_ATTEMPT = _call(1, request=1)
+_RETRY = _call(2, request=3)
+
+
+def test_a_relaunched_turn_charged_only_its_retry_passes():
+ ledger, usage = _turn_charged_for(_RETRY)
+
+ findings = _audit_turn(_RELAUNCHED, 1, [_FAILED_ATTEMPT, _RETRY], ledger, usage)
+
+ assert findings == []
+
+
+def test_charging_the_failed_attempt_is_reported():
+ ledger, usage = _turn_charged_for(_FAILED_ATTEMPT, _RETRY)
+
+ findings = _audit_turn(_RELAUNCHED, 1, [_FAILED_ATTEMPT, _RETRY], ledger, usage)
+
+ assert sorted(finding.check for finding in findings) == [
+ "charge",
+ "tokens",
+ "tokens",
+ ]
+ assert all("the turn owes" in finding.message for finding in findings)
+
+
+def _turn_charged_for(*calls: Call) -> tuple[PlatformLedger, list[Usage]]:
+ """Turn 1's ledger and session usage, as if the platform charged ``calls``."""
+ microdollars = usd_to_microdollars(sum(call.invoice_usd for call in calls))
+ input_tokens = sum(call.usage.input_tokens for call in calls)
+ output_tokens = sum(call.usage.output_tokens for call in calls)
+ cache_read = sum(call.usage.cache_read_tokens for call in calls)
+ cache_write = sum(call.usage.cache_write_tokens for call in calls)
+ ledger = PlatformLedger()
+ ledger.spend.append(
+ SpendCharge(turn=1, user_id="user", microdollars=microdollars, skip_daily=False)
+ )
+ ledger.tree.append(TreeCharge(turn=1, tree_id="tree", microdollars=microdollars))
+ ledger.cost_log.append(
+ CostLogRow(
+ turn=1,
+ entry=PlatformCostEntry(
+ user_id="user",
+ chat_session_id="chat",
+ provider="anthropic",
+ cost_microdollars=microdollars,
+ input_tokens=input_tokens,
+ output_tokens=output_tokens,
+ cache_read_tokens=cache_read,
+ cache_creation_tokens=cache_write,
+ tracking_type="cost_usd",
+ ),
+ )
+ )
+ usage = Usage(
+ prompt_tokens=input_tokens,
+ completion_tokens=output_tokens,
+ total_tokens=input_tokens + output_tokens,
+ cache_read_tokens=cache_read,
+ cache_creation_tokens=cache_write,
+ )
+ return ledger, [usage]
diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py
index e3ef689725..20a55a8fb9 100644
--- a/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py
+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py
@@ -56,7 +56,9 @@ class Scenario(BaseModel):
calls_per_turn: tuple[int, int, int] = (1, 2, 1)
fault: PlannedFault | None = None
# Whether the platform relaunches the turn after the fault, rather than
- # its client retrying the one request.
+ # its client retrying the one request. The fault then ends the failed
+ # attempt, so every call after it is the retry's: the CLI's own retries
+ # must be off (``cli_max_retries=0``) for this to hold.
fault_relaunches_turn: bool = False
# CLAUDE_CODE_MAX_RETRIES for the CLI; None keeps its own default.
cli_max_retries: int | None = None
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 73220f6542..7e771a9d6d 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -1813,7 +1813,9 @@ async def _apply_building_mode_restart(
if building_suffix
else _BUILDING_MODE_UNAVAILABLE_CONTINUATION
)
- # Fresh adapter, same carry-over rules as a transient retry.
+ # Fresh adapter, carrying over the same flags as a transient retry. Unlike
+ # a transient retry, this relaunch keeps the CLI cost baseline: it is
+ # deliberate, so the interrupted run's spend is the user's.
# NOTE: the transcript builder is NOT restored — its partial
# entries are real; the relaunched run's `append_user` adds
# the original message once more, which only surfaces in the
@@ -4175,7 +4177,9 @@ async def _run_stream_attempt(
yield StreamStatus(message="Starting the assistant…")
# An interruption can precede ResultMessage. A building-mode relaunch keeps
# the baseline, so its next result includes that unreported spend; a
- # transient retry clears it first (see ``_do_transient_backoff``).
+ # transient retry clears it first (see ``_do_transient_backoff``). The
+ # baseline is the total the CLI restores, and only ``--resume`` restores
+ # one: a launch with ``session_id`` starts at zero, so don't read its file.
if state.usage.cli_session_total_usd is None:
baseline = await asyncio.to_thread(
read_cli_session_usage, ctx.sdk_cwd, state.options.resume, ctx.log_prefix