AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): charge a retried turn only for the attempt that succeeded (#15399)
Nicholas Tindleb12c70dd8aaa
diff --git a/.github/scripts/backend-allowed-skips.json b/.github/scripts/backend-allowed-skips.jsonindex f144772edf..0c5c103051 100644--- a/.github/scripts/backend-allowed-skips.json+++ b/.github/scripts/backend-allowed-skips.json@@ -107,8 +107,6 @@"backend.blocks.test.test_block.test_block_ids_valid[TwitterUnpinListBlock]","backend.blocks.test.test_block.test_block_ids_valid[TwitterUpdateListBlock]","backend.copilot.baseline.service_test.test_baseline_multi_turn",- "backend.copilot.billing_e2e.thread_billing_test.test_each_turn_is_charged_what_its_provider_billed[sdk-anthropic-dropped-after-tool]",- "backend.copilot.billing_e2e.thread_billing_test.test_each_turn_is_charged_what_its_provider_billed[sdk-anthropic-overloaded-after-tool]","backend.copilot.graphiti.client_test.TestHyphenInGroupIdRegression.test_build_fulltext_query_escapes_hyphens_in_group_id","backend.copilot.sdk.cli_openrouter_compat_test.test_bare_cli_does_not_send_openrouter_incompatible_features","backend.copilot.sdk.tool_adapter_test.TestBug1DuplicateExecution.test_old_code_duplicates_on_arg_mismatch",diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/audit.py b/autogpt_platform/backend/backend/copilot/billing_e2e/audit.pyindex a110e04df1..a1ae97c5e1 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/audit.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/audit.py@@ -98,14 +98,15 @@ def _audit_turn(message=f"turn {turn}: the provider billed {len(calls)} calls",)]- findings = _audit_spend(scenario, turn, calls, ledger)+ owed = scenario.owed_calls(turn, calls)+ findings = _audit_spend(scenario, turn, owed, ledger)rows = ledger.rows_in(turn)if len(rows) != 1:message = f"turn {turn}: {len(rows)} cost-log rows, expected 1"return [*findings, Finding(check="charge", message=message)]findings += _audit_row(scenario, turn, rows[0], ledger)if scenario.reports_usage:- findings += _audit_tokens(turn, calls, rows[0], session_usage)+ findings += _audit_tokens(turn, owed, rows[0], session_usage)return findings@@ -129,7 +130,7 @@ def _audit_spend(messages = []if len(spend) != 1 or abs(spend[0] - (expected or 0)) > tolerance:messages.append(- f"turn {turn}: charged {spend} microdollars; the provider billed "+ f"turn {turn}: charged {spend} microdollars; the turn owes "f"{expected} ({scenario.charge}) for {len(calls)} call(s)")if tree != spend:@@ -162,7 +163,7 @@ def _audit_row(def _audit_tokens(turn: int, calls: list[Call], row: PlatformCostEntry, session_usage: list[Usage]) -> list[Finding]:- billed = (+ owed = (sum(call.usage.input_tokens for call in calls),sum(call.usage.output_tokens for call in calls),sum(call.usage.cache_read_tokens for call in calls),@@ -184,12 +185,8 @@ def _audit_tokens(for u in session_usage]messages = []- if logged != billed:- messages.append(- f"turn {turn}: logged tokens {logged}, provider billed {billed}"- )- if recorded != [billed]:- messages.append(- f"turn {turn}: session usage {recorded}, provider billed {billed}"- )+ if logged != owed:+ messages.append(f"turn {turn}: logged tokens {logged}, the turn owes {owed}")+ if recorded != [owed]:+ messages.append(f"turn {turn}: session usage {recorded}, the turn owes {owed}")return [Finding(check="tokens", message=message) for message in messages]diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/invoice.py b/autogpt_platform/backend/backend/copilot/billing_e2e/invoice.pyindex 008f6ae17f..9275cfb44b 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/invoice.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/invoice.py@@ -79,6 +79,8 @@ class Call(BaseModel):"""One request the provider served."""turn: int+ # The turn's model request it answered, numbered like PlannedFault.request.+ request: introute: Routemodel: strusage: Usagediff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/provider.py b/autogpt_platform/backend/backend/copilot/billing_e2e/provider.pyindex 2cbb127314..872ff2ed83 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/provider.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/provider.py@@ -222,6 +222,7 @@ class FakeProvider:raise web.HTTPBadRequest(text=f"unknown model {model}")call = Call(turn=self.turn,+ request=self._requests_per_turn.get(self.turn, 0),route=route,model=model,usage=usage,diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.pyindex 553481782d..e3ef689725 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py@@ -10,6 +10,10 @@ charge is expected to rest on:the CLI's price (Anthropic rates) would be far off.- ``none``: the user's own subscription pays, or nobody does; the platformmust not charge the user's spend budgets at all.++A turn the platform relaunches after a provider failure owes only the calls+billed after the failure. The failed attempt is the platform's to absorb, not+the user's."""from typing import Any, Literal@@ -51,10 +55,21 @@ class Scenario(BaseModel):# Calls the provider bills per turn: one, a tool round of two, one.calls_per_turn: tuple[int, int, int] = (1, 2, 1)fault: PlannedFault | None = None+ # Whether the platform relaunches the turn after the fault, rather than+ # its client retrying the one request.+ fault_relaunches_turn: bool = False# CLAUDE_CODE_MAX_RETRIES for the CLI; None keeps its own default.cli_max_retries: int | None = Noneknown_gap: KnownGap | None = None+ def owed_calls(self, turn: int, calls: list[Call]) -> list[Call]:+ """The turn's billed calls the user pays for: after a relaunch, only+ the attempt that succeeded."""+ fault = self.fault+ if fault is None or fault.turn != turn or not self.fault_relaunches_turn:+ return calls+ return [call for call in calls if call.request > fault.request]+@propertydef needs_cli(self) -> bool:return self.engine == "sdk"@@ -88,7 +103,6 @@ def _sdk_anthropic_failing(scenario_id: str,fault: PlannedFault,calls_per_turn: tuple[int, int, int],- known_gap: KnownGap | None = None,) -> Scenario:"""The direct-Anthropic thread with one request failed and the CLI's ownretries off, so the platform's handling of the failure is what runs."""@@ -96,39 +110,29 @@ def _sdk_anthropic_failing(update={"id": scenario_id,"fault": fault,+ "fault_relaunches_turn": True,"calls_per_turn": calls_per_turn,"cli_max_retries": 0,- "known_gap": known_gap,})-_RETRY_TOKENS = KnownGap(- check="tokens",- reason=(- "A retried turn is charged the failed attempt's spend, but its "- "cost-log row and session usage keep only the retry's tokens"- ),-)-SCENARIOS = [_SDK_ANTHROPIC,# The tool round's second call is refused: the platform relaunches the CLI- # on the session, and the turn owes the first call and the retry's.+ # on the session. The provider billed the tool round's first call and the+ # retry's; the turn owes only the retry's._sdk_anthropic_failing("sdk-anthropic-overloaded-after-tool",PlannedFault(kind="overloaded", turn=1, request=2),calls_per_turn=(1, 2, 1),- known_gap=_RETRY_TOKENS,),- # The connection drops instead, which is transient: the platform stops+ # The connection drops instead, which is transient too: the platform stops# reading before the CLI's result and relaunches the CLI on the session.- # The turn owes the first call and the retry's._sdk_anthropic_failing("sdk-anthropic-dropped-after-tool",PlannedFault(kind="dropped", turn=1, request=2),calls_per_turn=(1, 2, 1),- known_gap=_RETRY_TOKENS,),# A resumed turn whose first request drops is relaunched before anything# streamed, and charged its one successful call.diff --git a/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py b/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.pyindex d8d169ea8f..191eb63252 100644--- a/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py@@ -784,8 +784,10 @@ class TestDoTransientBackoff:mock_cls.assert_called_once_with(message_id="msg-1", session_id="sess-1")assert state.adapter is new_adapter- async def test_preserves_usage_after_yield(self):- """A resumed retry must retain both reported spend and its CLI baseline."""+ async def test_keeps_reported_usage_and_clears_the_cli_baseline(self):+ """A retry keeps the spend already reported, but re-reads the CLI+ baseline at relaunch, so the failed attempt's unreported spend is not+ charged."""from unittest.mock import AsyncMock, MagicMock, patchfrom backend.copilot.sdk.service import _do_transient_backoff@@ -812,7 +814,7 @@ class TestDoTransientBackoff:cache_creation_tokens=30,cost_usd=0.02,cli_cost_usd=0.1,- cli_session_total_usd=1.1,+ cli_session_total_usd=None,)diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex 39a80ced09..73220f6542 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -2442,7 +2442,10 @@ async def _do_transient_backoff(Yields a single :class:`StreamStatus` so the caller can forward it tothe client, then sleeps for *backoff* seconds and resets ``state.adapter``.- Usage and the CLI cost baseline survive because the retry resumes the session.+ Usage already reported survives the retry, but the CLI cost baseline does+ not: the relaunch re-reads the total the resumed session restores, so+ whatever the failed attempt spent without reporting a result is not+ charged. The user pays for the attempt that succeeds, not for our failure.Extracted from both exception handlers in the retry loop to removenear-identical code duplication.@@ -2453,6 +2456,7 @@ async def _do_transient_backoff(message_id=message_id,session_id=session_id,)+ state.usage.cli_session_total_usd = Nonedef _is_fallback_stderr(line: str) -> bool:@@ -4169,8 +4173,9 @@ async def _run_stream_attempt(# CLI subprocess spawn + MCP init can take seconds on cold starts —# narrate it so the status doesn't sit on the context-prep message.yield StreamStatus(message="Starting the assistant…")- # An interruption can precede ResultMessage. Keep the baseline across- # resumed retries so the next result includes that unreported spend.+ # An interruption can precede ResultMessage. A building-mode relaunch keeps+ # the baseline, so its next result includes that unreported spend; a+ # transient retry clears it first (see ``_do_transient_backoff``).if state.usage.cli_session_total_usd is None:baseline = await asyncio.to_thread(read_cli_session_usage, ctx.sdk_cwd, state.options.resume, ctx.log_prefixdiff --git a/autogpt_platform/backend/backend/copilot/sdk/turn_cost_retry_test.py b/autogpt_platform/backend/backend/copilot/sdk/turn_cost_retry_test.pyindex 12e64dddfb..9aee735ba8 100644--- a/autogpt_platform/backend/backend/copilot/sdk/turn_cost_retry_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/turn_cost_retry_test.py@@ -1,4 +1,9 @@-"""Retrying a platform turn must retain spend from the failed attempt."""+"""What a turn that retried is charged.++A transient failure is ours to absorb: the turn is charged only the attempt+that succeeded. A context retry also keeps the overflow result it was billed+for, since that result was reported before the retry.+"""import contextlibfrom datetime import UTC, datetime@@ -20,10 +25,17 @@ from .turn_cost_test import _CALL_COST, _SDK_CWD, _SESSION_ID, _SVC, _result@pytest.mark.asyncio@pytest.mark.parametrize(- "failure", ["transient", "transient-assistant", "context-result", "context-zero"]+ ("failure", "charged_calls"),+ [+ # The failed attempt spent one call and the retry one: only the retry.+ ("transient", 1),+ ("transient-assistant", 1),+ ("context-result", 3),+ ("context-zero", 2),+ ],)-async def test_retry_retains_failed_attempt_spend(- failure, tmp_path, monkeypatch, caplog+async def test_retried_turn_charge(+ failure, charged_calls, tmp_path, monkeypatch, caplog):monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(tmp_path))session_file = Path(cli_session_path(_SDK_CWD, _SESSION_ID))@@ -122,7 +134,7 @@ async def test_retry_retains_failed_attempt_spend(]assert not any(isinstance(event, StreamError) for event in events)persist.assert_awaited_once()- expected = (3 if failure == "context-result" else 2) * _CALL_COST+ expected = charged_calls * _CALL_COSTassert persist.await_args.kwargs["cost_usd"] == pytest.approx(expected)if failure.startswith("context"):assert persist.await_args.kwargs["prompt_tokens"] == (