AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
dx(backend/copilot): check each chat route charges a thread what its provider billed (#15379)
Nicholas Tindlec9cd58e1c6ce
diff --git a/.github/scripts/backend-allowed-skips.json b/.github/scripts/backend-allowed-skips.jsonindex 0c5c103051..88edd78a62 100644--- a/.github/scripts/backend-allowed-skips.json+++ b/.github/scripts/backend-allowed-skips.json@@ -107,6 +107,8 @@"backend.blocks.test.test_block.test_block_ids_valid[TwitterUnpinListBlock]","backend.blocks.test.test_block.test_block_ids_valid[TwitterUpdateListBlock]","backend.copilot.baseline.service_test.test_baseline_multi_turn",+ "backend.copilot.billing_e2e.thread_billing_test.test_each_turn_is_charged_what_its_provider_billed[sdk-anthropic-dropped-after-tool]",+ "backend.copilot.billing_e2e.thread_billing_test.test_each_turn_is_charged_what_its_provider_billed[sdk-anthropic-overloaded-on-resume]","backend.copilot.graphiti.client_test.TestHyphenInGroupIdRegression.test_build_fulltext_query_escapes_hyphens_in_group_id","backend.copilot.sdk.cli_openrouter_compat_test.test_bare_cli_does_not_send_openrouter_incompatible_features","backend.copilot.sdk.tool_adapter_test.TestBug1DuplicateExecution.test_old_code_duplicates_on_arg_mismatch",diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/__init__.py b/autogpt_platform/backend/backend/copilot/billing_e2e/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/audit.py b/autogpt_platform/backend/backend/copilot/billing_e2e/audit.pynew file mode 100644index 0000000000..a110e04df1--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/audit.py@@ -0,0 +1,195 @@+"""A finished thread checked against its provider's invoice, turn by turn."""++from pydantic import BaseModel++from backend.copilot.model import ChatSession, Usage+from backend.data.platform_cost import PlatformCostEntry, usd_to_microdollars++from .invoice import Call+from .ledger import PlatformLedger+from .provider import FakeProvider+from .scenarios import Check, Scenario, expected_charge_usd++# Per call: float pricing in the CLI and in each sum, rounded once per turn.+_MICRODOLLAR_TOLERANCE_PER_CALL = 2+++class Finding(BaseModel):+ check: Check+ message: str+++def audit_thread(+ scenario: Scenario,+ provider: FakeProvider,+ ledger: PlatformLedger,+ session: ChatSession,+ usage_by_turn: list[list[Usage]],+ cli_resumed: list[bool] | None,+) -> list[Finding]:+ """Every way the thread strayed from its invoice or its planned path.++ ``cli_resumed`` says, for each CLI launch of an SDK thread, whether it+ resumed the chat's session; None for engines without a CLI.+ """+ findings = _audit_path(provider, cli_resumed, turns=len(usage_by_turn))+ findings += _audit_attribution(session, ledger)+ for turn, session_usage in enumerate(usage_by_turn):+ findings += _audit_turn(+ scenario, turn, provider.calls_in(turn), ledger, session_usage+ )+ return findings+++def _audit_attribution(session: ChatSession, ledger: PlatformLedger) -> list[Finding]:+ """Every charge on this chat's user and its daily budget, every row on+ this chat."""+ messages = [+ f"turn {charge.turn}: charged user {charge.user_id!r}, "+ f"skipping the daily budget: {charge.skip_daily}"+ for charge in ledger.spend+ if charge.user_id != session.user_id or charge.skip_daily+ ]+ messages += [+ f"turn {row.turn}: cost-log row for user {row.entry.user_id!r}, "+ f"chat {row.entry.chat_session_id!r}"+ for row in ledger.cost_log+ if (row.entry.user_id, row.entry.chat_session_id)+ != (session.user_id, session.session_id)+ ]+ return [Finding(check="charge", message=message) for message in messages]+++def _audit_path(+ provider: FakeProvider, cli_resumed: list[bool] | None, *, turns: int+) -> list[Finding]:+ """Whether the thread took the path it meant to.++ Without this the charge checks could pass vacuously: an SDK turn that+ never resumed has no restored total to over-charge from, and a fault that+ never fired tests nothing.+ """+ messages = [f"unexpected provider request: {r}" for r in provider.unexpected]+ if provider.faults_served != provider.faults:+ messages.append(+ f"faults served {provider.faults_served}, planned {provider.faults}"+ )+ # Every launch after the first, retries included, resumes the session.+ if cli_resumed is not None and (+ len(cli_resumed) < turns or cli_resumed[0] or not all(cli_resumed[1:])+ ):+ messages.append(f"CLI launches resumed the session: {cli_resumed}")+ return [Finding(check="path", message=message) for message in messages]+++def _audit_turn(+ scenario: Scenario,+ turn: int,+ calls: list[Call],+ ledger: PlatformLedger,+ session_usage: list[Usage],+) -> list[Finding]:+ if len(calls) != scenario.calls_per_turn[turn]:+ # The turn did not run as planned; checking its charge would only+ # compare against the wrong invoice.+ return [+ Finding(+ check="path",+ message=f"turn {turn}: the provider billed {len(calls)} calls",+ )+ ]+ findings = _audit_spend(scenario, turn, calls, ledger)+ rows = ledger.rows_in(turn)+ if len(rows) != 1:+ message = f"turn {turn}: {len(rows)} cost-log rows, expected 1"+ return [*findings, Finding(check="charge", message=message)]+ findings += _audit_row(scenario, turn, rows[0], ledger)+ if scenario.reports_usage:+ findings += _audit_tokens(turn, calls, rows[0], session_usage)+ return findings+++def _audit_spend(+ scenario: Scenario, turn: int, calls: list[Call], ledger: PlatformLedger+) -> list[Finding]:+ spend = [charge.microdollars for charge in ledger.spend_in(turn)]+ tree = [charge.microdollars for charge in ledger.tree_in(turn)]+ if scenario.charge == "none":+ if not spend and not tree:+ return []+ message = (+ f"turn {turn}: charged {spend} to spend and {tree} to the tree; "+ f"nobody bills the platform for this route"+ )+ return [Finding(check="charge", message=message)]+ expected = usd_to_microdollars(+ sum(expected_charge_usd(scenario.charge, call) for call in calls)+ )+ tolerance = _MICRODOLLAR_TOLERANCE_PER_CALL * len(calls)+ messages = []+ if len(spend) != 1 or abs(spend[0] - (expected or 0)) > tolerance:+ messages.append(+ f"turn {turn}: charged {spend} microdollars; the provider billed "+ f"{expected} ({scenario.charge}) for {len(calls)} call(s)"+ )+ if tree != spend:+ messages.append(f"turn {turn}: the tree was charged {tree}, spend {spend}")+ return [Finding(check="charge", message=message) for message in messages]+++def _audit_row(+ scenario: Scenario, turn: int, row: PlatformCostEntry, ledger: PlatformLedger+) -> list[Finding]:+ messages = []+ if row.provider != scenario.cost_log_provider:+ messages.append(f"turn {turn}: cost-log provider {row.provider!r}")+ if scenario.charge == "none":+ if row.tracking_type != "tokens" or row.cost_microdollars:+ messages.append(+ f"turn {turn}: logged {row.tracking_type} {row.cost_microdollars} "+ f"for a route the platform does not pay for"+ )+ else:+ charged = [charge.microdollars for charge in ledger.spend_in(turn)]+ if row.tracking_type != "cost_usd" or [row.cost_microdollars] != charged:+ messages.append(+ f"turn {turn}: logged {row.tracking_type} {row.cost_microdollars}, "+ f"charged {charged}"+ )+ return [Finding(check="charge", message=message) for message in messages]+++def _audit_tokens(+ turn: int, calls: list[Call], row: PlatformCostEntry, session_usage: list[Usage]+) -> list[Finding]:+ billed = (+ sum(call.usage.input_tokens for call in calls),+ sum(call.usage.output_tokens for call in calls),+ sum(call.usage.cache_read_tokens for call in calls),+ sum(call.usage.cache_write_tokens for call in calls),+ )+ logged = (+ row.input_tokens or 0,+ row.output_tokens or 0,+ row.cache_read_tokens or 0,+ row.cache_creation_tokens or 0,+ )+ recorded = [+ (+ u.prompt_tokens,+ u.completion_tokens,+ u.cache_read_tokens,+ u.cache_creation_tokens,+ )+ for u in session_usage+ ]+ messages = []+ if logged != billed:+ messages.append(+ f"turn {turn}: logged tokens {logged}, provider billed {billed}"+ )+ if recorded != [billed]:+ messages.append(+ f"turn {turn}: session usage {recorded}, provider billed {billed}"+ )+ return [Finding(check="tokens", message=message) for message in messages]diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/baseline_turns.py b/autogpt_platform/backend/backend/copilot/billing_e2e/baseline_turns.pynew file mode 100644index 0000000000..c3deee5f86--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/baseline_turns.py@@ -0,0 +1,87 @@+"""Baseline-engine turns: ``stream_chat_completion_baseline`` over the OpenAI SDK.++A real ``AsyncOpenAI`` client streams from the fake provider, and the+baseline's own usage and cost parsing reads what it sends. Stubbed:+persistence, transcript storage, the system prompt, flag lookups and the+copilot's tools, which answer the provider's tool call with a canned result.+"""++import uuid+from typing import Any+from unittest.mock import AsyncMock++import pytest+from openai import AsyncOpenAI++from backend.copilot import pending_message_helpers+from backend.copilot.baseline import service as baseline+from backend.copilot.config import ChatConfig+from backend.copilot.model import ChatSession+from backend.copilot.model_router import ResolvedModel+from backend.copilot.response_model import StreamToolOutputAvailable+from backend.copilot.tree import root_envelope++from .wire import TOOL_NAME+++class BaselineTurns:+ def __init__(+ self,+ monkeypatch: pytest.MonkeyPatch,+ config: ChatConfig,+ *,+ base_url: str,+ model: str,+ ) -> None:+ client = AsyncOpenAI(api_key="sk-billing-e2e", base_url=base_url)+ save = AsyncMock(side_effect=lambda session, *_, **__: session)+ stubs: dict[str, Any] = {+ "config": config,+ "_get_main_client": lambda: client,+ "resolve_model_route": AsyncMock(+ return_value=ResolvedModel(model=model, source="env")+ ),+ "execute_tool": AsyncMock(side_effect=_answer_tool),+ "upsert_chat_session": save,+ "download_transcript": AsyncMock(return_value=None),+ "upload_transcript": AsyncMock(),+ "drain_pending_safe": AsyncMock(return_value=[]),+ "drain_pending_messages": AsyncMock(return_value=[]),+ "_build_system_prompt": AsyncMock(+ return_value=("You are a test assistant. Reply briefly.", None)+ ),+ "build_budget_ctx": AsyncMock(return_value=None),+ "build_session_context": AsyncMock(return_value=""),+ "build_skills_context": AsyncMock(return_value=""),+ "build_skills_update_notice": AsyncMock(return_value=None),+ "inject_user_context": AsyncMock(return_value=None),+ "clear_pending_question": AsyncMock(),+ "resolve_answered": AsyncMock(return_value=[]),+ "build_turn_budget_block": AsyncMock(return_value=""),+ "is_enabled_for_user": AsyncMock(return_value=False),+ "is_feature_enabled": AsyncMock(return_value=False),+ }+ for name, value in stubs.items():+ monkeypatch.setattr(baseline, name, value)+ monkeypatch.setattr(pending_message_helpers, "upsert_chat_session", save)++ async def __call__(+ self, session: ChatSession, prompt: str, *, new_pod: bool+ ) -> None:+ # The baseline keeps nothing on disk between turns, so a new pod is+ # the same as the old one.+ async for _ in baseline.stream_chat_completion_baseline(+ session.session_id,+ message=prompt,+ is_user_message=True,+ user_id=session.user_id,+ session=session,+ envelope=root_envelope(str(uuid.uuid4()), session_id=session.session_id),+ ):+ pass+++async def _answer_tool(*, tool_call_id: str, **_: Any) -> StreamToolOutputAvailable:+ return StreamToolOutputAvailable(+ toolCallId=tool_call_id, toolName=TOOL_NAME, output="The fact is 42."+ )diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/conftest.py b/autogpt_platform/backend/backend/copilot/billing_e2e/conftest.pynew file mode 100644index 0000000000..eecb29ca91--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/conftest.py@@ -0,0 +1,22 @@+"""The fake provider and the platform ledger each billing scenario runs on."""++from collections.abc import AsyncIterator++import pytest+import pytest_asyncio++from .ledger import PlatformLedger+from .provider import FakeProvider+++@pytest_asyncio.fixture(loop_scope="function")+async def provider() -> AsyncIterator[FakeProvider]:+ async with FakeProvider() as fake:+ yield fake+++def ledger(monkeypatch: pytest.MonkeyPatch) -> PlatformLedger:+ ledger = PlatformLedger()+ ledger.install(monkeypatch)+ return ledgerdiff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/invoice.py b/autogpt_platform/backend/backend/copilot/billing_e2e/invoice.pynew file mode 100644index 0000000000..008f6ae17f--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/invoice.py@@ -0,0 +1,118 @@+"""What the fake provider bills for each call it serves.++The prices are written out here instead of read from ``anthropic_rates.json``,+the LLM catalog or ``moonshot.py``. An invoice priced by the code under test+could not catch that code pricing a call wrongly.+"""++from typing import Literal++from pydantic import BaseModel++Route = Literal["anthropic", "openrouter", "ollama", "codex", "graph"]+CacheTtl = Literal["5m", "1h"]++SONNET = "claude-sonnet-5-5"+KIMI = "moonshotai/kimi-k2.6"+++class Usage(BaseModel):+ """Token counts for one call; ``input_tokens`` excludes the cache buckets."""++ input_tokens: int+ output_tokens: int+ cache_read_tokens: int = 0+ cache_write_tokens: int = 0+++class Price(BaseModel):+ """USD per million tokens."""++ input: float+ output: float+ cache_read: float+ cache_write_5m: float+ cache_write_1h: float++ def of(self, usage: Usage, ttl: CacheTtl | None) -> float:+ write = self.cache_write_1h if ttl == "1h" else self.cache_write_5m+ return (+ usage.input_tokens * self.input+ + usage.output_tokens * self.output+ + usage.cache_read_tokens * self.cache_read+ + usage.cache_write_tokens * write+ ) / 1_000_000+++# Anthropic's list prices. The bundled CLI prices its calls from its own copy.+ANTHROPIC_PRICES: dict[str, Price] = {+ SONNET: Price(+ input=2.0, output=10.0, cache_read=0.2, cache_write_5m=2.5, cache_write_1h=4.0+ ),+}++# OpenRouter's prices, a tenth above Anthropic's list on purpose: a turn charged+# an estimate where it should have been charged this invoice shows up as a+# mismatch instead of matching by coincidence.+OPENROUTER_PRICES: dict[str, Price] = {+ f"anthropic/{SONNET}": Price(+ input=2.2, output=11.0, cache_read=0.22, cache_write_5m=2.75, cache_write_1h=4.4+ ),+ KIMI: Price(+ input=0.66,+ output=3.08,+ cache_read=0.165,+ cache_write_5m=0.66,+ cache_write_1h=0.66,+ ),+}++# The rate card the platform falls back to for Kimi K2.6 when it has no+# invoice: cache tokens count as input. Mirrors ``moonshot.py``'s documented+# default rather than importing it.+MOONSHOT_FALLBACK = Price(+ input=0.60, output=2.80, cache_read=0.60, cache_write_5m=0.60, cache_write_1h=0.60+)+++class Call(BaseModel):+ """One request the provider served."""++ turn: int+ route: Route+ model: str+ usage: Usage+ cache_ttl: CacheTtl | None+ invoice_usd: float+ generation_id: str | None = None+++def scripted_usage(index: int, *, cached: bool) -> Usage:+ """Usage for the provider's ``index``-th call.++ Distinct for every call, so a charge carried over from another call or+ turn cannot match by coincidence. A request that asked for no caching is+ billed the same tokens as fresh input.+ """+ fresh = 1_200 + 37 * index+ read = 40_000 + 1_500 * index+ write = 3_000 + 250 * index+ output = 150 + 11 * index+ if not cached:+ return Usage(input_tokens=fresh + read + write, output_tokens=output)+ return Usage(+ input_tokens=fresh,+ output_tokens=output,+ cache_read_tokens=read,+ cache_write_tokens=write,+ )+++def price_table(route: Route) -> dict[str, Price] | None:+ """The table ``route`` bills from; None where nobody bills the platform."""+ if route == "anthropic":+ return ANTHROPIC_PRICES+ if route == "openrouter":+ return OPENROUTER_PRICES+ # Ollama is self-hosted; Codex and Microsoft 365 bill the user's own plan.+ return Nonediff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/ledger.py b/autogpt_platform/backend/backend/copilot/billing_e2e/ledger.pynew file mode 100644index 0000000000..2af593b7ca--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/ledger.py@@ -0,0 +1,102 @@+"""What the platform charged, captured where ``persist_and_record_usage`` writes.++Each write lands in one of three places: the user's daily and weekly spend+counters, the turn tree's budget, and a ``PlatformCostLog`` row. The ledger+replaces exactly those three writes, so every decision about what to charge+still runs for real, and files each write under the turn that made it.+"""++import asyncio++import pytest+from pydantic import BaseModel++from backend.copilot import token_tracking+from backend.copilot.baseline import service as baseline_service+from backend.copilot.sdk import service as sdk_service+from backend.copilot.tree import TurnEnvelope+from backend.data.platform_cost import PlatformCostEntry+++class SpendCharge(BaseModel):+ turn: int+ user_id: str+ microdollars: int+ skip_daily: bool+++class TreeCharge(BaseModel):+ turn: int+ tree_id: str+ microdollars: int+++class CostLogRow(BaseModel):+ turn: int+ entry: PlatformCostEntry+++class PlatformLedger:+ def __init__(self) -> None:+ self.turn = 0+ self.spend: list[SpendCharge] = []+ self.tree: list[TreeCharge] = []+ self.cost_log: list[CostLogRow] = []++ def install(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(token_tracking, "record_cost_usage", self._record_spend)+ monkeypatch.setattr(token_tracking, "charge_turn", self._charge_tree)+ monkeypatch.setattr(token_tracking, "platform_cost_db", lambda: self)++ def spend_in(self, turn: int) -> list[SpendCharge]:+ return [charge for charge in self.spend if charge.turn == turn]++ def tree_in(self, turn: int) -> list[TreeCharge]:+ return [charge for charge in self.tree if charge.turn == turn]++ def rows_in(self, turn: int) -> list[PlatformCostEntry]:+ return [row.entry for row in self.cost_log if row.turn == turn]++ async def log_platform_cost(self, entry: PlatformCostEntry) -> None:+ self.cost_log.append(CostLogRow(turn=self.turn, entry=entry))++ async def settle(self) -> None:+ """Wait for the writes a turn defers past its own end.++ The OpenRouter reconcile charges from a background task, and every+ cost-log row is written by one; both must land under the turn that+ started them.+ """+ while pending := _deferred_tasks():+ await asyncio.gather(*pending)++ async def _record_spend(+ self, user_id: str, cost_microdollars: int, *, skip_daily: bool = False+ ) -> None:+ self.spend.append(+ SpendCharge(+ turn=self.turn,+ user_id=user_id,+ microdollars=cost_microdollars,+ skip_daily=skip_daily,+ )+ )++ async def _charge_tree(self, envelope: TurnEnvelope, microdollars: int) -> None:+ self.tree.append(+ TreeCharge(+ turn=self.turn, tree_id=envelope.tree_id, microdollars=microdollars+ )+ )+++def _deferred_tasks() -> list[asyncio.Task]:+ with token_tracking._pending_log_tasks_lock:+ logs = list(token_tracking._pending_log_tasks)+ loop = asyncio.get_running_loop()+ tasks = [+ *logs,+ *sdk_service._background_tasks,+ *baseline_service._background_tasks,+ ]+ return [task for task in tasks if task.get_loop() is loop and not task.done()]diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/m365_turns.py b/autogpt_platform/backend/backend/copilot/billing_e2e/m365_turns.pynew file mode 100644index 0000000000..8912a8c1f4--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/m365_turns.py@@ -0,0 +1,58 @@+"""Microsoft 365 Copilot turns over Graph's chatOverStream, served by the fake."""++from functools import partial+from unittest.mock import AsyncMock, MagicMock++import pytest+from pydantic import SecretStr++from backend.copilot.model import ChatSession+from backend.data.model import OAuth2Credentials+from backend.integrations.credential_lease import CredentialLease+from backend.integrations.microsoft_365_copilot import service as m365+from backend.integrations.microsoft_365_copilot.client import Microsoft365CopilotClient+from backend.integrations.oauth.microsoft_365_copilot import (+ Microsoft365CopilotDeviceAuthHandler,+)+from backend.integrations.providers import ProviderName++from .provider import FakeProvider++MICROSOFT_365_CREDENTIAL_ID = "microsoft-365-credential"+++class Microsoft365Turns:+ def __init__(self, monkeypatch: pytest.MonkeyPatch, provider: FakeProvider) -> None:+ monkeypatch.setattr(+ m365,+ "Microsoft365CopilotClient",+ partial(Microsoft365CopilotClient, base_url=f"{provider.url}/graph"),+ )+ monkeypatch.setattr(+ m365, "upsert_chat_session", AsyncMock(side_effect=lambda session: session)+ )+ monkeypatch.setattr(m365, "_get_timezone", AsyncMock(return_value="UTC"))+ credentials = OAuth2Credentials(+ id=MICROSOFT_365_CREDENTIAL_ID,+ provider=ProviderName.MICROSOFT_365_COPILOT,+ access_token=SecretStr("graph-billing-e2e"),+ scopes=list(Microsoft365CopilotDeviceAuthHandler.CHAT_SCOPES),+ title="Microsoft 365 Copilot",+ )+ self._lease = CredentialLease(+ credentials, lock=MagicMock(), checkpoint_callback=AsyncMock()+ )++ async def __call__(+ self, session: ChatSession, prompt: str, *, new_pod: bool+ ) -> None:+ # Graph keeps the conversation; nothing of it lives on a pod.+ async for _ in m365.stream_chat_completion_microsoft_365(+ session.session_id,+ message=prompt,+ is_user_message=True,+ user_id=session.user_id,+ session=session,+ credential_lease=self._lease,+ ):+ passdiff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/provider.py b/autogpt_platform/backend/backend/copilot/billing_e2e/provider.pynew file mode 100644index 0000000000..2cbb127314--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/provider.py@@ -0,0 +1,241 @@+"""One localhost server standing in for every upstream a chat turn can call.++It answers in each upstream's wire format and keeps its own invoice: every+call it served, the turn it served it in, and what that call costs the+platform at the provider's prices. That invoice is what the platform's+charges are checked against.+"""++import uuid+from functools import partial+from typing import Literal++from aiohttp import web+from pydantic import BaseModel++from . import wire+from .invoice import CacheTtl, Call, Route, Usage, price_table, scripted_usage+from .replies import GREETING, anthropic_reply, cache_ttl, chat_reply, responses_reply+++class PlannedFault(BaseModel):+ """A model request the provider fails, before doing any work it bills."""++ kind: Literal["overloaded", "dropped"]+ turn: int+ # Counts every model request of the turn, failed ones included, from 1.+ request: int+++class FakeProvider:+ def __init__(self) -> None:+ self.turn = 0+ self.calls: list[Call] = []+ self.unexpected: list[str] = []+ self.generation_lookups_fail = False+ self.faults: list[PlannedFault] = []+ self.faults_served: list[PlannedFault] = []+ self.url = ""+ self._runner: web.AppRunner | None = None+ self._requests_per_turn: dict[int, int] = {}++ async def __aenter__(self) -> "FakeProvider":+ self._runner = web.AppRunner(self._app())+ await self._runner.setup()+ await web.TCPSite(self._runner, "127.0.0.1", 0).start()+ host, port = self._runner.addresses[0][:2]+ self.url = f"http://{host}:{port}"+ return self++ async def __aexit__(self, *_: object) -> None:+ if self._runner is not None:+ await self._runner.cleanup()++ def calls_in(self, turn: int) -> list[Call]:+ return [call for call in self.calls if call.turn == turn]++ def _app(self) -> web.Application:+ app = web.Application()+ upstreams: tuple[tuple[str, Route], ...] = (+ ("/anthropic", "anthropic"),+ ("/openrouter/api", "openrouter"),+ ("/ollama", "ollama"),+ )+ for prefix, route in upstreams:+ app.router.add_post(+ f"{prefix}/v1/chat/completions", partial(self._chat, route=route)+ )+ if route != "ollama":+ app.router.add_post(+ f"{prefix}/v1/messages", partial(self._messages, route=route)+ )+ app.router.add_post("/codex/responses", self._responses)+ app.router.add_post("/graph/copilot/conversations", self._graph_conversation)+ app.router.add_post(+ "/graph/copilot/conversations/{id}/chatOverStream", self._graph_chat+ )+ app.router.add_get("/openrouter/api/v1/generation", self._generation)+ app.router.add_get("/ollama/api/ps", self._ollama_loaded_models)+ # The CLI checks it can reach its base URL before a session starts.+ app.router.add_route("HEAD", "/{prefix:.*}/api/hello", self._reachable)+ app.router.add_route("*", "/{tail:.*}", self._unexpected)+ return app++ async def _messages(self, request: web.Request, route: Route) -> web.StreamResponse:+ if failed := self._fault(request):+ return failed+ body = await request.json()+ ttl = cache_ttl(body)+ usage = scripted_usage(len(self.calls), cached=ttl is not None)+ generation_id = f"gen-{uuid.uuid4().hex}" if route == "openrouter" else None+ self._bill(route, body["model"], usage, ttl, generation_id)+ sse = wire.anthropic_messages(+ message_id=generation_id or f"msg_{uuid.uuid4().hex}",+ model=body["model"],+ reply=anthropic_reply(body["messages"]),+ usage=usage,+ ttl=ttl,+ )+ return await _stream(request, sse)++ async def _chat(self, request: web.Request, route: Route) -> web.StreamResponse:+ if failed := self._fault(request):+ return failed+ body = await request.json()+ ttl = cache_ttl(body)+ usage = scripted_usage(len(self.calls), cached=ttl is not None)+ call = self._bill(route, body["model"], usage, ttl, None)+ sse = wire.chat_completion_chunks(+ model=body["model"],+ reply=chat_reply(body["messages"]),+ usage=usage,+ cost=call.invoice_usd if route == "openrouter" else None,+ cache_write_field={+ "openrouter": "cache_write_tokens",+ "anthropic": "cache_creation_input_tokens",+ }.get(route),+ )+ return await _stream(request, sse)++ async def _responses(self, request: web.Request) -> web.StreamResponse:+ if failed := self._fault(request):+ return failed+ body = await request.json()+ scripted = scripted_usage(len(self.calls), cached=True)+ usage = Usage(+ input_tokens=scripted.input_tokens + scripted.cache_write_tokens,+ output_tokens=scripted.output_tokens,+ cache_read_tokens=scripted.cache_read_tokens,+ )+ self._bill("codex", body["model"], usage, None, None)+ sse = wire.responses_events(+ response_id=f"resp_{uuid.uuid4().hex}",+ model=body["model"],+ reply=responses_reply(body["input"]),+ usage=usage,+ )+ return await _stream(request, sse)++ async def _graph_conversation(self, _request: web.Request) -> web.Response:+ return web.json_response({"id": f"conv-{uuid.uuid4().hex[:12]}"}, status=201)++ async def _graph_chat(self, request: web.Request) -> web.StreamResponse:+ body = await request.json()+ prompt = body["message"]["text"]+ # Graph reports no usage; the platform estimates it from the text.+ self._bill(+ "graph",+ "microsoft-365-copilot",+ Usage(input_tokens=0, output_tokens=0),+ None,+ None,+ )+ sse = wire.graph_chat_snapshots(+ conversation_id=request.match_info["id"], prompt=prompt, answer=GREETING+ )+ return await _stream(request, sse)++ async def _generation(self, request: web.Request) -> web.Response:+ generation_id = request.query.get("id", "")+ call = next((c for c in self.calls if c.generation_id == generation_id), None)+ if self.generation_lookups_fail:+ # A permanent refusal, which the reconcile does not retry.+ return web.json_response({"error": {"code": 403}}, status=403)+ if call is None:+ self.unexpected.append(f"generation lookup for unknown {generation_id}")+ return web.json_response({"error": {"code": 404}}, status=404)+ return web.json_response(+ {+ "data": {+ "id": generation_id,+ "model": call.model,+ "total_cost": call.invoice_usd,+ }+ }+ )++ async def _ollama_loaded_models(self, _request: web.Request) -> web.Response:+ # The local transport reads the loaded model's window before a turn.+ return web.json_response(+ {"models": [{"name": "llama3.1:8b", "context_length": 131_072}]}+ )++ async def _reachable(self, _request: web.Request) -> web.Response:+ return web.Response(status=200)++ async def _unexpected(self, request: web.Request) -> web.Response:+ self.unexpected.append(f"{request.method} {request.path}")+ return web.json_response({"error": {"message": "not served"}}, status=404)++ def _fault(self, request: web.Request) -> web.Response | None:+ """Fail this model request if the plan says so: a 529, or a connection+ that drops before any answer."""+ nth = self._requests_per_turn.get(self.turn, 0) + 1+ self._requests_per_turn[self.turn] = nth+ fault = next(+ (f for f in self.faults if (f.turn, f.request) == (self.turn, nth)), None+ )+ if fault is None:+ return None+ self.faults_served.append(fault)+ if fault.kind == "dropped" and request.transport is not None:+ request.transport.abort()+ return web.json_response(+ {+ "type": "error",+ "error": {"type": "overloaded_error", "message": "Overloaded"},+ },+ status=529,+ )++ def _bill(+ self,+ route: Route,+ model: str,+ usage: Usage,+ ttl: CacheTtl | None,+ generation_id: str | None,+ ) -> Call:+ table = price_table(route)+ if table is not None and model not in table:+ self.unexpected.append(f"{route} has no price for {model}")+ raise web.HTTPBadRequest(text=f"unknown model {model}")+ call = Call(+ turn=self.turn,+ route=route,+ model=model,+ usage=usage,+ cache_ttl=ttl,+ invoice_usd=table[model].of(usage, ttl) if table is not None else 0.0,+ generation_id=generation_id,+ )+ self.calls.append(call)+ return call+++async def _stream(request: web.Request, body: str) -> web.StreamResponse:+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})+ await response.prepare(request)+ await response.write(body.encode())+ await response.write_eof()+ return responsediff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/replies.py b/autogpt_platform/backend/backend/copilot/billing_e2e/replies.pynew file mode 100644index 0000000000..242527729c--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/replies.py@@ -0,0 +1,81 @@+"""What the fake model answers, read off each request the way a model would.++A user message carrying ``TOOL_MARKER`` gets one tool call; the tool's result+gets a closing answer; anything else gets a greeting.+"""++import json+from collections.abc import Iterator+from typing import Any++from .invoice import CacheTtl+from .wire import Reply++TOOL_MARKER = "[use-tool]"+GREETING = "Hello from the fake provider."+AFTER_TOOL = "Here is what the lookup found."+# The Codex gateway hands the whole chat over as one message: a preamble,+# then the conversation as a JSON array of Anthropic-shaped messages.+_GATEWAY_TRANSCRIPT_MARKER = "The transcript is JSON:\n"+++def anthropic_reply(messages: list[dict[str, Any]]) -> Reply:+ # The CLI can follow the user's message with a system reminder, so the+ # last user message is the one to answer.+ content = _latest(messages, {"user"}).get("content")+ if isinstance(content, list) and any(+ isinstance(block, dict) and block.get("type") == "tool_result"+ for block in content+ ):+ return Reply(kind="text", text=AFTER_TOOL)+ return _reply_to(json.dumps(content))+++def chat_reply(messages: list[dict[str, Any]]) -> Reply:+ latest = _latest(messages, {"user", "tool"})+ if latest.get("role") == "tool":+ return Reply(kind="text", text=AFTER_TOOL)+ return _reply_to(json.dumps(latest.get("content")))+++def responses_reply(items: list[dict[str, Any]]) -> Reply:+ last = items[-1] if items else {}+ if last.get("type") == "function_call_output":+ return Reply(kind="text", text=AFTER_TOOL)+ prompt = "".join(+ part.get("text", "")+ for part in last.get("content") or []+ if isinstance(part, dict)+ )+ _, found, transcript = prompt.partition(_GATEWAY_TRANSCRIPT_MARKER)+ return anthropic_reply(json.loads(transcript)) if found else _reply_to(prompt)+++def cache_ttl(body: object) -> CacheTtl | None:+ """The cache lifetime the request asked for; None when it asked for none."""+ ttls = set(_cache_control_ttls(body))+ if not ttls:+ return None+ return "1h" if "1h" in ttls else "5m"+++def _latest(messages: list[dict[str, Any]], roles: set[str]) -> dict[str, Any]:+ return next((m for m in reversed(messages) if m.get("role") in roles), {})+++def _reply_to(latest_user_message: str) -> Reply:+ if TOOL_MARKER in latest_user_message:+ return Reply(kind="tool")+ return Reply(kind="text", text=GREETING)+++def _cache_control_ttls(node: object) -> Iterator[str]:+ if isinstance(node, dict):+ control = node.get("cache_control")+ if isinstance(control, dict):+ yield str(control.get("ttl") or "5m")+ for value in node.values():+ yield from _cache_control_ttls(value)+ elif isinstance(node, list):+ for value in node:+ yield from _cache_control_ttls(value)diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.pynew file mode 100644index 0000000000..55f9e2c929--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py@@ -0,0 +1,272 @@+"""Every route a chat turn can take, and what each should charge per call.++A row names the engine, how the deployment is configured, and the basis its+charge is expected to rest on:++- ``invoice``: what the provider billed for the call.+- ``cli_estimate``: the CLI's own price for the tokens, which the platform+ falls back to when it has no invoice to read.+- ``moonshot_estimate``: the platform's Kimi rate card, the fallback where+ the CLI's price (Anthropic rates) would be far off.+- ``none``: the user's own subscription pays, or nobody does; the platform+ must not charge the user's spend budgets at all.+"""++from typing import Any, Literal++from pydantic import BaseModel++from .invoice import ANTHROPIC_PRICES, KIMI, MOONSHOT_FALLBACK, SONNET, Call+from .provider import PlannedFault++ChargeBasis = Literal["invoice", "cli_estimate", "moonshot_estimate", "none"]+# What a finding is about: whether the thread took its planned path, what the+# turn was charged, or the tokens recorded against it.+Check = Literal["path", "charge", "tokens"]+++class KnownGap(BaseModel):+ """A known defect: this row's ``check`` fails until it is fixed.++ The other checks still hold the row to account, and a gap that stops+ reproducing fails the test so it gets removed.+ """++ check: Check+ reason: str+++class Scenario(BaseModel):+ id: str+ engine: Literal["sdk", "baseline", "microsoft_365"]+ model: str+ charge: ChargeBasis+ cost_log_provider: str+ # ChatConfig fields; ``{provider}`` in a string is the fake's base URL.+ config: dict[str, Any] = {}+ # Where the baseline's OpenAI client points, under the fake's base URL.+ client_path: str = ""+ codex: bool = False+ generation_lookups_fail: bool = False+ # Calls the provider bills per turn: one, a tool round of two, one.+ calls_per_turn: tuple[int, int, int] = (1, 2, 1)+ fault: PlannedFault | None = None+ # CLAUDE_CODE_MAX_RETRIES for the CLI; None keeps its own default.+ cli_max_retries: int | None = None+ known_gap: KnownGap | None = None++ @property+ def needs_cli(self) -> bool:+ return self.engine == "sdk"++ @property+ def reports_usage(self) -> bool:+ """Whether the upstream reports token counts the platform logs as is."""+ return self.engine != "microsoft_365"+++_OPENROUTER = {+ "use_openrouter": True,+ "api_key": "sk-or-billing-e2e",+ "base_url": "{provider}/openrouter/api/v1",+}+# The CLI reads its key from ANTHROPIC_API_KEY; the platform only checks one+# is configured.+_DIRECT = {"use_openrouter": False, "api_key": "sk-ant-billing-e2e"}++_SDK_ANTHROPIC = Scenario(+ id="sdk-anthropic",+ engine="sdk",+ model=SONNET,+ charge="invoice",+ cost_log_provider="anthropic",+ config={**_DIRECT, "claude_agent_model": SONNET},+)+++def _sdk_anthropic_failing(+ scenario_id: str,+ fault: PlannedFault,+ calls_per_turn: tuple[int, int, int],+ known_gap: KnownGap | None = None,+) -> Scenario:+ """The direct-Anthropic thread with one request failed and the CLI's own+ retries off, so the platform's handling of the failure is what runs."""+ return _SDK_ANTHROPIC.model_copy(+ update={+ "id": scenario_id,+ "fault": fault,+ "calls_per_turn": calls_per_turn,+ "cli_max_retries": 0,+ "known_gap": known_gap,+ }+ )+++_CLI_529_NOT_TRANSIENT = (+ "The bundled CLI words a 529 as 'API Error: 529 Overloaded. This is a "+ "server-side issue...', which is_transient_api_error does not match (it "+ "looks for 'status code 529'), so the platform never retries it: the "+ "reply is the CLI's error text"+)++SCENARIOS = [+ _SDK_ANTHROPIC,+ # The tool round's second call is refused. Unmatched as transient (see+ # _CLI_529_NOT_TRANSIENT), the CLI's error becomes the reply, and the+ # turn still owes the first call.+ _sdk_anthropic_failing(+ "sdk-anthropic-overloaded-after-tool",+ PlannedFault(kind="overloaded", turn=1, request=2),+ calls_per_turn=(1, 1, 1),+ ),+ # The connection drops instead, which is transient: the platform stops+ # reading before the CLI's result and relaunches the CLI on the session.+ # The turn owes the first call and the retry's.+ _sdk_anthropic_failing(+ "sdk-anthropic-dropped-after-tool",+ PlannedFault(kind="dropped", turn=1, request=2),+ calls_per_turn=(1, 2, 1),+ known_gap=KnownGap(+ check="tokens",+ reason=(+ "A retried turn is charged the failed attempt's spend, but its "+ "cost-log row and session usage keep only the retry's tokens"+ ),+ ),+ ),+ # A resumed turn whose first request drops is relaunched before anything+ # streamed, and charged its one successful call.+ _sdk_anthropic_failing(+ "sdk-anthropic-dropped-on-resume",+ PlannedFault(kind="dropped", turn=2, request=1),+ calls_per_turn=(1, 2, 1),+ ),+ _sdk_anthropic_failing(+ "sdk-anthropic-overloaded-on-resume",+ PlannedFault(kind="overloaded", turn=2, request=1),+ calls_per_turn=(1, 2, 1),+ known_gap=KnownGap(check="path", reason=_CLI_529_NOT_TRANSIENT),+ ),+ Scenario(+ id="sdk-openrouter",+ engine="sdk",+ model=f"anthropic/{SONNET}",+ charge="invoice",+ cost_log_provider="open_router",+ config={**_OPENROUTER, "claude_agent_model": f"anthropic/{SONNET}"},+ ),+ Scenario(+ id="sdk-openrouter-lookups-refused",+ engine="sdk",+ model=f"anthropic/{SONNET}",+ charge="cli_estimate",+ cost_log_provider="open_router",+ config={**_OPENROUTER, "claude_agent_model": f"anthropic/{SONNET}"},+ generation_lookups_fail=True,+ ),+ Scenario(+ id="sdk-openrouter-no-reconcile",+ engine="sdk",+ model=f"anthropic/{SONNET}",+ charge="cli_estimate",+ cost_log_provider="open_router",+ config={+ **_OPENROUTER,+ "claude_agent_model": f"anthropic/{SONNET}",+ "sdk_reconcile_openrouter_cost": False,+ },+ ),+ Scenario(+ id="sdk-openrouter-kimi",+ engine="sdk",+ model=KIMI,+ charge="invoice",+ cost_log_provider="open_router",+ config={**_OPENROUTER, "claude_agent_model": KIMI},+ ),+ Scenario(+ id="sdk-openrouter-kimi-no-reconcile",+ engine="sdk",+ model=KIMI,+ charge="moonshot_estimate",+ cost_log_provider="open_router",+ config={+ **_OPENROUTER,+ "claude_agent_model": KIMI,+ "sdk_reconcile_openrouter_cost": False,+ },+ ),+ Scenario(+ id="sdk-codex",+ engine="sdk",+ model="",+ charge="none",+ cost_log_provider="codex",+ config=_DIRECT,+ codex=True,+ ),+ Scenario(+ id="baseline-openrouter",+ engine="baseline",+ model=f"anthropic/{SONNET}",+ charge="invoice",+ cost_log_provider="open_router",+ config=_OPENROUTER,+ client_path="/openrouter/api/v1",+ ),+ Scenario(+ # The OpenAI client retries the refused request itself: the turn+ # completes, and the retry is billed once.+ id="baseline-openrouter-overloaded-once",+ engine="baseline",+ model=f"anthropic/{SONNET}",+ charge="invoice",+ cost_log_provider="open_router",+ config=_OPENROUTER,+ client_path="/openrouter/api/v1",+ fault=PlannedFault(kind="overloaded", turn=1, request=2),+ ),+ Scenario(+ id="baseline-anthropic",+ engine="baseline",+ model=SONNET,+ charge="invoice",+ cost_log_provider="anthropic",+ config=_DIRECT,+ client_path="/anthropic/v1",+ ),+ Scenario(+ id="baseline-ollama",+ engine="baseline",+ model="llama3.1:8b",+ charge="none",+ cost_log_provider="ollama",+ config={+ "use_local": True,+ "api_key": "ollama",+ "base_url": "{provider}/ollama/v1",+ },+ client_path="/ollama/v1",+ ),+ Scenario(+ id="microsoft-365-copilot",+ engine="microsoft_365",+ model="microsoft-365-copilot",+ charge="none",+ cost_log_provider="microsoft_365_copilot",+ # Graph runs no tools, so the tool turn is one call too.+ calls_per_turn=(1, 1, 1),+ ),+]+++def expected_charge_usd(basis: ChargeBasis, call: Call) -> float:+ if basis == "invoice":+ return call.invoice_usd+ if basis == "cli_estimate":+ list_price = ANTHROPIC_PRICES[call.model.removeprefix("anthropic/")]+ return list_price.of(call.usage, call.cache_ttl)+ if basis == "moonshot_estimate":+ return MOONSHOT_FALLBACK.of(call.usage, None)+ return 0.0diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/sdk_turns.py b/autogpt_platform/backend/backend/copilot/billing_e2e/sdk_turns.pynew file mode 100644index 0000000000..767a9967bb--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/sdk_turns.py@@ -0,0 +1,218 @@+"""SDK-engine turns: ``stream_chat_completion_sdk`` driving the bundled CLI.++The CLI really runs and really talks to the fake provider, directly or+through the Codex gateway. Stubbed: persistence, locks, transcript storage,+the copilot's own tools and hooks, and per-user flag lookups. Model+resolution, the CLI environment, retries, resume and every cost decision run+as in production.+"""++import shutil+import uuid+from contextlib import ExitStack+from pathlib import Path+from typing import Any+from unittest.mock import AsyncMock, MagicMock, patch++import pytest+from claude_agent_sdk import ClaudeAgentOptions, ClaudeSDKClient+from openai import AsyncOpenAI+from pydantic import SecretStr++from backend.copilot.config import ChatConfig+from backend.copilot.model import ChatSession+from backend.copilot.sdk import env as sdk_env+from backend.copilot.sdk import openrouter_cost+from backend.copilot.sdk import service as sdk_service+from backend.copilot.transcript import TranscriptDownload+from backend.copilot.tree import root_envelope+from backend.data.llm_registry.llm_models import LLMModel+from backend.data.model import OAuth2Credentials+from backend.integrations.codex.auth_bundle import CodexAuthBundleV1, CodexAuthTokensV1+from backend.integrations.codex.credential_codec import credentials_from_bundle+from backend.integrations.codex.http_session import CodexHttpSession+from backend.integrations.codex.models import CodexModelInfo+from backend.integrations.codex.transport import CodexAgentSession, CodexCredentialLease++from .provider import FakeProvider++CODEX_MODEL = LLMModel.GPT5_6_TERRA.value+CODEX_CREDENTIAL_ID = "codex-credential"+++class SdkTurns:+ def __init__(+ self,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+ provider: FakeProvider,+ config: ChatConfig,+ *,+ codex: bool = False,+ cli_max_retries: int | None = None,+ ) -> None:+ self._config_dir = tmp_path / "home" / ".claude"+ self._cwd = tmp_path / "copilot-cwd"+ self._cwd.mkdir()+ self._config = config+ self._transcripts: dict[str, TranscriptDownload] = {}+ self._lease = _codex_lease(provider) if codex else None+ # The CLI session each launch resumed, None for a fresh one.+ self.resumed: list[str | None] = []+ # The CLI inherits these, and the platform reads the CLI's session+ # files from the same place.+ monkeypatch.setenv("HOME", str(tmp_path / "home"))+ monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(self._config_dir))+ # Where a direct-Anthropic CLI sends its requests; the key is never+ # checked. OpenRouter and Codex turns override both.+ monkeypatch.setenv("ANTHROPIC_BASE_URL", f"{provider.url}/anthropic")+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant-billing-e2e")+ if cli_max_retries is not None:+ monkeypatch.setenv("CLAUDE_CODE_MAX_RETRIES", str(cli_max_retries))+ monkeypatch.setattr(sdk_env, "config", config)+ monkeypatch.setattr(+ openrouter_cost,+ "_GENERATION_URL",+ f"{provider.url}/openrouter/api/v1/generation",+ )++ async def __call__(+ self, session: ChatSession, prompt: str, *, new_pod: bool+ ) -> None:+ if new_pod:+ # Only what the turn uploaded survives a move to another pod.+ shutil.rmtree(self._config_dir / "projects", ignore_errors=True)+ with ExitStack() as stack:+ for name, value in self._stubs(session).items():+ stack.enter_context(patch.object(sdk_service, name, value))+ async for _ in sdk_service.stream_chat_completion_sdk(+ session_id=session.session_id,+ message=prompt,+ is_user_message=True,+ user_id=session.user_id,+ session=session,+ envelope=root_envelope(+ str(uuid.uuid4()), session_id=session.session_id+ ),+ credential_lease=self._lease,+ ):+ pass++ def _stubs(self, session: ChatSession) -> dict[str, Any]:+ return {+ "config": self._config,+ "ClaudeSDKClient": MagicMock(side_effect=self._client),+ "AsyncClusterLock": MagicMock(side_effect=_lock),+ "_make_sdk_cwd": MagicMock(return_value=str(self._cwd)),+ "get_chat_session": AsyncMock(return_value=session),+ "upsert_chat_session": AsyncMock(return_value=session),+ "download_transcript": AsyncMock(side_effect=self._download),+ "upload_transcript": AsyncMock(side_effect=self._upload),+ "compact_transcript": AsyncMock(return_value=None),+ "get_redis_async": AsyncMock(),+ "propagate_attributes": MagicMock(),+ "_build_system_prompt": AsyncMock(+ return_value=("You are a test assistant. Reply briefly.", None)+ ),+ "build_skills_context": AsyncMock(return_value=""),+ "build_skills_update_notice": AsyncMock(return_value=None),+ "build_session_context": AsyncMock(return_value=""),+ "inject_user_context": AsyncMock(return_value=None),+ "create_copilot_mcp_server": MagicMock(),+ "create_security_hooks": MagicMock(),+ "get_copilot_tool_names": MagicMock(return_value=[]),+ "get_sdk_disallowed_tools": MagicMock(return_value=[]),+ "get_user_tier": AsyncMock(return_value=None),+ "_resolve_dynamic_max_budget_usd": AsyncMock(return_value=100.0),+ "build_turn_budget_block": AsyncMock(return_value=""),+ "drain_pending_safe": AsyncMock(return_value=[]),+ "drain_pending_for_persist": AsyncMock(return_value=[]),+ "resolve_answered": AsyncMock(return_value=[]),+ "clear_pending_question": AsyncMock(),+ }++ def _client(+ self, *args: Any, options: ClaudeAgentOptions, **kwargs: Any+ ) -> ClaudeSDKClient:+ self.resumed.append(options.resume)+ # The copilot's own tools and hooks are not what is under test; the+ # CLI answers the provider's tool call with "no such tool" instead.+ options.mcp_servers = {}+ options.hooks = None+ options.allowed_tools = []+ options.disallowed_tools = []+ return ClaudeSDKClient(*args, options=options, **kwargs)++ async def _upload(+ self,+ *,+ session_id: str,+ content: bytes,+ message_count: int,+ mode: Any,+ **_: Any,+ ) -> None:+ self._transcripts[session_id] = TranscriptDownload(+ content=content, message_count=message_count, mode=mode+ )++ async def _download(+ self, user_id: str, session_id: str, *_: Any, **__: Any+ ) -> TranscriptDownload | None:+ return self._transcripts.get(session_id)+++def _lock(*_: Any, owner_id: str = "", **__: Any) -> MagicMock:+ return MagicMock(+ try_acquire=AsyncMock(return_value=owner_id),+ refresh=AsyncMock(),+ release=AsyncMock(),+ )+++class _CodexAccount(CodexAgentSession):+ """The production Codex session over the fake Responses endpoint.++ Only the model list is answered here: production fetches it through the+ platform's guarded HTTP client, which refuses loopback addresses.+ """++ async def models(self) -> list[CodexModelInfo]:+ return [+ CodexModelInfo(+ model=CODEX_MODEL,+ display_name="Codex",+ is_default=True,+ hidden=False,+ default_reasoning_effort="medium",+ supported_reasoning_efforts=["low", "medium", "high"],+ )+ ]+++def _codex_lease(provider: FakeProvider) -> CodexCredentialLease:+ credentials = _codex_credentials()+ client = AsyncOpenAI(api_key="codex-billing-e2e", base_url=f"{provider.url}/codex")+ session = CodexHttpSession(+ credentials,+ turn_timeout_seconds=120,+ tool_timeout_seconds=60,+ client=client,+ )+ return CodexCredentialLease(credentials, _CodexAccount(session, credentials))+++def _codex_credentials() -> OAuth2Credentials:+ # An unsigned JWT: the platform reads its claims, never its signature.+ token = "eyJhbGciOiJub25lIn0.eyJleHAiOjk5OTk5OTk5OTksImVtYWlsIjoiYUBiLmMifQ.sig"+ bundle = CodexAuthBundleV1(+ tokens=CodexAuthTokensV1(+ id_token=SecretStr(token),+ access_token=SecretStr(token),+ refresh_token=SecretStr("refresh"),+ ),+ codex_runtime_version="http",+ )+ return credentials_from_bundle(bundle).model_copy(+ update={"id": CODEX_CREDENTIAL_ID}+ )diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/thread_billing_test.py b/autogpt_platform/backend/backend/copilot/billing_e2e/thread_billing_test.pynew file mode 100644index 0000000000..1b55308807--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/thread_billing_test.py@@ -0,0 +1,152 @@+"""A three-turn chat on every route, each turn charged what its provider billed.++Turn 1 answers once. Turn 2 asks for a tool, so the model is called twice.+Turn 3 runs on a pod that kept nothing on disk: the SDK engine restores its+CLI session from storage and resumes it, the path on which a resumed turn was+charged the whole chat's spend so far until #15342.++The provider keeps its own invoice; the ledger records what the platform+charged. Each turn must charge the user's spend counters and the turn tree+once, for the turn's own calls, and write one cost-log row whose tokens are+the provider's.++The SDK scenarios run the bundled Claude Code CLI, so they skip where there is+none (the Windows wheel ships without one); CI runs them on Linux.+"""++from pathlib import Path++import pytest++from backend.copilot.baseline import service as baseline_service+from backend.copilot.config import ChatConfig+from backend.copilot.model import ChatMessage, ChatSession, Usage+from backend.copilot.sdk import service as sdk_service+from backend.copilot.sdk.cli_openrouter_compat_test import _resolve_cli_path++from .audit import audit_thread+from .baseline_turns import BaselineTurns+from .ledger import PlatformLedger+from .m365_turns import MICROSOFT_365_CREDENTIAL_ID, Microsoft365Turns+from .provider import FakeProvider+from .replies import TOOL_MARKER+from .scenarios import SCENARIOS, Scenario+from .sdk_turns import CODEX_CREDENTIAL_ID, SdkTurns++PROMPTS = ("Say hello.", f"Look something up for me. {TOOL_MARKER}", "Thanks!")+NEW_POD_TURN = 2+++# Its own event loop: a background task another test left on the shared+# session loop must not write into this thread's ledger mid-run.[email protected](loop_scope="function")[email protected]("scenario", SCENARIOS, ids=lambda scenario: scenario.id)+async def test_each_turn_is_charged_what_its_provider_billed(+ scenario: Scenario,+ provider: FakeProvider,+ ledger: PlatformLedger,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ if scenario.needs_cli and _resolve_cli_path() is None:+ pytest.skip("No Claude Code CLI binary available")+ provider.generation_lookups_fail = scenario.generation_lookups_fail+ provider.faults = [scenario.fault] if scenario.fault else []+ run_turn = _turn_runner(scenario, provider, tmp_path, monkeypatch)+ session = _session(scenario)+ usage_by_turn: list[list[Usage]] = []++ for turn, prompt in enumerate(PROMPTS):+ provider.turn = ledger.turn = turn+ session.messages.append(ChatMessage(role="user", content=prompt))+ usage_before = len(session.usage)+ await run_turn(session, prompt, new_pod=turn == NEW_POD_TURN)+ await ledger.settle()+ usage_by_turn.append(session.usage[usage_before:])++ cli_resumed = (+ [launch == session.session_id for launch in run_turn.resumed]+ if isinstance(run_turn, SdkTurns)+ else None+ )+ findings = audit_thread(+ scenario, provider, ledger, session, usage_by_turn, cli_resumed+ )+ gap = scenario.known_gap+ unexplained = [f.message for f in findings if gap is None or f.check != gap.check]+ assert not unexplained, "\n".join(unexplained)+ if gap is not None:+ assert any(+ f.check == gap.check for f in findings+ ), f"no longer reproduces, so drop this known gap: {gap.reason}"+ pytest.xfail(gap.reason)+++def _turn_runner(+ scenario: Scenario,+ provider: FakeProvider,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> SdkTurns | BaselineTurns | Microsoft365Turns:+ if scenario.engine == "microsoft_365":+ return Microsoft365Turns(monkeypatch, provider)+ if scenario.engine == "baseline":+ return BaselineTurns(+ monkeypatch,+ _config(baseline_service.config, scenario, provider),+ base_url=f"{provider.url}{scenario.client_path}",+ model=scenario.model,+ )+ return SdkTurns(+ tmp_path,+ monkeypatch,+ provider,+ _config(sdk_service.config, scenario, provider),+ codex=scenario.codex,+ cli_max_retries=scenario.cli_max_retries,+ )+++def _config(base: ChatConfig, scenario: Scenario, provider: FakeProvider) -> ChatConfig:+ overrides = {+ name: value.format(provider=provider.url) if isinstance(value, str) else value+ for name, value in scenario.config.items()+ }+ # Pinned so the deployment's own settings cannot change the path a+ # scenario takes; each scenario overrides what it is about.+ return base.model_copy(+ update={+ "use_local": False,+ "use_claude_code_subscription": False,+ "use_e2b_sandbox": False,+ "claude_agent_fallback_model": None,+ "claude_agent_cli_path": None,+ "claude_agent_use_resume": True,+ "claude_agent_max_transient_retries": 3,+ "sdk_reconcile_openrouter_cost": True,+ **overrides,+ }+ )+++def _session(scenario: Scenario) -> ChatSession:+ if scenario.codex:+ session = ChatSession.new(+ "billing-e2e-user",+ dry_run=False,+ llm_auth_provider="codex",+ llm_credential_id=CODEX_CREDENTIAL_ID,+ )+ elif scenario.engine == "microsoft_365":+ session = ChatSession.new(+ "billing-e2e-user",+ dry_run=False,+ llm_auth_provider="microsoft_365_copilot",+ llm_credential_id=MICROSOFT_365_CREDENTIAL_ID,+ )+ else:+ session = ChatSession.new("billing-e2e-user", dry_run=False)+ # A titled chat skips title generation, a separate call this suite does+ # not account for.+ session.title = "Billing e2e"+ return sessiondiff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/wire.py b/autogpt_platform/backend/backend/copilot/billing_e2e/wire.pynew file mode 100644index 0000000000..93b64017c7--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/wire.py@@ -0,0 +1,217 @@+"""Streamed response bodies in each upstream's own format."""++import json+import uuid+from typing import Any, Literal++from pydantic import BaseModel++from .invoice import CacheTtl, Usage++# A tool no engine has, so every engine answers it with an error result and+# asks the model again: a second billed call in the same turn.+TOOL_NAME = "lookup_fact"+TOOL_ARGUMENTS = {"query": "billing"}+++class Reply(BaseModel):+ kind: Literal["text", "tool"]+ text: str = ""+++def anthropic_messages(+ *, message_id: str, model: str, reply: Reply, usage: Usage, ttl: CacheTtl | None+) -> str:+ """Anthropic Messages SSE, as Anthropic and OpenRouter's /messages stream it."""+ start_usage = {+ "input_tokens": usage.input_tokens,+ "output_tokens": 0,+ "cache_read_input_tokens": usage.cache_read_tokens,+ "cache_creation_input_tokens": usage.cache_write_tokens,+ "cache_creation": {+ "ephemeral_5m_input_tokens": usage.cache_write_tokens if ttl == "5m" else 0,+ "ephemeral_1h_input_tokens": usage.cache_write_tokens if ttl == "1h" else 0,+ },+ }+ if reply.kind == "tool":+ block: dict[str, Any] = {+ "type": "tool_use",+ "id": f"toolu_{uuid.uuid4().hex[:24]}",+ "name": TOOL_NAME,+ "input": {},+ }+ delta = {"type": "input_json_delta", "partial_json": json.dumps(TOOL_ARGUMENTS)}+ else:+ block = {"type": "text", "text": ""}+ delta = {"type": "text_delta", "text": reply.text}+ events = [+ {+ "type": "message_start",+ "message": {+ "id": message_id,+ "type": "message",+ "role": "assistant",+ "model": model,+ "content": [],+ "stop_reason": None,+ "stop_sequence": None,+ "usage": start_usage,+ },+ },+ {"type": "content_block_start", "index": 0, "content_block": block},+ {"type": "content_block_delta", "index": 0, "delta": delta},+ {"type": "content_block_stop", "index": 0},+ {+ "type": "message_delta",+ "delta": {+ "stop_reason": "tool_use" if reply.kind == "tool" else "end_turn",+ "stop_sequence": None,+ },+ "usage": {"output_tokens": usage.output_tokens},+ },+ {"type": "message_stop"},+ ]+ return "".join(f"event: {e['type']}\ndata: {json.dumps(e)}\n\n" for e in events)+++def chat_completion_chunks(+ *,+ model: str,+ reply: Reply,+ usage: Usage,+ cost: float | None,+ cache_write_field: str | None,+) -> str:+ """OpenAI chat-completion SSE, ending in a usage-only chunk.++ ``prompt_tokens`` includes the cache buckets, as the OpenAI shape does;+ ``cost`` is OpenRouter's extension and ``cache_write_field`` names the+ cache-write count the way that upstream spells it.+ """+ base = {+ "id": f"chatcmpl-{uuid.uuid4().hex}",+ "object": "chat.completion.chunk",+ "created": 0,+ "model": model,+ }+ if reply.kind == "tool":+ delta: dict[str, Any] = {+ "role": "assistant",+ "tool_calls": [+ {+ "index": 0,+ "id": f"call_{uuid.uuid4().hex[:24]}",+ "type": "function",+ "function": {+ "name": TOOL_NAME,+ "arguments": json.dumps(TOOL_ARGUMENTS),+ },+ }+ ],+ }+ else:+ delta = {"role": "assistant", "content": reply.text}+ prompt_tokens = (+ usage.input_tokens + usage.cache_read_tokens + usage.cache_write_tokens+ )+ usage_body: dict[str, Any] = {+ "prompt_tokens": prompt_tokens,+ "completion_tokens": usage.output_tokens,+ "total_tokens": prompt_tokens + usage.output_tokens,+ }+ if cache_write_field is not None:+ usage_body["prompt_tokens_details"] = {+ "cached_tokens": usage.cache_read_tokens,+ cache_write_field: usage.cache_write_tokens,+ }+ if cost is not None:+ usage_body["cost"] = cost+ finish = "tool_calls" if reply.kind == "tool" else "stop"+ chunks = [+ {**base, "choices": [{"index": 0, "delta": delta, "finish_reason": None}]},+ {**base, "choices": [{"index": 0, "delta": {}, "finish_reason": finish}]},+ {**base, "choices": [], "usage": usage_body},+ ]+ return "".join(f"data: {json.dumps(c)}\n\n" for c in chunks) + "data: [DONE]\n\n"+++def responses_events(+ *, response_id: str, model: str, reply: Reply, usage: Usage+) -> str:+ """OpenAI Responses SSE, as ChatGPT's Codex backend streams it.++ OpenAI caching has no write bucket: ``input_tokens`` counts every input+ token and ``cached_tokens`` the cached subset of them.+ """+ if reply.kind == "tool":+ item: dict[str, Any] = {+ "type": "function_call",+ "id": f"fc_{uuid.uuid4().hex}",+ "call_id": f"call_{uuid.uuid4().hex[:24]}",+ "name": TOOL_NAME,+ "arguments": json.dumps(TOOL_ARGUMENTS),+ "status": "completed",+ }+ deltas: list[dict[str, Any]] = []+ else:+ item = {+ "type": "message",+ "id": f"msg_{uuid.uuid4().hex}",+ "role": "assistant",+ "status": "completed",+ "content": [{"type": "output_text", "text": reply.text, "annotations": []}],+ }+ deltas = [+ {+ "type": "response.output_text.delta",+ "item_id": item["id"],+ "output_index": 0,+ "content_index": 0,+ "delta": reply.text,+ }+ ]+ response = {+ "id": response_id,+ "object": "response",+ "created_at": 0,+ "model": model,+ "status": "completed",+ "output": [item],+ "usage": {+ "input_tokens": usage.input_tokens + usage.cache_read_tokens,+ "input_tokens_details": {"cached_tokens": usage.cache_read_tokens},+ "output_tokens": usage.output_tokens,+ "output_tokens_details": {"reasoning_tokens": 0},+ "total_tokens": usage.input_tokens+ + usage.cache_read_tokens+ + usage.output_tokens,+ },+ }+ events = [+ {+ "type": "response.created",+ "response": {**response, "status": "in_progress", "output": []},+ },+ *deltas,+ {"type": "response.output_item.done", "output_index": 0, "item": item},+ {"type": "response.completed", "response": response},+ ]+ return "".join(f"event: {e['type']}\ndata: {json.dumps(e)}\n\n" for e in events)+++def graph_chat_snapshots(*, conversation_id: str, prompt: str, answer: str) -> str:+ """Microsoft Graph's chatOverStream: whole-conversation snapshots, the+ reply growing a word at a time."""+ words = answer.split(" ")+ snapshots = [+ {+ "id": conversation_id,+ "state": "active",+ "messages": [+ {"id": f"{conversation_id}-prompt", "text": prompt},+ {"id": f"{conversation_id}-reply", "text": " ".join(words[:count])},+ ],+ }+ for count in range(1, len(words) + 1)+ ]+ return "".join(f"data: {json.dumps(s)}\n\n" for s in snapshots)