Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): retain Codex usage for interrupted turns (#15720)

Nicholas Tindle238cdc42f9d9
diff --git a/autogpt_platform/backend/backend/copilot/sdk/codex_compat_gateway.py b/autogpt_platform/backend/backend/copilot/sdk/codex_compat_gateway.py
index f7b3695f5e..b7ac117ede 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/codex_compat_gateway.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/codex_compat_gateway.py
@@ -146,6 +146,7 @@ class _Conversation(_GatewayState):
response_lock: asyncio.Lock = Field(default_factory=asyncio.Lock)
task: asyncio.Task[None] | None = None
result: CodexInvocationResult | None = None
+ usage: CodexTokenUsage | None = None
replays: dict[str, _Replay] = Field(default_factory=dict)
@@ -238,7 +239,6 @@ class CodexAnthropicGateway:
if not self._results:
return None
latest = self._results[-1]
- usage = _sum_usage(result.usage for result in self._results)
durations = [
result.duration_ms
for result in self._results
@@ -247,7 +247,7 @@ class CodexAnthropicGateway:
return latest.model_copy(
update={
"duration_ms": sum(durations) if durations else None,
- "usage": usage,
+ "usage": self.usage,
}
)
@@ -255,6 +255,14 @@ class CodexAnthropicGateway:
def results(self) -> tuple[CodexInvocationResult, ...]:
return tuple(self._results)
+ @property
+ def usage(self) -> CodexTokenUsage | None:
+ """Reported usage across completed and interrupted conversations.
+
+ Conversations remain retained for this gateway's lifetime (one SDK turn).
+ """
+ return _sum_usage(c.usage for c in self._conversations.values())
+
async def start(self) -> None:
await self.__aenter__()
@@ -537,6 +545,9 @@ class CodexAnthropicGateway:
async def handle_event(event: CodexStreamEvent) -> None:
if event.type == "text_delta" and event.delta:
await conversation.queue.put(_TextDelta(text=event.delta))
+ elif event.type == "usage":
+ # Do not suspend here: usage must survive cancellation during cleanup.
+ conversation.usage = _sum_usage((conversation.usage, event.usage))
async def handle_tool(call: CodexDynamicToolCall) -> CodexDynamicToolResult:
future: asyncio.Future[CodexDynamicToolResult] = (
@@ -572,6 +583,8 @@ class CodexAnthropicGateway:
handle_event,
)
conversation.result = result
+ if conversation.usage is None:
+ conversation.usage = result.usage
self._results.append(result)
await conversation.queue.put(_Completed(result=result))
except asyncio.CancelledError:
diff --git a/autogpt_platform/backend/backend/copilot/sdk/codex_usage_persistence_test.py b/autogpt_platform/backend/backend/copilot/sdk/codex_usage_persistence_test.py
new file mode 100644
index 0000000000..4f07ccaed7
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/sdk/codex_usage_persistence_test.py
@@ -0,0 +1,196 @@
+"""Codex interruption reaches the real session and cost-log accounting path."""
+
+import asyncio
+import contextlib
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from claude_agent_sdk import AssistantMessage, ResultMessage, TextBlock
+from pydantic import SecretStr
+
+from backend.copilot.model import ChatMessage, ChatSession
+from backend.copilot.response_model import StreamTextDelta
+from backend.copilot.sdk.service import stream_chat_completion_sdk
+from backend.copilot.token_tracking import persist_and_record_usage
+from backend.data.model import OAuth2Credentials
+from backend.integrations.codex.transport import CodexCredentialLease
+
+from .codex_usage_test import PAYLOAD, run_conversation, usage_gateway
+from .retry_scenarios_test import _make_sdk_patches
+
+_SVC = "backend.copilot.sdk.service"
+_TRACKING = "backend.copilot.token_tracking"
+
+
[email protected]("ending", ["failure", "cancel", "generator_close", "success"])
+async def test_sdk_finally_records_subscription_usage_once(ending):
+ session, lease = _session_and_lease()
+ waiting = asyncio.Event()
+ async with usage_gateway("success" if ending == "success" else "failed") as gateway:
+ client = _client(gateway, ending, waiting, session.session_id)
+ with contextlib.ExitStack() as stack:
+ _stub_sdk(stack, session, gateway, client)
+ persist, save, cost_log, credits, budget = _record_accounting(stack)
+ stream = stream_chat_completion_sdk(
+ session_id=session.session_id,
+ message="hello",
+ is_user_message=True,
+ user_id=session.user_id,
+ session=session,
+ credential_lease=lease,
+ )
+ await _consume_turn(stream, ending, waiting)
+
+ persist.assert_awaited_once()
+ save.assert_awaited_with(session)
+ cost_log.assert_called_once()
+ credits.assert_not_awaited()
+ budget.assert_not_awaited()
+ _assert_usage(
+ session, cost_log.call_args.args[0], lease, 2 if ending == "success" else 1
+ )
+
+
+def _session_and_lease():
+ session = ChatSession(
+ session_id="codex-usage-session",
+ user_id="test-user",
+ usage=[],
+ started_at=datetime.now(UTC),
+ updated_at=datetime.now(UTC),
+ messages=[ChatMessage(role="user", content="hello")],
+ title="Usage regression",
+ )
+ lease = CodexCredentialLease(
+ OAuth2Credentials(
+ id="codex-usage-credential",
+ provider="codex",
+ title="ChatGPT",
+ access_token=SecretStr("test"),
+ scopes=[],
+ ),
+ MagicMock(),
+ )
+ return session, lease
+
+
+def _client(gateway, ending, waiting, session_id):
+ async def receive():
+ if ending in ("cancel", "generator_close"):
+ conversation = gateway._start_conversation(PAYLOAD)
+ await asyncio.wait_for(conversation.queue.get(), timeout=2)
+ waiting.set()
+ if ending == "cancel":
+ await asyncio.Event().wait()
+ else:
+ await run_conversation(gateway)
+ if ending == "failure":
+ raise ValueError("scripted transport failure")
+ yield AssistantMessage(content=[TextBlock(text="partial")], model=gateway.model)
+ yield ResultMessage(
+ subtype="success",
+ duration_ms=1,
+ duration_api_ms=1,
+ is_error=False,
+ num_turns=1,
+ session_id=session_id,
+ usage={"input_tokens": 999, "output_tokens": 999},
+ total_cost_usd=99,
+ result="done",
+ )
+
+ client = MagicMock()
+ client.query = AsyncMock()
+ client.receive_response = receive
+ context = AsyncMock()
+ context.__aenter__.return_value = client
+ context.__aexit__.return_value = None
+ return context
+
+
+def _stub_sdk(stack, session, gateway, client):
+ for target, kwargs in _make_sdk_patches(
+ session,
+ original_transcript="",
+ compacted_transcript=None,
+ client_side_effect=lambda **_: client,
+ ):
+ stack.enter_context(patch(target, **kwargs))
+ stack.enter_context(patch(f"{_SVC}.resolve_answered", new=_no_answered_cards))
+ for name, value in (
+ ("inject_user_context", None),
+ ("build_session_context", ""),
+ ("build_skills_update_notice", None),
+ ("capture_chat_turn", None),
+ ("resolve_codex_model_route", (gateway.model, None, "env")),
+ ):
+ stack.enter_context(
+ patch(f"{_SVC}.{name}", new_callable=AsyncMock, return_value=value)
+ )
+ stack.enter_context(patch(f"{_SVC}.gateway_for_lease", return_value=gateway))
+
+
+async def _no_answered_cards(*_, **__):
+ return
+ yield
+
+
+def _record_accounting(stack):
+ persist = stack.enter_context(
+ patch(f"{_SVC}.persist_and_record_usage", wraps=persist_and_record_usage)
+ )
+ save = stack.enter_context(
+ patch(f"{_SVC}.upsert_chat_session", new_callable=AsyncMock)
+ )
+ cost_log = stack.enter_context(patch(f"{_TRACKING}._schedule_cost_log"))
+ credits = stack.enter_context(
+ patch(f"{_TRACKING}.record_credit_billed_cost", new_callable=AsyncMock)
+ )
+ budget = stack.enter_context(
+ patch(f"{_TRACKING}.record_cost_usage", new_callable=AsyncMock)
+ )
+ return persist, save, cost_log, credits, budget
+
+
+async def _consume_turn(stream, ending, waiting):
+ async def consume():
+ async for event in stream:
+ if ending == "generator_close" and isinstance(event, StreamTextDelta):
+ await stream.aclose()
+ break
+
+ task = asyncio.create_task(consume())
+ try:
+ if ending == "cancel":
+ await asyncio.wait_for(waiting.wait(), timeout=5)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ else:
+ await asyncio.wait_for(task, timeout=5)
+ finally:
+ if not task.done():
+ task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await task
+ await stream.aclose()
+
+
+def _assert_usage(session, entry, lease, calls):
+ assert len(session.usage) == 1
+ usage = session.usage[0]
+ assert usage.prompt_tokens == 60 * calls
+ assert usage.completion_tokens == 20 * calls
+ assert usage.cache_read_tokens == 40 * calls
+ assert usage.total_tokens == 80 * calls
+ assert usage.cache_creation_tokens == 0
+ assert entry.input_tokens == 60 * calls
+ assert entry.output_tokens == 20 * calls
+ assert entry.cache_read_tokens == 40 * calls
+ assert entry.provider == "codex"
+ assert entry.credential_id == lease.credentials.id
+ assert entry.cost_microdollars is None
+ assert entry.metadata["billing_mode"] == "user_subscription"
+ assert entry.metadata["execution_path"] == "codex_claude_sdk"
diff --git a/autogpt_platform/backend/backend/copilot/sdk/codex_usage_test.py b/autogpt_platform/backend/backend/copilot/sdk/codex_usage_test.py
new file mode 100644
index 0000000000..cc78b9bddb
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/sdk/codex_usage_test.py
@@ -0,0 +1,231 @@
+"""Reported Codex usage survives an interrupted tool loop, once per call."""
+
+import asyncio
+import json
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+
+import httpx
+import pytest
+from openai import AsyncOpenAI
+from pydantic import SecretStr
+
+from backend.copilot.sdk.codex_compat_gateway import CodexAnthropicGateway
+from backend.copilot.sdk.service import _codex_gateway_usage
+from backend.data.model import OAuth2Credentials
+from backend.integrations.codex.http_session import CodexHttpSession
+from backend.integrations.codex.models import CodexDynamicToolResult, CodexTokenUsage
+
+USAGE = {
+ "input_tokens": 100,
+ "input_tokens_details": {"cached_tokens": 40},
+ "output_tokens": 20,
+ "output_tokens_details": {"reasoning_tokens": 5},
+ "total_tokens": 120,
+}
+PAYLOAD: dict[str, object] = {
+ "messages": [{"role": "user", "content": "hello"}],
+ "tools": [
+ {"name": "lookup", "description": "Look up", "input_schema": {"type": "object"}}
+ ],
+}
+
+
[email protected]("ending", ["failed", "disconnect", "timeout", "cancel"])
+async def test_interrupted_tool_loop_retains_completed_call_usage(ending):
+ async with usage_gateway(ending) as gateway:
+ await run_conversation(gateway, cancel_at_tool=ending == "cancel")
+
+ assert gateway.result is None
+ assert _codex_gateway_usage(gateway) == expected_usage(1)
+
+
[email protected]("ending", ["success", "failed_with_usage", "incomplete"])
+async def test_response_snapshots_and_final_result_are_counted_once(ending):
+ async with usage_gateway(ending) as gateway:
+ await run_conversation(gateway)
+
+ assert _codex_gateway_usage(gateway) == expected_usage(2)
+
+
+async def test_an_early_failure_has_no_usage():
+ async with usage_gateway("early_failure") as gateway:
+ await run_conversation(gateway)
+
+ assert _codex_gateway_usage(gateway) is None
+
+
+async def test_a_retry_keeps_failed_usage_without_recounting_success():
+ async with usage_gateway("failed_then_success") as gateway:
+ await run_conversation(gateway)
+ await run_conversation(gateway)
+
+ assert _codex_gateway_usage(gateway) == expected_usage(3)
+
+
+ "ending", ["stream_disconnect", "stream_timeout", "stream_cancel", "stream_failed"]
+)
+async def test_interrupted_stream_retains_usage_and_closes_response(ending):
+ stream = _InterruptedStream(ending)
+ async with usage_gateway(ending, response_stream=stream) as gateway:
+ await run_conversation(
+ gateway, cancel_during_response=ending == "stream_cancel"
+ )
+ assert stream.closed
+
+ assert gateway.result is None
+ assert _codex_gateway_usage(gateway) == expected_usage(2)
+
+
+async def run_conversation(
+ gateway: CodexAnthropicGateway,
+ *,
+ cancel_at_tool: bool = False,
+ cancel_during_response: bool = False,
+) -> None:
+ conversation = gateway._start_conversation(PAYLOAD)
+ assert conversation.task is not None
+ await asyncio.wait_for(conversation.queue.get(), timeout=2)
+ if cancel_at_tool:
+ await gateway.close()
+ assert conversation.task.cancelled()
+ return
+ for record in conversation.pending.values():
+ record.future.set_result(CodexDynamicToolResult(content="found"))
+ if cancel_during_response:
+ await asyncio.wait_for(conversation.queue.get(), timeout=2)
+ await gateway.close()
+ assert conversation.task.cancelled()
+ return
+ await asyncio.wait_for(conversation.task, timeout=2)
+
+
+@asynccontextmanager
+async def usage_gateway(
+ ending: str, *, response_stream: httpx.AsyncByteStream | None = None
+) -> AsyncIterator[CodexAnthropicGateway]:
+ calls = 0
+
+ async def respond(_request: httpx.Request) -> httpx.Response:
+ nonlocal calls
+ calls += 1
+ return await _provider_response(ending, calls, response_stream)
+
+ async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http:
+ session = CodexHttpSession(
+ OAuth2Credentials(
+ id="codex-usage-test",
+ provider="codex",
+ title="ChatGPT",
+ access_token=SecretStr("test"),
+ scopes=[],
+ ),
+ turn_timeout_seconds=1 if ending in ("timeout", "stream_timeout") else 5,
+ tool_timeout_seconds=5,
+ client=AsyncOpenAI(
+ api_key="test",
+ base_url="https://codex.test",
+ http_client=http,
+ max_retries=0,
+ ),
+ )
+ async with CodexAnthropicGateway(
+ agent_session=session, model="gpt-5.6-sol"
+ ) as gateway:
+ yield gateway
+
+
+async def _provider_response(
+ ending: str, calls: int, response_stream: httpx.AsyncByteStream | None
+) -> httpx.Response:
+ if ending == "early_failure" or (calls % 2 == 0 and ending == "disconnect"):
+ raise httpx.ReadError("provider disconnected")
+ if calls % 2 == 0 and ending == "timeout":
+ await asyncio.Event().wait()
+ if calls % 2 == 0 and ending.startswith("stream_"):
+ return httpx.Response(
+ 200,
+ headers={"content-type": "text/event-stream"},
+ stream=response_stream or _InterruptedStream(ending),
+ )
+ events = _response_events(ending, calls)
+ return httpx.Response(
+ 200,
+ headers={"content-type": "text/event-stream"},
+ content="".join(f"data: {json.dumps(event)}\n\n" for event in events),
+ )
+
+
+def _response_events(ending: str, calls: int) -> list[dict[str, object]]:
+ tool = calls % 2 == 1
+ success = ending == "success" or (ending == "failed_then_success" and calls > 2)
+ kind = "response.completed" if tool or success else "response.failed"
+ if not tool and ending == "incomplete":
+ kind = "response.incomplete"
+ usage = (
+ USAGE
+ if tool or success or ending in ("failed_with_usage", "incomplete")
+ else None
+ )
+ response = {"id": f"resp-{calls}", "model": "gpt-5.6-sol", "usage": usage}
+ events: list[dict[str, object]] = [
+ {
+ "type": "response.created",
+ "response": {**response, "usage": usage if tool or success else None},
+ }
+ ]
+ if tool:
+ events.append(
+ {
+ "type": "response.output_item.done",
+ "item": {
+ "type": "function_call",
+ "name": "lookup",
+ "arguments": "{}",
+ "call_id": f"call-{calls}",
+ },
+ }
+ )
+ events.append({"type": kind, "response": response})
+ return events
+
+
+class _InterruptedStream(httpx.AsyncByteStream):
+ def __init__(self, ending: str):
+ self.ending = ending
+ self.closed = False
+
+ async def __aiter__(self):
+ for event in (
+ {
+ "type": "response.in_progress",
+ "response": {"id": "partial", "usage": USAGE},
+ },
+ {"type": "response.output_text.delta", "delta": "partial"},
+ ):
+ yield f"data: {json.dumps(event)}\n\n".encode()
+ if self.ending == "stream_disconnect":
+ raise httpx.ReadError("stream disconnected")
+ if self.ending == "stream_failed":
+ failure = {"type": "response.failed", "response": {"usage": USAGE}}
+ yield f"data: {json.dumps(failure)}\n\n".encode()
+ await asyncio.Event().wait()
+
+ async def aclose(self) -> None:
+ self.closed = True
+
+
+def expected_usage(calls: int) -> CodexTokenUsage:
+ return CodexTokenUsage(
+ input_tokens=100 * calls,
+ cached_input_tokens=40 * calls,
+ output_tokens=20 * calls,
+ reasoning_output_tokens=5 * calls,
+ total_tokens=120 * calls,
+ )
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 752b6c2ec0..ced6ceea0e 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -2488,9 +2488,8 @@ async def _resolve_sdk_model_for_request(
def _codex_gateway_usage(
gateway: CodexAnthropicGateway,
) -> CodexTokenUsage | None:
- """Read authoritative aggregate usage from a completed Codex gateway."""
- result = gateway.result
- return result.usage if result is not None else None
+ """Read authoritative usage, including interrupted Codex conversations."""
+ return gateway.usage
def _reconciles_openrouter_cost(is_codex_transport: bool) -> bool:
diff --git a/autogpt_platform/backend/backend/integrations/codex/http_session.py b/autogpt_platform/backend/backend/integrations/codex/http_session.py
index 0319d56da6..05cc1a1266 100644
--- a/autogpt_platform/backend/backend/integrations/codex/http_session.py
+++ b/autogpt_platform/backend/backend/integrations/codex/http_session.py
@@ -204,8 +204,17 @@ class CodexHttpSession:
# with_raw_response returns LegacyAPIResponse for streaming requests;
# its synchronous parse() returns the AsyncStream to iterate.
stream = raw.parse()
- async for event in stream:
- await _consume_event(event, outcome, event_handler)
+ async with stream:
+ try:
+ async for event in stream:
+ await _consume_event(event, outcome, event_handler)
+ finally:
+ # Publish the last snapshot once per response, even on cancellation
+ # or failure, before another model call can lose this call's usage.
+ if event_handler is not None and outcome.usage is not None:
+ await event_handler(
+ CodexStreamEvent(type="usage", usage=outcome.usage)
+ )
return outcome
@@ -244,15 +253,22 @@ async def _consume_event(
)
return
- if kind in ("response.created", "response.completed", "response.in_progress"):
+ if kind in (
+ "response.created",
+ "response.completed",
+ "response.in_progress",
+ "response.incomplete",
+ "response.failed",
+ ):
response = getattr(event, "response", None)
- if response is None:
+ if response is not None:
+ outcome.response_id = getattr(response, "id", "") or outcome.response_id
+ outcome.model = getattr(response, "model", None) or outcome.model
+ if (raw_usage := getattr(response, "usage", None)) is not None:
+ outcome.usage = _usage_from(raw_usage)
+ # Failed responses fall through to raise after their usage is captured.
+ if kind != "response.failed":
return
- outcome.response_id = getattr(response, "id", "") or outcome.response_id
- outcome.model = getattr(response, "model", None) or outcome.model
- if (raw_usage := getattr(response, "usage", None)) is not None:
- outcome.usage = _usage_from(raw_usage)
- return
if kind in ("response.failed", "error"):
raise CodexTurnFailedError(_failure_message(event))
diff --git a/autogpt_platform/backend/backend/integrations/codex/http_session_test.py b/autogpt_platform/backend/backend/integrations/codex/http_session_test.py
index 8f3102cada..d5e64eaea4 100644
--- a/autogpt_platform/backend/backend/integrations/codex/http_session_test.py
+++ b/autogpt_platform/backend/backend/integrations/codex/http_session_test.py
@@ -51,11 +51,22 @@ class _RawResponse:
def parse(self):
# Match LegacyAPIResponse.parse, which synchronously returns the
# AsyncStream for an SSE response.
- async def stream():
- for event in self._events:
- yield event
+ return _Stream(self._events)
- return stream()
+
+class _Stream:
+ def __init__(self, events: list[_Event]) -> None:
+ self._events = events
+
+ async def __aiter__(self):
+ for event in self._events:
+ yield event
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *_exc) -> None:
+ pass
class _FakeResponses:
diff --git a/autogpt_platform/backend/backend/integrations/codex/models.py b/autogpt_platform/backend/backend/integrations/codex/models.py
index 876b02955c..3d11828f56 100644
--- a/autogpt_platform/backend/backend/integrations/codex/models.py
+++ b/autogpt_platform/backend/backend/integrations/codex/models.py
@@ -145,9 +145,13 @@ class CodexStreamEvent(BaseModel):
Replaces the CLI SDK's ``Notification`` on the HTTP path so nothing outside
this package has to import ``openai_codex`` -- that package hard-depends on
the 391 MB bundled binary, with no extra to opt out of it.
+
+ Usage events carry one response's last reported snapshot, emitted once when
+ its stream exits, rather than the invocation's running total.
"""
model_config = ConfigDict(extra="forbid")
- type: Literal["text_delta", "reasoning_delta"]
- delta: str
+ type: Literal["text_delta", "reasoning_delta", "usage"]
+ delta: str = ""
+ usage: CodexTokenUsage | None = None