Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): preserve interrupted SDK partial work on final-failure exit (#12918)

Zamil Majdy5b1d9763edec
diff --git a/autogpt_platform/backend/backend/copilot/sdk/interrupted_partial_test.py b/autogpt_platform/backend/backend/copilot/sdk/interrupted_partial_test.py
new file mode 100644
index 0000000000..3c69a8af04
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/sdk/interrupted_partial_test.py
@@ -0,0 +1,322 @@
+"""Tests for partial-work preservation when an SDK turn is interrupted.
+
+Covers the regression SECRT-2275 surfaced: when the SDK retry loop rolls
+back ``session.messages`` for a failed attempt (correct so a successful
+retry doesn't duplicate content) it MUST re-attach the rolled-back work on
+final-failure exit. Without that, the user's UI streamed tokens live then
+a refresh shows an empty turn — described by users as "the turn is gone".
+
+Tests target the ``_InterruptedAttempt`` dataclass + the orphan-tool flush
+directly. Full retry-loop coverage lives in ``retry_scenarios_test.py``.
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from backend.copilot.constants import (
+ COPILOT_ERROR_PREFIX,
+ COPILOT_RETRYABLE_ERROR_PREFIX,
+)
+from backend.copilot.model import ChatMessage, ChatSession
+from backend.copilot.response_model import StreamToolOutputAvailable
+
+from .service import (
+ _classify_final_failure,
+ _FinalFailure,
+ _flush_orphan_tool_uses_to_session,
+ _HandledErrorInfo,
+ _InterruptedAttempt,
+)
+
+
+def _make_session(messages: list[ChatMessage] | None = None) -> ChatSession:
+ session = ChatSession.new(user_id="user-1", dry_run=False)
+ session.messages = list(messages or [])
+ return session
+
+
+def _tool_output(tool_call_id: str, output) -> StreamToolOutputAvailable:
+ return StreamToolOutputAvailable(
+ toolCallId=tool_call_id, toolName="t", output=output
+ )
+
+
+def _adapter_with_unresolved(responses: list[StreamToolOutputAvailable]):
+ """Stub _RetryState whose adapter flushes the given responses."""
+ adapter = MagicMock()
+ adapter.has_unresolved_tool_calls = bool(responses)
+
+ def _flush(out: list) -> None:
+ out.extend(responses)
+ adapter.has_unresolved_tool_calls = False
+
+ adapter.flush_unresolved_tool_calls.side_effect = _flush
+ state = MagicMock()
+ state.adapter = adapter
+ return state
+
+
+def _builder_stub() -> MagicMock:
+ builder = MagicMock()
+ builder.restore = MagicMock()
+ return builder
+
+
+class TestInterruptedAttemptCapture:
+ def test_keeps_partial_when_no_marker_present(self):
+ session = _make_session(
+ [
+ ChatMessage(role="user", content="hi"),
+ ChatMessage(role="assistant", content="part-1"),
+ ]
+ )
+ attempt = _InterruptedAttempt()
+ attempt.capture(session, _builder_stub(), object(), pre_attempt_msg_count=1)
+ assert [m.content for m in attempt.partial] == ["part-1"]
+ assert [m.content for m in session.messages] == ["hi"]
+
+ def test_strips_trailing_error_marker(self):
+ # _run_stream_attempt may append a marker (idle timeout, circuit
+ # breaker) before raising _HandledStreamError. Carrying it forward
+ # would let finalize() replay it and then add its own.
+ marker = (
+ f"{COPILOT_RETRYABLE_ERROR_PREFIX} The session has been idle "
+ "for too long. Please try again."
+ )
+ session = _make_session(
+ [
+ ChatMessage(role="user", content="hi"),
+ ChatMessage(role="assistant", content="part-1"),
+ ChatMessage(role="assistant", content=marker),
+ ]
+ )
+ attempt = _InterruptedAttempt()
+ attempt.capture(session, _builder_stub(), object(), pre_attempt_msg_count=1)
+ assert [m.content for m in attempt.partial] == ["part-1"]
+
+ def test_strips_consecutive_error_markers(self):
+ session = _make_session(
+ [
+ ChatMessage(role="user", content="hi"),
+ ChatMessage(role="assistant", content="part-1"),
+ ChatMessage(role="assistant", content=f"{COPILOT_ERROR_PREFIX} a"),
+ ChatMessage(
+ role="assistant", content=f"{COPILOT_RETRYABLE_ERROR_PREFIX} b"
+ ),
+ ]
+ )
+ attempt = _InterruptedAttempt()
+ attempt.capture(session, _builder_stub(), object(), pre_attempt_msg_count=1)
+ assert [m.content for m in attempt.partial] == ["part-1"]
+
+ def test_preserves_non_marker_assistant(self):
+ session = _make_session(
+ [
+ ChatMessage(role="user", content="hi"),
+ ChatMessage(role="assistant", content="Important note"),
+ ]
+ )
+ attempt = _InterruptedAttempt()
+ attempt.capture(session, _builder_stub(), object(), pre_attempt_msg_count=1)
+ assert [m.content for m in attempt.partial] == ["Important note"]
+
+
+class TestInterruptedAttemptFinalize:
+ def test_appends_partial_then_marker(self):
+ session = _make_session([ChatMessage(role="user", content="hi")])
+ attempt = _InterruptedAttempt(
+ partial=[
+ ChatMessage(role="assistant", content="working"),
+ ChatMessage(role="tool", content="result", tool_call_id="t1"),
+ ]
+ )
+ attempt.finalize(session, state=None, display_msg="Boom", retryable=False)
+ roles = [m.role for m in session.messages]
+ assert roles == ["user", "assistant", "tool", "assistant"]
+ assert session.messages[-1].content.startswith(COPILOT_ERROR_PREFIX)
+ # partial consumed so a follow-up finalize() is a no-op for partial.
+ assert attempt.partial == []
+
+ def test_only_marker_when_partial_empty(self):
+ session = _make_session([ChatMessage(role="user", content="hi")])
+ attempt = _InterruptedAttempt()
+ attempt.finalize(session, state=None, display_msg="Boom", retryable=True)
+ assert len(session.messages) == 2
+ assert session.messages[-1].content.startswith(COPILOT_RETRYABLE_ERROR_PREFIX)
+
+ def test_noop_when_session_is_none(self):
+ attempt = _InterruptedAttempt(
+ partial=[ChatMessage(role="assistant", content="x")]
+ )
+ events = attempt.finalize(None, state=None, display_msg="Boom", retryable=False)
+ assert events == []
+
+ def test_flushes_unresolved_tools_between_partial_and_marker(self):
+ session = _make_session([ChatMessage(role="user", content="hi")])
+ attempt = _InterruptedAttempt(
+ partial=[
+ ChatMessage(
+ role="assistant",
+ content="calling",
+ tool_calls=[
+ {
+ "id": "t1",
+ "type": "function",
+ "function": {"name": "lookup", "arguments": "{}"},
+ }
+ ],
+ ),
+ ]
+ )
+ flushed = [_tool_output("t1", "interrupted")]
+ state = _adapter_with_unresolved(flushed)
+ events = attempt.finalize(
+ session, state=state, display_msg="Boom", retryable=False
+ )
+ roles = [m.role for m in session.messages]
+ assert roles == ["user", "assistant", "tool", "assistant"]
+ assert session.messages[2].tool_call_id == "t1"
+ assert session.messages[2].content == "interrupted"
+ # The same events that were persisted to history are returned to the
+ # caller so the caller can yield them to the client — without this
+ # the frontend's spinner widgets stay open until refresh because the
+ # adapter's has_unresolved_tool_calls flag is already flipped to False.
+ assert events == flushed
+
+ def test_clear_drops_both_partial_and_handled_error(self):
+ attempt = _InterruptedAttempt(
+ partial=[ChatMessage(role="assistant", content="x")],
+ handled_error=_HandledErrorInfo(
+ error_msg="m", code="c", retryable=True, already_yielded=False
+ ),
+ )
+ attempt.clear()
+ assert attempt.partial == []
+ assert attempt.handled_error is None
+
+
+class TestFlushOrphanToolUses:
+ def test_appends_synthetic_tool_results_for_unresolved(self):
+ session = _make_session()
+ flushed = [_tool_output("t1", "r1"), _tool_output("t2", {"ok": False})]
+ state = _adapter_with_unresolved(flushed)
+ events = _flush_orphan_tool_uses_to_session(session, state)
+ assert [m.tool_call_id for m in session.messages] == ["t1", "t2"]
+ # Dict outputs are JSON-encoded so structure survives the str-only
+ # ChatMessage content field for the next-turn LLM read.
+ assert session.messages[1].content == '{"ok": false}'
+ assert events == flushed
+
+ def test_noop_when_state_is_none(self):
+ session = _make_session()
+ events = _flush_orphan_tool_uses_to_session(session, None)
+ assert session.messages == []
+ assert events == []
+
+ def test_noop_when_no_unresolved(self):
+ adapter = MagicMock()
+ adapter.has_unresolved_tool_calls = False
+ state = MagicMock()
+ state.adapter = adapter
+ events = _flush_orphan_tool_uses_to_session(_make_session(), state)
+ adapter.flush_unresolved_tool_calls.assert_not_called()
+ assert events == []
+
+
+class TestClassifyFinalFailure:
+ """Ensures the history marker (via finalize) and the SSE StreamError yield
+ share one source of truth for display message + stream code — any drift
+ would let the chat bubble and the SSE event show different copy for the
+ same failure."""
+
+ def test_handled_error_wins(self):
+ interrupted = _InterruptedAttempt(
+ handled_error=_HandledErrorInfo(
+ error_msg="circuit tripped",
+ code="circuit_breaker",
+ retryable=False,
+ already_yielded=True,
+ )
+ )
+ result = _classify_final_failure(
+ interrupted,
+ attempts_exhausted=False,
+ transient_exhausted=False,
+ stream_err=RuntimeError("ignored"),
+ )
+ assert result == _FinalFailure(
+ display_msg="circuit tripped",
+ code="circuit_breaker",
+ retryable=False,
+ )
+
+ def test_attempts_exhausted(self):
+ result = _classify_final_failure(
+ _InterruptedAttempt(),
+ attempts_exhausted=True,
+ transient_exhausted=False,
+ stream_err=RuntimeError("x"),
+ )
+ assert result is not None
+ assert result.code == "all_attempts_exhausted"
+ assert result.retryable is False
+
+ def test_transient_exhausted(self):
+ result = _classify_final_failure(
+ _InterruptedAttempt(),
+ attempts_exhausted=False,
+ transient_exhausted=True,
+ stream_err=RuntimeError("x"),
+ )
+ assert result is not None
+ assert result.code == "transient_api_error"
+ assert result.retryable is True
+
+ def test_stream_err_fallback(self):
+ result = _classify_final_failure(
+ _InterruptedAttempt(),
+ attempts_exhausted=False,
+ transient_exhausted=False,
+ stream_err=RuntimeError("some sdk error"),
+ )
+ assert result is not None
+ assert result.code == "sdk_stream_error"
+ assert result.retryable is False
+
+ def test_returns_none_when_no_failure_recorded(self):
+ assert (
+ _classify_final_failure(
+ _InterruptedAttempt(),
+ attempts_exhausted=False,
+ transient_exhausted=False,
+ stream_err=None,
+ )
+ is None
+ )
+
+
+class TestRetryRollbackContract:
+ """End-to-end contract: capture on a rolled-back attempt + finalize yields
+ the exact content the user saw streaming live, plus the error marker."""
+
+ def test_capture_then_finalize_matches_streamed_sequence(self):
+ session = _make_session([ChatMessage(role="user", content="hi")])
+ pre = len(session.messages)
+ # Simulate incremental SDK appends during the attempt.
+ session.messages.extend(
+ [
+ ChatMessage(role="assistant", content="part-1"),
+ ChatMessage(role="assistant", content="part-2"),
+ ]
+ )
+ attempt = _InterruptedAttempt()
+ attempt.capture(session, _builder_stub(), object(), pre)
+ # Final-failure path — no retry, no success clear().
+ attempt.finalize(session, state=None, display_msg="Boom", retryable=False)
+ assert [m.content for m in session.messages] == [
+ "hi",
+ "part-1",
+ "part-2",
+ f"{COPILOT_ERROR_PREFIX} Boom",
+ ]
diff --git a/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py b/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py
index 2a15e9f1fc..ab6a9e7510 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py
@@ -166,7 +166,7 @@ class SDKResponseAdapter:
# are still executing concurrently and haven't finished yet.
is_tool_only = all(isinstance(b, ToolUseBlock) for b in sdk_message.content)
if not is_tool_only:
- self._flush_unresolved_tool_calls(responses)
+ self.flush_unresolved_tool_calls(responses)
# After tool results, the SDK sends a new AssistantMessage for the
# next LLM turn. Open a new step if the previous one was closed.
@@ -375,7 +375,7 @@ class SDKResponseAdapter:
self.step_open = False
elif isinstance(sdk_message, ResultMessage):
- self._flush_unresolved_tool_calls(responses)
+ self.flush_unresolved_tool_calls(responses)
# Thinking-only final turn guard: when the model's last LLM
# call after a tool result produced only a ``ThinkingBlock``
# (no ``TextBlock``, no ``ToolUseBlock``) the UI has nothing
@@ -703,7 +703,7 @@ class SDKResponseAdapter:
self._pending_thinking_delta = ""
self._pending_thinking_index = None
- def _flush_unresolved_tool_calls(self, responses: list[StreamBaseResponse]) -> None:
+ def flush_unresolved_tool_calls(self, responses: list[StreamBaseResponse]) -> None:
"""Emit outputs for tool calls that didn't receive a UserMessage result.
SDK built-in tools (WebSearch, Read, etc.) may be executed by the CLI
@@ -711,6 +711,12 @@ class SDKResponseAdapter:
``ToolResultBlock`` content. The ``PostToolUse`` hook stashes their
output, which we pop and emit here before the next ``AssistantMessage``
starts.
+
+ Callers that need to both record synthetic tool_results in history AND
+ yield the same events to the client should call this exactly once and
+ share the resulting list — the method mutates ``resolved_tool_calls``,
+ so a second call returns nothing and ``has_unresolved_tool_calls``
+ flips to ``False`` after the first invocation.
"""
unresolved = [
(tid, info.get("name", "unknown"))
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 6e2fb12048..6ab2143825 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -133,7 +133,7 @@ from ..transcript import (
upload_transcript,
validate_transcript,
)
-from ..transcript_builder import TranscriptBuilder
+from ..transcript_builder import TranscriptBuilder, TranscriptSnapshot
from .compaction import CompactionTracker, filter_compaction_messages
from .env import build_sdk_env # noqa: F401 — re-export for backward compat
from .openrouter_cost import record_turn_cost_from_openrouter
@@ -541,14 +541,7 @@ def _append_error_marker(
*,
retryable: bool = False,
) -> None:
- """Append a copilot error marker to *session* so it persists across refresh.
-
- Args:
- session: The chat session to append to (no-op if `None`).
- display_msg: User-visible error text.
- retryable: If `True`, use the retryable prefix so the frontend
- shows a "Try Again" button.
- """
+ """Append a copilot error marker to *session* so it persists across refresh."""
if session is None:
return
prefix = COPILOT_RETRYABLE_ERROR_PREFIX if retryable else COPILOT_ERROR_PREFIX
@@ -557,6 +550,181 @@ def _append_error_marker(
)
+def _is_error_marker(msg: ChatMessage) -> bool:
+ """True if *msg* is an error marker emitted by ``_append_error_marker``."""
+ if msg.role != "assistant" or not msg.content:
+ return False
+ return msg.content.startswith(COPILOT_ERROR_PREFIX) or msg.content.startswith(
+ COPILOT_RETRYABLE_ERROR_PREFIX
+ )
+
+
+@dataclass
+class _InterruptedAttempt:
+ """Captured state of a failed SDK attempt, carried across the retry loop.
+
+ The SDK always rolls back ``session.messages`` before deciding whether
+ to retry (so attempt #2 starts clean). That rollback would otherwise
+ discard everything the assistant produced — the user sees tokens stream
+ live, then a refresh shows nothing. This dataclass holds the rolled-back
+ messages plus the ``_HandledStreamError`` info needed to emit a final
+ ``StreamError`` once the loop decides not to retry.
+
+ The retry loop calls ``capture()`` on every failed attempt, ``clear()``
+ on a successful retry (so prior rolled-back content is not replayed),
+ and ``finalize()`` exactly once after the loop on final failure.
+ """
+
+ partial: list[ChatMessage] = dataclass_field(default_factory=list)
+ # Populated by the ``except _HandledStreamError`` branch so the post-loop
+ # block can restore the partial and (when the inner handler didn't) emit
+ # the client-facing StreamError. Transient errors deliberately suppress
+ # the early StreamError flash and rely on this post-loop emit.
+ handled_error: "_HandledErrorInfo | None" = None
+
+ def capture(
+ self,
+ session: ChatSession,
+ transcript_builder: "TranscriptBuilder",
+ transcript_snap: TranscriptSnapshot,
+ pre_attempt_msg_count: int,
+ ) -> None:
+ """Roll back ``session.messages`` + transcript, keeping the partial.
+
+ Trailing error markers appended inside ``_run_stream_attempt`` (idle
+ timeout, circuit breaker) are stripped: re-attaching them would make
+ the post-loop restore replay a stale marker before adding its own,
+ leaving duplicate error bubbles.
+ """
+ tail = list(session.messages[pre_attempt_msg_count:])
+ while tail and _is_error_marker(tail[-1]):
+ tail.pop()
+ self.partial = tail
+ session.messages = session.messages[:pre_attempt_msg_count]
+ transcript_builder.restore(transcript_snap)
+
+ def clear(self) -> None:
+ """Drop captured state — used on successful retry."""
+ self.partial = []
+ self.handled_error = None
+
+ def finalize(
+ self,
+ session: ChatSession | None,
+ state: "_RetryState | None",
+ display_msg: str,
+ *,
+ retryable: bool,
+ ) -> list[StreamBaseResponse]:
+ """Re-attach partial + synthetic tool_result rows + error marker.
+
+ Called exactly once after the retry loop on final-failure exit.
+ Idempotent on empty state, so it's safe to call on paths where no
+ rollback happened.
+
+ Returns the ``StreamBaseResponse`` events produced by the safety
+ flush so the caller can yield them to the client (the flush mutates
+ adapter state, so a second flush elsewhere would return nothing and
+ stale UI elements like spinners would stay open).
+ """
+ if session is None:
+ return []
+ if self.partial:
+ session.messages.extend(self.partial)
+ self.partial = []
+ events = _flush_orphan_tool_uses_to_session(session, state)
+ _append_error_marker(session, display_msg, retryable=retryable)
+ return events
+
+
+def _flush_orphan_tool_uses_to_session(
+ session: "ChatSession | None",
+ state: "_RetryState | None",
+) -> list[StreamBaseResponse]:
+ """Synthesize ``tool_result`` rows for ``tool_use`` blocks that never resolved.
+
+ Re-attached partial work may carry orphan ``tool_use`` blocks; without
+ matching ``tool_result`` rows the next turn's LLM call would error with
+ ``tool_use_id without tool_result``. The adapter's safety-flush produces
+ interrupted-marker results that satisfy the API contract.
+
+ Returns the flushed events so callers can yield them to the client
+ alongside persisting the synthetic rows in session history.
+ """
+ if session is None or state is None:
+ return []
+ if not state.adapter.has_unresolved_tool_calls:
+ return []
+ safety: list[StreamBaseResponse] = []
+ state.adapter.flush_unresolved_tool_calls(safety)
+ for resp in safety:
+ if isinstance(resp, StreamToolOutputAvailable):
+ content = (
+ resp.output
+ if isinstance(resp.output, str)
+ else json.dumps(resp.output, ensure_ascii=False)
+ )
+ session.messages.append(
+ ChatMessage(role="tool", content=content, tool_call_id=resp.toolCallId)
+ )
+ return safety
+
+
+@dataclass(frozen=True)
+class _FinalFailure:
+ """Display message + stream code + retryable flag for a final-failure exit.
+
+ Shared by the in-history error marker (via ``_InterruptedAttempt.finalize``)
+ and the client-facing ``StreamError`` SSE yield so the two stay in sync.
+ """
+
+ display_msg: str
+ code: str
+ retryable: bool
+
+
+def _classify_final_failure(
+ interrupted: _InterruptedAttempt,
+ attempts_exhausted: bool,
+ transient_exhausted: bool,
+ stream_err: BaseException | None,
+) -> _FinalFailure | None:
+ """Pick the display message, stream code, and retryable flag for the exit.
+
+ Returns ``None`` when no failure was recorded (success path) — the caller
+ should skip both the history marker and the SSE yield in that case.
+ """
+ if interrupted.handled_error is not None:
+ return _FinalFailure(
+ display_msg=interrupted.handled_error.error_msg,
+ code=interrupted.handled_error.code,
+ retryable=interrupted.handled_error.retryable,
+ )
+ if attempts_exhausted:
+ return _FinalFailure(
+ display_msg=(
+ "Your conversation is too long. "
+ "Please start a new chat or clear some history."
+ ),
+ code="all_attempts_exhausted",
+ retryable=False,
+ )
+ if transient_exhausted:
+ return _FinalFailure(
+ display_msg=FRIENDLY_TRANSIENT_MSG,
+ code="transient_api_error",
+ retryable=True,
+ )
+ if stream_err is not None:
+ safe_err = str(stream_err).replace("\n", " ").replace("\r", "")[:500]
+ return _FinalFailure(
+ display_msg=_friendly_error_text(safe_err),
+ code="sdk_stream_error",
+ retryable=False,
+ )
+ return None
+
+
def _setup_langfuse_otel() -> None:
"""Configure OTEL tracing for the Claude Agent SDK → Langfuse.
@@ -2006,6 +2174,21 @@ class _HandledStreamError(Exception):
self.already_yielded = already_yielded
+@dataclass(frozen=True)
+class _HandledErrorInfo:
+ """Carries a `_HandledStreamError`'s decisions out of the retry loop.
+
+ Set inside the `except _HandledStreamError` branch and consumed by the
+ post-loop block, which restores the partial and (if the inner handler
+ didn't already do it) yields the client-facing StreamError.
+ """
+
+ error_msg: str
+ code: str
+ retryable: bool
+ already_yielded: bool
+
+
@dataclass
class _EmptyToolBreakResult:
"""Result of checking for empty tool calls in a single AssistantMessage."""
@@ -2725,7 +2908,7 @@ async def _run_stream_attempt(
- len(state.adapter.resolved_tool_calls),
)
safety_responses: list[StreamBaseResponse] = []
- state.adapter._flush_unresolved_tool_calls(safety_responses)
+ state.adapter.flush_unresolved_tool_calls(safety_responses)
for response in safety_responses:
if isinstance(
response,
@@ -2994,7 +3177,7 @@ async def _maybe_prepend_builder_context(
return block + query_message if block else query_message
-async def stream_chat_completion_sdk(
+async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues]
session_id: str,
message: str | None = None,
is_user_message: bool = True,
@@ -3007,6 +3190,13 @@ async def stream_chat_completion_sdk(
request_arrival_at: float = 0.0,
**_kwargs: Any,
) -> AsyncGenerator[StreamBaseResponse, None]:
+ # Pyright's complexity heuristic bails on this ~1500 LoC function (retry
+ # loop with context-overflow fallback + transient backoff + partial-work
+ # preservation). Splitting the retry loop further hurts readability —
+ # branches share mutable state (session, adapter, transcript builder,
+ # usage accumulators) that doesn't pass cleanly through helpers. The
+ # suppression only silences the complexity bailout; real type errors in
+ # the function body still surface.
"""Stream chat completion using Claude Agent SDK.
Args:
@@ -3170,6 +3360,11 @@ async def stream_chat_completion_sdk(
turn_cost_usd: float | None = None
graphiti_enabled = False
pre_attempt_msg_count = 0
+ # State of the latest failed attempt: rolled-back messages + any
+ # _HandledStreamError info to emit on final-failure exit. The retry loop
+ # mutates this via capture()/clear(); the post-loop block calls
+ # finalize() once.
+ interrupted = _InterruptedAttempt()
# Defaults ensure the finally block can always reference these safely even when
# an early return (e.g. sdk_cwd error) skips their normal assignment below.
sdk_model: str | None = None
@@ -3830,6 +4025,9 @@ async def stream_chat_completion_sdk(
"using fallback model for this request"
)
yield event
+ # Discard any state captured from prior failed attempts so
+ # outer cleanup paths don't replay pre-retry content.
+ interrupted.clear()
break # Stream completed — exit retry loop
except asyncio.CancelledError:
logger.warning(
@@ -3845,8 +4043,12 @@ async def stream_chat_completion_sdk(
# session messages and set the error flag — do NOT set
# stream_err so the post-loop code won't emit a
# duplicate StreamError.
- session.messages = session.messages[:pre_attempt_msg_count]
- state.transcript_builder.restore(transcript_snap)
+ interrupted.capture(
+ session,
+ state.transcript_builder,
+ transcript_snap,
+ pre_attempt_msg_count,
+ )
# Check if this is a transient error we can retry with backoff.
# exc.code is the only reliable signal — str(exc) is always the
# static "Stream error handled — StreamError already yielded" message.
@@ -3880,27 +4082,13 @@ async def stream_chat_completion_sdk(
# attempt that no longer match session.messages. Skip upload
# so a future --resume doesn't replay rolled-back content.
skip_transcript_upload = True
- # Re-append the error marker so it survives the rollback
- # and is persisted by the finally block (see #2947655365).
- # Use the specific error message from the attempt (e.g.
- # circuit breaker msg) rather than always the generic one.
- _append_error_marker(
- session,
- exc.error_msg or FRIENDLY_TRANSIENT_MSG,
- retryable=True,
+ interrupted.handled_error = _HandledErrorInfo(
+ error_msg=exc.error_msg or FRIENDLY_TRANSIENT_MSG,
+ code=exc.code or "transient_api_error",
+ retryable=exc.retryable,
+ already_yielded=exc.already_yielded,
)
ended_with_stream_error = True
- # For transient errors the StreamError was deliberately NOT
- # yielded inside _run_stream_attempt (already_yielded=False)
- # so the client didn't see a premature error flash. Yield it
- # now that we know retries are exhausted.
- # For non-transient errors (circuit breaker, idle timeout)
- # already_yielded=True — do NOT yield again.
- if not exc.already_yielded:
- yield StreamError(
- errorText=exc.error_msg or FRIENDLY_TRANSIENT_MSG,
- code=exc.code or "transient_api_error",
- )
break
except Exception as e:
stream_err = e
@@ -3918,8 +4106,12 @@ async def stream_chat_completion_sdk(
stream_err,
exc_info=True,
)
- session.messages = session.messages[:pre_attempt_msg_count]
- state.transcript_builder.restore(transcript_snap)
+ interrupted.capture(
+ session,
+ state.transcript_builder,
+ transcript_snap,
+ pre_attempt_msg_count,
+ )
if events_yielded > 0:
# Events were already sent to the frontend and cannot be
# unsent. Retrying would produce duplicate/inconsistent
@@ -3957,9 +4149,6 @@ async def stream_chat_completion_sdk(
# at line ~2310.
transient_exhausted = True
skip_transcript_upload = True
- _append_error_marker(
- session, FRIENDLY_TRANSIENT_MSG, retryable=True
- )
ended_with_stream_error = True
break
@@ -3983,43 +4172,38 @@ async def stream_chat_completion_sdk(
_MAX_STREAM_ATTEMPTS,
stream_err,
)
-
- if ended_with_stream_error and state is not None:
- # Flush any unresolved tool calls so the frontend can close
- # stale UI elements (e.g. spinners) that were started before
- # the exception interrupted the stream.
- error_flush: list[StreamBaseResponse] = []
- state.adapter._end_text_if_open(error_flush)
- if state.adapter.has_unresolved_tool_calls:
- logger.warning(
- "%s Flushing %d unresolved tool(s) after stream error",
- log_prefix,
- len(state.adapter.current_tool_calls)
- - len(state.adapter.resolved_tool_calls),
+ # Consolidated final-failure handling. _classify_final_failure picks
+ # the display message + stream code + retryable flag, finalize() adds
+ # the history marker and produces the safety-flush events that close
+ # stale UI widgets on the client, and the StreamError yield below
+ # surfaces the same message over SSE. The _HandledStreamError path
+ # sets ``already_yielded=True`` for non-transient errors (circuit
+ # breaker, idle timeout) whose inner handler already yielded — skip
+ # the re-yield in that case.
+ if ended_with_stream_error:
+ failure = _classify_final_failure(
+ interrupted, attempts_exhausted, transient_exhausted, stream_err
+ )
+ if failure is not None:
+ cleanup_events: list[StreamBaseResponse] = []
+ if state is not None:
+ state.adapter._end_text_if_open(cleanup_events)
+ cleanup_events.extend(
+ interrupted.finalize(
+ session,
+ state,
+ failure.display_msg,
+ retryable=failure.retryable,
+ )
)
- state.adapter._flush_unresolved_tool_calls(error_flush)
- for response in error_flush:
- yield response
-
- if ended_with_stream_error and stream_err is not None:
- # Use distinct error codes depending on how the loop ended:
- # • "all_attempts_exhausted" — context compaction ran out of room
- # • "transient_api_error" — 429/5xx/ECONNRESET retries exhausted
- # • "sdk_stream_error" — non-context, non-transient fatal error
- safe_err = str(stream_err).replace("\n", " ").replace("\r", "")[:500]
- if attempts_exhausted:
- error_text = (
- "Your conversation is too long. "
- "Please start a new chat or clear some history."
+ for response in cleanup_events:
+ yield response
+ already_yielded = (
+ interrupted.handled_error is not None
+ and interrupted.handled_error.already_yielded
)
- error_code = "all_attempts_exhausted"
- elif transient_exhausted:
- error_text = FRIENDLY_TRANSIENT_MSG
- error_code = "transient_api_error"
- else:
- error_text = _friendly_error_text(safe_err)
- error_code = "sdk_stream_error"
- yield StreamError(errorText=error_text, code=error_code)
+ if not already_yielded:
+ yield StreamError(errorText=failure.display_msg, code=failure.code)
# Copy token usage from retry state to outer-scope accumulators
# so the finally block can persist them.
@@ -4101,12 +4285,13 @@ async def stream_chat_completion_sdk(
else:
display_msg, code = error_msg, "sdk_error"
- # Append error marker to session (non-invasive text parsing approach).
- # The finally block will persist the session with this error marker.
- # Skip if a marker was already appended inside the stream loop
- # (ended_with_stream_error) to avoid duplicate stale markers.
+ # Append error marker + restore any rolled-back partial when the retry
+ # loop didn't already finalize. ``interrupted`` is empty on success and
+ # on paths where the retry loop's own post-loop finalize() already ran,
+ # so this is a no-op for those and only kicks in for unhandled errors
+ # that bypass the retry-loop handlers entirely.
if not ended_with_stream_error:
- _append_error_marker(session, display_msg, retryable=is_transient)
+ interrupted.finalize(session, state, display_msg, retryable=is_transient)
logger.debug(
"%s Appended error marker, will be persisted in finally",
log_prefix,
diff --git a/autogpt_platform/backend/backend/copilot/transcript_builder.py b/autogpt_platform/backend/backend/copilot/transcript_builder.py
index 35abf639b1..ab25a92aeb 100644
--- a/autogpt_platform/backend/backend/copilot/transcript_builder.py
+++ b/autogpt_platform/backend/backend/copilot/transcript_builder.py
@@ -34,6 +34,9 @@ class TranscriptEntry(BaseModel):
message: dict[str, Any]
+TranscriptSnapshot = tuple[list[TranscriptEntry], str | None]
+
+
class TranscriptBuilder:
"""Build complete JSONL transcript from SDK messages.
@@ -224,7 +227,7 @@ class TranscriptBuilder:
lines = [entry.model_dump_json(exclude_none=True) for entry in self._entries]
return "\n".join(lines) + "\n"
- def snapshot(self) -> tuple[list[TranscriptEntry], str | None]:
+ def snapshot(self) -> TranscriptSnapshot:
"""Return a shallow snapshot of the current builder state.
Use with :meth:`restore` to roll back transcript mutations from a
@@ -235,7 +238,7 @@ class TranscriptBuilder:
"""
return list(self._entries), self._last_uuid
- def restore(self, snap: tuple[list[TranscriptEntry], str | None]) -> None:
+ def restore(self, snap: TranscriptSnapshot) -> None:
"""Restore builder state from a :meth:`snapshot`.
Replaces ``_entries`` and ``_last_uuid`` atomically so the builder
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/useCopilotStream.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/useCopilotStream.test.ts
index e56317bf04..f646a5b29e 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/useCopilotStream.test.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/useCopilotStream.test.ts
@@ -2,6 +2,7 @@ import { act, renderHook } from "@testing-library/react";
import type { UIMessage } from "ai";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { useCopilotUIStore } from "../store";
import { useCopilotStream } from "../useCopilotStream";
// Capture the args passed to ``useChat`` so tests can invoke onFinish/onError
@@ -67,6 +68,7 @@ vi.mock("@/components/molecules/Toast/use-toast", () => ({
vi.mock("@/services/environment", () => ({
environment: {
getAGPTServerBaseUrl: () => "http://localhost",
+ isServerSide: () => false,
},
}));
@@ -147,6 +149,80 @@ describe("useCopilotStream — reconnect debounce", () => {
expect(resumeStreamMock).toHaveBeenCalledTimes(2);
});
+ it("restores the unsent text and drops the optimistic user bubble on 429 usage-limit", async () => {
+ useCopilotUIStore.getState().setInitialPrompt(null);
+ const { result } = renderStream();
+
+ await act(async () => {
+ await result.current.sendMessage({ text: "recover me" });
+ });
+ expect(sdkSendMessageMock).toHaveBeenCalledTimes(1);
+
+ await act(async () => {
+ lastUseChatArgs!.onError!(
+ new Error(JSON.stringify({ detail: "Daily usage limit exceeded" })),
+ );
+ });
+
+ expect(useCopilotUIStore.getState().initialPrompt).toBe("recover me");
+ expect(result.current.rateLimitMessage).toContain("Daily usage limit");
+
+ const dropCall = setMessagesMock.mock.calls.find(([arg]) => {
+ if (typeof arg !== "function") return false;
+ const prev: UIMessage[] = [
+ {
+ id: "u1",
+ role: "user",
+ parts: [{ type: "text", text: "recover me" }],
+ } as unknown as UIMessage,
+ ];
+ const next = (arg as (p: UIMessage[]) => UIMessage[])(prev);
+ return next.length === 0;
+ });
+ expect(dropCall).toBeTruthy();
+
+ // Branch: when the trailing message is NOT a user bubble (e.g. an
+ // assistant turn already landed), the rollback updater must leave the
+ // list untouched so we don't clobber the assistant reply.
+ const updater = setMessagesMock.mock.calls
+ .map(([arg]) => arg)
+ .find(
+ (arg): arg is (p: UIMessage[]) => UIMessage[] =>
+ typeof arg === "function",
+ );
+ expect(updater).toBeDefined();
+ const assistantOnly: UIMessage[] = [
+ {
+ id: "a1",
+ role: "assistant",
+ parts: [{ type: "text", text: "ok" }],
+ } as unknown as UIMessage,
+ ];
+ expect(updater!(assistantOnly)).toBe(assistantOnly);
+ expect(updater!([])).toEqual([]);
+ });
+
+ it("skips composer restore on 429 when there is no captured unsent text", async () => {
+ useCopilotUIStore.getState().setInitialPrompt(null);
+ renderStream();
+
+ await act(async () => {
+ lastUseChatArgs!.onError!(
+ new Error(JSON.stringify({ detail: "Daily usage limit exceeded" })),
+ );
+ });
+
+ // No sendMessage was called → lastSubmittedMsgRef stays null → restore
+ // branch must be skipped. The store stays untouched and setMessages is
+ // NOT invoked with a rollback updater.
+ expect(useCopilotUIStore.getState().initialPrompt).toBeNull();
+ expect(sdkSendMessageMock).not.toHaveBeenCalled();
+ const rollbackUpdaterCall = setMessagesMock.mock.calls.find(
+ ([arg]) => typeof arg === "function",
+ );
+ expect(rollbackUpdaterCall).toBeUndefined();
+ });
+
it("does not debounce a reconnect that arrives after the window closes", async () => {
renderStream();
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotStream.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotStream.ts
index afef20c85a..d217a7789e 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotStream.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/useCopilotStream.ts
@@ -20,6 +20,7 @@ import {
disconnectSessionStream,
shouldDebounceReconnect,
} from "./helpers";
+import { useCopilotUIStore } from "./store";
import type { CopilotLlmModel, CopilotMode } from "./store";
import { useHydrateOnStreamEnd } from "./useHydrateOnStreamEnd";
@@ -62,6 +63,7 @@ export function useCopilotStream({
copilotModel,
}: UseCopilotStreamArgs) {
const queryClient = useQueryClient();
+ const setInitialPrompt = useCopilotUIStore((s) => s.setInitialPrompt);
const [rateLimitMessage, setRateLimitMessage] = useState<string | null>(null);
function dismissRateLimit() {
setRateLimitMessage(null);
@@ -269,6 +271,20 @@ export function useCopilotStream({
}
const isRateLimited = errorDetail.toLowerCase().includes("usage limit");
if (isRateLimited) {
+ // Backend raises 429 BEFORE persisting the user message, so the
+ // optimistic user bubble added by useChat is a lie. Restore the text
+ // into the composer (via the same store slot URL pre-fills use) and
+ // drop the unsent bubble so the user can edit/resend after reset.
+ const unsentText = lastSubmittedMsgRef.current;
+ if (unsentText) {
+ setInitialPrompt(unsentText);
+ lastSubmittedMsgRef.current = null;
+ setMessages((prev) => {
+ const last = prev[prev.length - 1];
+ if (last?.role === "user") return prev.slice(0, -1);
+ return prev;
+ });
+ }
setRateLimitMessage(
errorDetail ||
"You've reached your usage limit. Please try again later.",