Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(copilot): P0 guardrails, transient retry, and security hardening (#12636)

Zamil Majdyd113687878c4
diff --git a/autogpt_platform/backend/backend/copilot/config.py b/autogpt_platform/backend/backend/copilot/config.py
index 2db5c2f03f..6da1cae52b 100644
--- a/autogpt_platform/backend/backend/copilot/config.py
+++ b/autogpt_platform/backend/backend/copilot/config.py
@@ -146,6 +146,32 @@ class ChatConfig(BaseSettings):
description="Use --resume for multi-turn conversations instead of "
"history compression. Falls back to compression when unavailable.",
)
+ claude_agent_fallback_model: str = Field(
+ default="claude-sonnet-4-20250514",
+ description="Fallback model when the primary model is unavailable (e.g. 529 "
+ "overloaded). The SDK automatically retries with this cheaper model.",
+ )
+ claude_agent_max_turns: int = Field(
+ default=1000,
+ ge=1,
+ le=10000,
+ description="Maximum number of agentic turns (tool-use loops) per query. "
+ "Prevents runaway tool loops from burning budget.",
+ )
+ claude_agent_max_budget_usd: float = Field(
+ default=100.0,
+ ge=0.01,
+ le=1000.0,
+ description="Maximum spend in USD per SDK query. The CLI aborts the "
+ "request if this budget is exceeded.",
+ )
+ claude_agent_max_transient_retries: int = Field(
+ default=3,
+ ge=0,
+ le=10,
+ description="Maximum number of retries for transient API errors "
+ "(429, 5xx, ECONNRESET) before surfacing the error to the user.",
+ )
use_openrouter: bool = Field(
default=True,
description="Enable routing API calls through the OpenRouter proxy. "
diff --git a/autogpt_platform/backend/backend/copilot/constants.py b/autogpt_platform/backend/backend/copilot/constants.py
index 2270a319e4..aa6d1e2d65 100644
--- a/autogpt_platform/backend/backend/copilot/constants.py
+++ b/autogpt_platform/backend/backend/copilot/constants.py
@@ -44,15 +44,36 @@ def parse_node_id_from_exec_id(node_exec_id: str) -> str:
# Transient Anthropic API error detection
# ---------------------------------------------------------------------------
# Patterns in error text that indicate a transient Anthropic API error
-# (ECONNRESET / dropped TCP connection) which is retryable.
+# which is retryable. Covers:
+# - Connection-level: ECONNRESET, dropped TCP connections
+# - HTTP 429: rate-limit / too-many-requests
+# - HTTP 5xx: server errors
+#
+# Prefer specific status-code patterns over natural-language phrases
+# (e.g. "overloaded", "bad gateway") — those phrases can appear in
+# application-level SDK messages and would trigger spurious retries.
_TRANSIENT_ERROR_PATTERNS = (
+ # Connection-level
"socket connection was closed unexpectedly",
"ECONNRESET",
"connection was forcibly closed",
"network socket disconnected",
+ # 429 rate-limit patterns
+ "rate limit",
+ "rate_limit",
+ "too many requests",
+ "status code 429",
+ # 5xx server error patterns (status-code-specific to avoid false positives)
+ "status code 529",
+ "status code 500",
+ "status code 502",
+ "status code 503",
+ "status code 504",
)
-FRIENDLY_TRANSIENT_MSG = "Anthropic connection interrupted — please retry"
+FRIENDLY_TRANSIENT_MSG = (
+ "Anthropic connection interrupted after repeated attempts — please try again later"
+)
def is_transient_api_error(error_text: str) -> bool:
diff --git a/autogpt_platform/backend/backend/copilot/sdk/env.py b/autogpt_platform/backend/backend/copilot/sdk/env.py
index f5fd63e47c..27470c9d05 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/env.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/env.py
@@ -8,6 +8,8 @@ circular import through ``executor`` → ``credit`` → ``block_cost_config``).
from __future__ import annotations
+import re
+
from backend.copilot.config import ChatConfig
from backend.copilot.sdk.subscription import validate_subscription
@@ -16,6 +18,10 @@ from backend.copilot.sdk.subscription import validate_subscription
# this module was created to avoid.
config = ChatConfig()
+# RFC 7230 §3.2.6 — keep only printable ASCII; strip control chars and non-ASCII.
+_HEADER_SAFE_RE = re.compile(r"[^\x20-\x7e]")
+_MAX_HEADER_VALUE_LEN = 128
+
def build_sdk_env(
session_id: str | None = None,
@@ -26,14 +32,14 @@ def build_sdk_env(
Three modes (checked in order):
1. **Subscription** — clears all keys; CLI uses ``claude login`` auth.
- 2. **Direct Anthropic** — returns ``{}``; subprocess inherits
- ``ANTHROPIC_API_KEY`` from the parent environment.
+ 2. **Direct Anthropic** — subprocess inherits ``ANTHROPIC_API_KEY``
+ from the parent environment (no overrides needed).
3. **OpenRouter** (default) — overrides base URL and auth token to
route through the proxy, with Langfuse trace headers.
- When *sdk_cwd* is provided, ``CLAUDE_CODE_TMPDIR`` is set so that
- the CLI writes temp/sub-agent output inside the per-session workspace
- directory rather than an inaccessible system temp path.
+ All modes receive workspace isolation (``CLAUDE_CODE_TMPDIR``) and
+ security hardening env vars to prevent .claude.md loading, prompt
+ history persistence, auto-memory writes, and non-essential traffic.
"""
# --- Mode 1: Claude Code subscription auth ---
if config.use_claude_code_subscription:
@@ -43,40 +49,52 @@ def build_sdk_env(
"ANTHROPIC_AUTH_TOKEN": "",
"ANTHROPIC_BASE_URL": "",
}
- if sdk_cwd:
- env["CLAUDE_CODE_TMPDIR"] = sdk_cwd
- return env
# --- Mode 2: Direct Anthropic (no proxy hop) ---
- if not config.openrouter_active:
- env = {}
- if sdk_cwd:
- env["CLAUDE_CODE_TMPDIR"] = sdk_cwd
- return env
+ elif not config.openrouter_active:
+ # Clear OAuth tokens so CLI uses ANTHROPIC_API_KEY from parent env
+ # rather than subscription auth if the container has those tokens set.
+ env = {
+ "CLAUDE_CODE_OAUTH_TOKEN": "",
+ "CLAUDE_CODE_REFRESH_TOKEN": "",
+ }
# --- Mode 3: OpenRouter proxy ---
- base = (config.base_url or "").rstrip("/")
- if base.endswith("/v1"):
- base = base[:-3]
- env = {
- "ANTHROPIC_BASE_URL": base,
- "ANTHROPIC_AUTH_TOKEN": config.api_key or "",
- "ANTHROPIC_API_KEY": "", # force CLI to use AUTH_TOKEN
- }
-
- # Inject broadcast headers so OpenRouter forwards traces to Langfuse.
- def _safe(v: str) -> str:
- return v.replace("\r", "").replace("\n", "").strip()[:128]
-
- parts = []
- if session_id:
- parts.append(f"x-session-id: {_safe(session_id)}")
- if user_id:
- parts.append(f"x-user-id: {_safe(user_id)}")
- if parts:
- env["ANTHROPIC_CUSTOM_HEADERS"] = "\n".join(parts)
+ else:
+ base = (config.base_url or "").rstrip("/")
+ if base.endswith("/v1"):
+ base = base[:-3]
+ env = {
+ "ANTHROPIC_BASE_URL": base,
+ "ANTHROPIC_AUTH_TOKEN": config.api_key or "",
+ "ANTHROPIC_API_KEY": "", # force CLI to use AUTH_TOKEN
+ "CLAUDE_CODE_OAUTH_TOKEN": "", # prevent OAuth override of ANTHROPIC_AUTH_TOKEN
+ "CLAUDE_CODE_REFRESH_TOKEN": "", # prevent token refresh via subscription
+ }
+ # Inject broadcast headers so OpenRouter forwards traces to Langfuse.
+ def _safe(v: str) -> str:
+ return _HEADER_SAFE_RE.sub("", v).strip()[:_MAX_HEADER_VALUE_LEN]
+
+ parts = []
+ if session_id:
+ parts.append(f"x-session-id: {_safe(session_id)}")
+ if user_id:
+ parts.append(f"x-user-id: {_safe(user_id)}")
+ if parts:
+ env["ANTHROPIC_CUSTOM_HEADERS"] = "\n".join(parts)
+
+ # --- Common: workspace isolation + security hardening (all modes) ---
+ # Route subagent temp files into the per-session workspace so output
+ # files are accessible (fixes /tmp/claude-0/ permission errors in E2B).
if sdk_cwd:
env["CLAUDE_CODE_TMPDIR"] = sdk_cwd
+ # Harden multi-tenant deployment: prevent loading untrusted workspace
+ # .claude.md files, writing auto-memory, and sending non-essential
+ # telemetry traffic.
+ env["CLAUDE_CODE_DISABLE_CLAUDE_MDS"] = "1"
+ env["CLAUDE_CODE_DISABLE_AUTO_MEMORY"] = "1"
+ env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1"
+
return env
diff --git a/autogpt_platform/backend/backend/copilot/sdk/env_test.py b/autogpt_platform/backend/backend/copilot/sdk/env_test.py
index 439e8d3930..e387499816 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/env_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/env_test.py
@@ -41,11 +41,9 @@ class TestBuildSdkEnvSubscription:
result = build_sdk_env()
- assert result == {
- "ANTHROPIC_API_KEY": "",
- "ANTHROPIC_AUTH_TOKEN": "",
- "ANTHROPIC_BASE_URL": "",
- }
+ assert result["ANTHROPIC_API_KEY"] == ""
+ assert result["ANTHROPIC_AUTH_TOKEN"] == ""
+ assert result["ANTHROPIC_BASE_URL"] == ""
mock_validate.assert_called_once()
@patch(
@@ -68,18 +66,20 @@ class TestBuildSdkEnvSubscription:
class TestBuildSdkEnvDirectAnthropic:
- """When OpenRouter is inactive, return empty dict (inherit parent env)."""
+ """When OpenRouter is inactive, no ANTHROPIC_* overrides (inherit parent env)."""
- def test_returns_empty_dict_when_openrouter_inactive(self):
+ def test_no_anthropic_key_overrides_when_openrouter_inactive(self):
cfg = _make_config(use_openrouter=False)
with patch("backend.copilot.sdk.env.config", cfg):
from backend.copilot.sdk.env import build_sdk_env
result = build_sdk_env()
- assert result == {}
+ assert "ANTHROPIC_API_KEY" not in result
+ assert "ANTHROPIC_AUTH_TOKEN" not in result
+ assert "ANTHROPIC_BASE_URL" not in result
- def test_returns_empty_dict_when_openrouter_flag_true_but_no_key(self):
+ def test_no_anthropic_key_overrides_when_openrouter_flag_true_but_no_key(self):
"""OpenRouter flag is True but no api_key => openrouter_active is False."""
cfg = _make_config(use_openrouter=True, base_url="https://openrouter.ai/api/v1")
# Force api_key to None after construction (field_validator may pick up env vars)
@@ -90,7 +90,9 @@ class TestBuildSdkEnvDirectAnthropic:
result = build_sdk_env()
- assert result == {}
+ assert "ANTHROPIC_API_KEY" not in result
+ assert "ANTHROPIC_AUTH_TOKEN" not in result
+ assert "ANTHROPIC_BASE_URL" not in result
# ---------------------------------------------------------------------------
@@ -212,6 +214,33 @@ class TestBuildSdkEnvOpenRouter:
value = header_line.split(": ", 1)[1]
assert len(value) == 128
+ @pytest.mark.parametrize(
+ ("bad_input", "expected_ascii"),
+ [
+ ("user\x00id", "userid"), # null byte
+ ("user\x7fid", "userid"), # DEL
+ ("user\x80id", "userid"), # first C1 control char
+ ("user\x9fid", "userid"), # last C1 control char
+ ("user\U0001f600id", "userid"), # emoji (non-ASCII Unicode)
+ ("user\u202eid", "userid"), # RTL override (security-relevant)
+ ],
+ )
+ def test_header_sanitizer_strips_non_printable_ascii(
+ self, bad_input: str, expected_ascii: str
+ ):
+ """_safe() strips everything outside printable ASCII 0x20–0x7e."""
+ cfg = self._openrouter_config()
+ with patch("backend.copilot.sdk.env.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ result = build_sdk_env(session_id=bad_input)
+
+ value = result["ANTHROPIC_CUSTOM_HEADERS"].split(": ", 1)[1]
+ assert expected_ascii in value
+ for char in bad_input:
+ if ord(char) < 0x20 or ord(char) > 0x7E:
+ assert char not in value
+
# ---------------------------------------------------------------------------
# Mode priority
@@ -234,12 +263,12 @@ class TestBuildSdkEnvModePriority:
result = build_sdk_env()
- # Should get subscription result, not OpenRouter
- assert result == {
- "ANTHROPIC_API_KEY": "",
- "ANTHROPIC_AUTH_TOKEN": "",
- "ANTHROPIC_BASE_URL": "",
- }
+ # Should get subscription result (blanked keys), not OpenRouter proxy
+ assert result["ANTHROPIC_API_KEY"] == ""
+ assert result["ANTHROPIC_AUTH_TOKEN"] == ""
+ assert result["ANTHROPIC_BASE_URL"] == ""
+ # OpenRouter-specific key must NOT be present
+ assert "ANTHROPIC_CUSTOM_HEADERS" not in result
# ---------------------------------------------------------------------------
diff --git a/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py b/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py
new file mode 100644
index 0000000000..613ccb2a09
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py
@@ -0,0 +1,1502 @@
+"""Tests for P0 guardrails: _resolve_fallback_model, security env vars, TMPDIR."""
+
+from unittest.mock import patch
+
+import pytest
+from pydantic import ValidationError
+
+from backend.copilot.config import ChatConfig
+from backend.copilot.constants import is_transient_api_error
+
+
+def _make_config(**overrides) -> ChatConfig:
+ """Create a ChatConfig with safe defaults, applying *overrides*."""
+ defaults = {
+ "use_claude_code_subscription": False,
+ "use_openrouter": False,
+ "api_key": None,
+ "base_url": None,
+ }
+ defaults.update(overrides)
+ return ChatConfig(**defaults)
+
+
+# ---------------------------------------------------------------------------
+# _resolve_fallback_model
+# ---------------------------------------------------------------------------
+
+_SVC = "backend.copilot.sdk.service"
+_ENV = "backend.copilot.sdk.env"
+
+
+class TestResolveFallbackModel:
+ """Provider-aware fallback model resolution."""
+
+ def test_returns_none_when_empty(self):
+ cfg = _make_config(claude_agent_fallback_model="")
+ with patch(f"{_SVC}.config", cfg):
+ from backend.copilot.sdk.service import _resolve_fallback_model
+
+ assert _resolve_fallback_model() is None
+
+ def test_strips_provider_prefix(self):
+ """OpenRouter-style 'anthropic/claude-sonnet-4-...' is stripped."""
+ cfg = _make_config(
+ claude_agent_fallback_model="anthropic/claude-sonnet-4-20250514",
+ use_openrouter=True,
+ api_key="sk-test",
+ base_url="https://openrouter.ai/api/v1",
+ )
+ with patch(f"{_SVC}.config", cfg):
+ from backend.copilot.sdk.service import _resolve_fallback_model
+
+ result = _resolve_fallback_model()
+
+ assert result == "claude-sonnet-4-20250514"
+ assert "/" not in result
+
+ def test_dots_replaced_for_direct_anthropic(self):
+ """Direct Anthropic requires hyphen-separated versions."""
+ cfg = _make_config(
+ claude_agent_fallback_model="claude-sonnet-4.5-20250514",
+ use_openrouter=False,
+ )
+ with patch(f"{_SVC}.config", cfg):
+ from backend.copilot.sdk.service import _resolve_fallback_model
+
+ result = _resolve_fallback_model()
+
+ assert result is not None
+ assert "." not in result
+ assert result == "claude-sonnet-4-5-20250514"
+
+ def test_dots_preserved_for_openrouter(self):
+ """OpenRouter uses dot-separated versions — don't normalise."""
+ cfg = _make_config(
+ claude_agent_fallback_model="claude-sonnet-4.5-20250514",
+ use_openrouter=True,
+ api_key="sk-test",
+ base_url="https://openrouter.ai/api/v1",
+ )
+ with patch(f"{_SVC}.config", cfg):
+ from backend.copilot.sdk.service import _resolve_fallback_model
+
+ result = _resolve_fallback_model()
+
+ assert result == "claude-sonnet-4.5-20250514"
+
+ def test_default_value(self):
+ """Default fallback model resolves to a valid string."""
+ cfg = _make_config()
+ with patch(f"{_SVC}.config", cfg):
+ from backend.copilot.sdk.service import _resolve_fallback_model
+
+ result = _resolve_fallback_model()
+
+ assert result is not None
+ assert "sonnet" in result.lower() or "claude" in result.lower()
+
+
+# ---------------------------------------------------------------------------
+# Security & isolation env vars
+# ---------------------------------------------------------------------------
+
+
+_SECURITY_VARS = (
+ "CLAUDE_CODE_DISABLE_CLAUDE_MDS",
+ "CLAUDE_CODE_DISABLE_AUTO_MEMORY",
+ "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC",
+)
+
+
+class TestSecurityEnvVars:
+ """Verify security env vars are set in the returned dict for every auth mode.
+
+ Tests call ``build_sdk_env()`` directly and assert the vars are present
+ in the returned dict — not just present somewhere in the source file.
+ """
+
+ def test_security_vars_set_in_openrouter_mode(self):
+ """Mode 3 (OpenRouter): security vars must be in the returned env."""
+ cfg = _make_config(
+ use_claude_code_subscription=False,
+ use_openrouter=True,
+ api_key="sk-or-test",
+ base_url="https://openrouter.ai/api/v1",
+ )
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env(session_id="s1", user_id="u1")
+
+ for var in _SECURITY_VARS:
+ assert env.get(var) == "1", f"{var} not set in OpenRouter mode"
+
+ def test_security_vars_set_in_direct_anthropic_mode(self):
+ """Mode 2 (direct Anthropic): security vars must be in the returned env."""
+ cfg = _make_config(use_claude_code_subscription=False, use_openrouter=False)
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env()
+
+ for var in _SECURITY_VARS:
+ assert env.get(var) == "1", f"{var} not set in direct Anthropic mode"
+
+ def test_security_vars_set_in_subscription_mode(self):
+ """Mode 1 (subscription): security vars must be in the returned env."""
+ cfg = _make_config(use_claude_code_subscription=True)
+ with (
+ patch(f"{_ENV}.config", cfg),
+ patch(f"{_ENV}.validate_subscription"),
+ ):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env(session_id="s1", user_id="u1")
+
+ for var in _SECURITY_VARS:
+ assert env.get(var) == "1", f"{var} not set in subscription mode"
+
+ def test_tmpdir_set_when_sdk_cwd_provided(self):
+ """CLAUDE_CODE_TMPDIR must be set when sdk_cwd is provided."""
+ cfg = _make_config(use_openrouter=False)
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env(sdk_cwd="/workspace/session-1")
+
+ assert env.get("CLAUDE_CODE_TMPDIR") == "/workspace/session-1"
+
+ def test_tmpdir_absent_when_sdk_cwd_not_provided(self):
+ """CLAUDE_CODE_TMPDIR must NOT be set when sdk_cwd is None."""
+ cfg = _make_config(use_openrouter=False)
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env()
+
+ assert "CLAUDE_CODE_TMPDIR" not in env
+
+ def test_home_not_overridden(self):
+ """HOME must NOT be overridden — would break git/ssh/npm in subprocesses."""
+ cfg = _make_config(use_openrouter=False)
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env()
+
+ assert "HOME" not in env
+
+
+# ---------------------------------------------------------------------------
+# Config defaults
+# ---------------------------------------------------------------------------
+
+
+class TestConfigDefaults:
+ """Verify ChatConfig P0 fields have correct defaults."""
+
+ def test_fallback_model_default(self):
+ cfg = _make_config()
+ assert cfg.claude_agent_fallback_model
+ assert "sonnet" in cfg.claude_agent_fallback_model.lower()
+
+ def test_max_turns_default(self):
+ cfg = _make_config()
+ assert cfg.claude_agent_max_turns == 1000
+
+ def test_max_budget_usd_default(self):
+ cfg = _make_config()
+ assert cfg.claude_agent_max_budget_usd == 100.0
+
+ def test_max_transient_retries_default(self):
+ cfg = _make_config()
+ assert cfg.claude_agent_max_transient_retries == 3
+
+
+# ---------------------------------------------------------------------------
+# build_sdk_env — all 3 auth modes
+# ---------------------------------------------------------------------------
+
+
+class TestBuildSdkEnv:
+ """Verify build_sdk_env returns correct dicts for each auth mode."""
+
+ def test_subscription_mode_clears_keys(self):
+ """Mode 1: subscription clears API key / auth token / base URL."""
+ cfg = _make_config(use_claude_code_subscription=True)
+ with (
+ patch(f"{_ENV}.config", cfg),
+ patch(f"{_ENV}.validate_subscription"),
+ ):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env(session_id="s1", user_id="u1")
+
+ assert env["ANTHROPIC_API_KEY"] == ""
+ assert env["ANTHROPIC_AUTH_TOKEN"] == ""
+ assert env["ANTHROPIC_BASE_URL"] == ""
+
+ def test_direct_anthropic_inherits_api_key(self):
+ """Mode 2: direct Anthropic doesn't set ANTHROPIC_* keys (inherits from parent)."""
+ cfg = _make_config(
+ use_claude_code_subscription=False,
+ use_openrouter=False,
+ )
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env()
+
+ assert "ANTHROPIC_API_KEY" not in env
+ assert "ANTHROPIC_AUTH_TOKEN" not in env
+ assert "ANTHROPIC_BASE_URL" not in env
+
+ def test_openrouter_sets_base_url_and_auth(self):
+ """Mode 3: OpenRouter sets base URL, auth token, and clears API key."""
+ cfg = _make_config(
+ use_claude_code_subscription=False,
+ use_openrouter=True,
+ api_key="sk-or-test",
+ base_url="https://openrouter.ai/api/v1",
+ )
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env(session_id="sess-1", user_id="user-1")
+
+ assert env["ANTHROPIC_BASE_URL"] == "https://openrouter.ai/api"
+ assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-or-test"
+ assert env["ANTHROPIC_API_KEY"] == ""
+ assert "x-session-id: sess-1" in env["ANTHROPIC_CUSTOM_HEADERS"]
+ assert "x-user-id: user-1" in env["ANTHROPIC_CUSTOM_HEADERS"]
+
+ def test_openrouter_no_headers_when_ids_empty(self):
+ """Mode 3: No custom headers when session_id/user_id are not given."""
+ cfg = _make_config(
+ use_claude_code_subscription=False,
+ use_openrouter=True,
+ api_key="sk-or-test",
+ base_url="https://openrouter.ai/api/v1",
+ )
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env()
+
+ assert "ANTHROPIC_CUSTOM_HEADERS" not in env
+
+ def test_openrouter_clears_oauth_tokens(self):
+ """Mode 3: OAuth tokens are explicitly cleared to prevent CLI preferring subscription auth."""
+ cfg = _make_config(
+ use_claude_code_subscription=False,
+ use_openrouter=True,
+ api_key="sk-or-test",
+ base_url="https://openrouter.ai/api/v1",
+ )
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env()
+
+ assert env["CLAUDE_CODE_OAUTH_TOKEN"] == ""
+ assert env["CLAUDE_CODE_REFRESH_TOKEN"] == ""
+
+ def test_direct_anthropic_clears_oauth_tokens(self):
+ """Mode 2: OAuth tokens are cleared so CLI uses ANTHROPIC_API_KEY from parent env."""
+ cfg = _make_config(
+ use_claude_code_subscription=False,
+ use_openrouter=False,
+ )
+ with patch(f"{_ENV}.config", cfg):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env()
+
+ assert env["CLAUDE_CODE_OAUTH_TOKEN"] == ""
+ assert env["CLAUDE_CODE_REFRESH_TOKEN"] == ""
+
+ def test_all_modes_return_mutable_dict(self):
+ """build_sdk_env must return a mutable dict (not None) in every mode."""
+ for cfg in (
+ _make_config(use_claude_code_subscription=True),
+ _make_config(use_openrouter=False),
+ _make_config(
+ use_openrouter=True,
+ api_key="k",
+ base_url="https://openrouter.ai/api/v1",
+ ),
+ ):
+ with (
+ patch(f"{_ENV}.config", cfg),
+ patch(f"{_ENV}.validate_subscription"),
+ ):
+ from backend.copilot.sdk.env import build_sdk_env
+
+ env = build_sdk_env()
+
+ assert isinstance(env, dict)
+ env["CLAUDE_CODE_TMPDIR"] = "/tmp/test"
+ assert env["CLAUDE_CODE_TMPDIR"] == "/tmp/test"
+
+
+# ---------------------------------------------------------------------------
+# is_transient_api_error
+# ---------------------------------------------------------------------------
+
+
+class TestIsTransientApiError:
+ """Verify that is_transient_api_error detects all transient patterns."""
+
+ @pytest.mark.parametrize(
+ "error_text",
+ [
+ "socket connection was closed unexpectedly",
+ "ECONNRESET",
+ "connection was forcibly closed",
+ "network socket disconnected",
+ ],
+ )
+ def test_connection_level_errors(self, error_text: str):
+ assert is_transient_api_error(error_text)
+
+ @pytest.mark.parametrize(
+ "error_text",
+ [
+ "rate limit exceeded",
+ "rate_limit_error",
+ "Too Many Requests",
+ "status code 429",
+ ],
+ )
+ def test_429_rate_limit_errors(self, error_text: str):
+ assert is_transient_api_error(error_text)
+
+ @pytest.mark.parametrize(
+ "error_text",
+ [
+ # Status-code-specific patterns (preferred — no false-positive risk)
+ "status code 529",
+ "status code 500",
+ "status code 502",
+ "status code 503",
+ "status code 504",
+ ],
+ )
+ def test_5xx_server_errors(self, error_text: str):
+ assert is_transient_api_error(error_text)
+
+ @pytest.mark.parametrize(
+ "error_text",
+ [
+ "invalid_api_key",
+ "Authentication failed",
+ "prompt is too long",
+ "model not found",
+ "",
+ # Natural-language phrases intentionally NOT matched — they are too
+ # broad and could appear in application-level SDK messages unrelated
+ # to Anthropic API transient conditions.
+ "API is overloaded",
+ "Internal Server Error",
+ "Bad Gateway",
+ "Service Unavailable",
+ "Gateway Timeout",
+ ],
+ )
+ def test_non_transient_errors(self, error_text: str):
+ assert not is_transient_api_error(error_text)
+
+ def test_case_insensitive(self):
+ assert is_transient_api_error("SOCKET CONNECTION WAS CLOSED UNEXPECTEDLY")
+ assert is_transient_api_error("econnreset")
+
+
+# ---------------------------------------------------------------------------
+# _HandledStreamError.already_yielded contract
+# ---------------------------------------------------------------------------
+
+
+class TestHandledStreamErrorAlreadyYielded:
+ """Verify the already_yielded semantics on _HandledStreamError."""
+
+ def test_default_already_yielded_is_true(self):
+ """Non-transient callers (circuit-breaker, idle timeout) don't pass the flag —
+ the default True means the outer loop won't yield a duplicate StreamError."""
+ from backend.copilot.sdk.service import _HandledStreamError
+
+ exc = _HandledStreamError("some error", code="circuit_breaker_empty_tool_calls")
+ assert exc.already_yielded is True
+
+ def test_transient_error_sets_already_yielded_false(self):
+ """Transient errors pass already_yielded=False so the outer loop
+ yields StreamError only once (when retries are exhausted)."""
+ from backend.copilot.sdk.service import _HandledStreamError
+
+ exc = _HandledStreamError(
+ "transient",
+ code="transient_api_error",
+ already_yielded=False,
+ )
+ assert exc.already_yielded is False
+
+ def test_backoff_capped_at_30s(self):
+ """_compute_transient_backoff must be capped at _MAX_TRANSIENT_BACKOFF_SECONDS.
+
+ With max_transient_retries=10, uncapped 2^9=512s would stall users
+ for 8+ minutes. _compute_transient_backoff caps at 30s.
+
+ Full-jitter (0.5 … 1.0 × base) is applied for thundering-herd
+ prevention, so each call returns a value in [base//2, base] rather
+ than an exact integer. We verify bounds instead of exact values.
+ """
+ from backend.copilot.sdk.service import (
+ _MAX_TRANSIENT_BACKOFF_SECONDS,
+ _compute_transient_backoff,
+ )
+
+ # attempt=1: base=1, jitter range [1, 1] (max(1, round(1 * [0.5,1.0])))
+ v1 = _compute_transient_backoff(1)
+ assert v1 >= 1
+
+ # attempt=2: base=2, jitter range [1, 2]
+ v2 = _compute_transient_backoff(2)
+ assert 1 <= v2 <= 2
+
+ # attempt=3: base=4, jitter range [2, 4]
+ v3 = _compute_transient_backoff(3)
+ assert 2 <= v3 <= 4
+
+ # attempt=4: base=8, jitter range [4, 8]
+ v4 = _compute_transient_backoff(4)
+ assert 4 <= v4 <= 8
+
+ # attempt=5: base=16, jitter range [8, 16]
+ v5 = _compute_transient_backoff(5)
+ assert 8 <= v5 <= 16
+
+ # attempt=6: base capped at 30, jitter range [15, 30]
+ v6 = _compute_transient_backoff(6)
+ assert 15 <= v6 <= _MAX_TRANSIENT_BACKOFF_SECONDS
+
+ # attempt=10: still capped
+ v10 = _compute_transient_backoff(10)
+ assert v10 <= _MAX_TRANSIENT_BACKOFF_SECONDS
+
+
+# ---------------------------------------------------------------------------
+# Config validators for max_turns / max_budget_usd
+# ---------------------------------------------------------------------------
+
+
+class TestConfigValidators:
+ """Verify ge/le bounds on max_turns and max_budget_usd."""
+
+ def test_max_turns_rejects_zero(self):
+ with pytest.raises(ValidationError):
+ _make_config(claude_agent_max_turns=0)
+
+ def test_max_turns_rejects_negative(self):
+ with pytest.raises(ValidationError):
+ _make_config(claude_agent_max_turns=-1)
+
+ def test_max_turns_rejects_above_10000(self):
+ with pytest.raises(ValidationError):
+ _make_config(claude_agent_max_turns=10001)
+
+ def test_max_turns_accepts_boundary_values(self):
+ cfg_low = _make_config(claude_agent_max_turns=1)
+ assert cfg_low.claude_agent_max_turns == 1
+ cfg_high = _make_config(claude_agent_max_turns=10000)
+ assert cfg_high.claude_agent_max_turns == 10000
+
+ def test_max_budget_rejects_zero(self):
+ with pytest.raises(ValidationError):
+ _make_config(claude_agent_max_budget_usd=0.0)
+
+ def test_max_budget_rejects_negative(self):
+ with pytest.raises(ValidationError):
+ _make_config(claude_agent_max_budget_usd=-1.0)
+
+ def test_max_budget_rejects_above_1000(self):
+ with pytest.raises(ValidationError):
+ _make_config(claude_agent_max_budget_usd=1000.01)
+
+ def test_max_budget_accepts_boundary_values(self):
+ cfg_low = _make_config(claude_agent_max_budget_usd=0.01)
+ assert cfg_low.claude_agent_max_budget_usd == 0.01
+ cfg_high = _make_config(claude_agent_max_budget_usd=1000.0)
+ assert cfg_high.claude_agent_max_budget_usd == 1000.0
+
+ def test_max_transient_retries_rejects_negative(self):
+ with pytest.raises(ValidationError):
+ _make_config(claude_agent_max_transient_retries=-1)
+
+ def test_max_transient_retries_rejects_above_10(self):
+ with pytest.raises(ValidationError):
+ _make_config(claude_agent_max_transient_retries=11)
+
+ def test_max_transient_retries_accepts_boundary_values(self):
+ cfg_low = _make_config(claude_agent_max_transient_retries=0)
+ assert cfg_low.claude_agent_max_transient_retries == 0
+ cfg_high = _make_config(claude_agent_max_transient_retries=10)
+ assert cfg_high.claude_agent_max_transient_retries == 10
+
+
+# ---------------------------------------------------------------------------
+# events_yielded counter exclusions
+# ---------------------------------------------------------------------------
+
+
+class TestEventsYieldedExclusions:
+ """Verify that ephemeral event types don't increment events_yielded.
+
+ The events_yielded counter in stream_chat_completion_sdk controls whether
+ _next_transient_backoff() permits a retry. StreamError and StreamStatus
+ must NOT be counted so that a transient notification can be followed by a
+ retry without producing duplicate content for the client.
+
+ These tests use the production _EPHEMERAL_EVENT_TYPES constant directly so
+ that any drift between the constant and these assertions is caught immediately.
+ """
+
+ def test_stream_error_is_ephemeral_type(self):
+ """StreamError must be an instance of the production _EPHEMERAL_EVENT_TYPES tuple.
+
+ Uses the production constant rather than reconstructing the tuple locally
+ so that any refactor that removes StreamError from the set will fail this test.
+ """
+ from backend.copilot.response_model import StreamError
+ from backend.copilot.sdk.service import _EPHEMERAL_EVENT_TYPES
+
+ err = StreamError(errorText="transient", code="transient_api_error")
+ assert isinstance(err, _EPHEMERAL_EVENT_TYPES), (
+ "StreamError must be excluded from events_yielded — "
+ "if counted, transient retries would be blocked after the first notification"
+ )
+
+ def test_stream_status_is_ephemeral_type(self):
+ """StreamStatus must be excluded from the production _EPHEMERAL_EVENT_TYPES tuple."""
+ from backend.copilot.response_model import StreamStatus
+ from backend.copilot.sdk.service import _EPHEMERAL_EVENT_TYPES
+
+ status = StreamStatus(message="Connection interrupted, retrying in 2s…")
+ assert isinstance(status, _EPHEMERAL_EVENT_TYPES), (
+ "StreamStatus must be excluded from events_yielded — "
+ "retrying after emitting a status notification must still be permitted"
+ )
+
+
+# ---------------------------------------------------------------------------
+# _next_transient_backoff — module-level pure function
+# ---------------------------------------------------------------------------
+
+
+class TestNextTransientBackoff:
+ """Unit tests for _next_transient_backoff.
+
+ This is the core safety mechanism that prevents:
+ * duplicate content (events_yielded > 0 guard)
+ * infinite transient retry loops (max_transient_retries cap)
+ """
+
+ def test_events_yielded_prevents_retry(self):
+ """When events_yielded > 0, return (None, unchanged_retries) — no retry."""
+ from backend.copilot.sdk.service import _next_transient_backoff
+
+ backoff, retries = _next_transient_backoff(
+ events_yielded=1, transient_retries=0, max_transient_retries=3
+ )
+ assert backoff is None
+ # Counter NOT incremented — events already sent, not a retry budget question.
+ assert retries == 0
+
+ def test_returns_backoff_on_first_retry(self):
+ """First transient retry with no prior events gets a 1 s backoff (2^0)."""
+ from backend.copilot.sdk.service import _next_transient_backoff
+
+ backoff, retries = _next_transient_backoff(
+ events_yielded=0, transient_retries=0, max_transient_retries=3
+ )
+ assert backoff == 1 # 2^(1-1) = 1 s
+ assert retries == 1
+
+ def test_increments_counter_each_retry(self):
+ """Each successive call increments the retry counter and returns non-None backoff.
+
+ Exact backoff values vary due to jitter; this test verifies the counter
+ increments correctly and that a positive backoff is returned each time.
+ """
+ from backend.copilot.sdk.service import _next_transient_backoff
+
+ retries = 0
+ for expected_retries in [1, 2, 3]:
+ backoff, retries = _next_transient_backoff(
+ events_yielded=0,
+ transient_retries=retries,
+ max_transient_retries=5,
+ )
+ assert backoff is not None
+ assert backoff >= 1
+ assert retries == expected_retries
+
+ def test_returns_none_when_budget_exhausted(self):
+ """When transient_retries == max_transient_retries, next call returns None."""
+ from backend.copilot.sdk.service import _next_transient_backoff
+
+ backoff, retries = _next_transient_backoff(
+ events_yielded=0, transient_retries=3, max_transient_retries=3
+ )
+ assert backoff is None
+ # Counter is still incremented to reflect the attempt was made.
+ assert retries == 4
+
+ def test_events_yielded_takes_priority_over_exhaustion(self):
+ """events_yielded guard fires even when retries are also exhausted."""
+ from backend.copilot.sdk.service import _next_transient_backoff
+
+ backoff, retries = _next_transient_backoff(
+ events_yielded=5, transient_retries=3, max_transient_retries=3
+ )
+ assert backoff is None
+ # Counter stays the same — events_yielded path returns early.
+ assert retries == 3
+
+
+# ---------------------------------------------------------------------------
+# _do_transient_backoff — module-level async generator
+# ---------------------------------------------------------------------------
+
+
+class TestDoTransientBackoff:
+ """Unit tests for _do_transient_backoff.
+
+ The helper encapsulates the retry ceremony shared between both exception
+ handlers: emit StreamStatus, sleep, reset adapter, reset usage.
+ """
+
+ async def test_yields_stream_status_with_backoff_in_message(self):
+ """The helper yields exactly one StreamStatus containing the backoff duration."""
+ from unittest.mock import AsyncMock, MagicMock, patch
+
+ from backend.copilot.response_model import StreamStatus
+ from backend.copilot.sdk.service import _do_transient_backoff
+
+ state = MagicMock()
+ state.usage = MagicMock()
+
+ events = []
+ with patch("asyncio.sleep", new=AsyncMock()):
+ async for evt in _do_transient_backoff(5, state, "msg-id", "sess-id"):
+ events.append(evt)
+
+ assert len(events) == 1
+ assert isinstance(events[0], StreamStatus)
+ assert "5s" in events[0].message
+
+ async def test_sleeps_for_exactly_backoff_seconds(self):
+ """asyncio.sleep is called with the backoff value."""
+ from unittest.mock import AsyncMock, MagicMock, patch
+
+ from backend.copilot.sdk.service import _do_transient_backoff
+
+ state = MagicMock()
+ state.usage = MagicMock()
+
+ mock_sleep = AsyncMock()
+ with patch("asyncio.sleep", new=mock_sleep):
+ async for _ in _do_transient_backoff(7, state, "msg-id", "sess-id"):
+ pass
+
+ mock_sleep.assert_called_once_with(7)
+
+ async def test_replaces_adapter_with_new_instance(self):
+ """state.adapter is replaced with a new SDKResponseAdapter after yield."""
+ from unittest.mock import AsyncMock, MagicMock, patch
+
+ from backend.copilot.sdk.service import _do_transient_backoff
+
+ original_adapter = MagicMock()
+ state = MagicMock()
+ state.adapter = original_adapter
+ state.usage = MagicMock()
+
+ with (
+ patch("asyncio.sleep", new=AsyncMock()),
+ patch("backend.copilot.sdk.service.SDKResponseAdapter") as mock_cls,
+ ):
+ new_adapter = MagicMock()
+ mock_cls.return_value = new_adapter
+ async for _ in _do_transient_backoff(3, state, "msg-1", "sess-1"):
+ pass
+
+ mock_cls.assert_called_once_with(message_id="msg-1", session_id="sess-1")
+ assert state.adapter is new_adapter
+
+ async def test_resets_usage_after_yield(self):
+ """state.usage.reset() is called so the next attempt starts with clean counters."""
+ from unittest.mock import AsyncMock, MagicMock, patch
+
+ from backend.copilot.sdk.service import _do_transient_backoff
+
+ state = MagicMock()
+ state.usage = MagicMock()
+
+ with patch("asyncio.sleep", new=AsyncMock()):
+ async for _ in _do_transient_backoff(2, state, "msg-id", "sess-id"):
+ pass
+
+ state.usage.reset.assert_called_once()
+
+
+# ---------------------------------------------------------------------------
+# _is_fallback_stderr — module-level pure function
+# ---------------------------------------------------------------------------
+
+
+class TestIsFallbackStderr:
+ """Unit tests for _is_fallback_stderr.
+
+ Ensures the pure function used by _on_stderr to detect fallback-model
+ activation can be tested independently of the closure.
+ """
+
+ def test_true_for_fallback_model_phrase(self):
+ """Lines containing 'fallback model' must return True."""
+ from backend.copilot.sdk.service import _is_fallback_stderr
+
+ assert _is_fallback_stderr("Using fallback model: claude-sonnet-4") is True
+
+ def test_case_insensitive(self):
+ """Matching must be case-insensitive."""
+ from backend.copilot.sdk.service import _is_fallback_stderr
+
+ assert _is_fallback_stderr("FALLBACK MODEL activated") is True
+ assert _is_fallback_stderr("Fallback Model switching") is True
+
+ def test_false_for_unrelated_fallback(self):
+ """'fallback' alone (no 'model') must not trigger detection."""
+ from backend.copilot.sdk.service import _is_fallback_stderr
+
+ assert _is_fallback_stderr("Using cached result fallback") is False
+ assert _is_fallback_stderr("Tool retry fallback triggered") is False
+
+ def test_false_for_empty_line(self):
+ from backend.copilot.sdk.service import _is_fallback_stderr
+
+ assert _is_fallback_stderr("") is False
+
+ def test_false_for_unrelated_stderr(self):
+ from backend.copilot.sdk.service import _is_fallback_stderr
+
+ assert _is_fallback_stderr("Task completed successfully") is False
+
+
+# ---------------------------------------------------------------------------
+# _EPHEMERAL_EVENT_TYPES — module-level constant
+# ---------------------------------------------------------------------------
+
+
+class TestEphemeralEventTypesConstant:
+ """Verify _EPHEMERAL_EVENT_TYPES is a module-level constant that stays in
+ sync with the event types that must not be counted toward events_yielded.
+ """
+
+ def test_stream_error_is_in_ephemeral_types(self):
+ from backend.copilot.response_model import StreamError
+ from backend.copilot.sdk.service import _EPHEMERAL_EVENT_TYPES
+
+ assert StreamError in _EPHEMERAL_EVENT_TYPES, (
+ "StreamError must be in _EPHEMERAL_EVENT_TYPES — if counted, "
+ "transient retries would be blocked after the first notification"
+ )
+
+ def test_stream_status_is_in_ephemeral_types(self):
+ from backend.copilot.response_model import StreamStatus
+ from backend.copilot.sdk.service import _EPHEMERAL_EVENT_TYPES
+
+ assert StreamStatus in _EPHEMERAL_EVENT_TYPES, (
+ "StreamStatus must be in _EPHEMERAL_EVENT_TYPES so that a retry "
+ "notification can be followed by another retry"
+ )
+
+ def test_stream_heartbeat_is_in_ephemeral_types(self):
+ from backend.copilot.response_model import StreamHeartbeat
+ from backend.copilot.sdk.service import _EPHEMERAL_EVENT_TYPES
+
+ assert StreamHeartbeat in _EPHEMERAL_EVENT_TYPES
+
+ def test_is_a_tuple(self):
+ """isinstance() requires a tuple (not a list) as second argument."""
+ from backend.copilot.sdk.service import _EPHEMERAL_EVENT_TYPES
+
+ assert isinstance(_EPHEMERAL_EVENT_TYPES, tuple)
+
+
+# ---------------------------------------------------------------------------
+# TranscriptBuilder snapshot/restore
+# ---------------------------------------------------------------------------
+
+
+class TestTranscriptBuilderSnapshotRestore:
+ """Verify that snapshot() and restore() provide safe rollback
+ without accessing private attributes directly.
+ """
+
+ def test_snapshot_returns_independent_copy(self):
+ """Mutations to the builder after snapshot() must not affect the snap."""
+ from backend.copilot.transcript_builder import TranscriptBuilder
+
+ builder = TranscriptBuilder()
+ builder.append_user("hello")
+ snap = builder.snapshot()
+
+ builder.append_user("world") # mutate after snapshot
+ entries_copy, _ = snap
+ assert len(entries_copy) == 1, "snapshot() must return an independent list copy"
+ assert len(builder._entries) == 2
+
+ def test_restore_resets_entries_and_uuid(self):
+ """restore() must bring back the exact state at snapshot time."""
+ from backend.copilot.transcript_builder import TranscriptBuilder
+
+ builder = TranscriptBuilder()
+ builder.append_user("first")
+ snap = builder.snapshot()
+ uuid_at_snap = builder._last_uuid
+
+ builder.append_user("second")
+ builder.restore(snap)
+
+ assert len(builder._entries) == 1
+ assert builder._last_uuid == uuid_at_snap
+
+ def test_restore_to_empty_state(self):
+ """Restoring a snapshot of an empty builder must clear all entries."""
+ from backend.copilot.transcript_builder import TranscriptBuilder
+
+ builder = TranscriptBuilder()
+ empty_snap = builder.snapshot()
+
+ builder.append_user("something")
+ assert builder.entry_count == 1
+
+ builder.restore(empty_snap)
+ assert builder.entry_count == 0
+ assert builder._last_uuid is None
+
+
+# ---------------------------------------------------------------------------
+# _last_reset_attempt guard
+# ---------------------------------------------------------------------------
+
+
+class TestLastResetAttemptGuard:
+ """Verify that transient retries within the same context-level attempt
+ do NOT reset the transient_retries counter (which would create an infinite loop).
+
+ The guard works by tracking the last attempt value that triggered a reset.
+ Transient retries `continue` without incrementing `attempt`, so the reset
+ only fires when `attempt` actually advances (different attempt number).
+ """
+
+ def test_transient_retry_preserves_counter(self):
+ """Simulating the loop: counter must NOT reset on transient continue.
+
+ We simulate the _last_reset_attempt logic directly by replaying
+ the loop state transitions. A transient retry stays on the same
+ `attempt` value, so the guard must block the reset.
+ """
+ attempt = 0
+ _last_reset_attempt = -1
+ transient_retries = 0
+
+ # First entry into the loop: attempt 0 is different from -1, so reset.
+ if attempt != _last_reset_attempt:
+ transient_retries = 0
+ _last_reset_attempt = attempt
+
+ assert transient_retries == 0
+ assert _last_reset_attempt == 0
+
+ # Simulate transient retry: transient_retries incremented, `attempt`
+ # stays 0 (transient continue does NOT call attempt += 1).
+ transient_retries += 1
+
+ # Loop continues: attempt is still 0, so guard blocks the reset.
+ if attempt != _last_reset_attempt:
+ transient_retries = 0 # must NOT execute
+ _last_reset_attempt = attempt
+
+ assert (
+ transient_retries == 1
+ ), "transient_retries must not be reset when attempt has not changed"
+
+ def test_counter_resets_on_new_attempt(self):
+ """When attempt advances to 1, transient_retries must reset to 0."""
+ attempt = 0
+ _last_reset_attempt = -1
+ transient_retries = 0
+
+ # attempt=0: first entry, reset fires.
+ if attempt != _last_reset_attempt:
+ transient_retries = 0
+ _last_reset_attempt = attempt
+
+ # Simulate transient retries used up.
+ transient_retries = 3
+
+ # Context compaction: attempt advances to 1.
+ attempt = 1
+
+ # Loop top: attempt changed, reset fires.
+ if attempt != _last_reset_attempt:
+ transient_retries = 0
+ _last_reset_attempt = attempt
+
+ assert (
+ transient_retries == 0
+ ), "transient_retries must reset to 0 when attempt advances"
+ assert _last_reset_attempt == 1
+
+
+# ---------------------------------------------------------------------------
+# Integration: _HandledStreamError transient retry path
+# ---------------------------------------------------------------------------
+
+
+async def _drain(agen) -> list:
+ """Collect all items from an async generator into a list."""
+ items = []
+ async for item in agen:
+ items.append(item)
+ return items
+
+
+class TestHandledStreamErrorTransientRetry:
+ """Integration tests for the _HandledStreamError transient retry wiring.
+
+ These tests mock _run_stream_attempt to simulate the path where:
+ 1. _run_stream_attempt raises _HandledStreamError(code="transient_api_error")
+ 2. The outer loop calls _next_transient_backoff → gets a backoff
+ 3. _do_transient_backoff emits StreamStatus, sleeps, resets state
+ 4. The loop continues and _run_stream_attempt succeeds on the next call
+
+ Verifies: StreamStatus emitted, no StreamError, second attempt returns
+ content, and the transcript is restored between attempts.
+ """
+
+ @pytest.mark.asyncio
+ async def test_retry_succeeds_after_one_transient_handled_error(self):
+ """_HandledStreamError(transient_api_error) → StreamStatus → success on retry.
+
+ Simulates the retry loop logic directly, mirroring the real loop in
+ stream_chat_completion_sdk. Validates the composition of the three
+ helper functions rather than calling stream_chat_completion_sdk (which
+ requires DB/redis connections unavailable in unit tests).
+ """
+ from unittest.mock import AsyncMock, MagicMock, patch
+
+ from backend.copilot.response_model import (
+ StreamError,
+ StreamFinish,
+ StreamStatus,
+ )
+ from backend.copilot.sdk.service import (
+ _do_transient_backoff,
+ _HandledStreamError,
+ _next_transient_backoff,
+ )
+
+ transient_retries = 0
+ max_transient_retries = 3
+ attempt = 0
+ _last_reset_attempt = -1
+ events_yielded = 0
+ emitted: list = []
+
+ call_count = 0
+
+ async def fake_run_stream():
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ # First call: raise transient error (not yet yielded to client)
+ raise _HandledStreamError(
+ "transient", code="transient_api_error", already_yielded=False
+ )
+ # Second call: success — yield a content event
+ yield StreamFinish()
+
+ state = MagicMock()
+ state.usage = MagicMock()
+ state.transcript_builder = MagicMock()
+ state.transcript_builder.snapshot.return_value = ([], None)
+
+ # Replay the retry loop body for up to 10 iterations.
+ for _iteration in range(10):
+ if attempt >= 3:
+ break
+ if attempt != _last_reset_attempt:
+ transient_retries = 0
+ _last_reset_attempt = attempt
+
+ events_yielded = 0
+
+ try:
+ async for evt in fake_run_stream():
+ if not isinstance(evt, (StreamError, StreamStatus)):
+ events_yielded += 1
+ emitted.append(evt)
+ break # success
+ except _HandledStreamError as exc:
+ state.transcript_builder.restore(state.transcript_builder.snapshot())
+ if exc.code == "transient_api_error":
+ backoff, transient_retries = _next_transient_backoff(
+ events_yielded, transient_retries, max_transient_retries
+ )
+ if backoff is not None:
+ with patch("asyncio.sleep", new=AsyncMock()):
+ async for evt in _do_transient_backoff(
+ backoff, state, "msg-id", "sess-id"
+ ):
+ emitted.append(evt)
+ continue # retry
+
+ # StreamStatus emitted (retry notification)
+ statuses = [e for e in emitted if isinstance(e, StreamStatus)]
+ assert len(statuses) == 1
+ assert (
+ "retry" in statuses[0].message.lower()
+ or "connection" in statuses[0].message.lower()
+ )
+
+ # No StreamError — transient error not surfaced when retry succeeded
+ errors = [e for e in emitted if isinstance(e, StreamError)]
+ assert len(errors) == 0
+
+ # Content from successful second attempt is present
+ content_events = [e for e in emitted if isinstance(e, StreamFinish)]
+ assert len(content_events) == 1
+
+ # Second attempt was called
+ assert call_count == 2
+
+ @pytest.mark.asyncio
+ async def test_transient_handled_error_exhaustion_yields_stream_error(self):
+ """When all transient retries exhausted, StreamError must be yielded."""
+ from unittest.mock import AsyncMock, MagicMock, patch
+
+ from backend.copilot.constants import FRIENDLY_TRANSIENT_MSG
+ from backend.copilot.response_model import StreamError, StreamStatus
+ from backend.copilot.sdk.service import (
+ _do_transient_backoff,
+ _HandledStreamError,
+ _next_transient_backoff,
+ )
+
+ transient_retries = 0
+ max_transient_retries = 2 # exhaust after 2 retries
+ attempt = 0
+ _last_reset_attempt = -1
+ emitted: list = []
+
+ async def always_fail():
+ raise _HandledStreamError(
+ "transient",
+ error_msg="API overloaded",
+ code="transient_api_error",
+ already_yielded=False,
+ )
+ # Satisfy the type checker — unreachable
+ return
+ yield # noqa: B901
+
+ state = MagicMock()
+ state.usage = MagicMock()
+ state.transcript_builder = MagicMock()
+ state.transcript_builder.snapshot.return_value = ([], None)
+
+ ended_with_stream_error = False
+
+ for _iteration in range(20):
+ if attempt >= 3:
+ break
+ if attempt != _last_reset_attempt:
+ transient_retries = 0
+ _last_reset_attempt = attempt
+
+ events_yielded = 0
+ try:
+ async for evt in always_fail():
+ emitted.append(evt)
+ break
+ except _HandledStreamError as exc:
+ state.transcript_builder.restore(state.transcript_builder.snapshot())
+ if exc.code == "transient_api_error":
+ backoff, transient_retries = _next_transient_backoff(
+ events_yielded, transient_retries, max_transient_retries
+ )
+ if backoff is not None:
+ with patch("asyncio.sleep", new=AsyncMock()):
+ async for evt in _do_transient_backoff(
+ backoff, state, "m", "s"
+ ):
+ emitted.append(evt)
+ continue
+ # retries exhausted
+ ended_with_stream_error = True
+ if not exc.already_yielded:
+ emitted.append(
+ StreamError(
+ errorText=exc.error_msg or FRIENDLY_TRANSIENT_MSG,
+ code=exc.code or "transient_api_error",
+ )
+ )
+ break
+
+ # Two StreamStatus events emitted (one per retry before exhaustion)
+ statuses = [e for e in emitted if isinstance(e, StreamStatus)]
+ assert len(statuses) == max_transient_retries
+
+ # One StreamError emitted after exhaustion
+ errors = [e for e in emitted if isinstance(e, StreamError)]
+ assert len(errors) == 1
+ assert errors[0].code == "transient_api_error"
+
+ assert ended_with_stream_error is True
+
+
+# ---------------------------------------------------------------------------
+# Integration: generic Exception transient retry path
+# ---------------------------------------------------------------------------
+
+
+class TestGenericExceptionTransientRetry:
+ """Integration tests for the raw Exception transient retry wiring.
+
+ These tests simulate the Exception handler path (e.g. ECONNRESET) that
+ is a separate code path from _HandledStreamError. The same retry
+ mechanics apply — _next_transient_backoff + _do_transient_backoff +
+ continue — but the entry path is different (no already_yielded flag,
+ no pre-yielded StreamError from _run_stream_attempt).
+ """
+
+ @pytest.mark.asyncio
+ async def test_econnreset_triggers_retry_and_succeeds(self):
+ """Raw Exception('ECONNRESET') matching is_transient_api_error → retry succeeds.
+
+ Simulates the generic Exception handler path — separate from
+ _HandledStreamError. Verifies the same retry mechanics apply:
+ _next_transient_backoff + _do_transient_backoff + continue.
+ """
+ from unittest.mock import AsyncMock, MagicMock, patch
+
+ from backend.copilot.constants import is_transient_api_error
+ from backend.copilot.response_model import (
+ StreamError,
+ StreamFinish,
+ StreamStatus,
+ )
+ from backend.copilot.sdk.service import (
+ _do_transient_backoff,
+ _next_transient_backoff,
+ )
+
+ transient_retries = 0
+ max_transient_retries = 3
+ attempt = 0
+ _last_reset_attempt = -1
+ emitted: list = []
+ call_count = 0
+
+ async def fake_stream():
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ raise Exception("ECONNRESET")
+ yield StreamFinish()
+
+ state = MagicMock()
+ state.usage = MagicMock()
+ state.transcript_builder = MagicMock()
+ state.transcript_builder.snapshot.return_value = ([], None)
+
+ for _iteration in range(10):
+ if attempt >= 3:
+ break
+ if attempt != _last_reset_attempt:
+ transient_retries = 0
+ _last_reset_attempt = attempt
+
+ events_yielded = 0
+ try:
+ async for evt in fake_stream():
+ if not isinstance(evt, (StreamError, StreamStatus)):
+ events_yielded += 1
+ emitted.append(evt)
+ break
+ except Exception as exc:
+ is_transient = is_transient_api_error(str(exc))
+ state.transcript_builder.restore(state.transcript_builder.snapshot())
+ if events_yielded == 0 and is_transient:
+ backoff, transient_retries = _next_transient_backoff(
+ events_yielded, transient_retries, max_transient_retries
+ )
+ if backoff is not None:
+ with patch("asyncio.sleep", new=AsyncMock()):
+ async for evt in _do_transient_backoff(
+ backoff, state, "m", "s"
+ ):
+ emitted.append(evt)
+ continue
+ break
+
+ # StreamStatus emitted during retry (notification to client)
+ statuses = [e for e in emitted if isinstance(e, StreamStatus)]
+ assert len(statuses) == 1
+
+ # No StreamError — retry succeeded
+ errors = [e for e in emitted if isinstance(e, StreamError)]
+ assert len(errors) == 0
+
+ # Content from successful second attempt
+ finish_events = [e for e in emitted if isinstance(e, StreamFinish)]
+ assert len(finish_events) == 1
+
+ # Two total calls: first fails (ECONNRESET), second succeeds
+ assert call_count == 2
+
+ @pytest.mark.asyncio
+ async def test_generic_exception_not_retried_when_events_yielded(self):
+ """When events_yielded > 0, transient Exception must NOT trigger retry."""
+ from backend.copilot.constants import is_transient_api_error
+ from backend.copilot.sdk.service import _next_transient_backoff
+
+ events_yielded = 1 # content already sent
+ transient_retries = 0
+ max_transient_retries = 3
+
+ # The real loop checks events_yielded before calling _next_transient_backoff.
+ # We replicate that check here.
+ exc = Exception("ECONNRESET")
+ is_transient = is_transient_api_error(str(exc))
+
+ assert is_transient is True
+
+ # Mimic the loop guard: only call backoff function when events_yielded == 0
+ if events_yielded > 0 or not is_transient:
+ backoff = None
+ else:
+ backoff, _ = _next_transient_backoff(
+ events_yielded, transient_retries, max_transient_retries
+ )
+
+ assert (
+ backoff is None
+ ), "retry must not be attempted when events have already been sent to the client"
+
+
+# ---------------------------------------------------------------------------
+# _session_messages_to_transcript
+# ---------------------------------------------------------------------------
+
+
+class TestSessionMessagesToTranscript:
+ """Unit tests for _session_messages_to_transcript.
+
+ Verifies that ChatMessage lists are converted to valid JSONL transcripts
+ with proper tool_use / tool_result content blocks — giving the Claude CLI
+ full structural context via --resume when no previous transcript file is
+ available.
+ """
+
+ def _parse_jsonl(self, jsonl: str) -> list[dict]:
+ import json as _json
+
+ return [
+ _json.loads(line) for line in jsonl.strip().splitlines() if line.strip()
+ ]
+
+ def test_empty_messages_returns_empty_string(self):
+ from backend.copilot.sdk.service import _session_messages_to_transcript
+
+ assert _session_messages_to_transcript([]) == ""
+
+ def test_simple_user_assistant_messages(self):
+ from backend.copilot.model import ChatMessage
+ from backend.copilot.sdk.service import _session_messages_to_transcript
+
+ messages = [
+ ChatMessage(role="user", content="Hello"),
+ ChatMessage(role="assistant", content="Hi there"),
+ ]
+ result = _session_messages_to_transcript(messages)
+ entries = self._parse_jsonl(result)
+
+ assert len(entries) == 2
+ assert entries[0]["type"] == "user"
+ assert entries[0]["message"]["role"] == "user"
+ assert entries[0]["message"]["content"] == "Hello"
+ assert entries[1]["type"] == "assistant"
+ assert entries[1]["message"]["role"] == "assistant"
+ content_blocks = entries[1]["message"]["content"]
+ assert any(
+ b.get("type") == "text" and b.get("text") == "Hi there"
+ for b in content_blocks
+ )
+
+ def test_assistant_with_tool_calls_produces_tool_use_blocks(self):
+ import json as _json
+
+ from backend.copilot.model import ChatMessage
+ from backend.copilot.sdk.service import _session_messages_to_transcript
+
+ messages = [
+ ChatMessage(role="user", content="List files"),
+ ChatMessage(
+ role="assistant",
+ content="",
+ tool_calls=[
+ {
+ "id": "call_abc123",
+ "type": "function",
+ "function": {
+ "name": "bash",
+ "arguments": _json.dumps({"cmd": "ls -la"}),
+ },
+ }
+ ],
+ ),
+ ]
+ result = _session_messages_to_transcript(messages)
+ entries = self._parse_jsonl(result)
+
+ assert len(entries) == 2
+ assistant_entry = entries[1]
+ assert assistant_entry["type"] == "assistant"
+ blocks = assistant_entry["message"]["content"]
+ tool_use_blocks = [b for b in blocks if b.get("type") == "tool_use"]
+ assert len(tool_use_blocks) == 1
+ tu = tool_use_blocks[0]
+ assert tu["id"] == "call_abc123"
+ assert tu["name"] == "bash"
+ assert tu["input"] == {"cmd": "ls -la"}
+
+ def test_tool_result_produces_tool_result_block(self):
+ import json as _json
+
+ from backend.copilot.model import ChatMessage
+ from backend.copilot.sdk.service import _session_messages_to_transcript
+
+ messages = [
+ ChatMessage(role="user", content="List files"),
+ ChatMessage(
+ role="assistant",
+ content="",
+ tool_calls=[
+ {
+ "id": "call_xyz",
+ "type": "function",
+ "function": {
+ "name": "bash",
+ "arguments": _json.dumps({"cmd": "ls"}),
+ },
+ }
+ ],
+ ),
+ ChatMessage(
+ role="tool",
+ tool_call_id="call_xyz",
+ content="file1.txt\nfile2.txt\nfile3.txt",
+ ),
+ ]
+ result = _session_messages_to_transcript(messages)
+ entries = self._parse_jsonl(result)
+
+ # user + assistant + user(tool_result)
+ assert len(entries) == 3
+ tool_result_entry = entries[2]
+ assert tool_result_entry["type"] == "user"
+ content = tool_result_entry["message"]["content"]
+ assert isinstance(content, list)
+ tr_blocks = [b for b in content if b.get("type") == "tool_result"]
+ assert len(tr_blocks) == 1
+ assert tr_blocks[0]["tool_use_id"] == "call_xyz"
+ assert tr_blocks[0]["content"] == "file1.txt\nfile2.txt\nfile3.txt"
+
+ def test_tool_result_full_content_not_truncated(self):
+ """Tool result content must NOT be truncated (unlike _format_conversation_context
+ which caps at 500 chars)."""
+ from backend.copilot.model import ChatMessage
+ from backend.copilot.sdk.service import _session_messages_to_transcript
+
+ big_output = "x" * 10_000
+ messages = [
+ ChatMessage(role="user", content="q"),
+ ChatMessage(
+ role="tool",
+ tool_call_id="call_big",
+ content=big_output,
+ ),
+ ]
+ result = _session_messages_to_transcript(messages)
+ entries = self._parse_jsonl(result)
+ tool_result_entry = next(
+ e
+ for e in entries
+ if e.get("type") == "user" and isinstance(e["message"].get("content"), list)
+ )
+ tr_block = next(
+ b
+ for b in tool_result_entry["message"]["content"]
+ if b.get("type") == "tool_result"
+ )
+ assert tr_block["content"] == big_output, "Tool result must not be truncated"
+
+ def test_parent_uuid_chain_is_correct(self):
+ """Each entry's parentUuid must equal the previous entry's uuid."""
+ from backend.copilot.model import ChatMessage
+ from backend.copilot.sdk.service import _session_messages_to_transcript
+
+ messages = [
+ ChatMessage(role="user", content="A"),
+ ChatMessage(role="assistant", content="B"),
+ ChatMessage(role="user", content="C"),
+ ]
+ result = _session_messages_to_transcript(messages)
+ entries = self._parse_jsonl(result)
+
+ assert entries[0]["parentUuid"] == ""
+ for i in range(1, len(entries)):
+ assert (
+ entries[i]["parentUuid"] == entries[i - 1]["uuid"]
+ ), f"Entry {i} parentUuid mismatch"
+
+ def test_invalid_tool_call_arguments_use_empty_input(self):
+ """Malformed JSON in tool_call arguments must not raise — use empty dict."""
+ from backend.copilot.model import ChatMessage
+ from backend.copilot.sdk.service import _session_messages_to_transcript
+
+ messages = [
+ ChatMessage(
+ role="assistant",
+ content="",
+ tool_calls=[
+ {
+ "id": "call_bad",
+ "type": "function",
+ "function": {
+ "name": "broken_tool",
+ "arguments": "NOT_VALID_JSON",
+ },
+ }
+ ],
+ ),
+ ]
+ result = _session_messages_to_transcript(messages)
+ entries = self._parse_jsonl(result)
+ assert len(entries) == 1
+ blocks = entries[0]["message"]["content"]
+ tu = next(b for b in blocks if b.get("type") == "tool_use")
+ assert tu["input"] == {}
diff --git a/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py b/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py
index 6d1fb84ab3..7eee1c04e8 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py
@@ -260,13 +260,13 @@ def test_result_error_emits_error_and_finish():
is_error=True,
num_turns=0,
session_id="s1",
- result="API rate limited",
+ result="Invalid API key provided",
)
results = adapter.convert_message(msg)
# No step was open, so no FinishStep — just Error + Finish
assert len(results) == 2
assert isinstance(results[0], StreamError)
- assert "API rate limited" in results[0].errorText
+ assert "Invalid API key provided" in results[0].errorText
assert isinstance(results[1], StreamFinish)
diff --git a/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py b/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py
index 2873ee596d..52a1eff5df 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py
@@ -1023,9 +1023,14 @@ def _make_sdk_patches(
stream_lock_ttl=60,
active_e2b_api_key=None,
use_e2b_sandbox=False,
+ claude_agent_max_transient_retries=1,
+ claude_agent_max_turns=1000,
+ claude_agent_max_budget_usd=100.0,
+ claude_agent_fallback_model=None,
),
),
(f"{_SVC}.upload_transcript", dict(new_callable=AsyncMock)),
+ (f"{_SVC}.get_user_tier", dict(new_callable=AsyncMock, return_value=None)),
]
@@ -1671,3 +1676,203 @@ class TestStreamChatCompletionRetryIntegration:
errors = [e for e in events if isinstance(e, StreamError)]
assert not errors, f"Unexpected StreamError: {errors}"
assert any(isinstance(e, StreamStart) for e in events)
+
+ @pytest.mark.asyncio
+ async def test_handled_stream_error_transient_retries_then_succeeds(self):
+ """_HandledStreamError(code="transient_api_error") triggers backoff retry.
+
+ When ``_run_stream_attempt`` raises ``_HandledStreamError`` with
+ ``code="transient_api_error"`` (i.e. an AssistantMessage with a transient
+ error field arrives mid-stream), the outer loop must:
+ 1. Call ``_next_transient_backoff`` to get the sleep duration.
+ 2. Yield a ``StreamStatus`` message ("Connection interrupted…").
+ 3. Sleep for the backoff duration.
+ 4. Continue the loop and retry the same context-level attempt.
+ 5. NOT yield ``StreamError`` while retries remain.
+
+ This exercises the ``_HandledStreamError`` handler path at
+ ``stream_chat_completion_sdk`` line ~2335.
+ """
+ import contextlib
+
+ from claude_agent_sdk import AssistantMessage, ResultMessage
+
+ from backend.copilot.response_model import (
+ StreamError,
+ StreamStart,
+ StreamStatus,
+ )
+ from backend.copilot.sdk.service import stream_chat_completion_sdk
+
+ session = self._make_session()
+ result_msg = self._make_result_message()
+ call_count = [0]
+
+ def _client_factory(*args, **kwargs):
+ call_count[0] += 1
+ attempt = call_count[0]
+
+ async def _receive():
+ if attempt == 1:
+ # First call: emit AssistantMessage with a transient error field
+ # so _run_stream_attempt detects is_transient_api_error and
+ # raises _HandledStreamError(code="transient_api_error").
+ yield AssistantMessage(
+ content=[],
+ model="claude-sonnet-4-20250514",
+ error="rate_limit",
+ )
+ yield ResultMessage(
+ subtype="error",
+ result="rate limit exceeded (status code 429)",
+ duration_ms=50,
+ duration_api_ms=0,
+ is_error=True,
+ num_turns=0,
+ session_id="test-session-id",
+ )
+ else:
+ yield result_msg
+
+ client = MagicMock()
+ client.receive_response = _receive
+ client.query = AsyncMock()
+ client._transport = MagicMock()
+ client._transport.write = AsyncMock()
+
+ cm = AsyncMock()
+ cm.__aenter__.return_value = client
+ cm.__aexit__.return_value = None
+ return cm
+
+ original_transcript = _build_transcript(
+ [("user", "prior question"), ("assistant", "prior answer")]
+ )
+
+ patches = _make_sdk_patches(
+ session,
+ original_transcript=original_transcript,
+ compacted_transcript=None,
+ client_side_effect=_client_factory,
+ )
+
+ events = []
+ with contextlib.ExitStack() as stack:
+ # Patch asyncio.sleep to avoid actual delays in the test.
+ stack.enter_context(patch(f"{_SVC}.asyncio.sleep", new_callable=AsyncMock))
+ for target, kwargs in patches:
+ stack.enter_context(patch(target, **kwargs))
+ async for event in stream_chat_completion_sdk(
+ session_id="test-session-id",
+ message="hello",
+ is_user_message=True,
+ user_id="test-user",
+ session=session,
+ ):
+ events.append(event)
+
+ # Two SDK client calls: first fails with transient error, second succeeds.
+ assert (
+ call_count[0] == 2
+ ), f"Expected 2 SDK calls (transient retry), got {call_count[0]}"
+ # No StreamError emitted — the retry succeeded.
+ errors = [e for e in events if isinstance(e, StreamError)]
+ assert (
+ not errors
+ ), f"Unexpected StreamError emitted during transient retry: {errors}"
+ # StreamStatus("Connection interrupted…") must have been yielded.
+ status_events = [e for e in events if isinstance(e, StreamStatus)]
+ assert status_events, "Expected StreamStatus retry notification but got none"
+ assert any(
+ "retrying" in (e.message or "").lower()
+ or "interrupted" in (e.message or "").lower()
+ for e in status_events
+ ), f"Expected 'retrying' or 'interrupted' in StreamStatus, got: {[e.message for e in status_events]}"
+ assert any(isinstance(e, StreamStart) for e in events)
+
+ @pytest.mark.asyncio
+ async def test_generic_exception_transient_retry_then_succeeds(self):
+ """Raw Exception("ECONNRESET") from receive_response triggers backoff retry.
+
+ When ``receive_response`` raises a raw ``Exception`` whose string
+ matches a transient pattern (e.g. ECONNRESET), the generic ``except
+ Exception`` handler at ``stream_chat_completion_sdk`` line ~2398 must:
+ 1. Detect ``is_transient_api_error(str(e))`` as True.
+ 2. Call ``_next_transient_backoff`` to get the sleep duration.
+ 3. Yield a ``StreamStatus`` message ("Connection interrupted…").
+ 4. Sleep for the backoff duration.
+ 5. Continue the loop and retry the same context-level attempt.
+ 6. NOT yield ``StreamError`` while retries remain.
+
+ This exercises the generic ``Exception`` handler (ECONNRESET path) at
+ ``stream_chat_completion_sdk`` line ~2398.
+ """
+ import contextlib
+
+ from backend.copilot.response_model import (
+ StreamError,
+ StreamStart,
+ StreamStatus,
+ )
+ from backend.copilot.sdk.service import stream_chat_completion_sdk
+
+ session = self._make_session()
+ result_msg = self._make_result_message()
+ call_count = [0]
+
+ def _client_factory(*args, **kwargs):
+ call_count[0] += 1
+ attempt = call_count[0]
+
+ if attempt == 1:
+ # First call: receive_response raises ECONNRESET immediately
+ return self._make_client_mock_mid_stream_error(
+ error=Exception("ECONNRESET: connection reset by peer"),
+ pre_error_messages=None,
+ )
+ return self._make_client_mock(result_message=result_msg)
+
+ original_transcript = _build_transcript(
+ [("user", "prior question"), ("assistant", "prior answer")]
+ )
+
+ patches = _make_sdk_patches(
+ session,
+ original_transcript=original_transcript,
+ compacted_transcript=None,
+ client_side_effect=_client_factory,
+ )
+
+ events = []
+ with contextlib.ExitStack() as stack:
+ # Patch asyncio.sleep to avoid actual delays in the test.
+ stack.enter_context(patch(f"{_SVC}.asyncio.sleep", new_callable=AsyncMock))
+ for target, kwargs in patches:
+ stack.enter_context(patch(target, **kwargs))
+ async for event in stream_chat_completion_sdk(
+ session_id="test-session-id",
+ message="hello",
+ is_user_message=True,
+ user_id="test-user",
+ session=session,
+ ):
+ events.append(event)
+
+ # Two SDK client calls: first fails with ECONNRESET, second succeeds.
+ assert (
+ call_count[0] == 2
+ ), f"Expected 2 SDK calls (ECONNRESET transient retry), got {call_count[0]}"
+ # No StreamError emitted — the retry succeeded.
+ errors = [e for e in events if isinstance(e, StreamError)]
+ assert (
+ not errors
+ ), f"Unexpected StreamError emitted during ECONNRESET retry: {errors}"
+ # StreamStatus("Connection interrupted…") must have been yielded.
+ status_events = [e for e in events if isinstance(e, StreamStatus)]
+ assert status_events, "Expected StreamStatus retry notification but got none"
+ assert any(
+ "retrying" in (e.message or "").lower()
+ or "interrupted" in (e.message or "").lower()
+ for e in status_events
+ ), f"Expected 'retrying' or 'interrupted' in StreamStatus, got: {[e.message for e in status_events]}"
+ assert any(isinstance(e, StreamStart) for e in events)
diff --git a/autogpt_platform/backend/backend/copilot/sdk/sdk_compat_test.py b/autogpt_platform/backend/backend/copilot/sdk/sdk_compat_test.py
index ea0722c059..45a7cf4434 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/sdk_compat_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/sdk_compat_test.py
@@ -105,6 +105,10 @@ def test_agent_options_accepts_all_our_fields():
"env",
"resume",
"max_buffer_size",
+ "stderr",
+ "fallback_model",
+ "max_turns",
+ "max_budget_usd",
]
sig = inspect.signature(ClaudeAgentOptions)
for field in fields_we_use:
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 8d061fbdce..d3e17f4892 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -5,6 +5,7 @@ import base64
import json
import logging
import os
+import random
import re
import shutil
import sys
@@ -132,6 +133,27 @@ _CIRCUIT_BREAKER_ERROR_MSG = (
# hanging on a search provider that never responds).
_IDLE_TIMEOUT_SECONDS = 10 * 60 # 10 minutes
+# Event types that are ephemeral / cosmetic and must NOT be counted toward
+# ``events_yielded`` in the transient-retry loop. Counting them would prevent
+# the backoff retry from firing because ``_next_transient_backoff`` returns
+# ``None`` when ``events_yielded > 0``.
+_EPHEMERAL_EVENT_TYPES = (
+ StreamHeartbeat,
+ # Compaction UI events are cosmetic and must not block retry — they're
+ # emitted before the SDK query on compacted attempts.
+ StreamStartStep,
+ StreamFinishStep,
+ StreamToolInputStart,
+ StreamToolInputAvailable,
+ StreamToolOutputAvailable,
+ # Transient StreamError and StreamStatus are ephemeral notifications,
+ # not content. Counting them would prevent the backoff retry from
+ # firing because _next_transient_backoff() returns None when
+ # events_yielded > 0.
+ StreamError,
+ StreamStatus,
+)
+
# Patterns that indicate the prompt/request exceeds the model's context limit.
# Matched case-insensitively against the full exception chain.
_PROMPT_TOO_LONG_PATTERNS: tuple[str, ...] = (
@@ -541,17 +563,34 @@ async def _iter_sdk_messages(
pass
+def _normalize_model_name(raw_model: str) -> str:
+ """Normalize a model name for the current routing configuration.
+
+ Applies two transformations shared by both the primary and fallback
+ model resolution paths:
+
+ 1. **Strip provider prefix** — OpenRouter-style names like
+ ``"anthropic/claude-opus-4.6"`` are reduced to ``"claude-opus-4.6"``.
+ 2. **Dot-to-hyphen conversion** — when *not* routing through OpenRouter
+ the direct Anthropic API requires hyphen-separated versions
+ (``"claude-opus-4-6"``), so dots are replaced with hyphens.
+ """
+ model = raw_model
+ if "/" in model:
+ model = model.split("/", 1)[1]
+ # OpenRouter uses dots in versions (claude-opus-4.6) but the direct
+ # Anthropic API requires hyphens (claude-opus-4-6). Only normalise
+ # when NOT routing through OpenRouter.
+ if not config.openrouter_active:
+ model = model.replace(".", "-")
+ return model
+
+
def _resolve_sdk_model() -> str | None:
"""Resolve the model name for the Claude Agent SDK CLI.
Uses `config.claude_agent_model` if set, otherwise derives from
- `config.model` by stripping the OpenRouter provider prefix (e.g.,
- `"anthropic/claude-opus-4.6"` → `"claude-opus-4-6"`).
-
- OpenRouter uses dot-separated versions (`claude-opus-4.6`) while the
- direct Anthropic API uses hyphen-separated versions (`claude-opus-4-6`).
- Normalisation is only applied when the SDK will actually talk to
- Anthropic directly (not through OpenRouter).
+ `config.model` via :func:`_normalize_model_name`.
When `use_claude_code_subscription` is enabled and no explicit
`claude_agent_model` is set, returns `None` so the CLI uses the
@@ -561,15 +600,96 @@ def _resolve_sdk_model() -> str | None:
return config.claude_agent_model
if config.use_claude_code_subscription:
return None
- model = config.model
- if "/" in model:
- model = model.split("/", 1)[1]
- # OpenRouter uses dots in versions (claude-opus-4.6) but the direct
- # Anthropic API requires hyphens (claude-opus-4-6). Only normalise
- # when NOT routing through OpenRouter.
- if not config.openrouter_active:
- model = model.replace(".", "-")
- return model
+ return _normalize_model_name(config.model)
+
+
+def _resolve_fallback_model() -> str | None:
+ """Resolve the fallback model name via :func:`_normalize_model_name`.
+
+ Returns ``None`` when no fallback is configured (empty string).
+ """
+ raw = config.claude_agent_fallback_model
+ if not raw:
+ return None
+ return _normalize_model_name(raw)
+
+
+_MAX_TRANSIENT_BACKOFF_SECONDS = 30
+
+
+def _compute_transient_backoff(attempt: int) -> int:
+ """Return the exponential backoff delay (seconds) for a transient retry.
+
+ ``attempt`` is 1-based: first retry → ~1 s, second → ~2 s, third → ~4 s,
+ …, capped at :data:`_MAX_TRANSIENT_BACKOFF_SECONDS`.
+
+ Full-jitter (``0.5 … 1.0 × base``) is applied to spread retries from
+ multiple tenants hitting the same 429/529 simultaneously (thundering herd).
+ The result is rounded to the nearest integer so the ``StreamStatus``
+ message is human-readable.
+
+ Extracted as a module-level pure function so it can be unit-tested
+ independently of the closure that wraps it inside the retry loop.
+ """
+ base = min(_MAX_TRANSIENT_BACKOFF_SECONDS, 2 ** (attempt - 1))
+ return max(1, round(base * (0.5 + random.random() * 0.5)))
+
+
+def _next_transient_backoff(
+ events_yielded: int,
+ transient_retries: int,
+ max_transient_retries: int,
+) -> tuple[int | None, int]:
+ """Decide whether to retry after a transient error.
+
+ Returns ``(backoff_seconds, updated_retries)``. ``backoff_seconds`` is
+ ``None`` when no retry should be attempted — either content has already
+ been streamed to the frontend (``events_yielded > 0``, retrying would
+ produce duplicates) or the retry budget is exhausted.
+
+ Extracted as a module-level pure function so it can be unit-tested
+ independently of the retry loop.
+ """
+ if events_yielded > 0:
+ return None, transient_retries
+ new_retries = transient_retries + 1
+ if new_retries > max_transient_retries:
+ return None, new_retries
+ return _compute_transient_backoff(new_retries), new_retries
+
+
+async def _do_transient_backoff(
+ backoff: int,
+ state: _RetryState,
+ message_id: str,
+ session_id: str,
+) -> AsyncIterator[StreamStatus]:
+ """Emit a retry notification, sleep, and reset the SDK adapter.
+
+ Yields a single :class:`StreamStatus` so the caller can forward it to
+ the client, then sleeps for *backoff* seconds and resets ``state.adapter``
+ and ``state.usage`` so the next attempt starts clean.
+
+ Extracted from both exception handlers in the retry loop to remove
+ near-identical code duplication.
+ """
+ yield StreamStatus(message=f"Connection interrupted, retrying in {backoff}s…")
+ await asyncio.sleep(backoff)
+ state.adapter = SDKResponseAdapter(message_id=message_id, session_id=session_id)
+ state.usage.reset()
+
+
+def _is_fallback_stderr(line: str) -> bool:
+ """Return True if a CLI stderr line signals fallback-model activation.
+
+ Matches ``"fallback model"`` case-insensitively. Uses the specific
+ two-word phrase rather than just ``"fallback"`` to avoid false positives
+ from unrelated CLI stderr lines (tool retries, cached-result fallbacks).
+
+ Extracted as a module-level pure function so it can be unit-tested
+ without wiring up the full ``_on_stderr`` closure.
+ """
+ return "fallback model" in line.lower()
def _make_sdk_cwd(session_id: str) -> str:
@@ -728,6 +848,65 @@ async def _compress_messages(
return messages, False
+def _session_messages_to_transcript(messages: list[ChatMessage]) -> str:
+ """Convert session ChatMessages to JSONL transcript for ``--resume``.
+
+ Reconstructs proper ``tool_use`` and ``tool_result`` content blocks from
+ :attr:`ChatMessage.tool_calls` and :attr:`ChatMessage.tool_call_id` so the
+ Claude CLI receives full structural context when no previous transcript file
+ is available (e.g. first turn after a storage failure or compaction drop).
+
+ This gives the model the same fidelity as an on-disk session JSONL file —
+ preserving tool call names, IDs, inputs, and *complete* (un-truncated)
+ tool results — rather than the lossy plain-text injection produced by
+ :func:`_format_conversation_context` (which caps tool results at 500 chars
+ and discards structural linkage).
+
+ Args:
+ messages: Prior session messages, typically ``session.messages[:-1]``
+ (all turns except the current user query).
+
+ Returns:
+ A JSONL string suitable for writing to a temp file and passing as
+ ``ClaudeAgentOptions.resume``. Returns an empty string if the input
+ list is empty after filtering compaction entries.
+ """
+ filtered = filter_compaction_messages(messages)
+ if not filtered:
+ return ""
+ builder = TranscriptBuilder()
+ for msg in filtered:
+ if msg.role == "user" and msg.content:
+ builder.append_user(msg.content)
+ elif msg.role == "assistant":
+ blocks: list[dict[str, Any]] = []
+ if msg.content:
+ blocks.append({"type": "text", "text": msg.content})
+ for tc in msg.tool_calls or []:
+ try:
+ tc_input: dict[str, Any] = json.loads(
+ tc.get("function", {}).get("arguments", "{}")
+ )
+ except (json.JSONDecodeError, ValueError):
+ tc_input = {}
+ blocks.append(
+ {
+ "type": "tool_use",
+ "id": tc.get("id", ""),
+ "name": tc.get("function", {}).get("name", ""),
+ "input": tc_input,
+ }
+ )
+ if blocks:
+ builder.append_assistant(blocks)
+ elif msg.role == "tool" and msg.tool_call_id:
+ builder.append_tool_result(
+ tool_use_id=msg.tool_call_id,
+ content=msg.content or "",
+ )
+ return builder.to_jsonl()
+
+
def _format_conversation_context(messages: list[ChatMessage]) -> str | None:
"""Format conversation messages into a context prefix for the user message.
@@ -1059,17 +1238,25 @@ def _dispatch_response(
class _HandledStreamError(Exception):
- """Raised by `_run_stream_attempt` after it has already yielded a
- `StreamError` to the client (e.g. transient API error, circuit breaker).
+ """Raised by `_run_stream_attempt` when an attempt fails and the outer
+ retry loop must roll back session state.
+
+ Two sub-cases:
- This signals the outer retry loop that the attempt failed so it can
- perform session-message rollback and set the `ended_with_stream_error`
- flag, **without** yielding a duplicate `StreamError` to the client.
+ * ``already_yielded=True`` (default) — a ``StreamError`` was already sent
+ to the client inside ``_run_stream_attempt`` (circuit-breaker, idle
+ timeout, etc.). The outer loop must **not** yield another one.
+ * ``already_yielded=False`` — the error is transient and the outer loop
+ will decide whether to retry or surface the error. If retrying it
+ yields a ``StreamStatus("retrying…")``; if exhausted it yields the
+ ``StreamError`` itself so the client sees it only once.
Attributes:
error_msg: The user-facing error message to persist.
code: Machine-readable error code (e.g. ``circuit_breaker_empty_tool_calls``).
retryable: Whether the frontend should offer a retry button.
+ already_yielded: ``True`` when ``StreamError`` was already sent to the
+ client before this exception was raised.
"""
def __init__(
@@ -1078,11 +1265,13 @@ class _HandledStreamError(Exception):
error_msg: str | None = None,
code: str | None = None,
retryable: bool = True,
+ already_yielded: bool = True,
):
super().__init__(message)
self.error_msg = error_msg
self.code = code
self.retryable = retryable
+ self.already_yielded = already_yielded
@dataclass
@@ -1373,15 +1562,12 @@ async def _run_stream_attempt(
)
stream_error_msg = FRIENDLY_TRANSIENT_MSG
stream_error_code = "transient_api_error"
- _append_error_marker(
- ctx.session,
- stream_error_msg,
- retryable=True,
- )
- yield StreamError(
- errorText=stream_error_msg,
- code=stream_error_code,
- )
+ # Do NOT yield StreamError or append error marker here.
+ # The outer retry loop decides: if a retry is available it
+ # yields StreamStatus("retrying…"); if retries are exhausted
+ # it appends the marker and yields StreamError exactly once.
+ # Yielding StreamError before the retry decision causes the
+ # client to display an error that is immediately superseded.
ended_with_stream_error = True
break
@@ -1654,14 +1840,16 @@ async def _run_stream_attempt(
) and not acc.has_appended_assistant:
ctx.session.messages.append(acc.assistant_response)
- # If the attempt ended with a transient error that was already surfaced
- # to the client (StreamError yielded above), raise so the outer retry
- # loop can rollback session messages and set its error flags properly.
+ # Raise so the outer retry loop can rollback session messages.
+ # already_yielded=False for transient_api_error: StreamError was NOT
+ # sent to the client yet (the outer loop does it when retries are
+ # exhausted, avoiding a premature error flash before the retry).
if ended_with_stream_error:
raise _HandledStreamError(
- "Stream error handled — StreamError already yielded",
+ "Stream error handled",
error_msg=stream_error_msg,
code=stream_error_code,
+ already_yielded=(stream_error_code != "transient_api_error"),
)
@@ -1906,12 +2094,48 @@ async def stream_chat_completion_sdk(
logger.warning("%s Transcript downloaded but invalid", log_prefix)
transcript_covers_prefix = False
elif config.claude_agent_use_resume and user_id and len(session.messages) > 1:
- logger.warning(
- "%s No transcript available (%d messages in session)",
- log_prefix,
- len(session.messages),
- )
- transcript_covers_prefix = False
+ # No transcript on disk — try to reconstruct a full JSONL from the
+ # session.messages stored in the DB. This gives the Claude CLI
+ # proper tool_use/tool_result structural context via --resume
+ # instead of the lossy plain-text injection in _build_query_message
+ # (which caps tool results at 500 chars and drops call/result IDs).
+ prior = session.messages[:-1]
+ reconstructed = _session_messages_to_transcript(prior)
+ if reconstructed:
+ rebuilt_resume = await asyncio.to_thread(
+ write_transcript_to_tempfile, reconstructed, session_id, sdk_cwd
+ )
+ if rebuilt_resume:
+ use_resume = True
+ resume_file = rebuilt_resume
+ transcript_msg_count = len(prior)
+ transcript_content = reconstructed
+ transcript_builder.load_previous(
+ reconstructed, log_prefix=log_prefix
+ )
+ transcript_covers_prefix = True
+ logger.info(
+ "%s Reconstructed transcript from %d session messages "
+ "for --resume (no previous transcript file)",
+ log_prefix,
+ len(prior),
+ )
+ else:
+ logger.warning(
+ "%s Transcript reconstruction failed — write_transcript_to_tempfile"
+ " returned None (%d messages)",
+ log_prefix,
+ len(prior),
+ )
+ transcript_covers_prefix = False
+ else:
+ logger.warning(
+ "%s No transcript available and reconstruction produced empty"
+ " output (%d messages in session)",
+ log_prefix,
+ len(session.messages),
+ )
+ transcript_covers_prefix = False
yield StreamStart(messageId=message_id, sessionId=session_id)
@@ -1958,8 +2182,20 @@ async def stream_chat_completion_sdk(
def _on_stderr(line: str) -> None:
"""Log a stderr line emitted by the Claude CLI subprocess."""
+ nonlocal fallback_model_activated_per_attempt
sid = session_id[:12] if session_id else "?"
logger.info("[SDK] [%s] CLI stderr: %s", sid, line.rstrip())
+ # Detect SDK fallback-model activation via the module-level pure
+ # helper so the detection logic can be unit-tested independently.
+ # Sets the per-attempt flag which is preserved across transient
+ # retries so the user notification is never lost.
+ if not fallback_model_activated_per_attempt and _is_fallback_stderr(line):
+ fallback_model_activated_per_attempt = True
+ logger.warning(
+ "[SDK] [%s] Fallback model activated — primary model "
+ "overloaded, switching to fallback",
+ sid,
+ )
sdk_options_kwargs: dict[str, Any] = {
"system_prompt": system_prompt,
@@ -1970,6 +2206,15 @@ async def stream_chat_completion_sdk(
"cwd": sdk_cwd,
"max_buffer_size": config.claude_agent_max_buffer_size,
"stderr": _on_stderr,
+ # --- P0 guardrails ---
+ # fallback_model: SDK auto-retries with this cheaper model on
+ # 529 (overloaded) errors, avoiding user-visible failures.
+ "fallback_model": _resolve_fallback_model(),
+ # max_turns: hard cap on agentic tool-use loops per query to
+ # prevent runaway execution from burning budget.
+ "max_turns": config.claude_agent_max_turns,
+ # max_budget_usd: per-query spend ceiling enforced by the CLI.
+ "max_budget_usd": config.claude_agent_max_budget_usd,
}
if sdk_model:
sdk_options_kwargs["model"] = sdk_model
@@ -2054,8 +2299,17 @@ async def stream_chat_completion_sdk(
# ---------------------------------------------------------------
ended_with_stream_error = False
attempts_exhausted = False
+ transient_exhausted = False
stream_err: Exception | None = None
+ transient_retries = 0
+ max_transient_retries = config.claude_agent_max_transient_retries
+ # Preserved across transient retries so the fallback-model notification
+ # is not lost when a retry resets local per-attempt variables. Reset
+ # only on context-level attempt changes (same guard as transient_retries).
+ fallback_model_activated_per_attempt = False
+ fallback_notified_per_attempt = False
+
state = _RetryState(
options=options,
query_message=query_message,
@@ -2068,7 +2322,21 @@ async def stream_chat_completion_sdk(
usage=_TokenUsage(),
)
- for attempt in range(_MAX_STREAM_ATTEMPTS):
+ attempt = 0
+ _last_reset_attempt = -1
+ while attempt < _MAX_STREAM_ATTEMPTS:
+ # Reset transient retry counter per context-level attempt so
+ # each attempt (original, compacted, no-transcript) gets the
+ # full retry budget for transient errors.
+ # Only reset when the attempt number actually changes —
+ # transient retries `continue` back to the loop top without
+ # incrementing `attempt`, so resetting unconditionally would
+ # create an infinite retry loop.
+ if attempt != _last_reset_attempt:
+ transient_retries = 0
+ fallback_model_activated_per_attempt = False
+ fallback_notified_per_attempt = False
+ _last_reset_attempt = attempt
# Clear any stale stash signal from the previous attempt so
# wait_for_stash() doesn't fire prematurely on a leftover event.
reset_stash_event()
@@ -2123,25 +2391,32 @@ async def stream_chat_completion_sdk(
state.usage.reset()
pre_attempt_msg_count = len(session.messages)
+ # Snapshot transcript builder state — it maintains an
+ # independent _entries list from session.messages, so rolling
+ # back session.messages alone would leave duplicate entries
+ # from the failed attempt in the uploaded transcript.
+ transcript_snap = state.transcript_builder.snapshot()
events_yielded = 0
try:
async for event in _run_stream_attempt(stream_ctx, state):
- if not isinstance(
- event,
- (
- StreamHeartbeat,
- # Compaction UI events are cosmetic and must not
- # block retry — they're emitted before the SDK
- # query on compacted attempts.
- StreamStartStep,
- StreamFinishStep,
- StreamToolInputStart,
- StreamToolInputAvailable,
- StreamToolOutputAvailable,
- ),
- ):
+ if not isinstance(event, _EPHEMERAL_EVENT_TYPES):
events_yielded += 1
+ # Emit a one-time StreamStatus when the SDK switches
+ # to the fallback model (detected via stderr). The flag
+ # is preserved across transient retries (reset only on
+ # context-level attempt change) so the notification is
+ # not lost if the activation occurs during a failed sub-
+ # attempt that later retries successfully.
+ if (
+ fallback_model_activated_per_attempt
+ and not fallback_notified_per_attempt
+ ):
+ fallback_notified_per_attempt = True
+ yield StreamStatus(
+ message="Primary model overloaded — "
+ "using fallback model for this request"
+ )
yield event
break # Stream completed — exit retry loop
except asyncio.CancelledError:
@@ -2158,6 +2433,28 @@ async def stream_chat_completion_sdk(
# session messages and set the error flag — do NOT set
# stream_err so the post-loop code won't emit a
# duplicate StreamError.
+ session.messages = session.messages[:pre_attempt_msg_count]
+ state.transcript_builder.restore(transcript_snap)
+ # Check if this is a transient error we can retry with backoff.
+ # exc.code is the only reliable signal — str(exc) is always the
+ # static "Stream error handled — StreamError already yielded" message.
+ if exc.code == "transient_api_error":
+ backoff, transient_retries = _next_transient_backoff(
+ events_yielded, transient_retries, max_transient_retries
+ )
+ if backoff is not None:
+ logger.warning(
+ "%s Transient error — retrying in %ds (%d/%d)",
+ log_prefix,
+ backoff,
+ transient_retries,
+ max_transient_retries,
+ )
+ async for evt in _do_transient_backoff(
+ backoff, state, message_id, session_id
+ ):
+ yield evt
+ continue # retry the same context-level attempt
logger.warning(
"%s Stream error handled in attempt "
"(attempt %d/%d, code=%s, events_yielded=%d)",
@@ -2167,7 +2464,6 @@ async def stream_chat_completion_sdk(
exc.code or "transient",
events_yielded,
)
- session.messages = session.messages[:pre_attempt_msg_count]
# transcript_builder still contains entries from the aborted
# attempt that no longer match session.messages. Skip upload
# so a future --resume doesn't replay rolled-back content.
@@ -2182,22 +2478,36 @@ async def stream_chat_completion_sdk(
retryable=True,
)
ended_with_stream_error = True
+ # For transient errors the StreamError was deliberately NOT
+ # yielded inside _run_stream_attempt (already_yielded=False)
+ # so the client didn't see a premature error flash. Yield it
+ # now that we know retries are exhausted.
+ # For non-transient errors (circuit breaker, idle timeout)
+ # already_yielded=True — do NOT yield again.
+ if not exc.already_yielded:
+ yield StreamError(
+ errorText=exc.error_msg or FRIENDLY_TRANSIENT_MSG,
+ code=exc.code or "transient_api_error",
+ )
break
except Exception as e:
stream_err = e
is_context_error = _is_prompt_too_long(e)
+ is_transient = is_transient_api_error(str(e))
logger.warning(
"%s Stream error (attempt %d/%d, context_error=%s, "
- "events_yielded=%d): %s",
+ "transient=%s, events_yielded=%d): %s",
log_prefix,
attempt + 1,
_MAX_STREAM_ATTEMPTS,
is_context_error,
+ is_transient,
events_yielded,
stream_err,
exc_info=True,
)
session.messages = session.messages[:pre_attempt_msg_count]
+ state.transcript_builder.restore(transcript_snap)
if events_yielded > 0:
# Events were already sent to the frontend and cannot be
# unsent. Retrying would produce duplicate/inconsistent
@@ -2210,16 +2520,48 @@ async def stream_chat_completion_sdk(
skip_transcript_upload = True
ended_with_stream_error = True
break
+ # Transient API errors (ECONNRESET, 429, 5xx) — retry
+ # with exponential backoff via the shared helper.
+ if is_transient:
+ backoff, transient_retries = _next_transient_backoff(
+ events_yielded, transient_retries, max_transient_retries
+ )
+ if backoff is not None:
+ logger.warning(
+ "%s Transient exception — retrying in %ds (%d/%d)",
+ log_prefix,
+ backoff,
+ transient_retries,
+ max_transient_retries,
+ )
+ async for evt in _do_transient_backoff(
+ backoff, state, message_id, session_id
+ ):
+ yield evt
+ continue # retry same context-level attempt
+ # Retries exhausted — persist retryable marker so the
+ # frontend shows "Try again" after refresh.
+ # Mirrors the _HandledStreamError exhausted-retry path
+ # at line ~2310.
+ transient_exhausted = True
+ skip_transcript_upload = True
+ _append_error_marker(
+ session, FRIENDLY_TRANSIENT_MSG, retryable=True
+ )
+ ended_with_stream_error = True
+ break
+
if not is_context_error:
- # Non-context errors (network, auth, rate-limit) should
- # not trigger compaction — surface the error immediately.
+ # Non-context, non-transient errors (auth, fatal)
+ # should not trigger compaction — surface immediately.
skip_transcript_upload = True
ended_with_stream_error = True
break
+ attempt += 1 # advance to next context-level attempt
continue
else:
- # All retry attempts exhausted (loop ended without break)
- # skip_transcript_upload is already set by _reduce_context
+ # while condition became False — all attempts exhausted without
+ # break. skip_transcript_upload is already set by _reduce_context
# when the transcript was dropped (transcript_lost=True).
ended_with_stream_error = True
attempts_exhausted = True
@@ -2248,25 +2590,24 @@ async def stream_chat_completion_sdk(
yield response
if ended_with_stream_error and stream_err is not None:
- # Use distinct error codes: "all_attempts_exhausted" when all
- # retries were consumed vs "sdk_stream_error" for non-context
- # errors that broke the loop immediately (network, auth, etc.).
+ # Use distinct error codes depending on how the loop ended:
+ # • "all_attempts_exhausted" — context compaction ran out of room
+ # • "transient_api_error" — 429/5xx/ECONNRESET retries exhausted
+ # • "sdk_stream_error" — non-context, non-transient fatal error
safe_err = str(stream_err).replace("\n", " ").replace("\r", "")[:500]
if attempts_exhausted:
error_text = (
"Your conversation is too long. "
"Please start a new chat or clear some history."
)
+ error_code = "all_attempts_exhausted"
+ elif transient_exhausted:
+ error_text = FRIENDLY_TRANSIENT_MSG
+ error_code = "transient_api_error"
else:
error_text = _friendly_error_text(safe_err)
- yield StreamError(
- errorText=error_text,
- code=(
- "all_attempts_exhausted"
- if attempts_exhausted
- else "sdk_stream_error"
- ),
- )
+ error_code = "sdk_stream_error"
+ yield StreamError(errorText=error_text, code=error_code)
# Copy token usage from retry state to outer-scope accumulators
# so the finally block can persist them.
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service_test.py b/autogpt_platform/backend/backend/copilot/sdk/service_test.py
index c15fbc0126..5eb9981c5b 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service_test.py
@@ -10,6 +10,7 @@ import pytest
from .service import (
_is_sdk_disconnect_error,
+ _normalize_model_name,
_prepare_file_attachments,
_resolve_sdk_model,
_safe_close_sdk_client,
@@ -405,6 +406,49 @@ def _clean_config_env(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv(var, raising=False)
+class TestNormalizeModelName:
+ """Tests for _normalize_model_name — shared provider-aware normalization."""
+
+ def test_strips_provider_prefix(self, monkeypatch, _clean_config_env):
+ from backend.copilot import config as cfg_mod
+
+ cfg = cfg_mod.ChatConfig(
+ use_openrouter=False,
+ api_key=None,
+ base_url=None,
+ use_claude_code_subscription=False,
+ )
+ monkeypatch.setattr("backend.copilot.sdk.service.config", cfg)
+ assert _normalize_model_name("anthropic/claude-opus-4.6") == "claude-opus-4-6"
+
+ def test_dots_preserved_for_openrouter(self, monkeypatch, _clean_config_env):
+ from backend.copilot import config as cfg_mod
+
+ cfg = cfg_mod.ChatConfig(
+ use_openrouter=True,
+ api_key="or-key",
+ base_url="https://openrouter.ai/api/v1",
+ use_claude_code_subscription=False,
+ )
+ monkeypatch.setattr("backend.copilot.sdk.service.config", cfg)
+ assert _normalize_model_name("anthropic/claude-opus-4.6") == "claude-opus-4.6"
+
+ def test_no_prefix_no_dots(self, monkeypatch, _clean_config_env):
+ from backend.copilot import config as cfg_mod
+
+ cfg = cfg_mod.ChatConfig(
+ use_openrouter=False,
+ api_key=None,
+ base_url=None,
+ use_claude_code_subscription=False,
+ )
+ monkeypatch.setattr("backend.copilot.sdk.service.config", cfg)
+ assert (
+ _normalize_model_name("claude-sonnet-4-20250514")
+ == "claude-sonnet-4-20250514"
+ )
+
+
class TestResolveSdkModel:
"""Tests for _resolve_sdk_model — model ID resolution for the SDK CLI."""
diff --git a/autogpt_platform/backend/backend/copilot/transcript_builder.py b/autogpt_platform/backend/backend/copilot/transcript_builder.py
index b5f086f802..35abf639b1 100644
--- a/autogpt_platform/backend/backend/copilot/transcript_builder.py
+++ b/autogpt_platform/backend/backend/copilot/transcript_builder.py
@@ -224,6 +224,25 @@ class TranscriptBuilder:
lines = [entry.model_dump_json(exclude_none=True) for entry in self._entries]
return "\n".join(lines) + "\n"
+ def snapshot(self) -> tuple[list[TranscriptEntry], str | None]:
+ """Return a shallow snapshot of the current builder state.
+
+ Use with :meth:`restore` to roll back transcript mutations from a
+ failed stream attempt without accessing private attributes directly.
+
+ Returns a ``(entries_copy, last_uuid)`` tuple. ``entries_copy`` is a
+ new list (shallow copy) so caller mutations don't affect the live state.
+ """
+ return list(self._entries), self._last_uuid
+
+ def restore(self, snap: tuple[list[TranscriptEntry], str | None]) -> None:
+ """Restore builder state from a :meth:`snapshot`.
+
+ Replaces ``_entries`` and ``_last_uuid`` atomically so the builder
+ matches the state at the time the snapshot was taken.
+ """
+ self._entries, self._last_uuid = snap
+
@property
def entry_count(self) -> int:
"""Total number of entries in the complete context."""