AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): retry Anthropic 5xx and overload errors the Claude CLI reports (#15381)
Nicholas Tindle54cea3114d5b
diff --git a/.github/scripts/backend-allowed-skips.json b/.github/scripts/backend-allowed-skips.jsonindex 88edd78a62..f144772edf 100644--- a/.github/scripts/backend-allowed-skips.json+++ b/.github/scripts/backend-allowed-skips.json@@ -108,7 +108,7 @@"backend.blocks.test.test_block.test_block_ids_valid[TwitterUpdateListBlock]","backend.copilot.baseline.service_test.test_baseline_multi_turn","backend.copilot.billing_e2e.thread_billing_test.test_each_turn_is_charged_what_its_provider_billed[sdk-anthropic-dropped-after-tool]",- "backend.copilot.billing_e2e.thread_billing_test.test_each_turn_is_charged_what_its_provider_billed[sdk-anthropic-overloaded-on-resume]",+ "backend.copilot.billing_e2e.thread_billing_test.test_each_turn_is_charged_what_its_provider_billed[sdk-anthropic-overloaded-after-tool]","backend.copilot.graphiti.client_test.TestHyphenInGroupIdRegression.test_build_fulltext_query_escapes_hyphens_in_group_id","backend.copilot.sdk.cli_openrouter_compat_test.test_bare_cli_does_not_send_openrouter_incompatible_features","backend.copilot.sdk.tool_adapter_test.TestBug1DuplicateExecution.test_old_code_duplicates_on_arg_mismatch",diff --git a/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.pyindex 55f9e2c929..553481782d 100644--- a/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py+++ b/autogpt_platform/backend/backend/copilot/billing_e2e/scenarios.py@@ -103,22 +103,23 @@ def _sdk_anthropic_failing()-_CLI_529_NOT_TRANSIENT = (- "The bundled CLI words a 529 as 'API Error: 529 Overloaded. This is a "- "server-side issue...', which is_transient_api_error does not match (it "- "looks for 'status code 529'), so the platform never retries it: the "- "reply is the CLI's error text"+_RETRY_TOKENS = KnownGap(+ check="tokens",+ reason=(+ "A retried turn is charged the failed attempt's spend, but its "+ "cost-log row and session usage keep only the retry's tokens"+ ),)SCENARIOS = [_SDK_ANTHROPIC,- # The tool round's second call is refused. Unmatched as transient (see- # _CLI_529_NOT_TRANSIENT), the CLI's error becomes the reply, and the- # turn still owes the first call.+ # The tool round's second call is refused: the platform relaunches the CLI+ # on the session, and the turn owes the first call and the retry's._sdk_anthropic_failing("sdk-anthropic-overloaded-after-tool",PlannedFault(kind="overloaded", turn=1, request=2),- calls_per_turn=(1, 1, 1),+ calls_per_turn=(1, 2, 1),+ known_gap=_RETRY_TOKENS,),# The connection drops instead, which is transient: the platform stops# reading before the CLI's result and relaunches the CLI on the session.@@ -127,13 +128,7 @@ SCENARIOS = ["sdk-anthropic-dropped-after-tool",PlannedFault(kind="dropped", turn=1, request=2),calls_per_turn=(1, 2, 1),- known_gap=KnownGap(- check="tokens",- reason=(- "A retried turn is charged the failed attempt's spend, but its "- "cost-log row and session usage keep only the retry's tokens"- ),- ),+ known_gap=_RETRY_TOKENS,),# A resumed turn whose first request drops is relaunched before anything# streamed, and charged its one successful call.@@ -146,7 +141,6 @@ SCENARIOS = ["sdk-anthropic-overloaded-on-resume",PlannedFault(kind="overloaded", turn=2, request=1),calls_per_turn=(1, 2, 1),- known_gap=KnownGap(check="path", reason=_CLI_529_NOT_TRANSIENT),),Scenario(id="sdk-openrouter",diff --git a/autogpt_platform/backend/backend/copilot/constants.py b/autogpt_platform/backend/backend/copilot/constants.pyindex 4d20792e17..8163afff53 100644--- a/autogpt_platform/backend/backend/copilot/constants.py+++ b/autogpt_platform/backend/backend/copilot/constants.py@@ -146,3 +146,18 @@ def is_transient_api_error(error_text: str) -> bool:"""Return True if *error_text* matches a known transient Anthropic API error."""lower = error_text.lower()return any(pat.lower() in lower for pat in _TRANSIENT_ERROR_PATTERNS)+++# ``AssistantMessage.error`` categories the CLI gives an API error worth+# retrying: a 5xx or overload, and a 429. The category is set only on a failed+# API call, so unlike a phrase it cannot match application text, and it does+# not move with the CLI's wording: 2.1.284 reports a 529 as "API Error: 529+# Overloaded. This is a server-side issue...", with no status-code phrase for+# the patterns above to find.+_TRANSIENT_ERROR_CATEGORIES = frozenset({"server_error", "rate_limit"})+++def is_transient_assistant_error(category: str, content: str) -> bool:+ """Return True if an API error the CLI reported in an ``AssistantMessage``+ is transient, by its category or by the text it carries."""+ return category in _TRANSIENT_ERROR_CATEGORIES or is_transient_api_error(content)diff --git a/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py b/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.pyindex fce5cf920c..d8d169ea8f 100644--- a/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/p0_guardrails_test.py@@ -6,7 +6,10 @@ import pytestfrom pydantic import ValidationErrorfrom backend.copilot.config import ChatConfig-from backend.copilot.constants import is_transient_api_error+from backend.copilot.constants import (+ is_transient_api_error,+ is_transient_assistant_error,+)from backend.copilot.model import ChatSessionfrom backend.copilot.response_model import StreamCompactionProgressfrom backend.copilot.sdk.compaction import CompactionTracker@@ -431,6 +434,35 @@ class TestIsTransientApiError:assert is_transient_api_error("econnreset")+# The bundled CLI's (2.1.284) wording for API errors it gives up on.+_CLI_529 = (+ "API Error: 529 Overloaded. This is a server-side issue, usually temporary "+ "— try again in a moment."+)+_CLI_AT_CAPACITY = (+ "API Error: Repeated 529 Overloaded errors. The API is at capacity — this "+ "is usually temporary. Try again in a moment."+)+++class TestIsTransientAssistantError:+ """An API error the CLI reports in an AssistantMessage is transient by its+ category, whatever the CLI's current wording says."""++ @pytest.mark.parametrize("content", [_CLI_529, _CLI_AT_CAPACITY, ""])+ @pytest.mark.parametrize("category", ["server_error", "rate_limit"])+ def test_retryable_categories(self, category: str, content: str):+ assert is_transient_assistant_error(category, content)++ @pytest.mark.parametrize(+ "category",+ ["authentication_failed", "billing_error", "invalid_request", "unknown"],+ )+ def test_other_categories_need_a_transient_text(self, category: str):+ assert not is_transient_assistant_error(category, "Prompt is too long")+ assert is_transient_assistant_error(category, "upstream status code 503")++# ---------------------------------------------------------------------------# _HandledStreamError.already_yielded contract# ---------------------------------------------------------------------------diff --git a/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py b/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.pyindex 9d1d555f5a..d69beeec5a 100644--- a/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/retry_scenarios_test.py@@ -1741,7 +1741,23 @@ class TestStreamChatCompletionRetryIntegration:assert any(isinstance(e, StreamStart) for e in events)@pytest.mark.asyncio- async def test_handled_stream_error_transient_retries_then_succeeds(self):+ @pytest.mark.parametrize(+ "error, text",+ [+ ("rate_limit", "rate limit exceeded (status code 429)"),+ # How the bundled CLI (2.1.284) words a 529: no status-code+ # phrase for the text patterns to find, only the category.+ (+ "server_error",+ "API Error: 529 Overloaded. This is a server-side issue, "+ "usually temporary — try again in a moment.",+ ),+ ],+ ids=["rate_limit", "server_error"],+ )+ async def test_handled_stream_error_transient_retries_then_succeeds(+ self, error: str, text: str+ ):"""_HandledStreamError(code="transient_api_error") triggers backoff retry.When ``_run_stream_attempt`` raises ``_HandledStreamError`` with@@ -1758,7 +1774,7 @@ class TestStreamChatCompletionRetryIntegration:"""import contextlib- from claude_agent_sdk import AssistantMessage, ResultMessage+ from claude_agent_sdk import AssistantMessage, ResultMessage, TextBlockfrom backend.copilot.response_model import (StreamError,@@ -1778,16 +1794,16 @@ class TestStreamChatCompletionRetryIntegration:async def _receive():if attempt == 1:# First call: emit AssistantMessage with a transient error field- # so _run_stream_attempt detects is_transient_api_error and+ # so _run_stream_attempt detects is_transient_assistant_error and# raises _HandledStreamError(code="transient_api_error").yield AssistantMessage(- content=[],+ content=[TextBlock(text=text)],model="claude-sonnet-4-6",- error="rate_limit",+ error=error,)yield ResultMessage(subtype="error",- result="rate limit exceeded (status code 429)",+ result=text,duration_ms=50,duration_api_ms=0,is_error=True,diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex 14958d410e..39a80ced09 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -99,6 +99,7 @@ from ..constants import (STREAM_INCOMPLETE_MARKER,STREAM_LOCK_PREFIX,is_transient_api_error,+ is_transient_assistant_error,)from ..session_cleanup import prune_orphan_tool_callsfrom ..context import encode_cwd_for_cli, get_workspace_manager@@ -597,14 +598,12 @@ async def _consume_sdk_until_done()raise RuntimeError("Prompt is too long")- # Intercept transient API errors (socket closed,- # ECONNRESET) — replace the raw message with a- # user-friendly error text and use the retryable- # error prefix so the frontend shows a retry button.- # Check both the error field and content for patterns.- if is_transient_api_error(error_text) or is_transient_api_error(- error_preview- ):+ # Intercept transient API errors (5xx, overloaded, 429,+ # socket closed, ECONNRESET) — replace the raw message with+ # a user-friendly error text and use the retryable error+ # prefix so the frontend shows a retry button. The error+ # field carries the CLI's category; the content its text.+ if is_transient_assistant_error(category=error_text, content=error_preview):logger.warning("%s Transient Anthropic API error detected, ""suppressing raw error text",