Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(platform): don't show users a provider's raw out-of-credits error (#14882)

Bentlyfecf5c987b46
diff --git a/autogpt_platform/backend/backend/blocks/llm.py b/autogpt_platform/backend/backend/blocks/llm.py
index de94536be0..fb79a4d3c5 100644
--- a/autogpt_platform/backend/backend/blocks/llm.py
+++ b/autogpt_platform/backend/backend/blocks/llm.py
@@ -43,6 +43,7 @@ from backend.data.model import (
NodeExecutionStats,
SchemaField,
)
+from backend.integrations.credentials_store import is_system_credential
from backend.integrations.providers import ProviderName, provider_key
from backend.util import json
@@ -52,6 +53,11 @@ from backend.util import json
# them here so existing ``from backend.blocks.llm import
# ToolContentBlock`` imports keep working.
from backend.util.llm.conversions import ToolCall, ToolContentBlock
+from backend.util.llm.provider_billing import (
+ ProviderUnavailableError,
+ is_provider_out_of_credits,
+ report_provider_out_of_credits,
+)
from backend.util.llm.providers import timeout_error
from backend.util.llm.saturation import track_llm_call
from backend.util.logging import TruncatedLogger
@@ -313,6 +319,29 @@ async def llm_call(
if isinstance(e, asyncio.TimeoutError):
raise _block_timeout_error(llm_model, max_tokens) from e
raise
+ except Exception as e:
+ if is_provider_out_of_credits(e) and _is_platform_credential(credentials):
+ ctx = execution_context
+ report_provider_out_of_credits(
+ provider=provider_key(llm_model.metadata.provider),
+ model=llm_model.value,
+ surface="block",
+ error=e,
+ user_id=ctx.user_id if ctx else None,
+ graph_exec_id=ctx.graph_exec_id if ctx else None,
+ node_exec_id=ctx.node_exec_id if ctx else None,
+ )
+ raise ProviderUnavailableError() from e
+ raise
+
+
+def _is_platform_credential(credentials: APIKeyCredentials | None) -> bool:
+ """A key the platform pays for, as opposed to one the user brought.
+
+ A user's own key running dry is theirs to top up, so the provider's
+ wording is exactly what they need and is left alone.
+ """
+ return credentials is not None and is_system_credential(credentials.id)
async def _llm_call(
@@ -779,6 +808,11 @@ class AIStructuredResponseGeneratorBlock(AIBlockBase):
yield "prompt", self.prompt
return
except Exception as e:
+ if isinstance(e, ProviderUnavailableError):
+ # Already alerted at the llm_call seam, and an empty
+ # account stays empty however many times we ask.
+ error_feedback_message = str(e)
+ break
is_user_error = (
isinstance(e, (anthropic.APIStatusError, openai.APIStatusError))
and e.status_code in USER_ERROR_STATUS_CODES
diff --git a/autogpt_platform/backend/backend/blocks/test/test_llm_provider_out_of_credits.py b/autogpt_platform/backend/backend/blocks/test/test_llm_provider_out_of_credits.py
new file mode 100644
index 0000000000..9273615c4e
--- /dev/null
+++ b/autogpt_platform/backend/backend/blocks/test/test_llm_provider_out_of_credits.py
@@ -0,0 +1,144 @@
+"""A platform-owned provider account running dry must not reach the user raw.
+
+OpenRouter answers an empty account with a 402 telling the reader to buy
+credits on openrouter.ai. On a platform key that bill is ours, so the block
+run shows one plain sentence and we get paged; on the user's own key the
+provider's wording is exactly what they need, so it is left alone.
+"""
+
+import logging
+from typing import cast
+from unittest.mock import AsyncMock, patch
+
+import httpx
+import openai
+import pytest
+
+import backend.blocks.llm as llm
+from backend.integrations.credentials_store import open_router_credentials
+
+_OPENROUTER_402_MESSAGE = (
+ "Insufficient credits. This account never purchased credits. Make sure "
+ "your key is on the correct account or org, and if so, purchase more at "
+ "https://openrouter.ai/settings/credits"
+)
+
+
+def _openrouter_402() -> openai.APIStatusError:
+ body = {"error": {"message": _OPENROUTER_402_MESSAGE, "code": 402}}
+ response = httpx.Response(
+ 402,
+ json=body,
+ request=httpx.Request("POST", "https://openrouter.ai/api/v1/chat"),
+ )
+ return openai.APIStatusError(
+ f"Error code: 402 - {body}", response=response, body=body["error"]
+ )
+
+
+def _user_key() -> llm.APIKeyCredentials:
+ return open_router_credentials.model_copy(update={"id": "user-owned-key"})
+
+
+def _input() -> llm.AIStructuredResponseGeneratorBlock.Input:
+ return llm.AIStructuredResponseGeneratorBlock.Input(
+ prompt="Test",
+ expected_format={"key": "desc"},
+ model=llm.LLMModel("openai/gpt-oss-120b"),
+ credentials=cast(llm.AICredentials, llm.TEST_CREDENTIALS_INPUT),
+ retry=3,
+ )
+
+
+async def test_platform_key_402_shows_platform_message_and_alerts(caplog):
+ inner = AsyncMock(side_effect=_openrouter_402())
+ with (
+ patch.object(llm, "_llm_call", new=inner),
+ caplog.at_level(logging.ERROR),
+ pytest.raises(RuntimeError) as exc_info,
+ ):
+ async for _ in llm.AIStructuredResponseGeneratorBlock().run(
+ _input(), credentials=open_router_credentials
+ ):
+ pass
+
+ message = str(exc_info.value)
+ assert "temporarily unavailable" in message
+ assert "openrouter" not in message.lower()
+ assert "credits" not in message.lower()
+ # An empty account stays empty: one call, not the whole retry budget.
+ assert inner.await_count == 1
+ alerts = [
+ r
+ for r in caplog.records
+ if r.getMessage() == "LLM provider account is out of credits"
+ ]
+ assert len(alerts) == 1
+ assert alerts[0].levelno == logging.ERROR
+ fields = alerts[0].__dict__["json_fields"]
+ assert fields["provider"] == "open_router"
+ assert fields["model"] == "openai/gpt-oss-120b"
+ assert fields["surface"] == "block"
+
+
+async def test_user_key_402_keeps_the_providers_wording(caplog):
+ inner = AsyncMock(side_effect=_openrouter_402())
+ with (
+ patch.object(llm, "_llm_call", new=inner),
+ caplog.at_level(logging.ERROR),
+ pytest.raises(RuntimeError) as exc_info,
+ ):
+ async for _ in llm.AIStructuredResponseGeneratorBlock().run(
+ _input(), credentials=_user_key()
+ ):
+ pass
+
+ assert "openrouter.ai/settings/credits" in str(exc_info.value)
+ assert not [
+ r
+ for r in caplog.records
+ if r.getMessage() == "LLM provider account is out of credits"
+ ]
+
+
+async def test_llm_call_raises_platform_error_for_orchestrator_callers():
+ """Callers that use ``llm_call`` directly (orchestrator, AI condition)
+ surface ``str(exc)`` as the block error, so the seam itself must clean it.
+ """
+ with (
+ patch.object(llm, "_llm_call", new=AsyncMock(side_effect=_openrouter_402())),
+ pytest.raises(RuntimeError) as exc_info,
+ ):
+ await llm.llm_call(
+ credentials=open_router_credentials,
+ llm_model=llm.LLMModel("openai/gpt-oss-120b"),
+ prompt=[{"role": "user", "content": "hi"}],
+ max_tokens=10,
+ )
+
+ assert "temporarily unavailable" in str(exc_info.value)
+ assert isinstance(exc_info.value.__cause__, openai.APIStatusError)
+
+
+async def test_non_billing_errors_pass_through_untouched():
+ boom = openai.APIStatusError(
+ "Error code: 500 - upstream",
+ response=httpx.Response(
+ 500, request=httpx.Request("POST", "https://openrouter.ai/api/v1/chat")
+ ),
+ body=None,
+ )
+ with (
+ patch.object(llm, "_llm_call", new=AsyncMock(side_effect=boom)),
+ pytest.raises(openai.APIStatusError),
+ ):
+ await llm.llm_call(
+ credentials=open_router_credentials,
+ llm_model=llm.LLMModel("openai/gpt-oss-120b"),
+ prompt=[{"role": "user", "content": "hi"}],
+ max_tokens=10,
+ )
diff --git a/autogpt_platform/backend/backend/copilot/baseline/provider_out_of_credits_test.py b/autogpt_platform/backend/backend/copilot/baseline/provider_out_of_credits_test.py
new file mode 100644
index 0000000000..106f944086
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/baseline/provider_out_of_credits_test.py
@@ -0,0 +1,109 @@
+"""An empty platform OpenRouter account must not reach the chat as raw text.
+
+OpenRouter answers with a 402 that tells the reader to buy credits on
+openrouter.ai. On the platform route that bill is ours, so the user sees one
+plain sentence, the turn is retryable, and nothing offers to "switch
+connection" as if it were their own limit.
+"""
+
+from collections.abc import Iterator
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import openai
+import pytest
+
+from backend.copilot.baseline import service
+from backend.copilot.context import set_execution_context
+from backend.copilot.model import ChatMessage, ChatSession
+from backend.copilot.model_router import ResolvedModel
+from backend.copilot.response_model import StreamError, StreamProviderFailure
+
+_OPENROUTER_402_BODY = {
+ "message": "Insufficient credits. This account never purchased credits. "
+ "Make sure your key is on the correct account or org, and if so, purchase "
+ "more at https://openrouter.ai/settings/credits",
+ "code": 402,
+}
+
+
+def _openrouter_402() -> openai.APIStatusError:
+ response = httpx.Response(
+ 402, request=httpx.Request("POST", "https://openrouter.ai/api/v1/chat")
+ )
+ return openai.APIStatusError(
+ f"Error code: 402 - {{'error': {_OPENROUTER_402_BODY}}}",
+ response=response,
+ body=_OPENROUTER_402_BODY,
+ )
+
+
+def baseline_io(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[ChatSession]]:
+ persisted: list[ChatSession] = []
+
+ async def save(session: ChatSession) -> ChatSession:
+ persisted.append(session.model_copy(deep=True))
+ return session
+
+ monkeypatch.setattr(
+ service,
+ "config",
+ service.config.model_copy(
+ update={"use_e2b_sandbox": False, "use_local": False}
+ ),
+ )
+ for name, value in {
+ "drain_pending_safe": [],
+ "resolve_model_route": ResolvedModel(
+ model="anthropic/claude-sonnet-4-6", source="env"
+ ),
+ "_build_system_prompt": ("System prompt", None),
+ "is_enabled_for_user": False,
+ "is_feature_enabled": False,
+ "persist_and_record_usage": None,
+ }.items():
+ monkeypatch.setattr(service, name, AsyncMock(return_value=value))
+ monkeypatch.setattr(service, "_get_main_client", MagicMock())
+ monkeypatch.setattr(service, "upsert_chat_session", AsyncMock(side_effect=save))
+ monkeypatch.setattr(
+ service, "call_provider_stream", AsyncMock(side_effect=_openrouter_402())
+ )
+ try:
+ yield persisted
+ finally:
+ set_execution_context(None, None)
+
+
+async def _run_turn() -> tuple[list[object], ChatSession]:
+ session = ChatSession.new("user-1", dry_run=False)
+ session.messages.append(ChatMessage(role="user", content="hello"))
+ events = [
+ event
+ async for event in service.stream_chat_completion_baseline(
+ session.session_id,
+ user_id="user-1",
+ session=session,
+ is_user_message=False,
+ )
+ ]
+ return events, session
+
+
+async def test_platform_402_shows_the_platform_message(
+ baseline_io: list[ChatSession],
+) -> None:
+ events, session = await _run_turn()
+
+ [error] = [e for e in events if isinstance(e, StreamError)]
+ assert error.code == "provider_unavailable"
+ assert "temporarily unavailable" in error.errorText
+ assert "openrouter" not in error.errorText.lower()
+ # Not the user's limit, so no "switch connection" envelope.
+ assert not [e for e in events if isinstance(e, StreamProviderFailure)]
+
+ # The persisted marker is what a reload renders.
+ marker = session.messages[-1].content or ""
+ assert "temporarily unavailable" in marker
+ assert "openrouter" not in marker.lower()
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py
index a6b9ba79f8..5cc7a98dfa 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py
@@ -167,6 +167,12 @@ from backend.copilot.transcript_builder import TranscriptBuilder
from backend.util import json as util_json
from backend.util.exceptions import NotFoundError
from backend.util.feature_flag import Flag, is_feature_enabled
+from backend.util.llm.provider_billing import (
+ PROVIDER_UNAVAILABLE_CODE,
+ PROVIDER_UNAVAILABLE_MESSAGE,
+ is_provider_out_of_credits,
+ report_provider_out_of_credits,
+)
from backend.util.llm.providers import call_provider_stream
from backend.util.prompt import (
compress_context,
@@ -1462,6 +1468,15 @@ def _humanize_baseline_error(e: Exception) -> str:
return str(e) or type(e).__name__
+def _is_platform_out_of_credits(e: Exception, auth_provider: str | None) -> bool:
+ """An out-of-credits refusal on the account the platform pays for.
+
+ A linked subscription (Codex) running out is the user's own limit and
+ keeps its typed envelope; only the platform route is our outage.
+ """
+ return auth_provider in (None, "platform") and is_provider_out_of_credits(e)
+
+
def should_upload_transcript(user_id: str | None, upload_safe: bool) -> bool:
"""Return ``True`` when the caller should upload the final transcript.
@@ -2651,8 +2666,24 @@ async def stream_chat_completion_baseline(
state.assistant_text += fallback_text
except Exception as e:
_stream_error = True
- error_msg = _humanize_baseline_error(e)
+ out_of_credits = _is_platform_out_of_credits(
+ e, session.metadata.llm_auth_provider if session else None
+ )
+ error_msg = (
+ PROVIDER_UNAVAILABLE_MESSAGE
+ if out_of_credits
+ else _humanize_baseline_error(e)
+ )
logger.error("[Baseline] Streaming error: %s", error_msg, exc_info=True)
+ if out_of_credits:
+ report_provider_out_of_credits(
+ provider=config.effective_transport,
+ model=active_model,
+ surface="copilot_baseline",
+ error=e,
+ session_id=session_id,
+ user_id=user_id,
+ )
# Drain any queued tail events (reasoning/text close + finish step)
# that ``_baseline_llm_caller``'s finally block pushed before the
# sentinel arrived — without this the frontend would be missing the
@@ -2661,11 +2692,17 @@ async def stream_chat_completion_baseline(
evt = state.pending_events.get_nowait()
if evt is not None:
yield evt
- failure = classify_provider_failure(
- e,
- auth_provider=session.metadata.llm_auth_provider if session else None,
- credential_id=session.metadata.llm_credential_id if session else None,
- message=error_msg,
+ # An empty platform account is not the user's limit, so it must not
+ # reach the "switch connection / wait for your limit" envelope.
+ failure = (
+ None
+ if out_of_credits
+ else classify_provider_failure(
+ e,
+ auth_provider=(session.metadata.llm_auth_provider if session else None),
+ credential_id=session.metadata.llm_credential_id if session else None,
+ message=error_msg,
+ )
)
# Written before the error is yielded, and deliberately not in
# ``finally``. The consumer breaks out of its loop the moment it sees
@@ -2694,6 +2731,8 @@ async def stream_chat_completion_baseline(
# in hand by the time the turn is reported failed.
yield StreamProviderFailure(failure=failure.as_part())
yield StreamError(errorText=error_msg, code=failure.kind.value)
+ elif out_of_credits:
+ yield StreamError(errorText=error_msg, code=PROVIDER_UNAVAILABLE_CODE)
else:
yield StreamError(errorText=error_msg, code="baseline_error")
finally:
diff --git a/autogpt_platform/backend/backend/copilot/sdk/provider_out_of_credits_test.py b/autogpt_platform/backend/backend/copilot/sdk/provider_out_of_credits_test.py
new file mode 100644
index 0000000000..0b63cea050
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/sdk/provider_out_of_credits_test.py
@@ -0,0 +1,359 @@
+"""The CLI reporting an empty platform provider account must not reach the chat raw.
+
+The Claude CLI turns a provider billing refusal into a synthetic
+``AssistantMessage`` whose text is the provider's own wording ("Credit
+balance is too low", OpenRouter's "purchase more at openrouter.ai..."). Left
+alone, the adapter streams that as the assistant's reply.
+"""
+
+import contextlib
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from claude_agent_sdk import AssistantMessage, ResultError, ResultMessage, TextBlock
+
+from backend.copilot.model import ChatMessage, ChatSession
+from backend.copilot.response_model import StreamError, StreamTextDelta
+from backend.copilot.sdk.service import (
+ _classify_final_failure,
+ _FinalFailure,
+ _InterruptedAttempt,
+ stream_chat_completion_sdk,
+)
+from backend.util.llm.provider_billing import PROVIDER_UNAVAILABLE_MESSAGE
+
+from .conftest import build_test_transcript
+from .retry_scenarios_test import _make_sdk_patches
+
+_OPENROUTER_402 = (
+ 'API Error: 402 {"error":{"message":"Insufficient credits. Purchase more at '
+ 'https://openrouter.ai/settings/credits","code":402}}'
+)
+
+
+def _session() -> ChatSession:
+ return ChatSession(
+ session_id="test-session-id",
+ user_id="test-user",
+ usage=[],
+ started_at=datetime.now(UTC),
+ updated_at=datetime.now(UTC),
+ messages=[ChatMessage(role="user", content="hello")],
+ )
+
+
+def _result(
+ text: str, usage: dict | None = None, cost_usd: float | None = None
+) -> ResultMessage:
+ return ResultMessage(
+ subtype="success",
+ result=text,
+ duration_ms=100,
+ duration_api_ms=0,
+ is_error=True,
+ num_turns=1,
+ session_id="test-session-id",
+ total_cost_usd=cost_usd,
+ usage=usage,
+ )
+
+
+async def _run_turn(messages: list):
+ session = _session()
+ attempts = [0]
+
+ def _client_factory(*args, **kwargs):
+ attempts[0] += 1
+
+ async def _receive():
+ for message in messages:
+ if isinstance(message, BaseException):
+ raise message
+ yield message
+
+ client = MagicMock()
+ client._transport = MagicMock()
+ client._transport.write = AsyncMock()
+ client.query = AsyncMock()
+ client.receive_response = _receive
+ cm = AsyncMock()
+ cm.__aenter__.return_value = client
+ cm.__aexit__.return_value = None
+ return cm
+
+ patches = _make_sdk_patches(
+ session,
+ original_transcript=build_test_transcript(
+ [("user", "prior question"), ("assistant", "prior answer")]
+ ),
+ compacted_transcript=None,
+ client_side_effect=_client_factory,
+ )
+ events = []
+ with contextlib.ExitStack() as stack:
+ stack.enter_context(
+ patch("backend.copilot.sdk.service.asyncio.sleep", new_callable=AsyncMock)
+ )
+ for target, kwargs in patches:
+ stack.enter_context(patch(target, **kwargs))
+ async for event in stream_chat_completion_sdk(
+ session_id="test-session-id",
+ message="hello",
+ is_user_message=True,
+ user_id="test-user",
+ session=session,
+ ):
+ events.append(event)
+ return events, attempts[0], session
+
+
+ "refusal,result_text",
+ [
+ pytest.param(
+ AssistantMessage(
+ content=[TextBlock(text="Credit balance is too low")],
+ model="<synthetic>",
+ error="billing_error",
+ ),
+ "Credit balance is too low",
+ id="anthropic-billing-error",
+ ),
+ pytest.param(
+ AssistantMessage(
+ content=[TextBlock(text=_OPENROUTER_402)],
+ model="<synthetic>",
+ error="unknown",
+ ),
+ _OPENROUTER_402,
+ id="openrouter-402",
+ ),
+ ],
+)
+async def test_billing_refusal_shows_platform_message(refusal, result_text):
+ events, attempts, session = await _run_turn([refusal, _result(result_text)])
+
+ streamed_text = "".join(
+ e.delta for e in events if isinstance(e, StreamTextDelta)
+ ).lower()
+ assert "credit" not in streamed_text
+ assert "openrouter" not in streamed_text
+
+ errors = [e for e in events if isinstance(e, StreamError)]
+ assert len(errors) == 1
+ assert errors[0].code == "provider_unavailable"
+ assert "temporarily unavailable" in errors[0].errorText
+ assert "openrouter" not in errors[0].errorText.lower()
+ assert attempts == 1
+
+ # The row a reload renders carries the same message, not the raw text.
+ marker = session.messages[-1].content or ""
+ assert "temporarily unavailable" in marker
+ assert "openrouter" not in marker.lower()
+
+
+_ROUNDS_USAGE = {
+ "input_tokens": 1200,
+ "output_tokens": 300,
+ "cache_read_input_tokens": 50,
+ "cache_creation_input_tokens": 10,
+}
+
+
+ "messages",
+ [
+ pytest.param(
+ [
+ AssistantMessage(
+ content=[TextBlock(text="Let me look that up.")],
+ model="claude-sonnet-4-6",
+ ),
+ AssistantMessage(
+ content=[TextBlock(text="Credit balance is too low")],
+ model="<synthetic>",
+ error="billing_error",
+ ),
+ _result("Credit balance is too low", _ROUNDS_USAGE, 0.042),
+ ],
+ id="refusal-after-a-paid-round",
+ ),
+ pytest.param(
+ [_result(_OPENROUTER_402, _ROUNDS_USAGE, 0.042)],
+ id="refusal-is-the-result",
+ ),
+ ],
+)
+async def test_billing_refusal_still_records_the_turns_usage(messages):
+ # Rounds the provider already served in this turn are billed on the
+ # final ResultMessage, which must still be read after the refusal.
+ with patch(
+ "backend.copilot.sdk.service.persist_and_record_usage",
+ new_callable=AsyncMock,
+ ) as persist:
+ events, _, _ = await _run_turn(messages)
+
+ errors = [e for e in events if isinstance(e, StreamError)]
+ assert [e.code for e in errors] == ["provider_unavailable"]
+ persist.assert_awaited_once()
+ usage = persist.await_args.kwargs
+ assert usage["prompt_tokens"] == 1200
+ assert usage["completion_tokens"] == 300
+ assert usage["cache_read_tokens"] == 50
+ assert usage["cache_creation_tokens"] == 10
+ assert usage["cost_usd"] == 0.042
+
+
+def _raised_402() -> ResultError:
+ # How the SDK raises the CLI's error result once the CLI exits non-zero.
+ return ResultError(
+ f"Claude Code returned an error result: {_OPENROUTER_402}",
+ data={
+ "type": "result",
+ "subtype": "success",
+ "is_error": True,
+ "result": _OPENROUTER_402,
+ "api_error_status": 402,
+ "terminal_reason": "api_error",
+ },
+ exit_code=1,
+ )
+
+
+def test_raised_billing_error_maps_to_platform_message():
+ failure = _classify_final_failure(
+ _InterruptedAttempt(),
+ attempts_exhausted=False,
+ transient_exhausted=False,
+ stream_err=_raised_402(),
+ )
+ assert failure == _FinalFailure(
+ display_msg=PROVIDER_UNAVAILABLE_MESSAGE,
+ code="provider_unavailable",
+ retryable=True,
+ )
+
+
+def _raised_402_quoting_a_rate_limit() -> ResultError:
+ # A billing refusal whose text also matches a transient pattern, so the
+ # turn retries it as transient until the retries run out.
+ text = f"{_OPENROUTER_402} (upstream: rate limit)"
+ return ResultError(
+ f"Claude Code returned an error result: {text}",
+ data={
+ "type": "result",
+ "subtype": "success",
+ "is_error": True,
+ "result": text,
+ "api_error_status": 402,
+ "terminal_reason": "api_error",
+ },
+ exit_code=1,
+ )
+
+
+ "attempts_exhausted,transient_exhausted",
+ [
+ pytest.param(False, True, id="transient-retries-exhausted"),
+ pytest.param(True, False, id="context-attempts-exhausted"),
+ ],
+)
+def test_billing_refusal_outranks_an_exhausted_retry_verdict(
+ attempts_exhausted, transient_exhausted
+):
+ failure = _classify_final_failure(
+ _InterruptedAttempt(),
+ attempts_exhausted=attempts_exhausted,
+ transient_exhausted=transient_exhausted,
+ stream_err=_raised_402_quoting_a_rate_limit(),
+ )
+ assert failure == _FinalFailure(
+ display_msg=PROVIDER_UNAVAILABLE_MESSAGE,
+ code="provider_unavailable",
+ retryable=True,
+ )
+
+
+def test_codex_billing_refusal_after_transient_retries_stays_transient():
+ failure = _classify_final_failure(
+ _InterruptedAttempt(),
+ attempts_exhausted=False,
+ transient_exhausted=True,
+ stream_err=_raised_402_quoting_a_rate_limit(),
+ platform_route=False,
+ )
+ assert failure is not None
+ assert failure.code == "transient_api_error"
+
+
+async def test_raised_billing_refusal_retried_as_transient_ends_as_billing():
+ with patch("backend.copilot.sdk.service.report_provider_out_of_credits") as report:
+ events, attempts, session = await _run_turn(
+ [_raised_402_quoting_a_rate_limit()]
+ )
+
+ assert attempts == 2 # the one transient retry the test config allows
+ errors = [e for e in events if isinstance(e, StreamError)]
+ assert [e.code for e in errors] == ["provider_unavailable"]
+ assert "openrouter" not in errors[0].errorText.lower()
+ report.assert_called_once()
+ marker = session.messages[-1].content or ""
+ assert "temporarily unavailable" in marker
+
+
+def test_codex_billing_error_keeps_the_providers_wording():
+ # A linked ChatGPT subscription running out is the user's own limit.
+ failure = _classify_final_failure(
+ _InterruptedAttempt(),
+ attempts_exhausted=False,
+ transient_exhausted=False,
+ stream_err=_raised_402(),
+ platform_route=False,
+ )
+ assert failure is not None
+ assert failure.code == "sdk_stream_error"
+ assert "openrouter.ai" in failure.display_msg
+
+
+ "stream_err",
+ [
+ pytest.param(
+ Exception(f"Command failed: {_OPENROUTER_402}"), id="bare-exception"
+ ),
+ pytest.param(
+ ValueError(
+ "Tool input rejected: user pasted 'visit "
+ "openrouter.ai/settings/credits, requires more credits'"
+ ),
+ id="exception-quoting-user-text",
+ ),
+ pytest.param(
+ ResultError(
+ "Claude Code returned an error result: tool failed",
+ data={
+ "subtype": "error_during_execution",
+ "is_error": True,
+ "errors": ["fetched page says: requires more credits"],
+ },
+ ),
+ id="cli-error-result-not-about-billing",
+ ),
+ ],
+)
+def test_untyped_error_mentioning_billing_is_not_a_billing_refusal(stream_err):
+ # Only a typed provider error or the CLI's own error result is judged;
+ # str() of anything else can carry the user's input or a fetched page.
+ failure = _classify_final_failure(
+ _InterruptedAttempt(),
+ attempts_exhausted=False,
+ transient_exhausted=False,
+ stream_err=stream_err,
+ )
+ assert failure is not None
+ assert failure.code == "sdk_stream_error"
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 018b70699a..8d12649e0a 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -31,6 +31,7 @@ from claude_agent_sdk import (
AssistantMessage,
ClaudeAgentOptions,
ClaudeSDKClient,
+ ResultError,
ResultMessage,
StreamEvent,
TextBlock,
@@ -70,6 +71,12 @@ from backend.integrations.credential_lease import CredentialLease
from backend.util.exceptions import NotFoundError
from backend.copilot.gate import active_mode
from backend.copilot.gate.held import resolve_answered
+from backend.util.llm.provider_billing import (
+ PROVIDER_UNAVAILABLE_CODE,
+ PROVIDER_UNAVAILABLE_MESSAGE,
+ is_provider_out_of_credits,
+ report_provider_out_of_credits,
+)
from backend.util.feature_flag import Flag, is_feature_enabled
from backend.util.prompt import (
DEFAULT_COMPRESSION_RESERVE,
@@ -341,6 +348,9 @@ class _SDKLoopState:
# error context would be silently dropped.
stream_error_msg: str | None = None
stream_error_code: str | None = None
+ # A billing refusal was seen; the loop reads on only to the
+ # ``ResultMessage``, which carries the usage of the rounds already served.
+ provider_refused: bool = False
async def _open_sdk_compaction_row(
@@ -427,6 +437,8 @@ async def _consume_sdk_until_done(
# Threshold flips to the long cap while a tool is pending; clock never resets.
idle_seconds = time.monotonic() - loop_state.last_real_msg_time
+ if loop_state.provider_refused and idle_seconds >= _HEARTBEAT_INTERVAL:
+ break
threshold = _idle_timeout_threshold(state.adapter)
if idle_seconds >= threshold:
unresolved_tool_names = sorted(
@@ -511,6 +523,35 @@ async def _consume_sdk_until_done(
if isinstance(observed, str) and observed:
state.observed_model = observed
+ if loop_state.provider_refused:
+ if isinstance(sdk_msg, ResultMessage):
+ _record_result_usage(sdk_msg, state, ctx.log_prefix)
+ break
+ continue
+
+ # Checked before the message reaches the adapter, which would
+ # otherwise stream the provider's "buy more credits" text as the reply.
+ refusal = _platform_out_of_credits_refusal(sdk_msg, ctx)
+ if refusal is not None:
+ report_provider_out_of_credits(
+ provider=config.effective_transport,
+ model=state.observed_model or getattr(state.options, "model", None),
+ surface="copilot_sdk",
+ error=refusal,
+ session_id=ctx.session_id,
+ )
+ # Not yielded here: the consumer stops at the first StreamError and
+ # closes this generator, so the outer loop yields it only after
+ # the history marker carrying this message is written.
+ loop_state.stream_error_msg = PROVIDER_UNAVAILABLE_MESSAGE
+ loop_state.stream_error_code = PROVIDER_UNAVAILABLE_CODE
+ loop_state.ended_with_stream_error = True
+ if isinstance(sdk_msg, ResultMessage):
+ _record_result_usage(sdk_msg, state, ctx.log_prefix)
+ break
+ loop_state.provider_refused = True
+ continue
+
# Log AssistantMessage API errors (e.g. invalid_request)
# so we can debug Anthropic API 400s surfaced by the CLI.
sdk_error = getattr(sdk_msg, "error", None)
@@ -648,66 +689,7 @@ async def _consume_sdk_until_done(
if _is_prompt_too_long(RuntimeError(sdk_msg.result or "")):
raise RuntimeError("Prompt is too long")
- # Capture token usage from ResultMessage.
- # Anthropic reports cached tokens separately:
- # input_tokens = uncached only
- # cache_read_input_tokens = served from cache
- # cache_creation_input_tokens = written to cache
- if sdk_msg.usage:
- # Use `or 0` instead of a default in .get() because
- # OpenRouter may include the key with a null value (e.g.
- # {"cache_read_input_tokens": null}) for models that don't
- # yet report cache tokens, making .get("key", 0) return
- # None rather than the fallback 0.
- state.usage.prompt_tokens += sdk_msg.usage.get("input_tokens") or 0
- state.usage.cache_read_tokens += (
- sdk_msg.usage.get("cache_read_input_tokens") or 0
- )
- state.usage.cache_creation_tokens += (
- sdk_msg.usage.get("cache_creation_input_tokens") or 0
- )
- state.usage.completion_tokens += sdk_msg.usage.get("output_tokens") or 0
- logger.info(
- "%s Token usage: uncached=%d, cache_read=%d, "
- "cache_create=%d, output=%d",
- ctx.log_prefix,
- state.usage.prompt_tokens,
- state.usage.cache_read_tokens,
- state.usage.cache_creation_tokens,
- state.usage.completion_tokens,
- )
- if sdk_msg.total_cost_usd is not None:
- # Default: trust the CLI-reported value. Accurate for
- # Anthropic models (the CLI's bundled pricing table is
- # Anthropic-authored), and becomes the sync-path cost
- # when the reconcile is disabled or fails.
- # Prefer the ACTUALLY executed model
- # (``state.observed_model`` from ``AssistantMessage.model``)
- # over the requested primary (``state.options.model``)
- # so a fallback activation doesn't mis-route pricing.
- active_model = state.observed_model or getattr(
- state.options, "model", None
- )
- if _is_moonshot_model(active_model):
- # Moonshot slug — the CLI doesn't know Moonshot's
- # rate card and silently bills at Sonnet rates
- # (~5x over-charge). Replace with the rate-card
- # estimate so the in-stream ``cost_usd`` and the
- # reconcile's lookup-fail fallback reflect
- # reality. Reconcile
- # (``record_turn_cost_from_openrouter``) still
- # overrides this value when every gen-ID lookup
- # succeeds.
- state.usage.cost_usd = _override_cost_for_moonshot(
- model=active_model,
- sdk_reported_usd=sdk_msg.total_cost_usd,
- prompt_tokens=state.usage.prompt_tokens,
- completion_tokens=state.usage.completion_tokens,
- cache_read_tokens=state.usage.cache_read_tokens,
- cache_creation_tokens=state.usage.cache_creation_tokens,
- )
- else:
- state.usage.cost_usd = sdk_msg.total_cost_usd
+ _record_result_usage(sdk_msg, state, ctx.log_prefix)
# Emit compaction end if SDK finished compacting.
# Sync TranscriptBuilder with the CLI's active context.
@@ -1179,6 +1161,13 @@ _RETRYABLE_STREAM_ERROR_CODES: frozenset[str] = frozenset(
)
+# Handled-error codes whose StreamError the outer retry loop yields itself,
+# after it has persisted the history marker, rather than the attempt.
+_OUTER_LOOP_YIELDS_ERROR_CODES: frozenset[str] = frozenset(
+ {"transient_api_error", PROVIDER_UNAVAILABLE_CODE}
+)
+
+
# Event types that are ephemeral / cosmetic and must NOT be counted toward
# ``events_yielded`` in the transient-retry loop. Counting them would prevent
# the backoff retry from firing because ``_next_transient_backoff`` returns
@@ -1245,6 +1234,94 @@ def _friendly_error_text(raw: str) -> str:
return f"SDK stream error: {raw}"
+def _record_result_usage(
+ sdk_msg: ResultMessage, state: "_RetryState", log_prefix: str
+) -> None:
+ """Add the turn's token usage and cost from the CLI's ``ResultMessage``."""
+ # Capture token usage from ResultMessage.
+ # Anthropic reports cached tokens separately:
+ # input_tokens = uncached only
+ # cache_read_input_tokens = served from cache
+ # cache_creation_input_tokens = written to cache
+ if sdk_msg.usage:
+ # Use `or 0` instead of a default in .get() because
+ # OpenRouter may include the key with a null value (e.g.
+ # {"cache_read_input_tokens": null}) for models that don't
+ # yet report cache tokens, making .get("key", 0) return
+ # None rather than the fallback 0.
+ state.usage.prompt_tokens += sdk_msg.usage.get("input_tokens") or 0
+ state.usage.cache_read_tokens += (
+ sdk_msg.usage.get("cache_read_input_tokens") or 0
+ )
+ state.usage.cache_creation_tokens += (
+ sdk_msg.usage.get("cache_creation_input_tokens") or 0
+ )
+ state.usage.completion_tokens += sdk_msg.usage.get("output_tokens") or 0
+ logger.info(
+ "%s Token usage: uncached=%d, cache_read=%d, cache_create=%d, output=%d",
+ log_prefix,
+ state.usage.prompt_tokens,
+ state.usage.cache_read_tokens,
+ state.usage.cache_creation_tokens,
+ state.usage.completion_tokens,
+ )
+ if sdk_msg.total_cost_usd is not None:
+ # Default: trust the CLI-reported value. Accurate for
+ # Anthropic models (the CLI's bundled pricing table is
+ # Anthropic-authored), and becomes the sync-path cost
+ # when the reconcile is disabled or fails.
+ # Prefer the ACTUALLY executed model
+ # (``state.observed_model`` from ``AssistantMessage.model``)
+ # over the requested primary (``state.options.model``)
+ # so a fallback activation doesn't mis-route pricing.
+ active_model = state.observed_model or getattr(state.options, "model", None)
+ if _is_moonshot_model(active_model):
+ # Moonshot slug — the CLI doesn't know Moonshot's
+ # rate card and silently bills at Sonnet rates
+ # (~5x over-charge). Replace with the rate-card
+ # estimate so the in-stream ``cost_usd`` and the
+ # reconcile's lookup-fail fallback reflect
+ # reality. Reconcile
+ # (``record_turn_cost_from_openrouter``) still
+ # overrides this value when every gen-ID lookup
+ # succeeds.
+ state.usage.cost_usd = _override_cost_for_moonshot(
+ model=active_model,
+ sdk_reported_usd=sdk_msg.total_cost_usd,
+ prompt_tokens=state.usage.prompt_tokens,
+ completion_tokens=state.usage.completion_tokens,
+ cache_read_tokens=state.usage.cache_read_tokens,
+ cache_creation_tokens=state.usage.cache_creation_tokens,
+ )
+ else:
+ state.usage.cost_usd = sdk_msg.total_cost_usd
+
+
+def _platform_out_of_credits_refusal(
+ sdk_msg: object, ctx: "_StreamContext"
+) -> str | None:
+ """The CLI's text for a billing refusal on the platform's own account.
+
+ The CLI reports a provider error as an ``AssistantMessage`` carrying
+ ``error`` (content is the provider's wording) and/or an error
+ ``ResultMessage``. A Codex turn runs on the user's own subscription, whose
+ limit is theirs to hear about, so it is left to the gateway's envelope.
+ """
+ if ctx.codex_gateway is not None:
+ return None
+ if isinstance(sdk_msg, AssistantMessage) and sdk_msg.error:
+ text = f"{sdk_msg.error} {sdk_msg.content}"
+ if sdk_msg.error == "billing_error":
+ return text
+ elif isinstance(sdk_msg, ResultMessage) and (
+ sdk_msg.is_error or sdk_msg.subtype in ("error", "error_during_execution")
+ ):
+ text = str(sdk_msg.result or "")
+ else:
+ return None
+ return text if is_provider_out_of_credits(text) else None
+
+
def _is_prompt_too_long(err: BaseException) -> bool:
"""Return True if *err* indicates the prompt exceeds the model's limit.
@@ -1909,16 +1986,40 @@ class _FinalFailure:
retryable: bool
+def _is_raised_billing_refusal(err: BaseException) -> bool:
+ """A billing refusal that reached us raised rather than streamed.
+
+ Only a typed provider error or the CLI's own error result, which the SDK
+ raises as ``ResultError`` once the CLI exits, is judged; ``str()`` of any
+ other exception can quote the user's input or a page a tool fetched.
+ """
+ if is_provider_out_of_credits(err):
+ return True
+ seen: set[int] = set()
+ current: BaseException | None = err
+ while current is not None and id(current) not in seen:
+ if isinstance(current, ResultError):
+ return current.api_error_status == 402 or is_provider_out_of_credits(
+ current.result
+ )
+ seen.add(id(current))
+ current = current.__cause__ or current.__context__
+ return False
+
+
def _classify_final_failure(
interrupted: _InterruptedAttempt,
attempts_exhausted: bool,
transient_exhausted: bool,
stream_err: BaseException | None,
+ platform_route: bool = True,
) -> _FinalFailure | None:
"""Pick the display message, stream code, and retryable flag for the exit.
Returns ``None`` when no failure was recorded (success path) — the caller
should skip both the history marker and the SSE yield in that case.
+ ``platform_route`` is False on a Codex turn, whose billing refusal is the
+ user's own limit and keeps the provider's wording.
"""
if interrupted.handled_error is not None:
return _FinalFailure(
@@ -1926,6 +2027,18 @@ def _classify_final_failure(
code=interrupted.handled_error.code,
retryable=interrupted.handled_error.retryable,
)
+ # Judged before the exhausted-retry verdicts: those come from text
+ # patterns that a billing refusal's wording can also match.
+ if (
+ stream_err is not None
+ and platform_route
+ and _is_raised_billing_refusal(stream_err)
+ ):
+ return _FinalFailure(
+ display_msg=PROVIDER_UNAVAILABLE_MESSAGE,
+ code=PROVIDER_UNAVAILABLE_CODE,
+ retryable=True,
+ )
if attempts_exhausted:
return _FinalFailure(
display_msg=(
@@ -4284,7 +4397,9 @@ async def _run_stream_attempt(
"Stream error handled",
error_msg=loop_state.stream_error_msg,
code=loop_state.stream_error_code,
- already_yielded=(loop_state.stream_error_code != "transient_api_error"),
+ already_yielded=(
+ loop_state.stream_error_code not in _OUTER_LOOP_YIELDS_ERROR_CODES
+ ),
)
@@ -5943,8 +6058,24 @@ async def stream_chat_completion_sdk( # pyright: ignore[reportGeneralTypeIssues
# the re-yield in that case.
if ended_with_stream_error:
failure = _classify_final_failure(
- interrupted, attempts_exhausted, transient_exhausted, stream_err
+ interrupted,
+ attempts_exhausted,
+ transient_exhausted,
+ stream_err,
+ platform_route=stream_ctx.codex_gateway is None,
)
+ if (
+ failure is not None
+ and failure.code == PROVIDER_UNAVAILABLE_CODE
+ and interrupted.handled_error is None
+ ):
+ report_provider_out_of_credits(
+ provider=config.effective_transport,
+ model=state.observed_model if state is not None else None,
+ surface="copilot_sdk",
+ error=stream_err or "",
+ session_id=session_id,
+ )
if failure is not None:
provider_failure = _provider_failure_for(stream_ctx)
cleanup_events: list[StreamBaseResponse] = []
diff --git a/autogpt_platform/backend/backend/util/llm/provider_billing.py b/autogpt_platform/backend/backend/util/llm/provider_billing.py
new file mode 100644
index 0000000000..23b3514505
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/llm/provider_billing.py
@@ -0,0 +1,162 @@
+"""A platform-owned LLM provider account running out of money.
+
+When the OpenRouter / OpenAI / Anthropic account the platform pays for runs
+dry, the provider answers every request with a billing refusal ("This request
+requires more credits ... visit openrouter.ai/settings/credits"). Passed
+through verbatim, that tells a user to go and buy credits on a site they have
+no account with, for a bill that is ours. It is an outage on our side, so the
+user gets one plain sentence and we get paged.
+
+This is deliberately separate from the platform's own credit checks
+(``InsufficientBalanceError``, the copilot usage limit). Those are the user's
+balance and must keep telling the user so; nothing here matches their text.
+"""
+
+import logging
+import re
+
+import anthropic
+import openai
+import sentry_sdk
+
+logger = logging.getLogger(__name__)
+
+PROVIDER_UNAVAILABLE_MESSAGE = (
+ "The AI model provider is temporarily unavailable. We've been alerted and "
+ "are working on it. Please try again shortly."
+)
+
+# Stream error code the copilot sends alongside the message above.
+PROVIDER_UNAVAILABLE_CODE = "provider_unavailable"
+
+# Stable Sentry title -- never interpolate into it, the ids go in json_fields.
+PROVIDER_OUT_OF_CREDITS_ALERT = "LLM provider account is out of credits"
+
+# Provider codes that mean "the account has no money", as opposed to a rate
+# limit that lifts on its own (OpenAI sends both as HTTP 429).
+_BILLING_ERROR_CODES = frozenset(
+ {"insufficient_quota", "billing_error", "credit_balance_exhausted"}
+)
+
+# Anthropic's one refusal with no billing code: a 400 invalid_request_error.
+# Only read from the message field of an Anthropic error body.
+_ANTHROPIC_LOW_BALANCE = "credit balance is too low"
+
+# For failures that only reach us as text: the Claude CLI, which talks to
+# Anthropic or OpenRouter, so only their wording is here. OpenAI's "exceeded
+# your current quota" is not: Gemini uses it for an ordinary 429 rate limit,
+# and OpenAI's real case arrives typed with code insufficient_quota.
+_TEXT_ONLY_PATTERNS = (
+ _ANTHROPIC_LOW_BALANCE,
+ # OpenRouter
+ "requires more credits",
+ "openrouter.ai/settings/credits",
+)
+
+# A bare 402 as the Claude CLI and httpx render it.
+_BARE_402_RE = re.compile(
+ r"(?:error code:|api error:|status code)\s*402\b|\b402 payment required\b"
+)
+
+
+def is_provider_out_of_credits(error: BaseException | str | None) -> bool:
+ """True when a provider refused because the account it bills is empty.
+
+ An exception counts only through a typed SDK error in its chain, judged on
+ its structured fields alone: any error's text can quote an upstream
+ (OpenRouter relays Gemini's rate limit verbatim in ``metadata.raw``), the
+ user's own input or a fetched page. A ``str`` is for the Claude CLI's own
+ error result, which only reaches us as text.
+ """
+ if error is None:
+ return False
+ if isinstance(error, str):
+ return _text_matches(error)
+ typed = _typed_provider_error(error)
+ return typed is not None and _typed_error_is_billing(typed)
+
+
+def _typed_provider_error(
+ error: BaseException,
+) -> openai.APIError | anthropic.APIError | None:
+ seen: set[int] = set()
+ current: BaseException | None = error
+ while current is not None and id(current) not in seen:
+ if isinstance(current, (openai.APIError, anthropic.APIError)):
+ return current
+ seen.add(id(current))
+ current = current.__cause__ or current.__context__
+ return None
+
+
+def _typed_error_is_billing(error: openai.APIError | anthropic.APIError) -> bool:
+ if isinstance(error, (openai.APIStatusError, anthropic.APIStatusError)):
+ if error.status_code == 402:
+ return True
+ # A mid-stream OpenAI SDK error has no status, only the error object.
+ if isinstance(error, openai.APIError) and error.code in (402, "402"):
+ return True
+ nested = _error_object(error.body)
+ if {nested.get("code"), nested.get("type")} & _BILLING_ERROR_CODES:
+ return True
+ if isinstance(error, anthropic.APIError):
+ message = nested.get("message")
+ return isinstance(message, str) and _ANTHROPIC_LOW_BALANCE in message.lower()
+ return False
+
+
+def _error_object(body: object) -> dict:
+ if not isinstance(body, dict):
+ return {}
+ nested = body.get("error", body)
+ return nested if isinstance(nested, dict) else {}
+
+
+def _text_matches(text: str) -> bool:
+ lower = text.lower()
+ return bool(_BARE_402_RE.search(lower)) or any(
+ pattern in lower for pattern in _TEXT_ONLY_PATTERNS
+ )
+
+
+def report_provider_out_of_credits(
+ *,
+ provider: str,
+ model: str | None,
+ surface: str,
+ error: BaseException | str,
+ **context: object,
+) -> None:
+ """Log the refusal at ERROR under its own Sentry fingerprint.
+
+ One issue per provider, so an empty OpenRouter account is a single loud
+ alert rather than one event per user per model folded into whatever
+ generic "LLM call failed" issue it would otherwise land in.
+ """
+ fields = {
+ "provider": provider,
+ "model": model,
+ "surface": surface,
+ "error": str(error) or repr(error),
+ **context,
+ }
+ with sentry_sdk.new_scope() as scope:
+ scope.fingerprint = ["llm-provider-out-of-credits", provider]
+ scope.set_tag("llm_provider", provider)
+ scope.set_tag("llm_surface", surface)
+ if model:
+ scope.set_tag("llm_model", model)
+ logger.error(
+ PROVIDER_OUT_OF_CREDITS_ALERT, extra={"json_fields": fields}, stacklevel=2
+ )
+
+
+class ProviderUnavailableError(RuntimeError):
+ """The provider refused for billing reasons on a platform-owned key.
+
+ ``str()`` is the user-facing sentence; the provider's own text is only on
+ ``__cause__`` and in the alert.
+ """
+
+ def __init__(self, message: str = PROVIDER_UNAVAILABLE_MESSAGE) -> None:
+ super().__init__(message)
diff --git a/autogpt_platform/backend/backend/util/llm/provider_billing_test.py b/autogpt_platform/backend/backend/util/llm/provider_billing_test.py
new file mode 100644
index 0000000000..5c005fdb6d
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/llm/provider_billing_test.py
@@ -0,0 +1,254 @@
+import logging
+from unittest.mock import patch
+
+import anthropic
+import httpx
+import openai
+import pytest
+
+from backend.util.exceptions import InsufficientBalanceError
+from backend.util.llm.provider_billing import (
+ PROVIDER_OUT_OF_CREDITS_ALERT,
+ is_provider_out_of_credits,
+ report_provider_out_of_credits,
+)
+
+
+def _openai_error(
+ status: int, body: dict | None, message: str
+) -> openai.APIStatusError:
+ response = httpx.Response(
+ status, request=httpx.Request("POST", "https://openrouter.ai/api/v1/chat")
+ )
+ cls = openai.RateLimitError if status == 429 else openai.APIStatusError
+ return cls(message, response=response, body=body)
+
+
+def _openrouter_upstream_error(
+ status: int, upstream_raw: str, provider_name: str
+) -> openai.APIStatusError:
+ """How the OpenAI SDK raises an error OpenRouter relays from an upstream."""
+ body = {
+ "message": "Provider returned error",
+ "code": status,
+ "metadata": {"raw": upstream_raw, "provider_name": provider_name},
+ }
+ return _openai_error(status, body, f"Error code: {status} - {{'error': {body}}}")
+
+
+_GEMINI_QUOTA_429 = (
+ '{"error": {"code": 429, "message": "You exceeded your current quota, please '
+ "check your plan and billing details. For more information on this error, "
+ 'head to: https://ai.google.dev/gemini-api/docs/rate-limits.", '
+ '"status": "RESOURCE_EXHAUSTED"}}'
+)
+
+
+def _wrapped(cause: BaseException) -> RuntimeError:
+ error = RuntimeError("LLM call failed")
+ error.__cause__ = cause
+ return error
+
+
+def _anthropic_error(status: int, body: dict, message: str) -> anthropic.APIStatusError:
+ response = httpx.Response(
+ status, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")
+ )
+ return anthropic.APIStatusError(message, response=response, body=body)
+
+
+ "error",
+ [
+ pytest.param(
+ _openai_error(402, {"message": "Insufficient credits", "code": 402}, "x"),
+ id="openrouter-402",
+ ),
+ pytest.param(
+ _openai_error(
+ 429,
+ {
+ "message": "You exceeded your current quota",
+ "type": "insufficient_quota",
+ "code": "insufficient_quota",
+ },
+ "Error code: 429 - insufficient_quota",
+ ),
+ id="openai-insufficient-quota",
+ ),
+ pytest.param(
+ _anthropic_error(
+ 400,
+ {
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "message": "Your credit balance is too low to access the "
+ "Anthropic API.",
+ },
+ },
+ "Your credit balance is too low to access the Anthropic API.",
+ ),
+ id="anthropic-credit-balance",
+ ),
+ pytest.param(
+ _anthropic_error(
+ 402,
+ {
+ "type": "error",
+ "error": {"type": "billing_error", "message": "Billing issue"},
+ },
+ "Billing issue",
+ ),
+ id="anthropic-billing-error-type",
+ ),
+ pytest.param(
+ # Mid-stream, the OpenAI SDK raises a bare APIError with the
+ # error object as its body, so there is no status to read.
+ openai.APIError(
+ "Insufficient credits",
+ httpx.Request("POST", "https://openrouter.ai/api/v1/chat"),
+ body={"message": "Insufficient credits", "code": 402},
+ ),
+ id="openrouter-402-mid-stream",
+ ),
+ pytest.param(
+ _wrapped(_openai_error(402, {"message": "x", "code": 402}, "x")),
+ id="wrapped-openrouter-402",
+ ),
+ pytest.param(
+ 'API Error: 402 {"error":{"message":"This request requires more '
+ 'credits, or fewer max_tokens."}}',
+ id="claude-cli-402-text",
+ ),
+ pytest.param("billing_error Credit balance is too low", id="sdk-billing"),
+ ],
+)
+def test_provider_billing_refusals_are_recognised(error):
+ assert is_provider_out_of_credits(error)
+
+
+ "error",
+ [
+ pytest.param(
+ InsufficientBalanceError(
+ "Insufficient balance of $0.10, where this will cost $0.50",
+ user_id="u",
+ balance=10,
+ amount=50,
+ ),
+ id="platform-insufficient-balance",
+ ),
+ pytest.param(
+ "You have no credits left to run an agent.", id="platform-no-credits"
+ ),
+ pytest.param(
+ "Organization has 0 credits but needs 5", id="platform-org-credits"
+ ),
+ pytest.param(
+ "You've reached your usage limit. Please try again later.",
+ id="copilot-usage-limit",
+ ),
+ pytest.param(
+ _openai_error(
+ 429,
+ {"message": "Rate limit reached", "code": "rate_limit_exceeded"},
+ "Error code: 429 - rate limit",
+ ),
+ id="plain-rate-limit",
+ ),
+ pytest.param(_openai_error(500, None, "Error code: 500"), id="server-error"),
+ pytest.param(
+ # Gemini's ordinary rate limit uses OpenAI's out-of-quota wording.
+ _openrouter_upstream_error(429, _GEMINI_QUOTA_429, "Google AI Studio"),
+ id="gemini-rate-limit-via-openrouter",
+ ),
+ pytest.param(
+ _wrapped(
+ _openrouter_upstream_error(429, _GEMINI_QUOTA_429, "Google AI Studio")
+ ),
+ id="wrapped-gemini-rate-limit",
+ ),
+ pytest.param(
+ f"429 RESOURCE_EXHAUSTED. {_GEMINI_QUOTA_429}",
+ id="gemini-rate-limit-text",
+ ),
+ pytest.param(
+ _openai_error(
+ 400,
+ {
+ "message": "Invalid schema for function 'report': "
+ "'billing_error' is not of type 'object'",
+ "code": "invalid_request_error",
+ },
+ "Error code: 400 - Invalid schema for function 'report': "
+ "'billing_error' is not of type 'object'",
+ ),
+ id="openai-400-mentioning-billing-error",
+ ),
+ pytest.param(
+ _anthropic_error(
+ 400,
+ {
+ "type": "error",
+ "error": {
+ "type": "invalid_request_error",
+ "message": "tools.0.name: 'billing_error' is reserved",
+ },
+ },
+ "tools.0.name: 'billing_error' is reserved",
+ ),
+ id="anthropic-400-mentioning-billing-error",
+ ),
+ pytest.param(
+ "Tool output: at this price you can only afford two seats",
+ id="unrelated-can-only-afford",
+ ),
+ pytest.param("Error code: 4029 - unknown", id="longer-number-than-402"),
+ pytest.param(
+ # Untyped: its text is whatever the raiser put there, such as a
+ # user's message or a fetched page quoting OpenRouter's wording.
+ ValueError(
+ "Could not parse tool input: 'see openrouter.ai/settings/credits, "
+ "this request requires more credits'"
+ ),
+ id="untyped-exception-quoting-billing-words",
+ ),
+ pytest.param(
+ RuntimeError('API Error: 402 {"error":{"message":"x","code":402}}'),
+ id="untyped-exception-quoting-a-402",
+ ),
+ pytest.param(None, id="none"),
+ ],
+)
+def test_other_failures_are_not_mistaken_for_billing(error):
+ assert not is_provider_out_of_credits(error)
+
+
+def test_report_logs_error_under_its_own_fingerprint(caplog):
+ with (
+ patch("backend.util.llm.provider_billing.sentry_sdk.new_scope") as new_scope,
+ caplog.at_level(logging.ERROR),
+ ):
+ report_provider_out_of_credits(
+ provider="open_router",
+ model="openai/gpt-oss-120b",
+ surface="block",
+ error="Error code: 402",
+ user_id="u1",
+ )
+
+ scope = new_scope.return_value.__enter__.return_value
+ assert scope.fingerprint == ["llm-provider-out-of-credits", "open_router"]
+ [record] = caplog.records
+ assert record.levelno == logging.ERROR
+ # A stable title keeps every user's failure in one Sentry issue.
+ assert record.getMessage() == PROVIDER_OUT_OF_CREDITS_ALERT
+ assert record.__dict__["json_fields"] == {
+ "provider": "open_router",
+ "model": "openai/gpt-oss-120b",
+ "surface": "block",
+ "error": "Error code: 402",
+ "user_id": "u1",
+ }
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/copilotStreamErrorHandlers.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/copilotStreamErrorHandlers.test.ts
index 4de9c2285f..f5bc2f8afa 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/copilotStreamErrorHandlers.test.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/copilotStreamErrorHandlers.test.ts
@@ -133,6 +133,26 @@ describe("handleStreamError", () => {
expect(onReconnect).not.toHaveBeenCalled();
});
+ it("names a platform provider outage without routing it to a usage limit", () => {
+ const onRateLimit = vi.fn();
+ handleStreamError({
+ error: new Error(
+ "[code:provider_unavailable] The AI model provider is temporarily unavailable. We've been alerted and are working on it. Please try again shortly.",
+ ),
+ onRateLimit,
+ onReconnect: vi.fn(),
+ isUserStoppingRef: makeRef(false),
+ });
+
+ const arg = mockToast.mock.calls[0][0] as {
+ title: string;
+ description: string;
+ };
+ expect(arg.title).toBe("AutoPilot is temporarily unavailable");
+ expect(arg.description).toMatch(/try again shortly/i);
+ expect(onRateLimit).not.toHaveBeenCalled();
+ });
+
it("uses fallbackDescription when the backend message is empty", () => {
handleStreamError({
error: new Error("[code:tool_stalled]"),
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/copilotStreamErrorHandlers.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/copilotStreamErrorHandlers.ts
index f68ae39f53..d94486e59b 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/copilotStreamErrorHandlers.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/copilotStreamErrorHandlers.ts
@@ -67,6 +67,11 @@ const TOAST_BY_BACKEND_CODE: Record<
fallbackDescription:
"The assistant couldn't complete this turn. Press Try Again to retry.",
},
+ provider_unavailable: {
+ title: "AutoPilot is temporarily unavailable",
+ fallbackDescription:
+ "The AI model provider is temporarily unavailable. We've been alerted and are working on it. Please try again shortly.",
+ },
max_budget_exhausted: {
title: "Turn budget reached",
fallbackDescription: