The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""Provider error bodies observed in real sessions, through the real SDK exception types.
Every row is a body Hermes actually received (relay traces + agent.log, scrubbed of request ids
and account data), raised as the exception class the OpenAI SDK / httpx raises for it, so a
classifier change is checked against what providers really send rather than hand-written
strings. Add a row whenever a new production body shows up in a bug report or trace.
OpenRouter's "can only afford N tokens" 402 (#49769) stays ``billing`` here; the main loop's
recovery chain retries it once with the lowered cap (see that test's docstring).
The last test is a body main still mishandles, as a run-time xfail keyed on the current wrong
outcome, so it turns into a passing regression test the moment its fix lands:
* #85005 — "<model> is not a multimodal model" (vLLM/text-only endpoints) is not recognised as
an image rejection, so the image is never stripped and every retry fails identically.
"""
from __future__ import annotations
import httpx
import openai
import pytest
from agent.error_classifier import FailoverReason, classify_api_error
from agent.message_sanitization import _looks_like_image_content_rejection
from tests.e2e.core._pending_fixes import known_failure
_REQUEST = httpx.Request("POST", "https://example.invalid/v1/chat/completions")
def _status(cls: type, code: int, body: dict) -> Exception:
return cls(f"Error code: {code} - {body}", response=httpx.Response(code, request=_REQUEST, json=body), body=body)
AFFORD_402 = {"error": {"message": (
"This request requires more credits, or fewer max_tokens. You requested up to 128000 tokens, but can only "
"afford 81664. To increase, visit https://openrouter.ai/settings/credits and upgrade to a paid account"),
"code": 402}}
NOT_MULTIMODAL_400 = {"error": {"message": "glm-5.2-fp8 is not a multimodal model", "type": "BadRequestError",
"param": None, "code": 400}}
# (id, provider, exception, expected reason, retryable, should_fallback)
OBSERVED = [
("connection-error", "custom", openai.APIConnectionError(request=_REQUEST), FailoverReason.timeout, True, False),
("broken-pipe", "nous", httpx.ReadError("[Errno 32] Broken pipe", request=_REQUEST),
FailoverReason.timeout, True, False),
("peer-closed-mid-body", "custom", httpx.RemoteProtocolError(
"peer closed connection without sending complete message body (incomplete chunked read)", request=_REQUEST),
FailoverReason.timeout, True, False),
("request-timeout", "nous", openai.APITimeoutError(request=_REQUEST), FailoverReason.timeout, True, False),
("nous-404-no-credits", "nous", _status(openai.NotFoundError, 404, {"status": 404, "message": (
"Model 'anthropic/claude-opus-5' requires available credits. Your account balance is too low to use paid "
"models — add credits at https://portal.nousresearch.com or pick a free model.")}),
FailoverReason.billing, False, True),
("nous-401-invalid-key", "nous", _status(openai.AuthenticationError, 401, {"status": 401, "message": (
"Your API key is invalid, blocked or out of funds. Please go visit the portal to sort that out: "
"https://portal.nousresearch.com ")}),
FailoverReason.auth, False, True),
("cloudflare-502", "nous", _status(openai.InternalServerError, 502, {
"type": "https://developers.cloudflare.com/support/troubleshooting/http-status-codes/cloudflare-5xx-errors/"
"error-502/", "title": "Error 502: Bad gateway", "status": 502,
"detail": "The origin web server returned an invalid or incomplete response to Cloudflare.",
"error_code": 502, "error_name": "origin_bad_gateway", "error_category": "origin"}),
FailoverReason.server_error, True, False),
("nous-500-unexpected", "nous", _status(openai.InternalServerError, 500, {"status": 500, "message": (
"Something unexpected happened while processing your request. Please try again in a moment, or contact us "
"if the issue persists.")}),
FailoverReason.server_error, True, False),
("router-no-workers", "custom", _status(openai.InternalServerError, 500, {"error": {
"message": "No available workers (all circuits open or unhealthy)"}}),
FailoverReason.server_error, True, False),
]
@pytest.mark.parametrize("provider,error,reason,retryable,fallback", [r[1:] for r in OBSERVED],
ids=[r[0] for r in OBSERVED])
def test_observed_body_routes_to_its_recovery(provider, error, reason, retryable, fallback):
result = classify_api_error(error, provider=provider, model="test-model")
assert (result.reason, result.retryable, result.should_fallback) == (reason, retryable, fallback)
def test_affordable_402_classifies_as_billing_with_a_parseable_budget():
"""#49769 is recovered in the main loop, not by the classifier: the 402 stays ``billing`` and the
recovery chain retries once with the cap lowered to the budget it names
(``tests/agent/test_affordable_402_main_loop.py`` drives that retry end to end)."""
from agent.auxiliary_client import _affordable_max_tokens_from_error
error = _status(openai.APIStatusError, 402, AFFORD_402)
result = classify_api_error(error, provider="openrouter", model="anthropic/claude-opus-5.5")
assert result.reason == FailoverReason.billing
assert _affordable_max_tokens_from_error(error) == 81664 - 64
def test_not_a_multimodal_model_is_an_image_rejection():
body = str(_status(openai.BadRequestError, 400, NOT_MULTIMODAL_400))
with known_failure(r"not recognised", "#85005: text-only endpoint image rejection not recognised"):
assert _looks_like_image_content_rejection(body), f"image rejection not recognised: {body[:120]}"