The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
"""Regression tests for #86570 / #116323: gateway provider error connection messaging."""
import pytest
from gateway.config import Platform
from gateway.run import (
_GATEWAY_CONNECTION_ERROR_RE,
_gateway_provider_error_reply,
_looks_like_gateway_provider_error,
_sanitize_gateway_final_response,
)
# Terminal-path envelopes for an ESTABLISHED connection that died mid-transfer. Whether the
# endpoint is up is unknowable from these (the same turn may already have been answered by it).
INTERRUPTED_ENVELOPES = (
"API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset by peer",
"API call failed after 3 retries: ConnectionResetError: [Errno 104] Connection reset by peer",
"API call failed after 3 retries: httpx.RemoteProtocolError: peer closed connection "
"without sending complete message body",
"API call failed after 3 retries: httpx.ReadError: server disconnected without sending a response",
)
# Nothing accepted the connection / no path to the host: "the endpoint is not up" IS the diagnosis
# here, and it is the case the #86570 wording was written for.
UNREACHABLE_ENVELOPES = (
"API call failed after 3 retries: httpx.ConnectError: [Errno 111] Connection refused",
"API call failed after 3 retries: ConnectionError: [WinError 10061] No connection could "
"be made because the target machine actively refused it",
"API call failed after 3 retries: httpx.ConnectError: [Errno 113] No route to host",
)
# Connection-shaped, but the SDK flattened the cause away; neither diagnosis is supported.
AMBIGUOUS_ENVELOPES = (
"openai.APIConnectionError: Connection error.",
"API call failed after 3 retries: openai.APIConnectionError: Connection error.",
"❌ API failed after 3 retries — openai.APIConnectionError: Connection error.",
)
# Claims that the configured endpoint stopped/never came up. Asserting one of these about a
# mid-transfer reset sends the user to debug a server that is answering.
_ENDPOINT_DOWN_CLAIMS = ("not running", "not responding", "unreachable", "not started", "is down")
def _claims_the_endpoint_is_down(reply: str) -> bool:
return any(claim in reply.lower() for claim in _ENDPOINT_DOWN_CLAIMS)
class TestGatewayConnectionErrorReply:
def test_connection_error_strings_produce_specific_reply(self):
"""A connect that was REFUSED/unroutable is the local-endpoint-down case of #86570.
A bare ``openai.APIConnectionError`` is not: the SDK kept no cause, so it is equally a
dropped response from a live endpoint. It stays a recognised provider envelope, but the
endpoint-down diagnosis is no longer asserted for it (see the distinct-categories test).
"""
samples = [
"httpx.ConnectError: connection refused",
"ConnectionError: [WinError 10061] No connection could be made",
"Errno 111 Connection refused",
"All connection attempts failed: Connection refused",
]
for text in samples:
assert _looks_like_gateway_provider_error(text), text
reply = _gateway_provider_error_reply(text)
assert reply == _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0]), text
assert "/retry" in reply, text
assert _looks_like_gateway_provider_error("openai.APIConnectionError")
def test_broad_connection_phrases_still_map_once_classified(self):
"""Reply selector keeps the full phrase set; the gate does not."""
for text in (
"cannot connect to http://127.0.0.1:8033/v1",
"failed to establish a new connection",
):
reply = _gateway_provider_error_reply(text)
assert reply == _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0]), text
def test_prose_cannot_connect_is_not_a_provider_error(self):
text = (
"cannot connect to the office VPN from this cafe, "
"so I used the backup notes instead"
)
assert not _looks_like_gateway_provider_error(text)
def test_other_errors_keep_generic_reply(self):
for text in (
"RuntimeError: model returned empty content",
"Exception: unknown provider",
"HTTP 500 internal server error",
):
if _looks_like_gateway_provider_error(text):
reply = _gateway_provider_error_reply(text)
assert reply != _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0]), text
def test_connection_regex_does_not_match_non_connection_error(self):
assert not _GATEWAY_CONNECTION_ERROR_RE.search("Rate limited after 3 retries")
assert not _GATEWAY_CONNECTION_ERROR_RE.search("Provider authentication failed")
def test_auth_and_rate_limit_preserved(self):
auth_reply = _gateway_provider_error_reply("provider authentication failed")
assert "sign-in" in auth_reply.lower() and "/login" in auth_reply
assert "rate-limiting" in _gateway_provider_error_reply(
"rate limited after 3 retries"
).lower()
def test_every_reply_names_a_slash_command_and_no_jargon(self):
"""Each shaped reply must give the chat user something they can run; 'provider' and
'gateway logs' are operator words (the log pointer is the `hermes logs` command)."""
from agent.i18n import t
from gateway.run import _PROVIDER_ERROR_REPLIES
replies = [t(reply_key) for _, reply_key in _PROVIDER_ERROR_REPLIES] + [_gateway_provider_error_reply("zzz")]
for reply in replies:
assert any(cmd in reply for cmd in ("/login", "/retry", "/model")), reply
assert "provider" not in reply.lower(), reply
def test_three_connection_causes_are_three_distinct_categories(self):
"""A dropped response, a refused connect and a cause-free error are different failures.
Contract, not wording: each cause gets ONE reply, the three replies differ, and only the
refused/unroutable one may claim the endpoint is down (that claim is pinned to the
refusal samples by ``test_connection_error_strings_produce_specific_reply`` above).
"""
interrupted = {_gateway_provider_error_reply(e) for e in INTERRUPTED_ENVELOPES}
unreachable = {_gateway_provider_error_reply(e) for e in UNREACHABLE_ENVELOPES}
ambiguous = {_gateway_provider_error_reply(e) for e in AMBIGUOUS_ENVELOPES}
assert len(interrupted) == 1, interrupted
assert len(unreachable) == 1, unreachable
assert len(ambiguous) == 1, ambiguous
assert len(interrupted | unreachable | ambiguous) == 3
for reply in interrupted | ambiguous:
assert reply.strip()
assert not _claims_the_endpoint_is_down(reply), reply
@pytest.mark.parametrize("platform", [Platform.TELEGRAM, "slack", "feishu"])
def test_interrupted_connection_delivery_keeps_precedence_and_redaction(self, platform):
"""The new category rides the real chat path, and takes nothing from the other rows."""
raw_reset = (
"API call failed after 3 retries: httpx.ReadError: [Errno 104] Connection reset "
"by peer (Authorization: Bearer sk-ABCDEF0123456789abcdef0123)"
)
sanitized = _sanitize_gateway_final_response(platform, raw_reset)
assert sanitized.strip()
assert "sk-ABCDEF" not in sanitized
assert "Errno 104" not in sanitized
assert not _claims_the_endpoint_is_down(sanitized), sanitized
assert sanitized != _gateway_provider_error_reply(UNREACHABLE_ENVELOPES[0])
# Auth beats policy beats rate-limit beats connection: an envelope carrying BOTH its own
# marker and connection wording keeps the category it had before the connection split.
for tainted, clean in (
("API call failed after 3 retries: HTTP 401 incorrect api key provided; "
"connection reset by peer on the retry", "provider authentication failed"),
("API call failed after 3 retries: HTTP 400 request was blocked under the provider "
"safety policy; connection reset by peer", "request was blocked under the safety policy"),
("API call failed after 3 retries: HTTP 429 rate limit exceeded for this model; "
"connection reset by peer", "rate limited after 3 retries"),
):
assert _sanitize_gateway_final_response(platform, tainted) == (
_gateway_provider_error_reply(clean)), tainted
# Programmatic consumers still get the bottom exception, byte for byte.
assert _sanitize_gateway_final_response("local", raw_reset) == raw_reset
class TestQuotaExhaustedIsNotAnAuthFailure:
"""A 429/quota envelope must never send the user to re-authenticate valid credentials, and a
long reset window must be named instead of "wait a moment" (#89401)."""
def test_quota_envelope_with_auth_preamble_names_reset_window(self):
reply = _gateway_provider_error_reply(
"⚠️ Provider authentication failed: Codex provider quota exhausted (429); "
"retry after 116168s. Credentials are still valid.")
assert "/login" not in reply and "resets in ~33h" in reply
body_reply = _gateway_provider_error_reply(
"API call failed after 3 retries: Error code: 429 - "
"{'error': {'type': 'usage_limit_reached', 'resets_in_seconds': 30995}}")
assert "resets in ~9h" in body_reply
assert "resets in ~5h" in _gateway_provider_error_reply("429 weekly limit reached. Resets in 4hr 5min")
# A bare 401 inside a timestamp is not a sign-in failure; every real 401 envelope still is,
# including the ``HTTP 401: Unauthorized`` shape _summarize_api_error emits (control).
assert "rate-limiting" in _gateway_provider_error_reply("API call failed at 05:14:15,401 status 429")
assert "/login" not in _gateway_provider_error_reply("request 05:14:15,401 failed")
for text in ("HTTP 401: Unauthorized", "returned 401.", "HTTP 401, re-authenticate", "Error code: 401 - token rejected"):
assert "/login" in _gateway_provider_error_reply(text), text
def test_turn_runner_resolution_quota_failure_names_reset_window_not_login(self, monkeypatch):
"""Production entry: TurnRunner.run_sync with credential resolution raising the quota
RuntimeError the gateway wraps around a ``codex_rate_limited`` AuthError."""
from types import SimpleNamespace
import gateway.run as gateway_run
from gateway.config import Platform
from gateway.run_turn_runner import TurnRunner
from gateway.session import SessionSource
from gateway.turn_context import TurnContext
from hermes_cli.auth import AuthError
from hermes_cli.auth_constants import CODEX_RATE_LIMITED_CODE
def _resolve(**_kwargs):
try:
raise AuthError("Codex provider quota exhausted (429); retry after 116168s. Credentials are still valid.",
code=CODEX_RATE_LIMITED_CODE, relogin_required=False)
except AuthError as exc:
raise RuntimeError(str(exc)) from exc
monkeypatch.setattr(gateway_run, "_current_max_iterations", lambda: 30)
runner = SimpleNamespace(_resolve_session_agent_runtime=_resolve,
_get_system_prompt_for_channel=lambda *a, **k: "",
_ephemeral_system_prompt="")
ctx = TurnContext(source=SessionSource(platform=Platform.SLACK, chat_id="C1", chat_type="dm"),
session_key="slack:C1", user_config={}, message="Hi")
result = TurnRunner(runner, ctx).run_sync()
reply = result["final_response"]
assert "/login" not in reply and "resets in ~33h" in reply and result["api_calls"] == 0
def test_turn_runner_resolution_failure_is_a_failed_turn_with_local_detail(self, monkeypatch):
"""A pre-agent resolution failure is a FAILED result (one-shot exits non-zero, the turn
is closed in the transcript), and a LOCAL session sees the resolver's own diagnosis —
``hermes chat -q`` on a bare ``custom`` must print the no-credentials error and exit
non-zero, not ``rc 0`` with a /login hint (C11 bare_custom_without_base_url_fails_fast)."""
from types import SimpleNamespace
import gateway.run as gateway_run
from gateway.config import Platform
from gateway.run_turn_runner import TurnRunner
from gateway.session import SessionSource
from gateway.turn_context import TurnContext
detail = "provider 'custom' resolved without credentials (no endpoint or API key configured)."
def _resolve(**_kwargs):
raise RuntimeError(detail)
monkeypatch.setattr(gateway_run, "_current_max_iterations", lambda: 30)
runner = SimpleNamespace(_resolve_session_agent_runtime=_resolve,
_get_system_prompt_for_channel=lambda *a, **k: "",
_ephemeral_system_prompt="", _adapter_for_source=lambda _s: None)
for platform, chat_id in ((Platform.LOCAL, "local-1"), (Platform.SLACK, "C1")):
ctx = TurnContext(source=SessionSource(platform=platform, chat_id=chat_id, chat_type="dm"),
session_key=f"{platform.value}:{chat_id}", user_config={}, message="Hi")
result = TurnRunner(runner, ctx).run_sync()
assert result["failed"] is True and result["completed"] is False and result["api_calls"] == 0
assert result["failure_reason"] and result["error"] == detail
if platform == Platform.LOCAL:
assert detail in result["final_response"] and "/login" not in result["final_response"]
else:
assert detail not in result["final_response"] and "/login" in result["final_response"]
def test_turn_runner_host_side_resolution_failure_does_not_send_user_to_login(monkeypatch):
"""A local OSError under resolution (the CA bundle deleted by an install-environment rotation)
is not a credential verdict: messaging users get the gateway-log reply, never /login (#140344).
An AuthError in the chain keeps the sign-in reply, also when it wraps a network OSError."""
from types import SimpleNamespace
import gateway.run as gateway_run
from gateway.run_turn_runner import TurnRunner
from gateway.session import SessionSource
from gateway.turn_context import TurnContext
from hermes_cli.auth import AuthError
def _raising(cause):
def _resolve(**_kwargs):
try:
raise cause
except Exception as exc:
raise RuntimeError(str(exc)) from exc
return _resolve
monkeypatch.setattr(gateway_run, "_current_max_iterations", lambda: 30)
replies = {}
def _auth_over_oserror():
try:
raise ConnectionError("refresh endpoint unreachable")
except ConnectionError as net:
raise AuthError("Nous refresh failed; sign in again.") from net
def _mixed_cause():
try:
_auth_over_oserror()
except AuthError as exc:
return exc
for label, cause in (("oserror", FileNotFoundError(2, "No such file or directory")),
("auth", AuthError("No usable credentials found for provider 'nous'.")),
("mixed", _mixed_cause())):
runner = SimpleNamespace(_resolve_session_agent_runtime=_raising(cause),
_get_system_prompt_for_channel=lambda *a, **k: "",
_ephemeral_system_prompt="", _adapter_for_source=lambda _s: None)
ctx = TurnContext(source=SessionSource(platform=Platform.DISCORD, chat_id="T1", chat_type="thread"),
session_key="discord:T1", user_config={}, message="retry")
result = TurnRunner(runner, ctx).run_sync()
assert result["failed"] is True and result["api_calls"] == 0
replies[label] = (result["final_response"], result["failure_reason"], result["failure_retryable"])
assert "/login" not in replies["oserror"][0] and "hermes logs" in replies["oserror"][0]
assert replies["oserror"][1:] == ("host_error", False)
from agent.error_surface import LAYER_GATEWAY, _REASON_TO_LAYER
assert _REASON_TO_LAYER["host_error"] == LAYER_GATEWAY # no "Switch provider" for a host fault
for label in ("auth", "mixed"): # an AuthError anywhere in the chain is a credential verdict
assert "/login" in replies[label][0] and replies[label][1] == "auth_permanent", label