The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): stamp a host-side resolution failure as a gateway-layer error
kshitijk4poor7e9eb8d9a30f
diff --git a/agent/error_surface.py b/agent/error_surface.pyindex 985c97411bf..2b25512b1ef 100644--- a/agent/error_surface.py+++ b/agent/error_surface.py@@ -37,7 +37,7 @@ LAYER_DISK = "disk"_REASON_TO_LAYER = {"auth": LAYER_AUTH, "auth_permanent": LAYER_AUTH, "billing": LAYER_BILLING, "billing_unverified": LAYER_BILLING,"loop_error": LAYER_GATEWAY, "interpreter_shutdown": LAYER_GATEWAY, "session_busy": LAYER_GATEWAY,- "gateway_connection": LAYER_GATEWAY,+ "gateway_connection": LAYER_GATEWAY, "host_error": LAYER_GATEWAY,"truncated": LAYER_PROVIDER, "empty_response": LAYER_PROVIDER, "invalid_response": LAYER_PROVIDER,"context_overflow": LAYER_PROVIDER, # a bigger-window model IS the fix, so Switch provider applies}diff --git a/agent/turn_failure_copy.py b/agent/turn_failure_copy.pyindex 655ea2772a0..8ff6a690faa 100644--- a/agent/turn_failure_copy.py+++ b/agent/turn_failure_copy.py@@ -18,7 +18,7 @@ from hermes_constants import display_hermes_home# Failure codes minted by loop sites that are not provider verdicts (see module docstring).SITE_FAILURE_CODES = frozenset({"context_overflow", "truncated", "invalid_response", "empty_response", "loop_error",- "interpreter_shutdown", "session_busy", "gateway_connection",+ "interpreter_shutdown", "session_busy", "gateway_connection", "host_error",})diff --git a/gateway/run_turn_runner.py b/gateway/run_turn_runner.pyindex b0b2af7936a..8e766a570fc 100644--- a/gateway/run_turn_runner.py+++ b/gateway/run_turn_runner.py@@ -131,8 +131,8 @@ def _settle_api_runtime_lock(agent, result, captured):result.update(final_response='', failed=True, completed=False, error=str(exc))-def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str, str]:- """``(chat text, failure_reason)`` for a model/credential resolution failure before the turn."""+def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str, str, bool]:+ """``(chat text, failure_reason, retryable)`` for a model/credential resolution failure before the turn."""from agent.error_classifier import FailoverReasonfrom hermes_cli.auth import AuthError, is_rate_limited_auth_errorfrom hermes_state_runtime import RuntimeStoreError@@ -142,15 +142,15 @@ def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str,# daemon restart): keep it in the reply so clients can act on it, and do not suggest# /login — the profile's own credentials were never in play.return (f"⚠️ This session's launch credentials are no longer available ({exc.reason}), so this "- "message wasn't processed. Start a new session from the CLI to bind them again.", auth_permanent)+ "message wasn't processed. Start a new session from the CLI to bind them again.", auth_permanent, False)if is_rate_limited_auth_error(exc.__cause__):# Quota cap with valid credentials: /login cannot help; name the reset window (#89401).from gateway.run import _gateway_provider_error_reply- return _gateway_provider_error_reply(str(exc)), auth_permanent+ return _gateway_provider_error_reply(str(exc)), auth_permanent, Falseif platform == Platform.LOCAL:# The local operator reads the terminal: the resolver's own sentence ("provider 'custom'# resolved without credentials ...") is the diagnosis; /login is not.- return f"⚠️ {exc}", auth_permanent+ return f"⚠️ {exc}", auth_permanent, Falseseen: set[int] = set()host = auth = Falselink: BaseException | None = exc@@ -162,8 +162,10 @@ def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str,if host and not auth:# A host-side failure (a CA bundle deleted under a long-lived process, an unreadable file)# with no credential verdict: /login cannot fix it, the gateway log can (#140344).- return t("gateway.errors.generic_failed"), FailoverReason.unknown.value- return t("gateway.errors.no_credentials"), auth_permanent+ # host_error sits on the gateway layer, so clients don't offer "Switch provider"; not retryable,+ # since a deleted bundle or unreadable store fails the same way until the host is fixed.+ return t("gateway.errors.generic_failed"), "host_error", False+ return t("gateway.errors.no_credentials"), auth_permanent, Falseclass TurnRunner(GatewayTurnProgressMixin, GatewaySessionAgentMixin):@@ -1227,9 +1229,9 @@ class TurnRunner(GatewayTurnProgressMixin, GatewaySessionAgentMixin):# turn, and an API receipt reports failure, never a completed turn with an apology.logger.warning("Model resolution failed for session %s: %s", ctx.session_key or "", exc, exc_info=True)from agent.turn_failure_copy import stamp_failure- text, reason = _resolution_failure_reply(exc, ctx.source.platform)+ text, reason, retryable = _resolution_failure_reply(exc, ctx.source.platform)return stamp_failure({"final_response": text, "messages": [], "api_calls": 0, "tools": [],- "failed": True, "completed": False, "error": str(exc)}, reason, False)+ "failed": True, "completed": False, "error": str(exc)}, reason, retryable)from gateway.session_authorities import active_authorityfrom gateway.session_worker_construct import session_routingpr = session_routing(active_authority(runner), policy) if policy else runner._provider_routingdiff --git a/tests/gateway/test_local_model_connection_reply.py b/tests/gateway/test_local_model_connection_reply.pyindex 7fd814b1b84..73420534935 100644--- a/tests/gateway/test_local_model_connection_reply.py+++ b/tests/gateway/test_local_model_connection_reply.py@@ -294,8 +294,10 @@ def test_turn_runner_host_side_resolution_failure_does_not_send_user_to_login(mosession_key="discord:T1", user_config={}, message="retry")result = TurnRunner(runner, ctx).run_sync()assert result["failed"] is True and result["api_calls"] == 0- replies[label] = (result["final_response"], result["failure_reason"])+ replies[label] = (result["final_response"], result["failure_reason"], result["failure_retryable"])assert "/login" not in replies["oserror"][0] and "hermes logs" in replies["oserror"][0]- assert replies["oserror"][1] == "unknown"+ assert replies["oserror"][1:] == ("host_error", False)+ from agent.error_surface import LAYER_GATEWAY, _REASON_TO_LAYER+ assert _REASON_TO_LAYER["host_error"] == LAYER_GATEWAY # no "Switch provider" for a host faultfor label in ("auth", "mixed"): # an AuthError anywhere in the chain is a credential verdictassert "/login" in replies[label][0] and replies[label][1] == "auth_permanent", label