Skip to content

The agent that grows with you

fix(gateway): stamp a host-side resolution failure as a gateway-layer error

kshitijk4poor7e9eb8d9a30f
diff --git a/agent/error_surface.py b/agent/error_surface.py
index 985c97411bf..2b25512b1ef 100644
--- a/agent/error_surface.py
+++ b/agent/error_surface.py
@@ -37,7 +37,7 @@ LAYER_DISK = "disk"
_REASON_TO_LAYER = {
"auth": LAYER_AUTH, "auth_permanent": LAYER_AUTH, "billing": LAYER_BILLING, "billing_unverified": LAYER_BILLING,
"loop_error": LAYER_GATEWAY, "interpreter_shutdown": LAYER_GATEWAY, "session_busy": LAYER_GATEWAY,
- "gateway_connection": LAYER_GATEWAY,
+ "gateway_connection": LAYER_GATEWAY, "host_error": LAYER_GATEWAY,
"truncated": LAYER_PROVIDER, "empty_response": LAYER_PROVIDER, "invalid_response": LAYER_PROVIDER,
"context_overflow": LAYER_PROVIDER, # a bigger-window model IS the fix, so Switch provider applies
}
diff --git a/agent/turn_failure_copy.py b/agent/turn_failure_copy.py
index 655ea2772a0..8ff6a690faa 100644
--- a/agent/turn_failure_copy.py
+++ b/agent/turn_failure_copy.py
@@ -18,7 +18,7 @@ from hermes_constants import display_hermes_home
# Failure codes minted by loop sites that are not provider verdicts (see module docstring).
SITE_FAILURE_CODES = frozenset({
"context_overflow", "truncated", "invalid_response", "empty_response", "loop_error",
- "interpreter_shutdown", "session_busy", "gateway_connection",
+ "interpreter_shutdown", "session_busy", "gateway_connection", "host_error",
})
diff --git a/gateway/run_turn_runner.py b/gateway/run_turn_runner.py
index b0b2af7936a..8e766a570fc 100644
--- a/gateway/run_turn_runner.py
+++ b/gateway/run_turn_runner.py
@@ -131,8 +131,8 @@ def _settle_api_runtime_lock(agent, result, captured):
result.update(final_response='', failed=True, completed=False, error=str(exc))
-def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str, str]:
- """``(chat text, failure_reason)`` for a model/credential resolution failure before the turn."""
+def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str, str, bool]:
+ """``(chat text, failure_reason, retryable)`` for a model/credential resolution failure before the turn."""
from agent.error_classifier import FailoverReason
from hermes_cli.auth import AuthError, is_rate_limited_auth_error
from hermes_state_runtime import RuntimeStoreError
@@ -142,15 +142,15 @@ def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str,
# daemon restart): keep it in the reply so clients can act on it, and do not suggest
# /login — the profile's own credentials were never in play.
return (f"⚠️ This session's launch credentials are no longer available ({exc.reason}), so this "
- "message wasn't processed. Start a new session from the CLI to bind them again.", auth_permanent)
+ "message wasn't processed. Start a new session from the CLI to bind them again.", auth_permanent, False)
if is_rate_limited_auth_error(exc.__cause__):
# Quota cap with valid credentials: /login cannot help; name the reset window (#89401).
from gateway.run import _gateway_provider_error_reply
- return _gateway_provider_error_reply(str(exc)), auth_permanent
+ return _gateway_provider_error_reply(str(exc)), auth_permanent, False
if platform == Platform.LOCAL:
# The local operator reads the terminal: the resolver's own sentence ("provider 'custom'
# resolved without credentials ...") is the diagnosis; /login is not.
- return f"⚠️ {exc}", auth_permanent
+ return f"⚠️ {exc}", auth_permanent, False
seen: set[int] = set()
host = auth = False
link: BaseException | None = exc
@@ -162,8 +162,10 @@ def _resolution_failure_reply(exc: Exception, platform: Platform) -> tuple[str,
if host and not auth:
# A host-side failure (a CA bundle deleted under a long-lived process, an unreadable file)
# with no credential verdict: /login cannot fix it, the gateway log can (#140344).
- return t("gateway.errors.generic_failed"), FailoverReason.unknown.value
- return t("gateway.errors.no_credentials"), auth_permanent
+ # host_error sits on the gateway layer, so clients don't offer "Switch provider"; not retryable,
+ # since a deleted bundle or unreadable store fails the same way until the host is fixed.
+ return t("gateway.errors.generic_failed"), "host_error", False
+ return t("gateway.errors.no_credentials"), auth_permanent, False
class TurnRunner(GatewayTurnProgressMixin, GatewaySessionAgentMixin):
@@ -1227,9 +1229,9 @@ class TurnRunner(GatewayTurnProgressMixin, GatewaySessionAgentMixin):
# turn, and an API receipt reports failure, never a completed turn with an apology.
logger.warning("Model resolution failed for session %s: %s", ctx.session_key or "", exc, exc_info=True)
from agent.turn_failure_copy import stamp_failure
- text, reason = _resolution_failure_reply(exc, ctx.source.platform)
+ text, reason, retryable = _resolution_failure_reply(exc, ctx.source.platform)
return stamp_failure({"final_response": text, "messages": [], "api_calls": 0, "tools": [],
- "failed": True, "completed": False, "error": str(exc)}, reason, False)
+ "failed": True, "completed": False, "error": str(exc)}, reason, retryable)
from gateway.session_authorities import active_authority
from gateway.session_worker_construct import session_routing
pr = session_routing(active_authority(runner), policy) if policy else runner._provider_routing
diff --git a/tests/gateway/test_local_model_connection_reply.py b/tests/gateway/test_local_model_connection_reply.py
index 7fd814b1b84..73420534935 100644
--- a/tests/gateway/test_local_model_connection_reply.py
+++ b/tests/gateway/test_local_model_connection_reply.py
@@ -294,8 +294,10 @@ def test_turn_runner_host_side_resolution_failure_does_not_send_user_to_login(mo
session_key="discord:T1", user_config={}, message="retry")
result = TurnRunner(runner, ctx).run_sync()
assert result["failed"] is True and result["api_calls"] == 0
- replies[label] = (result["final_response"], result["failure_reason"])
+ replies[label] = (result["final_response"], result["failure_reason"], result["failure_retryable"])
assert "/login" not in replies["oserror"][0] and "hermes logs" in replies["oserror"][0]
- assert replies["oserror"][1] == "unknown"
+ assert replies["oserror"][1:] == ("host_error", False)
+ from agent.error_surface import LAYER_GATEWAY, _REASON_TO_LAYER
+ assert _REASON_TO_LAYER["host_error"] == LAYER_GATEWAY # no "Switch provider" for a host fault
for label in ("auth", "mixed"): # an AuthError anywhere in the chain is a credential verdict
assert "/login" in replies[label][0] and replies[label][1] == "auth_permanent", label