The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(streaming): cap kills name agent.stream_max_lifetime and skip the stale breaker
Tekniumeee418e089d9
diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.pyindex c682fbd4d76..7ec81fa2fef 100644--- a/agent/chat_completion_helpers.py+++ b/agent/chat_completion_helpers.py@@ -33,7 +33,7 @@ from agent.sdk_transform_bypass import bypass_chat_sdk_request_transformfrom agent.errors import EmptyStreamErrorfrom agent.chat_completion_local import _streams_llamacpp, stale_giveupfrom agent.chat_completion_headers import _chat_completion_capturing_headers-from agent.chat_completion_stream_monitor import StreamingWaitMonitor+from agent.chat_completion_stream_monitor import StreamingWaitMonitor, StreamLifetimeExceededfrom agent.transports.chat_completions import is_router_timeout_shim, router_timeout_shim_may_followfrom agent.fast_mode import effective_request_overridesfrom agent.turn_context import substitute_api_content@@ -3571,9 +3571,9 @@ class _StreamingCall(StreamingWaitMonitor):if self._request_cancelled["value"]:logger.debug("Streaming worker caught %s after request cancellation — exiting without retry.", type(e).__name__)return False- _is_timeout = isinstance(e, (_httpx.ReadTimeout, _httpx.ConnectTimeout, _httpx.PoolTimeout))- # ReadError: abort/reset mid-body (stale-kill shutdown under a parked reader,- # ECONNRESET) — the retry loop owns recovery.+ e = self._as_lifetime_error(e) # a cap kill: transient, and its copy names the setting+ _is_timeout = isinstance(e, (_httpx.ReadTimeout, _httpx.ConnectTimeout, _httpx.PoolTimeout, StreamLifetimeExceeded))+ # ReadError: abort/reset mid-body (stale-kill shutdown under a parked reader, ECONNRESET) — retry loop owns it.# anthropic.APIConnectionError: the Anthropic SDK wraps connect/read drops# (incl. stale-kill aborts) in its own type, not httpx's._is_conn_err = isinstance(e, (_httpx.ConnectError, _httpx.ReadError, _httpx.RemoteProtocolError, ConnectionError,diff --git a/agent/chat_completion_stream_monitor.py b/agent/chat_completion_stream_monitor.pyindex c6c45670c26..de36da61680 100644--- a/agent/chat_completion_stream_monitor.py+++ b/agent/chat_completion_stream_monitor.py@@ -15,7 +15,7 @@ from agent import chat_completion_wait_notice as wnfrom agent.model_metadata import is_local_endpointfrom agent.proxy_bypass import is_loopback_hostfrom agent.sleep_clock import SleepWatch-from agent.stream_max_lifetime import StreamLifetime, resolve_stream_max_lifetime+from agent.stream_max_lifetime import StreamLifetime, StreamLifetimeExceeded, resolve_stream_max_lifetimelogger = logging.getLogger(__name__)# Kills of one dispatched attempt whose worker never unwinds before the monitor gives up on it.@@ -32,6 +32,7 @@ class StreamingWaitMonitor:# are withheld (None = claim liveness as usual). A real chunk after it lifts the hold._liveness_withheld_at = None_lifetime = StreamLifetime(0.0) # opt-in agent.stream_max_lifetime; real clock set when the monitor starts+ _lifetime_kill = None # (attempt id, elapsed s) of the latest cap killdef _claims_liveness(self) -> bool:since = self._liveness_withheld_at@@ -192,8 +193,9 @@ class StreamingWaitMonitor:def _check_stream_lifetime(self) -> None:"""Opt-in ``agent.stream_max_lifetime``: a stream that keeps sending chunks but neverfinishes never trips the stale check. Kill it like a stale kill (cancel the attempt,- abort the request-local client, count it in the cross-turn breaker) so the retry loop- or the partial-delivery continuation takes over."""+ abort the request-local client) so the retry loop or the partial-delivery continuation+ takes over. Never counted in the cross-turn stale breaker: the provider WAS responding,+ and a streak of cap kills would refuse the next healthy request with nothing sent."""if not self._lifetime.cap: # off (the default): no per-tick workreturnwith self.stream_attempt_lock:@@ -208,13 +210,28 @@ class StreamingWaitMonitor:f"⚠️ Response still streaming after {int(elapsed)}s (stream_max_lifetime "f"{int(self._lifetime.cap)}s, model: {model}). Aborting...")killed = self._attempt_stream_response+ self._lifetime_kill = (attempt, elapsed) # before the abort: the woken worker reads it at oncewith contextlib.suppress(Exception):self._cancel_current_stream_attempt("stream_max_lifetime_kill")self.clients.close_once("stream_max_lifetime_kill")self._shutdown_stale_attempt_socket(killed)- self._count_stale_attempt()+ # Accounted, never bumped: a stale re-kill or /stop while it unwinds must not strike it either.+ self._stale_counted_attempts.add(attempt)self.last_chunk_time["t"] = time.time() # the stale timer restarts with the kill+ def _as_lifetime_error(self, e: Exception) -> Exception:+ """The worker's transport error for an attempt the cap killed, replaced by one that names+ ``agent.stream_max_lifetime`` (the retry, exhausted and partial-delivery lines show it)."""+ kill = self._lifetime_kill+ if kill is None or isinstance(e, StreamLifetimeExceeded): # no cap kill: touch nothing else+ return e+ with self.stream_attempt_lock:+ if kill[0] != int(self.stream_attempt_state["current"]):+ return e+ err = StreamLifetimeExceeded(kill[1], self._lifetime.cap)+ err.__cause__ = e+ return err+def _monitor_loop(self) -> None:_HEARTBEAT_INTERVAL = 30.0 # seconds between gateway activity touchesself._mon = SimpleNamespace(diff --git a/agent/stream_diag.py b/agent/stream_diag.pyindex 10af9ec5d71..f8dcdd8d08d 100644--- a/agent/stream_diag.py+++ b/agent/stream_diag.py@@ -207,6 +207,9 @@ def connect_exhausted_notice(error: BaseException, *, attempts: int, base_url: Anames the host, the attempt count and the serialized request size so a body-size limit isdistinguishable from an outage without reading agent.log."""from urllib.parse import urlparse+ from agent.stream_max_lifetime import StreamLifetimeExceeded+ if isinstance(error, StreamLifetimeExceeded): # the stream DID open; the opt-in cap cut every attempt+ return f"❌ {error} ({attempts} attempt{'s' if attempts != 1 else ''})"request = _failed_request(error)host = urlparse(str(getattr(request, "url", None) or base_url or "")).hostname or "the endpoint"size = _request_body_bytes(request)diff --git a/agent/stream_max_lifetime.py b/agent/stream_max_lifetime.pyindex 1b0c8d45d7d..712f63e3a6c 100644--- a/agent/stream_max_lifetime.py+++ b/agent/stream_max_lifetime.py@@ -40,6 +40,17 @@ def resolve_stream_max_lifetime(stale_timeout: float | None) -> float:return cap+class StreamLifetimeExceeded(TimeoutError):+ """A stream attempt cut by ``agent.stream_max_lifetime``: the stream opened and was still+ delivering chunks, so the user-facing error names the setting, not the endpoint."""++ def __init__(self, elapsed: float, cap: float):+ super().__init__(+ f"Response still streaming after {elapsed:.0f}s, cut by agent.stream_max_lifetime ({cap:g}s). "+ f"Raise agent.stream_max_lifetime, or set it to 0 to turn the cap off, if this model "+ f"legitimately streams this long.")++class StreamLifetime:"""Per-call clock: restarts whenever a new stream attempt is dispatched, so eachretry gets the full cap, and after each kill (the killed attempt may unwind slowly)."""diff --git a/tests/agent/test_stream_max_lifetime.py b/tests/agent/test_stream_max_lifetime.pyindex ec59bd3096f..52be8958b2f 100644--- a/tests/agent/test_stream_max_lifetime.py+++ b/tests/agent/test_stream_max_lifetime.py@@ -7,17 +7,17 @@ import threadingimport timefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServer-import httpximport pytestimport run_agentfrom agent.chat_completion_helpers import interruptible_streaming_api_call-from agent.stream_max_lifetime import resolve_stream_max_lifetime+from agent.stream_max_lifetime import StreamLifetimeExceeded, resolve_stream_max_lifetimefrom hermes_cli.config_defaults import DEFAULT_CONFIGclass _DripWire:- """Real OpenAI-wire SSE server: one content chunk every 50 ms for ~8 s, then a clean finish."""+ """Real OpenAI-wire SSE server: one content chunk every 50 ms for ~8 s, then a clean finish+ (a "short" prompt gets one chunk and the finish)."""def __init__(self):class Handler(BaseHTTPRequestHandler):@@ -25,7 +25,8 @@ class _DripWire:passdef do_POST(self):- self.rfile.read(int(self.headers.get("content-length", 0)))+ body = json.loads(self.rfile.read(int(self.headers.get("content-length", 0))) or b"{}")+ short = (body.get("messages") or [{}])[-1].get("content") == "short"if not self.path.endswith("/chat/completions"):self.send_response(404)self.end_headers()@@ -37,7 +38,7 @@ class _DripWire:"choices": [{"index": 0, "delta": {"content": "."}, "finish_reason": None}]}fin = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}try:- for _ in range(160):+ for _ in range(1 if short else 160):self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())self.wfile.flush()time.sleep(0.05) # health: allow HX013 -- the drip cadence is the behaviour under test@@ -63,17 +64,24 @@ def test_cap_cuts_a_stream_that_never_goes_quiet(drip_wire, monkeypatch):monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "1.0")monkeypatch.setenv("HERMES_STREAM_MAX_LIFETIME", "1.5")monkeypatch.setenv("HERMES_STREAM_RETRIES", "0")+ monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "2")agent = run_agent.AIAgent(api_key="test-key", base_url=drip_wire.base_url, model="m", provider="custom",platform="cli", quiet_mode=True, skip_context_files=True, skip_memory=True,enabled_toolsets=[], max_iterations=1,)- started = time.time()- # Without the cap the drip completes normally after ~8 s; with it the call ends early.- # Killed like a stale kill: the undelivered stream surfaces the transport error to the loop.- with pytest.raises(httpx.RemoteProtocolError):- interruptible_streaming_api_call(agent, {"model": "m", "messages": [{"role": "user", "content": "hi"}]})- assert time.time() - started < 5.0+ def call(prompt):+ return interruptible_streaming_api_call(agent, {"model": "m", "messages": [{"role": "user", "content": prompt}]})++ # Without the cap the drip completes normally after ~8 s; with it each call ends early, with+ # an error naming the setting (the stream opened, so no "endpoint unreachable" copy).+ for _ in range(2):+ started = time.time()+ with pytest.raises(StreamLifetimeExceeded, match="agent.stream_max_lifetime"):+ call("hi")+ assert time.time() - started < 5.0+ # Cap kills are not stale kills: the breaker (giveup 2) must not refuse a healthy request.+ assert call("short").choices[0].message.content == "."def test_cap_is_off_by_default_and_never_undercuts_the_stale_timeout(monkeypatch):diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.mdindex a98e075efb8..7528fa348c5 100644--- a/website/docs/reference/environment-variables.md+++ b/website/docs/reference/environment-variables.md@@ -830,7 +830,7 @@ Advanced per-platform knobs for throttling the outbound message batcher. Most us| `HERMES_STREAM_READ_TIMEOUT` | Streaming socket read timeout in seconds (default: `120`). Auto-increased to `HERMES_API_TIMEOUT` for local providers. Increase if local LLMs time out during long code generation. || `HERMES_STREAM_STALE_TIMEOUT` | Stale stream detection timeout in seconds (default: `180`). Auto-disabled for local providers. Triggers connection kill if no chunks arrive within this window. || `HERMES_LOCAL_STREAM_STALE_TIMEOUT` | Stale stream ceiling for local providers (Ollama, oMLX, llama-cpp) in seconds (default: `900`). When the base stale timeout is at its default and a local endpoint is detected, this finite ceiling replaces the former infinite disable so a wedged local server eventually trips the detector instead of hanging forever. Also configurable via `agent.local_stream_stale_timeout` in `config.yaml`. |-| `HERMES_STREAM_MAX_LIFETIME` | Opt-in total wall-clock cap in seconds for one streaming response attempt (default: `0`, off). Unlike the stale-stream detector it never resets on chunk arrival, so it stops a stream that keeps sending data but never finishes. Never fires before the effective stale-stream timeout. Also configurable via `agent.stream_max_lifetime` in `config.yaml`. |+| `HERMES_STREAM_MAX_LIFETIME` | Opt-in total wall-clock cap in seconds for one streaming response attempt (default: `0`, off). Unlike the stale-stream detector it never resets on chunk arrival, so it stops a stream that keeps sending data but never finishes. Never fires before the effective stale-stream timeout. Applies to Chat Completions and Anthropic Messages streams only; Codex/Responses (`codex_responses`, including custom Responses providers) and Bedrock streams are not capped. A capped attempt is retried and does not count toward `HERMES_STREAM_STALE_GIVEUP`. Also configurable via `agent.stream_max_lifetime` in `config.yaml`. || `HERMES_STREAM_RETRIES` | Number of mid-stream reconnect attempts on transient network errors (default: `3`). || `HERMES_STREAM_STALE_GIVEUP` | Cross-turn circuit breaker: after this many consecutive stale kills (streaming or non-streaming) with no completed response, abort each call immediately with an actionable error instead of re-waiting out the stale timeout (default: `5`, `0` disables). Resets on any completed response, `/model` switch, fallback activation, or turn-start primary restore. || `HERMES_AGENT_TIMEOUT` | Gateway inactivity timeout for a running agent in seconds (default: `1800`, 30 minutes). Resets on every tool call and streamed token. Set to `0` to disable. |diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.mdindex 7c4685b5e99..a7609572844 100644--- a/website/docs/user-guide/configuration.md+++ b/website/docs/user-guide/configuration.md@@ -1380,7 +1380,7 @@ The **stale stream detection** kills connections that receive SSE keep-alive pinThe **Responses first-event watchdog** (Codex / `codex_responses` transport, including custom providers declared with the Responses transport) aborts and reconnects a request that accepts the connection but emits no stream event within 120 seconds. A local server prefilling a large context legitimately stays silent longer than that, so on local endpoints the implicit default is raised to the same ceiling as the stale stream detector (`agent.local_stream_stale_timeout` / `HERMES_LOCAL_STREAM_STALE_TIMEOUT`, 900s). An explicit `HERMES_CODEX_TTFB_TIMEOUT_SECONDS` is always used as-is (`0` disables the watchdog).-The **stream max lifetime** is an opt-in cap on the total wall-clock time of one streaming response. The stale detector resets on every chunk, so a stream that keeps sending data but never finishes (a gateway trickling chunks, a runaway generation) is never cut. Set `agent.stream_max_lifetime` (seconds) to cut such a stream; the connection is killed the same way a stale kill is, and the retry or partial-response continuation takes over. It never fires before the effective stale-stream timeout. Leave it at `0` (the default) if you run long reasoning turns that legitimately stream for a long time.+The **stream max lifetime** is an opt-in cap on the total wall-clock time of one streaming response. The stale detector resets on every chunk, so a stream that keeps sending data but never finishes (a gateway trickling chunks, a runaway generation) is never cut. Set `agent.stream_max_lifetime` (seconds) to cut such a stream; the connection is killed the same way a stale kill is, and the retry or partial-response continuation takes over. It never fires before the effective stale-stream timeout, and a cut attempt does not count toward the stale-stream circuit breaker (`HERMES_STREAM_STALE_GIVEUP`), since the provider was responding. The cap covers Chat Completions and Anthropic Messages streams only; Codex/Responses streams (`codex_responses`, including custom providers declared with the Responses transport) and Bedrock streams are not capped. Leave it at `0` (the default) if you run long reasoning turns that legitimately stream for a long time.The **stale non-stream detection** kills non-streaming calls that produce no response for too long. By default Hermes disables this on local endpoints to avoid false positives during long prefills. If you explicitly set `providers.<id>.stale_timeout_seconds`, `providers.<id>.models.<model>.stale_timeout_seconds`, or `HERMES_API_CALL_STALE_TIMEOUT`, that explicit value is honored even on local endpoints.