Skip to content

The agent that grows with you

fix(streaming): cap kills name agent.stream_max_lifetime and skip the stale breaker

Tekniumeee418e089d9
diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py
index c682fbd4d76..7ec81fa2fef 100644
--- a/agent/chat_completion_helpers.py
+++ b/agent/chat_completion_helpers.py
@@ -33,7 +33,7 @@ from agent.sdk_transform_bypass import bypass_chat_sdk_request_transform
from agent.errors import EmptyStreamError
from agent.chat_completion_local import _streams_llamacpp, stale_giveup
from agent.chat_completion_headers import _chat_completion_capturing_headers
-from agent.chat_completion_stream_monitor import StreamingWaitMonitor
+from agent.chat_completion_stream_monitor import StreamingWaitMonitor, StreamLifetimeExceeded
from agent.transports.chat_completions import is_router_timeout_shim, router_timeout_shim_may_follow
from agent.fast_mode import effective_request_overrides
from agent.turn_context import substitute_api_content
@@ -3571,9 +3571,9 @@ class _StreamingCall(StreamingWaitMonitor):
if self._request_cancelled["value"]:
logger.debug("Streaming worker caught %s after request cancellation — exiting without retry.", type(e).__name__)
return False
- _is_timeout = isinstance(e, (_httpx.ReadTimeout, _httpx.ConnectTimeout, _httpx.PoolTimeout))
- # ReadError: abort/reset mid-body (stale-kill shutdown under a parked reader,
- # ECONNRESET) — the retry loop owns recovery.
+ e = self._as_lifetime_error(e) # a cap kill: transient, and its copy names the setting
+ _is_timeout = isinstance(e, (_httpx.ReadTimeout, _httpx.ConnectTimeout, _httpx.PoolTimeout, StreamLifetimeExceeded))
+ # ReadError: abort/reset mid-body (stale-kill shutdown under a parked reader, ECONNRESET) — retry loop owns it.
# anthropic.APIConnectionError: the Anthropic SDK wraps connect/read drops
# (incl. stale-kill aborts) in its own type, not httpx's.
_is_conn_err = isinstance(e, (_httpx.ConnectError, _httpx.ReadError, _httpx.RemoteProtocolError, ConnectionError,
diff --git a/agent/chat_completion_stream_monitor.py b/agent/chat_completion_stream_monitor.py
index c6c45670c26..de36da61680 100644
--- a/agent/chat_completion_stream_monitor.py
+++ b/agent/chat_completion_stream_monitor.py
@@ -15,7 +15,7 @@ from agent import chat_completion_wait_notice as wn
from agent.model_metadata import is_local_endpoint
from agent.proxy_bypass import is_loopback_host
from agent.sleep_clock import SleepWatch
-from agent.stream_max_lifetime import StreamLifetime, resolve_stream_max_lifetime
+from agent.stream_max_lifetime import StreamLifetime, StreamLifetimeExceeded, resolve_stream_max_lifetime
logger = logging.getLogger(__name__)
# Kills of one dispatched attempt whose worker never unwinds before the monitor gives up on it.
@@ -32,6 +32,7 @@ class StreamingWaitMonitor:
# are withheld (None = claim liveness as usual). A real chunk after it lifts the hold.
_liveness_withheld_at = None
_lifetime = StreamLifetime(0.0) # opt-in agent.stream_max_lifetime; real clock set when the monitor starts
+ _lifetime_kill = None # (attempt id, elapsed s) of the latest cap kill
def _claims_liveness(self) -> bool:
since = self._liveness_withheld_at
@@ -192,8 +193,9 @@ class StreamingWaitMonitor:
def _check_stream_lifetime(self) -> None:
"""Opt-in ``agent.stream_max_lifetime``: a stream that keeps sending chunks but never
finishes never trips the stale check. Kill it like a stale kill (cancel the attempt,
- abort the request-local client, count it in the cross-turn breaker) so the retry loop
- or the partial-delivery continuation takes over."""
+ abort the request-local client) so the retry loop or the partial-delivery continuation
+ takes over. Never counted in the cross-turn stale breaker: the provider WAS responding,
+ and a streak of cap kills would refuse the next healthy request with nothing sent."""
if not self._lifetime.cap: # off (the default): no per-tick work
return
with self.stream_attempt_lock:
@@ -208,13 +210,28 @@ class StreamingWaitMonitor:
f"⚠️ Response still streaming after {int(elapsed)}s (stream_max_lifetime "
f"{int(self._lifetime.cap)}s, model: {model}). Aborting...")
killed = self._attempt_stream_response
+ self._lifetime_kill = (attempt, elapsed) # before the abort: the woken worker reads it at once
with contextlib.suppress(Exception):
self._cancel_current_stream_attempt("stream_max_lifetime_kill")
self.clients.close_once("stream_max_lifetime_kill")
self._shutdown_stale_attempt_socket(killed)
- self._count_stale_attempt()
+ # Accounted, never bumped: a stale re-kill or /stop while it unwinds must not strike it either.
+ self._stale_counted_attempts.add(attempt)
self.last_chunk_time["t"] = time.time() # the stale timer restarts with the kill
+ def _as_lifetime_error(self, e: Exception) -> Exception:
+ """The worker's transport error for an attempt the cap killed, replaced by one that names
+ ``agent.stream_max_lifetime`` (the retry, exhausted and partial-delivery lines show it)."""
+ kill = self._lifetime_kill
+ if kill is None or isinstance(e, StreamLifetimeExceeded): # no cap kill: touch nothing else
+ return e
+ with self.stream_attempt_lock:
+ if kill[0] != int(self.stream_attempt_state["current"]):
+ return e
+ err = StreamLifetimeExceeded(kill[1], self._lifetime.cap)
+ err.__cause__ = e
+ return err
+
def _monitor_loop(self) -> None:
_HEARTBEAT_INTERVAL = 30.0 # seconds between gateway activity touches
self._mon = SimpleNamespace(
diff --git a/agent/stream_diag.py b/agent/stream_diag.py
index 10af9ec5d71..f8dcdd8d08d 100644
--- a/agent/stream_diag.py
+++ b/agent/stream_diag.py
@@ -207,6 +207,9 @@ def connect_exhausted_notice(error: BaseException, *, attempts: int, base_url: A
names the host, the attempt count and the serialized request size so a body-size limit is
distinguishable from an outage without reading agent.log."""
from urllib.parse import urlparse
+ from agent.stream_max_lifetime import StreamLifetimeExceeded
+ if isinstance(error, StreamLifetimeExceeded): # the stream DID open; the opt-in cap cut every attempt
+ return f"❌ {error} ({attempts} attempt{'s' if attempts != 1 else ''})"
request = _failed_request(error)
host = urlparse(str(getattr(request, "url", None) or base_url or "")).hostname or "the endpoint"
size = _request_body_bytes(request)
diff --git a/agent/stream_max_lifetime.py b/agent/stream_max_lifetime.py
index 1b0c8d45d7d..712f63e3a6c 100644
--- a/agent/stream_max_lifetime.py
+++ b/agent/stream_max_lifetime.py
@@ -40,6 +40,17 @@ def resolve_stream_max_lifetime(stale_timeout: float | None) -> float:
return cap
+class StreamLifetimeExceeded(TimeoutError):
+ """A stream attempt cut by ``agent.stream_max_lifetime``: the stream opened and was still
+ delivering chunks, so the user-facing error names the setting, not the endpoint."""
+
+ def __init__(self, elapsed: float, cap: float):
+ super().__init__(
+ f"Response still streaming after {elapsed:.0f}s, cut by agent.stream_max_lifetime ({cap:g}s). "
+ f"Raise agent.stream_max_lifetime, or set it to 0 to turn the cap off, if this model "
+ f"legitimately streams this long.")
+
+
class StreamLifetime:
"""Per-call clock: restarts whenever a new stream attempt is dispatched, so each
retry gets the full cap, and after each kill (the killed attempt may unwind slowly)."""
diff --git a/tests/agent/test_stream_max_lifetime.py b/tests/agent/test_stream_max_lifetime.py
index ec59bd3096f..52be8958b2f 100644
--- a/tests/agent/test_stream_max_lifetime.py
+++ b/tests/agent/test_stream_max_lifetime.py
@@ -7,17 +7,17 @@ import threading
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
-import httpx
import pytest
import run_agent
from agent.chat_completion_helpers import interruptible_streaming_api_call
-from agent.stream_max_lifetime import resolve_stream_max_lifetime
+from agent.stream_max_lifetime import StreamLifetimeExceeded, resolve_stream_max_lifetime
from hermes_cli.config_defaults import DEFAULT_CONFIG
class _DripWire:
- """Real OpenAI-wire SSE server: one content chunk every 50 ms for ~8 s, then a clean finish."""
+ """Real OpenAI-wire SSE server: one content chunk every 50 ms for ~8 s, then a clean finish
+ (a "short" prompt gets one chunk and the finish)."""
def __init__(self):
class Handler(BaseHTTPRequestHandler):
@@ -25,7 +25,8 @@ class _DripWire:
pass
def do_POST(self):
- self.rfile.read(int(self.headers.get("content-length", 0)))
+ body = json.loads(self.rfile.read(int(self.headers.get("content-length", 0))) or b"{}")
+ short = (body.get("messages") or [{}])[-1].get("content") == "short"
if not self.path.endswith("/chat/completions"):
self.send_response(404)
self.end_headers()
@@ -37,7 +38,7 @@ class _DripWire:
"choices": [{"index": 0, "delta": {"content": "."}, "finish_reason": None}]}
fin = {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}
try:
- for _ in range(160):
+ for _ in range(1 if short else 160):
self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())
self.wfile.flush()
time.sleep(0.05) # health: allow HX013 -- the drip cadence is the behaviour under test
@@ -63,17 +64,24 @@ def test_cap_cuts_a_stream_that_never_goes_quiet(drip_wire, monkeypatch):
monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "1.0")
monkeypatch.setenv("HERMES_STREAM_MAX_LIFETIME", "1.5")
monkeypatch.setenv("HERMES_STREAM_RETRIES", "0")
+ monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "2")
agent = run_agent.AIAgent(
api_key="test-key", base_url=drip_wire.base_url, model="m", provider="custom",
platform="cli", quiet_mode=True, skip_context_files=True, skip_memory=True,
enabled_toolsets=[], max_iterations=1,
)
- started = time.time()
- # Without the cap the drip completes normally after ~8 s; with it the call ends early.
- # Killed like a stale kill: the undelivered stream surfaces the transport error to the loop.
- with pytest.raises(httpx.RemoteProtocolError):
- interruptible_streaming_api_call(agent, {"model": "m", "messages": [{"role": "user", "content": "hi"}]})
- assert time.time() - started < 5.0
+ def call(prompt):
+ return interruptible_streaming_api_call(agent, {"model": "m", "messages": [{"role": "user", "content": prompt}]})
+
+ # Without the cap the drip completes normally after ~8 s; with it each call ends early, with
+ # an error naming the setting (the stream opened, so no "endpoint unreachable" copy).
+ for _ in range(2):
+ started = time.time()
+ with pytest.raises(StreamLifetimeExceeded, match="agent.stream_max_lifetime"):
+ call("hi")
+ assert time.time() - started < 5.0
+ # Cap kills are not stale kills: the breaker (giveup 2) must not refuse a healthy request.
+ assert call("short").choices[0].message.content == "."
def test_cap_is_off_by_default_and_never_undercuts_the_stale_timeout(monkeypatch):
diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md
index a98e075efb8..7528fa348c5 100644
--- a/website/docs/reference/environment-variables.md
+++ b/website/docs/reference/environment-variables.md
@@ -830,7 +830,7 @@ Advanced per-platform knobs for throttling the outbound message batcher. Most us
| `HERMES_STREAM_READ_TIMEOUT` | Streaming socket read timeout in seconds (default: `120`). Auto-increased to `HERMES_API_TIMEOUT` for local providers. Increase if local LLMs time out during long code generation. |
| `HERMES_STREAM_STALE_TIMEOUT` | Stale stream detection timeout in seconds (default: `180`). Auto-disabled for local providers. Triggers connection kill if no chunks arrive within this window. |
| `HERMES_LOCAL_STREAM_STALE_TIMEOUT` | Stale stream ceiling for local providers (Ollama, oMLX, llama-cpp) in seconds (default: `900`). When the base stale timeout is at its default and a local endpoint is detected, this finite ceiling replaces the former infinite disable so a wedged local server eventually trips the detector instead of hanging forever. Also configurable via `agent.local_stream_stale_timeout` in `config.yaml`. |
-| `HERMES_STREAM_MAX_LIFETIME` | Opt-in total wall-clock cap in seconds for one streaming response attempt (default: `0`, off). Unlike the stale-stream detector it never resets on chunk arrival, so it stops a stream that keeps sending data but never finishes. Never fires before the effective stale-stream timeout. Also configurable via `agent.stream_max_lifetime` in `config.yaml`. |
+| `HERMES_STREAM_MAX_LIFETIME` | Opt-in total wall-clock cap in seconds for one streaming response attempt (default: `0`, off). Unlike the stale-stream detector it never resets on chunk arrival, so it stops a stream that keeps sending data but never finishes. Never fires before the effective stale-stream timeout. Applies to Chat Completions and Anthropic Messages streams only; Codex/Responses (`codex_responses`, including custom Responses providers) and Bedrock streams are not capped. A capped attempt is retried and does not count toward `HERMES_STREAM_STALE_GIVEUP`. Also configurable via `agent.stream_max_lifetime` in `config.yaml`. |
| `HERMES_STREAM_RETRIES` | Number of mid-stream reconnect attempts on transient network errors (default: `3`). |
| `HERMES_STREAM_STALE_GIVEUP` | Cross-turn circuit breaker: after this many consecutive stale kills (streaming or non-streaming) with no completed response, abort each call immediately with an actionable error instead of re-waiting out the stale timeout (default: `5`, `0` disables). Resets on any completed response, `/model` switch, fallback activation, or turn-start primary restore. |
| `HERMES_AGENT_TIMEOUT` | Gateway inactivity timeout for a running agent in seconds (default: `1800`, 30 minutes). Resets on every tool call and streamed token. Set to `0` to disable. |
diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md
index 7c4685b5e99..a7609572844 100644
--- a/website/docs/user-guide/configuration.md
+++ b/website/docs/user-guide/configuration.md
@@ -1380,7 +1380,7 @@ The **stale stream detection** kills connections that receive SSE keep-alive pin
The **Responses first-event watchdog** (Codex / `codex_responses` transport, including custom providers declared with the Responses transport) aborts and reconnects a request that accepts the connection but emits no stream event within 120 seconds. A local server prefilling a large context legitimately stays silent longer than that, so on local endpoints the implicit default is raised to the same ceiling as the stale stream detector (`agent.local_stream_stale_timeout` / `HERMES_LOCAL_STREAM_STALE_TIMEOUT`, 900s). An explicit `HERMES_CODEX_TTFB_TIMEOUT_SECONDS` is always used as-is (`0` disables the watchdog).
-The **stream max lifetime** is an opt-in cap on the total wall-clock time of one streaming response. The stale detector resets on every chunk, so a stream that keeps sending data but never finishes (a gateway trickling chunks, a runaway generation) is never cut. Set `agent.stream_max_lifetime` (seconds) to cut such a stream; the connection is killed the same way a stale kill is, and the retry or partial-response continuation takes over. It never fires before the effective stale-stream timeout. Leave it at `0` (the default) if you run long reasoning turns that legitimately stream for a long time.
+The **stream max lifetime** is an opt-in cap on the total wall-clock time of one streaming response. The stale detector resets on every chunk, so a stream that keeps sending data but never finishes (a gateway trickling chunks, a runaway generation) is never cut. Set `agent.stream_max_lifetime` (seconds) to cut such a stream; the connection is killed the same way a stale kill is, and the retry or partial-response continuation takes over. It never fires before the effective stale-stream timeout, and a cut attempt does not count toward the stale-stream circuit breaker (`HERMES_STREAM_STALE_GIVEUP`), since the provider was responding. The cap covers Chat Completions and Anthropic Messages streams only; Codex/Responses streams (`codex_responses`, including custom providers declared with the Responses transport) and Bedrock streams are not capped. Leave it at `0` (the default) if you run long reasoning turns that legitimately stream for a long time.
The **stale non-stream detection** kills non-streaming calls that produce no response for too long. By default Hermes disables this on local endpoints to avoid false positives during long prefills. If you explicitly set `providers.<id>.stale_timeout_seconds`, `providers.<id>.models.<model>.stale_timeout_seconds`, or `HERMES_API_CALL_STALE_TIMEOUT`, that explicit value is honored even on local endpoints.