The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(streaming): count only kills of an attempt the worker is still inside
Teknium18d604189084
diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.pyindex 7ec81fa2fef..3f016294197 100644--- a/agent/chat_completion_helpers.py+++ b/agent/chat_completion_helpers.py@@ -3728,8 +3728,7 @@ class _StreamingCall(StreamingWaitMonitor):while _stream_attempt < _max_stream_retries + self._compat_retries:_stream_attempt += 1stream_attempt_id = self._start_stream_attempt()- # Otherwise /stop closes the connection and the retry opens a- # FRESH one, blocking up to a full read timeout per attempt.+ # Else /stop closes the socket and the retry opens a fresh one (a full read timeout each).if self.agent._interrupt_requested:self._cancel_current_stream_attempt("interrupt_before_stream_retry")raise InterruptedError("Agent interrupted before stream retry")@@ -3738,6 +3737,7 @@ class _StreamingCall(StreamingWaitMonitor):self.agent, lambda: self._call_wire(stream_attempt_id))return # successexcept Exception as e:+ self._released_attempt = stream_attempt_id # unwound: kills during the backoff are not wedge killsself._close_managed_stream()if not self._handle_stream_error(e, _stream_attempt, _max_stream_retries):returndiff --git a/agent/chat_completion_stream_monitor.py b/agent/chat_completion_stream_monitor.pyindex f55ac534395..82e09fbcbf3 100644--- a/agent/chat_completion_stream_monitor.py+++ b/agent/chat_completion_stream_monitor.py@@ -33,6 +33,7 @@ class StreamingWaitMonitor:_liveness_withheld_at = None_lifetime = StreamLifetime(0.0) # opt-in agent.stream_max_lifetime; real clock set when the monitor starts_lifetime_kill = None # (attempt id, elapsed s) of the latest cap kill+ _released_attempt = 0 # the attempt the worker's except path last caught (it let go of it)def _claims_liveness(self) -> bool:since = self._liveness_withheld_at@@ -41,8 +42,9 @@ class StreamingWaitMonitor:def _rearm_or_abandon_after_kill(self, elapsed: float) -> None:"""After a stale kill: re-arm the stale timer and announce the reconnect, unless thekill bookkeeping gave up on a wedged attempt (see ``_note_attempt_kill``)."""- # Reset the timer so we don't kill repeatedly while the worker unwinds.- self.last_chunk_time["t"] = time.time()+ # Reset the timer so we don't kill repeatedly while the worker unwinds; the cap clock+ # too, so a cap at the stale timeout does not cut (and count) the same window again.+ self.last_chunk_time["t"] = self._lifetime.started = time.time()if self._note_attempt_kill(elapsed, "stale"):returnif self._claims_liveness(): # the reconnect notice touches activity too (_emit_wait_notice)@@ -58,6 +60,8 @@ class StreamingWaitMonitor:the monitor stops touching activity and the owner's inactivity watchdog ends it."""with self.stream_attempt_lock:attempt = int(self.stream_attempt_state["current"])+ if attempt == self._released_attempt: # retry backoff: the worker is not stuck in it+ return Falserekill = bool(attempt) and attempt == self._last_killed_attemptself._last_killed_attempt = attemptself._same_attempt_kills = self._same_attempt_kills + 1 if rekill else 1@@ -207,11 +211,12 @@ class StreamingWaitMonitor:abort the request-local client) so the retry loop or the partial-delivery continuationtakes over. Never counted in the cross-turn stale breaker: the provider WAS responding,and a streak of cap kills would refuse the next healthy request with nothing sent."""- if not self._lifetime.cap: # off (the default): no per-tick work+ if not self._lifetime.cap or self._call_done.is_set(): # off (the default) or call overreturnwith self.stream_attempt_lock:attempt = int(self.stream_attempt_state["current"])- elapsed = self._lifetime.exceeded(attempt) if attempt else None+ # A released attempt (retry backoff) has no stream left to cut.+ elapsed = self._lifetime.exceeded(attempt) if attempt and attempt != self._released_attempt else Noneif elapsed is None:returnmodel = self.api_kwargs.get("model", "unknown")diff --git a/tests/agent/test_stream_stale_abandon.py b/tests/agent/test_stream_stale_abandon.pyindex 18f3eb29865..c45a4ce9bd5 100644--- a/tests/agent/test_stream_stale_abandon.py+++ b/tests/agent/test_stream_stale_abandon.py@@ -277,20 +277,26 @@ def test_abandon_joins_the_worker_for_relay_teardown_like_stop(wedged, monkeypat@pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning")-def test_wedged_worker_under_the_lifetime_cap_is_abandoned_with_stale_off(wedged, monkeypatch, tmp_path, caplog):[email protected]("stale,cap", [("inf", "1"), ("0.3", "0.3")], ids=["stale_off", "cap_at_stale"])+def test_wedged_worker_under_the_lifetime_cap_is_abandoned_once(wedged, monkeypatch, tmp_path, caplog, stale, cap):"""agent.stream_max_lifetime kills that cannot unwind the worker get the same give-up as- stale kills: with the stale detector off, the cap alone must still end the turn."""+ stale kills: with the stale detector off, the cap alone must still end the turn. With the+ cap at the stale timeout both checks fire on the same tick; one silent window is one kill,+ and the call is abandoned once, with the stale error (no chunk ever arrived)."""from agent.stream_max_lifetime import StreamLifetimeExceeded- monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "inf")- (tmp_path / "config.yaml").write_text("agent:\n stream_max_lifetime: 1\n")+ monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", stale)+ (tmp_path / "config.yaml").write_text(f"agent:\n stream_max_lifetime: {cap}\n")monkeypatch.setenv("HERMES_HOME", str(tmp_path))with caplog.at_level("WARNING"):caller, outcome = _call_in_thread(wedged)caller.join(timeout=10.0)+ capped = stale == "inf"assert not caller.is_alive(), "the cap re-killed the same wedged attempt forever"- assert isinstance(outcome.get("error"), StreamLifetimeExceeded), outcome- assert _log_count(caplog, "stream_max_lifetime 1s") == 3 # _STALE_ABANDON_KILLS- assert _log_count(caplog, "Stream stale for") == 0- assert wedged._consecutive_stale_streams == 0, "cap kills must not feed the stale breaker"+ assert isinstance(outcome.get("error"), StreamLifetimeExceeded) is capped, outcome+ assert isinstance(outcome.get("error"), TimeoutError), outcome+ assert _log_count(caplog, "Stream still running after") == (3 if capped else 0) # _STALE_ABANDON_KILLS+ assert _log_count(caplog, "Stream stale for") == (0 if capped else 3)+ assert _log_count(caplog, "abandoning the request") == 1+ assert wedged._consecutive_stale_streams == (0 if capped else 1), "cap kills must not feed the stale breaker"