The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(streaming): give up on a worker that stream_max_lifetime kills cannot unwind
Tekniumc552db90bc9f
diff --git a/agent/chat_completion_stream_monitor.py b/agent/chat_completion_stream_monitor.pyindex 81fdd8aee81..f55ac534395 100644--- a/agent/chat_completion_stream_monitor.py+++ b/agent/chat_completion_stream_monitor.py@@ -40,13 +40,22 @@ class StreamingWaitMonitor:def _rearm_or_abandon_after_kill(self, elapsed: float) -> None:"""After a stale kill: re-arm the stale timer and announce the reconnect, unless the- same dispatched attempt has now been killed ``_STALE_ABANDON_KILLS`` times without- unwinding. Then the request is parked where a socket shutdown cannot reach it, and- re-killing it every window would stall the turn forever: a worker thread is abandoned;- an inline call (cron, delegated children) runs on the caller's thread and cannot be,- so the monitor stops touching activity and the owner's inactivity watchdog ends it."""+ kill bookkeeping gave up on a wedged attempt (see ``_note_attempt_kill``)."""# Reset the timer so we don't kill repeatedly while the worker unwinds.self.last_chunk_time["t"] = time.time()+ if self._note_attempt_kill(elapsed, "stale"):+ return+ if self._claims_liveness(): # the reconnect notice touches activity too (_emit_wait_notice)+ self.agent._emit_diagnostic_wait(f"⚠ no output from provider for {int(elapsed)}s — reconnecting...")+ self.agent._touch_activity(f"stale stream detected after {int(elapsed)}s, reconnecting")++ def _note_attempt_kill(self, elapsed: float, kind: str) -> bool:+ """Count a stale or ``stream_max_lifetime`` kill of the current attempt. Once the same+ dispatched attempt has been killed ``_STALE_ABANDON_KILLS`` times without unwinding,+ the request is parked where a socket shutdown cannot reach it, and re-killing it every+ window would stall the turn forever: a worker thread is abandoned (returns True); an+ inline call (cron, delegated children) runs on the caller's thread and cannot be, so+ the monitor stops touching activity and the owner's inactivity watchdog ends it."""with self.stream_attempt_lock:attempt = int(self.stream_attempt_state["current"])rekill = bool(attempt) and attempt == self._last_killed_attempt@@ -54,19 +63,18 @@ class StreamingWaitMonitor:self._same_attempt_kills = self._same_attempt_kills + 1 if rekill else 1if not rekill:self._liveness_withheld_at = None- if attempt and self._same_attempt_kills >= _STALE_ABANDON_KILLS:- if self.worker is not None:- self._abandon_wedged_worker(elapsed, attempt)- return- if self._liveness_withheld_at is None:- logger.error(- "Inline streaming call did not unwind after %d stale kills of attempt %s; no longer "- "reporting it as active, so the inactivity watchdog can end it. model=%s",- self._same_attempt_kills, attempt, self.api_kwargs.get("model", "unknown"))- self._liveness_withheld_at = self.last_chunk_time["t"]- if self._claims_liveness(): # the reconnect notice touches activity too (_emit_wait_notice)- self.agent._emit_diagnostic_wait(f"⚠ no output from provider for {int(elapsed)}s — reconnecting...")- self.agent._touch_activity(f"stale stream detected after {int(elapsed)}s, reconnecting")+ if not attempt or self._same_attempt_kills < _STALE_ABANDON_KILLS:+ return False+ if self.worker is not None:+ self._abandon_wedged_worker(elapsed, attempt, kind)+ return True+ if self._liveness_withheld_at is None:+ logger.error(+ "Inline streaming call did not unwind after %d %s kills of attempt %s; no longer "+ "reporting it as active, so the inactivity watchdog can end it. model=%s",+ self._same_attempt_kills, kind, attempt, self.api_kwargs.get("model", "unknown"))+ self._liveness_withheld_at = self.last_chunk_time["t"]+ return Falsedef _forget_invisible_deltas(self, exc) -> bool:"""Deltas fired but nothing visible reached a consumer (whitespace/think-only deltas,@@ -85,17 +93,20 @@ class StreamingWaitMonitor:self.first_delta_fired["done"] = Falsereturn tool_in_flight- def _abandon_wedged_worker(self, elapsed: float, attempt: int) -> None:+ def _abandon_wedged_worker(self, elapsed: float, attempt: int, kind: str) -> None:"""Like the /stop path, leave the (daemon) worker behind and end this call with aretryable timeout; ``run()`` discards anything the worker writes later."""logger.error(- "Streaming worker did not unwind after %d stale kills of attempt %s (%.0fs since last chunk); "- "abandoning the request. model=%s", self._same_attempt_kills, attempt, elapsed,+ "Streaming worker did not unwind after %d %s kills of attempt %s (%.0fs); "+ "abandoning the request. model=%s", self._same_attempt_kills, kind, attempt, elapsed,self.api_kwargs.get("model", "unknown"))self._request_cancelled["value"] = True- self._abandon_error = TimeoutError(- f"Provider stream stalled: no chunks and the request worker did not unwind after "- f"{self._same_attempt_kills} stale-stream kills; request abandoned.")+ if kind == "stale":+ self._abandon_error = TimeoutError(+ f"Provider stream stalled: no chunks and the request worker did not unwind after "+ f"{self._same_attempt_kills} stale-stream kills; request abandoned.")+ else: # the copy names the setting, like a cap kill that did unwind+ self._abandon_error = StreamLifetimeExceeded(elapsed, self._lifetime.cap)self._call_done.set()def _poll_local_load_notice(self, now: float) -> bool:@@ -218,6 +229,9 @@ class StreamingWaitMonitor:# Accounted, never bumped: a stale re-kill or /stop while it unwinds must not strike it either.self._stale_counted_attempts.add(attempt)self.last_chunk_time["t"] = time.time() # the stale timer restarts with the kill+ # A worker the abort cannot unwind would be re-cut every window forever (the stale+ # detector may be off): same give-up as a stale kill, without its reconnect notice.+ self._note_attempt_kill(elapsed, "stream_max_lifetime")def _as_lifetime_error(self, e: Exception) -> Exception:"""The worker's transport error for an attempt the cap killed, replaced by one that namesdiff --git a/tests/agent/test_stream_stale_abandon.py b/tests/agent/test_stream_stale_abandon.pyindex bdbfc692e3b..18f3eb29865 100644--- a/tests/agent/test_stream_stale_abandon.py+++ b/tests/agent/test_stream_stale_abandon.py@@ -274,3 +274,23 @@ def test_abandon_joins_the_worker_for_relay_teardown_like_stop(wedged, monkeypatassert isinstance(outcome.get("error"), TimeoutError), outcomeassert len(joined) == 1 and joined[0][0] is not None and joined[0][1] == "Abandoned streaming"++[email protected]("ignore::pytest.PytestUnhandledThreadExceptionWarning")+def test_wedged_worker_under_the_lifetime_cap_is_abandoned_with_stale_off(wedged, monkeypatch, tmp_path, caplog):+ """agent.stream_max_lifetime kills that cannot unwind the worker get the same give-up as+ stale kills: with the stale detector off, the cap alone must still end the turn."""+ from agent.stream_max_lifetime import StreamLifetimeExceeded++ monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "inf")+ (tmp_path / "config.yaml").write_text("agent:\n stream_max_lifetime: 1\n")+ monkeypatch.setenv("HERMES_HOME", str(tmp_path))+ with caplog.at_level("WARNING"):+ caller, outcome = _call_in_thread(wedged)+ caller.join(timeout=10.0)++ assert not caller.is_alive(), "the cap re-killed the same wedged attempt forever"+ assert isinstance(outcome.get("error"), StreamLifetimeExceeded), outcome+ assert _log_count(caplog, "stream_max_lifetime 1s") == 3 # _STALE_ABANDON_KILLS+ assert _log_count(caplog, "Stream stale for") == 0+ assert wedged._consecutive_stale_streams == 0, "cap kills must not feed the stale breaker"