The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(agent): abandon a streaming worker that stale kills cannot unwind
unsupportedpastelsea751e5c8108
diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.pyindex 5ef48920097..c7455371c96 100644--- a/agent/chat_completion_helpers.py+++ b/agent/chat_completion_helpers.py@@ -3843,10 +3843,7 @@ class _StreamingCall(StreamingWaitMonitor):self.clients.close_once("stale_stream_kill")self._shutdown_stale_attempt_socket(_killed_response)self._count_stale_attempt()- # Reset the timer so we don't kill repeatedly while the worker unwinds.- self.last_chunk_time["t"] = time.time()- self.agent._emit_diagnostic_wait(f"⚠ no output from provider for {int(elapsed)}s — reconnecting...")- self.agent._touch_activity(f"stale stream detected after {int(elapsed)}s, reconnecting")+ self._rearm_or_abandon_after_kill(elapsed)def _abort_for_interrupt(self, stale_elapsed: float) -> None:"""/stop seen by the monitor: mark cancelled, abort the request-local@@ -3975,6 +3972,8 @@ class _StreamingCall(StreamingWaitMonitor):raise InterruptedError("Agent interrupted during streaming API call")if self.agent._interrupt_requested: # worker returned early before the monitor saw the flagraise InterruptedError("Agent interrupted during streaming API call (post-worker)")+ if self._abandon_error is not None: # the abandoned worker may still write into ``result``+ self.result = {**self.result, "response": None, "error": self._abandon_error}if self.result["error"] is not None:if self.deltas_were_sent["yes"]:return self._partial_stream_stub()diff --git a/agent/chat_completion_stream_monitor.py b/agent/chat_completion_stream_monitor.pyindex d0c1965682e..920848ca6ae 100644--- a/agent/chat_completion_stream_monitor.py+++ b/agent/chat_completion_stream_monitor.py@@ -9,11 +9,50 @@ from agent import chat_completion_wait_notice as wnfrom agent.model_metadata import is_local_endpointfrom agent.proxy_bypass import is_loopback_hostfrom agent.sleep_clock import SleepWatch+from utils import env_intlogger = logging.getLogger(__name__)class StreamingWaitMonitor:+ # Stale-kill bookkeeping: consecutive kills of the SAME dispatched attempt, and the error+ # that ends a call whose worker those kills could not unwind.+ _last_killed_attempt = 0+ _same_attempt_kills = 0+ _abandon_error = None++ def _rearm_or_abandon_after_kill(self, elapsed: float) -> None:+ """After a stale kill: re-arm the stale timer and announce the reconnect, unless the+ same dispatched attempt has now been killed ``HERMES_STREAM_STALE_ABANDON_KILLS``+ times with its worker still alive. Then the worker is parked where a socket shutdown+ cannot reach it, and re-killing it every window would stall the turn forever."""+ # Reset the timer so we don't kill repeatedly while the worker unwinds.+ self.last_chunk_time["t"] = time.time()+ with self.stream_attempt_lock:+ attempt = int(self.stream_attempt_state["current"])+ rekill = bool(attempt) and attempt == self._last_killed_attempt+ self._last_killed_attempt = attempt+ self._same_attempt_kills = self._same_attempt_kills + 1 if rekill else 1+ if rekill and self.worker is not None and self._same_attempt_kills >= env_int(+ "HERMES_STREAM_STALE_ABANDON_KILLS", 3):+ self._abandon_wedged_worker(elapsed, attempt)+ return+ self.agent._emit_diagnostic_wait(f"⚠ no output from provider for {int(elapsed)}s — reconnecting...")+ self.agent._touch_activity(f"stale stream detected after {int(elapsed)}s, reconnecting")++ def _abandon_wedged_worker(self, elapsed: float, attempt: int) -> None:+ """Like the /stop path, leave the (daemon) worker behind and end this call with a+ retryable timeout; ``run()`` discards anything the worker writes later."""+ logger.error(+ "Streaming worker did not unwind after %d stale kills of attempt %s (%.0fs since last chunk); "+ "abandoning the request. model=%s", self._same_attempt_kills, attempt, elapsed,+ self.api_kwargs.get("model", "unknown"))+ self._request_cancelled["value"] = True+ self._abandon_error = TimeoutError(+ f"Provider stream stalled: no chunks and the request worker did not unwind after "+ f"{self._same_attempt_kills} stale-stream kills; request abandoned.")+ self._call_done.set()+def _poll_local_load_notice(self, now: float) -> bool:"""Managed local server: surface a cold model's weight-load progressinstead of the 60s neutral "waiting on <model>" notice. Polled ~1s only while nodiff --git a/tests/agent/test_stream_stale_abandon.py b/tests/agent/test_stream_stale_abandon.pynew file mode 100644index 00000000000..603222c87d1--- /dev/null+++ b/tests/agent/test_stream_stale_abandon.py@@ -0,0 +1,203 @@+"""A streaming worker that stale kills cannot unwind must not stall the turn forever.++Incident: the monitor killed the same Anthropic stream attempt every stale window for+hours. The worker never unwound (parked outside the socket read, so the socket+shutdown could not wake it), ``_count_stale_attempt`` counts each attempt once so the+stale streak stayed at 1, ``_check_stale_giveup`` only runs before a call, and only+/stop ended the turn. Now the call abandons the worker after+``HERMES_STREAM_STALE_ABANDON_KILLS`` kills of the same attempt and raises a retryable+``TimeoutError``, the way the /stop path already leaves a worker behind.+"""++import threading+from types import SimpleNamespace+from unittest.mock import MagicMock, patch++import httpx+import pytest++from agent import chat_completion_helpers as helpers++_ABANDONED = "request abandoned"+++def _anthropic_agent():+ from run_agent import AIAgent++ agent = AIAgent(api_key="test-key", base_url="https://example.com/v1", model="claude-opus-4-7",+ quiet_mode=True, skip_context_files=True, skip_memory=True)+ agent.api_mode = "anthropic_messages"+ agent._anthropic_client = MagicMock()+ agent._anthropic_api_key = "test-anthropic-key"+ agent._create_request_anthropic_client = lambda *a, **k: agent._anthropic_client+ agent._consecutive_stale_streams = 0+ return agent+++def _stream_of(events_factory):+ def _open(*_args, **_kwargs):+ cm, raw = MagicMock(), MagicMock()+ raw.response = None+ raw.__iter__ = MagicMock(return_value=events_factory())+ cm.__enter__ = MagicMock(return_value=raw)+ cm.__exit__ = MagicMock(return_value=False)+ return cm+ return _open+++def _call_in_thread(agent):+ outcome: dict = {}++ def _run():+ try:+ outcome["value"] = agent._interruptible_streaming_api_call({}) # no model: no opus stale floor+ except BaseException as exc: # recorded for the assertions+ outcome["error"] = exc++ caller = threading.Thread(target=_run, name="stale-abandon-caller", daemon=True)+ caller.start()+ return caller, outcome+++def _log_count(caplog, needle):+ return sum(needle in r.getMessage() for r in caplog.records)+++def wedged(monkeypatch):+ """An Anthropic stream whose worker blocks on an Event that neither the client abort+ nor the socket shutdown can set: a worker parked outside the socket read."""+ monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "0.3")+ monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "5")+ agent = _anthropic_agent()+ agent._abort_request_anthropic_client = lambda client, reason=None: None+ release = threading.Event()++ def _events():+ release.wait(timeout=30.0)+ yield SimpleNamespace(type="message_stop")++ agent._anthropic_client.messages.stream.side_effect = _stream_of(_events)+ yield agent+ release.set() # let the abandoned daemon worker finish++[email protected]("ignore::pytest.PytestUnhandledThreadExceptionWarning")+def test_wedged_worker_is_abandoned_with_a_retryable_timeout(wedged, caplog):+ with caplog.at_level("WARNING"):+ caller, outcome = _call_in_thread(wedged)+ caller.join(timeout=10.0)++ assert not caller.is_alive(), "the call never ended: the same wedged attempt was re-killed forever"+ assert isinstance(outcome.get("error"), TimeoutError), outcome+ assert _ABANDONED in str(outcome["error"])+ assert _log_count(caplog, "Stream stale for") == 3 # HERMES_STREAM_STALE_ABANDON_KILLS default+ assert _log_count(caplog, "abandoning the request") == 1+ assert wedged._consecutive_stale_streams == 1, "one wedged attempt must still count once"++[email protected]("ignore::pytest.PytestUnhandledThreadExceptionWarning")+def test_late_write_from_the_abandoned_worker_does_not_replace_the_error(wedged, monkeypatch):+ real_abandon = helpers._StreamingCall._abandon_wedged_worker+ late = SimpleNamespace(late=True)++ def _abandon_then_late_write(self, *args):+ real_abandon(self, *args)+ self.result["response"] = late # the daemon worker wakes up and writes after the verdict++ monkeypatch.setattr(helpers._StreamingCall, "_abandon_wedged_worker", _abandon_then_late_write)+ caller, outcome = _call_in_thread(wedged)+ caller.join(timeout=10.0)++ assert not caller.is_alive()+ assert "value" not in outcome, "a late write from the abandoned worker became the call's result"+ assert isinstance(outcome.get("error"), TimeoutError), outcome++[email protected]("ignore::pytest.PytestUnhandledThreadExceptionWarning")+def test_worker_that_unwinds_after_a_kill_still_retries(monkeypatch, caplog):+ """The common path: the kill unblocks the read, the worker retries with a fresh+ attempt, and no attempt is killed twice, so nothing is abandoned."""+ monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "0.3")+ monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "5")+ monkeypatch.setenv("HERMES_STREAM_RETRIES", "2")+ agent = _anthropic_agent()+ unblock = threading.Event()+ agent._abort_request_anthropic_client = lambda client, reason=None: unblock.set()+ opened = []++ def _events():+ opened.append(1)+ unblock.wait(timeout=10.0)+ unblock.clear()+ raise httpx.RemoteProtocolError("peer closed connection without sending complete message body")+ yield # pragma: no cover - generator marker++ agent._anthropic_client.messages.stream.side_effect = _stream_of(_events)+ with caplog.at_level("WARNING"):+ caller, outcome = _call_in_thread(agent)+ caller.join(timeout=15.0)+ unblock.set()++ assert not caller.is_alive(), "the recoverable call never ended"+ assert _ABANDONED not in str(outcome.get("error")), outcome+ assert len(opened) == 3, f"each kill should lead to a fresh attempt (1 + 2 retries), saw {len(opened)}"+ assert _log_count(caplog, "abandoning the request") == 0+++# ── turn level: the abandon error is retried and the turn completes ──────────+++def _resp(content):+ msg = SimpleNamespace(content=content, tool_calls=None)+ return SimpleNamespace(choices=[SimpleNamespace(message=msg, finish_reason="stop")],+ model="test/model", usage=None)+++def _abandon_error():+ """The exact error a real abandonment produces."""+ call = helpers._StreamingCall.__new__(helpers._StreamingCall)+ call.api_kwargs, call._same_attempt_kills = {"model": "m"}, 3+ call._request_cancelled, call._call_done = {"value": False}, threading.Event()+ call._abandon_wedged_worker(720.0, 1)+ assert call._request_cancelled["value"] and call._call_done.is_set()+ return call._abandon_error+++def test_turn_retries_an_abandoned_stream_and_completes(monkeypatch):+ monkeypatch.setattr("hermes_cli.plugins.discover_plugins", lambda: None)+ session_ends = []+ monkeypatch.setattr("hermes_cli.lifecycle.invoke_hook",+ lambda name, **kw: session_ends.append(kw) if name == "on_session_end" else [])+ tool = {"type": "function", "function": {"name": "web_search", "description": "web_search",+ "parameters": {"type": "object", "properties": {}}}}+ from run_agent import AIAgent++ with (patch("model_tools.get_tool_definitions", return_value=[tool]),+ patch("model_tools.check_toolset_requirements", return_value={}),+ patch("agent.process_bootstrap.OpenAI")):+ agent = AIAgent(api_key="test-key-1234567890", base_url="https://openrouter.ai/api/v1",+ quiet_mode=True, skip_context_files=True, skip_memory=True)+ agent.client = MagicMock()+ agent._cached_system_prompt = "You are helpful."+ agent._use_prompt_caching = False+ agent.compression_enabled = False+ agent.save_trajectories = False+ agent._persist_session = lambda *a, **k: None+ agent._save_trajectory = lambda *a, **k: None+ abandoned = _abandon_error()+ calls = []++ def _api(api_kwargs):+ calls.append(1)+ if len(calls) == 1:+ raise abandoned+ return _resp("done after retry")++ agent._interruptible_api_call = _api+ with patch("run_agent.time.sleep", return_value=None), patch("time.sleep", return_value=None):+ result = agent.run_conversation("hi")++ assert len(calls) == 2+ assert result.get("completed") is True and result.get("final_response") == "done after retry"+ assert len(session_ends) == 1 and session_ends[0]["completed"] is True