The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(agent): fixed abandon limit, quiet a wedged inline call, keep the outcome
unsupportedpastels74877912d4dd
diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.pyindex ae964c9f32b..b4ca409edb2 100644--- a/agent/chat_completion_helpers.py+++ b/agent/chat_completion_helpers.py@@ -3872,14 +3872,13 @@ class _StreamingCall(StreamingWaitMonitor):returnself._stream_stale_timeout = _cloud_stale_timeout_for(self.agent, self.api_kwargs)- def _partial_stream_stub(self):+ def _partial_stream_stub(self, error):"""Tokens already reached the platform: a finish_reason="length" stub fires thecontinuation machinery; tool_calls=None blocks executing incomplete calls.Content may be EMPTY (dropped tool call, overflow) — the loop skips appending anempty stub and only sends the nudge (placeholder text leaked into the stitchedresponse). A text-only death with 0 visible chars never gets here: the errorhandler reclassifies it as undelivered (#112419)."""- error = self.result["error"]_partial_text = (getattr(self.agent, "_current_streamed_assistant_text", "") or "").strip() or None_partial_names = list(self.result.get("partial_tool_names") or [])if _partial_names:@@ -3957,19 +3956,23 @@ class _StreamingCall(StreamingWaitMonitor):raise InterruptedError("Agent interrupted during streaming API call")if self.agent._interrupt_requested: # worker returned early before the monitor saw the flagraise InterruptedError("Agent interrupted during streaming API call (post-worker)")- if self._abandon_error is not None: # the abandoned worker may still write into ``result``- self.result = {**self.result, "response": None, "error": self._abandon_error}- self._forget_invisible_deltas(self._abandon_error) # nothing visible: retry, not a stub- if self.result["error"] is not None:+ # Read the outcome once: an abandoned worker can still write into ``self.result`` later.+ error, response = self.result["error"], self.result["response"]+ if self._abandon_error is not None:+ # Same bounded Relay join as /stop (#81521), before the retry opens a new scope.+ _join_worker_for_relay_teardown(self.worker, label="Abandoned streaming")+ error, response = self._abandon_error, None+ self._forget_invisible_deltas(error) # nothing visible: retry, not a stub+ if error is not None:if self.deltas_were_sent["yes"]:- return self._partial_stream_stub()- raise self.result["error"]- if self.result["response"] is not None:+ return self._partial_stream_stub(error)+ raise error+ if response is not None:_reset_stale_streak(self.agent) # provider proved responsive: clear the breaker# Propagate first-chunk timing for the ``post_api_request`` hook.if isinstance(self.clients.diag, dict) and self.clients.diag.get("first_chunk_at"):self.agent._last_api_first_chunk_at = float(self.clients.diag["first_chunk_at"])- return self.result["response"]+ return responsedef interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta=None):diff --git a/agent/chat_completion_stream_monitor.py b/agent/chat_completion_stream_monitor.pyindex 7f5ca23673c..bcbdf891e38 100644--- a/agent/chat_completion_stream_monitor.py+++ b/agent/chat_completion_stream_monitor.py@@ -1,4 +1,9 @@-"""Display and heartbeat phase of the request-local streaming monitor."""+"""Display and heartbeat phase of the request-local streaming monitor.++Also holds the stale-kill follow-up (re-arm, abandon a wedged worker, stop claiming liveness+for a wedged inline call) and the shared "no visible text was delivered" check+(``_forget_invisible_deltas``) used by the error and abandon paths.+"""import loggingimport time@@ -9,10 +14,11 @@ from agent import chat_completion_wait_notice as wnfrom agent.model_metadata import is_local_endpointfrom agent.proxy_bypass import is_loopback_hostfrom agent.sleep_clock import SleepWatch-from utils import env_int-logger = logging.getLogger(__name__)+# Kills of one dispatched attempt whose worker never unwinds before the monitor gives up on it.+_STALE_ABANDON_KILLS = 3+class StreamingWaitMonitor:# Stale-kill bookkeeping: consecutive kills of the SAME dispatched attempt, and the error@@ -20,12 +26,21 @@ class StreamingWaitMonitor:_last_killed_attempt = 0_same_attempt_kills = 0_abandon_error = None+ # Inline call wedged past the limit: the stale-timer re-arm time after which activity touches+ # are withheld (None = claim liveness as usual). A real chunk after it lifts the hold.+ _liveness_withheld_at = None++ def _claims_liveness(self) -> bool:+ since = self._liveness_withheld_at+ return since is None or self.last_chunk_time["t"] > sincedef _rearm_or_abandon_after_kill(self, elapsed: float) -> None:"""After a stale kill: re-arm the stale timer and announce the reconnect, unless the- same dispatched attempt has now been killed ``HERMES_STREAM_STALE_ABANDON_KILLS``- times with its worker still alive. Then the worker is parked where a socket shutdown- cannot reach it, and re-killing it every window would stall the turn forever."""+ same dispatched attempt has now been killed ``_STALE_ABANDON_KILLS`` times without+ unwinding. Then the request is parked where a socket shutdown cannot reach it, and+ re-killing it every window would stall the turn forever: a worker thread is abandoned;+ an inline call (cron, delegated children) runs on the caller's thread and cannot be,+ so the monitor stops touching activity and the owner's inactivity watchdog ends it."""# Reset the timer so we don't kill repeatedly while the worker unwinds.self.last_chunk_time["t"] = time.time()with self.stream_attempt_lock:@@ -33,12 +48,21 @@ class StreamingWaitMonitor:rekill = bool(attempt) and attempt == self._last_killed_attemptself._last_killed_attempt = attemptself._same_attempt_kills = self._same_attempt_kills + 1 if rekill else 1- limit = env_int("HERMES_STREAM_STALE_ABANDON_KILLS", 3) # 0 or less: never abandon- if attempt and self.worker is not None and 0 < limit <= self._same_attempt_kills:- self._abandon_wedged_worker(elapsed, attempt)- return+ if not rekill:+ self._liveness_withheld_at = None+ if attempt and self._same_attempt_kills >= _STALE_ABANDON_KILLS:+ if self.worker is not None:+ self._abandon_wedged_worker(elapsed, attempt)+ return+ if self._liveness_withheld_at is None:+ logger.error(+ "Inline streaming call did not unwind after %d stale kills of attempt %s; no longer "+ "reporting it as active, so the inactivity watchdog can end it. model=%s",+ self._same_attempt_kills, attempt, self.api_kwargs.get("model", "unknown"))+ self._liveness_withheld_at = self.last_chunk_time["t"]self.agent._emit_diagnostic_wait(f"⚠ no output from provider for {int(elapsed)}s — reconnecting...")- self.agent._touch_activity(f"stale stream detected after {int(elapsed)}s, reconnecting")+ if self._claims_liveness():+ self.agent._touch_activity(f"stale stream detected after {int(elapsed)}s, reconnecting")def _forget_invisible_deltas(self, exc) -> bool:"""Deltas fired but nothing visible reached a consumer (whitespace/think-only deltas,@@ -142,7 +166,9 @@ class StreamingWaitMonitor:def _heartbeat(self, waiting_secs: int) -> None:"""Gateway inactivity heartbeat: the start-to-first-chunk gap (thinking,- local prefill) can exceed the gateway timeout."""+ local prefill) can exceed the gateway timeout. Withheld for a wedged inline call."""+ if not self._claims_liveness():+ returnif waiting_secs >= 60.0:# No chunks for 60s+: say WHAT the wait is and WHEN recovery kicks in —# once per silence, not every heartbeat (#92550).diff --git a/tests/agent/test_stream_stale_abandon.py b/tests/agent/test_stream_stale_abandon.pyindex c8772f63970..c84a4e6c549 100644--- a/tests/agent/test_stream_stale_abandon.py+++ b/tests/agent/test_stream_stale_abandon.py@@ -4,9 +4,10 @@ Incident: the monitor killed the same Anthropic stream attempt every stale windohours. The worker never unwound (parked outside the socket read, so the socketshutdown could not wake it), ``_count_stale_attempt`` counts each attempt once so thestale streak stayed at 1, ``_check_stale_giveup`` only runs before a call, and only-/stop ended the turn. Now the call abandons the worker after-``HERMES_STREAM_STALE_ABANDON_KILLS`` kills of the same attempt and raises a retryable-``TimeoutError``, the way the /stop path already leaves a worker behind.+/stop ended the turn. Now the call abandons the worker after ``_STALE_ABANDON_KILLS``+kills of the same attempt and raises a retryable ``TimeoutError``, the way the /stop path+already leaves a worker behind. An inline call (cron, delegated children) cannot be+abandoned, so it stops reporting activity and the owner's inactivity watchdog ends it."""import threading@@ -17,6 +18,7 @@ import httpximport pytestfrom agent import chat_completion_helpers as helpers+from agent import chat_completion_stream_monitor as monitor_ABANDONED = "request abandoned"@@ -91,33 +93,20 @@ def test_wedged_worker_is_abandoned_with_a_retryable_timeout(wedged, caplog):assert not caller.is_alive(), "the call never ended: the same wedged attempt was re-killed forever"assert isinstance(outcome.get("error"), TimeoutError), outcomeassert _ABANDONED in str(outcome["error"])- assert _log_count(caplog, "Stream stale for") == 3 # HERMES_STREAM_STALE_ABANDON_KILLS default+ assert _log_count(caplog, "Stream stale for") == 3 # _STALE_ABANDON_KILLSassert _log_count(caplog, "abandoning the request") == 1assert wedged._consecutive_stale_streams == 1, "one wedged attempt must still count once"[email protected]("ignore::pytest.PytestUnhandledThreadExceptionWarning")-def test_late_write_from_the_abandoned_worker_does_not_replace_the_error(wedged, monkeypatch):- real_abandon = helpers._StreamingCall._abandon_wedged_worker- late = SimpleNamespace(late=True)-- def _abandon_then_late_write(self, *args):- real_abandon(self, *args)- self.result["response"] = late # the daemon worker wakes up and writes after the verdict-- monkeypatch.setattr(helpers._StreamingCall, "_abandon_wedged_worker", _abandon_then_late_write)- caller, outcome = _call_in_thread(wedged)- caller.join(timeout=10.0)-- assert not caller.is_alive()- assert "value" not in outcome, "a late write from the abandoned worker became the call's result"- assert isinstance(outcome.get("error"), TimeoutError), outcome--@pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning")def test_worker_that_unwinds_after_a_kill_still_retries(monkeypatch, caplog):"""The common path: the kill unblocks the read, the worker retries with a fresh- attempt, and no attempt is killed twice, so nothing is abandoned."""+ attempt, and no attempt is killed twice, so nothing is abandoned.++ Relies on ``tests/agent/conftest.py`` zeroing ``jittered_backoff``: the attempt id moves+ only when the worker STARTS its next attempt, so with this 0.3 s stale window kills+ landing in a real 1-4 s retry backoff would count as the same attempt. In production+ the stale window (180 s or more) dwarfs that backoff."""monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "0.3")monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "5")monkeypatch.setenv("HERMES_STREAM_RETRIES", "2")@@ -145,67 +134,9 @@ def test_worker_that_unwinds_after_a_kill_still_retries(monkeypatch, caplog):assert _log_count(caplog, "abandoning the request") == 0-# ── turn level: the abandon error is retried and the turn completes ──────────---def _resp(content):- msg = SimpleNamespace(content=content, tool_calls=None)- return SimpleNamespace(choices=[SimpleNamespace(message=msg, finish_reason="stop")],- model="test/model", usage=None)---def _abandon_error():- """The exact error a real abandonment produces."""- call = helpers._StreamingCall.__new__(helpers._StreamingCall)- call.api_kwargs, call._same_attempt_kills = {"model": "m"}, 3- call._request_cancelled, call._call_done = {"value": False}, threading.Event()- call._abandon_wedged_worker(720.0, 1)- assert call._request_cancelled["value"] and call._call_done.is_set()- return call._abandon_error---def test_turn_retries_an_abandoned_stream_and_completes(monkeypatch):- monkeypatch.setattr("hermes_cli.plugins.discover_plugins", lambda: None)- session_ends = []- monkeypatch.setattr("hermes_cli.lifecycle.invoke_hook",- lambda name, **kw: session_ends.append(kw) if name == "on_session_end" else [])- tool = {"type": "function", "function": {"name": "web_search", "description": "web_search",- "parameters": {"type": "object", "properties": {}}}}- from run_agent import AIAgent-- with (patch("model_tools.get_tool_definitions", return_value=[tool]),- patch("model_tools.check_toolset_requirements", return_value={}),- patch("agent.process_bootstrap.OpenAI")):- agent = AIAgent(api_key="test-key-1234567890", base_url="https://openrouter.ai/api/v1",- quiet_mode=True, skip_context_files=True, skip_memory=True)- agent.client = MagicMock()- agent._cached_system_prompt = "You are helpful."- agent._use_prompt_caching = False- agent.compression_enabled = False- agent.save_trajectories = False- agent._persist_session = lambda *a, **k: None- agent._save_trajectory = lambda *a, **k: None- abandoned = _abandon_error()- calls = []-- def _api(api_kwargs):- calls.append(1)- if len(calls) == 1:- raise abandoned- return _resp("done after retry")-- agent._interruptible_api_call = _api- with patch("run_agent.time.sleep", return_value=None), patch("time.sleep", return_value=None):- result = agent.run_conversation("hi")-- assert len(calls) == 2- assert result.get("completed") is True and result.get("final_response") == "done after retry"- assert len(session_ends) == 1 and session_ends[0]["completed"] is True--@pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning")def test_abandon_limit_of_one_abandons_on_the_first_kill(wedged, monkeypatch, caplog):- monkeypatch.setenv("HERMES_STREAM_STALE_ABANDON_KILLS", "1")+ monkeypatch.setattr(monitor, "_STALE_ABANDON_KILLS", 1, raising=False)with caplog.at_level("WARNING"):caller, outcome = _call_in_thread(wedged)caller.join(timeout=10.0)@@ -244,3 +175,95 @@ def test_abandon_after_invisible_deltas_retries_instead_of_an_empty_stub(monkeypassert not caller.is_alive()assert "value" not in outcome, f"an empty partial stub was returned: {outcome.get('value')!r}"assert isinstance(outcome.get("error"), TimeoutError), outcome++[email protected]("ignore::pytest.PytestUnhandledThreadExceptionWarning")+def test_wedged_inline_call_stops_reporting_activity(wedged, monkeypatch, caplog):+ """Cron turns and delegated children run the request inline, so there is no worker to+ abandon. Past the kill limit the monitor must stop touching activity (kills and+ heartbeat), so the cron / delegate inactivity watchdog can end the job."""+ monkeypatch.setattr(helpers, "should_use_direct_api_call", lambda agent: True)+ monkeypatch.setattr(monitor, "_STALE_ABANDON_KILLS", 3, raising=False)+ touches, calls = [], []+ real_touch = wedged._touch_activity+ monkeypatch.setattr(wedged, "_touch_activity", lambda desc, **kw: (touches.append(desc), real_touch(desc, **kw)))+ real_rearm = helpers._StreamingCall._rearm_or_abandon_after_kill++ def _rearm(self, elapsed):+ calls.append(self)+ real_rearm(self, elapsed)++ monkeypatch.setattr(helpers._StreamingCall, "_rearm_or_abandon_after_kill", _rearm)+ with caplog.at_level("WARNING"):+ caller, _outcome = _call_in_thread(wedged)+ caller.join(timeout=2.5) # about 8 stale windows: well past the limit+ kills = _log_count(caplog, "Stream stale for")+ stale_touches = [t for t in touches if t.startswith("stale stream detected")]+ call = calls[-1]+ before = len(touches)+ call._heartbeat(10)+ call._heartbeat(90)+ heartbeat_touches = len(touches) - before++ assert caller.is_alive(), "an inline call runs on the caller's thread and is not abandoned"+ assert kills > 3+ assert len(stale_touches) == 2, f"kills past the limit still reported activity: {stale_touches}"+ assert heartbeat_touches == 0, "the heartbeat kept reporting a wedged inline call as active"+ assert _log_count(caplog, "no longer reporting it as active") == 1+ assert _log_count(caplog, "abandoning the request") == 0+ # A real chunk lifts the hold.+ call.last_chunk_time["t"] = call._liveness_withheld_at + 1+ call._heartbeat(10)+ assert len(touches) > before++[email protected]("ignore::pytest.PytestUnhandledThreadExceptionWarning")+def test_abandon_after_visible_text_returns_a_stub_classified_as_the_abandon(monkeypatch):+ """Visible text already reached the user: the abandon ends in the partial-delivery stub,+ and the stub is classified from the abandon error, not from whatever the worker left."""+ monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "0.3")+ monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "5")+ agent = _anthropic_agent()+ agent._abort_request_anthropic_client = lambda client, reason=None: None+ agent._current_streamed_assistant_text = "Hello there"+ monkeypatch.setattr(helpers._StreamingCall, "_visible_text_delivered", lambda self: True)+ classified = []+ from agent import error_classifier++ real_classify = error_classifier.classify_api_error+ monkeypatch.setattr(error_classifier, "classify_api_error",+ lambda error, **kw: (classified.append(error), real_classify(error, **kw))[1])+ release = threading.Event()++ def _events():+ yield SimpleNamespace(type="content_block_start", index=0,+ content_block=SimpleNamespace(type="text", text=""))+ yield SimpleNamespace(type="content_block_delta", index=0,+ delta=SimpleNamespace(type="text_delta", text="Hello there"))+ release.wait(timeout=30.0)+ yield SimpleNamespace(type="message_stop")++ agent._anthropic_client.messages.stream.side_effect = _stream_of(_events)+ try:+ caller, outcome = _call_in_thread(agent)+ caller.join(timeout=10.0)+ finally:+ release.set()++ assert not caller.is_alive()+ assert "value" in outcome, outcome+ assert classified and isinstance(classified[-1], TimeoutError) and _ABANDONED in str(classified[-1])++[email protected]("ignore::pytest.PytestUnhandledThreadExceptionWarning")+def test_abandon_joins_the_worker_for_relay_teardown_like_stop(wedged, monkeypatch):+ """The in-turn retry opens a new Relay scope, so the abandon gets the same bounded+ join as /stop before raising (#81521)."""+ joined = []+ monkeypatch.setattr(helpers, "_join_worker_for_relay_teardown",+ lambda worker, *, label: joined.append((worker, label)))+ caller, outcome = _call_in_thread(wedged)+ caller.join(timeout=10.0)++ assert isinstance(outcome.get("error"), TimeoutError), outcome+ assert len(joined) == 1 and joined[0][0] is not None and joined[0][1] == "Abandoned streaming"