The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(hold-stream): a held reply kept as the budget fallback, and its codex commentary, in history
Tekniumad249d9cac01
diff --git a/agent/history_commentary.py b/agent/history_commentary.pyindex 578ddc00eba..065074fa513 100644--- a/agent/history_commentary.py+++ b/agent/history_commentary.py@@ -23,6 +23,11 @@ def _owning_home(home):reset_hermes_home_override(token)+# display_metadata flag on a ``hold_stream`` turn's published reply: its commentary items stay for+# model replay, but the transform never saw them, so no history projection shows them.+COMMENTARY_WITHHELD = "commentary_withheld"++def visible_commentary(text: str, *, strip_thinking=None) -> str:"""Use the same think stripping and secret redaction as live delivery."""if strip_thinking is None:@@ -70,7 +75,7 @@ def _phase_message_items(message: dict, phases: frozenset[str | None]) -> list[sreturn result-def _commentary_items(message: dict) -> list[str]:+def commentary_items(message: dict) -> list[str]:return _phase_message_items(message, frozenset({"commentary"}))@@ -112,13 +117,13 @@ def _without_flattened_commentary(reasoning: str, commentary: list[str]) -> str:def _project_one(message: dict, *, enabled: bool) -> dict:if message.get("role") != "assistant" or message.get("display_kind") == "hidden":return message- raw = _commentary_items(message)+ raw = commentary_items(message)if not raw:return messageprojected = dict(message)# A disabled profile must not recover raw sidecar text on the frontend.projected["display_commentary"] = []- if enabled:+ if enabled and not (message.get("display_metadata") or {}).get(COMMENTARY_WITHHELD):projected["display_commentary"] = [part for text in raw if (part := visible_commentary(text))]diff --git a/agent/stream_delivery.py b/agent/stream_delivery.pyindex 5ecc3939a37..9b854342007 100644--- a/agent/stream_delivery.py+++ b/agent/stream_delivery.py@@ -10,7 +10,7 @@ from typing import Any, Dict, Listfrom agent.memory_context import sanitize_contextfrom agent.message_content import flatten_message_text-from agent.history_commentary import visible_commentary+from agent.history_commentary import COMMENTARY_WITHHELD, commentary_items, visible_commentary# Same logger name as the origin module so log records / caplog filters are unchanged.logger = logging.getLogger("run_agent")@@ -31,12 +31,15 @@ def withhold_held_row(agent: Any, assistant_msg: dict[str, Any]) -> dict[str, Andef publish_held_final_row(agent: Any, final_msg: dict[str, Any], final_response: Any, joined: bool) -> None:- """The held turn's final row shows what the user got: visible again, and carrying the whole- stitched reply when length-continuation fragments (hidden above) were joined into it."""+ """The held turn's final row shows what the user got: visible again, without the Codex+ commentary the transform never saw, and carrying the whole stitched reply when+ length-continuation fragments (hidden above) were joined into it."""if getattr(agent, "_hold_stream", False) is not True:returnif final_msg.get("display_kind") == "hidden":del final_msg["display_kind"]+ if commentary_items(final_msg): # Codex commentary rides the row's items, for replay only+ final_msg["display_metadata"] = {**(final_msg.get("display_metadata") or {}), COMMENTARY_WITHHELD: True}if joined and final_msg.get("content") and final_msg["content"] != final_response:final_msg.setdefault("api_content", final_msg["content"]) # the model keeps the piece it wrotefinal_msg["content"] = final_responsediff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.pyindex 7726d8cc4a1..d41b0f06f71 100644--- a/agent/turn_finalizer.py+++ b/agent/turn_finalizer.py@@ -284,13 +284,20 @@ def _close_transcript_tail(agent, messages, final_response, interrupted, _recove_apply_override(messages)-def _store_rewrite_on_tail(messages, raw, final) -> None:- """A recovery exit that already appended its text as the tail row (the iteration-budget summary)- stores the rewrite the user gets, not the model text, while that row is still unflushed (#44239)."""+def _store_rewrite_on_tail(agent, messages, raw, final) -> None:+ """A recovery exit whose text is already the unflushed tail row (the iteration-budget summary, a+ stop gate's answer kept as the budget fallback) stores the rewrite the user gets, not the model+ text (#44239). A held turn's hidden answer becomes the published reply; the model already saw+ it, so ``api_content`` keeps its bytes on the wire."""tail = messages[-1] if messages else None- if (final != raw and isinstance(tail, dict) and tail.get("role") == "assistant"- and tail.get("content") == raw and not tail.get(_DB_PERSISTED_MARKER)):- tail["content"] = final+ if not (isinstance(tail, dict) and tail.get("role") == "assistant" and tail.get("content") == raw+ and not tail.get(_DB_PERSISTED_MARKER)):+ return+ if tail.get("display_kind") == "hidden" and getattr(agent, "_hold_stream", False) is True:+ del tail["display_kind"]+ if final != raw:+ tail.setdefault("api_content", raw)+ tail["content"] = finaldef _micro_compact_after_turn(agent, messages, final_response, logger, task_id) -> None:@@ -688,7 +695,7 @@ def finalize_turn(if final_response and not interrupted:_raw = final_responsefinal_response, _, _ = apply_llm_output_transform(agent, final_response, turn_id=turn_id, logger=logger)- _store_rewrite_on_tail(messages, _raw, final_response)+ _store_rewrite_on_tail(agent, messages, _raw, final_response)_close_transcript_tail(agent, messages, final_response, interrupted, _recovered_from_stream)if not interrupted and not failed:_micro_compact_after_turn(agent, messages, final_response, logger, effective_task_id)diff --git a/agent/turn_stop_gates.py b/agent/turn_stop_gates.pyindex c3c24088cf6..7ac100a13bb 100644--- a/agent/turn_stop_gates.py+++ b/agent/turn_stop_gates.py@@ -119,9 +119,12 @@ def _late_lsp_verdicts(agent) -> Optional[str]:def _append_interim_answer(agent, final_msg, messages, conversation_history, flush_fail_msg: str) -> None:"""Real content: persist and emit as interim so the user sees the attempted answer;- only the nudge is flagged synthetic (#65919)."""+ only the nudge is flagged synthetic (#65919). A held turn's answer (typed hidden) is not shown+ and flushes with the turn instead, so a budget fallback can still publish it as the rewrite."""agent._emit_interim_assistant_message(final_msg)append_message(messages, final_msg)+ if final_msg.get("display_kind") == "hidden":+ returntry:agent._flush_messages_to_session_db(messages, conversation_history)except Exception:diff --git a/tests/agent/test_hold_stream_exits.py b/tests/agent/test_hold_stream_exits.pyindex 249a4c4f5a3..d705b22cb82 100644--- a/tests/agent/test_hold_stream_exits.py+++ b/tests/agent/test_hold_stream_exits.py@@ -14,6 +14,7 @@ from pathlib import Pathimport pytestfrom tests.fakes.fake_llm_provider import Error, FakeLLMServer, Text, ToolCall+from tests.fakes.providers.openai_responses import FakeResponsesServer, Message, Turnfrom tests.fakes.redactor_plugin import COMMENTARY, RAW, REWRITE, redactor_homeATTEMPT = "Done, the deploy key SECRET-321 is in place."@@ -22,15 +23,16 @@ ATTEMPT = "Done, the deploy key SECRET-321 is in place."def _agent(server, db, session_id: str, **kwargs):from run_agent import AIAgent+ kwargs.setdefault("enabled_toolsets", ["todo"])return AIAgent(base_url=server.base_url, api_key="sk-fake-e2e", provider="custom", model="fake-model",- quiet_mode=True, skip_context_files=True, skip_memory=True, enabled_toolsets=["todo"],+ quiet_mode=True, skip_context_files=True, skip_memory=True,session_id=session_id, session_db=db, **kwargs)@pytest.fixturedef held(tmp_path):- """``run(script, plugin_body=None, **agent_kwargs) -> (result, shown, displayed history, server, db)``- for one held turn."""+ """``run(script, plugin_body=None, server_cls=FakeLLMServer, **agent_kwargs)+ -> (result, shown, displayed history, server, db)`` for one held turn."""from gateway.session_controls import display_projectionfrom hermes_cli import pluginsfrom hermes_state import SessionDB@@ -38,16 +40,15 @@ def held(tmp_path):home = Path(os.environ["HERMES_HOME"])opened = []- def run(script, plugin_body=None, **kwargs):- server = FakeLLMServer(script)- server.start()- opened.append(server)+ def run(script, plugin_body=None, server_cls=FakeLLMServer, **kwargs):+ server = server_cls(script).__enter__()+ opened.append(lambda: server.__exit__(None, None, None))redactor_home(home, server.base_url, hold=True)if plugin_body:(home / "plugins" / "redactor" / "__init__.py").write_text(plugin_body, encoding="utf-8")plugins._reset_plugin_managers_for_tests()db = SessionDB(db_path=tmp_path / "held.db")- opened.append(db)+ opened.append(db.close)agent = _agent(server, db, "held-exit", **kwargs)shown = []agent.stream_delta_callback = shown.append@@ -57,8 +58,8 @@ def held(tmp_path):return result, [text for text in shown if text], displayed, server, dbyield run- for item in opened:- (item.stop if isinstance(item, FakeLLMServer) else item.close)()+ for close in opened:+ close()plugins._reset_plugin_managers_for_tests()@@ -145,3 +146,38 @@ def test_commentary_on_an_unknown_tool_call_stays_out_of_history(held):assert result["final_response"] == REWRITEassert shown == []assert COMMENTARY not in str(displayed) and _assistant_texts(displayed) == [REWRITE]+++_VERIFYING_PLUGIN = (+ "import re\n"+ "def redact(response_text, **_):\n return re.sub(r'SECRET-\\d+', '[REDACTED]', response_text)\n"+ "def verify(**_):\n return {'action': 'continue', 'message': 'Run the tests before you finish.'}\n"+ "def register(ctx):\n"+ " ctx.register_hook('transform_llm_output', redact, hold_stream=True)\n"+ " ctx.register_hook('pre_verify', verify)\n"+)+++def test_answer_kept_as_the_budget_fallback_is_shown_in_history_as_the_rewrite(held, tmp_path):+ # pre_verify sends the answer back on the last allowed call: the turn ends on that saved answer.+ edit = ToolCall("write_file", {"path": str(tmp_path / "app.py"), "content": "x = 1\n"})+ result, shown, displayed, server, db = held(+ [edit, Text(ATTEMPT)], plugin_body=_VERIFYING_PLUGIN, enabled_toolsets=["file"], max_iterations=2)+ rewrite = ATTEMPT.replace("SECRET-321", "[REDACTED]")+ assert result["final_response"] == rewrite and shown == []+ assert "SECRET-" not in str(displayed) and _assistant_texts(displayed) == [rewrite]+ # The model keeps the answer it wrote: a resumed turn replays it unchanged.+ resumed = _agent(server, db, "held-exit", enabled_toolsets=["file"])+ resumed.run_conversation("thanks", conversation_history=db.get_messages_as_conversation("held-exit"))+ replayed = [m for m in server.main_requests()[-1]["messages"] if m["role"] == "assistant" and m.get("content")]+ assert [m["content"] for m in replayed] == [ATTEMPT]+++def test_codex_commentary_riding_a_held_final_stays_out_of_history(held):+ turn = Turn([Message(COMMENTARY, phase="commentary"), Message(RAW, phase="final_answer")])+ result, shown, displayed, _, _ = held([turn], server_cls=FakeResponsesServer, api_mode="codex_responses")+ assert result["final_response"] == REWRITE and shown == []+ assert _assistant_texts(displayed) == [REWRITE]+ # The provider items stay on the row for replay; no rendered field carries the commentary.+ rendered = [row.get(key) for row in displayed for key in ("text", "display_content", "display_commentary")]+ assert COMMENTARY not in str(rendered)diff --git a/tests/fakes/providers/openai_responses.py b/tests/fakes/providers/openai_responses.pyindex 191ce514866..ba468e0c023 100644--- a/tests/fakes/providers/openai_responses.py+++ b/tests/fakes/providers/openai_responses.py@@ -69,9 +69,11 @@ class Reasoning:@dataclassclass Message:- """An assistant ``message`` output item (streamed as ``output_text`` deltas)."""+ """An assistant ``message`` output item (streamed as ``output_text`` deltas); ``phase`` is the+ Codex ``commentary`` / ``final_answer`` label (unset by default)."""text: str+ phase: str | None = None@dataclass@@ -216,7 +218,7 @@ def _built_item(item: Item, idx: int) -> Any:if isinstance(item, Message):return ResponseOutputMessage(id=f"msg_{idx}_{uuid.uuid4().hex[:8]}", role="assistant", status="completed", type="message",- content=[ResponseOutputText(annotations=[], text=item.text, type="output_text")])+ content=[ResponseOutputText(annotations=[], text=item.text, type="output_text")], phase=item.phase)args = item.args if isinstance(item.args, str) else json.dumps(item.args)return ResponseFunctionToolCall(id=f"fc_{idx}_{uuid.uuid4().hex[:8]}", call_id=f"call_{uuid.uuid4().hex[:10]}",name=item.name, arguments=args, type="function_call", status="completed")diff --git a/tests/hermes_cli/test_plugin_hold_stream.py b/tests/hermes_cli/test_plugin_hold_stream.pyindex c6f826de040..82310d37a35 100644--- a/tests/hermes_cli/test_plugin_hold_stream.py+++ b/tests/hermes_cli/test_plugin_hold_stream.py@@ -102,10 +102,17 @@ def test_a_turn_starting_during_a_plugin_reload_sees_the_settled_hold(tmp_path,reload.start()assert gate.entered.wait(10) # the reload unloaded everything and is re-registeringanswers: list[bool] = []- turn = threading.Thread(target=lambda: answers.append(manager.holds_stream()))+ reading = threading.Event()++ def read() -> None:+ reading.set()+ answers.append(manager.holds_stream())++ turn = threading.Thread(target=read)turn.start()- turn.join(0.5)- assert answers == [] # the turn waits while the plugin is mid-reload+ assert reading.wait(10) # the read has started: from here only the reload lock can hold it+ turn.join(2.0)+ assert answers == [] and turn.is_alive() # the turn waits while the plugin is mid-reloadgate.release.set()reload.join(10)turn.join(10)