Skip to content

Hermes Agent context engine: the main model writes the compaction handoff on its cached prompt prefix

Fix captured reasoning validation for issue 29

Elevatormusicdcaa94e5cefb
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 329b0a6..12d8af0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,7 @@ All notable changes are listed here. The format follows [Keep a Changelog](https
### Fixed
+- Accept unchanged stored `reasoning` text in captured Chat Completions assistant rows. Do not require reasoning padding on captured summary or notice rows that have no reasoning. Keep checks for changed text, reasoning details, and tool signatures, and keep the captured request prefix unchanged.
- Match captured content with the stored `api_content` text for context-injected rows. Keep role, tool, and native replay checks. Report each history refusal stage and use the fallback when the capture is ahead of the live history.
- Keep the original history when an indivisible native replay block exceeds the compaction budget. Check the threshold, reply reserve, and unknown request overhead before accepting the new history.
- Match Copilot hostnames at a domain boundary. A lookalike hostname no longer gets Copilot replay handling.
diff --git a/evidence/issue-29-synthetic-validation-20261011.json b/evidence/issue-29-synthetic-validation-20261011.json
new file mode 100644
index 0000000..9bafdac
--- /dev/null
+++ b/evidence/issue-29-synthetic-validation-20261011.json
@@ -0,0 +1,73 @@
+{
+ "issue": 29,
+ "synthetic": true,
+ "hermes_commit": "d4d2ab7b5d104166e7e22a02cafd12f1fe456a31",
+ "hermes_clean": true,
+ "test_python": "3.14.5",
+ "hermes_dependency_python": "3.14.7",
+ "ruff": "0.16.10",
+ "unit_tests": 411,
+ "property_tests": 13,
+ "policy_tests": 193,
+ "all_passed": true,
+ "loopback_suites": {
+ "compatibility": {
+ "passed": true,
+ "cases": 4,
+ "case_names": [
+ "capture_last",
+ "earlier_rewrite",
+ "later_rewrite",
+ "unknown_order"
+ ]
+ },
+ "plugin": {
+ "passed": true,
+ "cases": 10,
+ "case_names": [
+ "enable_manual_warm",
+ "manual_context",
+ "manual_whitespace",
+ "manual_reasoning",
+ "manual_fallback",
+ "manual_aborted",
+ "auto_tool_loop",
+ "auto_context_tool",
+ "auto_warm_failures",
+ "rollback"
+ ]
+ },
+ "provider": {
+ "passed": true,
+ "cases": 9,
+ "case_names": [
+ "responses_text",
+ "responses_tool",
+ "responses_incomplete",
+ "responses_codex",
+ "responses_codex_reasoning",
+ "anthropic_text",
+ "anthropic_tool",
+ "anthropic_refusal",
+ "anthropic_signed"
+ ]
+ },
+ "native": {
+ "passed": true,
+ "cases": 4,
+ "case_names": [
+ "responses_manual",
+ "responses_auto_tool",
+ "anthropic_manual",
+ "anthropic_auto_tool"
+ ]
+ }
+ },
+ "local_document_links": 45,
+ "agents_claude_byte_parity": true,
+ "sha256": {
+ "warm_compaction/warm.py": "86ab4ade20d513fe3bdfb1240b44d4dea151ad19326b62977d42eeede549d302",
+ "scripts/check_plugin_hermes.py": "0f6173d0841f29b959a7f1ca4ba0a7e9edcaf0d9fac4a1d2b9d631086eab7845"
+ },
+ "limits": "No live server, summary quality, cache reuse, or performance qualification."
+}
diff --git a/scripts/check_plugin_hermes.py b/scripts/check_plugin_hermes.py
index 91e5687..c89ef73 100644
--- a/scripts/check_plugin_hermes.py
+++ b/scripts/check_plugin_hermes.py
@@ -77,8 +77,8 @@ TOOL_PLUGIN = {
' "parameters": {"type": "object", "properties": {}, "required": []}})\n'
),
}
-SCENARIOS = ("enable_manual_warm", "manual_context", "manual_whitespace", "manual_fallback", "manual_aborted",
- "auto_tool_loop", "auto_context_tool", "auto_warm_failures", "rollback")
+SCENARIOS = ("enable_manual_warm", "manual_context", "manual_whitespace", "manual_reasoning", "manual_fallback",
+ "manual_aborted", "auto_tool_loop", "auto_context_tool", "auto_warm_failures", "rollback")
# The scenarios with automatic compaction in a tool loop.
AUTO_SCENARIOS = ("auto_tool_loop", "auto_context_tool", "auto_warm_failures")
NOTICE_MARK = "Warm compaction unavailable"
@@ -223,6 +223,8 @@ class FakeServer:
step = self.main_replies.pop(0) if self.main_replies else {"content": "OK."}
payload = completion(step.get("content"), step.get("tool"),
prompt_tokens=step.get("prompt_tokens", size))
+ if "reasoning_content" in step:
+ payload[0]["reasoning_content"] = step["reasoning_content"]
else:
payload = completion("Synthetic title", prompt_tokens=size)
return entry["status"], payload
@@ -243,6 +245,8 @@ class FakeServer:
delta = {"role": "assistant"}
if message.get("content") is not None:
delta["content"] = message["content"]
+ if "reasoning_content" in message:
+ delta["reasoning_content"] = message["reasoning_content"]
if message.get("tool_calls"):
delta["tool_calls"] = [{"index": index, **call} for index, call in enumerate(message["tool_calls"])]
parts = [event([{"index": 0, "delta": delta, "finish_reason": None}]),
@@ -371,6 +375,9 @@ def synthetic_history(pairs=30):
def plan_for(scenario):
+ if scenario == "manual_reasoning":
+ return {"main": [{"content": "The code word is BLUE-7.", "reasoning_content": "Synthetic first thoughts."},
+ {"content": "Next: report BLUE-7.", "reasoning_content": "Synthetic next thoughts."}]}
if scenario == "manual_whitespace":
return {"main": [{"content": "The code word is BLUE-7."}, {"content": "Hello."},
{"content": "Hello again."}]}
@@ -539,7 +546,13 @@ def run_phase(spec, session_dir):
statuses = []
sid = f"wc-{scenario}"
db.create_session(sid, "cli")
- db.append_messages_batch(sid, synthetic_history())
+ initial = synthetic_history()
+ if scenario == "manual_reasoning":
+ initial[1].update(reasoning="Synthetic earlier thoughts.", reasoning_content="Synthetic earlier thoughts.")
+ initial[3]["content"] = "[CONTEXT SUMMARY]: Synthetic earlier summary."
+ initial[3]["_compressed_summary"] = True
+ initial[5]["display_kind"] = "compaction_notice"
+ db.append_messages_batch(sid, initial)
agent = AIAgent(base_url=server.base_url, api_key="no-key-required", provider="custom",
api_mode="chat_completions", model=MODEL,
enabled_toolsets=["wc_test"] if scenario in AUTO_SCENARIOS else [],
@@ -560,6 +573,9 @@ def run_phase(spec, session_dir):
elif scenario == "manual_whitespace":
run_manual_whitespace(agent, engine, db, history, server, result, checks,
_compress_session_history, finalize_context_engine_compression_notification)
+ elif scenario == "manual_reasoning":
+ run_manual_reasoning(agent, engine, db, history, server, result, checks,
+ _compress_session_history, finalize_context_engine_compression_notification)
else:
run_manual(agent, engine, db, history, server, result, checks, scenario,
_compress_session_history, finalize_context_engine_compression_notification, statuses)
@@ -900,6 +916,61 @@ def run_manual_aborted(agent, engine, db, session, capture, server, result, chec
result["automatic_stop_contract"] = "not checked: Hermes has no documented terminal plugin abort result"
+def run_manual_reasoning(agent, engine, db, history, server, result, checks, compress, finalize):
+ """Repeat compaction with synthetic reasoning, summary, and notice rows on the real route."""
+ from agent import message_sanitization
+
+ modern = callable(getattr(message_sanitization, "route_reasoning_carriers", None))
+ result["reasoning_carrier_policy"] = "route_carriers" if modern else "legacy_echo"
+ if modern:
+ pad, carriers = agent._reasoning_replay_route()
+ expected_keys = set(carriers or ()) & {"reasoning", "reasoning_content"}
+ else:
+ pad = agent._needs_thinking_reasoning_pad()
+ expected_keys = {"reasoning_content"} if pad else set()
+ result["host_reasoning_padding"] = bool(pad)
+ messages = history
+ for index, question in enumerate((SEED_QUESTION, NEXT_QUESTION), 1):
+ if index > 1:
+ messages = list(messages) + synthetic_history(20)
+ reply = agent.run_conversation(question, system_message=SYSTEM, conversation_history=messages)
+ messages = list(reply.get("messages") or [])
+ label = f"turn{index}"
+ checks[f"{label}_reply"] = bool(str(reply.get("final_response") or "").strip())
+ capture = engine._store.latest(engine._wc_session_id) or {}
+ captured = copy.deepcopy(capture.get("body", {}).get("messages") or [])
+ assistants = [row for row in captured if row.get("role") == "assistant"]
+ text_keys = {key for row in assistants for key in ("reasoning", "reasoning_content") if key in row}
+ checks[f"{label}_host_reasoning_carriers"] = text_keys == expected_keys
+ checks[f"{label}_captured_padding_matches_route"] = (
+ all(row.get("reasoning_content") for row in assistants) if pad else any(
+ "reasoning" not in row and "reasoning_content" not in row for row in assistants))
+ before = copy.deepcopy(messages)
+ session = {"agent": agent, "history": messages, "history_lock": threading.Lock(),
+ "history_version": index, "session_key": agent.session_id}
+ warm_before = len(by_kind(server, "warm"))
+ removed, _usage = compress(session, "")
+ finalize(agent, committed=True)
+ checks[f"{label}_input_history_unchanged"] = messages == before
+ checks[f"{label}_warm_accepted"] = (engine.warm_last.get("path"), engine.warm_last.get("reason")) == (
+ "warm", "accepted")
+ checks[f"{label}_one_warm_request"] = len(by_kind(server, "warm")) == warm_before + 1
+ if checks[f"{label}_one_warm_request"]:
+ check_warm_request(server, checks, label)
+ sent = by_kind(server, "warm")[-1]["body"]["messages"]
+ checks[f"{label}_captured_prefix_unchanged"] = sent[:len(captured)] == captured
+ # The new reply keeps its synthetic text. Exact parity of all route carriers is a separate check.
+ checks[f"{label}_appended_reasoning_text"] = sent[len(captured)].get("reasoning_content") == (
+ "Synthetic first thoughts." if index == 1 else "Synthetic next thoughts.")
+ result[f"{label}_appended_reasoning_keys"] = sorted(
+ key for key in ("reasoning", "reasoning_content") if key in sent[len(captured)])
+ checks[f"{label}_removed_rows"] = removed > 0
+ messages = session["history"]
+ checks[f"{label}_summary_rows"] = summary_rows_ok(messages)
+ checks["two_compactions"] = engine.compression_count == 2
+ checks["no_fallback_request"] = not by_kind(server, "fallback")
+
+
def run_manual_whitespace(agent, engine, db, history, server, result, checks, compress, finalize):
"""Repeat manual compaction when Hermes strips a string only in the sent request."""
messages, outcomes = history, []
diff --git a/tests/test_wc_engine.py b/tests/test_wc_engine.py
index ef7f51d..b3ed097 100644
--- a/tests/test_wc_engine.py
+++ b/tests/test_wc_engine.py
@@ -181,6 +181,27 @@ class EngineTest(unittest.TestCase):
self.assertEqual(new[-2:], [rows[-1], reply])
self.assertEqual(self.llm.calls, [])
+ def test_mixed_reasoning_capture_keeps_the_warm_prefix_and_history(self):
+ rows = [assistant("Synthetic earlier summary.", _compressed_summary=True),
+ assistant("Synthetic compaction notice."), *old_turns(),
+ user("Synthetic question."), assistant("Synthetic answer.",
+ reasoning="Synthetic thoughts.",
+ reasoning_content="Synthetic thoughts."),
+ user("Synthetic next question.")]
+ reply = assistant("Synthetic final answer.")
+ self.seed(rows, reply)
+ messages = [*rows, reply]
+ before_messages = copy.deepcopy(messages)
+ before_capture = copy.deepcopy(self.store.latest("s1"))
+ self.engine.compress(messages)
+ self.assertEqual((self.engine.warm_last["path"], self.engine.warm_last["reason"]), ("warm", "accepted"))
+ prefix = before_capture["body"]["messages"]
+ sent = self.post.calls[0]["body"]["messages"]
+ self.assertEqual(json.dumps(sent[:len(prefix)], ensure_ascii=False),
+ json.dumps(prefix, ensure_ascii=False))
+ self.assertEqual((messages, self.store.latest("s1")), (before_messages, before_capture))
+ self.assertEqual(self.llm.calls, [])
+
def test_capture_ahead_uses_only_live_rows_for_fallback(self):
rows = [*old_turns(), user("synthetic live question")]
future = [assistant("synthetic unmatched future reply"), tool("future-call", "future result")]
diff --git a/tests/test_wc_warm.py b/tests/test_wc_warm.py
index c2d37ca..3da8048 100644
--- a/tests/test_wc_warm.py
+++ b/tests/test_wc_warm.py
@@ -466,6 +466,87 @@ class BuildRequestTest(unittest.TestCase):
self.build(capture, route)
self.assertEqual(caught.exception.code, "source_transform_unsupported")
+ def test_stored_and_sent_reasoning_text_keeps_the_captured_prefix(self):
+ self.rows = [user("Synthetic question."), assistant(
+ "Synthetic answer.", reasoning="Synthetic thoughts.", reasoning_content="Synthetic thoughts.")]
+ self.messages = [*self.rows, self.reply, tool("c1", "r1")]
+ capture = capture_for(self.rows, self.reply)
+ before_capture, before_messages = copy.deepcopy(capture), copy.deepcopy(self.messages)
+ body = self.build(capture)
+ sent = capture["body"]["messages"]
+ self.assertEqual(body["messages"][:len(sent)], sent)
+ self.assertEqual((capture, self.messages), (before_capture, before_messages))
+
+ def test_reasoning_free_rows_need_no_captured_echo_padding(self):
+ for extra in ({"_compressed_summary": True}, {"display_kind": "compaction_notice"}, {}):
+ with self.subTest(extra=extra):
+ self.rows = [assistant("Synthetic summary or notice.", **extra), user("Synthetic question."),
+ assistant("Synthetic answer.", reasoning_content="Synthetic thoughts.")]
+ self.messages = [*self.rows, self.reply, tool("c1", "r1")]
+ capture = capture_for(self.rows, self.reply)
+ # Hermes removes this local display field before sending the row.
+ capture["body"]["messages"][1].pop("display_kind", None)
+ before_capture, before_messages = copy.deepcopy(capture), copy.deepcopy(self.messages)
+ body = self.build(capture)
+ sent = capture["body"]["messages"]
+ self.assertEqual(body["messages"][:len(sent)], sent)
+ self.assertNotIn("reasoning_content", body["messages"][1])
+ self.assertEqual((capture, self.messages), (before_capture, before_messages))
+
+ def test_added_or_changed_reasoning_text_is_refused(self):
+ for stored in (None, "Synthetic thoughts."):
+ self.rows = [user("Synthetic question."), assistant(
+ "Synthetic answer.", **({"reasoning": stored} if stored is not None else {}))]
+ self.messages = [*self.rows, self.reply, tool("c1", "r1")]
+ for sent in ("Changed synthetic thoughts.", 1, True, ["Synthetic thoughts."], None):
+ with self.subTest(stored=stored, sent=sent):
+ capture = capture_for(self.rows, self.reply)
+ capture["body"]["messages"][2]["reasoning"] = sent
+ with self.assertRaises(WarmRefusal) as caught:
+ self.build(capture)
+ self.assertEqual(caught.exception.code, "source_transform_unsupported")
+
+ def test_missing_stored_reasoning_content_is_refused_on_a_mixed_route(self):
+ for field in ("reasoning_content", "reasoning"):
+ with self.subTest(field=field):
+ self.rows = [assistant("Synthetic earlier answer.", **{field: "Synthetic thoughts."}),
+ user("Synthetic question."),
+ assistant("Synthetic answer.", reasoning_content="Synthetic other thoughts.")]
+ self.messages = [*self.rows, self.reply, tool("c1", "r1")]
+ capture = capture_for(self.rows, self.reply)
+ capture["body"]["messages"][1].pop("reasoning_content", None)
+ capture["body"]["messages"][1].pop("reasoning", None)
+ with self.assertRaises(WarmRefusal) as caught:
+ self.build(capture)
+ self.assertEqual(caught.exception.code, "source_transform_unsupported")
+
+ def test_mixed_reasoning_capture_refuses_changed_details_and_signatures(self):
+ route = ("gemini-3-pro", "https://openrouter.ai/api/v1", ROUTE[2])
+ stored = assistant("Synthetic answer.", [("c0", "read", "{}")],
+ reasoning="Synthetic thoughts.", reasoning_content="Synthetic thoughts.",
+ reasoning_details=[{"type": "reasoning.encrypted", "data": "synthetic-data"}])
+ stored["tool_calls"][0]["extra_content"] = {"google": {"thought_signature": "synthetic-signature"}}
+ self.rows = [assistant("Synthetic summary.", _compressed_summary=True), stored,
+ tool("c0", "Synthetic result."), user("Synthetic question.")]
+ self.messages = [*self.rows, self.reply, tool("c1", "r1")]
+ self.build(capture_for(self.rows, self.reply, route=route), route)
+ changes = (
+ lambda row: row.update(reasoning="Changed synthetic thoughts."),
+ lambda row: row.update(reasoning_content="Changed synthetic thoughts."),
+ lambda row: row.pop("reasoning_details"),
+ lambda row: row.update(reasoning_details=[{"type": "reasoning.encrypted", "data": "changed"}]),
+ lambda row: row["tool_calls"][0].pop("extra_content"),
+ lambda row: row["tool_calls"][0].update(extra_content={"google": {"thought_signature": "changed"}}),
+ lambda row: row.update(untrusted_control="synthetic-value"),
+ )
+ for index, change in enumerate(changes):
+ with self.subTest(change=index):
+ capture = capture_for(self.rows, self.reply, route=route)
+ change(capture["body"]["messages"][2])
+ with self.assertRaises(WarmRefusal) as caught:
+ self.build(capture, route)
+ self.assertEqual(caught.exception.code, "source_transform_unsupported")
+
def test_accepts_reformatted_arguments(self):
capture = self._tool_round()
sent = capture["body"]["messages"]
diff --git a/warm_compaction/warm.py b/warm_compaction/warm.py
index 94fdec5..a4d3097 100644
--- a/warm_compaction/warm.py
+++ b/warm_compaction/warm.py
@@ -203,10 +203,21 @@ def check_source(body: dict[str, Any], history_rows: list, base_url: Any = None,
raise WarmRefusal("source_transform_unsupported")
# The replayed fields follow the captured request only: the new rows do not change what Hermes sent.
echo = needs_reasoning_echo(body, [])
- expected = [wire_row(row, echo, body.get("model"), base_url, native_type) for row in history_rows]
- if not all(_same_replay_fields(wire, want) and _no_extra_fields(wire, want)
- for wire, want in zip(sent[offset:], expected)):
- raise WarmRefusal("source_transform_unsupported")
+ for wire, row in zip(sent[offset:], history_rows):
+ want = wire_row(row, echo, body.get("model"), base_url, native_type)
+ # Hermes d4d2ab7b can send stored reasoning on local Chat Completions routes.
+ # Accept this field only when the sent assistant text equals the stored text.
+ reasoning = attr(row, "reasoning")
+ if (attr(row, "role") == "assistant" and isinstance(reasoning, str)
+ and attr(wire, "reasoning") == reasoning):
+ want["reasoning"] = reasoning
+ # A captured field on another row does not prove that every row needs a pad.
+ # Do not require a new pad when neither this sent row nor its source has reasoning.
+ if (attr(wire, "reasoning_content") is None and attr(row, "reasoning_content") is None
+ and reasoning is None):
+ want.pop("reasoning_content", None)
+ if not (_same_replay_fields(wire, want) and _no_extra_fields(wire, want)):
+ raise WarmRefusal("source_transform_unsupported")
def needs_reasoning_echo(body: dict[str, Any], rows: list) -> bool: