Skip to content

Hermes Agent context engine: the main model writes the compaction handoff on its cached prompt prefix

Send the warm request through the execution middleware, compare media parts, and bound copies in tokens

Elevatormusicdcb57f61b58c
diff --git a/README.md b/README.md
index 4c8f983..2289d06 100644
--- a/README.md
+++ b/README.md
@@ -138,7 +138,7 @@ warm_compaction: path=warm reason=accepted elapsed_s=10.656 prompt_tokens=108021
```
- `path` is `warm`, `fallback`, or `fixed`.
-- `reason` is `accepted` or the refusal code of the warm request: `disabled`, `no_capture`, `api_mode_unsupported`, `route_changed`, `settings_unsupported`, `source_transform_unsupported`, `history_changed`, `capacity`, `cancelled`, `provider_error`, `timeout`, `incomplete_response`, or `gate:<reason>`.
+- `reason` is `accepted` or the refusal code of the warm request: `disabled`, `no_capture`, `api_mode_unsupported`, `route_changed`, `settings_unsupported`, `source_transform_unsupported`, `history_changed`, `capacity`, `cancelled`, `middleware_unavailable`, `middleware_refused`, `middleware_changed_reply`, `provider_error`, `timeout`, `incomplete_response`, or `gate:<reason>`. The warm request goes through the Hermes `llm_execution` middleware, as a main request does.
- `cached_tokens` is `None` when the server does not report it. Then the cache reuse is unknown.
The engine status (`get_status()`) has the same values in `warm_last`. The log never contains message text, request bodies, or keys.
diff --git a/tests/test_wc_engine.py b/tests/test_wc_engine.py
index b21c253..dc993cd 100644
--- a/tests/test_wc_engine.py
+++ b/tests/test_wc_engine.py
@@ -145,6 +145,38 @@ class EngineTest(unittest.TestCase):
self.assertEqual((engine.warm_last["path"], engine.warm_last["reason"], engine.warm_last["prompt_tokens"]),
("fallback", "gate:heading_missing", 1000))
+ def test_warm_request_runs_through_the_execution_middleware(self):
+ seen = []
+
+ def audit(request=None, next_call=None, **context):
+ seen.append((len(request["messages"]), context.get("purpose"), context.get("api_request_id")))
+ return next_call()
+ wc_hermes_stub.EXECUTION_MIDDLEWARE.append(audit)
+ rows = old_turns()
+ reply = assistant("final")
+ self.seed(rows, reply)
+ self.engine.compress([*rows, reply])
+ self.assertEqual(self.engine.warm_last["path"], "warm")
+ self.assertEqual(seen, [(len(rows) + 3, "warm_compaction", None)])
+
+ def test_a_blocking_or_mocking_middleware_stops_the_warm_request(self):
+ def block(request=None, next_call=None, **context):
+ raise PermissionError("policy")
+
+ def mock(request=None, next_call=None, **context):
+ return SimpleNamespace(choices=[])
+ for middleware, reason in ((block, "middleware_refused"), (mock, "middleware_changed_reply")):
+ with self.subTest(reason):
+ wc_hermes_stub.EXECUTION_MIDDLEWARE[:] = [middleware]
+ self.post.calls.clear()
+ rows = old_turns()
+ reply = assistant("final")
+ self.seed(rows, reply)
+ self.engine.compress([*rows, reply])
+ self.assertEqual((self.engine.warm_last["path"], self.engine.warm_last["reason"]),
+ ("fallback", reason))
+ self.assertEqual(self.post.calls, [])
+
def test_fixed_summary_when_the_fallback_fails(self):
self.llm = FakeLlm(error=RuntimeError("down"))
engine = self.make()
diff --git a/tests/test_wc_fallback.py b/tests/test_wc_fallback.py
index db8e6cf..fe144ee 100644
--- a/tests/test_wc_fallback.py
+++ b/tests/test_wc_fallback.py
@@ -5,7 +5,7 @@ from types import SimpleNamespace
from warm_compaction.fallback import (
FALLBACK_INSTRUCTION, MAX_TOKENS, MIDDLE_MARK, TASK, TOOL_CHARS, TRANSCRIPT_CHARS, fixed_summary, llm_summary,
- transcript,
+ render_row, transcript,
)
from warm_compaction.handoff import HEADINGS
from wc_fixtures import assistant, tool, user
@@ -25,6 +25,12 @@ class FakeLlm:
class TranscriptTest(unittest.TestCase):
+ def test_think_blocks_are_removed_only_from_assistant_rows(self):
+ text = "<think>keep me</think> body"
+ self.assertNotIn("keep me", render_row(assistant(text)))
+ self.assertIn("keep me", render_row(user(text)))
+ self.assertIn("keep me", render_row(tool("c1", text)))
+
def test_earlier_summary_then_first_user_message_then_newest_rows(self):
rows = [user("[HERMES PREFIX] header", _compressed_summary=True),
assistant("[CONTEXT SUMMARY]:\nold summary\n\n--- END OF CONTEXT SUMMARY x", _compressed_summary=True),
diff --git a/tests/test_wc_layout.py b/tests/test_wc_layout.py
index 7775865..f5e1e89 100644
--- a/tests/test_wc_layout.py
+++ b/tests/test_wc_layout.py
@@ -57,6 +57,13 @@ class CopyTest(unittest.TestCase):
rows = [user("a" * 10), user("b" * (COPY_EACH + 50)), assistant("x"), user("c" * 10)]
self.assertEqual(copied_user_messages(rows, COPY_EACH + 15, PREFIXES), ["b" * COPY_EACH, "c" * 10])
+ def test_copies_stay_inside_the_token_budget(self):
+ # Six 4,000-character CJK messages are about 24,000 tokens. A 5,000-token budget keeps one.
+ rows = [user("\u4f60" * 4_000) for _ in range(6)]
+ self.assertEqual(len(copied_user_messages(rows, 24_000, PREFIXES)), 6)
+ self.assertEqual(len(copied_user_messages(rows, 24_000, PREFIXES, max_tokens=5_000)), 1)
+ self.assertEqual(copied_user_messages(rows, 24_000, PREFIXES, max_tokens=100), [])
+
class BuildTest(unittest.TestCase):
def test_two_summary_rows_then_the_tail_without_the_marker(self):
diff --git a/tests/test_wc_warm.py b/tests/test_wc_warm.py
index bf2298c..d827455 100644
--- a/tests/test_wc_warm.py
+++ b/tests/test_wc_warm.py
@@ -166,6 +166,21 @@ class BuildRequestTest(unittest.TestCase):
stripped["body"]["messages"][1].pop("name")
self.build(stripped)
+ def test_refuses_changed_media_parts(self):
+ image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}
+ other = {"type": "image_url", "image_url": {"url": "data:image/png;base64,BBBB"}}
+ for stored in ([{"type": "text", "text": "see this"}, image], [image]):
+ self.rows = [user(stored), assistant("a1"), user("u2")]
+ self.messages = [*self.rows, self.reply, tool("c1", "r1"), user("u3")]
+ self.build(capture_for(self.rows, self.reply))
+ for sent in ([part for part in stored if part is not image], [other if part is image else part
+ for part in stored]):
+ capture = capture_for(self.rows, self.reply)
+ capture["body"]["messages"][1]["content"] = sent
+ with self.subTest(stored=stored, sent=sent), self.assertRaises(WarmRefusal) as caught:
+ self.build(capture)
+ self.assertEqual(caught.exception.code, "source_transform_unsupported")
+
def test_accepts_request_time_context_and_reformatted_arguments(self):
capture = self._tool_round()
sent = capture["body"]["messages"]
diff --git a/tests/wc_hermes_stub.py b/tests/wc_hermes_stub.py
index 25a5098..8434b1e 100644
--- a/tests/wc_hermes_stub.py
+++ b/tests/wc_hermes_stub.py
@@ -101,8 +101,23 @@ HERMES_CLI = _module("hermes_cli")
PLUGINS = _module("hermes_cli.plugins", VALID_HOOKS={
"pre_api_request", "post_api_request", "on_session_start", "on_session_end", "on_session_finalize",
"on_session_reset"})
+EXECUTION_MIDDLEWARE = []
+
+
+def run_llm_execution_middleware(request, next_call, **context):
+ """The llm_execution chain of Hermes, without its failure reporting."""
+ def call_at(index, payload):
+ if index >= len(EXECUTION_MIDDLEWARE):
+ return next_call(payload)
+ return EXECUTION_MIDDLEWARE[index](
+ request=payload, next_call=lambda new=None: call_at(index + 1, payload if new is None else new),
+ **context)
+ return call_at(0, request)
+
+
MIDDLEWARE = _module("hermes_cli.middleware", VALID_MIDDLEWARE={
- "tool_request", "tool_execution", "llm_request", "llm_execution"})
+ "tool_request", "tool_execution", "llm_request", "llm_execution"},
+ run_llm_execution_middleware=run_llm_execution_middleware)
HERMES_CLI.plugins = PLUGINS
HERMES_CLI.middleware = MIDDLEWARE
MODULES = {
@@ -116,3 +131,4 @@ def install(test_case):
patcher = patch.dict(sys.modules, MODULES)
patcher.start()
test_case.addCleanup(patcher.stop)
+ test_case.addCleanup(EXECUTION_MIDDLEWARE.clear)
diff --git a/warm_compaction/engine.py b/warm_compaction/engine.py
index 3a9b029..e0e3de1 100644
--- a/warm_compaction/engine.py
+++ b/warm_compaction/engine.py
@@ -191,7 +191,8 @@ class WarmCompactionEngine(ContextEngine):
header_prefix=hermes_value("agent.context_compressor", "SUMMARY_PREFIX", handoff.LEGACY_PREFIX),
prefixes=prefixes,
end_marker=hermes_value("agent.context_compressor", "_SUMMARY_END_MARKER", HERMES_END_MARKER),
- marker=hermes_value("agent.context_compressor", "_DB_PERSISTED_MARKER", HERMES_DB_MARKER))
+ marker=hermes_value("agent.context_compressor", "_DB_PERSISTED_MARKER", HERMES_DB_MARKER),
+ copy_tokens=self._tail_tokens())
self.compression_count += 1
self._finish(record, started)
return new
@@ -208,7 +209,7 @@ class WarmCompactionEngine(ContextEngine):
handoff.build_instruction(focus_topic, memory))
if self._cancelled():
raise warm.WarmRefusal("cancelled")
- reply = warm.send(body, self._wc_route[1], self._wc_api_key, post=self._post)
+ reply = self._execute(body)
record.update(prompt_tokens=reply["prompt_tokens"], cached_tokens=reply["cached_tokens"])
text, reason = handoff.gate(reply, prefixes)
if text is None:
@@ -222,6 +223,30 @@ class WarmCompactionEngine(ContextEngine):
record.update(path="warm", reason="accepted")
return text
+ def _execute(self, body: dict[str, Any]) -> dict[str, Any]:
+ """Send the warm request through the Hermes llm_execution middleware, as Hermes sends a main request.
+ A middleware can audit, change, block, or replace the request. A block or a replaced reply stops the
+ warm request; the fallback summary then runs."""
+ try:
+ from hermes_cli.middleware import run_llm_execution_middleware
+ except Exception as error:
+ raise warm.WarmRefusal("middleware_unavailable") from error
+
+ def terminal(request: Any) -> dict[str, Any]:
+ return warm.send(request, self._wc_route[1], self._wc_api_key, post=self._post)
+ try:
+ reply = run_llm_execution_middleware(
+ body, terminal, original_request=body, purpose=NAME, api_request_id=None,
+ session_id=self._wc_session_id, model=self._wc_route[0], base_url=self._wc_route[1],
+ api_mode=self._wc_route[2])
+ except warm.WarmRefusal:
+ raise
+ except Exception as error:
+ raise warm.WarmRefusal("middleware_refused") from error
+ if not isinstance(reply, dict) or not {"content", "finish_reason", "prompt_tokens"} <= reply.keys():
+ raise warm.WarmRefusal("middleware_changed_reply")
+ return reply
+
def _prefixes(self) -> tuple[str, ...]:
current = hermes_value("agent.context_compressor", "SUMMARY_PREFIX", handoff.LEGACY_PREFIX)
historical = hermes_value("agent.context_compressor", "_HISTORICAL_SUMMARY_PREFIXES", ())
diff --git a/warm_compaction/fallback.py b/warm_compaction/fallback.py
index 9748853..8180309 100644
--- a/warm_compaction/fallback.py
+++ b/warm_compaction/fallback.py
@@ -86,7 +86,8 @@ def _summary_text(row: Any, prefixes: tuple[str, ...]) -> str:
def render_row(row: Any) -> str:
"""Return one transcript entry for a row."""
role = str(attr(row, "role") or "unknown")
- text = strip_think(plain_text(attr(row, "content"))).strip()
+ text = plain_text(attr(row, "content"))
+ text = (strip_think(text) if role == "assistant" else text).strip()
if role == "tool":
return "[tool result]\n" + _cut(text, TOOL_CHARS)
lines = [f"[{role}]"]
diff --git a/warm_compaction/layout.py b/warm_compaction/layout.py
index 0833259..dec902c 100644
--- a/warm_compaction/layout.py
+++ b/warm_compaction/layout.py
@@ -85,19 +85,23 @@ def tail_start(messages: list, tail_tokens: int, prefixes: Iterable[str]) -> tup
return start, {"role": "user", "content": copy.deepcopy(latest["content"])}
-def copied_user_messages(messages: list, total_chars: int, prefixes: Iterable[str]) -> list[str]:
- """Return the newest real user messages that fit in total_chars, each cut to COPY_EACH characters."""
+def copied_user_messages(messages: list, total_chars: int, prefixes: Iterable[str],
+ max_tokens: int | None = None) -> list[str]:
+ """Return the newest real user messages that fit in total_chars and in max_tokens (estimated), each cut
+ to COPY_EACH characters. The token limit keeps dense text, such as CJK text, inside the window."""
prefixes = tuple(prefixes)
chosen: list[str] = []
- used = 0
+ used = tokens = 0
for row in reversed(messages):
if not is_real_user(row, prefixes):
continue
text = plain_text(row.get("content")).strip()[:COPY_EACH]
- if used + len(text) > total_chars:
+ cost = estimate_tokens(text)
+ if used + len(text) > total_chars or (max_tokens is not None and tokens + cost > max_tokens):
break
chosen.append(text)
used += len(text)
+ tokens += cost
chosen.reverse()
return chosen
@@ -122,13 +126,14 @@ def _tail_row(row: Any, marker: str) -> Any:
def build(messages: list, summary_text: str, *, start: int, prepend: dict[str, Any] | None, copy_chars: int,
header_prefix: str, prefixes: Iterable[str], end_marker: str,
- marker: str = "_db_persisted") -> list | None:
+ marker: str = "_db_persisted", copy_tokens: int | None = None) -> list | None:
"""Return the new history, or None when no row comes before the tail."""
if start <= 0:
return None
prefixes = tuple(prefixes)
header = f"{header_prefix}\n\n{HEADER_TEXT}"
- body = summary_body(summary_text, copied_user_messages(messages[:start], copy_chars, prefixes), end_marker)
+ copies = copied_user_messages(messages[:start], copy_chars, prefixes, copy_tokens)
+ body = summary_body(summary_text, copies, end_marker)
rest = ([copy.deepcopy(prepend)] if prepend else []) + [_tail_row(row, marker) for row in messages[start:]]
if attr(rest[0], "role") != "user":
return [{"role": "user", "content": f"{header}\n\n{body}", "_compressed_summary": True}, *rest]
diff --git a/warm_compaction/warm.py b/warm_compaction/warm.py
index 4d3f6f0..68b7842 100644
--- a/warm_compaction/warm.py
+++ b/warm_compaction/warm.py
@@ -82,6 +82,14 @@ def _words(content: Any) -> str:
return " ".join(plain_text(content).split())
+def _media(content: Any) -> list:
+ """Return the parts of a content list that are not text, in order."""
+ if not isinstance(content, list):
+ return []
+ return [part for part in content if not (isinstance(part, str) or (
+ isinstance(part, dict) and part.get("type", "text") == "text" and isinstance(part.get("text"), str)))]
+
+
def _arguments(value: Any) -> Any:
"""Return the JSON value of tool-call arguments, so that a change of spacing or key order is not a change."""
if isinstance(value, str):
@@ -95,11 +103,13 @@ def _arguments(value: Any) -> Any:
def _same_row(wire: Any, row: Any) -> bool:
"""True when the sent row carries the stored row: the same shape, the same name or no name (the Hermes
transport removes the name from some rows), the stored text inside the sent text (Hermes can add
- request-time context to a row), and the same tool-call arguments."""
+ request-time context to a row), the same image, audio, and file parts, and the same tool-call arguments."""
if _shape(wire) != _shape(row) or attr(wire, "name") not in (None, attr(row, "name")):
return False
if _words(attr(row, "content")) not in _words(attr(wire, "content")):
return False
+ if _media(attr(wire, "content")) != _media(attr(row, "content")):
+ return False
return [_arguments(arguments) for _id, _name, arguments in tool_calls_of(wire)] == [
_arguments(arguments) for _id, _name, arguments in tool_calls_of(row)]