The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(state): a compacted tool result paints once, as its archived original
Teknium109a8f7c0363
diff --git a/hermes_state_compaction_display.py b/hermes_state_compaction_display.pynew file mode 100644index 00000000000..fb98ff55e81--- /dev/null+++ b/hermes_state_compaction_display.py@@ -0,0 +1,54 @@+"""Display reads paint a compacted tool result once, as its archived original (#132939, #135655).++Compaction can carry a tool result forward as a shortened stub: same ``message_uid`` and timestamp, short+body. A tool result keeps its payload in its display key on purpose (the archived output must never fold+INTO its stub), so the stub lands in a display slot (``display_order``) of its own and both copies painted.+The reads therefore skip a tool row when an archived (``compacted``) display-visible tool row of the same+session, ``message_uid`` and timestamp holds an EARLIER slot: that slot already paints the full original.++The rule uses identity only (session, uid, timestamp, slot), never bodies, and changes no stored row. An+in-place prune keeps its row's slot, so its stub shares the slot with the archived copies and the per-slot+rule (the live row wins) keeps showing the stub. A row without a slot (legacy, never backfilled) is never+skipped. The timestamp lets the probe seek ``(session_id, timestamp)``, so a page read stays per-row cheap.+"""++from __future__ import annotations++from typing import Any, Iterable++from agent.context_compressor import MODEL_ONLY_DISPLAY_METADATA_KEY+from hermes_state_common import _sql_json_extract++_ORIGINAL_VISIBLE_SQL = (+ f"COALESCE({_sql_json_extract('original.display_metadata', '$.' + MODEL_ONLY_DISPLAY_METADATA_KEY)}, 0) = 0")+++def not_a_later_tool_copy_sql(alias: str) -> str:+ """Filter that drops *alias* rows that are a later copy of an archived tool result."""+ return (f" AND NOT ({alias}.role = 'tool' AND {alias}.message_uid IS NOT NULL AND EXISTS ("+ f"SELECT 1 FROM messages AS original WHERE original.session_id = {alias}.session_id "+ f"AND original.timestamp = {alias}.timestamp AND original.message_uid = {alias}.message_uid "+ "AND original.role = 'tool' AND original.active = 0 AND original.compacted = 1 "+ f"AND +original.display_order < {alias}.display_order AND {_ORIGINAL_VISIBLE_SQL}))")+++def later_tool_copy_ids(db: Any, rows: Iterable[Any]) -> set[int]:+ """Python twin of :func:`not_a_later_tool_copy_sql` over display-set *rows*: the ids it skips.+ Empty when the rows carry no slot (``display_order``) or archive flag (``compacted``)."""+ tools = [row for row in rows if row["role"] == "tool"]+ if not tools or "compacted" not in tools[0].keys():+ return set()+ candidates = [row for row in tools if row["active"] or row["compacted"]]+ if not candidates or "display_order" not in candidates[0].keys():+ return set()++ def owner(row):+ return row["session_id"], row["message_uid"], row["timestamp"]++ first_slot: dict[Any, int] = {}+ for row in candidates:+ slot = row["display_order"]+ if row["message_uid"] and slot is not None and not row["active"] and not db._is_model_only_row(row):+ first_slot[owner(row)] = min(first_slot.get(owner(row), slot), slot)+ return {row["id"] for row in candidates if row["message_uid"] and row["display_order"] is not None+ and first_slot.get(owner(row), row["display_order"]) < row["display_order"]}diff --git a/hermes_state_messages.py b/hermes_state_messages.pyindex d89c734fc1c..220bbeabb06 100644--- a/hermes_state_messages.py+++ b/hermes_state_messages.py@@ -22,6 +22,7 @@ from hermes_cli.timefmt import coerce_epochfrom hermes_state_common import (_COMPRESSION_LOCK_ROW_SQL, _ENDED_ROW_SQL, _RESET_END_REASONS, _RESET_END_REASONS_SQL, _ended_by_compression,_json_or, _legacy_reset_child_sql, _placeholders, _sql_json_extract)+from hermes_state_compaction_display import later_tool_copy_ids, not_a_later_tool_copy_sqlfrom hermes_state_detached_results import SessionDetachedResultsMixinfrom hermes_state_todo import SessionTodoResultMixinfrom hermes_state_identity import (@@ -1144,8 +1145,9 @@ class SessionMessagesMixin(SessionDetachedResultsMixin, SessionTodoResultMixin):removed: dict[Any, bool] = {} # copy identity -> its newest copy is removed (rows come in id order)identities: dict[tuple[Any, ...], list[Any]] = {}removed_ts: set[Any] = set()+ rows = rows if isinstance(rows, list) else list(rows)+ later_copies = later_tool_copy_ids(self, rows) # a compacted tool result paints once, as its originalif drop_removed:- rows = rows if isinstance(rows, list) else list(rows)# Every display key carries the row's timestamp third, so a live row can share a removed row's identity# only at one of the removed rows' timestamps; hash just those (a sha256 per live row cost a warm# 30k-row resume ~0.17 s, RT-39). No removed row at all: nothing can hide a copy.@@ -1165,7 +1167,7 @@ class SessionMessagesMixin(SessionDetachedResultsMixin, SessionTodoResultMixin):removed.update(dict.fromkeys(copies, gone))if gone:continue- if self._is_model_only_row(row):+ if self._is_model_only_row(row) or row["id"] in later_copies:continuekey = key or self._display_dedupe_key(row)# Only the summary carrier folds by uid: other rewritten copies (a pruned tool stub) must keep@@ -1315,7 +1317,7 @@ class SessionMessagesMixin(SessionDetachedResultsMixin, SessionTodoResultMixin):return conn.execute(f"""WITH page AS (SELECT display_order FROM messages- WHERE session_id = ? AND (active = 1 OR compacted = 1){DISPLAY_VISIBLE_SQL}+ WHERE session_id = ? AND (active = 1 OR compacted = 1){DISPLAY_VISIBLE_SQL}{not_a_later_tool_copy_sql("messages")}GROUP BY display_order ORDER BY display_order {direction}LIMIT ? OFFSET ?)@@ -1324,7 +1326,7 @@ class SessionMessagesMixin(SessionDetachedResultsMixin, SessionTodoResultMixin):SELECT candidate.id FROM messages AS candidateWHERE candidate.session_id = ?AND candidate.display_order = page.display_order- AND (candidate.active = 1 OR candidate.compacted = 1){DISPLAY_VISIBLE_SQL}+ AND (candidate.active = 1 OR candidate.compacted = 1){DISPLAY_VISIBLE_SQL}{not_a_later_tool_copy_sql("candidate")}ORDER BY candidate.active DESC, candidate.id DESC LIMIT 1)ORDER BY page.display_order ASC""",@@ -1336,8 +1338,8 @@ class SessionMessagesMixin(SessionDetachedResultsMixin, SessionTodoResultMixin):``_display_rows_from_conn``'s set. Unindexed legacy rows count as one group (the readbackfills them), so the count is zero exactly when the read paints nothing."""row = self._read_one(- "SELECT COUNT(*) FROM (SELECT DISTINCT display_order FROM messages"- f" WHERE session_id = ?{_DISPLAY_ACTIVE_CLAUSE}{DISPLAY_VISIBLE_SQL})", (session_id,))+ "SELECT COUNT(*) FROM (SELECT DISTINCT display_order FROM messages WHERE session_id = ?"+ f"{_DISPLAY_ACTIVE_CLAUSE}{DISPLAY_VISIBLE_SQL}{not_a_later_tool_copy_sql('messages')})", (session_id,))return int(row[0])def _display_messages_from_conn(self, conn, session_id: str) -> Optional[list[dict[str, Any]]]:@@ -1485,7 +1487,7 @@ class SessionMessagesMixin(SessionDetachedResultsMixin, SessionTodoResultMixin):and would break tool-call adjacency). ``with_compacted``: the display set (``active_clause`` isignored) with the ``compacted`` column, plus the removed rows' identities for ``drop_removed``."""rows = self._read_all(- f"SELECT {'session_id, ' if with_session_id else ''}{'compacted, ' if with_compacted else ''}"+ f"SELECT {'session_id, ' if with_session_id or with_compacted else ''}{'compacted, display_order, ' if with_compacted else ''}"f"{self._CONVERSATION_ROW_COLUMNS} "f"FROM messages WHERE session_id IN ({_placeholders(session_ids)})"f"{_DISPLAY_ACTIVE_CLAUSE if with_compacted else active_clause} ORDER BY id", tuple(session_ids))diff --git a/hermes_state_timeline.py b/hermes_state_timeline.pyindex a305cd03e5f..181cfd19b5f 100644--- a/hermes_state_timeline.py+++ b/hermes_state_timeline.py@@ -7,6 +7,7 @@ from contextlib import contextmanagerfrom agent.compaction_display import project_compaction_message_for_displayfrom agent.context_compressor import user_originated_turn_view+from hermes_state_compaction_display import not_a_later_tool_copy_sqlfrom hermes_state_messages import DISPLAY_VISIBLE_SQL@@ -68,10 +69,10 @@ def _display_rows_sql(conn, session_id, *, users_only=False):SELECT (SELECT candidate.id FROM messages candidateWHERE candidate.session_id = :sidAND candidate.display_order = m.display_order- AND (candidate.active = 1 OR candidate.compacted = 1){DISPLAY_VISIBLE_SQL}+ AND (candidate.active = 1 OR candidate.compacted = 1){DISPLAY_VISIBLE_SQL}{not_a_later_tool_copy_sql('candidate')}ORDER BY candidate.active DESC, candidate.id DESC LIMIT 1) AS row_id,m.display_order AS sort_id- FROM messages m WHERE session_id = :sid AND (active = 1 OR compacted = 1){filters}+ FROM messages m WHERE session_id = :sid AND (active = 1 OR compacted = 1){filters}{not_a_later_tool_copy_sql('m')}GROUP BY m.display_order)"""return f"""WITH ranked AS (diff --git a/hermes_state_worker_compression.py b/hermes_state_worker_compression.pyindex 5b301db727e..46a0a142239 100644--- a/hermes_state_worker_compression.py+++ b/hermes_state_worker_compression.py@@ -190,7 +190,7 @@ def worker_history(db, conn, sid, payload):ids = view._resume_lineage_ids(target) if payload['include_ancestors'] else [target]active = db._active_clause(payload['include_inactive'], payload['include_compacted'])display = payload['include_compacted'] and not payload['include_inactive']- rows = conn.execute(f'SELECT {"compacted, " if display else ""}{db._CONVERSATION_ROW_COLUMNS} FROM messages '+ rows = conn.execute(f'SELECT {"session_id, compacted, display_order, " if display else ""}{db._CONVERSATION_ROW_COLUMNS} FROM messages 'f'WHERE session_id IN ({",".join("?" for _ in ids)}){active} ORDER BY id', ids).fetchall()if display: # the owner's display reads' rule: a message whose newest copy was removed stays hiddenfrom hermes_state_messages import SessionMessagesMixindiff --git a/tests/hermes_state/test_compacted_tool_result_paints_once.py b/tests/hermes_state/test_compacted_tool_result_paints_once.pynew file mode 100644index 00000000000..b42880336b9--- /dev/null+++ b/tests/hermes_state/test_compacted_tool_result_paints_once.py@@ -0,0 +1,50 @@+"""A compaction carries a tool result forward as a shortened stub (same ``message_uid``). Every display+read paints that tool result once, as the archived full original, while the model keeps the stub and+no stored row changes (#132939, #135655)."""+import pytest++from hermes_state import SessionDB+from hermes_state_timeline import get_session_messages_around++FULL = "FULL " * 1000+STUB = "[read_file] pruned stub"+++def db(tmp_path):+ instance = SessionDB(tmp_path / "state.db")+ yield instance+ instance.close()+++def _painted_tools(db, sid):+ _model, resumed = db.get_resume_conversations(sid)+ paged = db.get_messages(sid, include_compacted=True)+ shown = db.get_messages_as_conversation(sid, include_compacted=True)+ around = get_session_messages_around(db, sid, paged[0]["id"])["messages"]+ return [[m.get("content") for m in read if m["role"] == "tool"] for read in (resumed, paged, shown, around)]+++def test_compacted_tool_result_paints_once_as_its_archived_original(db):+ sid = "stub-display"+ db.create_session(sid, source="test")+ db.append_messages_batch(sid, [+ {"role": "user", "content": "read it", "timestamp": 100.0},+ {"role": "assistant", "content": "", "timestamp": 101.0,+ "tool_calls": [{"id": "call-1", "type": "function", "function": {"name": "read_file", "arguments": "{}"}}]},+ {"role": "tool", "content": FULL, "tool_call_id": "call-1", "tool_name": "read_file", "timestamp": 102.0,+ "display_metadata": {"tool_result_metadata": {"lines": 1000}}},+ {"role": "assistant", "content": "done reading", "timestamp": 103.0},+ ])+ for generation in (1, 2): # the stub is carried again by a later compaction+ history = db.get_messages_as_conversation(sid, include_row_ids=True)+ tool = next(m for m in history if m["role"] == "tool")+ tool["content"] = STUB+ db.archive_and_compact(sid, [{"role": "user", "content": f"summary {generation}", "timestamp": 150.0 + generation},+ *[m for m in history if not m.get("_compressed_summary")]])+ for bodies in _painted_tools(db, sid):+ assert bodies == [FULL]+ assert db.display_message_count(sid) == len(db.get_messages(sid, include_compacted=True))+ model_tool = [m for m in db.get_messages_as_conversation(sid) if m["role"] == "tool"]+ assert [m["content"] for m in model_tool] == [STUB]+ assert model_tool[0].get("display_metadata") == {"tool_result_metadata": {"lines": 1000}}