The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(state): a rewritten carry claims its tail row only when the compressor's rewrite explains it
Tekniumd652e53fdd1b
diff --git a/hermes_state_coverage.py b/hermes_state_coverage.pyindex 8171ad4a19d..216f8f81f4e 100644--- a/hermes_state_coverage.py+++ b/hermes_state_coverage.py@@ -7,6 +7,7 @@ held row cannot be named). Mixin bound via the MRO next to SessionMessagesMixin,from __future__ import annotations+from collections import Counterfrom typing import Any, Dict, List, Optional, Set, Tuplefrom agent.message_metadata import MESSAGE_UID, message_uid_or_none@@ -15,6 +16,33 @@ from hermes_state_common import _id_chunks, _placeholdersfrom hermes_state_messages import _parse_tool_calls+def _text_of(content: Any) -> str:+ """The text a message carries: a string, or its text parts joined; image parts carry none."""+ if isinstance(content, str):+ return content+ if isinstance(content, list):+ return "\n".join(+ str(part.get("text") or "") for part in content+ if isinstance(part, dict) and part.get("type") in {"text", "input_text", "output_text"})+ return ""+++def _images_replaced(stored: Any, held: Any) -> bool:+ """True when *held* is *stored* with at least one image part swapped for a text part, all else equal."""+ from agent.context_compressor import _is_image_part++ if not (isinstance(stored, list) and isinstance(held, list)) or len(stored) != len(held):+ return False+ swapped = False+ for was, now in zip(stored, held):+ if was == now:+ continue+ if not (_is_image_part(was) and isinstance(now, dict) and now.get("type") == "text"):+ return False+ swapped = True+ return swapped++class SessionCoverageMixin:"""Coverage helpers for SessionDB; relies on SessionMessagesMixin's codec/identity helpers."""@@ -27,22 +55,61 @@ class SessionCoverageMixin:clones it after the compacted set. A row the compressor ALSO held (flushed after the watermarkwas read, or left unnamed so coverage fell back to the watermark) is already in the compactedset as a carried copy; cloning it too stores the message twice, with a fresh id and timestamp- each (#137197). A claim never drops a different message, so each proof is checked against the- row: a dict that names a row by ``_row_id`` claims only that row; a ``message_uid`` match with other- content claims only when no active row outside *tail_ids* carries that uid (the compressor rewrote- its own carry: summary merged in, images stripped), else it is another version and is kept; identity plus- timestamp, or identity alone for a dict carrying no stamps at all, never claims a row whose uid- differs from the dict's. Each dict claims at most one row and each row is claimed once.-- ``weak`` holds the rows claimed by identity alone (with or without a timestamp) or by a uid whose content- differs: two surfaces can store the same text, and a rewritten carry is not the stored text, so the+ each (#137197). A claim never drops a different message, and a claim never leaves the model on an+ older copy, so claims run in two passes. The first settles every claim whose dict holds the row's+ stored content: ``_row_id``, then ``message_uid``, then identity plus timestamp, or identity alone+ for a dict carrying no stamps at all (never a row whose uid differs from the dict's). The second+ lets a still-unclaimed dict that names the row by ``_row_id`` or uid, with other content, claim it+ only when the compressor's own rewrite explains the difference (see ``_rewrite_explains``), exactly+ one tail row carries that uid and no active row outside *tail_ids* does (there the dict's original+ is that row, so the tail row is another version). Anything else is cloned: a duplicate, never a+ stale model view. Each dict claims at most one row and each row is claimed once.++ ``weak`` holds the rows claimed by identity alone (with or without a timestamp) or by a rewritten+ carry: two surfaces can store the same text, and a rewritten carry is not the stored text, so thecaller archives those as compacted history, never rewinds them out of display, and a wrong claim staysvisible."""if not tail_ids or not compacted_messages:return {}, set()- by_row: dict[int, tuple[int, tuple[Any, ...]]] = {}- by_uid: dict[str, tuple[int, tuple[Any, ...]]] = {}+ by_row, by_uid, by_key, bare = self._index_held_dicts(compacted_messages)+ rows = conn.execute(+ f"SELECT id, session_id, role, content, tool_call_id, tool_calls, timestamp, {MESSAGE_UID} FROM messages "+ f"WHERE id IN ({_placeholders(tail_ids)}) ORDER BY id", tail_ids,+ ).fetchall()+ identities = {+ int(row["id"]): self._row_identity(+ row["role"], self._decode_content(row["content"]), row["tool_call_id"],+ _parse_tool_calls(row["tool_calls"]))+ for row in rows}+ carried: dict[int, int] = {}+ weak: set[int] = set()+ for row in rows:+ row_id = int(row["id"])+ index, strong = self._claim_stored_copy(+ row, identities[row_id], by_row, by_uid, by_key, bare, set(carried.values()))+ if index is not None:+ carried[row_id] = index+ if not strong:+ weak.add(row_id)+ named_uids = {*by_uid, *(named[2] for named in by_row.values() if named[2] is not None)}+ held_elsewhere = self._uids_active_outside(conn, rows, tail_ids, named_uids)+ tail_uids = Counter(row[MESSAGE_UID] for row in rows if row[MESSAGE_UID])+ for row in rows:+ row_id, row_uid = int(row["id"]), row[MESSAGE_UID] or None+ if row_id in carried or row_uid is None or tail_uids[row_uid] != 1 or row_uid in held_elsewhere:+ continue+ index = self._rewritten_carry(row, row_uid, compacted_messages, by_row, by_uid, set(carried.values()))+ if index is not None:+ carried[row_id] = index+ weak.add(row_id)+ return carried, weak++ def _index_held_dicts(self, compacted_messages: list[dict[str, Any]]) -> tuple[dict, dict, dict, dict]:+ """Held dicts by the proof they carry: ``_row_id`` -> (index, identity, uid); uid -> [(index, identity)];+ (identity, timestamp) -> (index, uid); identity -> [index] for dicts with no stamp at all."""+ by_row: dict[int, tuple[int, tuple[Any, ...], Optional[str]]] = {}+ by_uid: dict[str, list[tuple[int, tuple[Any, ...]]]] = {}by_key: dict[tuple[Any, ...], tuple[int, Optional[str]]] = {}bare: dict[tuple[Any, ...], list[int]] = {}for index, message in enumerate(compacted_messages):@@ -52,45 +119,78 @@ class SessionCoverageMixin:message.get("role", "unknown"), message.get("content"), message.get("tool_call_id"),_parse_tool_calls(message.get("tool_calls")))row_id = message.get("_row_id")+ uid = message_uid_or_none(message)if isinstance(row_id, int) and not isinstance(row_id, bool) and row_id > 0:# Authoritative only while the stored identity still matches (a stale stamp names another row).- by_row.setdefault(row_id, (index, identity))+ by_row.setdefault(row_id, (index, identity, uid))continue- uid = message_uid_or_none(message)if uid is not None:- by_uid.setdefault(uid, (index, identity))+ by_uid.setdefault(uid, []).append((index, identity))timestamp = coerce_epoch(message.get("timestamp"), field="message timestamp")if timestamp is not None:by_key.setdefault((*identity, timestamp), (index, uid))elif uid is None:bare.setdefault(identity, []).append(index)- carried: dict[int, int] = {}- weak: set[int] = set()- claimed: set[int] = set()- rows = conn.execute(- f"SELECT id, session_id, role, content, tool_call_id, tool_calls, timestamp, {MESSAGE_UID} FROM messages "- f"WHERE id IN ({_placeholders(tail_ids)}) ORDER BY id", tail_ids,- ).fetchall()- held_elsewhere = self._uids_active_outside(conn, rows, tail_ids, by_uid)- for row in rows:- row_id, row_uid = int(row["id"]), row[MESSAGE_UID] or None- identity = self._row_identity(- row["role"], self._decode_content(row["content"]), row["tool_call_id"],- _parse_tool_calls(row["tool_calls"]))- index, strong = self._claim_tail_row(- row, row_id, row_uid, identity, by_row, by_uid, by_key, bare, held_elsewhere)- if index is not None and index not in claimed:- claimed.add(index)- carried[row_id] = index- if not strong:- weak.add(row_id)- return carried, weak+ return by_row, by_uid, by_key, bare++ @staticmethod+ def _claim_stored_copy(row, identity, by_row, by_uid, by_key, bare, claimed) -> tuple[Optional[int], bool]:+ """First pass: the unclaimed dict holding *row*'s stored content, and whether it names that row+ (``_row_id`` / uid) rather than matching its text alone."""+ named = by_row.get(int(row["id"]))+ if named is not None and named[1] == identity and named[0] not in claimed:+ return named[0], True+ row_uid = row[MESSAGE_UID] or None+ for index, held in by_uid.get(row_uid, ()) if row_uid is not None else ():+ if held == identity and index not in claimed:+ return index, True+ timestamp = coerce_epoch(row["timestamp"], field="message timestamp")+ keyed = by_key.get((*identity, timestamp)) if timestamp is not None else None+ # A keyed dict with a uid matches only through the uid above.+ if keyed is not None and keyed[1] is None and keyed[0] not in claimed:+ return keyed[0], False+ if queue := bare.get(identity):+ return queue.pop(0), False+ return None, False++ def _rewritten_carry(self, row, row_uid, compacted_messages, by_row, by_uid, claimed) -> Optional[int]:+ """Second pass: the one unclaimed dict naming *row* (``_row_id`` with the row's uid, else its uid alone)+ whose different content the compressor's rewrite of that row explains; None means clone."""+ named = by_row.get(int(row["id"]))+ if named is not None and named[2] == row_uid:+ candidates = [named[0]]+ else:+ candidates = [index for index, _identity in by_uid.get(row_uid, ())]+ candidates = [index for index in candidates if index not in claimed]+ if len(candidates) != 1:+ return None+ held = compacted_messages[candidates[0]]+ if self._rewrite_explains(row, held):+ return candidates[0]+ return None++ def _rewrite_explains(self, row, held: dict[str, Any]) -> bool:+ """True when *held* differs from stored *row* only the way the compressor rewrites a carry it keeps:+ image parts replaced by text placeholders, or the row's text kept verbatim inside a longer text+ (a summary merged in, prior-context header and markers around it). The summary flag alone is not+ enough: a flagged copy taken before an in-place rewrite of the row would put the older text back."""+ role = row["role"]+ if (held.get("role"), held.get("tool_call_id") or None) != (role, row["tool_call_id"] or None):+ return False+ if (_parse_tool_calls(held.get("tool_calls")) or None) != (_parse_tool_calls(row["tool_calls"]) or None):+ return False+ stored = self._loaded_view_content(role, self._decode_content(row["content"]))+ content = self._loaded_view_content(role, held.get("content"))+ if _images_replaced(stored, content):+ return True+ stored_text = _text_of(stored)+ return bool(stored_text.strip()) and stored_text in _text_of(content)@staticmethod- def _uids_active_outside(conn, rows, tail_ids: list[int], by_uid: dict[str, Any]) -> set[str]:+ def _uids_active_outside(conn, rows, tail_ids: list[int], named_uids) -> set[str]:"""uids of tail *rows* a held dict names that an ACTIVE row outside *tail_ids* (a covered row, below thewatermark) also carries: there the dict's own original is that row, so the tail row is another version."""- wanted = sorted({row[MESSAGE_UID] for row in rows if row[MESSAGE_UID] in by_uid})+ wanted = sorted({row[MESSAGE_UID] for row in rows if row[MESSAGE_UID] in named_uids})if not wanted:return set()tail, session_id = set(tail_ids), rows[0]["session_id"]@@ -103,27 +203,6 @@ class SessionCoverageMixin:if int(found["id"]) not in tail}- @staticmethod- def _claim_tail_row(- row, row_id, row_uid, identity, by_row, by_uid, by_key, bare, held_elsewhere,- ) -> tuple[Optional[int], bool]:- """The carrying dict index for one tail row and whether the proof names that row with its stored- content (``_row_id`` / uid); a uid carry the compressor rewrote is a weak claim."""- if (named := by_row.get(row_id)) is not None and named[1] == identity:- return named[0], True- if row_uid is not None and (by_id := by_uid.get(row_uid)) is not None:- if by_id[1] == identity:- return by_id[0], True- if row_uid not in held_elsewhere:- return by_id[0], False- timestamp = coerce_epoch(row["timestamp"], field="message timestamp")- if timestamp is not None and (keyed := by_key.get((*identity, timestamp))) is not None:- if keyed[1] is None: # a dict uid equal to the row's already matched above- return keyed[0], False- if queue := bare.get(identity):- return queue.pop(0), False- return None, False-@staticmethoddef _carried_tail_slots(carried: dict[int, int], compacted_count: int, tail_count: int) -> int:"""How many of the carried tail's *tail_count* dicts name a row in *carried*: those originals sitdiff --git a/tests/hermes_state/test_compaction_carried_tail_not_cloned.py b/tests/hermes_state/test_compaction_carried_tail_not_cloned.pyindex 43c34948930..df5665c8c19 100644--- a/tests/hermes_state/test_compaction_carried_tail_not_cloned.py+++ b/tests/hermes_state/test_compaction_carried_tail_not_cloned.py@@ -122,3 +122,46 @@ def test_a_colliding_stored_row_is_never_dropped(db: SessionDB) -> None:restored.pop("_db_persisted", None)db.append_messages_batch(SID, [restored])assert "continue, and also this" in [m["content"] for m in db.get_messages_as_conversation(SID)]+++def _row_rewritten_after_hold(db: SessionDB) -> tuple:+ """The compressor took a uid-only copy, then the row was rewritten in place (an @file expansion)."""+ db.append_message(SID, role="user", content="raw @file")+ row = db.get_messages_as_conversation(SID, include_row_ids=True)[-1]+ db.set_user_message_content(SID, row["_row_id"], "expanded file body")+ held = {"role": "user", "content": "raw @file", "message_uid": row["message_uid"]}+ return [_summary(), held], [SUMMARY_TEXT, "raw @file", "expanded file body"]+++def _edit_reinserted_with_its_uid(db: SessionDB) -> tuple:+ """Two tail rows share a uid (an edit re-inserted with it); the held copy is the edit."""+ db.append_message(SID, role="assistant", content="v1 draft")+ uid = db.get_messages_as_conversation(SID)[-1]["message_uid"]+ db.append_messages_batch(SID, [{"role": "assistant", "content": "v2 edited", "message_uid": uid}])+ held = {"role": "assistant", "content": "v2 edited", "message_uid": uid}+ return [_summary(), held], [SUMMARY_TEXT, "v2 edited", "v1 draft"]+++def _summary_merged_into_named_row(db: SessionDB) -> tuple:+ """The in-process compressor merged the summary into its copy of the row, which keeps ``_row_id``."""+ db.append_message(SID, role="user", content="steer during the summary call")+ row = db.get_messages_as_conversation(SID, include_row_ids=True)[-1]+ merged = f"[prior]\n{row['content']}\n\n{SUMMARY_TEXT}"+ return [{**row, "content": merged, "_compressed_summary": True}], [merged]++[email protected]("build", [+ _row_rewritten_after_hold, _edit_reinserted_with_its_uid, _summary_merged_into_named_row])+def test_a_rewritten_carry_claims_only_what_the_compressor_rewrote(db: SessionDB, build) -> None:+ """A held copy whose content differs from its tail row stands for that row only when the compressor's own+ rewrite explains the difference: the model never ends up on an older copy or on two copies of one+ message, and every stored version stays visible."""+ watermark = db.get_active_message_watermark(SID)+ compacted, expected_live = build(db)+ versions = [r["content"] for r in _rows(db) if r["active"] and r["id"] > watermark]++ db.archive_and_compact(SID, compacted, watermark=watermark, tail_count=1)++ assert [m["content"] for m in db.get_messages_as_conversation(SID)] == expected_live+ display = "\n".join(str(m["content"]) for m in db.get_messages(SID, include_compacted=True))+ assert versions and all(text in display for text in versions)