The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(state): the re-insert guard compares through the restore lens; a rewritten carry keeps its uid claim
Teknium33db60eefe16
diff --git a/agent/transcript_repair.py b/agent/transcript_repair.pyindex dbaf76ef4ef..6672ad09db0 100644--- a/agent/transcript_repair.py+++ b/agent/transcript_repair.py@@ -10,10 +10,11 @@ from typing import Any, Callable, Dict, List, Mappingfrom agent.context_compressor import _DB_PERSISTED_MARKERfrom agent.message_metadata import (- CANONICAL_ROW, DB_ROW_SNAPSHOT, MESSAGE_UID, RESTORED_FROM_STORE, copy_identity_fields, message_uid_or_none)+ ABSORBED_MESSAGE_UIDS, CANONICAL_ROW, DB_ROW_SNAPSHOT, MESSAGE_UID, RESTORED_FROM_STORE, TOOL_CALL_UIDS,+ copy_identity_fields, message_uid_or_none)from hermes_state_common import _id_chunks, _placeholdersfrom hermes_state_identity import _fill_missing_tool_call_uids, _restore_row_identity-from hermes_state_messages import _MESSAGE_WRITE_COLUMNS+from hermes_state_messages import _MESSAGE_WRITE_COLUMNS, SessionMessagesMixin# Durable payload columns a row-addressed rewrite may change: every INSERT column except row identity,@@ -142,15 +143,35 @@ def _active_uid_dedupe(return durable+# Owned columns a restore projection may leave off the dict: ``token_count`` is never decoded, the summary+# flag only on request, and the underscore identity maps are lost to any wire strip. A dict without a value+# says nothing about the column, so the row's value stands (the flush row turns an absent summary flag into+# False, so falsy is the absent case); a dict that carries a value is compared as usual.+_RESTORE_MAY_OMIT = (+ ("token_count", "token_count"), ("_compressed_summary", "_compressed_summary"),+ ("tool_call_uids", TOOL_CALL_UIDS), ("absorbed_message_uids", ABSORBED_MESSAGE_UIDS))++def _same_as_stored(msg: dict[str, Any], rows: list[sqlite3.Row],serialize_message_fn: Callable[[dict[str, Any], float], Mapping[str, Any]],+ encode_content_fn: Callable[[Any], Any], decode_content_fn: Callable[[Any], Any],) -> bool:- """*msg* serializes to the owned columns of one of its uid's active *rows*: the re-handed copy, not an edit."""+ """*msg* serializes to the owned columns of one of its uid's active *rows*: the re-handed copy, not an edit.++ The row is seen through the restore lens (``_loaded_view_content``: user/assistant text sanitized and+ stripped on load), because that is what the dict was rebuilt from; the dict's own content is never+ normalized, so text that differs after the lens is an edit and inserts.+ """for row in rows:- serialized = serialize_message_fn(msg, float(row["timestamp"]))- if serialized["token_count"] is None: # a restored dict never decodes token_count: unknown, not NULL- serialized = {**serialized, "token_count": row["token_count"]}+ serialized = dict(serialize_message_fn(msg, float(row["timestamp"])))+ for column, key in _RESTORE_MAY_OMIT:+ if not msg.get(key):+ serialized[column] = row[column]+ if serialized["content"] == encode_content_fn(+ SessionMessagesMixin._loaded_view_content(row["role"], decode_content_fn(row["content"]))+ ):+ serialized["content"] = row["content"]if all(row[column] == serialized[column] for column in _OWNED_COLUMNS):return Truereturn False@@ -194,7 +215,7 @@ def resolve_and_repair_transcript_batch(isinstance(msg, dict)and (stored := durable_uids.get(message_uid_or_none(msg) or ""))and not msg.get(_PERSIST_AFTER_ADMISSION_INTERRUPT)- and _same_as_stored(msg, stored, serialize_message_fn)+ and _same_as_stored(msg, stored, serialize_message_fn, encode_content_fn, decode_content_fn)):# Restored logical message, no row address and no CAS version (the wire strip removes# every underscore-prefixed field): its uid already names an ACTIVE durable row holdingdiff --git a/hermes_state_coverage.py b/hermes_state_coverage.pyindex b4916b68dd2..8171ad4a19d 100644--- a/hermes_state_coverage.py+++ b/hermes_state_coverage.py@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Set, Tuplefrom agent.message_metadata import MESSAGE_UID, message_uid_or_nonefrom hermes_cli.timefmt import coerce_epoch-from hermes_state_common import _placeholders+from hermes_state_common import _id_chunks, _placeholdersfrom hermes_state_messages import _parse_tool_calls@@ -28,14 +28,16 @@ class SessionCoverageMixin:was read, or left unnamed so coverage fell back to the watermark) is already in the compactedset as a carried copy; cloning it too stores the message twice, with a fresh id and timestampeach (#137197). A claim never drops a different message, so each proof is checked against the- row: a dict that names a row by ``_row_id`` claims only that row; a ``message_uid`` match also needs- the same identity (a same-uid row with other content is another version, kept); identity plus+ row: a dict that names a row by ``_row_id`` claims only that row; a ``message_uid`` match with other+ content claims only when no active row outside *tail_ids* carries that uid (the compressor rewrote+ its own carry: summary merged in, images stripped), else it is another version and is kept; identity plustimestamp, or identity alone for a dict carrying no stamps at all, never claims a row whose uiddiffers from the dict's. Each dict claims at most one row and each row is claimed once.- ``weak`` holds the rows claimed by identity alone (with or without a timestamp): two surfaces can- store the same text, so the caller archives those as compacted history, never rewinds them out of- display, and a wrong claim stays visible.+ ``weak`` holds the rows claimed by identity alone (with or without a timestamp) or by a uid whose content+ differs: two surfaces can store the same text, and a rewritten carry is not the stored text, so the+ caller archives those as compacted history, never rewinds them out of display, and a wrong claim stays+ visible."""if not tail_ids or not compacted_messages:return {}, set()@@ -65,15 +67,18 @@ class SessionCoverageMixin:carried: dict[int, int] = {}weak: set[int] = set()claimed: set[int] = set()- for row in conn.execute(- f"SELECT id, role, content, tool_call_id, tool_calls, timestamp, {MESSAGE_UID} FROM messages "+ rows = conn.execute(+ f"SELECT id, session_id, role, content, tool_call_id, tool_calls, timestamp, {MESSAGE_UID} FROM messages "f"WHERE id IN ({_placeholders(tail_ids)}) ORDER BY id", tail_ids,- ).fetchall():+ ).fetchall()+ held_elsewhere = self._uids_active_outside(conn, rows, tail_ids, by_uid)+ for row in rows:row_id, row_uid = int(row["id"]), row[MESSAGE_UID] or Noneidentity = self._row_identity(row["role"], self._decode_content(row["content"]), row["tool_call_id"],_parse_tool_calls(row["tool_calls"]))- index, strong = self._claim_tail_row(row, row_id, row_uid, identity, by_row, by_uid, by_key, bare)+ index, strong = self._claim_tail_row(+ row, row_id, row_uid, identity, by_row, by_uid, by_key, bare, held_elsewhere)if index is not None and index not in claimed:claimed.add(index)carried[row_id] = index@@ -82,12 +87,35 @@ class SessionCoverageMixin:return carried, weak@staticmethod- def _claim_tail_row(row, row_id, row_uid, identity, by_row, by_uid, by_key, bare) -> tuple[Optional[int], bool]:- """The carrying dict index for one tail row and whether the proof names that row (``_row_id`` / uid)."""+ def _uids_active_outside(conn, rows, tail_ids: list[int], by_uid: dict[str, Any]) -> set[str]:+ """uids of tail *rows* a held dict names that an ACTIVE row outside *tail_ids* (a covered row, below the+ watermark) also carries: there the dict's own original is that row, so the tail row is another version."""+ wanted = sorted({row[MESSAGE_UID] for row in rows if row[MESSAGE_UID] in by_uid})+ if not wanted:+ return set()+ tail, session_id = set(tail_ids), rows[0]["session_id"]+ return {+ found[MESSAGE_UID]+ for chunk in _id_chunks(wanted)+ for found in conn.execute(+ f"SELECT id, {MESSAGE_UID} FROM messages WHERE session_id = ? AND active = 1 "+ f"AND {MESSAGE_UID} IN ({_placeholders(chunk)})", (session_id, *chunk)).fetchall()+ if int(found["id"]) not in tail+ }++ @staticmethod+ def _claim_tail_row(+ row, row_id, row_uid, identity, by_row, by_uid, by_key, bare, held_elsewhere,+ ) -> tuple[Optional[int], bool]:+ """The carrying dict index for one tail row and whether the proof names that row with its stored+ content (``_row_id`` / uid); a uid carry the compressor rewrote is a weak claim."""if (named := by_row.get(row_id)) is not None and named[1] == identity:return named[0], True- if row_uid is not None and (by_id := by_uid.get(row_uid)) is not None and by_id[1] == identity:- return by_id[0], True+ if row_uid is not None and (by_id := by_uid.get(row_uid)) is not None:+ if by_id[1] == identity:+ return by_id[0], True+ if row_uid not in held_elsewhere:+ return by_id[0], Falsetimestamp = coerce_epoch(row["timestamp"], field="message timestamp")if timestamp is not None and (keyed := by_key.get((*identity, timestamp))) is not None:if keyed[1] is None: # a dict uid equal to the row's already matched abovediff --git a/tests/agent/test_flush_uid_reinsert_guard.py b/tests/agent/test_flush_uid_reinsert_guard.pyindex 5b55d113fa8..84e229d941d 100644--- a/tests/agent/test_flush_uid_reinsert_guard.py+++ b/tests/agent/test_flush_uid_reinsert_guard.py@@ -144,4 +144,46 @@ def test_restored_snapshot_rewrite_still_lands(tmp_path):"live edit on a snapshot-carrying restored dict was silently dropped: "f"final contents {contents}")- assert _row_count(db, sid) == 2, "the rewrite must not add rows"\ No newline at end of file+ assert _row_count(db, sid) == 2, "the rewrite must not add rows"++_READ_CALL = [{"id": "call_1", "type": "function", "function": {"name": "read_file", "arguments": "{\"path\": \"a.txt\"}"}}]++[email protected]("turn, strip_all", [+ pytest.param([{"role": "user", "content": "q"}, {"role": "assistant", "content": "Answer line.\n"}], False,+ id="assistant-trailing-newline"),+ pytest.param([{"role": "user", "content": " please do X\n"}, {"role": "assistant", "content": "done"}], False,+ id="user-edge-whitespace"),+ pytest.param([{"role": "user", "content": "<memory-context>\nfact: likes tea\n</memory-context>\n\nwhat do I like?"},+ {"role": "assistant", "content": "tea"}], False, id="user-memory-context-block"),+ pytest.param(None, False, id="compaction-summary-row"),+ pytest.param([{"role": "user", "content": "read it"},+ {"role": "assistant", "content": "Reading.", "tool_calls": _READ_CALL, "finish_reason": "tool_calls"},+ {"role": "tool", "content": "line1", "tool_call_id": "call_1"},+ {"role": "assistant", "content": "one line"}], True, id="tool-call-row-underscore-fields-lost"),+])+def test_restore_normalised_rows_are_not_reinserted_but_edits_land(tmp_path, turn, strip_all):+ """Restore hands back text sanitized and stripped, no summary flag and (across a wire) no underscore+ identity maps: the guard compares through that lens, so the copy is skipped while an edit still lands."""+ agent, db = _make_agent_with_db(tmp_path)+ sid = agent.session_id+ if turn is None:+ db.append_messages_batch(sid, [{"role": "user", "content": "q0"}, {"role": "assistant", "content": "a1"}])+ db.archive_and_compact(sid, [{"role": "user", "content": "[CONTEXT COMPACTION] summary",+ "_compressed_summary": True}, {"role": "assistant", "content": "a1"}])+ live = db.get_messages_as_conversation(sid)+ else:+ live = [{**m, "timestamp": 1000.0 + i} for i, m in enumerate(turn)]+ agent._flush_messages_to_session_db(live)+ active = lambda: db._read_one("SELECT COUNT(*) FROM messages WHERE session_id = ? AND active = 1", (sid,))[0]+ stored = active()++ restored = [{k: v for k, v in m.items() if not (k.startswith("_") if strip_all else k == _DB_PERSISTED_MARKER)}+ for m in db.get_messages_as_conversation(sid)]+ agent._flush_messages_to_session_db(live + restored)+ assert active() == stored, "a restored copy of a durable row was inserted again"++ edited = {k: v for k, v in db.get_messages_as_conversation(sid)[-1].items() if k != _DB_PERSISTED_MARKER}+ edited["content"] += " (edited)"+ agent._flush_messages_to_session_db(live + restored + [edited])+ assert edited["content"] in [m["content"] for m in db.get_messages_as_conversation(sid)]