Skip to content

The agent that grows with you

fix(state): the re-insert guard compares through the restore lens; a rewritten carry keeps its uid claim

Teknium33db60eefe16
diff --git a/agent/transcript_repair.py b/agent/transcript_repair.py
index dbaf76ef4ef..6672ad09db0 100644
--- a/agent/transcript_repair.py
+++ b/agent/transcript_repair.py
@@ -10,10 +10,11 @@ from typing import Any, Callable, Dict, List, Mapping
from agent.context_compressor import _DB_PERSISTED_MARKER
from agent.message_metadata import (
- CANONICAL_ROW, DB_ROW_SNAPSHOT, MESSAGE_UID, RESTORED_FROM_STORE, copy_identity_fields, message_uid_or_none)
+ ABSORBED_MESSAGE_UIDS, CANONICAL_ROW, DB_ROW_SNAPSHOT, MESSAGE_UID, RESTORED_FROM_STORE, TOOL_CALL_UIDS,
+ copy_identity_fields, message_uid_or_none)
from hermes_state_common import _id_chunks, _placeholders
from hermes_state_identity import _fill_missing_tool_call_uids, _restore_row_identity
-from hermes_state_messages import _MESSAGE_WRITE_COLUMNS
+from hermes_state_messages import _MESSAGE_WRITE_COLUMNS, SessionMessagesMixin
# Durable payload columns a row-addressed rewrite may change: every INSERT column except row identity,
@@ -142,15 +143,35 @@ def _active_uid_dedupe(
return durable
+# Owned columns a restore projection may leave off the dict: ``token_count`` is never decoded, the summary
+# flag only on request, and the underscore identity maps are lost to any wire strip. A dict without a value
+# says nothing about the column, so the row's value stands (the flush row turns an absent summary flag into
+# False, so falsy is the absent case); a dict that carries a value is compared as usual.
+_RESTORE_MAY_OMIT = (
+ ("token_count", "token_count"), ("_compressed_summary", "_compressed_summary"),
+ ("tool_call_uids", TOOL_CALL_UIDS), ("absorbed_message_uids", ABSORBED_MESSAGE_UIDS))
+
+
def _same_as_stored(
msg: dict[str, Any], rows: list[sqlite3.Row],
serialize_message_fn: Callable[[dict[str, Any], float], Mapping[str, Any]],
+ encode_content_fn: Callable[[Any], Any], decode_content_fn: Callable[[Any], Any],
) -> bool:
- """*msg* serializes to the owned columns of one of its uid's active *rows*: the re-handed copy, not an edit."""
+ """*msg* serializes to the owned columns of one of its uid's active *rows*: the re-handed copy, not an edit.
+
+ The row is seen through the restore lens (``_loaded_view_content``: user/assistant text sanitized and
+ stripped on load), because that is what the dict was rebuilt from; the dict's own content is never
+ normalized, so text that differs after the lens is an edit and inserts.
+ """
for row in rows:
- serialized = serialize_message_fn(msg, float(row["timestamp"]))
- if serialized["token_count"] is None: # a restored dict never decodes token_count: unknown, not NULL
- serialized = {**serialized, "token_count": row["token_count"]}
+ serialized = dict(serialize_message_fn(msg, float(row["timestamp"])))
+ for column, key in _RESTORE_MAY_OMIT:
+ if not msg.get(key):
+ serialized[column] = row[column]
+ if serialized["content"] == encode_content_fn(
+ SessionMessagesMixin._loaded_view_content(row["role"], decode_content_fn(row["content"]))
+ ):
+ serialized["content"] = row["content"]
if all(row[column] == serialized[column] for column in _OWNED_COLUMNS):
return True
return False
@@ -194,7 +215,7 @@ def resolve_and_repair_transcript_batch(
isinstance(msg, dict)
and (stored := durable_uids.get(message_uid_or_none(msg) or ""))
and not msg.get(_PERSIST_AFTER_ADMISSION_INTERRUPT)
- and _same_as_stored(msg, stored, serialize_message_fn)
+ and _same_as_stored(msg, stored, serialize_message_fn, encode_content_fn, decode_content_fn)
):
# Restored logical message, no row address and no CAS version (the wire strip removes
# every underscore-prefixed field): its uid already names an ACTIVE durable row holding
diff --git a/hermes_state_coverage.py b/hermes_state_coverage.py
index b4916b68dd2..8171ad4a19d 100644
--- a/hermes_state_coverage.py
+++ b/hermes_state_coverage.py
@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple
from agent.message_metadata import MESSAGE_UID, message_uid_or_none
from hermes_cli.timefmt import coerce_epoch
-from hermes_state_common import _placeholders
+from hermes_state_common import _id_chunks, _placeholders
from hermes_state_messages import _parse_tool_calls
@@ -28,14 +28,16 @@ class SessionCoverageMixin:
was read, or left unnamed so coverage fell back to the watermark) is already in the compacted
set as a carried copy; cloning it too stores the message twice, with a fresh id and timestamp
each (#137197). A claim never drops a different message, so each proof is checked against the
- row: a dict that names a row by ``_row_id`` claims only that row; a ``message_uid`` match also needs
- the same identity (a same-uid row with other content is another version, kept); identity plus
+ row: a dict that names a row by ``_row_id`` claims only that row; a ``message_uid`` match with other
+ content claims only when no active row outside *tail_ids* carries that uid (the compressor rewrote
+ its own carry: summary merged in, images stripped), else it is another version and is kept; identity plus
timestamp, or identity alone for a dict carrying no stamps at all, never claims a row whose uid
differs from the dict's. Each dict claims at most one row and each row is claimed once.
- ``weak`` holds the rows claimed by identity alone (with or without a timestamp): two surfaces can
- store the same text, so the caller archives those as compacted history, never rewinds them out of
- display, and a wrong claim stays visible.
+ ``weak`` holds the rows claimed by identity alone (with or without a timestamp) or by a uid whose content
+ differs: two surfaces can store the same text, and a rewritten carry is not the stored text, so the
+ caller archives those as compacted history, never rewinds them out of display, and a wrong claim stays
+ visible.
"""
if not tail_ids or not compacted_messages:
return {}, set()
@@ -65,15 +67,18 @@ class SessionCoverageMixin:
carried: dict[int, int] = {}
weak: set[int] = set()
claimed: set[int] = set()
- for row in conn.execute(
- f"SELECT id, role, content, tool_call_id, tool_calls, timestamp, {MESSAGE_UID} FROM messages "
+ rows = conn.execute(
+ f"SELECT id, session_id, role, content, tool_call_id, tool_calls, timestamp, {MESSAGE_UID} FROM messages "
f"WHERE id IN ({_placeholders(tail_ids)}) ORDER BY id", tail_ids,
- ).fetchall():
+ ).fetchall()
+ held_elsewhere = self._uids_active_outside(conn, rows, tail_ids, by_uid)
+ for row in rows:
row_id, row_uid = int(row["id"]), row[MESSAGE_UID] or None
identity = self._row_identity(
row["role"], self._decode_content(row["content"]), row["tool_call_id"],
_parse_tool_calls(row["tool_calls"]))
- index, strong = self._claim_tail_row(row, row_id, row_uid, identity, by_row, by_uid, by_key, bare)
+ index, strong = self._claim_tail_row(
+ row, row_id, row_uid, identity, by_row, by_uid, by_key, bare, held_elsewhere)
if index is not None and index not in claimed:
claimed.add(index)
carried[row_id] = index
@@ -82,12 +87,35 @@ class SessionCoverageMixin:
return carried, weak
@staticmethod
- def _claim_tail_row(row, row_id, row_uid, identity, by_row, by_uid, by_key, bare) -> tuple[Optional[int], bool]:
- """The carrying dict index for one tail row and whether the proof names that row (``_row_id`` / uid)."""
+ def _uids_active_outside(conn, rows, tail_ids: list[int], by_uid: dict[str, Any]) -> set[str]:
+ """uids of tail *rows* a held dict names that an ACTIVE row outside *tail_ids* (a covered row, below the
+ watermark) also carries: there the dict's own original is that row, so the tail row is another version."""
+ wanted = sorted({row[MESSAGE_UID] for row in rows if row[MESSAGE_UID] in by_uid})
+ if not wanted:
+ return set()
+ tail, session_id = set(tail_ids), rows[0]["session_id"]
+ return {
+ found[MESSAGE_UID]
+ for chunk in _id_chunks(wanted)
+ for found in conn.execute(
+ f"SELECT id, {MESSAGE_UID} FROM messages WHERE session_id = ? AND active = 1 "
+ f"AND {MESSAGE_UID} IN ({_placeholders(chunk)})", (session_id, *chunk)).fetchall()
+ if int(found["id"]) not in tail
+ }
+
+ @staticmethod
+ def _claim_tail_row(
+ row, row_id, row_uid, identity, by_row, by_uid, by_key, bare, held_elsewhere,
+ ) -> tuple[Optional[int], bool]:
+ """The carrying dict index for one tail row and whether the proof names that row with its stored
+ content (``_row_id`` / uid); a uid carry the compressor rewrote is a weak claim."""
if (named := by_row.get(row_id)) is not None and named[1] == identity:
return named[0], True
- if row_uid is not None and (by_id := by_uid.get(row_uid)) is not None and by_id[1] == identity:
- return by_id[0], True
+ if row_uid is not None and (by_id := by_uid.get(row_uid)) is not None:
+ if by_id[1] == identity:
+ return by_id[0], True
+ if row_uid not in held_elsewhere:
+ return by_id[0], False
timestamp = coerce_epoch(row["timestamp"], field="message timestamp")
if timestamp is not None and (keyed := by_key.get((*identity, timestamp))) is not None:
if keyed[1] is None: # a dict uid equal to the row's already matched above
diff --git a/tests/agent/test_flush_uid_reinsert_guard.py b/tests/agent/test_flush_uid_reinsert_guard.py
index 5b55d113fa8..84e229d941d 100644
--- a/tests/agent/test_flush_uid_reinsert_guard.py
+++ b/tests/agent/test_flush_uid_reinsert_guard.py
@@ -144,4 +144,46 @@ def test_restored_snapshot_rewrite_still_lands(tmp_path):
"live edit on a snapshot-carrying restored dict was silently dropped: "
f"final contents {contents}"
)
- assert _row_count(db, sid) == 2, "the rewrite must not add rows"
\ No newline at end of file
+ assert _row_count(db, sid) == 2, "the rewrite must not add rows"
+
+_READ_CALL = [{"id": "call_1", "type": "function", "function": {"name": "read_file", "arguments": "{\"path\": \"a.txt\"}"}}]
+
+
[email protected]("turn, strip_all", [
+ pytest.param([{"role": "user", "content": "q"}, {"role": "assistant", "content": "Answer line.\n"}], False,
+ id="assistant-trailing-newline"),
+ pytest.param([{"role": "user", "content": " please do X\n"}, {"role": "assistant", "content": "done"}], False,
+ id="user-edge-whitespace"),
+ pytest.param([{"role": "user", "content": "<memory-context>\nfact: likes tea\n</memory-context>\n\nwhat do I like?"},
+ {"role": "assistant", "content": "tea"}], False, id="user-memory-context-block"),
+ pytest.param(None, False, id="compaction-summary-row"),
+ pytest.param([{"role": "user", "content": "read it"},
+ {"role": "assistant", "content": "Reading.", "tool_calls": _READ_CALL, "finish_reason": "tool_calls"},
+ {"role": "tool", "content": "line1", "tool_call_id": "call_1"},
+ {"role": "assistant", "content": "one line"}], True, id="tool-call-row-underscore-fields-lost"),
+])
+def test_restore_normalised_rows_are_not_reinserted_but_edits_land(tmp_path, turn, strip_all):
+ """Restore hands back text sanitized and stripped, no summary flag and (across a wire) no underscore
+ identity maps: the guard compares through that lens, so the copy is skipped while an edit still lands."""
+ agent, db = _make_agent_with_db(tmp_path)
+ sid = agent.session_id
+ if turn is None:
+ db.append_messages_batch(sid, [{"role": "user", "content": "q0"}, {"role": "assistant", "content": "a1"}])
+ db.archive_and_compact(sid, [{"role": "user", "content": "[CONTEXT COMPACTION] summary",
+ "_compressed_summary": True}, {"role": "assistant", "content": "a1"}])
+ live = db.get_messages_as_conversation(sid)
+ else:
+ live = [{**m, "timestamp": 1000.0 + i} for i, m in enumerate(turn)]
+ agent._flush_messages_to_session_db(live)
+ active = lambda: db._read_one("SELECT COUNT(*) FROM messages WHERE session_id = ? AND active = 1", (sid,))[0]
+ stored = active()
+
+ restored = [{k: v for k, v in m.items() if not (k.startswith("_") if strip_all else k == _DB_PERSISTED_MARKER)}
+ for m in db.get_messages_as_conversation(sid)]
+ agent._flush_messages_to_session_db(live + restored)
+ assert active() == stored, "a restored copy of a durable row was inserted again"
+
+ edited = {k: v for k, v in db.get_messages_as_conversation(sid)[-1].items() if k != _DB_PERSISTED_MARKER}
+ edited["content"] += " (edited)"
+ agent._flush_messages_to_session_db(live + restored + [edited])
+ assert edited["content"] in [m["content"] for m in db.get_messages_as_conversation(sid)]