Skip to content

The agent that grows with you

Memory entries stay removable after the char limit is lowered (salvage #133029) (#144240)

Teknium26e8e3f4d51b
diff --git a/agent/learning_mutations.py b/agent/learning_mutations.py
index 415d22fa176..3539ca6fbf9 100644
--- a/agent/learning_mutations.py
+++ b/agent/learning_mutations.py
@@ -98,6 +98,7 @@ def _mutate_memory(node_id: str, replacement: str | None) -> dict[str, Any]:
def _apply(entries, limit):
from tools.memory_tool import ENTRY_DELIMITER
+ from tools.memory_tool_store import _error
_, chunks, local = _locate_memory(node_id)
text = chunks[local].strip()
@@ -106,11 +107,10 @@ def _mutate_memory(node_id: str, replacement: str | None) -> dict[str, Any]:
idx = entries.index(text)
new_entries = entries[:idx] + ([] if replacement is None else [replacement]) + entries[idx + 1:]
# Same cap the memory tool enforces on replace (never on remove: deleting is how a file
- # already over its total gets back under it). An over-limit entry reads as external drift
- # to every later mutation, so the tool's own remove/replace refuse until hand-fixed.
+ # already over its total gets back under it).
if replacement is not None and (total := len(ENTRY_DELIMITER.join(new_entries))) > limit:
- return {"success": False,
- "error": f"Replacement would put memory at {total:,}/{limit:,} chars. Shorten the new content."}
+ return _error(f"Replacement would put memory at {total:,}/{limit:,} chars. Shorten the new content.",
+ "over_budget")
return new_entries, message
result = load_on_disk_store()._mutate(_STORE_TARGETS[source], _apply)
diff --git a/tests/agent/test_learning_mutations.py b/tests/agent/test_learning_mutations.py
index f78b7beed48..964fda057d3 100644
--- a/tests/agent/test_learning_mutations.py
+++ b/tests/agent/test_learning_mutations.py
@@ -182,20 +182,18 @@ def test_edit_memory_keeps_concurrent_memory_tool_add(home, monkeypatch):
assert MemoryStore._read_file(home / "memories" / "MEMORY.md") == ["alpha rewritten", "beta note", "gamma note"]
[email protected]("mutate", [lambda: lm.delete_node("memory:memory:0"),
- lambda: lm.edit_node("memory:memory:0", "alpha rewritten")],
- ids=["delete", "edit"])
-def test_memory_drift_is_refused_with_backup_like_memory_tool(home, mutate):
+def test_memory_drift_is_refused_with_backup_like_memory_tool(home):
"""Free-form text appended without § delimiters (one "entry" past the whole-store
limit) is what the memory tool's drift guard exists for: snapshot to .bak, refuse,
- leave the file untouched. A Journey mutation must not flatten it silently."""
+ leave the file untouched. A Journey edit must not flatten it silently; a delete that
+ leaves that entry in place is a prune and goes through (#133028)."""
from tools.memory_tool_store import _drift_error
path = home / "memories" / "MEMORY.md"
raw = "alpha note\n§\nbeta note\n\n## Vendor Master\n" + "x" * 2300 + "\n"
path.write_text(raw, encoding="utf-8")
- res = mutate()
+ res = lm.edit_node("memory:memory:0", "alpha rewritten")
assert not res["ok"]
assert path.read_text(encoding="utf-8") == raw
@@ -203,6 +201,9 @@ def test_memory_drift_is_refused_with_backup_like_memory_tool(home, mutate):
assert backup.read_text(encoding="utf-8") == raw
assert res["message"] == _drift_error(path, str(backup))["error"]
+ assert lm.delete_node("memory:memory:0")["ok"]
+ assert path.read_text(encoding="utf-8") == raw.strip().split("\n§\n", 1)[1]
+
def test_unreadable_memory_file_is_refused_unchanged(home):
path = home / "memories" / "MEMORY.md"
diff --git a/tests/tools/test_memory_tool.py b/tests/tools/test_memory_tool.py
index a149af47e03..a1823dbd336 100644
--- a/tests/tools/test_memory_tool.py
+++ b/tests/tools/test_memory_tool.py
@@ -682,6 +682,37 @@ class TestExternalDriftGuard:
result = store.replace("memory", "Entry two", "Entry two replaced.")
assert result["success"] is True
+ def test_entries_stay_removable_after_the_limit_is_lowered(self, store, monkeypatch):
+ """An entry written under a larger limit is no external drift: pruning stays the way back
+ under a lowered limit."""
+ roomy = MemoryStore(memory_char_limit=2200, user_char_limit=1375)
+ work, home = ("Work:" + " firmware and vision pipelines;" * 16, # each > the fixture's 300
+ "Home:" + " garden, workshop and bikes;" * 16)
+ for entry in (work, home, "Prefers metric.", "Uses vim."):
+ assert roomy.add("user", entry)["success"]
+
+ pruned = json.loads(memory_tool(target="user", store=store, operations=[
+ {"action": "remove", "old_text": "Prefers metric"}, {"action": "remove", "old_text": "Uses vim"}]))
+ assert pruned["success"] is True
+ monkeypatch.setattr("tools.memory_tool_store.time.time", lambda: 1_700_000_000.0) # same second
+ first, second = store.remove("user", "Work:"), store.remove("user", "Home:")
+ assert first["success"] and second["success"] and first["removed_entry"] == work
+ assert MemoryStore._read_file(store._path_for("user")) == []
+ # Each delete of an oversized entry keeps its own snapshot, even within the same second.
+ assert first["drift_backup"] != second["drift_backup"]
+ assert work in Path(first["drift_backup"]).read_text(encoding="utf-8")
+
+ def test_removing_an_entry_with_glued_free_text_keeps_a_snapshot(self, store):
+ """Free text appended without a delimiter rides on the last entry; deleting that entry
+ must never take the text with it unless a snapshot holds it."""
+ store.add("memory", "User likes brevity.")
+ drifted = self._plant_drift(store).read_text(encoding="utf-8")
+
+ result = store.remove("memory", "User likes")
+
+ assert result["success"] is True
+ assert Path(result["drift_backup"]).read_text(encoding="utf-8") == drifted
+
def test_drift_guard_also_protects_user_target(self, store):
"""USER.md gets the same guarantee as MEMORY.md."""
store.add("user", "Some preference.")
diff --git a/tools/memory_tool_store.py b/tools/memory_tool_store.py
index 22616da69b0..03e84ff1b89 100644
--- a/tools/memory_tool_store.py
+++ b/tools/memory_tool_store.py
@@ -3,6 +3,7 @@ Entries are joined by ``ENTRY_DELIMITER``; budgets are in chars (model-independe
Module state that tests monkeypatch (``get_memory_dir``, ``fcntl``/``msvcrt``) stays
in ``tools.memory_tool`` and is read lazily."""
+import itertools
import logging
import os
import re
@@ -34,6 +35,7 @@ def _scan_memory_content(content: str) -> Optional[str]:
# Why the last refusal/failure in this context happened, for the shared metric only (a closed name from
# shared_metrics_contract.MEMORY_OP_FAILURE_CLASSES); never part of the result the model sees.
FAILURE_CLASS: ContextVar[str] = ContextVar("memory_failure_class", default="other")
+_BACKUP_FAILED = " (BACKUP FAILED — file unchanged on disk)"
def _error(message: str, failure_class: str, **extra) -> dict[str, Any]:
@@ -349,8 +351,8 @@ class MemoryStore:
"""Lock, re-read from disk, run ``mutate(entries, limit)`` -> ``(new_entries, message)``
or an error dict, then persist and return the success response. The reload aborts
on an existing-but-unreadable file (even append-only ``add`` rewrites the whole
- file) and, unless *skip_drift*, on external drift (flushing would discard
- un-roundtrippable content). Drift check and parse use the SAME raw snapshot —
+ file) and, unless *skip_drift*, refuses a write that adds or rewrites text while an
+ entry is over the whole-file limit (it may hold free text a flush would discard). Drift check and parse use the SAME raw snapshot —
a failed second read used to count as "no drift". The closure may return a
third value, a dict merged into the success payload (``_error``'s ``**extra``
convention) — e.g. the full text a replace overwrote (#117952). ANY dict the closure
@@ -361,19 +363,35 @@ class MemoryStore:
raw, read_ok = self._read_raw_checked(path)
if not read_ok:
return _read_failed_error(path)
- bak = None if skip_drift else self._detect_external_drift(target, raw)
- self._set_entries(target, list(dict.fromkeys(self._parse_entries(raw))))
- if bak:
- return _drift_error(path, bak)
+ oversized = not skip_drift and self._has_oversized_entry(target, raw)
+ before = list(dict.fromkeys(self._parse_entries(raw)))
+ self._set_entries(target, before)
+ FAILURE_CLASS.set("other") # read below: only this closure's refusal counts
result = mutate(self._entries_for(target), self._char_limit(target))
if isinstance(result, dict):
+ # Over budget beside an oversized entry: that entry is the obstacle, so report
+ # the drift (with its snapshot) rather than a budget the model cannot meet.
+ if oversized and result.get("success") is False and FAILURE_CLASS.get() == "over_budget":
+ return _drift_error(path, self._drift_backup(target, raw))
return result
+ extra_fields = dict(result[2]) if len(result) > 2 else {}
+ # An oversized entry is either free text appended without a delimiter or an entry written before the limit was lowered. Pruning is the way
+ # back under a lowered limit, so a write that only deletes entries proceeds; one that
+ # drops an oversized entry keeps the snapshot first, as it may hold appended text.
+ if oversized:
+ if not set(result[0]) <= set(before):
+ return _drift_error(path, self._drift_backup(target, raw))
+ limit = self._char_limit(target)
+ if any(len(e) > limit for e in before if e not in result[0]):
+ bak = self._drift_backup(target, raw)
+ if bak.endswith(_BACKUP_FAILED): # no copy, no delete of possibly glued text
+ return _drift_error(path, bak)
+ extra_fields["drift_backup"] = bak
self._set_entries(target, result[0])
from hermes_constants import mkdir_under_hermes_home
mkdir_under_hermes_home(path.parent)
self._write_file(path, result[0])
- extra_fields = result[2] if len(result) > 2 else {}
return self._success_response(target, result[1], **extra_fields)
def add(self, target: str, content: str) -> dict[str, Any]:
@@ -598,9 +616,10 @@ class MemoryStore:
f"entries into a shorter one instead of removing the last one. To delete the "
f"final entry deliberately, remove it alone in a one-item batch."), "would_empty")
new_total = len(ENTRY_DELIMITER.join(working)) # budget check against the FINAL state only
- # A one-item remove or duplicate add never grows the store, so like the legacy
- # remove/add it is not refused for excess left by a lowered cap.
- grows = not (len(ops) == 1 and (ops[0].get("action") == "remove" or working == entries))
+ # A remove-only batch or a lone duplicate add never grows the store, so like the
+ # legacy remove/add it is not refused for excess left by a lowered cap.
+ grows = not (all(op.get("action") == "remove" for op in ops)
+ or (len(ops) == 1 and working == entries))
if grows and new_total > limit:
if len(ops) == 1 and ops[0].get("action") == "add": # the advertised single add
return self._add_overflow(target, (ops[0].get("content") or ops[0].get("new_text") or "").strip(), limit)
@@ -694,19 +713,28 @@ class MemoryStore:
except OSError as e:
raise RuntimeError(f"Failed to write memory file {path}: {e}")
- def _detect_external_drift(self, target: str, raw: str) -> Optional[str]:
- """``.bak.<ts>`` snapshot path if *raw* shows external drift, else None. The one
- signal is an entry over the whole-file limit: no tool-written entry can be, so it is
- free-form text appended without delimiters, and replacing that "entry" would discard
- it. Layout differences (blank lines around ``§``, empty entries, padding) are NOT
- drift: parsing only strips whitespace, so the flush loses nothing but whitespace, and
- refusing there blocked every replace/remove on a hand-formatted file for good (#107270)."""
- if max(map(len, self._parse_entries(raw)), default=0) <= self._char_limit(target):
- return None
+ def _has_oversized_entry(self, target: str, raw: str) -> bool:
+ """True when one entry in *raw* is longer than the whole-file limit. A tool write
+ never produces one, so it is free-form text appended without delimiters, or an entry
+ written before the limit was lowered. Layout differences (blank lines around ``§``,
+ empty entries, padding) are NOT drift: parsing only strips whitespace, so the flush
+ loses nothing but whitespace, and refusing there blocked every replace/remove on a
+ hand-formatted file for good (#107270)."""
+ return max(map(len, self._parse_entries(raw)), default=0) > self._char_limit(target)
+
+ def _drift_backup(self, target: str, raw: str) -> str:
+ """``.bak.<ts>[.<n>]`` snapshot of *raw* before a drift refusal or a delete that drops an
+ oversized entry. Created exclusively: two snapshots in the same second never overwrite
+ each other, since a later delete's snapshot no longer holds what an earlier one removed."""
path = self._path_for(target)
- bak_path = path.with_suffix(path.suffix + f".bak.{int(time.time())}")
- try:
- bak_path.write_text(raw, encoding="utf-8")
- except OSError:
- return str(bak_path) + " (BACKUP FAILED — file unchanged on disk)"
- return str(bak_path)
+ stem = path.suffix + f".bak.{int(time.time())}"
+ for n in itertools.count():
+ bak_path = path.with_suffix(stem + (f".{n}" if n else ""))
+ try:
+ with open(bak_path, "x", encoding="utf-8") as fh:
+ fh.write(raw)
+ return str(bak_path)
+ except FileExistsError:
+ continue
+ except OSError:
+ return str(bak_path) + _BACKUP_FAILED