Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(toin): preserve stores that exceed the byte limit (#4589)

Endidf9cae66b6b19
diff --git a/headroom/telemetry/backends/filesystem.py b/headroom/telemetry/backends/filesystem.py
index 51841f4b1..e69bcc7d4 100644
--- a/headroom/telemetry/backends/filesystem.py
+++ b/headroom/telemetry/backends/filesystem.py
@@ -26,6 +26,8 @@ class FileSystemTOINBackend:
Args:
path: Path to the JSON storage file.
+ max_load_bytes: Maximum file size in bytes for loading and saving.
+ None disables the limit.
"""
def __init__(self, path: str, *, max_load_bytes: int | None = None) -> None:
@@ -48,7 +50,7 @@ class FileSystemTOINBackend:
):
logger.warning(
"TOIN data file %s exceeds the configured load limit (%d bytes); "
- "ignoring it until the store is rebuilt",
+ "leaving it unchanged",
self._path,
self._max_load_bytes,
)
@@ -70,9 +72,20 @@ class FileSystemTOINBackend:
data: Serialized TOIN data to persist.
Raises:
+ ValueError: The existing file or serialized data exceeds the size limit.
OSError: If persistence fails, so the caller can retain pending changes.
"""
try:
+ # A file skipped by load() must not be replaced with partial state.
+ if (
+ self._max_load_bytes is not None
+ and self._path.exists()
+ and self._path.stat().st_size > self._max_load_bytes
+ ):
+ raise ValueError(
+ f"Existing TOIN data file {self._path} exceeds the configured "
+ f"size limit ({self._max_load_bytes} bytes); refusing to overwrite it"
+ )
self._path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp_path = tempfile.mkstemp(dir=self._path.parent, prefix=".toin_", suffix=".tmp")
@@ -80,8 +93,17 @@ class FileSystemTOINBackend:
# Stream compact JSON directly to the temp file. Building a
# second multi-gigabyte string was the primary autosave RSS
# spike reported in #2886.
- with open(fd, "w") as f:
+ with open(fd, "w", newline="\n") as f:
json.dump(data, f, separators=(",", ":"))
+ # Check actual bytes after flushing, before replacing a valid store.
+ if (
+ self._max_load_bytes is not None
+ and Path(tmp_path).stat().st_size > self._max_load_bytes
+ ):
+ raise ValueError(
+ f"Serialized TOIN data exceeds the configured size limit "
+ f"({self._max_load_bytes} bytes)"
+ )
Path(tmp_path).replace(self._path)
except Exception:
try:
diff --git a/headroom/telemetry/toin.py b/headroom/telemetry/toin.py
index a62a53484..5273be0b8 100644
--- a/headroom/telemetry/toin.py
+++ b/headroom/telemetry/toin.py
@@ -489,7 +489,7 @@ class TOINConfig:
storage_path: str = field(default_factory=get_default_toin_storage_path)
auto_save_interval: int = 600 # Auto-save every 10 minutes
max_patterns: int = DEFAULT_MAX_PATTERNS
- max_storage_bytes: int = DEFAULT_MAX_STORAGE_BYTES
+ max_storage_bytes: int = DEFAULT_MAX_STORAGE_BYTES # Filesystem load and save limit
# Network learning thresholds
min_samples_for_recommendation: int = 10
diff --git a/tests/test_toin_retention.py b/tests/test_toin_retention.py
index f71a1fc88..2247552f1 100644
--- a/tests/test_toin_retention.py
+++ b/tests/test_toin_retention.py
@@ -358,6 +358,24 @@ def test_filesystem_backend_writes_compact_json(tmp_path) -> None:
assert " " not in path.read_text(encoding="utf-8")
+def test_filesystem_backend_writes_pin_lf(monkeypatch, tmp_path) -> None:
+ import headroom.telemetry.backends.filesystem as filesystem
+
+ calls: list[dict] = []
+ real_open = open
+
+ def recording_open(fd, *args, **kwargs):
+ calls.append(kwargs)
+ return real_open(fd, *args, **kwargs)
+
+ monkeypatch.setattr(filesystem, "open", recording_open, raising=False)
+ FileSystemTOINBackend(str(tmp_path / "toin.json")).save({"patterns": {}})
+
+ # Artifact bytes alone cannot detect a missing newline pin on POSIX.
+ assert [call.get("newline") for call in calls] == ["\n"]
+
+
def test_filesystem_backend_skips_oversized_store(tmp_path) -> None:
path = tmp_path / "toin.json"
path.write_text("x" * 100, encoding="utf-8")
@@ -365,3 +383,99 @@ def test_filesystem_backend_skips_oversized_store(tmp_path) -> None:
backend = FileSystemTOINBackend(str(path), max_load_bytes=10)
assert backend.load() == {}
assert path.exists()
+
+
+def test_filesystem_backend_preserves_oversized_store_on_save(tmp_path) -> None:
+ path = tmp_path / "toin.json"
+ original = b'{"patterns":{"existing":{"value":1}}}'
+ path.write_bytes(original)
+ backend = FileSystemTOINBackend(str(path), max_load_bytes=len(original) - 1)
+
+ assert backend.load() == {}
+ with pytest.raises(ValueError, match="limit"):
+ backend.save({})
+
+ assert path.read_bytes() == original
+ assert list(tmp_path.glob(".toin_*.tmp")) == []
+
+
[email protected]("existing", [False, True], ids=["new_store", "existing_store"])
+def test_filesystem_backend_rejects_oversized_output(tmp_path, existing) -> None:
+ path = tmp_path / "toin.json"
+ original = b"{}"
+ if existing:
+ path.write_bytes(original)
+ data = {"value": "x" * 100}
+ limit = len(json.dumps(data, separators=(",", ":")).encode("utf-8")) - 1
+ backend = FileSystemTOINBackend(str(path), max_load_bytes=limit)
+
+ with pytest.raises(ValueError, match="limit"):
+ backend.save(data)
+
+ if existing:
+ assert path.read_bytes() == original
+ else:
+ assert not path.exists()
+ assert list(tmp_path.glob(".toin_*.tmp")) == []
+
+
+def test_filesystem_backend_accepts_exact_byte_limit(tmp_path) -> None:
+ path = tmp_path / "toin.json"
+ data = {"value": "雪"}
+ serialized = json.dumps(data, separators=(",", ":")).encode("utf-8")
+ backend = FileSystemTOINBackend(str(path), max_load_bytes=len(serialized))
+
+ backend.save(data)
+ assert path.read_bytes() == serialized
+ assert backend.load() == data
+ # An existing file at the boundary can still be replaced.
+ backend.save({})
+ assert backend.load() == {}
+
+
+def test_filesystem_backend_zero_byte_limit_rejects_output(tmp_path) -> None:
+ path = tmp_path / "toin.json"
+ backend = FileSystemTOINBackend(str(path), max_load_bytes=0)
+
+ with pytest.raises(ValueError, match="limit"):
+ backend.save({})
+
+ assert not path.exists()
+ assert list(tmp_path.glob(".toin_*.tmp")) == []
+
+
+def test_filesystem_backend_without_byte_limit_allows_growth(tmp_path) -> None:
+ path = tmp_path / "toin.json"
+ backend = FileSystemTOINBackend(str(path), max_load_bytes=None)
+ backend.save({"value": "x" * 100})
+ data = {"value": "x" * 200}
+
+ backend.save(data)
+
+ assert backend.load() == data
+
+
[email protected]("oversized_on_load", [True, False], ids=["load_rejected", "store_grows"])
+def test_toin_size_rejection_preserves_store_and_dirty_state(
+ tmp_path, caplog, oversized_on_load
+) -> None:
+ path = tmp_path / "toin.json"
+ _saved_store(path, 3 if oversized_on_load else 1)
+ original = path.read_bytes()
+ limit = len(original) - 1 if oversized_on_load else len(original) + 128
+ toin = ToolIntelligenceNetwork(
+ TOINConfig(storage_path=str(path), max_storage_bytes=limit, auto_save_interval=0)
+ )
+ assert len(toin._patterns) == (0 if oversized_on_load else 1)
+ _record(toin, 99)
+ last_save_time = toin._last_save_time
+
+ with caplog.at_level(logging.WARNING, logger="headroom.telemetry.toin"):
+ toin.save()
+
+ assert path.read_bytes() == original
+ assert toin._dirty
+ assert toin._last_save_time == last_save_time
+ assert list(tmp_path.glob(".toin_*.tmp")) == []
+ [warning] = [r for r in caplog.records if getattr(r, "event", None) == "toin_save_failed"]
+ assert warning.error_type == "ValueError"