Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(output-savings): rewrite a pre-rule ledger once instead of warning on every /stats poll (#4196)

M Barry96e81ac6703d
diff --git a/headroom/cli/learn.py b/headroom/cli/learn.py
index e917b5b84..e38995dc3 100644
--- a/headroom/cli/learn.py
+++ b/headroom/cli/learn.py
@@ -755,9 +755,7 @@ def _run_verbosity(
# Seed the savings baseline: replace baseline, preserve any live
# treatment/control already accumulated.
ledger_path = ws / "output_savings.json"
- ledger = SavingsLedger.load(ledger_path)
- ledger.baseline = aggregated
- ledger.save(ledger_path)
+ SavingsLedger.store_baseline(ledger_path, aggregated)
click.echo(f"\n [WROTE] {ws / 'verbosity.json'} (level {best_profile.level})")
click.echo(
f" [WROTE] {ledger_path} (baseline: {aggregated.total_samples} samples, "
diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.py
index 333c03d48..129bbcbf3 100644
--- a/headroom/proxy/output_savings.py
+++ b/headroom/proxy/output_savings.py
@@ -41,6 +41,8 @@ from __future__ import annotations
import json
import logging
import math
+from collections.abc import Iterator
+from contextlib import contextmanager
from dataclasses import asdict, dataclass, field
from typing import Any
@@ -392,6 +394,52 @@ def _measured_supersedes(measured: SavingsEstimate, labelled_requests: int) -> b
return _measured_covers(measured, labelled_requests)
+@contextmanager
+def ledger_lock(path: Any) -> Iterator[None]:
+ """Serialize read-modify-write of the ledger file across processes.
+
+ The lock lives in a sidecar file because ``save`` replaces the ledger's
+ inode. Same platform split as ``cli.wrap._locked_file``.
+ """
+ import sys
+ from pathlib import Path
+
+ lock_path = Path(str(path) + ".lock")
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
+ with open(lock_path, "a+b") as handle:
+ if sys.platform == "win32":
+ import msvcrt
+ import os
+ import time
+
+ # msvcrt.locking locks bytes from the current position, so lock a
+ # real byte at offset 0. LK_LOCK raises after ~10 one-second
+ # retries, so loop on the non-blocking form instead.
+ if os.fstat(handle.fileno()).st_size == 0:
+ handle.write(b"0")
+ handle.flush()
+ handle.seek(0)
+ while True:
+ try:
+ msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
+ break
+ except OSError:
+ time.sleep(0.05)
+ try:
+ yield
+ finally:
+ handle.seek(0)
+ msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl
+
+ fcntl.flock(handle, fcntl.LOCK_EX)
+ try:
+ yield
+ finally:
+ fcntl.flock(handle, fcntl.LOCK_UN)
+
+
@dataclass
class SavingsLedger:
"""Accumulates shaped (treatment) and unshaped (control) observations and
@@ -406,6 +454,9 @@ class SavingsLedger:
baseline: BaselineModel = field(default_factory=BaselineModel)
treatment: dict[str, _Accum] = field(default_factory=dict)
control: dict[str, _Accum] = field(default_factory=dict)
+ # Set by ``from_dict`` when it dropped pre-rule arms, so the owner can
+ # persist the drop instead of re-dropping (and re-warning) on every load.
+ dropped_legacy_arms: bool = field(default=False, compare=False, repr=False)
# ---- recording -------------------------------------------------------
@@ -616,7 +667,7 @@ class SavingsLedger:
}
@classmethod
- def from_dict(cls, d: dict[str, Any]) -> SavingsLedger:
+ def from_dict(cls, d: dict[str, Any], *, warn_legacy: bool = True) -> SavingsLedger:
ledger = cls(baseline=BaselineModel.from_dict(d.get("baseline") or {}))
if not d.get("shaped_only"):
# Pre-rule arms cannot be told apart from shaped ones entry by
@@ -625,6 +676,8 @@ class SavingsLedger:
# baseline is kept: it is learned from pre-shaper history, costs a
# `learn --verbosity` run to rebuild, and was never the poisoned part.
if d.get("treatment") or d.get("control"):
+ ledger.dropped_legacy_arms = True
+ if ledger.dropped_legacy_arms and warn_legacy:
logger.warning(
"output-savings ledger predates shaped-only recording; "
"dropping %d treatment and %d control strata and "
@@ -651,14 +704,14 @@ class SavingsLedger:
fsutil.write_text(p, json.dumps(self.to_dict(), separators=(",", ":")))
@classmethod
- def load(cls, path: Any) -> SavingsLedger:
+ def load(cls, path: Any, *, warn_legacy: bool = True) -> SavingsLedger:
from pathlib import Path
p = Path(path)
if not p.exists():
return cls()
try:
- return cls.from_dict(json.loads(p.read_text()))
+ return cls.from_dict(json.loads(p.read_text()), warn_legacy=warn_legacy)
except (json.JSONDecodeError, ValueError, OSError) as exc:
# Fail open (empty ledger), but surface the loss — silently
# swallowing a corrupt file made lost history indistinguishable
@@ -666,6 +719,14 @@ class SavingsLedger:
logger.warning("output-savings ledger %s unreadable, starting empty: %s", p, exc)
return cls()
+ @classmethod
+ def store_baseline(cls, path: Any, baseline: BaselineModel) -> None:
+ """Replace the on-disk baseline, keeping the live arms already there."""
+ with ledger_lock(path):
+ ledger = cls.load(path)
+ ledger.baseline = baseline
+ ledger.save(path)
+
# --------------------------------------------------------------------------
# Live recording — rides the existing ``transforms_applied`` label channel so
@@ -692,6 +753,11 @@ class SavingsRecorder:
self._ledger = SavingsLedger.load(self._path)
self._flush_every = flush_every
self._since_flush = 0
+ if self._ledger.dropped_legacy_arms:
+ # Persist the drop now. With the shaper off nothing ever records,
+ # so waiting for a periodic flush would leave the legacy file on
+ # disk indefinitely.
+ self._migrate_legacy()
def record_from_labels(self, labels: Any, output_tokens: int) -> bool:
"""Record one outcome given its transforms_applied labels. Returns True
@@ -779,7 +845,10 @@ class SavingsRecorder:
same number of samples still takes effect, and the empty-disk guard keeps
a truncated file from wiping a baseline we already hold."""
try:
- disk = SavingsLedger.load(self._path)
+ # Only the baseline is read here, and the constructor already
+ # reported any legacy arms. Warning again would repeat on every
+ # /stats poll whenever the file can't be rewritten (stateless).
+ disk = SavingsLedger.load(self._path, warn_legacy=False)
except OSError:
return
if disk.baseline.total_samples == 0:
@@ -787,6 +856,21 @@ class SavingsRecorder:
if disk.baseline.to_dict() != self._ledger.baseline.to_dict():
self._ledger.baseline = disk.baseline
+ def _migrate_legacy(self) -> None:
+ from ..paths import process_is_stateless
+
+ if process_is_stateless():
+ return
+ try:
+ with ledger_lock(self._path):
+ # Re-read under the lock: another writer may have replaced the
+ # file since the first load, and its copy is the newer one.
+ self._ledger = SavingsLedger.load(self._path, warn_legacy=False)
+ if self._ledger.dropped_legacy_arms:
+ self._ledger.save(self._path)
+ except OSError:
+ pass
+
def _flush_locked(self) -> None:
from ..paths import process_is_stateless
@@ -795,8 +879,9 @@ class SavingsRecorder:
self._since_flush = 0
return
try:
- self._reload_baseline_locked()
- self._ledger.save(self._path)
+ with ledger_lock(self._path):
+ self._reload_baseline_locked()
+ self._ledger.save(self._path)
self._since_flush = 0
except OSError:
pass
diff --git a/tests/test_output_savings.py b/tests/test_output_savings.py
index e8ad6d749..543e2774e 100644
--- a/tests/test_output_savings.py
+++ b/tests/test_output_savings.py
@@ -17,6 +17,7 @@ from headroom.proxy.output_savings import (
conversation_label,
echo_ratio,
input_bucket,
+ ledger_lock,
model_family,
stratum_key,
stratum_label,
@@ -685,6 +686,181 @@ class TestLedgerPersistence:
ledger = SavingsLedger.load(p)
assert ledger.baseline.total_samples == 0
+ @staticmethod
+ def _pre_rule_ledger(path):
+ """A ledger written before shaped-only recording: arms, no marker."""
+ baseline = BaselineModel()
+ baseline.observe("opus|a|s|tools", 1000)
+ legacy = {
+ "baseline": baseline.to_dict(),
+ "treatment": {"opus|a|s|tools": {"n": 3, "sum": 2400.0, "sumsq": 1920000.0}},
+ "control": {},
+ }
+ path.write_text(json.dumps(legacy))
+ return legacy
+
+ @staticmethod
+ def _legacy_warnings(caplog):
+ return [r for r in caplog.records if "predates shaped-only" in r.getMessage()]
+
+ def test_pre_rule_ledger_warns_once_and_is_rewritten(self, tmp_path, caplog):
+ """/stats reloads the ledger on every poll. With the shaper disabled
+ nothing ever flushes, so the legacy file used to survive and warn on
+ every poll forever."""
+ import logging
+
+ path = tmp_path / "output_savings.json"
+ legacy = self._pre_rule_ledger(path)
+
+ with caplog.at_level(logging.WARNING):
+ recorder = SavingsRecorder(str(path))
+ for _ in range(3):
+ recorder.estimate()
+
+ assert len(self._legacy_warnings(caplog)) == 1
+ on_disk = json.loads(path.read_text())
+ assert on_disk["shaped_only"] is True
+ assert on_disk["treatment"] == {}
+ assert on_disk["baseline"] == legacy["baseline"]
+
+ def test_stateless_pre_rule_ledger_warns_once_and_is_left_alone(self, tmp_path, caplog):
+ import logging
+
+ from headroom import paths
+
+ path = tmp_path / "output_savings.json"
+ legacy = self._pre_rule_ledger(path)
+
+ paths.set_process_stateless(True)
+ try:
+ with caplog.at_level(logging.WARNING):
+ recorder = SavingsRecorder(str(path))
+ for _ in range(3):
+ recorder.estimate()
+ finally:
+ paths.set_process_stateless(False)
+
+ assert len(self._legacy_warnings(caplog)) == 1
+ assert json.loads(path.read_text()) == legacy
+ assert not (tmp_path / "output_savings.json.lock").exists()
+
+ def test_migration_keeps_a_baseline_learned_after_the_first_load(self, tmp_path, monkeypatch):
+ """learn replaced the file between the recorder's first load and its
+ migration write; the stale legacy baseline must not overwrite it."""
+ path = tmp_path / "output_savings.json"
+ self._pre_rule_ledger(path)
+ learned = BaselineModel()
+ for output_tokens in (400, 420, 380):
+ learned.observe("opus|b|s|tools", output_tokens)
+
+ real_load = SavingsLedger.load.__func__
+ loads = []
+
+ def load_then_learn(cls, p, **kwargs):
+ ledger = real_load(cls, p, **kwargs)
+ loads.append(p)
+ if len(loads) == 1:
+ SavingsLedger.store_baseline(path, learned)
+ return ledger
+
+ monkeypatch.setattr(SavingsLedger, "load", classmethod(load_then_learn))
+ recorder = SavingsRecorder(str(path))
+
+ on_disk = json.loads(path.read_text())
+ assert on_disk["baseline"] == learned.to_dict()
+ assert recorder._ledger.baseline.to_dict() == learned.to_dict()
+
+ def test_migration_holds_the_ledger_lock_while_writing(self, tmp_path, monkeypatch):
+ import threading
+
+ path = tmp_path / "output_savings.json"
+ self._pre_rule_ledger(path)
+ real_save = SavingsLedger.save
+ contended = []
+
+ def probing_save(self, p):
+ acquired = threading.Event()
+
+ def other_writer():
+ with ledger_lock(p):
+ acquired.set()
+
+ probe = threading.Thread(target=other_writer)
+ probe.start()
+ contended.append(not acquired.wait(timeout=0.3))
+ real_save(self, p)
+ probe_threads.append(probe)
+
+ probe_threads: list[threading.Thread] = []
+ monkeypatch.setattr(SavingsLedger, "save", probing_save)
+ SavingsRecorder(str(path))
+ for probe in probe_threads:
+ probe.join(timeout=5)
+
+ assert contended == [True]
+ assert json.loads(path.read_text())["shaped_only"] is True
+
+ def test_migration_keeps_a_baseline_learned_during_the_locked_reload(
+ self, tmp_path, monkeypatch
+ ):
+ """learn writes while the migration holds its re-read snapshot; it
+ must wait for the lock rather than be overwritten by that snapshot."""
+ import threading
+
+ path = tmp_path / "output_savings.json"
+ legacy = self._pre_rule_ledger(path)
+ learned = BaselineModel()
+ for output_tokens in (400, 420, 380):
+ learned.observe("opus|b|s|tools", output_tokens)
+
+ real_load = SavingsLedger.load.__func__
+ loads = []
+ writers: list[threading.Thread] = []
+
+ def learn_during_second_load(cls, p, **kwargs):
+ ledger = real_load(cls, p, **kwargs)
+ loads.append(p)
+ if len(loads) == 2:
+ writer = threading.Thread(target=SavingsLedger.store_baseline, args=(path, learned))
+ writer.start()
+ writer.join(timeout=0.3)
+ writers.append(writer)
+ return ledger
+
+ monkeypatch.setattr(SavingsLedger, "load", classmethod(learn_during_second_load))
+ recorder = SavingsRecorder(str(path))
+ assert len(writers) == 1
+ writers[0].join(timeout=5)
+ assert not writers[0].is_alive()
+
+ on_disk = json.loads(path.read_text())
+ assert on_disk["shaped_only"] is True
+ assert on_disk["baseline"] == learned.to_dict()
+ assert on_disk["baseline"] != legacy["baseline"]
+ recorder.estimate()
+ assert recorder._ledger.baseline.to_dict() == learned.to_dict()
+
+ def test_store_baseline_waits_for_the_ledger_lock(self, tmp_path):
+ """learn must not replace the file while another writer holds the lock
+ between its re-read and its save."""
+ import threading
+
+ path = tmp_path / "output_savings.json"
+ legacy = self._pre_rule_ledger(path)
+ learned = BaselineModel()
+ learned.observe("opus|b|s|tools", 400)
+
+ writer = threading.Thread(target=SavingsLedger.store_baseline, args=(path, learned))
+ with ledger_lock(path):
+ writer.start()
+ writer.join(timeout=0.3)
+ assert writer.is_alive()
+ assert json.loads(path.read_text()) == legacy
+ writer.join(timeout=5)
+
+ assert not writer.is_alive()
+ assert json.loads(path.read_text())["baseline"] == learned.to_dict()
+
# ---------------------------------------------------------------------------
# echo ratio (direct waste signal)
@@ -769,6 +945,21 @@ class TestRecorderBaselineReload:
# in-memory one over it.
assert SavingsLedger.load(path).baseline.total_samples == 4
+ def test_store_baseline_keeps_live_arms(self, tmp_path):
+ path = str(tmp_path / "output_savings.json")
+ key = self._key()
+
+ recorder = SavingsRecorder(path, flush_every=1)
+ recorder.record_from_labels([stratum_label("treatment", key), SHAPED], 200)
+
+ learned = BaselineModel()
+ learned.observe(key, 400)
+ SavingsLedger.store_baseline(path, learned)
+
+ on_disk = SavingsLedger.load(path)
+ assert on_disk.baseline.to_dict() == learned.to_dict()
+ assert on_disk.treatment[key].n == 1
+
def test_does_not_downgrade_to_empty_disk_baseline(self, tmp_path):
path = str(tmp_path / "output_savings.json")
key = self._key()