Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): make output-savings flush atomic and keep it off the event loop (#3231)

inixb9d7dcc3da3e
diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.py
index c2b78b1dc..ba94feee7 100644
--- a/headroom/proxy/outcome.py
+++ b/headroom/proxy/outcome.py
@@ -25,6 +25,7 @@ actually reports.
from __future__ import annotations
+import asyncio
import logging
from dataclasses import dataclass, field
from datetime import datetime, timezone
@@ -455,10 +456,17 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
from headroom.proxy.output_savings import get_recorder
_rec = get_recorder()
- _rec.record_from_labels(outcome.transforms_applied, outcome.output_tokens)
- output_tokens_saved_est = _rec.estimate_request_savings(
- outcome.transforms_applied, outcome.output_tokens
- )
+
+ def _record_and_estimate() -> int:
+ _rec.record_from_labels(outcome.transforms_applied, outcome.output_tokens)
+ return _rec.estimate_request_savings(
+ outcome.transforms_applied, outcome.output_tokens
+ )
+
+ # Both calls take the recorder lock, and the every-Nth record also
+ # does a full read-modify-write of the ledger file — run them
+ # together off the event loop (#18) so a slow flush can't stall it.
+ output_tokens_saved_est = await asyncio.to_thread(_record_and_estimate)
except Exception: # pragma: no cover - defensive
pass
diff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.py
index b0ee6baf3..5fe317cb0 100644
--- a/headroom/proxy/output_savings.py
+++ b/headroom/proxy/output_savings.py
@@ -39,6 +39,7 @@ Pure module: no I/O except explicit ``load``/``save``.
from __future__ import annotations
import json
+import logging
import math
from dataclasses import asdict, dataclass, field
from typing import Any
@@ -68,6 +69,8 @@ from .output_savings_policy import (
stratum_label as stratum_label,
)
+logger = logging.getLogger(__name__)
+
@dataclass
class _Accum:
@@ -328,9 +331,13 @@ class SavingsLedger:
def save(self, path: Any) -> None:
from pathlib import Path
+ from headroom import fsutil
+
p = Path(path)
p.parent.mkdir(parents=True, exist_ok=True)
- p.write_text(json.dumps(self.to_dict(), separators=(",", ":")))
+ # fsutil.write_text is atomic (temp file + os.replace), so a crash
+ # mid-write cannot truncate the ledger already on disk (#18).
+ fsutil.write_text(p, json.dumps(self.to_dict(), separators=(",", ":")))
@classmethod
def load(cls, path: Any) -> SavingsLedger:
@@ -341,7 +348,11 @@ class SavingsLedger:
return cls()
try:
return cls.from_dict(json.loads(p.read_text()))
- except (json.JSONDecodeError, ValueError, OSError):
+ except (json.JSONDecodeError, ValueError, OSError) as exc:
+ # Fail open (empty ledger), but surface the loss — silently
+ # swallowing a corrupt file made lost history indistinguishable
+ # from no history yet (#18).
+ logger.warning("output-savings ledger %s unreadable, starting empty: %s", p, exc)
return cls()
diff --git a/tests/test_output_savings.py b/tests/test_output_savings.py
index e9d67a826..f62bc3e37 100644
--- a/tests/test_output_savings.py
+++ b/tests/test_output_savings.py
@@ -389,12 +389,7 @@ class TestRecorderBaselineReload:
@staticmethod
def _key() -> str:
- return stratum_key(
- turn_kind="code",
- input_tokens=8000,
- model="claude-opus-4-8",
- has_tools=True,
- )
+ return SAMPLE_KEY
def test_adopts_baseline_learned_after_start(self, tmp_path):
path = str(tmp_path / "output_savings.json")
@@ -482,3 +477,99 @@ class TestRecorderBaselineReload:
relearned.save(path)
assert recorder.estimate().baseline_tokens > baseline_tokens_v1
+
+
+# ---------------------------------------------------------------------------
+# flush durability + event-loop safety
+# ---------------------------------------------------------------------------
+
+# Deterministic stratum key shared by the recorder tests below.
+SAMPLE_KEY = stratum_key(
+ turn_kind="code",
+ input_tokens=8000,
+ model="claude-opus-4-8",
+ has_tools=True,
+)
+
+
+class TestFlushDurability:
+ def test_crash_mid_write_leaves_previous_ledger_intact(self, tmp_path, monkeypatch):
+ import headroom.fsutil
+
+ path = str(tmp_path / "output_savings.json")
+ key = SAMPLE_KEY
+
+ recorder = SavingsRecorder(path, flush_every=1)
+ recorder.record_from_labels([stratum_label("treatment", key)], 200)
+ recorder.flush()
+ assert SavingsLedger.load(path).treatment[key].n == 1
+
+ def _die_before_rename(*args, **kwargs):
+ raise OSError(5, "simulated crash before rename")
+
+ monkeypatch.setattr(headroom.fsutil.os, "replace", _die_before_rename)
+ recorder.record_from_labels([stratum_label("treatment", key)], 210)
+ recorder.flush() # OSError swallowed by the recorder — fail-open by design
+
+ # The pre-crash sample must survive and no temp residue may be left
+ # behind: a failed save may not corrupt or clutter the ledger.
+ assert SavingsLedger.load(path).treatment[key].n == 1
+ assert not list(tmp_path.glob("*.tmp"))
+
+ def test_corrupt_ledger_warns_and_starts_empty(self, tmp_path, caplog):
+ import logging
+
+ path = tmp_path / "output_savings.json"
+ path.write_text("{not json")
+
+ with caplog.at_level(logging.WARNING):
+ SavingsRecorder(str(path))
+
+ assert caplog.records, "corrupt ledger was swallowed silently"
+
+ def test_emit_request_outcome_flushes_off_the_loop_thread(self, tmp_path, monkeypatch):
+ import asyncio
+ import threading
+
+ from headroom.proxy.outcome import RequestOutcome, emit_request_outcome
+
+ path = str(tmp_path / "output_savings.json")
+ recorder = SavingsRecorder(path, flush_every=1)
+ monkeypatch.setattr("headroom.proxy.output_savings.get_recorder", lambda: recorder)
+
+ saved_on_threads = []
+ real_save = SavingsLedger.save
+
+ def _spy_save(self, save_path):
+ saved_on_threads.append(threading.get_ident())
+ real_save(self, save_path)
+
+ monkeypatch.setattr(SavingsLedger, "save", _spy_save)
+
+ class _Metrics:
+ async def record_request(self, **kwargs):
+ pass
+
+ class _Handler:
+ def __init__(self):
+ self.metrics = _Metrics()
+ self.cost_tracker = None
+ self.logger = None
+
+ outcome = RequestOutcome(
+ request_id="req-shaper",
+ provider="openai",
+ model="gpt-5",
+ status_code=200,
+ original_tokens=100,
+ optimized_tokens=80,
+ output_tokens=50,
+ tokens_saved=20,
+ attempted_input_tokens=100,
+ transforms_applied=(stratum_label("treatment", SAMPLE_KEY),),
+ )
+ asyncio.run(emit_request_outcome(_Handler(), outcome))
+
+ loop_thread = threading.get_ident()
+ assert saved_on_threads, "flush never ran"
+ assert all(t != loop_thread for t in saved_on_threads)