Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): make output-savings flush atomic and keep it off the event loop (#3231)
inixb9d7dcc3da3e
diff --git a/headroom/proxy/outcome.py b/headroom/proxy/outcome.pyindex c2b78b1dc..ba94feee7 100644--- a/headroom/proxy/outcome.py+++ b/headroom/proxy/outcome.py@@ -25,6 +25,7 @@ actually reports.from __future__ import annotations+import asyncioimport loggingfrom dataclasses import dataclass, fieldfrom datetime import datetime, timezone@@ -455,10 +456,17 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:from headroom.proxy.output_savings import get_recorder_rec = get_recorder()- _rec.record_from_labels(outcome.transforms_applied, outcome.output_tokens)- output_tokens_saved_est = _rec.estimate_request_savings(- outcome.transforms_applied, outcome.output_tokens- )++ def _record_and_estimate() -> int:+ _rec.record_from_labels(outcome.transforms_applied, outcome.output_tokens)+ return _rec.estimate_request_savings(+ outcome.transforms_applied, outcome.output_tokens+ )++ # Both calls take the recorder lock, and the every-Nth record also+ # does a full read-modify-write of the ledger file — run them+ # together off the event loop (#18) so a slow flush can't stall it.+ output_tokens_saved_est = await asyncio.to_thread(_record_and_estimate)except Exception: # pragma: no cover - defensivepassdiff --git a/headroom/proxy/output_savings.py b/headroom/proxy/output_savings.pyindex b0ee6baf3..5fe317cb0 100644--- a/headroom/proxy/output_savings.py+++ b/headroom/proxy/output_savings.py@@ -39,6 +39,7 @@ Pure module: no I/O except explicit ``load``/``save``.from __future__ import annotationsimport json+import loggingimport mathfrom dataclasses import asdict, dataclass, fieldfrom typing import Any@@ -68,6 +69,8 @@ from .output_savings_policy import (stratum_label as stratum_label,)+logger = logging.getLogger(__name__)+@dataclassclass _Accum:@@ -328,9 +331,13 @@ class SavingsLedger:def save(self, path: Any) -> None:from pathlib import Path+ from headroom import fsutil+p = Path(path)p.parent.mkdir(parents=True, exist_ok=True)- p.write_text(json.dumps(self.to_dict(), separators=(",", ":")))+ # fsutil.write_text is atomic (temp file + os.replace), so a crash+ # mid-write cannot truncate the ledger already on disk (#18).+ fsutil.write_text(p, json.dumps(self.to_dict(), separators=(",", ":")))@classmethoddef load(cls, path: Any) -> SavingsLedger:@@ -341,7 +348,11 @@ class SavingsLedger:return cls()try:return cls.from_dict(json.loads(p.read_text()))- except (json.JSONDecodeError, ValueError, OSError):+ except (json.JSONDecodeError, ValueError, OSError) as exc:+ # Fail open (empty ledger), but surface the loss — silently+ # swallowing a corrupt file made lost history indistinguishable+ # from no history yet (#18).+ logger.warning("output-savings ledger %s unreadable, starting empty: %s", p, exc)return cls()diff --git a/tests/test_output_savings.py b/tests/test_output_savings.pyindex e9d67a826..f62bc3e37 100644--- a/tests/test_output_savings.py+++ b/tests/test_output_savings.py@@ -389,12 +389,7 @@ class TestRecorderBaselineReload:@staticmethoddef _key() -> str:- return stratum_key(- turn_kind="code",- input_tokens=8000,- model="claude-opus-4-8",- has_tools=True,- )+ return SAMPLE_KEYdef test_adopts_baseline_learned_after_start(self, tmp_path):path = str(tmp_path / "output_savings.json")@@ -482,3 +477,99 @@ class TestRecorderBaselineReload:relearned.save(path)assert recorder.estimate().baseline_tokens > baseline_tokens_v1+++# ---------------------------------------------------------------------------+# flush durability + event-loop safety+# ---------------------------------------------------------------------------++# Deterministic stratum key shared by the recorder tests below.+SAMPLE_KEY = stratum_key(+ turn_kind="code",+ input_tokens=8000,+ model="claude-opus-4-8",+ has_tools=True,+)+++class TestFlushDurability:+ def test_crash_mid_write_leaves_previous_ledger_intact(self, tmp_path, monkeypatch):+ import headroom.fsutil++ path = str(tmp_path / "output_savings.json")+ key = SAMPLE_KEY++ recorder = SavingsRecorder(path, flush_every=1)+ recorder.record_from_labels([stratum_label("treatment", key)], 200)+ recorder.flush()+ assert SavingsLedger.load(path).treatment[key].n == 1++ def _die_before_rename(*args, **kwargs):+ raise OSError(5, "simulated crash before rename")++ monkeypatch.setattr(headroom.fsutil.os, "replace", _die_before_rename)+ recorder.record_from_labels([stratum_label("treatment", key)], 210)+ recorder.flush() # OSError swallowed by the recorder — fail-open by design++ # The pre-crash sample must survive and no temp residue may be left+ # behind: a failed save may not corrupt or clutter the ledger.+ assert SavingsLedger.load(path).treatment[key].n == 1+ assert not list(tmp_path.glob("*.tmp"))++ def test_corrupt_ledger_warns_and_starts_empty(self, tmp_path, caplog):+ import logging++ path = tmp_path / "output_savings.json"+ path.write_text("{not json")++ with caplog.at_level(logging.WARNING):+ SavingsRecorder(str(path))++ assert caplog.records, "corrupt ledger was swallowed silently"++ def test_emit_request_outcome_flushes_off_the_loop_thread(self, tmp_path, monkeypatch):+ import asyncio+ import threading++ from headroom.proxy.outcome import RequestOutcome, emit_request_outcome++ path = str(tmp_path / "output_savings.json")+ recorder = SavingsRecorder(path, flush_every=1)+ monkeypatch.setattr("headroom.proxy.output_savings.get_recorder", lambda: recorder)++ saved_on_threads = []+ real_save = SavingsLedger.save++ def _spy_save(self, save_path):+ saved_on_threads.append(threading.get_ident())+ real_save(self, save_path)++ monkeypatch.setattr(SavingsLedger, "save", _spy_save)++ class _Metrics:+ async def record_request(self, **kwargs):+ pass++ class _Handler:+ def __init__(self):+ self.metrics = _Metrics()+ self.cost_tracker = None+ self.logger = None++ outcome = RequestOutcome(+ request_id="req-shaper",+ provider="openai",+ model="gpt-5",+ status_code=200,+ original_tokens=100,+ optimized_tokens=80,+ output_tokens=50,+ tokens_saved=20,+ attempted_input_tokens=100,+ transforms_applied=(stratum_label("treatment", SAMPLE_KEY),),+ )+ asyncio.run(emit_request_outcome(_Handler(), outcome))++ loop_thread = threading.get_ident()+ assert saved_on_threads, "flush never ran"+ assert all(t != loop_thread for t in saved_on_threads)