Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy/savings): append history point on cache-only savings too (#2194)

Ingmar Kruschd1258055893a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 85671853c..cb5a7f9d7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
### Fixed
+- **proxy/savings:** `SavingsTracker.record_request()` only appended a history point when `tokens_saved > 0` (headroom's own lossy compression). In `--mode cache`, `tokens_saved` is near-always 0 by design, since the frozen prefix is byte-replayed rather than compressed to keep the provider's prompt cache warm. That silently dropped every history point on a cache-mode deployment even when `cache_read_tokens`/`cache_savings_usd` were large, making `headroom-monthly`-style tooling read as a total savings collapse. The guard now fires on `tokens_saved` OR `cache_read_tokens`, and the appended entry carries `cache_read_tokens`/`cache_savings_usd` so downstream consumers can show them; `_normalize_history_entry` defaults both fields to 0/0.0 for legacy entries that predate this change.
- **litellm:** vendor-specific top-level fields on `/v1/chat/completions`, including vLLM's `chat_template_kwargs` for per-request Qwen3 thinking-mode toggles, now reach OpenAI-compatible backends through LiteLLM `extra_body` instead of being dropped by the standard-parameter allowlist ([#2128](https://github.com/headroomlabs-ai/headroom/issues/2128)).
- **cache aligner:** hash the actual frozen Claude Code prefix instead of only system-message text, so `stable_prefix_hash` / `prefix_changed` now surface prompt-cache churn when a cached tool-result block changes without any system-prompt edit ([#2085](https://github.com/headroomlabs-ai/headroom/issues/2085)).
- **proxy image compression:** run native image compression in a spawned subprocess so an OpenCV or KleidiCV crash now fails open to the original image payload instead of taking down the whole proxy process ([#2107](https://github.com/headroomlabs-ai/headroom/issues/2107)).
diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.py
index 927937945..725aafeb8 100644
--- a/headroom/proxy/savings_tracker.py
+++ b/headroom/proxy/savings_tracker.py
@@ -324,6 +324,8 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
timestamp: datetime | None = None
total_tokens_saved = 0
compression_savings_usd = 0.0
+ cache_read_tokens = 0
+ cache_savings_usd = 0.0
total_input_tokens = 0
total_input_cost_usd = 0.0
provider = PROVIDER_UNKNOWN
@@ -333,6 +335,11 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
timestamp = _parse_timestamp(entry.get("timestamp"))
total_tokens_saved = _coerce_int(entry.get("total_tokens_saved"))
compression_savings_usd = _coerce_float(entry.get("compression_savings_usd"))
+ # Older history points predate cache-savings tracking and omit these
+ # keys entirely; default to 0/0.0 rather than raising or dropping the
+ # entry, matching how every other field here handles legacy shapes.
+ cache_read_tokens = _coerce_int(entry.get("cache_read_tokens"))
+ cache_savings_usd = _coerce_float(entry.get("cache_savings_usd"))
total_input_tokens = _coerce_int(entry.get("total_input_tokens"))
total_input_cost_usd = _coerce_float(entry.get("total_input_cost_usd"))
provider = _normalize_provider(entry.get("provider"))
@@ -358,6 +365,8 @@ def _normalize_history_entry(entry: Any) -> dict[str, Any] | None:
"model": model,
"total_tokens_saved": total_tokens_saved,
"compression_savings_usd": round(compression_savings_usd, 6),
+ "cache_read_tokens": cache_read_tokens,
+ "cache_savings_usd": round(cache_savings_usd, 6),
"total_input_tokens": total_input_tokens,
"total_input_cost_usd": round(total_input_cost_usd, 6),
}
@@ -750,7 +759,13 @@ class SavingsTracker:
input_cost_usd_delta=delta_input_cost_usd,
)
- if delta_tokens_saved > 0:
+ # In --mode cache, headroom's own compression (tokens_saved) is
+ # near-always 0 by design — the frozen prefix is byte-replayed,
+ # not lossy-compressed, to keep Bedrock's prompt cache warm. Gating
+ # on tokens_saved alone silently dropped every history point on
+ # those requests even though real cache-read savings occurred.
+ # Append whenever either mechanism produced a saving.
+ if delta_tokens_saved > 0 or delta_cache_read_tokens > 0:
self._state["history"].append(
{
"timestamp": _to_utc_iso(timestamp_dt),
@@ -758,6 +773,8 @@ class SavingsTracker:
"model": _normalize_model(model),
"total_tokens_saved": lifetime["tokens_saved"],
"compression_savings_usd": lifetime["compression_savings_usd"],
+ "cache_read_tokens": lifetime["cache_read_tokens"],
+ "cache_savings_usd": lifetime["cache_savings_usd"],
"total_input_tokens": lifetime["total_input_tokens"],
"total_input_cost_usd": lifetime["total_input_cost_usd"],
}
diff --git a/tests/test_proxy_savings_history.py b/tests/test_proxy_savings_history.py
index 2d1bcdbc0..64db13086 100644
--- a/tests/test_proxy_savings_history.py
+++ b/tests/test_proxy_savings_history.py
@@ -51,6 +51,11 @@ def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatc
assert savings_tracker_module.get_default_savings_storage_path() == str(override_path)
monkeypatch.delenv(HEADROOM_SAVINGS_PATH_ENV_VAR, raising=False)
+ # HEADROOM_WORKSPACE_DIR overrides the default savings path too (see
+ # headroom/paths.py); unset it so this assertion checks the actual
+ # library default rather than whatever workspace a live deployment on
+ # this machine happens to have exported.
+ monkeypatch.delenv("HEADROOM_WORKSPACE_DIR", raising=False)
default_path = savings_tracker_module.get_default_savings_storage_path()
assert Path(default_path).as_posix().endswith(".headroom/proxy_savings.json")
@@ -73,6 +78,8 @@ def test_savings_tracker_helpers_normalize_inputs_and_paths(tmp_path, monkeypatc
"model": "unknown",
"total_tokens_saved": 12,
"compression_savings_usd": 0.5,
+ "cache_read_tokens": 0,
+ "cache_savings_usd": 0.0,
"total_input_tokens": 0,
"total_input_cost_usd": 0.0,
}
@@ -134,6 +141,8 @@ def test_savings_tracker_sanitizes_legacy_state_and_applies_retention(tmp_path):
"model": "unknown",
"total_tokens_saved": 30,
"compression_savings_usd": 0.03,
+ "cache_read_tokens": 0,
+ "cache_savings_usd": 0.0,
"total_input_tokens": 0,
"total_input_cost_usd": 0.0,
}
@@ -1878,3 +1887,83 @@ def test_non_finite_state_values_coerce_to_defaults(tmp_path):
timestamp="2026-07-02T00:00:00Z",
)
assert tracker.snapshot()["lifetime"]["cache_read_tokens"] == 5
+
+
+def test_cache_only_request_still_appends_a_history_point(tmp_path):
+ # Regression: in --mode cache, tokens_saved is ~always 0 (the frozen
+ # prefix is byte-replayed, not lossy-compressed, to keep the provider's
+ # prompt cache warm). The history-append guard used to gate on
+ # tokens_saved alone, so a cache-only deployment silently wrote zero
+ # history points regardless of real cache_read_tokens/cache_savings_usd —
+ # making headroom-monthly-style tooling read as a total collapse.
+ path = tmp_path / "proxy_savings.json"
+ tracker = SavingsTracker(path=str(path))
+
+ assert tracker.snapshot()["history"] == []
+
+ tracker.record_request(
+ model="claude-sonnet-5",
+ input_tokens=60_000,
+ tokens_saved=0,
+ cache_read_tokens=50_000,
+ timestamp="2026-07-13T22:15:00Z",
+ )
+
+ history = tracker.snapshot()["history"]
+ assert len(history) == 1
+ entry = history[0]
+ assert entry["total_tokens_saved"] == 0
+ assert entry["cache_read_tokens"] == 50_000
+ assert entry["cache_savings_usd"] > 0.0
+
+ # A request with neither compression nor cache savings still appends
+ # nothing — this isn't "always append," only "append on either saving."
+ tracker.record_request(
+ model="claude-sonnet-5",
+ input_tokens=60_000,
+ tokens_saved=0,
+ cache_read_tokens=0,
+ timestamp="2026-07-13T22:16:00Z",
+ )
+ assert len(tracker.snapshot()["history"]) == 1
+
+
+def test_normalize_history_entry_defaults_missing_cache_fields(tmp_path):
+ # Regression: history points written before cache-savings tracking existed
+ # have no cache_read_tokens/cache_savings_usd keys at all. Loading them
+ # back must default to 0/0.0, not raise or silently drop the entry.
+ path = tmp_path / "proxy_savings.json"
+ path.write_text(
+ json.dumps(
+ {
+ "schema_version": 4,
+ "lifetime": {
+ "requests": 1,
+ "tokens_saved": 40,
+ "compression_savings_usd": 0.04,
+ "total_input_tokens": 200,
+ "total_input_cost_usd": 0.4,
+ },
+ "history": [
+ {
+ "timestamp": "2026-07-01T00:00:00Z",
+ "provider": "anthropic",
+ "model": "claude-sonnet-5",
+ "total_tokens_saved": 40,
+ "compression_savings_usd": 0.04,
+ "total_input_tokens": 200,
+ "total_input_cost_usd": 0.4,
+ }
+ ],
+ "projects": {},
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ tracker = SavingsTracker(path=str(path))
+ history = tracker.snapshot()["history"]
+ assert len(history) == 1
+ assert history[0]["cache_read_tokens"] == 0
+ assert history[0]["cache_savings_usd"] == 0.0
+ assert history[0]["total_tokens_saved"] == 40