Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(policy): correct warm-cache penalty in net_mutation_gain to (S + dT) (#903)

Focused Instability0632eba6c3bd
diff --git a/crates/headroom-core/src/compression_policy.rs b/crates/headroom-core/src/compression_policy.rs
index 278d98cbc..4d9d786c6 100644
--- a/crates/headroom-core/src/compression_policy.rs
+++ b/crates/headroom-core/src/compression_policy.rs
@@ -239,21 +239,24 @@ impl CompressionPolicy {
/// removes `delta_t` tokens from a message whose cached suffix is
/// `suffix_tokens` long (#856).
///
- /// Mutating message K invalidates every cached token after it: the
- /// suffix is re-written once at the write multiplier instead of
- /// being read at the read multiplier, costing
- /// `P_alive · (w − r) · S`. In exchange, `delta_t` tokens are gone
- /// from the current write and every one of the `expected_reads`
- /// remaining reads of the chain, saving `ΔT · (w + r·(R − 1))`.
+ /// Mutating message K invalidates every cached token after it. When
+ /// the cache is warm the mutated ΔT tokens are themselves already
+ /// cache-written, so keeping them costs only reads (`ΔT · r · R`)
+ /// while mutating re-writes the suffix: alive-case saving is
+ /// `ΔT·r·R − (w−r)·S`. When the cache is dead there is no suffix
+ /// penalty and the full `ΔT·(w + r·(R−1))` is saved. Taking the
+ /// expectation over `P_alive`:
///
- /// gain = ΔT · (w + r·(R − 1)) − P_alive · (w − r) · S
+ /// gain = ΔT · (w + r·(R − 1)) − P_alive · (w − r) · (S + ΔT)
///
/// Sanity anchors (Anthropic w=1.25, r=0.1), matching the unit
- /// tests below: a 2K shave under a 50K warm suffix needs ~276
+ /// tests below: a 2K shave under a 50K warm suffix needs 287.5
/// remaining reads to pay off (rarely profitable); a 50K shave
- /// under a 10K suffix is profitable from the first write (its
- /// break-even read count is negative); a live-zone edit (S = 0)
- /// is always profitable.
+ /// under a 10K suffix breaks even at 2.3 reads (profitable in any
+ /// session with a few turns left); an edit with S = 0 is profitable
+ /// whenever at least one read remains. Callers gating not-yet-cached
+ /// content (live-zone edits) should bypass this formula — it prices
+ /// mutations of content the cache has already written.
///
/// Takes `&self` so a follow-up can apply per-mode margins; today
/// the arithmetic is mode-independent. Inputs are clamped:
@@ -276,7 +279,15 @@ impl CompressionPolicy {
} else {
p_alive.clamp(0.0, 1.0)
};
- (delta_t as f32) * (w + r * (reads - 1.0)) - alive * (w - r) * (suffix_tokens as f32)
+ // Corrected warm-case penalty (#856 follow-up): when the cache is
+ // alive, the ΔT tokens are already cache-written, so keeping them
+ // costs only reads — a mutation can avoid at most ΔT·r·R, not a
+ // fresh write. Blending alive (ΔT·r·R − (w−r)·S) and dead
+ // (ΔT·(w + r·(R−1))) cases over P_alive gives a penalty over
+ // S + ΔT, not S alone. The looser ·S form overstated gain by
+ // P_alive·(w−r)·ΔT — always pro-mutation, largest for big shaves.
+ (delta_t as f32) * (w + r * (reads - 1.0))
+ - alive * (w - r) * ((suffix_tokens as f32) + (delta_t as f32))
}
/// Decision form of [`Self::net_mutation_gain`]: mutate iff the
@@ -292,9 +303,13 @@ impl CompressionPolicy {
}
/// Remaining-read count at which a warm-cache (P_alive = 1)
- /// mutation breaks even:
+ /// mutation breaks even. With the corrected penalty this is exactly
///
- /// R = ((w − r) / r) · (S/ΔT − 1) ≈ 11.5 · S/ΔT for S ≫ ΔT
+ /// R = ((w − r) / r) · S/ΔT = 11.5 · S/ΔT (Anthropic 5-min)
+ ///
+ /// reproducing the #856 anchors precisely: 2K shave / 50K suffix →
+ /// 287.5 (~290 reads, rarely profitable); 50K shave / 10K suffix →
+ /// 2.3 (profitable in any session with a few turns left).
///
/// Useful for decision telemetry ("this edit pays off if the
/// session lasts N more turns"). Returns 0 when `delta_t` is 0
@@ -305,7 +320,7 @@ impl CompressionPolicy {
}
let w = CACHE_WRITE_MULTIPLIER;
let r = CACHE_READ_MULTIPLIER;
- ((w - r) / r) * ((suffix_tokens as f32) / (delta_t as f32) - 1.0)
+ ((w - r) / r) * ((suffix_tokens as f32) / (delta_t as f32))
}
}
@@ -414,31 +429,36 @@ mod tests {
#[test]
fn net_gain_small_shave_deep_suffix_is_loss() {
// Shave 2K under a 50K warm suffix at R=10 remaining reads:
- // 2000·(1.25 + 0.1·9) − 1.0·1.15·50000 = 4300 − 57500 = −53200.
+ // 2000·(1.25 + 0.1·9) − 1.0·1.15·52000 = 4300 − 59800 = −55500.
let p = CompressionPolicy::for_mode(AuthMode::Payg);
let gain = p.net_mutation_gain(2_000, 50_000, 10.0, 1.0);
- assert!((gain - (-53_200.0)).abs() < 1.0, "gain = {gain}");
+ assert!((gain - (-55_500.0)).abs() < 1.0, "gain = {gain}");
assert!(!p.should_mutate_deep(2_000, 50_000, 10.0, 1.0));
}
#[test]
fn net_gain_big_shave_shallow_suffix_is_win() {
// Shave 50K under a 10K warm suffix at R=3:
- // 50000·(1.25 + 0.1·2) − 1.0·1.15·10000 = 72500 − 11500 = 61000.
+ // 50000·(1.25 + 0.1·2) − 1.0·1.15·60000 = 72500 − 69000 = 3500.
+ // Tight but positive — consistent with the 2.3-read break-even.
let p = CompressionPolicy::for_mode(AuthMode::Payg);
let gain = p.net_mutation_gain(50_000, 10_000, 3.0, 1.0);
- assert!((gain - 61_000.0).abs() < 1.0, "gain = {gain}");
+ assert!((gain - 3_500.0).abs() < 1.0, "gain = {gain}");
assert!(p.should_mutate_deep(50_000, 10_000, 3.0, 1.0));
}
#[test]
- fn net_gain_live_zone_edit_always_profitable() {
- // S = 0 derives the existing Subscription live-zone policy as a
- // special case: nothing cached is invalidated, so any positive
- // shave wins even at R=0 (gain = ΔT·(w − r) > 0).
+ fn net_gain_no_suffix_edit_profitable_with_reads_remaining() {
+ // S = 0: nothing cached after the edit is invalidated. Warm-case
+ // saving is the avoided rereads, ΔT·r·R — positive whenever at
+ // least one read remains. At R=0 with a warm cache the gain is
+ // exactly 0 (already written, never read again): the boundary
+ // where mutating is pointless rather than harmful.
let p = CompressionPolicy::for_mode(AuthMode::Subscription);
- assert!(p.should_mutate_deep(1, 0, 0.0, 1.0));
- assert!(p.should_mutate_deep(2_000, 0, 0.0, 1.0));
+ assert!(p.should_mutate_deep(1, 0, 1.0, 1.0));
+ assert!(p.should_mutate_deep(2_000, 0, 1.0, 1.0));
+ let boundary = p.net_mutation_gain(2_000, 0, 0.0, 1.0);
+ assert!(boundary.abs() < f32::EPSILON, "boundary = {boundary}");
}
#[test]
@@ -472,13 +492,14 @@ mod tests {
#[test]
fn break_even_reads_matches_research_anchor() {
- // R = 11.5·(S/ΔT − 1): 2K shave / 50K suffix → 11.5·24 = 276
- // (rarely profitable); 50K shave / 10K suffix →
- // 11.5·(0.2 − 1) < 0 → profitable from the first read.
+ // R = 11.5·S/ΔT, the #856 anchors exactly: 2K shave / 50K
+ // suffix → 11.5·25 = 287.5 (rarely profitable); 50K shave /
+ // 10K suffix → 11.5·0.2 = 2.3 (profitable within a few turns).
let p = CompressionPolicy::for_mode(AuthMode::Payg);
let r = p.break_even_reads(2_000, 50_000);
- assert!((r - 276.0).abs() < 0.5, "break-even = {r}");
- assert!(p.break_even_reads(50_000, 10_000) < 0.0);
+ assert!((r - 287.5).abs() < 0.5, "break-even = {r}");
+ let shallow = p.break_even_reads(50_000, 10_000);
+ assert!((shallow - 2.3).abs() < 0.05, "break-even = {shallow}");
assert_eq!(p.break_even_reads(0, 10_000), 0.0);
}
}
diff --git a/headroom/transforms/compression_policy.py b/headroom/transforms/compression_policy.py
index 717cc56b8..626b40bc3 100644
--- a/headroom/transforms/compression_policy.py
+++ b/headroom/transforms/compression_policy.py
@@ -137,7 +137,12 @@ class CompressionPolicy:
Mirrors ``CompressionPolicy::net_mutation_gain`` in the Rust
crate (source of truth — see its docstring for the derivation)::
- gain = dT * (w + r*(R - 1)) - P_alive * (w - r) * S
+ gain = dT * (w + r*(R - 1)) - P_alive * (w - r) * (S + dT)
+
+ The warm-case penalty covers ``S + dT``: with a live cache the
+ ``dT`` tokens are already cache-written, so keeping them costs
+ only reads — a mutation avoids at most ``dT*r*R``, not a fresh
+ write.
Inputs are clamped: ``delta_t``/``suffix_tokens`` to ``>= 0``
(the Rust signature takes ``u32``), ``expected_reads`` to
@@ -152,7 +157,7 @@ class CompressionPolicy:
# f32::max in the Rust source of truth — guard explicitly.
reads = 0.0 if math.isnan(expected_reads) else max(expected_reads, 0.0)
alive = 1.0 if math.isnan(p_alive) else min(max(p_alive, 0.0), 1.0)
- return float(dt) * (w + r * (reads - 1.0)) - alive * (w - r) * float(suffix)
+ return float(dt) * (w + r * (reads - 1.0)) - alive * (w - r) * float(suffix + dt)
def should_mutate_deep(
self,
@@ -169,7 +174,10 @@ class CompressionPolicy:
"""Remaining-read count at which a warm-cache (``p_alive=1``)
mutation breaks even::
- R = ((w - r) / r) * (S/dT - 1) ~= 11.5 * S/dT for S >> dT
+ R = ((w - r) / r) * (S/dT) = 11.5 * S/dT (Anthropic 5-min)
+
+ With the corrected penalty this reproduces the #856 anchors
+ exactly: 2K/50K -> 287.5, 50K/10K -> 2.3.
Returns 0 when ``delta_t`` is ``<= 0`` (no savings — callers
gate on ``delta_t > 0``; the Rust signature takes ``u32``).
@@ -179,7 +187,7 @@ class CompressionPolicy:
return 0.0
w = CACHE_WRITE_MULTIPLIER
r = CACHE_READ_MULTIPLIER
- return ((w - r) / r) * (float(max(0, suffix_tokens)) / float(delta_t) - 1.0)
+ return ((w - r) / r) * (float(max(0, suffix_tokens)) / float(delta_t))
def policy_for_mode(mode: AuthMode) -> CompressionPolicy:
diff --git a/tests/test_compression_policy.py b/tests/test_compression_policy.py
index da6a5284a..cac6f02f9 100644
--- a/tests/test_compression_policy.py
+++ b/tests/test_compression_policy.py
@@ -169,25 +169,29 @@ class TestNetCostFormula:
"""
def test_small_shave_deep_suffix_is_loss(self):
- # 2000*(1.25 + 0.1*9) - 1.0*1.15*50000 = 4300 - 57500 = -53200.
+ # 2000*(1.25 + 0.1*9) - 1.0*1.15*52000 = 4300 - 59800 = -55500.
p = policy_for_mode(AuthMode.PAYG)
gain = p.net_mutation_gain(2_000, 50_000, 10.0, 1.0)
- assert abs(gain - (-53_200.0)) < 1.0
+ assert abs(gain - (-55_500.0)) < 1.0
assert not p.should_mutate_deep(2_000, 50_000, 10.0, 1.0)
def test_big_shave_shallow_suffix_is_win(self):
- # 50000*(1.25 + 0.1*2) - 1.0*1.15*10000 = 72500 - 11500 = 61000.
+ # 50000*(1.25 + 0.1*2) - 1.0*1.15*60000 = 72500 - 69000 = 3500.
+ # Tight but positive — consistent with the 2.3-read break-even.
p = policy_for_mode(AuthMode.PAYG)
gain = p.net_mutation_gain(50_000, 10_000, 3.0, 1.0)
- assert abs(gain - 61_000.0) < 1.0
+ assert abs(gain - 3_500.0) < 1.0
assert p.should_mutate_deep(50_000, 10_000, 3.0, 1.0)
- def test_live_zone_edit_always_profitable(self):
- # S = 0 derives the existing Subscription live-zone policy as a
- # special case of the formula.
+ def test_no_suffix_edit_profitable_with_reads_remaining(self):
+ # S = 0: warm-case saving is the avoided rereads, dT*r*R —
+ # positive whenever at least one read remains. At R=0 with a
+ # warm cache the gain is exactly 0 (already written, never read
+ # again): pointless rather than harmful.
p = policy_for_mode(AuthMode.SUBSCRIPTION)
- assert p.should_mutate_deep(1, 0, 0.0, 1.0)
- assert p.should_mutate_deep(2_000, 0, 0.0, 1.0)
+ assert p.should_mutate_deep(1, 0, 1.0, 1.0)
+ assert p.should_mutate_deep(2_000, 0, 1.0, 1.0)
+ assert abs(p.net_mutation_gain(2_000, 0, 0.0, 1.0)) < 1e-6
def test_cold_cache_ignores_suffix(self):
# P_alive = 0 (TTL lapsed): the idle-timer compaction window.
@@ -220,11 +224,11 @@ class TestNetCostFormula:
assert p.net_mutation_gain(2_000, -1, 5.0, 1.0) == p.net_mutation_gain(2_000, 0, 5.0, 1.0)
def test_break_even_reads_matches_research_anchor(self):
- # R = 11.5*(S/dT - 1): 2K/50K -> 276; 50K/10K -> negative
- # (profitable from the first read); dT=0 -> 0.
+ # R = 11.5*S/dT, the #856 anchors exactly: 2K/50K -> 287.5;
+ # 50K/10K -> 2.3; dT=0 -> 0.
p = policy_for_mode(AuthMode.PAYG)
- assert abs(p.break_even_reads(2_000, 50_000) - 276.0) < 0.5
- assert p.break_even_reads(50_000, 10_000) < 0.0
+ assert abs(p.break_even_reads(2_000, 50_000) - 287.5) < 0.5
+ assert abs(p.break_even_reads(50_000, 10_000) - 2.3) < 0.05
assert p.break_even_reads(0, 10_000) == 0.0
def test_constants_match_rust(self):