Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(policy): batch deep edits through one cache-bust (#856 P3a) (#1015)

Focused Instabilityc2e52fe7439b
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index c25cb7b3a..05b89cca6 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -1970,6 +1970,7 @@ class ContentRouter(Transform):
suffix_tokens: list[int],
route_counts: dict[str, int],
transforms_applied: list[str],
+ batch_state: dict[str, int | None] | None = None,
) -> bool:
"""Break-even gate for one candidate mutation (#856 P2, flag-gated).
@@ -1982,9 +1983,31 @@ class ContentRouter(Transform):
full-penalty assumption). Every decision is logged with its inputs
and counted in ``route_counts`` so the flag can be validated from
telemetry before any default-on.
+
+ #856 P3a (batch deep edits): a mutation at depth K busts the
+ provider's cached suffix after K, so every *later* candidate at a
+ deeper slot rides that same invalidation for free — mutating it adds
+ no incremental cache-bust cost. ``batch_state["floor"]`` tracks the
+ shallowest slot already admitted as a net-positive mutation. When the
+ current candidate sits strictly deeper than that floor, S is charged
+ as 0 (rather than the full invalidated suffix), so the break-even
+ formula admits it on the write/read economics alone. Charging S=0 via
+ the same ``net_mutation_gain`` (instead of blanket-admitting on
+ ``delta_t > 0``) keeps the decision conservative: it never admits a
+ mutation the real economics would reject. The floor is only set/lowered
+ by full-S admits, so a slot only ever rides free behind a genuinely
+ mutated shallower slot. Each batch admission emits the
+ ``router:netcost_batch_admit`` marker and the ``netcost_batch_admitted``
+ counter for telemetry.
"""
delta_t = max(0, original_tokens - compressed_tokens)
- suffix = suffix_tokens[slot_idx + 1]
+ # Batch reclaim: if a shallower slot was already admitted, its
+ # cache-bust already invalidated everything after it, including this
+ # slot — so charge S=0 here. Otherwise S is the full suffix after the
+ # candidate (P2 v1 estimator).
+ floor = batch_state.get("floor") if batch_state is not None else None
+ batch_reclaim = floor is not None and slot_idx > floor
+ suffix = 0 if batch_reclaim else suffix_tokens[slot_idx + 1]
policy = self._runtime_compression_policy
if policy is None:
from .compression_policy import policy_default_payg
@@ -2015,18 +2038,31 @@ class ContentRouter(Transform):
gain = float(policy.net_mutation_gain(delta_t, suffix, reads, p_alive))
allowed = gain > 0.0
logger.info(
- "NetCostPolicy slot=%d delta_t=%d suffix=%d reads=%.1f p_alive=%.2f gain=%.0f -> %s",
+ "NetCostPolicy slot=%d delta_t=%d suffix=%d reads=%.1f p_alive=%.2f "
+ "gain=%.0f batch_reclaim=%s -> %s",
slot_idx,
delta_t,
suffix,
reads,
p_alive,
gain,
+ batch_reclaim,
"mutate" if allowed else "skip",
)
if allowed:
route_counts.setdefault("netcost_allowed", 0)
route_counts["netcost_allowed"] += 1
+ if batch_reclaim:
+ # Rode a shallower edit's cache-bust for free — telemetry only;
+ # the floor is unchanged (this slot is deeper than the floor).
+ route_counts.setdefault("netcost_batch_admitted", 0)
+ route_counts["netcost_batch_admitted"] += 1
+ transforms_applied.append("router:netcost_batch_admit")
+ elif batch_state is not None:
+ # First/shallower full-S admit — open (or lower) the batch
+ # floor so deeper candidates can reclaim against it.
+ current = batch_state.get("floor")
+ batch_state["floor"] = slot_idx if current is None else min(current, slot_idx)
else:
route_counts.setdefault("netcost_skipped", 0)
route_counts["netcost_skipped"] += 1
@@ -2215,6 +2251,10 @@ class ContentRouter(Transform):
# token total of every message after the candidate.
netcost_enabled = os.environ.get("HEADROOM_NET_COST_POLICY") == "1"
netcost_suffix_tokens: list[int] = []
+ # #856 P3a: shared batch-reclaim state for this request. ``floor`` is
+ # the shallowest slot admitted as a net-positive mutation; once set,
+ # deeper candidates charge S=0 (their cache-bust is already paid).
+ netcost_batch_state: dict[str, int | None] = {"floor": None}
if netcost_enabled:
netcost_suffix_tokens = [0] * (num_messages + 1)
for j in range(num_messages - 1, -1, -1):
@@ -2411,6 +2451,7 @@ class ContentRouter(Transform):
suffix_tokens=netcost_suffix_tokens,
route_counts=route_counts,
transforms_applied=transforms_applied,
+ batch_state=netcost_batch_state,
):
# Net-cost gate: mutation would cost more in cache
# invalidation than it saves — leave untouched.
@@ -2492,6 +2533,7 @@ class ContentRouter(Transform):
suffix_tokens=netcost_suffix_tokens,
route_counts=route_counts,
transforms_applied=transforms_applied,
+ batch_state=netcost_batch_state,
):
result_slots[slot_idx] = message
continue
@@ -2547,6 +2589,8 @@ class ContentRouter(Transform):
parts.append(f"{route_counts['cache_hit']} cache hits")
if route_counts.get("cache_miss"):
parts.append(f"{route_counts['cache_miss']} cache misses")
+ if route_counts.get("netcost_batch_admitted"):
+ parts.append(f"{route_counts['netcost_batch_admitted']} netcost batch-admitted")
cs = self._cache.stats
if cs["cache_size"] > 0 or cs["cache_skip_size"] > 0:
parts.append(
diff --git a/tests/test_netcost_gate.py b/tests/test_netcost_gate.py
index 70564bf6a..54f3d35ca 100644
--- a/tests/test_netcost_gate.py
+++ b/tests/test_netcost_gate.py
@@ -262,3 +262,95 @@ class TestNetCostFrozenUnlock:
result = router.apply([dict(m) for m in messages], tokenizer, frozen_message_count=2)
assert result.messages[1]["content"] == original[1]["content"]
assert "router:netcost_frozen_unlock" not in result.transforms_applied
+
+
+class TestNetCostBatchReclaim:
+ """#856 P3a: batch deep edits -- once one net-positive edit is admitted at
+ slot K, deeper candidates ride that cache-bust for free (S charged as 0).
+
+ Tests are cache-cold (fresh router per fixture) so every candidate flows
+ through the parallel-merge pass in ascending slot order, which is where the
+ shared batch_state floor is set and then reclaimed.
+ """
+
+ @staticmethod
+ def _convo(slot1: str, slot2: str, filler_words: int) -> list[dict]:
+ # Two consecutive tool dumps (slots 1 and 2) followed by a filler
+ # suffix. Slot 1 is the shallower candidate; slot 2 the deeper one.
+ suffix = "analysis context word " * filler_words
+ return [
+ {"role": "user", "content": "fetch the records"},
+ {"role": "tool", "content": slot1},
+ {"role": "tool", "content": slot2},
+ {"role": "user", "content": suffix},
+ {"role": "user", "content": "summarize"},
+ ]
+
+ @staticmethod
+ def _compressed(result, original, idx: int) -> bool:
+ return result.messages[idx]["content"] != original[idx]["content"]
+
+ def test_flag_off_no_batch_marker(self, router, tokenizer, monkeypatch):
+ # Without the flag the batch path is inert -- no marker, no counter.
+ monkeypatch.delenv("HEADROOM_NET_COST_POLICY", raising=False)
+ messages = self._convo(_tool_json(2000), _tool_json(800), filler_words=5)
+ original = [dict(m) for m in messages]
+ result = router.apply([dict(m) for m in messages], tokenizer)
+ assert "router:netcost_batch_admit" not in result.transforms_applied
+ # Both deep edits still compress (no gate at all when flag off).
+ assert self._compressed(result, original, 1)
+ assert self._compressed(result, original, 2)
+
+ def test_deeper_edit_rides_free(self, router, tokenizer, monkeypatch):
+ # Slot 1 (huge shave) admits on its own merit and opens the floor;
+ # slot 2 then admits via the batch reclaim path and emits the marker.
+ monkeypatch.setenv("HEADROOM_NET_COST_POLICY", "1")
+ messages = self._convo(_tool_json(2000), _tool_json(800), filler_words=5)
+ original = [dict(m) for m in messages]
+ result = router.apply([dict(m) for m in messages], tokenizer)
+ assert self._compressed(result, original, 1)
+ assert self._compressed(result, original, 2)
+ markers = [t for t in result.transforms_applied if t == "router:netcost_batch_admit"]
+ # Exactly one deeper slot rode the floor for free.
+ assert len(markers) == 1
+
+ def test_batch_admits_otherwise_blocked_edit(self, router, tokenizer, monkeypatch):
+ # Slot 2 (modest shave, large suffix after it) would be blocked on its
+ # own S, but slot 1's admit already busted the suffix -- so slot 2 rides
+ # free. Pairs with test_no_prior_admit_keeps_block, which shows the same
+ # slot-2 config stays blocked when no shallower edit opens the floor.
+ monkeypatch.setenv("HEADROOM_NET_COST_POLICY", "1")
+ messages = self._convo(_tool_json(2000), _tool_json(300), filler_words=4000)
+ original = [dict(m) for m in messages]
+ result = router.apply([dict(m) for m in messages], tokenizer)
+ assert self._compressed(result, original, 1) # floor-setting admit
+ assert self._compressed(result, original, 2) # rode free
+ assert "router:netcost_batch_admit" in result.transforms_applied
+
+ def test_no_prior_admit_keeps_block(self, router, tokenizer, monkeypatch):
+ # Neither candidate beats its own S (both modest shaves under a huge
+ # suffix), so the floor is never opened and no slot rides free. Guards
+ # against a floor-init / off-by-one bug that would grant a free ride
+ # with no genuine shallower mutation behind it.
+ monkeypatch.setenv("HEADROOM_NET_COST_POLICY", "1")
+ messages = self._convo(_tool_json(300), _tool_json(300), filler_words=40000)
+ original = [dict(m) for m in messages]
+ result = router.apply([dict(m) for m in messages], tokenizer)
+ assert not self._compressed(result, original, 1)
+ assert not self._compressed(result, original, 2)
+ assert "router:netcost_batch_admit" not in result.transforms_applied
+
+ def test_frozen_unlock_and_batch_combine(self, router, tokenizer, monkeypatch):
+ # Two frozen string slots inside the prefix (frozen_message_count=3).
+ # Slot 1 unlocks and sets the floor; slot 2 unlocks AND rides free.
+ # Slot 2 carries both markers; the batch counter must not double-count.
+ monkeypatch.setenv("HEADROOM_NET_COST_POLICY", "1")
+ messages = self._convo(_tool_json(2000), _tool_json(800), filler_words=5)
+ original = [dict(m) for m in messages]
+ result = router.apply([dict(m) for m in messages], tokenizer, frozen_message_count=3)
+ assert self._compressed(result, original, 1)
+ assert self._compressed(result, original, 2)
+ unlocks = [t for t in result.transforms_applied if t == "router:netcost_frozen_unlock"]
+ batch = [t for t in result.transforms_applied if t == "router:netcost_batch_admit"]
+ assert len(unlocks) == 2 # both frozen slots opened
+ assert len(batch) == 1 # only the deeper one rode free -- no double-count