Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): keep cold-turn reasoning drops sticky so the next warm turn reuses the cached prefix (#4592)

Parideboy3544cf5b88a0
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py
index 4a5d286d8..3dae38386 100644
--- a/headroom/cache/prefix_tracker.py
+++ b/headroom/cache/prefix_tracker.py
@@ -1269,12 +1269,21 @@ class PrefixCacheTracker:
self._tokens_preserved: int = 0
self._compression_foregone_tokens: int = 0
+ # Sticky reasoning-drop floor (#4548): on a cold turn the OpenAI reasoning
+ # compactor drops non-kept prior turns outright (safe — the cold turn
+ # re-caches from scratch). The next warm turn must DROP those same turns
+ # again instead of re-Kompressing them, or the forwarded prefix no longer
+ # matches the cached cold-turn prefix and the cache misses. This is the
+ # highest assistant ordinal dropped + 1, i.e. the count of leading
+ # non-kept turns to keep dropped on subsequent warm turns.
+ self.reasoning_drop_upto: int = 0
+
def get_frozen_message_count(self) -> int:
"""How many leading messages to skip compression on the next turn.
Returns 0 on turn 0 (cold start) or if caching is disabled/below threshold.
"""
- turn_number, cached_tokens, cached_messages, _, _ = self._request_turn_state()
+ turn_number, cached_tokens, cached_messages, *_ = self._request_turn_state()
if not self.config.enabled:
return 0
if turn_number == 0:
@@ -1361,6 +1370,7 @@ class PrefixCacheTracker:
self._cached_message_count,
self._last_original_messages,
self._last_forwarded_messages,
+ self.reasoning_drop_upto,
)
def _request_turn_state(self) -> tuple[Any, ...]:
@@ -1380,7 +1390,14 @@ class PrefixCacheTracker:
resolved = _resolved_turn.get()
if resolved is not None and resolved[0] is self:
if resolved[1] in self._turn_states:
- self._turn_states[resolved[1]] = self._turn_state()
+ state = self._turn_state()
+ # Keep writes associated with the resolved request's floor,
+ # not a sibling fork that mutated the live value (#4548).
+ if resolved[2] is not None and len(resolved[2]) > 5:
+ state = list(state)
+ state[5] = resolved[2][5]
+ state = tuple(state)
+ self._turn_states[resolved[1]] = state
# The request has its response; later reads see the current state.
_resolved_turn.set((self, resolved[1], None))
@@ -1398,8 +1415,30 @@ class PrefixCacheTracker:
self._cached_message_count,
self._last_original_messages,
self._last_forwarded_messages,
+ self.reasoning_drop_upto,
) = state
+ def request_reasoning_drop_floor(self) -> int:
+ """Return the sticky reasoning-drop floor for this request's resolved turn."""
+ return int(self._request_turn_state()[5])
+
+ def raise_reasoning_drop_floor(self, floor: int) -> None:
+ """Raise the sticky reasoning-drop floor for this request.
+
+ A cold turn's drop must survive into the response recorded for this
+ request, without letting a sibling fork's live mutation leak in.
+ Updates the live floor (never shrinking it) and the resolved request's
+ turn snapshot so response recording persists the deliberate increase.
+ """
+ floor = int(floor)
+ self.reasoning_drop_upto = max(self.reasoning_drop_upto, floor)
+ resolved = _resolved_turn.get()
+ if resolved is not None and resolved[0] is self and resolved[2] is not None:
+ state = list(resolved[2])
+ if len(state) > 5:
+ state[5] = max(int(state[5]), floor)
+ _resolved_turn.set((self, resolved[1], tuple(state)))
+
def record_returned(
self,
original_messages: list[dict[str, Any]],
@@ -1705,13 +1744,18 @@ def _deref(table: list[Any], refs: list[int]) -> list[Any]:
def _decode_turn_state(table: list[Any], state: list[Any]) -> tuple[Any, ...]:
- turn, cached_tokens, cached_messages, original, forwarded = state
+ if len(state) >= 6:
+ turn, cached_tokens, cached_messages, original, forwarded, drop_upto = state
+ else:
+ turn, cached_tokens, cached_messages, original, forwarded = state
+ drop_upto = 0
return (
int(turn),
int(cached_tokens),
int(cached_messages),
_deref(table, original),
_deref(table, forwarded),
+ int(drop_upto),
)
@@ -2226,7 +2270,7 @@ class SessionTrackerStore:
return memo[id(messages)]
def state(s: tuple[Any, ...]) -> list[Any]:
- return [s[0], s[1], s[2], ref(s[3]), ref(s[4])]
+ return [s[0], s[1], s[2], ref(s[3]), ref(s[4]), s[5]]
maturation = tracker.read_maturation_manager
meta = {
@@ -2298,6 +2342,7 @@ class SessionTrackerStore:
tracker._cached_message_count,
tracker._last_original_messages,
tracker._last_forwarded_messages,
+ tracker.reasoning_drop_upto,
) = _decode_turn_state(table, meta["state"])
tracker._turn_states = {int(t): _decode_turn_state(table, s) for t, s in meta["turns"]}
tracker.restored_from_disk = True
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 808f7b051..d94233f81 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -4827,16 +4827,35 @@ class OpenAIHandlerMixin:
if _rc_router is not None
else None
)
- # Drop needs no compressor; Kompress (warm) does.
- if _rc_kompress is not None or _rc_cold:
+ # Sticky drop floor: turns dropped on a prior cold turn must stay
+ # dropped on the next warm turn, or the prefix no longer matches
+ # the cached cold-turn prefix and the cache misses (#4548).
+ _rc_drop_upto = (
+ openai_prefix_tracker.request_reasoning_drop_floor()
+ if openai_prefix_tracker is not None
+ else 0
+ )
+ # Drop needs no compressor; Kompress (warm) does. Sticky drop needs
+ # neither — it just re-applies a previously-dropped shape.
+ if _rc_kompress is not None or _rc_cold or _rc_drop_upto:
_rc_keep = int(os.environ.get("HEADROOM_THINKING_COMPACT_KEEP_LAST", "1"))
optimized_messages, _rc_stats = compact_reasoning_openai_chat(
optimized_messages,
kompress=_rc_kompress,
keep_last_turns=_rc_keep,
drop=_rc_cold,
+ drop_first_turns=_rc_drop_upto,
)
body["messages"] = optimized_messages
+ # On a cold turn drop=True drops every non-kept prior turn, so
+ # record that count as the new sticky floor for the next warm
+ # turn. Never shrink it — a later cold turn can only drop more.
+ # Keep the increase on this request's resolved state so response
+ # recording persists it without adopting a sibling fork's floor.
+ if _rc_cold and openai_prefix_tracker is not None:
+ openai_prefix_tracker.raise_reasoning_drop_floor(
+ max(_rc_drop_upto, _rc_stats.get("dropped_upto", 0))
+ )
if _rc_stats["turns_compacted"]:
transforms_applied.append(
f"openai:reasoning_{'drop' if _rc_cold else 'compact'}:{_rc_stats['turns_compacted']}"
diff --git a/headroom/transforms/thinking_compactor.py b/headroom/transforms/thinking_compactor.py
index c03afd2e7..9e9ac4464 100644
--- a/headroom/transforms/thinking_compactor.py
+++ b/headroom/transforms/thinking_compactor.py
@@ -250,6 +250,7 @@ def compact_reasoning_openai_chat(
keep_last_turns: int = 1,
min_words: int = 40,
drop: bool = False,
+ drop_first_turns: int = 0,
) -> tuple[list[dict[str, Any]], dict[str, int]]:
"""Compact plain-text reasoning in OpenAI-chat messages (Kimi / GLM / DeepSeek-R1).
@@ -264,27 +265,46 @@ def compact_reasoning_openai_chat(
``drop=False`` (warm): Kompress the reasoning (deterministic → cache-stable).
``drop=True`` (cold-prefix hook): remove the reasoning outright — the full
block, not just ~15% — safe because the cold turn re-caches from scratch.
+
+ ``drop_first_turns`` (sticky drop): the first ``drop_first_turns`` assistant
+ turns (0-based ordinal, excluding the kept last turns) use the DROP path even
+ on a warm turn, so a turn that was dropped on a prior cold turn is dropped
+ again rather than re-Kompressed — keeping the forwarded prefix byte-identical
+ to the cached cold-turn prefix (#4548). ``dropped_upto`` in the returned stats
+ is (highest dropped ordinal + 1), or 0 when nothing was dropped.
+
Shape-driven — no model gate; no-ops when no plain-text reasoning is present
(OpenAI's encrypted models, Kimi k2.6). Keeps the last ``keep_last_turns``
assistant turns intact (the active reasoning the model uses). Never raises.
"""
- stats = {"turns_compacted": 0, "blocks": 0, "words_before": 0, "words_after": 0}
- if kompress is None and not drop: # drop needs no compressor
+ stats = {
+ "turns_compacted": 0,
+ "blocks": 0,
+ "words_before": 0,
+ "words_after": 0,
+ "dropped_upto": 0,
+ }
+ if kompress is None and not drop and drop_first_turns == 0:
return messages, stats
asst_indices = [i for i, m in enumerate(messages) if m.get("role") == "assistant"]
keep = set(asst_indices[-keep_last_turns:]) if keep_last_turns > 0 else set()
+ ordinal_of = {idx: o for o, idx in enumerate(asst_indices)}
out: list[dict[str, Any]] = []
for i, m in enumerate(messages):
if m.get("role") != "assistant" or i in keep:
out.append(m)
continue
+ ordinal = ordinal_of[i]
+ # Sticky drop: turns dropped on a prior cold turn are dropped again on
+ # the next warm turn so the prefix stays byte-identical to the cache.
+ use_drop = drop or (ordinal < drop_first_turns)
changed = False
nm = dict(m)
# (1) Kimi: reasoning_content field (plain text, no signature → editable)
rc = m.get("reasoning_content")
if isinstance(rc, str) and len(rc.split()) >= min_words:
- if drop: # cold hook: drop the whole reasoning block
+ if use_drop: # cold hook: drop the whole reasoning block
nm["reasoning_content"] = ""
changed = True
stats["blocks"] += 1
@@ -300,7 +320,7 @@ def compact_reasoning_openai_chat(
# (2) GLM / DeepSeek-R1: inline <think>…</think> in string content
c = m.get("content")
if isinstance(c, str) and "<think>" in c:
- new_c, b, wb, wa = _compact_think_spans(c, kompress, min_words, drop=drop)
+ new_c, b, wb, wa = _compact_think_spans(c, kompress, min_words, drop=use_drop)
if b:
nm["content"] = new_c
changed = True
@@ -309,6 +329,8 @@ def compact_reasoning_openai_chat(
stats["words_after"] += wa
if changed:
stats["turns_compacted"] += 1
+ if use_drop:
+ stats["dropped_upto"] = max(stats["dropped_upto"], ordinal + 1)
out.append(nm)
else:
out.append(m)
diff --git a/tests/test_thinking_compact_sticky_drop.py b/tests/test_thinking_compact_sticky_drop.py
new file mode 100644
index 000000000..12940e2bf
--- /dev/null
+++ b/tests/test_thinking_compact_sticky_drop.py
@@ -0,0 +1,183 @@
+"""#4548: reasoning dropped on a cold turn stays dropped on the next warm turn."""
+
+import contextvars
+from types import SimpleNamespace
+
+from headroom.transforms.thinking_compactor import compact_reasoning_openai_chat
+
+
+class _HalfKompress:
+ def compress(self, text, allow_download=False):
+ words = text.split()
+ return SimpleNamespace(compressed=" ".join(words[: len(words) // 2]))
+
+
+def _history(n, tag):
+ msgs = []
+ for k in range(n):
+ msgs.append({"role": "user", "content": f"q{k}"})
+ reasoning = " ".join(f"{tag}{k}w{j}" for j in range(60))
+ msgs.append({"role": "assistant", "content": f"a{k}", "reasoning_content": reasoning})
+ return msgs
+
+
+def test_cold_drop_reports_dropped_upto():
+ out, stats = compact_reasoning_openai_chat(
+ _history(3, "c"), kompress=None, keep_last_turns=1, drop=True
+ )
+ assert [m.get("reasoning_content") for m in out[1::2]][:2] == ["", ""]
+ assert stats["dropped_upto"] == 2
+
+
+def test_warm_turn_after_cold_reuses_cold_prefix():
+ cold, stats = compact_reasoning_openai_chat(
+ _history(3, "w"), kompress=None, keep_last_turns=1, drop=True
+ )
+ warm, _ = compact_reasoning_openai_chat(
+ _history(4, "w"),
+ kompress=_HalfKompress(),
+ keep_last_turns=1,
+ drop=False,
+ drop_first_turns=stats["dropped_upto"],
+ )
+ assert warm[:4] == cold[:4] # previously dropped turns byte-identical
+ assert 0 < len(warm[5]["reasoning_content"].split()) < 60 # newer turn Kompressed
+ assert len(warm[7]["reasoning_content"].split()) == 60 # active turn kept
+
+
+def test_no_sticky_floor_is_unchanged():
+ out, stats = compact_reasoning_openai_chat(
+ _history(3, "u"), kompress=_HalfKompress(), keep_last_turns=1
+ )
+ assert all(len(m["reasoning_content"].split()) == 30 for m in out[1:4:2])
+ assert stats["dropped_upto"] == 0
+
+
+def test_inline_think_spans_stay_dropped():
+ think = "<think>" + " ".join(f"t{j}" for j in range(60)) + "</think>"
+ msgs = [
+ {"role": "user", "content": "q"},
+ {"role": "assistant", "content": think + "a0"},
+ {"role": "user", "content": "q"},
+ {"role": "assistant", "content": think + "a1"},
+ ]
+ out, _ = compact_reasoning_openai_chat(
+ msgs, kompress=_HalfKompress(), keep_last_turns=1, drop_first_turns=1
+ )
+ assert out[1]["content"] == "a0"
+ assert out[3]["content"] == think + "a1"
+
+
+def _tracker_with_floor(floor):
+ from headroom.cache.prefix_tracker import PrefixCacheTracker
+
+ tracker = PrefixCacheTracker("openai")
+ tracker.update_from_response(cache_read_tokens=0, cache_write_tokens=0, messages=[])
+ tracker.reasoning_drop_upto = floor
+ return tracker
+
+
+def test_floor_survives_shared_round_trip_and_warm_prefix_matches():
+ from headroom.cache.prefix_tracker import SessionTrackerStore
+
+ cold, stats = compact_reasoning_openai_chat(
+ _history(3, "r"), kompress=None, keep_last_turns=1, drop=True
+ )
+ store = SessionTrackerStore()
+ meta, table = store.encode_shared("k", _tracker_with_floor(stats["dropped_upto"]))
+ restored = store.decode_shared(meta, table)
+ assert restored.reasoning_drop_upto == 2
+ warm, _ = compact_reasoning_openai_chat(
+ _history(4, "r"),
+ kompress=_HalfKompress(),
+ keep_last_turns=1,
+ drop_first_turns=restored.reasoning_drop_upto,
+ )
+ assert warm[:4] == cold[:4]
+
+
+def test_old_metadata_without_floor_decodes_to_zero():
+ from headroom.cache.prefix_tracker import SessionTrackerStore
+
+ store = SessionTrackerStore()
+ meta, table = store.encode_shared("k", _tracker_with_floor(3))
+ meta["state"] = meta["state"][:5]
+ meta["turns"] = [[t, s[:5]] for t, s in meta["turns"]]
+ assert store.decode_shared(meta, table).reasoning_drop_upto == 0
+
+
+def test_resumed_turn_restores_its_own_floor():
+ tracker = _tracker_with_floor(2)
+ tracker._turn_states[1] = tracker._turn_state()
+ tracker.reasoning_drop_upto = 3 # a cold fork raised it
+ tracker._resume_turn(1)
+ assert tracker.reasoning_drop_upto == 2
+
+
+def test_request_floor_ignores_a_concurrent_fork_raising_the_live_floor():
+ from headroom.cache.prefix_tracker import _resolved_turn
+
+ tracker = _tracker_with_floor(2)
+ tracker._turn_states[1] = tracker._turn_state()
+ request_a = contextvars.copy_context()
+ request_a.run(_resolved_turn.set, (tracker, 1, tracker._turn_states[1]))
+ tracker.reasoning_drop_upto = 3 # sibling fork went cold after A resolved
+
+ floor = request_a.run(tracker.request_reasoning_drop_floor)
+ assert floor == 2
+ out, _ = compact_reasoning_openai_chat(
+ _history(4, "f"), kompress=_HalfKompress(), keep_last_turns=1, drop_first_turns=floor
+ )
+ assert len(out[5]["reasoning_content"].split()) == 30 # Kompressed, not dropped
+
+
+def test_cold_request_records_raised_floor_through_response():
+ from headroom.cache.prefix_tracker import _resolved_turn
+
+ tracker = _tracker_with_floor(0)
+ tracker._turn_states[1] = tracker._turn_state()
+ request = contextvars.copy_context()
+ request.run(_resolved_turn.set, (tracker, 1, tracker._turn_states[1]))
+
+ cold, stats = compact_reasoning_openai_chat(
+ _history(3, "r"), kompress=None, keep_last_turns=1, drop=True
+ )
+ floor = max(request.run(tracker.request_reasoning_drop_floor), stats["dropped_upto"])
+ request.run(tracker.raise_reasoning_drop_floor, floor)
+ request.run(tracker.update_from_response, 0, 0, cold)
+
+ assert tracker._turn_states[1][5] == 2
+ tracker._resume_turn(1)
+ warm, _ = compact_reasoning_openai_chat(
+ _history(4, "r"),
+ kompress=_HalfKompress(),
+ keep_last_turns=1,
+ drop=False,
+ drop_first_turns=tracker.request_reasoning_drop_floor(),
+ )
+ assert [m.get("reasoning_content") for m in warm[1::2]][:2] == ["", ""]
+ assert warm[:5] == cold[:5]
+
+
+def test_cold_request_floor_write_isolated_from_sibling_through_response():
+ from headroom.cache.prefix_tracker import _resolved_turn
+
+ tracker = _tracker_with_floor(2)
+ tracker._turn_states[1] = tracker._turn_state()
+ request_a = contextvars.copy_context()
+ request_a.run(_resolved_turn.set, (tracker, 1, tracker._turn_states[1]))
+ tracker.reasoning_drop_upto = 3 # sibling fork mutated live floor
+
+ floor = request_a.run(tracker.request_reasoning_drop_floor)
+ assert floor == 2
+ request_a.run(tracker.update_from_response, 0, 0, [])
+
+ assert tracker._turn_states[1][5] == 2
+ tracker._resume_turn(1)
+ out, _ = compact_reasoning_openai_chat(
+ _history(4, "s"),
+ kompress=_HalfKompress(),
+ keep_last_turns=1,
+ drop_first_turns=tracker.request_reasoning_drop_floor(),
+ )
+ assert len(out[5]["reasoning_content"].split()) == 30 # Kompressed, not dropped