Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): keep cold-turn reasoning drops sticky so the next warm turn reuses the cached prefix (#4592)
Parideboy3544cf5b88a0
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.pyindex 4a5d286d8..3dae38386 100644--- a/headroom/cache/prefix_tracker.py+++ b/headroom/cache/prefix_tracker.py@@ -1269,12 +1269,21 @@ class PrefixCacheTracker:self._tokens_preserved: int = 0self._compression_foregone_tokens: int = 0+ # Sticky reasoning-drop floor (#4548): on a cold turn the OpenAI reasoning+ # compactor drops non-kept prior turns outright (safe — the cold turn+ # re-caches from scratch). The next warm turn must DROP those same turns+ # again instead of re-Kompressing them, or the forwarded prefix no longer+ # matches the cached cold-turn prefix and the cache misses. This is the+ # highest assistant ordinal dropped + 1, i.e. the count of leading+ # non-kept turns to keep dropped on subsequent warm turns.+ self.reasoning_drop_upto: int = 0+def get_frozen_message_count(self) -> int:"""How many leading messages to skip compression on the next turn.Returns 0 on turn 0 (cold start) or if caching is disabled/below threshold."""- turn_number, cached_tokens, cached_messages, _, _ = self._request_turn_state()+ turn_number, cached_tokens, cached_messages, *_ = self._request_turn_state()if not self.config.enabled:return 0if turn_number == 0:@@ -1361,6 +1370,7 @@ class PrefixCacheTracker:self._cached_message_count,self._last_original_messages,self._last_forwarded_messages,+ self.reasoning_drop_upto,)def _request_turn_state(self) -> tuple[Any, ...]:@@ -1380,7 +1390,14 @@ class PrefixCacheTracker:resolved = _resolved_turn.get()if resolved is not None and resolved[0] is self:if resolved[1] in self._turn_states:- self._turn_states[resolved[1]] = self._turn_state()+ state = self._turn_state()+ # Keep writes associated with the resolved request's floor,+ # not a sibling fork that mutated the live value (#4548).+ if resolved[2] is not None and len(resolved[2]) > 5:+ state = list(state)+ state[5] = resolved[2][5]+ state = tuple(state)+ self._turn_states[resolved[1]] = state# The request has its response; later reads see the current state._resolved_turn.set((self, resolved[1], None))@@ -1398,8 +1415,30 @@ class PrefixCacheTracker:self._cached_message_count,self._last_original_messages,self._last_forwarded_messages,+ self.reasoning_drop_upto,) = state+ def request_reasoning_drop_floor(self) -> int:+ """Return the sticky reasoning-drop floor for this request's resolved turn."""+ return int(self._request_turn_state()[5])++ def raise_reasoning_drop_floor(self, floor: int) -> None:+ """Raise the sticky reasoning-drop floor for this request.++ A cold turn's drop must survive into the response recorded for this+ request, without letting a sibling fork's live mutation leak in.+ Updates the live floor (never shrinking it) and the resolved request's+ turn snapshot so response recording persists the deliberate increase.+ """+ floor = int(floor)+ self.reasoning_drop_upto = max(self.reasoning_drop_upto, floor)+ resolved = _resolved_turn.get()+ if resolved is not None and resolved[0] is self and resolved[2] is not None:+ state = list(resolved[2])+ if len(state) > 5:+ state[5] = max(int(state[5]), floor)+ _resolved_turn.set((self, resolved[1], tuple(state)))+def record_returned(self,original_messages: list[dict[str, Any]],@@ -1705,13 +1744,18 @@ def _deref(table: list[Any], refs: list[int]) -> list[Any]:def _decode_turn_state(table: list[Any], state: list[Any]) -> tuple[Any, ...]:- turn, cached_tokens, cached_messages, original, forwarded = state+ if len(state) >= 6:+ turn, cached_tokens, cached_messages, original, forwarded, drop_upto = state+ else:+ turn, cached_tokens, cached_messages, original, forwarded = state+ drop_upto = 0return (int(turn),int(cached_tokens),int(cached_messages),_deref(table, original),_deref(table, forwarded),+ int(drop_upto),)@@ -2226,7 +2270,7 @@ class SessionTrackerStore:return memo[id(messages)]def state(s: tuple[Any, ...]) -> list[Any]:- return [s[0], s[1], s[2], ref(s[3]), ref(s[4])]+ return [s[0], s[1], s[2], ref(s[3]), ref(s[4]), s[5]]maturation = tracker.read_maturation_managermeta = {@@ -2298,6 +2342,7 @@ class SessionTrackerStore:tracker._cached_message_count,tracker._last_original_messages,tracker._last_forwarded_messages,+ tracker.reasoning_drop_upto,) = _decode_turn_state(table, meta["state"])tracker._turn_states = {int(t): _decode_turn_state(table, s) for t, s in meta["turns"]}tracker.restored_from_disk = Truediff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex 808f7b051..d94233f81 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -4827,16 +4827,35 @@ class OpenAIHandlerMixin:if _rc_router is not Noneelse None)- # Drop needs no compressor; Kompress (warm) does.- if _rc_kompress is not None or _rc_cold:+ # Sticky drop floor: turns dropped on a prior cold turn must stay+ # dropped on the next warm turn, or the prefix no longer matches+ # the cached cold-turn prefix and the cache misses (#4548).+ _rc_drop_upto = (+ openai_prefix_tracker.request_reasoning_drop_floor()+ if openai_prefix_tracker is not None+ else 0+ )+ # Drop needs no compressor; Kompress (warm) does. Sticky drop needs+ # neither — it just re-applies a previously-dropped shape.+ if _rc_kompress is not None or _rc_cold or _rc_drop_upto:_rc_keep = int(os.environ.get("HEADROOM_THINKING_COMPACT_KEEP_LAST", "1"))optimized_messages, _rc_stats = compact_reasoning_openai_chat(optimized_messages,kompress=_rc_kompress,keep_last_turns=_rc_keep,drop=_rc_cold,+ drop_first_turns=_rc_drop_upto,)body["messages"] = optimized_messages+ # On a cold turn drop=True drops every non-kept prior turn, so+ # record that count as the new sticky floor for the next warm+ # turn. Never shrink it — a later cold turn can only drop more.+ # Keep the increase on this request's resolved state so response+ # recording persists it without adopting a sibling fork's floor.+ if _rc_cold and openai_prefix_tracker is not None:+ openai_prefix_tracker.raise_reasoning_drop_floor(+ max(_rc_drop_upto, _rc_stats.get("dropped_upto", 0))+ )if _rc_stats["turns_compacted"]:transforms_applied.append(f"openai:reasoning_{'drop' if _rc_cold else 'compact'}:{_rc_stats['turns_compacted']}"diff --git a/headroom/transforms/thinking_compactor.py b/headroom/transforms/thinking_compactor.pyindex c03afd2e7..9e9ac4464 100644--- a/headroom/transforms/thinking_compactor.py+++ b/headroom/transforms/thinking_compactor.py@@ -250,6 +250,7 @@ def compact_reasoning_openai_chat(keep_last_turns: int = 1,min_words: int = 40,drop: bool = False,+ drop_first_turns: int = 0,) -> tuple[list[dict[str, Any]], dict[str, int]]:"""Compact plain-text reasoning in OpenAI-chat messages (Kimi / GLM / DeepSeek-R1).@@ -264,27 +265,46 @@ def compact_reasoning_openai_chat(``drop=False`` (warm): Kompress the reasoning (deterministic → cache-stable).``drop=True`` (cold-prefix hook): remove the reasoning outright — the fullblock, not just ~15% — safe because the cold turn re-caches from scratch.++ ``drop_first_turns`` (sticky drop): the first ``drop_first_turns`` assistant+ turns (0-based ordinal, excluding the kept last turns) use the DROP path even+ on a warm turn, so a turn that was dropped on a prior cold turn is dropped+ again rather than re-Kompressed — keeping the forwarded prefix byte-identical+ to the cached cold-turn prefix (#4548). ``dropped_upto`` in the returned stats+ is (highest dropped ordinal + 1), or 0 when nothing was dropped.+Shape-driven — no model gate; no-ops when no plain-text reasoning is present(OpenAI's encrypted models, Kimi k2.6). Keeps the last ``keep_last_turns``assistant turns intact (the active reasoning the model uses). Never raises."""- stats = {"turns_compacted": 0, "blocks": 0, "words_before": 0, "words_after": 0}- if kompress is None and not drop: # drop needs no compressor+ stats = {+ "turns_compacted": 0,+ "blocks": 0,+ "words_before": 0,+ "words_after": 0,+ "dropped_upto": 0,+ }+ if kompress is None and not drop and drop_first_turns == 0:return messages, statsasst_indices = [i for i, m in enumerate(messages) if m.get("role") == "assistant"]keep = set(asst_indices[-keep_last_turns:]) if keep_last_turns > 0 else set()+ ordinal_of = {idx: o for o, idx in enumerate(asst_indices)}out: list[dict[str, Any]] = []for i, m in enumerate(messages):if m.get("role") != "assistant" or i in keep:out.append(m)continue+ ordinal = ordinal_of[i]+ # Sticky drop: turns dropped on a prior cold turn are dropped again on+ # the next warm turn so the prefix stays byte-identical to the cache.+ use_drop = drop or (ordinal < drop_first_turns)changed = Falsenm = dict(m)# (1) Kimi: reasoning_content field (plain text, no signature → editable)rc = m.get("reasoning_content")if isinstance(rc, str) and len(rc.split()) >= min_words:- if drop: # cold hook: drop the whole reasoning block+ if use_drop: # cold hook: drop the whole reasoning blocknm["reasoning_content"] = ""changed = Truestats["blocks"] += 1@@ -300,7 +320,7 @@ def compact_reasoning_openai_chat(# (2) GLM / DeepSeek-R1: inline <think>…</think> in string contentc = m.get("content")if isinstance(c, str) and "<think>" in c:- new_c, b, wb, wa = _compact_think_spans(c, kompress, min_words, drop=drop)+ new_c, b, wb, wa = _compact_think_spans(c, kompress, min_words, drop=use_drop)if b:nm["content"] = new_cchanged = True@@ -309,6 +329,8 @@ def compact_reasoning_openai_chat(stats["words_after"] += waif changed:stats["turns_compacted"] += 1+ if use_drop:+ stats["dropped_upto"] = max(stats["dropped_upto"], ordinal + 1)out.append(nm)else:out.append(m)diff --git a/tests/test_thinking_compact_sticky_drop.py b/tests/test_thinking_compact_sticky_drop.pynew file mode 100644index 000000000..12940e2bf--- /dev/null+++ b/tests/test_thinking_compact_sticky_drop.py@@ -0,0 +1,183 @@+"""#4548: reasoning dropped on a cold turn stays dropped on the next warm turn."""++import contextvars+from types import SimpleNamespace++from headroom.transforms.thinking_compactor import compact_reasoning_openai_chat+++class _HalfKompress:+ def compress(self, text, allow_download=False):+ words = text.split()+ return SimpleNamespace(compressed=" ".join(words[: len(words) // 2]))+++def _history(n, tag):+ msgs = []+ for k in range(n):+ msgs.append({"role": "user", "content": f"q{k}"})+ reasoning = " ".join(f"{tag}{k}w{j}" for j in range(60))+ msgs.append({"role": "assistant", "content": f"a{k}", "reasoning_content": reasoning})+ return msgs+++def test_cold_drop_reports_dropped_upto():+ out, stats = compact_reasoning_openai_chat(+ _history(3, "c"), kompress=None, keep_last_turns=1, drop=True+ )+ assert [m.get("reasoning_content") for m in out[1::2]][:2] == ["", ""]+ assert stats["dropped_upto"] == 2+++def test_warm_turn_after_cold_reuses_cold_prefix():+ cold, stats = compact_reasoning_openai_chat(+ _history(3, "w"), kompress=None, keep_last_turns=1, drop=True+ )+ warm, _ = compact_reasoning_openai_chat(+ _history(4, "w"),+ kompress=_HalfKompress(),+ keep_last_turns=1,+ drop=False,+ drop_first_turns=stats["dropped_upto"],+ )+ assert warm[:4] == cold[:4] # previously dropped turns byte-identical+ assert 0 < len(warm[5]["reasoning_content"].split()) < 60 # newer turn Kompressed+ assert len(warm[7]["reasoning_content"].split()) == 60 # active turn kept+++def test_no_sticky_floor_is_unchanged():+ out, stats = compact_reasoning_openai_chat(+ _history(3, "u"), kompress=_HalfKompress(), keep_last_turns=1+ )+ assert all(len(m["reasoning_content"].split()) == 30 for m in out[1:4:2])+ assert stats["dropped_upto"] == 0+++def test_inline_think_spans_stay_dropped():+ think = "<think>" + " ".join(f"t{j}" for j in range(60)) + "</think>"+ msgs = [+ {"role": "user", "content": "q"},+ {"role": "assistant", "content": think + "a0"},+ {"role": "user", "content": "q"},+ {"role": "assistant", "content": think + "a1"},+ ]+ out, _ = compact_reasoning_openai_chat(+ msgs, kompress=_HalfKompress(), keep_last_turns=1, drop_first_turns=1+ )+ assert out[1]["content"] == "a0"+ assert out[3]["content"] == think + "a1"+++def _tracker_with_floor(floor):+ from headroom.cache.prefix_tracker import PrefixCacheTracker++ tracker = PrefixCacheTracker("openai")+ tracker.update_from_response(cache_read_tokens=0, cache_write_tokens=0, messages=[])+ tracker.reasoning_drop_upto = floor+ return tracker+++def test_floor_survives_shared_round_trip_and_warm_prefix_matches():+ from headroom.cache.prefix_tracker import SessionTrackerStore++ cold, stats = compact_reasoning_openai_chat(+ _history(3, "r"), kompress=None, keep_last_turns=1, drop=True+ )+ store = SessionTrackerStore()+ meta, table = store.encode_shared("k", _tracker_with_floor(stats["dropped_upto"]))+ restored = store.decode_shared(meta, table)+ assert restored.reasoning_drop_upto == 2+ warm, _ = compact_reasoning_openai_chat(+ _history(4, "r"),+ kompress=_HalfKompress(),+ keep_last_turns=1,+ drop_first_turns=restored.reasoning_drop_upto,+ )+ assert warm[:4] == cold[:4]+++def test_old_metadata_without_floor_decodes_to_zero():+ from headroom.cache.prefix_tracker import SessionTrackerStore++ store = SessionTrackerStore()+ meta, table = store.encode_shared("k", _tracker_with_floor(3))+ meta["state"] = meta["state"][:5]+ meta["turns"] = [[t, s[:5]] for t, s in meta["turns"]]+ assert store.decode_shared(meta, table).reasoning_drop_upto == 0+++def test_resumed_turn_restores_its_own_floor():+ tracker = _tracker_with_floor(2)+ tracker._turn_states[1] = tracker._turn_state()+ tracker.reasoning_drop_upto = 3 # a cold fork raised it+ tracker._resume_turn(1)+ assert tracker.reasoning_drop_upto == 2+++def test_request_floor_ignores_a_concurrent_fork_raising_the_live_floor():+ from headroom.cache.prefix_tracker import _resolved_turn++ tracker = _tracker_with_floor(2)+ tracker._turn_states[1] = tracker._turn_state()+ request_a = contextvars.copy_context()+ request_a.run(_resolved_turn.set, (tracker, 1, tracker._turn_states[1]))+ tracker.reasoning_drop_upto = 3 # sibling fork went cold after A resolved++ floor = request_a.run(tracker.request_reasoning_drop_floor)+ assert floor == 2+ out, _ = compact_reasoning_openai_chat(+ _history(4, "f"), kompress=_HalfKompress(), keep_last_turns=1, drop_first_turns=floor+ )+ assert len(out[5]["reasoning_content"].split()) == 30 # Kompressed, not dropped+++def test_cold_request_records_raised_floor_through_response():+ from headroom.cache.prefix_tracker import _resolved_turn++ tracker = _tracker_with_floor(0)+ tracker._turn_states[1] = tracker._turn_state()+ request = contextvars.copy_context()+ request.run(_resolved_turn.set, (tracker, 1, tracker._turn_states[1]))++ cold, stats = compact_reasoning_openai_chat(+ _history(3, "r"), kompress=None, keep_last_turns=1, drop=True+ )+ floor = max(request.run(tracker.request_reasoning_drop_floor), stats["dropped_upto"])+ request.run(tracker.raise_reasoning_drop_floor, floor)+ request.run(tracker.update_from_response, 0, 0, cold)++ assert tracker._turn_states[1][5] == 2+ tracker._resume_turn(1)+ warm, _ = compact_reasoning_openai_chat(+ _history(4, "r"),+ kompress=_HalfKompress(),+ keep_last_turns=1,+ drop=False,+ drop_first_turns=tracker.request_reasoning_drop_floor(),+ )+ assert [m.get("reasoning_content") for m in warm[1::2]][:2] == ["", ""]+ assert warm[:5] == cold[:5]+++def test_cold_request_floor_write_isolated_from_sibling_through_response():+ from headroom.cache.prefix_tracker import _resolved_turn++ tracker = _tracker_with_floor(2)+ tracker._turn_states[1] = tracker._turn_state()+ request_a = contextvars.copy_context()+ request_a.run(_resolved_turn.set, (tracker, 1, tracker._turn_states[1]))+ tracker.reasoning_drop_upto = 3 # sibling fork mutated live floor++ floor = request_a.run(tracker.request_reasoning_drop_floor)+ assert floor == 2+ request_a.run(tracker.update_from_response, 0, 0, [])++ assert tracker._turn_states[1][5] == 2+ tracker._resume_turn(1)+ out, _ = compact_reasoning_openai_chat(+ _history(4, "s"),+ kompress=_HalfKompress(),+ keep_last_turns=1,+ drop_first_turns=tracker.request_reasoning_drop_floor(),+ )+ assert len(out[5]["reasoning_content"].split()) == 30 # Kompressed, not dropped