Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(thinking): isolate compaction memo by compressor instance (#4569)

Yuchuan He5dcb1e4adec4
diff --git a/headroom/transforms/thinking_compactor.py b/headroom/transforms/thinking_compactor.py
index e279b7a68..46fda82d9 100644
--- a/headroom/transforms/thinking_compactor.py
+++ b/headroom/transforms/thinking_compactor.py
@@ -30,16 +30,18 @@ from __future__ import annotations
import hashlib
import logging
+import weakref
from collections import OrderedDict
from typing import Any
log = logging.getLogger(__name__)
-# original-thinking-hash -> compacted text. Deterministic memo so the same
+# (compressor identity, original-thinking-hash) -> (weak reference, compacted text).
+# Deterministic memo so the same
# thinking always yields the same bytes (cache stability), even if Kompress is
# nondeterministic. Bounded; ONNX Kompress is deterministic so eviction+recompute
# is byte-identical anyway. ponytail: crude LRU cap, fine given determinism.
-_COMPACT_CACHE: OrderedDict[str, str] = OrderedDict()
+_COMPACT_CACHE: OrderedDict[tuple[int, str], tuple[weakref.ReferenceType[Any], str]] = OrderedDict()
_CACHE_CAP = 8192
# Prefix on the emitted text block so the compaction is legible to the model
@@ -77,11 +79,12 @@ def bills_prior_thinking(model: str) -> bool:
def _memo_compact(text: str, kompress: Any) -> str | None:
"""Deterministically compact ``text`` via Kompress; None if no gain/failure."""
- key = hashlib.sha1(text.encode("utf-8", "replace")).hexdigest()
+ key = (id(kompress), hashlib.sha1(text.encode("utf-8", "replace")).hexdigest())
cached = _COMPACT_CACHE.get(key)
- if cached is not None:
+ # Verify identity as well as id: a collected compressor's id may be reused.
+ if cached is not None and cached[0]() is kompress:
_COMPACT_CACHE.move_to_end(key)
- return cached
+ return cached[1]
try:
# allow_download=False: never block the request thread on a cold model
# (mirrors the proxy convention in ContentRouter._get_kompress callers).
@@ -92,7 +95,13 @@ def _memo_compact(text: str, kompress: Any) -> str | None:
return None
if not isinstance(compacted, str) or not compacted:
return None
- _COMPACT_CACHE[key] = compacted
+ try:
+ compressor_ref = weakref.ref(kompress)
+ except TypeError:
+ # Duck-typed compressors need not support weak references. Do not keep
+ # model instances alive just to memoize their output.
+ return compacted
+ _COMPACT_CACHE[key] = (compressor_ref, compacted)
_COMPACT_CACHE.move_to_end(key)
while len(_COMPACT_CACHE) > _CACHE_CAP:
_COMPACT_CACHE.popitem(last=False)
diff --git a/tests/test_thinking_compactor_cache.py b/tests/test_thinking_compactor_cache.py
new file mode 100644
index 000000000..b0eb2e94e
--- /dev/null
+++ b/tests/test_thinking_compactor_cache.py
@@ -0,0 +1,88 @@
+"""Compaction memoization must respect the compressor selected by the caller."""
+
+from __future__ import annotations
+
+import gc
+import weakref
+from types import SimpleNamespace
+
+import pytest
+
+from headroom.transforms import thinking_compactor as tc
+
+
+class Compressor:
+ def __init__(self, output):
+ self.output = output
+ self.calls = 0
+
+ def compress(self, text, **kwargs):
+ self.calls += 1
+ return SimpleNamespace(compressed=self.output)
+
+
[email protected](autouse=True)
+def clear_cache():
+ tc._COMPACT_CACHE.clear()
+ yield
+ tc._COMPACT_CACHE.clear()
+
+
+def compact(compressor, shape="field"):
+ reasoning = "reasoning " * 60
+ if shape == "field":
+ messages = [{"role": "assistant", "content": "answer", "reasoning_content": reasoning}]
+ result, _ = tc.compact_reasoning_openai_chat(
+ messages, kompress=compressor, keep_last_turns=0
+ )
+ return result[0]["reasoning_content"]
+ messages = [{"role": "assistant", "content": [{"type": "thinking", "thinking": reasoning}]}]
+ result, _ = tc.compact_thinking_to_text(messages, kompress=compressor, keep_last_turns=0)
+ return result[0]["content"][0]["text"]
+
+
[email protected]("shape", ["field", "thinking"])
+def test_different_compressors_do_not_share_compacted_output(shape):
+ first = Compressor("first backend summary")
+ second = Compressor("second backend summary")
+
+ assert "first backend summary" in compact(first, shape)
+ assert "second backend summary" in compact(second, shape)
+ assert first.calls == second.calls == 1
+
+
+def test_repeated_calls_keep_the_selected_compressors_memo():
+ compressor = Compressor("stable summary")
+ assert compact(compressor) == compact(compressor)
+ assert compressor.calls == 1
+
+
+def test_cache_does_not_keep_unused_compressors_alive():
+ compressor = Compressor("summary")
+ compact(compressor)
+ reference = weakref.ref(compressor)
+ del compressor
+ gc.collect()
+ assert reference() is None
+
+
+def test_compressor_without_weakref_support_still_works():
+ class SlottedCompressor:
+ __slots__ = ()
+
+ def compress(self, text, **kwargs):
+ return SimpleNamespace(compressed="slotted summary")
+
+ compressor = SlottedCompressor()
+ assert compact(compressor) == "slotted summary"
+ assert compact(compressor) == "slotted summary"
+
+
+def test_cache_limit_is_shared_across_compressors(monkeypatch):
+ monkeypatch.setattr(tc, "_CACHE_CAP", 1)
+ first, second = Compressor("first summary"), Compressor("second summary")
+ compact(first)
+ compact(second)
+ assert len(tc._COMPACT_CACHE) == 1
+ assert compact(first) == "first summary"
+ assert first.calls == 2