Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(thinking): isolate compaction memo by compressor instance (#4569)
Yuchuan He5dcb1e4adec4
diff --git a/headroom/transforms/thinking_compactor.py b/headroom/transforms/thinking_compactor.pyindex e279b7a68..46fda82d9 100644--- a/headroom/transforms/thinking_compactor.py+++ b/headroom/transforms/thinking_compactor.py@@ -30,16 +30,18 @@ from __future__ import annotationsimport hashlibimport logging+import weakreffrom collections import OrderedDictfrom typing import Anylog = logging.getLogger(__name__)-# original-thinking-hash -> compacted text. Deterministic memo so the same+# (compressor identity, original-thinking-hash) -> (weak reference, compacted text).+# Deterministic memo so the same# thinking always yields the same bytes (cache stability), even if Kompress is# nondeterministic. Bounded; ONNX Kompress is deterministic so eviction+recompute# is byte-identical anyway. ponytail: crude LRU cap, fine given determinism.-_COMPACT_CACHE: OrderedDict[str, str] = OrderedDict()+_COMPACT_CACHE: OrderedDict[tuple[int, str], tuple[weakref.ReferenceType[Any], str]] = OrderedDict()_CACHE_CAP = 8192# Prefix on the emitted text block so the compaction is legible to the model@@ -77,11 +79,12 @@ def bills_prior_thinking(model: str) -> bool:def _memo_compact(text: str, kompress: Any) -> str | None:"""Deterministically compact ``text`` via Kompress; None if no gain/failure."""- key = hashlib.sha1(text.encode("utf-8", "replace")).hexdigest()+ key = (id(kompress), hashlib.sha1(text.encode("utf-8", "replace")).hexdigest())cached = _COMPACT_CACHE.get(key)- if cached is not None:+ # Verify identity as well as id: a collected compressor's id may be reused.+ if cached is not None and cached[0]() is kompress:_COMPACT_CACHE.move_to_end(key)- return cached+ return cached[1]try:# allow_download=False: never block the request thread on a cold model# (mirrors the proxy convention in ContentRouter._get_kompress callers).@@ -92,7 +95,13 @@ def _memo_compact(text: str, kompress: Any) -> str | None:return Noneif not isinstance(compacted, str) or not compacted:return None- _COMPACT_CACHE[key] = compacted+ try:+ compressor_ref = weakref.ref(kompress)+ except TypeError:+ # Duck-typed compressors need not support weak references. Do not keep+ # model instances alive just to memoize their output.+ return compacted+ _COMPACT_CACHE[key] = (compressor_ref, compacted)_COMPACT_CACHE.move_to_end(key)while len(_COMPACT_CACHE) > _CACHE_CAP:_COMPACT_CACHE.popitem(last=False)diff --git a/tests/test_thinking_compactor_cache.py b/tests/test_thinking_compactor_cache.pynew file mode 100644index 000000000..b0eb2e94e--- /dev/null+++ b/tests/test_thinking_compactor_cache.py@@ -0,0 +1,88 @@+"""Compaction memoization must respect the compressor selected by the caller."""++from __future__ import annotations++import gc+import weakref+from types import SimpleNamespace++import pytest++from headroom.transforms import thinking_compactor as tc+++class Compressor:+ def __init__(self, output):+ self.output = output+ self.calls = 0++ def compress(self, text, **kwargs):+ self.calls += 1+ return SimpleNamespace(compressed=self.output)++[email protected](autouse=True)+def clear_cache():+ tc._COMPACT_CACHE.clear()+ yield+ tc._COMPACT_CACHE.clear()+++def compact(compressor, shape="field"):+ reasoning = "reasoning " * 60+ if shape == "field":+ messages = [{"role": "assistant", "content": "answer", "reasoning_content": reasoning}]+ result, _ = tc.compact_reasoning_openai_chat(+ messages, kompress=compressor, keep_last_turns=0+ )+ return result[0]["reasoning_content"]+ messages = [{"role": "assistant", "content": [{"type": "thinking", "thinking": reasoning}]}]+ result, _ = tc.compact_thinking_to_text(messages, kompress=compressor, keep_last_turns=0)+ return result[0]["content"][0]["text"]++[email protected]("shape", ["field", "thinking"])+def test_different_compressors_do_not_share_compacted_output(shape):+ first = Compressor("first backend summary")+ second = Compressor("second backend summary")++ assert "first backend summary" in compact(first, shape)+ assert "second backend summary" in compact(second, shape)+ assert first.calls == second.calls == 1+++def test_repeated_calls_keep_the_selected_compressors_memo():+ compressor = Compressor("stable summary")+ assert compact(compressor) == compact(compressor)+ assert compressor.calls == 1+++def test_cache_does_not_keep_unused_compressors_alive():+ compressor = Compressor("summary")+ compact(compressor)+ reference = weakref.ref(compressor)+ del compressor+ gc.collect()+ assert reference() is None+++def test_compressor_without_weakref_support_still_works():+ class SlottedCompressor:+ __slots__ = ()++ def compress(self, text, **kwargs):+ return SimpleNamespace(compressed="slotted summary")++ compressor = SlottedCompressor()+ assert compact(compressor) == "slotted summary"+ assert compact(compressor) == "slotted summary"+++def test_cache_limit_is_shared_across_compressors(monkeypatch):+ monkeypatch.setattr(tc, "_CACHE_CAP", 1)+ first, second = Compressor("first summary"), Compressor("second summary")+ compact(first)+ compact(second)+ assert len(tc._COMPACT_CACHE) == 1+ assert compact(first) == "first summary"+ assert first.calls == 2