Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(kompress): let the next window decide the words past the token cut (#4669)
gglucassbfbadec8f9f5
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex 08c74e33f..3cd98a241 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -87,8 +87,6 @@ _KOMPRESS_MUST_KEEP_RE = re.compile(r"|(?i:\b(?:and|or|nor|xor)\b)")_KOMPRESS_MUST_KEEP_ENV = "HEADROOM_KOMPRESS_MUST_KEEP"-#: Tokenizer budget per chunk. Words past it are never scored (#4309).-_KOMPRESS_MAX_TOKENS = 512KOMPRESS_BACKEND_ENV = "HEADROOM_KOMPRESS_BACKEND"KOMPRESS_ONNX_FILENAME_ENV = "HEADROOM_KOMPRESS_ONNX_FILENAME"@@ -144,39 +142,6 @@ def _add_kompress_must_keep_words(kept_ids.add(word_idx + chunk_start)-def _add_truncated_words(- kept_ids: set[int],- word_ids: list[int | None],- chunk_words: list[str],- chunk_start: int,- token_count: int,-) -> None:- """Keep the words the tokenizer's token cut took away.-- Chunks are cut by words (``chunk_words``) but tokenized with- ``truncation=True, max_length=512``. Words past token 512 get no- ``word_id``, so they are never scored and never make ``kept_ids`` -- they- are dropped silently, regardless of importance, and the loss is- positional: whatever sits at the end of the chunk disappears (#4309).-- The cut can also land inside a word. Its visible prefix counts as- covered, so the model scores part of a word and may drop the whole of- it, taking the unseen suffix with it. The last covered word is kept as- well whenever the chunk reached the token limit.- """- covered = max((wid for wid in word_ids if wid is not None), default=-1) + 1- first_dropped = covered- if token_count >= _KOMPRESS_MAX_TOKENS and covered > 0:- first_dropped = covered - 1- for local_idx in range(first_dropped, len(chunk_words)):- kept_ids.add(local_idx + chunk_start)---def _real_token_count(attention_row: Any) -> int:- """Number of real (non-padding) tokens in an attention-mask row."""- return sum(1 for mask in attention_row if mask)--_TABULAR_ROW_SPLIT_RE = re.compile(r"\s{2,}")@@ -2094,7 +2059,6 @@ class KompressCompressor(Transform):input_ids = encoding["input_ids"]attention_mask = encoding["attention_mask"]word_ids = encoding.word_ids(batch_index=0)- token_count = _real_token_count(attention_mask[0])# One row, so no padding: every token in it is real.scored = _words_seen_whole(word_ids, len(word_ids), len(chunk_words))if scored == 0:@@ -2193,11 +2157,6 @@ class KompressCompressor(Transform):continueif bool(mask_list[idx]):kept_ids.add(wid + chunk_start)- # The words past the tokenizer's token cut were never- # scored, so the model cannot have kept them. A forced- # ratio is a hard budget the caller asked for, so only the- # model-decides path takes them back (#4309).- _add_truncated_words(kept_ids, word_ids, chunk_words, chunk_start, token_count)# Hard override: always keep must-keep tokens regardless of model score.# Numbers, error names, paths, and flags carry meaning agents cannot@@ -2632,17 +2591,6 @@ class KompressCompressor(Transform):for wid, score in word_scores.items():if score > self.config.score_threshold:kept_ids_per_text[text_idx].add(wid + chunk_start)- # The words past the tokenizer's token cut were never- # scored, so the model cannot have kept them. A forced- # ratio is a hard budget the caller asked for, so only- # the model-decides path takes them back (#4309).- _add_truncated_words(- kept_ids_per_text[text_idx],- word_ids,- chunk_words,- chunk_start,- _real_token_count(attention_mask[batch_idx]),- )_add_kompress_must_keep_words(kept_ids_per_text[text_idx], chunk_words, chunk_startdiff --git a/tests/test_kompress_must_keep.py b/tests/test_kompress_must_keep.pyindex b0e991ab2..e596d8b27 100644--- a/tests/test_kompress_must_keep.py+++ b/tests/test_kompress_must_keep.py@@ -233,13 +233,15 @@ class _TruncatingTok:class TestTruncatedWordsCompression:- """Words past the tokenizer's 512-token cut are kept, not dropped (#4309).+ """Words past the tokenizer's 512-token cut are never dropped unseen (#4309).Chunks are cut by words (``chunk_words``) but tokenized with``truncation=True, max_length=512``. A chunk of token-heavy words hitsthe token limit well before its word budget, so the tail of the chunk hadno score and was dropped silently from the default proxy path- (``target_ratio=None``).+ (``target_ratio=None``). The sequential path now scores that tail as the+ next window and the model's decision on it stands; the batch path builds+ its chunks before tokenizing, so it keeps the tail verbatim instead."""# One chunk is `chunk_words` words long, so 350 words is exactly one.@@ -265,22 +267,24 @@ class TestTruncatedWordsCompression:kept = compressor.compress(" ".join(words)).compressed.split()return kept, words- def test_words_past_the_truncation_point_survive(self, monkeypatch):+ def test_words_past_the_truncation_point_are_scored(self, monkeypatch):# Two tokens per word: the 350-word chunk needs 700 tokens, so the- # tokenizer cuts it at 512. The 256 words it covered were scored, the- # other 94 were never seen and used to vanish from the output.+ # tokenizer cuts it at 512. Words 0-254 are scored in the first+ # window; word 255 (the cut may have split it) and the 94 the+ # tokenizer never saw start the next window. The fake model keeps the+ # first word of each window, so the second window's decision decides+ # their fate, not a blanket keep.kept, words = self._compress(monkeypatch, tokens_per_word=2)- # 256 words were covered; the cut landed at a word boundary here, but- # the last covered word is kept too, because the cut may have fallen- # inside it.- assert kept == [words[0]] + words[255:]+ assert kept == [words[0], words[255]]- def test_the_word_the_cut_landed_in_is_kept(self, monkeypatch):+ def test_the_word_the_cut_landed_in_is_scored_whole(self, monkeypatch):# Three tokens per word: the cut at 512 falls inside word 170, whose# last token never reached the tokenizer. The model only saw part of- # that word, so dropping it would take its unseen suffix with it.+ # that word, so it is not judged there: it opens the next window and+ # is scored whole. That window is cut inside word 340 the same way,+ # and the last ten words fit in a third.kept, words = self._compress(monkeypatch, tokens_per_word=3)- assert kept == [words[0]] + words[170:]+ assert kept == [words[0], words[170], words[340]]def test_fully_covered_chunk_is_unchanged(self, monkeypatch):# One token per word: the chunk fits in 512 tokens, so nothing was