Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(kompress): let the next window decide the words past the token cut (#4669)

gglucassbfbadec8f9f5
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 08c74e33f..3cd98a241 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -87,8 +87,6 @@ _KOMPRESS_MUST_KEEP_RE = re.compile(
r"|(?i:\b(?:and|or|nor|xor)\b)"
)
_KOMPRESS_MUST_KEEP_ENV = "HEADROOM_KOMPRESS_MUST_KEEP"
-#: Tokenizer budget per chunk. Words past it are never scored (#4309).
-_KOMPRESS_MAX_TOKENS = 512
KOMPRESS_BACKEND_ENV = "HEADROOM_KOMPRESS_BACKEND"
KOMPRESS_ONNX_FILENAME_ENV = "HEADROOM_KOMPRESS_ONNX_FILENAME"
@@ -144,39 +142,6 @@ def _add_kompress_must_keep_words(
kept_ids.add(word_idx + chunk_start)
-def _add_truncated_words(
- kept_ids: set[int],
- word_ids: list[int | None],
- chunk_words: list[str],
- chunk_start: int,
- token_count: int,
-) -> None:
- """Keep the words the tokenizer's token cut took away.
-
- Chunks are cut by words (``chunk_words``) but tokenized with
- ``truncation=True, max_length=512``. Words past token 512 get no
- ``word_id``, so they are never scored and never make ``kept_ids`` -- they
- are dropped silently, regardless of importance, and the loss is
- positional: whatever sits at the end of the chunk disappears (#4309).
-
- The cut can also land inside a word. Its visible prefix counts as
- covered, so the model scores part of a word and may drop the whole of
- it, taking the unseen suffix with it. The last covered word is kept as
- well whenever the chunk reached the token limit.
- """
- covered = max((wid for wid in word_ids if wid is not None), default=-1) + 1
- first_dropped = covered
- if token_count >= _KOMPRESS_MAX_TOKENS and covered > 0:
- first_dropped = covered - 1
- for local_idx in range(first_dropped, len(chunk_words)):
- kept_ids.add(local_idx + chunk_start)
-
-
-def _real_token_count(attention_row: Any) -> int:
- """Number of real (non-padding) tokens in an attention-mask row."""
- return sum(1 for mask in attention_row if mask)
-
-
_TABULAR_ROW_SPLIT_RE = re.compile(r"\s{2,}")
@@ -2094,7 +2059,6 @@ class KompressCompressor(Transform):
input_ids = encoding["input_ids"]
attention_mask = encoding["attention_mask"]
word_ids = encoding.word_ids(batch_index=0)
- token_count = _real_token_count(attention_mask[0])
# One row, so no padding: every token in it is real.
scored = _words_seen_whole(word_ids, len(word_ids), len(chunk_words))
if scored == 0:
@@ -2193,11 +2157,6 @@ class KompressCompressor(Transform):
continue
if bool(mask_list[idx]):
kept_ids.add(wid + chunk_start)
- # The words past the tokenizer's token cut were never
- # scored, so the model cannot have kept them. A forced
- # ratio is a hard budget the caller asked for, so only the
- # model-decides path takes them back (#4309).
- _add_truncated_words(kept_ids, word_ids, chunk_words, chunk_start, token_count)
# Hard override: always keep must-keep tokens regardless of model score.
# Numbers, error names, paths, and flags carry meaning agents cannot
@@ -2632,17 +2591,6 @@ class KompressCompressor(Transform):
for wid, score in word_scores.items():
if score > self.config.score_threshold:
kept_ids_per_text[text_idx].add(wid + chunk_start)
- # The words past the tokenizer's token cut were never
- # scored, so the model cannot have kept them. A forced
- # ratio is a hard budget the caller asked for, so only
- # the model-decides path takes them back (#4309).
- _add_truncated_words(
- kept_ids_per_text[text_idx],
- word_ids,
- chunk_words,
- chunk_start,
- _real_token_count(attention_mask[batch_idx]),
- )
_add_kompress_must_keep_words(
kept_ids_per_text[text_idx], chunk_words, chunk_start
diff --git a/tests/test_kompress_must_keep.py b/tests/test_kompress_must_keep.py
index b0e991ab2..e596d8b27 100644
--- a/tests/test_kompress_must_keep.py
+++ b/tests/test_kompress_must_keep.py
@@ -233,13 +233,15 @@ class _TruncatingTok:
class TestTruncatedWordsCompression:
- """Words past the tokenizer's 512-token cut are kept, not dropped (#4309).
+ """Words past the tokenizer's 512-token cut are never dropped unseen (#4309).
Chunks are cut by words (``chunk_words``) but tokenized with
``truncation=True, max_length=512``. A chunk of token-heavy words hits
the token limit well before its word budget, so the tail of the chunk had
no score and was dropped silently from the default proxy path
- (``target_ratio=None``).
+ (``target_ratio=None``). The sequential path now scores that tail as the
+ next window and the model's decision on it stands; the batch path builds
+ its chunks before tokenizing, so it keeps the tail verbatim instead.
"""
# One chunk is `chunk_words` words long, so 350 words is exactly one.
@@ -265,22 +267,24 @@ class TestTruncatedWordsCompression:
kept = compressor.compress(" ".join(words)).compressed.split()
return kept, words
- def test_words_past_the_truncation_point_survive(self, monkeypatch):
+ def test_words_past_the_truncation_point_are_scored(self, monkeypatch):
# Two tokens per word: the 350-word chunk needs 700 tokens, so the
- # tokenizer cuts it at 512. The 256 words it covered were scored, the
- # other 94 were never seen and used to vanish from the output.
+ # tokenizer cuts it at 512. Words 0-254 are scored in the first
+ # window; word 255 (the cut may have split it) and the 94 the
+ # tokenizer never saw start the next window. The fake model keeps the
+ # first word of each window, so the second window's decision decides
+ # their fate, not a blanket keep.
kept, words = self._compress(monkeypatch, tokens_per_word=2)
- # 256 words were covered; the cut landed at a word boundary here, but
- # the last covered word is kept too, because the cut may have fallen
- # inside it.
- assert kept == [words[0]] + words[255:]
+ assert kept == [words[0], words[255]]
- def test_the_word_the_cut_landed_in_is_kept(self, monkeypatch):
+ def test_the_word_the_cut_landed_in_is_scored_whole(self, monkeypatch):
# Three tokens per word: the cut at 512 falls inside word 170, whose
# last token never reached the tokenizer. The model only saw part of
- # that word, so dropping it would take its unseen suffix with it.
+ # that word, so it is not judged there: it opens the next window and
+ # is scored whole. That window is cut inside word 340 the same way,
+ # and the last ten words fit in a third.
kept, words = self._compress(monkeypatch, tokens_per_word=3)
- assert kept == [words[0]] + words[170:]
+ assert kept == [words[0], words[170], words[340]]
def test_fully_covered_chunk_is_unchanged(self, monkeypatch):
# One token per word: the chunk fits in 512 tokens, so nothing was