Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(transforms/kompress): preserve line boundaries and tabular output in Kompress (#3119)

slxng1758fe2ed2b545e4
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.py
index 49588eac4..5b679b658 100644
--- a/headroom/transforms/kompress_compressor.py
+++ b/headroom/transforms/kompress_compressor.py
@@ -17,6 +17,7 @@ from __future__ import annotations
import contextlib
import gc
import hashlib
+import itertools
import logging
import os
import re
@@ -98,6 +99,56 @@ def _add_kompress_must_keep_words(
kept_ids.add(word_idx + chunk_start)
+_TABULAR_ROW_SPLIT_RE = re.compile(r"\s{2,}")
+
+
+def _words_by_line(content: str) -> tuple[list[str], list[int]]:
+ """Split into words like content.split(), plus each word's source line index.
+
+ Reconstruction must never join words that came from different source lines
+ onto one line -- doing so can glue unrelated facts together (#3099).
+ """
+ words: list[str] = []
+ line_ids: list[int] = []
+ for line_idx, line in enumerate(content.split("\n")):
+ for word in line.split():
+ words.append(word)
+ line_ids.append(line_idx)
+ return words, line_ids
+
+
+def _join_kept_words_by_line(words: list[str], line_ids: list[int], kept_ids: set[int]) -> str:
+ """Rejoin kept words, starting a new line wherever they crossed a line boundary."""
+ parts: list[str] = []
+ prev_line: int | None = None
+ for i in sorted(kept_ids):
+ if prev_line is not None:
+ parts.append("\n" if line_ids[i] != prev_line else " ")
+ parts.append(words[i])
+ prev_line = line_ids[i]
+ return "".join(parts)
+
+
+def _tabular_protected_line_ids(content: str) -> set[int]:
+ """Flag lines in a run of >=3 consecutive lines with a consistent fixed-width
+ column count (ls -l, ps, etc.), so compression keeps each such row whole
+ instead of dropping individual fields inconsistently row to row (#3099).
+ Shares the must-keep toggle: HEADROOM_KOMPRESS_MUST_KEEP=0 disables this too.
+ """
+ if os.environ.get(_KOMPRESS_MUST_KEEP_ENV, "1") == "0":
+ return set()
+ cols_per_line = [
+ len(_TABULAR_ROW_SPLIT_RE.split(line.strip())) if line.strip() else 0
+ for line in content.split("\n")
+ ]
+ protected: set[int] = set()
+ for cols, group in itertools.groupby(enumerate(cols_per_line), key=lambda p: p[1]):
+ indices = [idx for idx, _ in group]
+ if cols >= 3 and len(indices) >= 3:
+ protected.update(indices)
+ return protected
+
+
# ONNX artifacts are resolved against the model repo in this order, falling
# through on download miss OR session-load failure:
#
@@ -1602,12 +1653,14 @@ class KompressCompressor(Transform):
KompressResult with compressed text.
"""
t_deadline = time.perf_counter() if _deadline_started_at is None else _deadline_started_at
- words = content.split()
+ words, line_ids = _words_by_line(content)
n_words = len(words)
if n_words < max(10, self.config.min_input_words) or self._degraded_reason is not None:
return self._passthrough(content, n_words)
+ protected_line_ids = _tabular_protected_line_ids(content)
+
# Cooperative wall-clock budget (#1171): kompress ONNX inference is
# O(tokens) and non-preemptible once the request's asyncio timeout fires,
# so one large block can run for minutes holding a worker (the leak ->
@@ -1788,6 +1841,10 @@ class KompressCompressor(Transform):
# reconstruct from context. Disable via HEADROOM_KOMPRESS_MUST_KEEP=0.
_add_kompress_must_keep_words(kept_ids, chunk_words, chunk_start)
+ for i, line_id in enumerate(line_ids):
+ if line_id in protected_line_ids:
+ kept_ids.add(i)
+
if not kept_ids:
if inference_ms >= 1000.0:
logger.info(
@@ -1802,7 +1859,7 @@ class KompressCompressor(Transform):
return self._passthrough(content, n_words)
compressed_words = [words[w] for w in sorted(kept_ids) if w < n_words]
- compressed = " ".join(compressed_words)
+ compressed = _join_kept_words_by_line(words, line_ids, kept_ids)
compressed_count = len(compressed_words)
cache_key: str | None = None
original_tokens = n_words
@@ -2021,7 +2078,14 @@ class KompressCompressor(Transform):
]
results: list[KompressResult | None] = [None] * n
- word_lists: list[list[str]] = [c.split() for c in contents]
+ word_lists: list[list[str]] = []
+ line_id_lists: list[list[int]] = []
+ protected_line_ids_list: list[set[int]] = []
+ for c in contents:
+ words, line_ids = _words_by_line(c)
+ word_lists.append(words)
+ line_id_lists.append(line_ids)
+ protected_line_ids_list.append(_tabular_protected_line_ids(c))
# Short texts short-circuit to passthrough — no model call needed.
max_chunk_words = self.config.chunk_words
@@ -2203,14 +2267,19 @@ class KompressCompressor(Transform):
continue
content = contents[text_idx]
words = word_lists[text_idx]
+ line_ids = line_id_lists[text_idx]
n_words = len(words)
+ for i, line_id in enumerate(line_ids):
+ if line_id in protected_line_ids_list[text_idx]:
+ kept_ids.add(i)
+
if not kept_ids:
results[text_idx] = self._passthrough(content, n_words)
continue
compressed_words = [words[w] for w in sorted(kept_ids) if w < n_words]
- compressed = " ".join(compressed_words)
+ compressed = _join_kept_words_by_line(words, line_ids, kept_ids)
compressed_count = len(compressed_words)
cache_key: str | None = None
original_tokens = n_words
diff --git a/tests/test_kompress_line_integrity.py b/tests/test_kompress_line_integrity.py
new file mode 100644
index 000000000..076f84c84
--- /dev/null
+++ b/tests/test_kompress_line_integrity.py
@@ -0,0 +1,108 @@
+"""Regression tests for issue #3099: compression merging words across line
+boundaries, and dropping fields from tabular rows inconsistently row to row.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from headroom.transforms import kompress_compressor as kc
+from headroom.transforms.kompress_compressor import KompressCompressor, KompressConfig
+
+
+class _Enc(dict):
+ def word_ids(self, batch_index=0):
+ return self["_word_ids"][batch_index]
+
+
+class _Tok:
+ def __call__(self, chunk_words, **kw):
+ if chunk_words and isinstance(chunk_words[0], list):
+ batch_words = chunk_words
+ else:
+ batch_words = [chunk_words]
+ return _Enc(
+ input_ids=[[0] * len(words) for words in batch_words],
+ attention_mask=[[1] * len(words) for words in batch_words],
+ _word_ids=[list(range(len(words))) for words in batch_words],
+ )
+
+
+class _KeepIndicesModel:
+ """Fake model that keeps exactly the given (global, single-chunk) word indices."""
+
+ def __init__(self, keep_indices: set[int]):
+ self._keep = keep_indices
+
+ def get_keep_mask(self, input_ids, attention_mask):
+ return [[idx in self._keep for idx in range(len(row))] for row in input_ids]
+
+ def get_scores(self, input_ids, attention_mask):
+ return [[1.0 if idx in self._keep else 0.0 for idx in range(len(row))] for row in input_ids]
+
+
+def _install_fake_kompress(monkeypatch, keep_indices: set[int], *, must_keep: bool = False) -> None:
+ model = _KeepIndicesModel(keep_indices)
+ monkeypatch.setattr(kc, "_load_kompress", lambda *a, **k: (model, _Tok(), "onnx"))
+ monkeypatch.setattr(kc, "_model_device_type", lambda *a, **k: "cpu")
+ if must_keep:
+ monkeypatch.delenv(kc._KOMPRESS_MUST_KEEP_ENV, raising=False)
+ else:
+ monkeypatch.setenv(kc._KOMPRESS_MUST_KEEP_ENV, "0")
+
+
+def _compress(monkeypatch, content: str, path: str):
+ compressor = KompressCompressor(KompressConfig(enable_ccr=False, min_input_words=10))
+ if path == "compress":
+ monkeypatch.setattr(compressor, "_should_batch_single_content", lambda *a, **k: False)
+ return compressor.compress(content)
+ monkeypatch.setattr(compressor, "_should_use_sequential_fallback", lambda: False)
+ [result] = compressor.compress_batch([content], batch_size=8)
+ return result
+
+
[email protected]("path", ["compress", "compress_batch"])
+def test_compress_does_not_merge_words_across_lines(monkeypatch, path):
+ """Reproduces #3099 bug 1: two commands' output on separate lines.
+
+ The model keeps only the last word of line 1 ("rg") and the first word of
+ line 2 ("/opt/homebrew/bin/ast-grep"). Before the fix, these landed adjacent
+ in the flat kept-word list and were joined with a space onto one line,
+ fabricating "rg /opt/homebrew/bin/ast-grep" -- read literally, that implies
+ rg resolves to that path, which is false.
+ """
+ line1 = "alpha beta gamma delta epsilon zeta eta rg"
+ line2 = "/opt/homebrew/bin/ast-grep theta iota kappa lambda mu nu xi"
+ content = f"{line1}\n{line2}"
+
+ _install_fake_kompress(monkeypatch, keep_indices={7, 8})
+
+ result = _compress(monkeypatch, content, path)
+
+ assert "rg /opt/homebrew/bin/ast-grep" not in result.compressed
+ assert result.compressed.split("\n") == ["rg", "/opt/homebrew/bin/ast-grep"]
+
+
[email protected]("path", ["compress", "compress_batch"])
+def test_compress_keeps_tabular_row_intact(monkeypatch, path):
+ """Reproduces #3099 bug 2: an `ls -la`-shaped listing.
+
+ The model keeps nothing on its own, so absent row protection only the
+ must-keep regex would save permission bits/numbers/filenames -- exactly
+ the reported symptom of some fields (the group column, the month)
+ disappearing inconsistently. Once a row is part of a >=3-row run of
+ consistent fixed-width columns, the whole row must survive intact.
+ """
+ rows = [
+ "-rw-r--r--@ 1 user staff 4921 Aug 12 20:14 daemon.log",
+ "-rw-r--r--@ 1 user staff 10485 Aug 12 20:14 other.log",
+ "-rw-r--r--@ 1 user staff 512 Aug 12 20:15 third.log",
+ ]
+ content = "\n".join(rows)
+
+ _install_fake_kompress(monkeypatch, keep_indices=set(), must_keep=True)
+
+ result = _compress(monkeypatch, content, path)
+
+ expected = "\n".join(" ".join(row.split()) for row in rows)
+ assert result.compressed == expected