Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(transforms/kompress): preserve line boundaries and tabular output in Kompress (#3119)
slxng1758fe2ed2b545e4
diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex 49588eac4..5b679b658 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -17,6 +17,7 @@ from __future__ import annotationsimport contextlibimport gcimport hashlib+import itertoolsimport loggingimport osimport re@@ -98,6 +99,56 @@ def _add_kompress_must_keep_words(kept_ids.add(word_idx + chunk_start)+_TABULAR_ROW_SPLIT_RE = re.compile(r"\s{2,}")+++def _words_by_line(content: str) -> tuple[list[str], list[int]]:+ """Split into words like content.split(), plus each word's source line index.++ Reconstruction must never join words that came from different source lines+ onto one line -- doing so can glue unrelated facts together (#3099).+ """+ words: list[str] = []+ line_ids: list[int] = []+ for line_idx, line in enumerate(content.split("\n")):+ for word in line.split():+ words.append(word)+ line_ids.append(line_idx)+ return words, line_ids+++def _join_kept_words_by_line(words: list[str], line_ids: list[int], kept_ids: set[int]) -> str:+ """Rejoin kept words, starting a new line wherever they crossed a line boundary."""+ parts: list[str] = []+ prev_line: int | None = None+ for i in sorted(kept_ids):+ if prev_line is not None:+ parts.append("\n" if line_ids[i] != prev_line else " ")+ parts.append(words[i])+ prev_line = line_ids[i]+ return "".join(parts)+++def _tabular_protected_line_ids(content: str) -> set[int]:+ """Flag lines in a run of >=3 consecutive lines with a consistent fixed-width+ column count (ls -l, ps, etc.), so compression keeps each such row whole+ instead of dropping individual fields inconsistently row to row (#3099).+ Shares the must-keep toggle: HEADROOM_KOMPRESS_MUST_KEEP=0 disables this too.+ """+ if os.environ.get(_KOMPRESS_MUST_KEEP_ENV, "1") == "0":+ return set()+ cols_per_line = [+ len(_TABULAR_ROW_SPLIT_RE.split(line.strip())) if line.strip() else 0+ for line in content.split("\n")+ ]+ protected: set[int] = set()+ for cols, group in itertools.groupby(enumerate(cols_per_line), key=lambda p: p[1]):+ indices = [idx for idx, _ in group]+ if cols >= 3 and len(indices) >= 3:+ protected.update(indices)+ return protected++# ONNX artifacts are resolved against the model repo in this order, falling# through on download miss OR session-load failure:#@@ -1602,12 +1653,14 @@ class KompressCompressor(Transform):KompressResult with compressed text."""t_deadline = time.perf_counter() if _deadline_started_at is None else _deadline_started_at- words = content.split()+ words, line_ids = _words_by_line(content)n_words = len(words)if n_words < max(10, self.config.min_input_words) or self._degraded_reason is not None:return self._passthrough(content, n_words)+ protected_line_ids = _tabular_protected_line_ids(content)+# Cooperative wall-clock budget (#1171): kompress ONNX inference is# O(tokens) and non-preemptible once the request's asyncio timeout fires,# so one large block can run for minutes holding a worker (the leak ->@@ -1788,6 +1841,10 @@ class KompressCompressor(Transform):# reconstruct from context. Disable via HEADROOM_KOMPRESS_MUST_KEEP=0._add_kompress_must_keep_words(kept_ids, chunk_words, chunk_start)+ for i, line_id in enumerate(line_ids):+ if line_id in protected_line_ids:+ kept_ids.add(i)+if not kept_ids:if inference_ms >= 1000.0:logger.info(@@ -1802,7 +1859,7 @@ class KompressCompressor(Transform):return self._passthrough(content, n_words)compressed_words = [words[w] for w in sorted(kept_ids) if w < n_words]- compressed = " ".join(compressed_words)+ compressed = _join_kept_words_by_line(words, line_ids, kept_ids)compressed_count = len(compressed_words)cache_key: str | None = Noneoriginal_tokens = n_words@@ -2021,7 +2078,14 @@ class KompressCompressor(Transform):]results: list[KompressResult | None] = [None] * n- word_lists: list[list[str]] = [c.split() for c in contents]+ word_lists: list[list[str]] = []+ line_id_lists: list[list[int]] = []+ protected_line_ids_list: list[set[int]] = []+ for c in contents:+ words, line_ids = _words_by_line(c)+ word_lists.append(words)+ line_id_lists.append(line_ids)+ protected_line_ids_list.append(_tabular_protected_line_ids(c))# Short texts short-circuit to passthrough — no model call needed.max_chunk_words = self.config.chunk_words@@ -2203,14 +2267,19 @@ class KompressCompressor(Transform):continuecontent = contents[text_idx]words = word_lists[text_idx]+ line_ids = line_id_lists[text_idx]n_words = len(words)+ for i, line_id in enumerate(line_ids):+ if line_id in protected_line_ids_list[text_idx]:+ kept_ids.add(i)+if not kept_ids:results[text_idx] = self._passthrough(content, n_words)continuecompressed_words = [words[w] for w in sorted(kept_ids) if w < n_words]- compressed = " ".join(compressed_words)+ compressed = _join_kept_words_by_line(words, line_ids, kept_ids)compressed_count = len(compressed_words)cache_key: str | None = Noneoriginal_tokens = n_wordsdiff --git a/tests/test_kompress_line_integrity.py b/tests/test_kompress_line_integrity.pynew file mode 100644index 000000000..076f84c84--- /dev/null+++ b/tests/test_kompress_line_integrity.py@@ -0,0 +1,108 @@+"""Regression tests for issue #3099: compression merging words across line+boundaries, and dropping fields from tabular rows inconsistently row to row.+"""++from __future__ import annotations++import pytest++from headroom.transforms import kompress_compressor as kc+from headroom.transforms.kompress_compressor import KompressCompressor, KompressConfig+++class _Enc(dict):+ def word_ids(self, batch_index=0):+ return self["_word_ids"][batch_index]+++class _Tok:+ def __call__(self, chunk_words, **kw):+ if chunk_words and isinstance(chunk_words[0], list):+ batch_words = chunk_words+ else:+ batch_words = [chunk_words]+ return _Enc(+ input_ids=[[0] * len(words) for words in batch_words],+ attention_mask=[[1] * len(words) for words in batch_words],+ _word_ids=[list(range(len(words))) for words in batch_words],+ )+++class _KeepIndicesModel:+ """Fake model that keeps exactly the given (global, single-chunk) word indices."""++ def __init__(self, keep_indices: set[int]):+ self._keep = keep_indices++ def get_keep_mask(self, input_ids, attention_mask):+ return [[idx in self._keep for idx in range(len(row))] for row in input_ids]++ def get_scores(self, input_ids, attention_mask):+ return [[1.0 if idx in self._keep else 0.0 for idx in range(len(row))] for row in input_ids]+++def _install_fake_kompress(monkeypatch, keep_indices: set[int], *, must_keep: bool = False) -> None:+ model = _KeepIndicesModel(keep_indices)+ monkeypatch.setattr(kc, "_load_kompress", lambda *a, **k: (model, _Tok(), "onnx"))+ monkeypatch.setattr(kc, "_model_device_type", lambda *a, **k: "cpu")+ if must_keep:+ monkeypatch.delenv(kc._KOMPRESS_MUST_KEEP_ENV, raising=False)+ else:+ monkeypatch.setenv(kc._KOMPRESS_MUST_KEEP_ENV, "0")+++def _compress(monkeypatch, content: str, path: str):+ compressor = KompressCompressor(KompressConfig(enable_ccr=False, min_input_words=10))+ if path == "compress":+ monkeypatch.setattr(compressor, "_should_batch_single_content", lambda *a, **k: False)+ return compressor.compress(content)+ monkeypatch.setattr(compressor, "_should_use_sequential_fallback", lambda: False)+ [result] = compressor.compress_batch([content], batch_size=8)+ return result++[email protected]("path", ["compress", "compress_batch"])+def test_compress_does_not_merge_words_across_lines(monkeypatch, path):+ """Reproduces #3099 bug 1: two commands' output on separate lines.++ The model keeps only the last word of line 1 ("rg") and the first word of+ line 2 ("/opt/homebrew/bin/ast-grep"). Before the fix, these landed adjacent+ in the flat kept-word list and were joined with a space onto one line,+ fabricating "rg /opt/homebrew/bin/ast-grep" -- read literally, that implies+ rg resolves to that path, which is false.+ """+ line1 = "alpha beta gamma delta epsilon zeta eta rg"+ line2 = "/opt/homebrew/bin/ast-grep theta iota kappa lambda mu nu xi"+ content = f"{line1}\n{line2}"++ _install_fake_kompress(monkeypatch, keep_indices={7, 8})++ result = _compress(monkeypatch, content, path)++ assert "rg /opt/homebrew/bin/ast-grep" not in result.compressed+ assert result.compressed.split("\n") == ["rg", "/opt/homebrew/bin/ast-grep"]++[email protected]("path", ["compress", "compress_batch"])+def test_compress_keeps_tabular_row_intact(monkeypatch, path):+ """Reproduces #3099 bug 2: an `ls -la`-shaped listing.++ The model keeps nothing on its own, so absent row protection only the+ must-keep regex would save permission bits/numbers/filenames -- exactly+ the reported symptom of some fields (the group column, the month)+ disappearing inconsistently. Once a row is part of a >=3-row run of+ consistent fixed-width columns, the whole row must survive intact.+ """+ rows = [+ "-rw-r--r--@ 1 user staff 4921 Aug 12 20:14 daemon.log",+ "-rw-r--r--@ 1 user staff 10485 Aug 12 20:14 other.log",+ "-rw-r--r--@ 1 user staff 512 Aug 12 20:15 third.log",+ ]+ content = "\n".join(rows)++ _install_fake_kompress(monkeypatch, keep_indices=set(), must_keep=True)++ result = _compress(monkeypatch, content, path)++ expected = "\n".join(" ".join(row.split()) for row in rows)+ assert result.compressed == expected