Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(lossless): factor shared directory prefix in the grep search fold (#2547)
Tejas Chopra7dc9a978ca97
diff --git a/headroom/transforms/lossless_compaction.py b/headroom/transforms/lossless_compaction.pyindex 573f09aee..76c167e15 100644--- a/headroom/transforms/lossless_compaction.py+++ b/headroom/transforms/lossless_compaction.py@@ -25,6 +25,8 @@ __all__ = ["unfold_repeated_blocks","search_heading","search_unheading",+ "search_dir_heading",+ "search_dir_unheading","diff_strip_index","compact_lossless",]@@ -283,6 +285,74 @@ def search_unheading(text: str) -> str:return _join(out, had_trailing)+# A dir-heading data row: ``<base>:<line>:<content>`` where base has no '/'.+_DIR_DATA_RE = re.compile(r"^(?P<base>[^/\n:]+):(?P<line>\d+):(?P<content>.*)$")+++def search_dir_heading(text: str) -> str:+ """Fold grep ``path:line:content`` rows by DIRECTORY.++ Consecutive rows whose path shares a parent directory collapse to that+ directory once (a header ending in ``/``), then ``base:line:content`` rows+ beneath it. Complements :func:`search_heading` (which factors a repeated+ *file*): this factors a repeated *directory* across distinct files — the+ common ``grep -rn`` case where each file has a single match, so file-heading+ saves nothing but the shared directory repeats on every row. Rows whose path+ has no ``/`` pass through untouched. Exactly reversed by+ :func:`search_dir_unheading`; ``compact_lossless`` verifies the round-trip.+ """+ lines, had_trailing = _split_keep_trailing(text)+ if not lines:+ return text+ out: list[str] = []+ current_dir: str | None = None+ for line in lines:+ m = _GREP_ROW_RE.match(line)+ if m and "/" in m.group("path"):+ path = m.group("path")+ cut = path.rindex("/") + 1+ dir_part, base = path[:cut], path[cut:]+ if dir_part != current_dir:+ out.append(dir_part)+ current_dir = dir_part+ out.append(f"{base}:{m.group('line')}:{m.group('content')}")+ else:+ out.append(line)+ current_dir = None+ return _join(out, had_trailing)+++def search_dir_unheading(text: str) -> str:+ """Exact inverse of :func:`search_dir_heading`.++ A *header* is a line ending in ``/`` immediately followed by a+ ``base:line:content`` data row; it is consumed and re-prefixed onto each+ following data row until a non-data line appears.+ """+ lines, had_trailing = _split_keep_trailing(text)+ if not lines:+ return text+ out: list[str] = []+ current_dir: str | None = None+ n = len(lines)+ i = 0+ while i < n:+ line = lines[i]+ data = _DIR_DATA_RE.match(line)+ if current_dir is not None and data:+ out.append(f"{current_dir}{line}")+ i += 1+ continue+ if line.endswith("/") and i + 1 < n and _DIR_DATA_RE.match(lines[i + 1]):+ current_dir = line+ i += 1+ continue+ current_dir = None+ out.append(line)+ i += 1+ return _join(out, had_trailing)++def diff_strip_index(text: str) -> str:"""Drop ``index <sha>..<sha>`` lines from a unified diff (still applies)."""lines, had_trailing = _split_keep_trailing(text)@@ -387,10 +457,18 @@ def compact_lossless(content: str, kind: str) -> str:return candidate if _smaller(candidate, content) else contentif kind == "search":- candidate = search_heading(content)- if search_unheading(candidate) != content:- return content- return candidate if _smaller(candidate, content) else content+ # Two independent folds; keep the smaller that round-trips exactly.+ # search_heading factors a repeated FILE (many matches in one file);+ # search_dir_heading factors a repeated DIRECTORY (one match each+ # across many files in a dir — the grep -rn case the file fold misses).+ best = content+ for candidate, inverse in (+ (search_heading(content), search_unheading),+ (search_dir_heading(content), search_dir_unheading),+ ):+ if inverse(candidate) == content and _smaller(candidate, best):+ best = candidate+ return bestif kind == "paths":# Pure path listings (find/ls -1/rg -l): fold repeated parent dirs.diff --git a/tests/test_bash_search_lossless_fold.py b/tests/test_bash_search_lossless_fold.pyindex 0094fa84d..e2e74b361 100644--- a/tests/test_bash_search_lossless_fold.py+++ b/tests/test_bash_search_lossless_fold.py@@ -221,3 +221,45 @@ def test_experimental_read_keep_ratio_flag_and_gating(monkeypatch):assert r_on._experimental_compress_read("z" * 500) is None# sub-floor content never attemptedassert r_on._experimental_compress_read("short") is None+++# --- directory-prefix fold: grep -rn across many distinct files ---+from headroom.transforms.lossless_compaction import ( # noqa: E402+ compact_lossless,+ search_dir_heading,+ search_dir_unheading,+)+++def test_search_dir_fold_factors_directory_across_distinct_files() -> None:+ # Sorted grep -rn output: same-dir files are consecutive, one match each, so+ # the file-heading fold saves nothing but the shared directory repeats on+ # every row. The dir fold factors it out — byte-losslessly.+ grep = (+ "\n".join(f"headroom/proxy/mod_{i:02d}.py:{i + 1}: x = compress(p)" for i in range(12))+ + "\n"+ )+ folded = compact_lossless(grep, "search")+ assert len(folded) < len(grep) # actually shrank (0% before this fold)+ assert "headroom/proxy/" in folded # directory factored to a header line+ assert search_dir_unheading(folded) == grep # exact byte round-trip+ assert search_dir_unheading(search_dir_heading(grep)) == grep+++def test_search_dir_fold_roundtrips_mixed_and_passthrough() -> None:+ mixed = (+ "src/a/x.py:1:hit one\nsrc/a/y.py:2:hit two\n"+ "== a plain banner ==\n"+ "src/b/z.py:3:content with a colon: value\nnoslash.py:4:pathless row\n"+ )+ out = compact_lossless(mixed, "search")+ assert search_dir_unheading(out) == mixed or search_unheading(out) == mixed or out == mixed+++def test_search_file_fold_still_wins_for_many_matches_one_file() -> None:+ # Many matches in ONE file: the file fold is smaller, and compact_lossless+ # keeps whichever candidate round-trips and is smallest.+ grep = "\n".join(f"headroom/proxy/server.py:{i}: line {i}" for i in range(1, 40)) + "\n"+ out = compact_lossless(grep, "search")+ assert len(out) < len(grep)+ assert search_unheading(out) == grep