Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
"""Dense-line elision: minified/base64/RSC lines that no compressor can shrink."""
from __future__ import annotations
import pytest
from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
from headroom.transforms.dense_line_elider import elide_dense_lines
@pytest.fixture
def tokenizer():
from headroom.providers import OpenAIProvider
from headroom.tokenizer import Tokenizer
return Tokenizer(OpenAIProvider().get_token_counter("gpt-4o"), "gpt-4o")
MINIFIED_JS = (
"!function(){var e=window,t=e.document,n=t.createElement('div');"
"n.className='x';for(var r=0;r<1e3;r++){n.appendChild(t.createTextNode(r))}"
"e.__x=n;var a=e.location.search.replace(/^\\?/,'').split('&');"
) * 16
BASE64 = (
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="
* 12
)
PROSE = "This is an ordinary paragraph with plenty of spaces in it, " * 12
CODE = "\n".join(f" value_{i} = compute(arg_{i}, other_{i}) # comment {i}" for i in range(40))
def test_prose_code_and_short_lines_untouched():
for text in (PROSE, CODE, "short", "a" * 299):
assert elide_dense_lines(text) == (text, 0)
# Chinese, Japanese and Thai prose carries no spaces between words, so a
# paragraph has the same space ratio as a minified bundle.
CJK_PROSE = {
"zh": "人工智能是计算机科学的一个分支,它企图了解智能的实质,并生产出一种新的能以人类智能相似的方式做出反应的智能机器。"
* 8,
"ja": "人工知能は、人間の知的能力をコンピュータ上で実現する様々な技術やソフトウェアの総称であり、研究は現在も続いている。"
* 8,
"th": "ปัญญาประดิษฐ์เป็นสาขาหนึ่งของวิทยาการคอมพิวเตอร์ที่ศึกษาการสร้างเครื่องจักรที่สามารถคิดและเรียนรู้ได้เหมือนมนุษย์"
* 8,
}
@pytest.mark.parametrize("lang", sorted(CJK_PROSE))
def test_space_free_scripts_are_prose_not_dense(lang):
text = "\n\n".join([CJK_PROSE[lang]] * 6)
assert elide_dense_lines(text) == (text, 0)
def test_cjk_prose_wrapped_in_long_ascii_markup_is_kept():
"""Scraped HTML: long ASCII attributes can outweigh the prose in a line."""
url = "https://news.example.com/articles/2026/10/19/" + "a1b2c3d4" * 37
line = f'<p data-article-url="{url}">{CJK_PROSE["zh"][:84]}</p>'
assert len(line) >= 300 and 0.1 < sum(ord(c) > 0x7F for c in line) / len(line) < 0.2
text = "\n".join([line] * 6)
assert elide_dense_lines(text) == (text, 0)
def test_dense_lines_still_elided_next_to_cjk_prose():
text = f"{CJK_PROSE['zh']}\n{MINIFIED_JS}\n{BASE64}\n"
out, n = elide_dense_lines(text)
assert n == 2
assert CJK_PROSE["zh"] in out
def test_dense_lines_elided_others_kept():
text = f"Script completed\nOutput:\n{MINIFIED_JS}\n{PROSE}\n{BASE64}\n"
out, n = elide_dense_lines(text)
assert n == 2
lines = out.split("\n")
assert lines[0] == "Script completed" and lines[3] == PROSE and lines[5] == ""
assert lines[2].startswith(MINIFIED_JS[:160]) and lines[2].endswith(MINIFIED_JS[-80:])
assert "chars of dense machine-generated content elided" in lines[2]
assert len(out) < len(text) // 2
def test_router_applies_elision_when_no_compressor_wins():
html = (
"<!doctype html><html><head><script>" + MINIFIED_JS + "</script></head><body></body></html>"
)
router = ContentRouter(ContentRouterConfig(enable_kompress=False, min_section_tokens=10))
result = router.compress(html, context="tool_result")
assert "content elided" in result.compressed
assert len(result.compressed) < len(html) // 2
assert result.compressed.strip(), "must never blank a non-empty block"
off = ContentRouter(
ContentRouterConfig(
enable_kompress=False, enable_dense_line_elision=False, min_section_tokens=10
)
)
assert "content elided" not in off.compress(html, context="tool_result").compressed
# These exercise the DENSE-LINE ELIDER, so they must pin the router to it.
# Without `enable_html_extractor=False` the strategy chosen depends on whether
# the optional `html` extra is installed: with `trafilatura` present the router
# picks HTMLExtractor instead, which strips <script>/<style> and returns only
# the visible text -- so the assertions below were silently testing a different
# compressor. CI does not install that extra (it runs `--extra proxy`), so both
# tests passed there and failed in any full-extras checkout, including at the
# commit that introduced them.
#
# NOTE the reason they fail under HTMLExtractor is a real defect, not just a
# routing surprise: on `<html><head><script>{3.2kB}</script></head>
# <body>hi there</body></html>` it emits `hi there` -- 3232 chars to 8, logged
# as 99.8% savings -- with NO `Retrieve original: hash=` marker, so the script
# is unrecoverable through CCR. With an empty <body> its output is blank, the
# router rejects it ("must never blank a non-empty block") and falls back to
# elision, which is why only the prose cases fail. Tracked separately; pinning
# the strategy here keeps that bug from hiding inside an elider test.
def test_elided_block_carries_a_retrievable_marker():
from headroom.cache.compression_store import get_compression_store
html = "<html><head><script>" + MINIFIED_JS + "</script></head><body>hi there</body></html>"
router = ContentRouter(
ContentRouterConfig(
enable_kompress=False, enable_html_extractor=False, min_section_tokens=10
)
)
out = router.compress(html, context="tool_result").compressed
assert "Retrieve original: hash=" in out
key = out.rsplit("hash=", 1)[1].rstrip("]\n")
assert get_compression_store().retrieve(key) is not None
def test_messages_path_keeps_elided_tool_result(tokenizer):
"""The Claude path discards marker-less lossy results (#1307); elision must survive it."""
body = (
"<!doctype html><html><head><script>"
+ MINIFIED_JS * 3
+ "</script></head><body><p>x</p></body></html>"
)
messages = [
{"role": "user", "content": "check the site"},
{
"role": "assistant",
"content": [
{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"command": "curl -s x"}}
],
},
{
"role": "user",
"content": [{"type": "tool_result", "tool_use_id": "t1", "content": body}],
},
{"role": "assistant", "content": "ok"},
{"role": "user", "content": "and now?"},
]
# Same strategy pin as the test above: the elider, not whichever compressor
# the installed extras happen to make available.
router = ContentRouter(
ContentRouterConfig(
enable_kompress=False, enable_html_extractor=False, min_section_tokens=10
)
)
result = router.apply(messages, tokenizer)
block = result.messages[2]["content"][0]["content"]
text = block if isinstance(block, str) else block[0]["text"]
assert "content elided" in text and "Retrieve original: hash=" in text
assert len(text) < len(body) // 2
def test_single_dense_values_and_tsv_are_kept():
"""A lone JWT / signed URL / PATH / modulus is a value the agent asked for; TSV is data."""
jwt = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJzdWIiOiIxMjM0NTY3ODkwIn0" * 40 + ".sig"
url = "https://d1.cloudfront.net/v.mp4?Policy=" + "A" * 500 + "&Signature=" + "b" * 128
path = "PATH=" + ":".join(f"/opt/homebrew/opt/package{i}/bin" for i in range(24))
modulus = "Modulus: " + ":".join("ab" for _ in range(256))
tsv = "\n".join("\t".join(f"column_{i}_value_{j}" for i in range(35)) for j in range(5))
for text in (jwt, url, path, modulus, tsv, "Output:\n" + tsv * 3):
assert elide_dense_lines(text) == (text, 0)
def test_compact_json_lines_are_left_to_smart_crusher():
objs = " ".join(
f'{{"file":"src/m_{i}.py","line":{i},"text":"repeated search payload"}}' for i in range(160)
)
assert elide_dense_lines(objs) == (objs, 0)
assert elide_dense_lines("[" + ",".join(f'{{"id":{i}}}' for i in range(200)) + "]")[1] == 0
def test_lossless_mode_never_elides():
html = "<html><head><script>" + MINIFIED_JS + "</script></head><body>hi</body></html>"
router = ContentRouter(
ContentRouterConfig(enable_kompress=False, lossless=True, min_section_tokens=10)
)
assert "content elided" not in router.compress(html, context="tool_result").compressed
def test_search_output_keeps_line_structure():
grep = "src/bundle.js\n1:" + MINIFIED_JS + "\n2:export default x\nsrc/app.py\n10:def main():\n"
out, n = elide_dense_lines(grep)
lines = out.split("\n")
assert n == 1 and lines[0] == "src/bundle.js" and lines[1].startswith("1:!function")
assert lines[2:] == ["2:export default x", "src/app.py", "10:def main():", ""]