Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(transforms): stop the dense-line elider cutting Chinese, Japanese and Thai prose (#4295)
Rudy Mizrahi Celekli7c4af752adaf
diff --git a/headroom/transforms/dense_line_elider.py b/headroom/transforms/dense_line_elider.pyindex 7d61439c3..82929e56e 100644--- a/headroom/transforms/dense_line_elider.py+++ b/headroom/transforms/dense_line_elider.py@@ -21,6 +21,11 @@ from __future__ import annotationsMIN_LINE_CHARS = 300MAX_SPACE_RATIO = 0.06+# Chinese, Japanese and Thai prose puts no spaces between words, so a long+# paragraph passes the space test. Minified assets, base64 and RSC payloads are+# almost entirely ASCII, so a line with more non-ASCII than this is text, even+# when HTML markup or long URLs around the prose make up most of the line.+MAX_NON_ASCII_RATIO = 0.1# A single dense line (a JWT, a signed URL, a PATH, an RSA modulus) is a value# the agent asked for, not a dump: a block is only elided when its dense lines# add up to at least this many chars. Real bundle dumps are tens of KB.@@ -41,6 +46,8 @@ def is_dense_line(line: str) -> bool:# asked for; minified assets and encoded blobs never carry tabs.if n < MIN_LINE_CHARS or "\t" in line or (line.count(" ") / n) >= MAX_SPACE_RATIO:return False+ if not line.isascii() and n - len(line.encode("ascii", "ignore")) > n * MAX_NON_ASCII_RATIO:+ return Falsestripped = line.strip()if stripped[:1] in "{[" and stripped[-1:] in "}]":return Falsediff --git a/tests/test_transforms/test_dense_line_elider.py b/tests/test_transforms/test_dense_line_elider.pyindex c00f58ff3..519b898ef 100644--- a/tests/test_transforms/test_dense_line_elider.py+++ b/tests/test_transforms/test_dense_line_elider.py@@ -34,6 +34,40 @@ def test_prose_code_and_short_lines_untouched():assert elide_dense_lines(text) == (text, 0)+# Chinese, Japanese and Thai prose carries no spaces between words, so a+# paragraph has the same space ratio as a minified bundle.+CJK_PROSE = {+ "zh": "人工智能是计算机科学的一个分支,它企图了解智能的实质,并生产出一种新的能以人类智能相似的方式做出反应的智能机器。"+ * 8,+ "ja": "人工知能は、人間の知的能力をコンピュータ上で実現する様々な技術やソフトウェアの総称であり、研究は現在も続いている。"+ * 8,+ "th": "ปัญญาประดิษฐ์เป็นสาขาหนึ่งของวิทยาการคอมพิวเตอร์ที่ศึกษาการสร้างเครื่องจักรที่สามารถคิดและเรียนรู้ได้เหมือนมนุษย์"+ * 8,+}++[email protected]("lang", sorted(CJK_PROSE))+def test_space_free_scripts_are_prose_not_dense(lang):+ text = "\n\n".join([CJK_PROSE[lang]] * 6)+ assert elide_dense_lines(text) == (text, 0)+++def test_cjk_prose_wrapped_in_long_ascii_markup_is_kept():+ """Scraped HTML: long ASCII attributes can outweigh the prose in a line."""+ url = "https://news.example.com/articles/2026/10/19/" + "a1b2c3d4" * 37+ line = f'<p data-article-url="{url}">{CJK_PROSE["zh"][:84]}</p>'+ assert len(line) >= 300 and 0.1 < sum(ord(c) > 0x7F for c in line) / len(line) < 0.2+ text = "\n".join([line] * 6)+ assert elide_dense_lines(text) == (text, 0)+++def test_dense_lines_still_elided_next_to_cjk_prose():+ text = f"{CJK_PROSE['zh']}\n{MINIFIED_JS}\n{BASE64}\n"+ out, n = elide_dense_lines(text)+ assert n == 2+ assert CJK_PROSE["zh"] in out++def test_dense_lines_elided_others_kept():text = f"Script completed\nOutput:\n{MINIFIED_JS}\n{PROSE}\n{BASE64}\n"out, n = elide_dense_lines(text)