Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(parser): stop counting HTML comments twice in waste signals (#3942)
Zhibo Zhang (Amur)d0e9c4ca4810
diff --git a/headroom/parser.py b/headroom/parser.pyindex 920c26c20..36bd99211 100644--- a/headroom/parser.py+++ b/headroom/parser.py@@ -163,8 +163,10 @@ def detect_waste_signals(text: str, tokenizer: Tokenizer) -> WasteSignals:if not text:return signals- # HTML tags and comments- html_matches = HTML_TAG_PATTERN.findall(text) + HTML_COMMENT_PATTERN.findall(text)+ # HTML tags and comments. Comments are removed before tag matching so a+ # comment is counted once (and tags inside comments are not counted).+ comment_matches = HTML_COMMENT_PATTERN.findall(text)+ html_matches = HTML_TAG_PATTERN.findall(HTML_COMMENT_PATTERN.sub("", text)) + comment_matchesif html_matches:html_text = "".join(html_matches)signals.html_noise_tokens = tokenizer.count_text(html_text)diff --git a/tests/test_parser.py b/tests/test_parser.pyindex 2c0152b22..9deed38d7 100644--- a/tests/test_parser.py+++ b/tests/test_parser.py@@ -268,6 +268,37 @@ class TestDetectWasteSignals:assert signals.base64_tokens == 0assert signals.json_bloat_tokens == 0+ def test_html_comment_counted_once(self, mock_tokenizer):+ """A comment matches both HTML_TAG_PATTERN and HTML_COMMENT_PATTERN;+ it must contribute its token count once, not twice."""+ comment = "<!--" + "x" * 600 + "-->"+ text = "before " + comment + " after"+ signals = detect_waste_signals(text, mock_tokenizer)+ assert signals.html_noise_tokens == (len(comment) // 4 + 1)++ def test_tags_inside_comments_not_counted(self, mock_tokenizer):+ """Tags inside a comment are part of the comment, not extra noise."""+ comment = '<!-- <img src="diagram.png" alt="chart"> -->'+ text = "text " + comment + " end"+ signals = detect_waste_signals(text, mock_tokenizer)+ assert signals.html_noise_tokens == (len(comment) // 4 + 1)++ def test_mixed_tags_and_comments_counted_once_each(self, mock_tokenizer):+ """Plain tags and comments each contribute exactly once."""+ tag_open = '<div class="a">'+ tag_close = "</div>"+ comment = "<!-- note -->"+ text = tag_open + "content" + tag_close + comment + tag_open + "more" + tag_close+ signals = detect_waste_signals(text, mock_tokenizer)+ expected = 2 * len(tag_open) + 2 * len(tag_close) + len(comment)+ assert signals.html_noise_tokens == (expected // 4 + 1)++ def test_plain_tags_unchanged(self, mock_tokenizer):+ """Regular tags keep their existing count when no comments are present."""+ text = "<div>Hello</div>"+ signals = detect_waste_signals(text, mock_tokenizer)+ assert signals.html_noise_tokens == (len("<div>") + len("</div>")) // 4 + 1+# --- TestIsRagContent ---