Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(parser): stop counting HTML comments twice in waste signals (#3942)

Zhibo Zhang (Amur)d0e9c4ca4810
diff --git a/headroom/parser.py b/headroom/parser.py
index 920c26c20..36bd99211 100644
--- a/headroom/parser.py
+++ b/headroom/parser.py
@@ -163,8 +163,10 @@ def detect_waste_signals(text: str, tokenizer: Tokenizer) -> WasteSignals:
if not text:
return signals
- # HTML tags and comments
- html_matches = HTML_TAG_PATTERN.findall(text) + HTML_COMMENT_PATTERN.findall(text)
+ # HTML tags and comments. Comments are removed before tag matching so a
+ # comment is counted once (and tags inside comments are not counted).
+ comment_matches = HTML_COMMENT_PATTERN.findall(text)
+ html_matches = HTML_TAG_PATTERN.findall(HTML_COMMENT_PATTERN.sub("", text)) + comment_matches
if html_matches:
html_text = "".join(html_matches)
signals.html_noise_tokens = tokenizer.count_text(html_text)
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 2c0152b22..9deed38d7 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -268,6 +268,37 @@ class TestDetectWasteSignals:
assert signals.base64_tokens == 0
assert signals.json_bloat_tokens == 0
+ def test_html_comment_counted_once(self, mock_tokenizer):
+ """A comment matches both HTML_TAG_PATTERN and HTML_COMMENT_PATTERN;
+ it must contribute its token count once, not twice."""
+ comment = "<!--" + "x" * 600 + "-->"
+ text = "before " + comment + " after"
+ signals = detect_waste_signals(text, mock_tokenizer)
+ assert signals.html_noise_tokens == (len(comment) // 4 + 1)
+
+ def test_tags_inside_comments_not_counted(self, mock_tokenizer):
+ """Tags inside a comment are part of the comment, not extra noise."""
+ comment = '<!-- <img src="diagram.png" alt="chart"> -->'
+ text = "text " + comment + " end"
+ signals = detect_waste_signals(text, mock_tokenizer)
+ assert signals.html_noise_tokens == (len(comment) // 4 + 1)
+
+ def test_mixed_tags_and_comments_counted_once_each(self, mock_tokenizer):
+ """Plain tags and comments each contribute exactly once."""
+ tag_open = '<div class="a">'
+ tag_close = "</div>"
+ comment = "<!-- note -->"
+ text = tag_open + "content" + tag_close + comment + tag_open + "more" + tag_close
+ signals = detect_waste_signals(text, mock_tokenizer)
+ expected = 2 * len(tag_open) + 2 * len(tag_close) + len(comment)
+ assert signals.html_noise_tokens == (expected // 4 + 1)
+
+ def test_plain_tags_unchanged(self, mock_tokenizer):
+ """Regular tags keep their existing count when no comments are present."""
+ text = "<div>Hello</div>"
+ signals = detect_waste_signals(text, mock_tokenizer)
+ assert signals.html_noise_tokens == (len("<div>") + len("</div>")) // 4 + 1
+
# --- TestIsRagContent ---