Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(parser): whitespace waste signal always reported zero (#1102)

nyxst4ckf19bc9a9c344
diff --git a/headroom/parser.py b/headroom/parser.py
index a51d435a3..920c26c20 100644
--- a/headroom/parser.py
+++ b/headroom/parser.py
@@ -180,7 +180,9 @@ def detect_waste_signals(text: str, tokenizer: Tokenizer) -> WasteSignals:
if ws_matches:
# Count tokens that could be saved by normalizing whitespace to single spaces
ws_text = "".join(ws_matches)
- normalized_text = " ".join(ws_matches)
+ # Each matched run (4+ spaces/tabs or 3+ newlines) collapses to a
+ # single space, so the normalized form is one space per run.
+ normalized_text = " " * len(ws_matches)
signals.whitespace_tokens = max(
0, tokenizer.count_text(ws_text) - tokenizer.count_text(normalized_text)
)
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 8342f249e..2c0152b22 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -217,9 +217,28 @@ class TestDetectWasteSignals:
assert signals.base64_tokens > 0
def test_detect_excessive_whitespace(self, mock_tokenizer, whitespace_waste_text):
- """Detects excessive whitespace as waste."""
+ """Detects excessive whitespace as waste.
+
+ The fixture has a 4-newline run and a 6-space run. Each run collapses
+ to a single space when normalized, so there are real savings to report:
+ ws_text "\\n\\n\\n\\n " (10 chars -> 3 tokens) vs normalized " "
+ (2 chars -> 1 token) = 2 tokens saved with the mock tokenizer.
+ """
signals = detect_waste_signals(whitespace_waste_text, mock_tokenizer)
- assert signals.whitespace_tokens >= 0 # May be 0 if normalized tokens <= matches
+ assert signals.whitespace_tokens == 2
+
+ def test_whitespace_savings_scale_with_run_length(self, mock_tokenizer):
+ """Long whitespace runs report proportionally large savings.
+
+ Regression guard: normalizing must collapse each matched run to a
+ single space, not just join the runs together (which left the runs
+ intact and made the signal always report 0).
+ """
+ text = "ERROR" + " " * 200 + "stack" + "\n" * 60 + "end"
+ signals = detect_waste_signals(text, mock_tokenizer)
+ # 260 whitespace chars collapse to 2 spaces; with ~1 token/4 chars the
+ # savings are large and clearly non-zero.
+ assert signals.whitespace_tokens > 50
def test_detect_json_bloat(self, mock_tokenizer, json_bloat_text):
"""Detects large JSON blocks as bloat."""