Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(parser): whitespace waste signal always reported zero (#1102)
nyxst4ckf19bc9a9c344
diff --git a/headroom/parser.py b/headroom/parser.pyindex a51d435a3..920c26c20 100644--- a/headroom/parser.py+++ b/headroom/parser.py@@ -180,7 +180,9 @@ def detect_waste_signals(text: str, tokenizer: Tokenizer) -> WasteSignals:if ws_matches:# Count tokens that could be saved by normalizing whitespace to single spacesws_text = "".join(ws_matches)- normalized_text = " ".join(ws_matches)+ # Each matched run (4+ spaces/tabs or 3+ newlines) collapses to a+ # single space, so the normalized form is one space per run.+ normalized_text = " " * len(ws_matches)signals.whitespace_tokens = max(0, tokenizer.count_text(ws_text) - tokenizer.count_text(normalized_text))diff --git a/tests/test_parser.py b/tests/test_parser.pyindex 8342f249e..2c0152b22 100644--- a/tests/test_parser.py+++ b/tests/test_parser.py@@ -217,9 +217,28 @@ class TestDetectWasteSignals:assert signals.base64_tokens > 0def test_detect_excessive_whitespace(self, mock_tokenizer, whitespace_waste_text):- """Detects excessive whitespace as waste."""+ """Detects excessive whitespace as waste.++ The fixture has a 4-newline run and a 6-space run. Each run collapses+ to a single space when normalized, so there are real savings to report:+ ws_text "\\n\\n\\n\\n " (10 chars -> 3 tokens) vs normalized " "+ (2 chars -> 1 token) = 2 tokens saved with the mock tokenizer.+ """signals = detect_waste_signals(whitespace_waste_text, mock_tokenizer)- assert signals.whitespace_tokens >= 0 # May be 0 if normalized tokens <= matches+ assert signals.whitespace_tokens == 2++ def test_whitespace_savings_scale_with_run_length(self, mock_tokenizer):+ """Long whitespace runs report proportionally large savings.++ Regression guard: normalizing must collapse each matched run to a+ single space, not just join the runs together (which left the runs+ intact and made the signal always report 0).+ """+ text = "ERROR" + " " * 200 + "stack" + "\n" * 60 + "end"+ signals = detect_waste_signals(text, mock_tokenizer)+ # 260 whitespace chars collapse to 2 spaces; with ~1 token/4 chars the+ # savings are large and clearly non-zero.+ assert signals.whitespace_tokens > 50def test_detect_json_bloat(self, mock_tokenizer, json_bloat_text):"""Detects large JSON blocks as bloat."""