Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix: correct Go AST compression bugs and CODE_AWARE token accounting (#1668)
r00t3f14eac06014
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 062c43fe5..a1096dffb 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -700,6 +700,7 @@ class HeadroomProxy(profile_kwargs = proxy_pipeline_kwargs(config)router_config = ContentRouterConfig(enable_code_aware=config.code_aware_enabled,+ prefer_code_aware_for_code=_get_env_bool("HEADROOM_PREFER_CODE_AWARE_FOR_CODE", True),tool_profiles=config.tool_profiles,read_lifecycle=ReadLifecycleConfig(enabled=config.read_lifecycle),smart_crusher_max_items_after_crush=cast(diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.pyindex 03acb1648..ce3b442b7 100644--- a/headroom/transforms/code_compressor.py+++ b/headroom/transforms/code_compressor.py@@ -1438,7 +1438,11 @@ class CodeAwareCompressor(Transform):if _brace_in_signature:# Opening brace already in signature line — just find closingpass- elif body_lines and body_lines[0].strip().startswith("{"):+ elif body_lines and body_lines[0].strip().endswith("{"):+ # Matches both a bare `{` line and a multi-line signature's+ # closing line (e.g. Go's `) error {`), where the brace+ # shares a line with the closing paren/return type rather+ # than starting one of its own.opening_brace_line = body_lines[0]body_lines = body_lines[1:]if body_lines and body_lines[-1].strip().endswith("}"):@@ -1552,6 +1556,17 @@ class CodeAwareCompressor(Transform):# Skip unnamed tokens (tree-sitter anonymous nodes like braces)if not child.is_named:continue+ # Some grammars (e.g. Go) wrap all body statements in one generic+ # list node instead of exposing them as direct siblings of the+ # block. Treating that wrapper as a single statement makes its+ # row range swallow the block's own closing brace line, causing+ # a duplicated `}` later. Unwrap it into its real statements.+ if child.type == "statement_list":+ for inner in child.children:+ if inner.type in _SKIP_TYPES or not inner.is_named:+ continue+ body_stmts.append((inner.start_point[0], inner.end_point[0]))+ continuebody_stmts.append((child.start_point[0], child.end_point[0]))# Calculate lines per statement and keep whole statements until budgetdiff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 30b299b84..eb03be94e 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -862,7 +862,9 @@ class ContentRouterConfig:enable_image_optimizer: bool = True # Image token optimization# Routing preferences- prefer_code_aware_for_code: bool = False # Disabled: let code pass through unmangled+ prefer_code_aware_for_code: bool = (+ True # Route code to CodeAware over Kompress for higher, syntax-safe compression+ )# Route ALL compressible content to Kompress, skipping per-type selection.# Tool exclusion (Read/Glob/...) and reversibility gates still apply.force_kompress_all: bool = False@@ -1997,7 +1999,10 @@ class ContentRouter(Transform):if compressor:compressor_name = type(compressor).__name__result = compressor.compress(content, language=language, context=context)- compressed, compressed_tokens = result.compressed, result.compressed_tokens+ compressed, compressed_tokens = (+ result.compressed,+ len(result.compressed.split()),+ )decision_reason = "code_aware"if compressed is None:# Fallback to Kompressdiff --git a/tests/test_code_aware_regressions.py b/tests/test_code_aware_regressions.pynew file mode 100644index 000000000..632d8fc80--- /dev/null+++ b/tests/test_code_aware_regressions.py@@ -0,0 +1,143 @@+"""Regression tests requested in review of PR #1668 (Go AST compression bugs+and CODE_AWARE token accounting) and PR #1670 (prefer_code_aware_for_code+default flip).++1. Go `statement_list` unwrapping — a Go function body no longer produces a+ duplicated closing brace when truncated.+2. Multi-line Go signature (`) error {`) — the brace-bearing signature line+ survives truncation instead of being silently dropped.+3. ContentRouter CODE_AWARE token accounting — compressed_tokens must come+ from the same word-count metric as original_tokens, not the compressor's+ own (differently-scaled) estimator, or real savings get misread as none.+4. prefer_code_aware_for_code defaults to True, both on the dataclass and via+ the HEADROOM_PREFER_CODE_AWARE_FOR_CODE env var.+"""++from __future__ import annotations++import pytest++from headroom.proxy.server import HeadroomProxy, ProxyConfig+from headroom.transforms.code_compressor import (+ CodeAwareCompressor,+ CodeCompressorConfig,+ _check_tree_sitter_available,+)+from headroom.transforms.content_router import (+ CompressionStrategy,+ ContentRouter,+ ContentRouterConfig,+)++pytestmark = pytest.mark.skipif(+ not _check_tree_sitter_available(),+ reason="tree-sitter not installed (pip install headroom-ai[code])",+)++GO_FUNC = """package main++func Compute(+\ta int,+\tb int,+) error {+\tx := a + b+\ty := x * 2+\tz := y - a+\tw := z + b+\treturn nil+}+"""+++def _compress_go(**config_overrides: object):+ config = CodeCompressorConfig(+ min_tokens_for_compression=0,+ max_body_lines=2,+ semantic_analysis=False,+ **config_overrides,+ )+ compressor = CodeAwareCompressor(config)+ return compressor.compress(GO_FUNC, language="go")+++def test_go_statement_list_unwrap_no_duplicate_closing_brace() -> None:+ """Go wraps a block's statements in one `statement_list` node; treating+ that as a single statement made its row range swallow the block's own+ closing brace, producing a duplicate `}` in the compressed output."""+ result = _compress_go()++ closing_braces = [line for line in result.compressed.splitlines() if line.strip() == "}"]+ assert len(closing_braces) == 1, (+ f"expected exactly one closing brace line, got {len(closing_braces)}:\n{result.compressed}"+ )+++def test_go_multiline_signature_brace_preserved() -> None:+ """The multi-line signature's closing line (`) error {`) shares its row+ with the brace instead of starting one of its own. Detecting the brace+ via startswith("{") missed this and silently dropped the line; it must+ survive truncation via endswith("{")."""+ result = _compress_go()++ assert result.compressed.count(") error {") == 1, (+ f"multi-line signature line missing or duplicated:\n{result.compressed}"+ )+++class _FakeCodeCompressor:+ def __init__(self, compressed: str) -> None:+ self._compressed = compressed++ def compress(self, content: str, language=None, context=""):+ class _Result:+ pass++ r = _Result()+ r.compressed = self._compressed+ # Deliberately inflated/differently-scaled "own" token estimate —+ # simulates a compressor whose internal counter isn't comparable to+ # the router's len(text.split()) word count.+ r.compressed_tokens = 10_000_000+ return r+++def test_content_router_code_aware_token_accounting_matches_word_count() -> None:+ """compressed_tokens must be derived from len(result.compressed.split()),+ the same metric as original_tokens, not the compressor's own estimator.+ Using the mismatched estimator made a real compression look like "no+ savings" and forced a needless fallback to Kompress."""+ original = "word " * 200 # 200 words+ compressed_text = "word " * 50 # a real, large reduction++ router = ContentRouter(ContentRouterConfig(enable_code_aware=True))+ router._code_compressor = _FakeCodeCompressor(compressed_text)++ compressed, compressed_tokens, strategy_chain = router._apply_strategy_to_content(+ original, CompressionStrategy.CODE_AWARE, context=""+ )++ assert compressed == compressed_text+ assert compressed_tokens == len(compressed_text.split())+ assert strategy_chain == ["code_aware"], (+ f"real compression must not trigger a Kompress fallback, got {strategy_chain}"+ )+++def test_prefer_code_aware_for_code_dataclass_defaults_true() -> None:+ assert ContentRouterConfig().prefer_code_aware_for_code is True+++def test_prefer_code_aware_for_code_env_var_defaults_true(monkeypatch) -> None:+ monkeypatch.delenv("HEADROOM_PREFER_CODE_AWARE_FOR_CODE", raising=False)++ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ code_aware_enabled=False,+ )+ proxy = HeadroomProxy(config)+ router = proxy.anthropic_pipeline.transforms[-1]++ assert router.config.prefer_code_aware_for_code is True