Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(code-compressor): recover valid Python rewrites after local syntax rejection (#2202)
Rod Boevdbbef4bd41f1
diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.pyindex 1023d1d12..e955bd74f 100644--- a/headroom/transforms/code_compressor.py+++ b/headroom/transforms/code_compressor.py@@ -45,6 +45,7 @@ from __future__ import annotationsimport loggingimport reimport threading+from collections.abc import Callablefrom dataclasses import dataclass, fieldfrom enum import Enumfrom typing import Any@@ -1196,6 +1197,20 @@ class CodeAwareCompressor(Transform):# Verify syntax validity (checks both ERROR and MISSING nodes)syntax_valid = self._verify_syntax(compressed, detected_lang)+ original_syntax_valid = True++ if not syntax_valid and detected_lang == CodeLanguage.PYTHON:+ original_syntax_valid = self._verify_syntax(code, detected_lang)+ if original_syntax_valid:+ compressed, structure, symbol_scores = self._compress_with_ast(+ code,+ detected_lang,+ context,+ tokenizer,+ recover_invalid_python_nodes=True,+ )+ compressed_tokens = self._estimate_tokens(compressed, tokenizer)+ syntax_valid = self._verify_syntax(compressed, detected_lang)# If syntax invalid, return original (never serve broken code)if not syntax_valid:@@ -1294,6 +1309,7 @@ class CodeAwareCompressor(Transform):language: CodeLanguage,context: str,tokenizer: Tokenizer | None = None,+ recover_invalid_python_nodes: bool = False,) -> tuple[str, CodeStructure, dict[str, float]]:"""Compress code using AST parsing with symbol importance analysis.@@ -1312,6 +1328,23 @@ class CodeAwareCompressor(Transform):parser = _get_parser(language.value)tree = parser.parse(bytes(code, "utf-8"))root = tree.root_node+ candidate_validator: Callable[[Any, str], str] | None = None++ if recover_invalid_python_nodes and language == CodeLanguage.PYTHON:+ code_bytes = code.encode("utf-8")++ def candidate_validator(node: Any, candidate_text: str) -> str:+ original_text = _slice_code_bytes(code, node.start_byte, node.end_byte)+ if candidate_text == original_text:+ return candidate_text+ candidate_module = (+ code_bytes[: node.start_byte]+ + candidate_text.encode("utf-8")+ + code_bytes[node.end_byte :]+ ).decode("utf-8")+ if self._verify_syntax(candidate_module, CodeLanguage.PYTHON):+ return candidate_text+ return original_text# Analyze symbol importance and allocate compression budgetanalysis = self._analyze_symbol_importance(root, code, language, context)@@ -1321,7 +1354,13 @@ class CodeAwareCompressor(Transform):lang_config = _LANG_CONFIGS.get(language)if lang_config:structure = self._extract_structure(- root, code, language, lang_config, body_limits, analysis+ root,+ code,+ language,+ lang_config,+ body_limits,+ analysis,+ candidate_validator=candidate_validator,)else:structure = self._extract_generic_structure(root, code)@@ -1351,6 +1390,7 @@ class CodeAwareCompressor(Transform):lang_config: LangConfig,body_limits: dict[str, int],analysis: _SymbolAnalysis,+ candidate_validator: Callable[[Any, str], str] | None = None,) -> CodeStructure:"""Extract structure from AST using data-driven language config.@@ -1360,6 +1400,11 @@ class CodeAwareCompressor(Transform):structure = CodeStructure()captured_byte_ranges: list[tuple[int, int]] = []+ def _validated_candidate(node: Any, compressed: str) -> str:+ if candidate_validator is None:+ return compressed+ return candidate_validator(node, compressed)+def visit(node: Any) -> None:node_type = node.type@@ -1396,7 +1441,11 @@ class CodeAwareCompressor(Transform):export_prefix = _slice_code_bytes(code, node.start_byte, child.start_byte)export_suffix = _slice_code_bytes(code, child.end_byte, node.end_byte)structure.function_signatures.append(- leading + export_prefix + compressed + export_suffix+ leading+ + _validated_candidate(+ node,+ export_prefix + compressed + export_suffix,+ ))breakif not has_func_or_class:@@ -1421,16 +1470,21 @@ class CodeAwareCompressor(Transform):child, code, language, lang_config, body_limits, analysis)if decorator_text and definition_compressed:- full_def = leading + "\n".join(decorator_text) + "\n" + definition_compressed+ full_def = _validated_candidate(+ node,+ "\n".join(decorator_text) + "\n" + definition_compressed,+ )# Route to correct list based on inner definition typefor child in node.children:if child.type in lang_config.class_nodes:- structure.class_definitions.append(full_def)+ structure.class_definitions.append(leading + full_def)breakelse:- structure.function_signatures.append(full_def)+ structure.function_signatures.append(leading + full_def)elif definition_compressed:- structure.function_signatures.append(leading + definition_compressed)+ structure.function_signatures.append(+ leading + _validated_candidate(node, definition_compressed)+ )captured_byte_ranges.append((node.start_byte, node.end_byte))return@@ -1440,7 +1494,9 @@ class CodeAwareCompressor(Transform):compressed = self._compress_function_ast(node, code, language, lang_config, body_limits, analysis)- structure.function_signatures.append(leading + compressed)+ structure.function_signatures.append(+ leading + _validated_candidate(node, compressed)+ )captured_byte_ranges.append((node.start_byte, node.end_byte))return@@ -1448,7 +1504,7 @@ class CodeAwareCompressor(Transform):compressed = self._compress_class_ast(node, code, language, lang_config, body_limits, analysis)- structure.class_definitions.append(compressed)+ structure.class_definitions.append(_validated_candidate(node, compressed))captured_byte_ranges.append((node.start_byte, node.end_byte))return@@ -1458,7 +1514,7 @@ class CodeAwareCompressor(Transform):compressed = self._compress_class_ast(node, code, language, lang_config, body_limits, analysis)- structure.class_definitions.append(leading + compressed)+ structure.class_definitions.append(leading + _validated_candidate(node, compressed))captured_byte_ranges.append((node.start_byte, node.end_byte))trailing_semicolon = _get_same_line_trailing_semicolon(node)if trailing_semicolon is not None:diff --git a/tests/test_transforms/test_code_compressor.py b/tests/test_transforms/test_code_compressor.pyindex 574b438a1..33c71fac3 100644--- a/tests/test_transforms/test_code_compressor.py+++ b/tests/test_transforms/test_code_compressor.py@@ -14,6 +14,7 @@ from unittest.mock import patchimport pytest+import headroom.transforms.code_compressor as ccfrom headroom.transforms.code_compressor import (CodeAwareCompressor,CodeCompressionResult,@@ -1535,6 +1536,38 @@ class TestRealASTRuns:))+ def _recovery_compressor(self, **overrides):+ defaults = {+ "min_tokens_for_compression": 1,+ "max_body_lines": 2,+ "enable_ccr": False,+ "semantic_analysis": False,+ }+ defaults.update(overrides)+ return CodeAwareCompressor(CodeCompressorConfig(**defaults))++ def _python_recovery_fixture(self) -> str:+ return textwrap.dedent(+ """\+ from pathlib import Path++ def expand_search_roots(user_root: str) -> list[Path]:+ root = Path(user_root)+ candidates = [root]+ for child in root.iterdir():+ candidates.append(child.resolve())+ return candidates++ def load_user_overrides(config_path: str) -> dict[str, str]:+ config: dict[str, str] = {}+ for line in Path(config_path).read_text().splitlines():+ if "=" in line:+ key, value = line.split("=", 1)+ config[key.strip()] = value.strip()+ return config+ """+ )+def test_get_parser_returns_stock_node_api(self):"""The parser must yield nodes with the stock tree_sitter property APIthat the tree-walking code relies on (``.type``/``.children``/...)."""@@ -1612,6 +1645,101 @@ class TestRealASTRuns:# Output is still valid Python.compile(result.compressed, "<test>", "exec")+ def test_python_invalid_node_falls_back_locally(self):+ """One invalid Python rewrite must preserve only that definition."""+ compressor = self._recovery_compressor()+ code = self._python_recovery_fixture()+ original_compress_function_ast = compressor._compress_function_ast++ def _patched(node, code_text, language, lang_config, body_limits, analysis):+ name = cc._get_definition_name(node)+ if name == "expand_search_roots":+ return "def expand_search_roots(user_root: str) -> list[Path]:\n if True\n"+ if name == "load_user_overrides":+ return (+ "def load_user_overrides(config_path: str) -> dict[str, str]:\n"+ " config: dict[str, str] = {}\n"+ " # [4 lines omitted]\n"+ " return config"+ )+ return original_compress_function_ast(+ node,+ code_text,+ language,+ lang_config,+ body_limits,+ analysis,+ )++ with patch.object(compressor, "_compress_function_ast", side_effect=_patched):+ result = compressor.compress(code, language="python")++ assert result.syntax_valid is True+ assert result.compression_ratio < 1.0+ assert result.compressed != code+ assert "if True" not in result.compressed+ assert "for child in root.iterdir():" in result.compressed+ assert "# [4 lines omitted]" in result.compressed+ compile(result.compressed, "<test>", "exec")++ def test_python_recovery_does_not_block_valid_modern_syntax(self):+ """Valid decorators, nested defs, and match syntax still compress."""+ compressor = self._recovery_compressor(max_body_lines=4)+ code = textwrap.dedent(+ """\+ from dataclasses import dataclass++ def traced(fn):+ return fn++ @dataclass+ class Command:+ name: str+ payload: dict[str, int]++ @traced+ def route_command(command: Command) -> str:+ def normalize(value: str) -> str:+ return value.strip().lower()++ match normalize(command.name):+ case "ping":+ return "pong"+ case "echo":+ return str(command.payload)+ case _:+ return "unknown"+ """+ )++ result = compressor.compress(code, language="python")++ assert result.syntax_valid is True+ assert result.compression_ratio < 1.0+ assert "@traced" in result.compressed+ assert "class Command:" in result.compressed+ assert "def route_command(command: Command) -> str:" in result.compressed+ compile(result.compressed, "<test>", "exec")++ def test_python_recovery_still_returns_original_when_all_candidates_invalid(self):+ """Recovery keeps the terminal whole-file fail-safe."""+ compressor = self._recovery_compressor()+ code = self._python_recovery_fixture()++ def _patched(node, _code_text, _language, _lang_config, _body_limits, _analysis):+ name = cc._get_definition_name(node) or "broken"+ return f"def {name}(:\n pass"++ with patch.object(compressor, "_compress_function_ast", side_effect=_patched):+ result = compressor.compress(code, language="python")++ assert result.syntax_valid is True+ assert result.compression_ratio == 1.0+ assert "(:\n" not in result.compressed+ assert "for child in root.iterdir():" in result.compressed+ assert 'key, value = line.split("=", 1)' in result.compressed+ compile(result.compressed, "<test>", "exec")+def test_get_node_text_uses_utf8_byte_offsets(self):"""tree-sitter byte offsets must not be sliced as Python str indexes."""from headroom.transforms.code_compressor import _get_node_text, _get_parser