Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(compression): keep container bodies compressible in code handler (#890)
Ashish16ed73bca68e
diff --git a/headroom/compression/handlers/code_handler.py b/headroom/compression/handlers/code_handler.pyindex 697edaeb8..b25d94da3 100644--- a/headroom/compression/handlers/code_handler.py+++ b/headroom/compression/handlers/code_handler.py@@ -63,6 +63,50 @@ def _get_parser(language: str) -> Any:return _tree_sitter_parsers[language]+# tree-sitter API compatibility. tree-sitter-language-pack switched to a+# Rust binding (>=1.0) where node accessors are METHODS (kind(),+# start_byte(), child(i)) and parse() takes str; the classic pybind API+# uses attributes (.type, .start_byte, .children) and parse(bytes).+# Without this shim the tree-sitter path raises TypeError on modern+# installs and silently falls back to regex.+++def _ts_parse(parser: Any, content: str) -> Any:+ try:+ return parser.parse(content.encode("utf-8"))+ except TypeError:+ return parser.parse(content)+++def _ts_root(tree: Any) -> Any:+ root = tree.root_node+ return root() if callable(root) else root+++def _ts_kind(node: Any) -> str:+ kind = getattr(node, "type", None)+ if isinstance(kind, str):+ return kind+ return str(node.kind())+++def _ts_start_byte(node: Any) -> int:+ start = node.start_byte+ return int(start()) if callable(start) else int(start)+++def _ts_end_byte(node: Any) -> int:+ end = node.end_byte+ return int(end()) if callable(end) else int(end)+++def _ts_children(node: Any) -> list[Any]:+ children = getattr(node, "children", None)+ if children is not None and not callable(children):+ return list(children)+ return [node.child(i) for i in range(node.child_count())]++class CodeLanguage(Enum):"""Supported programming languages."""@@ -175,6 +219,23 @@ _SIGNATURE_PATTERNS: dict[str, list[re.Pattern[str]]] = {],}+# Body child node types for container definitions (classes, impls,+# traits). A container's span up to its body is structural (the+# signature); the body itself is NOT marked — recursion into the body+# emits signature spans for nested functions/methods, leaving their+# bodies compressible.+_CONTAINER_BODY_TYPES: frozenset[str] = frozenset(+ {+ "block", # python class body+ "statement_block", # js/ts+ "compound_statement", # c/cpp+ "class_body", # js/ts/java class body+ "interface_body", # java/ts interface body+ "declaration_list", # rust impl/trait body+ "enum_body", # java enum body+ }+)+# Import patterns for fallback_IMPORT_PATTERNS: dict[str, re.Pattern[str]] = {"python": re.compile(r"^\s*(import\s+\w+|from\s+\w+\s+import)", re.MULTILINE),@@ -301,15 +362,16 @@ class CodeStructureHandler(BaseStructureHandler):HandlerResult with mask."""parser = _get_parser(language)- tree = parser.parse(content.encode("utf-8"))+ tree = _ts_parse(parser, content)# Collect structural spansspans: list[CodeSpan] = []def visit_node(node: Any, depth: int = 0) -> None:"""Visit AST node and collect structural spans."""- node_type = node.type+ node_type = _ts_kind(node)structural_types = _STRUCTURAL_NODE_TYPES.get(language, set())+ children = _ts_children(node)# Check if this is a structural node typeif node_type in structural_types:@@ -317,8 +379,8 @@ class CodeStructureHandler(BaseStructureHandler):if "function" in node_type or "method" in node_type:# Find the body node and exclude itbody_node = None- for child in node.children:- if child.type in ("block", "statement_block", "compound_statement"):+ for child in children:+ if _ts_kind(child) in ("block", "statement_block", "compound_statement"):body_node = childbreak@@ -326,8 +388,8 @@ class CodeStructureHandler(BaseStructureHandler):# Signature is from start to body startspans.append(CodeSpan(- start=node.start_byte,- end=body_node.start_byte,+ start=_ts_start_byte(node),+ end=_ts_start_byte(body_node),role="signature",is_structural=True,)@@ -335,8 +397,8 @@ class CodeStructureHandler(BaseStructureHandler):# Body is compressiblespans.append(CodeSpan(- start=body_node.start_byte,- end=body_node.end_byte,+ start=_ts_start_byte(body_node),+ end=_ts_end_byte(body_node),role="body",is_structural=False,)@@ -345,37 +407,75 @@ class CodeStructureHandler(BaseStructureHandler):# No body found, preserve whole thingspans.append(CodeSpan(- start=node.start_byte,- end=node.end_byte,+ start=_ts_start_byte(node),+ end=_ts_end_byte(node),role=node_type,is_structural=True,))+ elif node_type == "decorated_definition":+ # Wrapper around decorator(s) + definition. Emit no+ # span: recursion marks the decorators and gives the+ # inner function its signature/body split. A whole-+ # node span here would preserve the function body.+ passelse:- # Non-function structural nodes- spans.append(- CodeSpan(- start=node.start_byte,- end=node.end_byte,- role=node_type,- is_structural=True,+ # Container definitions (class, impl, trait): the+ # signature runs to the body start; the body is NOT+ # marked, so nested function bodies stay compressible+ # (recursion emits their signature spans). Leaf+ # declarations (imports, type aliases, structs) have+ # no such body child and are preserved whole.+ body_node = None+ for child in children:+ if _ts_kind(child) in _CONTAINER_BODY_TYPES:+ body_node = child+ break++ if body_node is not None:+ spans.append(+ CodeSpan(+ start=_ts_start_byte(node),+ end=_ts_start_byte(body_node),+ role="signature",+ is_structural=True,+ ))+ else:+ spans.append(+ CodeSpan(+ start=_ts_start_byte(node),+ end=_ts_end_byte(node),+ role=node_type,+ is_structural=True,+ )+ )+ elif node_type == "decorator":+ # Decorators are structural (preserved) on their own so+ # the decorated_definition wrapper doesn't need a span.+ spans.append(+ CodeSpan(+ start=_ts_start_byte(node),+ end=_ts_end_byte(node),+ role="decorator",+ is_structural=True,)+ )elif node_type == "comment" and self.preserve_comments:spans.append(CodeSpan(- start=node.start_byte,- end=node.end_byte,+ start=_ts_start_byte(node),+ end=_ts_end_byte(node),role="comment",is_structural=True,))# Recurse into children- for child in node.children:+ for child in children:visit_node(child, depth + 1)- visit_node(tree.root_node)+ visit_node(_ts_root(tree))# Build mask from spansmask = self._spans_to_mask(spans, len(content))diff --git a/tests/test_compression/test_code_handler.py b/tests/test_compression/test_code_handler.pynew file mode 100644index 000000000..556317e6a--- /dev/null+++ b/tests/test_compression/test_code_handler.py@@ -0,0 +1,151 @@+"""Tests for code structure handler."""++import pytest++from headroom.compression.handlers.code_handler import (+ CodeStructureHandler,+ is_tree_sitter_available,+)++requires_tree_sitter = pytest.mark.skipif(+ not is_tree_sitter_available(),+ reason="tree-sitter-language-pack not installed",+)+++class TestCanHandle:+ @pytest.fixture+ def handler(self):+ return CodeStructureHandler()++ def test_detects_python(self, handler):+ assert handler.can_handle("def foo():\n pass\n") is True++ def test_detects_javascript(self, handler):+ assert handler.can_handle("function foo() { return 1; }") is True++ def test_rejects_prose(self, handler):+ assert handler.can_handle("This is a plain sentence.") is False+++class TestRegexFallback:+ """Regex path runs regardless of tree-sitter availability."""++ @pytest.fixture+ def handler(self):+ return CodeStructureHandler(use_tree_sitter=False)++ def test_python_signature_preserved_body_compressible(self, handler):+ code = "def hello(name: str) -> str:\n message = name\n return message\n"+ result = handler.get_mask(code, language="python")++ assert result.metadata["parser"] == "regex"+ sig = "def hello(name: str) -> str:"+ start = code.index(sig)+ assert all(result.mask.mask[i] for i in range(start, start + len(sig)))++ body_char = code.index("message = name")+ assert result.mask.mask[body_char] is False++ def test_python_import_preserved(self, handler):+ code = "import os\n\nx = 1\n"+ result = handler.get_mask(code, language="python")+ assert all(result.mask.mask[i] for i in range(len("import os")))+++@requires_tree_sitter+class TestTreeSitterContainers:+ """Container bodies must stay compressible (signature-only spans).++ Regression: class_definition / decorated_definition / impl_item were+ marked structural over their FULL span, so every method body inside a+ class (i.e. most real code) was preserved and compression no-opped at+ confidence 0.95.+ """++ @pytest.fixture+ def handler(self):+ return CodeStructureHandler()++ def test_class_method_bodies_compressible(self, handler):+ code = (+ "class Foo:\n"+ " def method_a(self):\n"+ " body_line_a = 1\n"+ " return body_line_a\n"+ "\n"+ " def method_b(self):\n"+ " body_line_b = 2\n"+ " return body_line_b\n"+ )+ result = handler.get_mask(code, language="python")+ assert result.metadata["parser"] == "tree-sitter"++ # Class signature and method signatures preserved+ assert all(result.mask.mask[i] for i in range(len("class Foo:")))+ sig = "def method_a(self):"+ start = code.index(sig)+ assert all(result.mask.mask[i] for i in range(start, start + len(sig)))++ # Method bodies compressible+ for body in ("body_line_a = 1", "body_line_b = 2"):+ start = code.index(body)+ assert not any(result.mask.mask[i] for i in range(start, start + len(body))), (+ f"method body {body!r} must be compressible"+ )++ def test_decorated_function_body_compressible(self, handler):+ code = "@decorator\ndef decorated():\n body_line = 4\n return body_line\n"+ result = handler.get_mask(code, language="python")++ # Decorator and signature preserved+ assert all(result.mask.mask[i] for i in range(len("@decorator")))+ sig = "def decorated():"+ start = code.index(sig)+ assert all(result.mask.mask[i] for i in range(start, start + len(sig)))++ # Body compressible+ start = code.index("body_line = 4")+ assert not any(result.mask.mask[i] for i in range(start, start + len("body_line = 4"))), (+ "decorated function body must be compressible"+ )++ def test_module_function_body_compressible(self, handler):+ code = "def standalone():\n body_line = 3\n return body_line\n"+ result = handler.get_mask(code, language="python")++ start = code.index("body_line = 3")+ assert not any(result.mask.mask[i] for i in range(start, start + len("body_line = 3")))++ def test_rust_impl_method_bodies_compressible(self, handler):+ code = (+ "struct Foo { x: i32 }\n"+ "impl Foo {\n"+ " fn method(&self) -> i32 {\n"+ " let body_line = 5;\n"+ " body_line\n"+ " }\n"+ "}\n"+ )+ result = handler.get_mask(code, language="rust")++ # impl signature preserved+ start = code.index("impl Foo")+ assert all(result.mask.mask[i] for i in range(start, start + len("impl Foo")))++ # method body compressible+ start = code.index("let body_line = 5;")+ assert not any(+ result.mask.mask[i] for i in range(start, start + len("let body_line = 5;"))+ ), "impl method body must be compressible"++ def test_preservation_ratio_sane_for_class_code(self, handler):+ """A class with substantial method bodies should NOT preserve+ everything — the whole point of the handler."""+ body = "\n".join(f" line_{i} = {i}" for i in range(20))+ code = f"class Big:\n def method(self):\n{body}\n return 0\n"+ result = handler.get_mask(code, language="python")+ assert result.preservation_ratio < 0.5, (+ f"class code preserved {result.preservation_ratio:.0%} — "+ "container bodies are leaking into the structural mask"+ )