Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(code): keep Rust attributes attached to their items (#4624)
X7c150ec6b061
diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.pyindex 2483e6680..1e664beab 100644--- a/headroom/transforms/code_compressor.py+++ b/headroom/transforms/code_compressor.py@@ -2534,35 +2534,51 @@ def _get_node_text(node: Any, code: str) -> str:_COMMENT_NODE_TYPES = frozenset({"comment", "line_comment", "block_comment"})+# Grammar siblings that decorate the following declaration instead of standing+# alone (Rust outer attributes: #[derive(...)], #[inline], #[cfg(...)]).+_ATTACHED_NODE_TYPES = frozenset({"attribute_item"})def _get_leading_comment_text(node: Any, code: str, captured_byte_ranges: list[tuple[int, int]]) -> str:- """Collect contiguous doc-comment siblings immediately preceding a node.-- Doc comments are top-level siblings of the declaration they document, not- children of it. Left uncaptured, they fall through to the leftover- top-level sweep and get grouped separately from the declarations they- document instead of staying attached. Only comments with no blank line- before the node (or the next comment) are treated as attached.+ """Collect contiguous comment and attribute siblings preceding a node.++ Doc comments and Rust outer attributes are top-level siblings of the+ declaration they decorate, not children of it. Left uncaptured, they fall+ through to the leftover top-level sweep and get grouped separately from the+ declarations they decorate instead of staying attached.++ Comments only attach when there is no blank line before the node (or the+ next sibling). Attributes always attach: Rust permits whitespace and+ comments between an outer attribute and the item it decorates, and+ detaching it produces ``expected item after attributes`` at the end of the+ output. Detached comments are skipped while scanning so an attribute+ behind one is still reached, without capturing those comments."""- comments: list[Any] = []+ preamble: list[Any] = []sibling = getattr(node, "prev_sibling", None)anchor_row = node.start_point[0]- while (- sibling is not None- and sibling.type in _COMMENT_NODE_TYPES- and (anchor_row - sibling.end_point[0] <= 1)+ while sibling is not None and (+ sibling.type in _COMMENT_NODE_TYPES or sibling.type in _ATTACHED_NODE_TYPES):- comments.append(sibling)- anchor_row = sibling.start_point[0]+ if sibling.type in _ATTACHED_NODE_TYPES:+ preamble.append(sibling)+ anchor_row = sibling.start_point[0]+ elif anchor_row - sibling.end_point[0] <= 1:+ preamble.append(sibling)+ anchor_row = sibling.start_point[0]+ # A detached comment is skipped, not fatal: an outer attribute behind+ # it still decorates this node.sibling = getattr(sibling, "prev_sibling", None)- if not comments:+ if not preamble:return ""- comments.reverse()- captured_byte_ranges.extend((c.start_byte, c.end_byte) for c in comments)- return "\n".join(_get_node_text(c, code) for c in comments) + "\n"+ preamble.reverse()+ captured_byte_ranges.extend((c.start_byte, c.end_byte) for c in preamble)+ # Some grammars (Rust) include the trailing newline in a line comment's+ # node text; strip it so joining the preamble does not add blank lines.+ texts = [_get_node_text(c, code).rstrip("\n") for c in preamble]+ return "\n".join(texts) + "\n"def _get_node_lines(node: Any, code_lines: list[str]) -> list[str]:diff --git a/tests/test_transforms/test_code_compressor.py b/tests/test_transforms/test_code_compressor.pyindex 131d68a97..167288ba1 100644--- a/tests/test_transforms/test_code_compressor.py+++ b/tests/test_transforms/test_code_compressor.py@@ -2321,3 +2321,108 @@ class TestPhpSupport:code = "<?php\nclass Broken {\n public function oops( {\n"result = self._compressor().compress(code, language="php")assert result.compressed == code++[email protected](not TREE_SITTER_INSTALLED, reason="tree-sitter grammar pack not installed")+class TestRustAttributeAttachment:+ """Rust outer attributes (``#[derive(...)]``, ``#[inline]``, ``#[cfg(...)]``)+ are siblings of the item they decorate, like doc comments. Left+ uncaptured, they fall through to the leftover sweep and drift away from+ the item — commonly after the compressed function, which rustc rejects+ with ``expected item after attributes``.+ """++ def _compressor(self):+ return CodeAwareCompressor(+ CodeCompressorConfig(+ min_tokens_for_compression=1,+ max_body_lines=1,+ enable_ccr=False,+ semantic_analysis=False,+ )+ )++ def test_attributes_stay_attached_to_their_items(self):+ code = (+ "#[derive(Debug, Clone, PartialEq)]\n"+ "pub struct Item {\n"+ " pub id: u64,\n"+ " pub name: String,\n"+ "}\n"+ "\n"+ "#[inline]\n"+ '#[cfg(feature = "extra")]\n'+ "pub fn log_item(item: &Item, sink: &mut Vec<u64>) {\n"+ " sink.push(item.id);\n"+ " sink.push(item.id + 1);\n"+ " sink.push(item.id + 2);\n"+ " sink.push(item.id + 3);\n"+ "}\n"+ )++ result = self._compressor().compress(code, language="rust")++ assert result.compression_ratio < 1.0+ assert "#[derive(Debug, Clone, PartialEq)]\npub struct Item {" in result.compressed+ assert '#[inline]\n#[cfg(feature = "extra")]\npub fn log_item' in result.compressed++ def test_doc_comment_stays_ahead_of_attributes(self):+ code = (+ "/// Records one item.\n"+ "#[inline]\n"+ "pub fn record(item: &Item, sink: &mut Vec<u64>) {\n"+ " sink.push(item.id);\n"+ " sink.push(item.id + 1);\n"+ " sink.push(item.id + 2);\n"+ "}\n"+ )++ result = self._compressor().compress(code, language="rust")++ assert result.compression_ratio < 1.0+ assert "/// Records one item.\n#[inline]\npub fn record" in result.compressed++ def test_attribute_attaches_across_blank_line(self):+ """Rust allows whitespace between an outer attribute and its item."""+ code = (+ "pub struct Item { pub id: u64 }\n"+ "\n"+ "#[inline]\n"+ "\n"+ "pub fn record(item: &Item, sink: &mut Vec<u64>) {\n"+ " sink.push(item.id);\n"+ " sink.push(item.id + 1);\n"+ " sink.push(item.id + 2);\n"+ "}\n"+ )++ result = self._compressor().compress(code, language="rust")++ assert result.compression_ratio < 1.0+ assert result.syntax_valid is True+ assert "#[inline]\npub fn record" in result.compressed+ assert not result.compressed.rstrip().endswith("#[inline]")++ def test_attribute_attaches_through_detached_comment(self):+ """A comment between an attribute and its item must not detach it."""+ code = (+ "pub struct Item { pub id: u64 }\n"+ "\n"+ "#[inline]\n"+ "\n"+ "// Keep with the function below.\n"+ "\n"+ "pub fn record(item: &Item, sink: &mut Vec<u64>) {\n"+ " sink.push(item.id);\n"+ " sink.push(item.id + 1);\n"+ " sink.push(item.id + 2);\n"+ " sink.push(item.id + 3);\n"+ "}\n"+ )++ result = self._compressor().compress(code, language="rust")++ assert result.compression_ratio < 1.0+ assert result.syntax_valid is True+ assert "#[inline]\npub fn record" in result.compressed+ assert not result.compressed.rstrip().endswith("#[inline]")