Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(code): keep Rust attributes attached to their items (#4624)

X7c150ec6b061
diff --git a/headroom/transforms/code_compressor.py b/headroom/transforms/code_compressor.py
index 2483e6680..1e664beab 100644
--- a/headroom/transforms/code_compressor.py
+++ b/headroom/transforms/code_compressor.py
@@ -2534,35 +2534,51 @@ def _get_node_text(node: Any, code: str) -> str:
_COMMENT_NODE_TYPES = frozenset({"comment", "line_comment", "block_comment"})
+# Grammar siblings that decorate the following declaration instead of standing
+# alone (Rust outer attributes: #[derive(...)], #[inline], #[cfg(...)]).
+_ATTACHED_NODE_TYPES = frozenset({"attribute_item"})
def _get_leading_comment_text(
node: Any, code: str, captured_byte_ranges: list[tuple[int, int]]
) -> str:
- """Collect contiguous doc-comment siblings immediately preceding a node.
-
- Doc comments are top-level siblings of the declaration they document, not
- children of it. Left uncaptured, they fall through to the leftover
- top-level sweep and get grouped separately from the declarations they
- document instead of staying attached. Only comments with no blank line
- before the node (or the next comment) are treated as attached.
+ """Collect contiguous comment and attribute siblings preceding a node.
+
+ Doc comments and Rust outer attributes are top-level siblings of the
+ declaration they decorate, not children of it. Left uncaptured, they fall
+ through to the leftover top-level sweep and get grouped separately from the
+ declarations they decorate instead of staying attached.
+
+ Comments only attach when there is no blank line before the node (or the
+ next sibling). Attributes always attach: Rust permits whitespace and
+ comments between an outer attribute and the item it decorates, and
+ detaching it produces ``expected item after attributes`` at the end of the
+ output. Detached comments are skipped while scanning so an attribute
+ behind one is still reached, without capturing those comments.
"""
- comments: list[Any] = []
+ preamble: list[Any] = []
sibling = getattr(node, "prev_sibling", None)
anchor_row = node.start_point[0]
- while (
- sibling is not None
- and sibling.type in _COMMENT_NODE_TYPES
- and (anchor_row - sibling.end_point[0] <= 1)
+ while sibling is not None and (
+ sibling.type in _COMMENT_NODE_TYPES or sibling.type in _ATTACHED_NODE_TYPES
):
- comments.append(sibling)
- anchor_row = sibling.start_point[0]
+ if sibling.type in _ATTACHED_NODE_TYPES:
+ preamble.append(sibling)
+ anchor_row = sibling.start_point[0]
+ elif anchor_row - sibling.end_point[0] <= 1:
+ preamble.append(sibling)
+ anchor_row = sibling.start_point[0]
+ # A detached comment is skipped, not fatal: an outer attribute behind
+ # it still decorates this node.
sibling = getattr(sibling, "prev_sibling", None)
- if not comments:
+ if not preamble:
return ""
- comments.reverse()
- captured_byte_ranges.extend((c.start_byte, c.end_byte) for c in comments)
- return "\n".join(_get_node_text(c, code) for c in comments) + "\n"
+ preamble.reverse()
+ captured_byte_ranges.extend((c.start_byte, c.end_byte) for c in preamble)
+ # Some grammars (Rust) include the trailing newline in a line comment's
+ # node text; strip it so joining the preamble does not add blank lines.
+ texts = [_get_node_text(c, code).rstrip("\n") for c in preamble]
+ return "\n".join(texts) + "\n"
def _get_node_lines(node: Any, code_lines: list[str]) -> list[str]:
diff --git a/tests/test_transforms/test_code_compressor.py b/tests/test_transforms/test_code_compressor.py
index 131d68a97..167288ba1 100644
--- a/tests/test_transforms/test_code_compressor.py
+++ b/tests/test_transforms/test_code_compressor.py
@@ -2321,3 +2321,108 @@ class TestPhpSupport:
code = "<?php\nclass Broken {\n public function oops( {\n"
result = self._compressor().compress(code, language="php")
assert result.compressed == code
+
+
[email protected](not TREE_SITTER_INSTALLED, reason="tree-sitter grammar pack not installed")
+class TestRustAttributeAttachment:
+ """Rust outer attributes (``#[derive(...)]``, ``#[inline]``, ``#[cfg(...)]``)
+ are siblings of the item they decorate, like doc comments. Left
+ uncaptured, they fall through to the leftover sweep and drift away from
+ the item — commonly after the compressed function, which rustc rejects
+ with ``expected item after attributes``.
+ """
+
+ def _compressor(self):
+ return CodeAwareCompressor(
+ CodeCompressorConfig(
+ min_tokens_for_compression=1,
+ max_body_lines=1,
+ enable_ccr=False,
+ semantic_analysis=False,
+ )
+ )
+
+ def test_attributes_stay_attached_to_their_items(self):
+ code = (
+ "#[derive(Debug, Clone, PartialEq)]\n"
+ "pub struct Item {\n"
+ " pub id: u64,\n"
+ " pub name: String,\n"
+ "}\n"
+ "\n"
+ "#[inline]\n"
+ '#[cfg(feature = "extra")]\n'
+ "pub fn log_item(item: &Item, sink: &mut Vec<u64>) {\n"
+ " sink.push(item.id);\n"
+ " sink.push(item.id + 1);\n"
+ " sink.push(item.id + 2);\n"
+ " sink.push(item.id + 3);\n"
+ "}\n"
+ )
+
+ result = self._compressor().compress(code, language="rust")
+
+ assert result.compression_ratio < 1.0
+ assert "#[derive(Debug, Clone, PartialEq)]\npub struct Item {" in result.compressed
+ assert '#[inline]\n#[cfg(feature = "extra")]\npub fn log_item' in result.compressed
+
+ def test_doc_comment_stays_ahead_of_attributes(self):
+ code = (
+ "/// Records one item.\n"
+ "#[inline]\n"
+ "pub fn record(item: &Item, sink: &mut Vec<u64>) {\n"
+ " sink.push(item.id);\n"
+ " sink.push(item.id + 1);\n"
+ " sink.push(item.id + 2);\n"
+ "}\n"
+ )
+
+ result = self._compressor().compress(code, language="rust")
+
+ assert result.compression_ratio < 1.0
+ assert "/// Records one item.\n#[inline]\npub fn record" in result.compressed
+
+ def test_attribute_attaches_across_blank_line(self):
+ """Rust allows whitespace between an outer attribute and its item."""
+ code = (
+ "pub struct Item { pub id: u64 }\n"
+ "\n"
+ "#[inline]\n"
+ "\n"
+ "pub fn record(item: &Item, sink: &mut Vec<u64>) {\n"
+ " sink.push(item.id);\n"
+ " sink.push(item.id + 1);\n"
+ " sink.push(item.id + 2);\n"
+ "}\n"
+ )
+
+ result = self._compressor().compress(code, language="rust")
+
+ assert result.compression_ratio < 1.0
+ assert result.syntax_valid is True
+ assert "#[inline]\npub fn record" in result.compressed
+ assert not result.compressed.rstrip().endswith("#[inline]")
+
+ def test_attribute_attaches_through_detached_comment(self):
+ """A comment between an attribute and its item must not detach it."""
+ code = (
+ "pub struct Item { pub id: u64 }\n"
+ "\n"
+ "#[inline]\n"
+ "\n"
+ "// Keep with the function below.\n"
+ "\n"
+ "pub fn record(item: &Item, sink: &mut Vec<u64>) {\n"
+ " sink.push(item.id);\n"
+ " sink.push(item.id + 1);\n"
+ " sink.push(item.id + 2);\n"
+ " sink.push(item.id + 3);\n"
+ "}\n"
+ )
+
+ result = self._compressor().compress(code, language="rust")
+
+ assert result.compression_ratio < 1.0
+ assert result.syntax_valid is True
+ assert "#[inline]\npub fn record" in result.compressed
+ assert not result.compressed.rstrip().endswith("#[inline]")