Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(transforms): preserve YAML block scalar data during comment elision (#4498)

Rudy Mizrahi Celekli534cec0e14db
diff --git a/headroom/transforms/config_compressor.py b/headroom/transforms/config_compressor.py
index 92e623478..09d01b4d3 100644
--- a/headroom/transforms/config_compressor.py
+++ b/headroom/transforms/config_compressor.py
@@ -53,7 +53,12 @@ _COMMENT_RES = {
# Content where whole-line elision is unsafe: a '#' line inside a YAML block
# scalar or a TOML multi-line string is data, not a comment. Detection is
# deliberately over-broad — when in doubt, Tier 2 stays off.
-_YAML_BLOCK_SCALAR_RE = re.compile(r":\s*[|>][+-]?\d*\s*$", re.MULTILINE)
+# Block scalars can be sequence items as well as mapping values; indentation
+# and chomping indicators may appear in either order, and a header can carry
+# a trailing comment. Conservatively skip elision for every such header.
+_YAML_BLOCK_SCALAR_RE = re.compile(
+ r"(?:^|[ \t:])[|>][1-9+-]*[ \t]*(?:#[^\r\n]*)?\r?$", re.MULTILINE
+)
_TOML_MULTILINE_RE = re.compile(r'"""|\'\'\'')
diff --git a/tests/test_config_yaml_scalar_elision.py b/tests/test_config_yaml_scalar_elision.py
new file mode 100644
index 000000000..449c94c91
--- /dev/null
+++ b/tests/test_config_yaml_scalar_elision.py
@@ -0,0 +1,45 @@
+"""YAML scalar contents are data even when a line starts with a hash."""
+
+import pytest
+
+from headroom.transforms.config_compressor import ConfigCompressor
+
+
[email protected]("newline", ["\n", "\r\n"])
+ "header",
+ [
+ " - |-",
+ " - >+",
+ " - | # literal script",
+ " - >2- # folded script",
+ " value: | # literal script",
+ " value: >- # folded script",
+ " value: |2-",
+ " value: >2+",
+ " value: |-2",
+ " value: !script |",
+ ],
+)
+def test_yaml_scalar_hash_lines_survive_compression(header: str, newline: str) -> None:
+ content = (
+ "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: demo\nscripts:\n"
+ + header
+ + "\n"
+ + "".join(
+ f" # note {i}: this is scalar content, not a removable YAML comment\n"
+ for i in range(12)
+ )
+ + " echo ready\n"
+ )
+ content = content.replace("\n", newline)
+ result = ConfigCompressor().compress(content)
+ assert result.flavor == "yaml"
+ assert result.lines_elided == 0
+ assert result.ccr_hash is None
+ assert result.compressed == content
+
+
+def test_plain_yaml_comments_remain_eligible_for_elision() -> None:
+ content = "metadata:\n name: demo\n value: plain\n# ordinary comment\n"
+ assert ConfigCompressor._elision_safe(content, "yaml")