Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(router): preserve read protection for quiet sed options (#4521)

Yuchuan He312ffec8ead5
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 276e6ad9a..994dd341d 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -35,6 +35,7 @@ Pipeline Usage:
from __future__ import annotations
import asyncio
+import getopt
import hashlib
import itertools
import json
@@ -42,6 +43,7 @@ import logging
import math
import os
import re
+import shlex
import sys
import threading
import time
@@ -907,8 +909,8 @@ def _segment_is_read(c: str) -> bool:
return _is_read_command(" ".join(rest[j + 1 :]).strip("'\""))
return False
is_read = prog in _READ_VERBS or (
- # `sed -n '1,20p' file` prints a range (read); bare `sed` is a stream editor.
- prog == "sed" and bool(re.search(r"(^|\s)-n(\s|$)", c))
+ # Quiet mode prints selected file contents; bare sed is a stream editor.
+ prog == "sed" and _sed_is_quiet_read(rest)
)
if not is_read:
return False
@@ -916,6 +918,75 @@ def _segment_is_read(c: str) -> bool:
return not _LOCKFILE_RE.search(c)
+def _strip_shell_comment(command: str) -> str:
+ """Strip a comment at an unquoted word boundary, retaining literal hashes."""
+ quote = ""
+ escaped = False
+ word_start = True
+ for i, char in enumerate(command):
+ if escaped:
+ escaped = False
+ continue
+ if char == "\\" and quote != "'":
+ escaped = True
+ word_start = False
+ elif quote:
+ if char == quote:
+ quote = ""
+ elif char in {"'", '"'}:
+ quote = char
+ word_start = False
+ elif char == "#" and word_start:
+ return command[:i]
+ else:
+ word_start = char.isspace()
+ return command
+
+
+def _sed_is_quiet_read(rest: list[str]) -> bool:
+ """Recognize quiet sed options without treating script arguments as flags."""
+ try:
+ # _bash_program retains shell quotes in its whitespace-split arguments.
+ # Reconstruct them before parsing options, including grouped flags (-ne).
+ args = shlex.split(_strip_shell_comment(" ".join(rest)))
+ except ValueError:
+ return False
+ # GNU -l takes a line length; BSD -l is a line-buffering flag. Recognize
+ # either dialect so existing BSD reads do not lose their protection.
+ is_quiet = False
+ for short_options in ("anErsuzbiIe:f:l:", "anErsuzbliIe:f:"):
+ try:
+ options, _ = getopt.gnu_getopt(
+ args,
+ short_options,
+ [
+ "quiet",
+ "silent",
+ "expression=",
+ "file=",
+ "line-length=",
+ "regexp-extended",
+ "separate",
+ "unbuffered",
+ "null-data",
+ "binary",
+ "posix",
+ "sandbox",
+ "debug",
+ "follow-symlinks",
+ "in-place",
+ ],
+ )
+ except getopt.GetoptError:
+ continue
+ # Check both dialects: GNU -l may consume a BSD -n or in-place flag.
+ # Any in-place interpretation takes precedence over quiet mode.
+ if any(opt in {"-i", "-I", "--in-place"} for opt, _ in options):
+ return False
+ is_quiet |= any(opt in {"-n", "--quiet", "--silent"} for opt, _ in options)
+ return is_quiet
+
+
# Shell wrappers that prefix the real program — peeled to find it. Shell
# grammar, not tunable policy: rtk (the user's token proxy), sudo/env/timeout/…
_SHELL_WRAPPERS = frozenset(
diff --git a/tests/test_transforms/test_read_command_detection.py b/tests/test_transforms/test_read_command_detection.py
index 03758634e..7419a2c36 100644
--- a/tests/test_transforms/test_read_command_detection.py
+++ b/tests/test_transforms/test_read_command_detection.py
@@ -7,9 +7,16 @@ the first program was inspected, so `wc -l a.py && sed -n '1,60p' a.py` was
classified as a `wc` run and the file content was word-dropped by Kompress.
"""
+import json
+
import pytest
-from headroom.transforms.content_router import _is_read_command
+from headroom.tokenizers.estimator import EstimatingTokenCounter
+from headroom.transforms.content_router import (
+ ContentRouter,
+ ContentRouterConfig,
+ _is_read_command,
+)
@pytest.mark.parametrize(
@@ -19,6 +26,19 @@ from headroom.transforms.content_router import _is_read_command
"head -80 foo.py",
"tail -n 50 foo.py",
"sed -n '1,60p' foo.py",
+ "sed -ne '1,60p' foo.py",
+ "sed -En '1,60p' foo.py",
+ "sed --quiet '1,60p' foo.py",
+ "sed --silent '1,60p' foo.py",
+ "sed --quiet --expression='1,60p' foo.py",
+ "timeout 30 sed -ne '1,60p' foo.py",
+ "sed -n --follow-symlinks '1,60p' foo.py",
+ "sed -a -n '1,60p' foo.py", # BSD sed.
+ "sed -n -e '1,60p' foo.py -l", # BSD line buffering (no argument).
+ "sed -l -n '1,60p' foo.py",
+ "sed -ln '1,60p' foo.py",
+ "sed -l 80 -n '1,60p' foo.py", # GNU line length takes an argument.
+ "sed -nl80 '1,60p' foo.py",
"cd /repo && cat foo.py",
"sudo cat foo.py",
"rtk cat foo.py",
@@ -75,6 +95,19 @@ def test_chained_reads_are_detected(command: str) -> None:
"cat foo.py > copy.py && echo done",
"sed -n '1,60p' foo.py >> out.txt",
"sed 's/a/b/' foo.py",
+ "sed -n1,60p foo.py", # Invalid: a script cannot be attached to -n.
+ "sed -ni '1,60p' foo.py",
+ "sed -n -i '1,60p' foo.py",
+ "sed -n -I .bak '1,60p' foo.py", # BSD in-place editing.
+ "sed -n -I '' '1,60p' foo.py",
+ "sed -n -l -i '' '1,60p' foo.py", # BSD -l must not hide in-place flags.
+ "sed -n -l -I .bak '1,60p' foo.py",
+ "sed -in '1,60p' foo.py",
+ "sed --quiet --in-place '1,60p' foo.py",
+ "sed --quiet --in-place=.bak '1,60p' foo.py",
+ "sed -e 's/a/ -n /' foo.py", # Script text is not a quiet option.
+ "sed -e ' -n ' foo.py",
+ "sed -- -n foo.py", # Option parsing ends at --.
# Lockfiles are regenerated, never byte-patched.
"cat uv.lock",
# `-n` from a sibling segment must not qualify a bare `sed`.
@@ -95,3 +128,73 @@ def test_heredoc_body_is_not_split_into_a_bogus_read() -> None:
read — the command as a whole writes a file."""
command = "cat > foo.py <<EOF\nprint(1); head -5 bar.py\nEOF"
assert _is_read_command(command) is False
+
+
[email protected]("flag", ["-n", "-ne", "-En", "--quiet", "--silent"])
[email protected]("comment", ["", " # avoid -i here", " # --unknown", " # author's note"])
+def test_quiet_sed_output_reaches_read_protection(flag: str, comment: str, monkeypatch) -> None:
+ monkeypatch.setenv("HEADROOM_PROTECT_READS", "1")
+ monkeypatch.setenv("HEADROOM_SHELL_RESULT_MIN_CHARS", "2000")
+ # Sixty source lines exceed the small-shell shortcut and reach read protection.
+ content = "".join(
+ f"def compute_{i}(value):\n"
+ " return value * 2 # Preserve this source exactly for a later patch.\n"
+ for i in range(30)
+ )
+ assert len(content) >= 2000
+ messages = [
+ {"role": "user", "content": "Read the module."},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "read-1",
+ "type": "function",
+ "function": {
+ "name": "Bash",
+ "arguments": json.dumps({"command": f"sed {flag} '1,60p' foo.py{comment}"}),
+ },
+ }
+ ],
+ },
+ {"role": "tool", "tool_call_id": "read-1", "content": content},
+ ]
+ router = ContentRouter(
+ ContentRouterConfig(
+ enable_code_aware=False,
+ enable_kompress=False,
+ protect_analysis_context=False,
+ protect_recent_reads_fraction=0.0,
+ )
+ )
+ result = router.apply(messages, EstimatingTokenCounter(), protect_recent=0)
+ assert "router:shell_small_result_verbatim" not in result.transforms_applied
+ assert "router:read_protected" in result.transforms_applied
+ assert result.messages[2]["content"] == content
+
+
+ ("command", "expected"),
+ [
+ ("sed -n '1,60p' foo.py # avoid -i here", True),
+ ("sed -n '1,60p' foo.py # --unknown", True),
+ ("sed -n '1,60p' foo.py # author's note", True),
+ ("sed -n '1,60p' foo.py # unclosed \"quote", True),
+ ("sed -n '1,60p' 'foo#bar.py'", True),
+ ('sed -n "1,60p" "foo#bar.py"', True),
+ ("sed -n '1,60p' foo#bar.py", True),
+ (r"sed -n '1,60p' foo\#bar.py", True),
+ ("sed -n '/# marker/p' foo.py", True),
+ # Literal hashes must not hide real in-place options that follow them.
+ ("sed -n '1,60p' foo#bar.py -i", False),
+ ("sed -n '1,60p' 'foo#bar.py' -i", False),
+ (r"sed -n '1,60p' foo\#bar.py -i", False),
+ (r"sed -n '1,60p' foo\ #bar.py -i", False),
+ ("sed -n '/# marker/p' foo.py -i", False),
+ ("sed -n '1,60p' foo.py -i # --quiet", False),
+ ("sed -n 'unterminated # quote", False),
+ ],
+)
+def test_sed_shell_comments_and_literal_hashes(command: str, expected: bool) -> None:
+ assert _is_read_command(command) is expected