Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(router): preserve read protection for quiet sed options (#4521)
Yuchuan He312ffec8ead5
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 276e6ad9a..994dd341d 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -35,6 +35,7 @@ Pipeline Usage:from __future__ import annotationsimport asyncio+import getoptimport hashlibimport itertoolsimport json@@ -42,6 +43,7 @@ import loggingimport mathimport osimport re+import shleximport sysimport threadingimport time@@ -907,8 +909,8 @@ def _segment_is_read(c: str) -> bool:return _is_read_command(" ".join(rest[j + 1 :]).strip("'\""))return Falseis_read = prog in _READ_VERBS or (- # `sed -n '1,20p' file` prints a range (read); bare `sed` is a stream editor.- prog == "sed" and bool(re.search(r"(^|\s)-n(\s|$)", c))+ # Quiet mode prints selected file contents; bare sed is a stream editor.+ prog == "sed" and _sed_is_quiet_read(rest))if not is_read:return False@@ -916,6 +918,75 @@ def _segment_is_read(c: str) -> bool:return not _LOCKFILE_RE.search(c)+def _strip_shell_comment(command: str) -> str:+ """Strip a comment at an unquoted word boundary, retaining literal hashes."""+ quote = ""+ escaped = False+ word_start = True+ for i, char in enumerate(command):+ if escaped:+ escaped = False+ continue+ if char == "\\" and quote != "'":+ escaped = True+ word_start = False+ elif quote:+ if char == quote:+ quote = ""+ elif char in {"'", '"'}:+ quote = char+ word_start = False+ elif char == "#" and word_start:+ return command[:i]+ else:+ word_start = char.isspace()+ return command+++def _sed_is_quiet_read(rest: list[str]) -> bool:+ """Recognize quiet sed options without treating script arguments as flags."""+ try:+ # _bash_program retains shell quotes in its whitespace-split arguments.+ # Reconstruct them before parsing options, including grouped flags (-ne).+ args = shlex.split(_strip_shell_comment(" ".join(rest)))+ except ValueError:+ return False+ # GNU -l takes a line length; BSD -l is a line-buffering flag. Recognize+ # either dialect so existing BSD reads do not lose their protection.+ is_quiet = False+ for short_options in ("anErsuzbiIe:f:l:", "anErsuzbliIe:f:"):+ try:+ options, _ = getopt.gnu_getopt(+ args,+ short_options,+ [+ "quiet",+ "silent",+ "expression=",+ "file=",+ "line-length=",+ "regexp-extended",+ "separate",+ "unbuffered",+ "null-data",+ "binary",+ "posix",+ "sandbox",+ "debug",+ "follow-symlinks",+ "in-place",+ ],+ )+ except getopt.GetoptError:+ continue+ # Check both dialects: GNU -l may consume a BSD -n or in-place flag.+ # Any in-place interpretation takes precedence over quiet mode.+ if any(opt in {"-i", "-I", "--in-place"} for opt, _ in options):+ return False+ is_quiet |= any(opt in {"-n", "--quiet", "--silent"} for opt, _ in options)+ return is_quiet++# Shell wrappers that prefix the real program — peeled to find it. Shell# grammar, not tunable policy: rtk (the user's token proxy), sudo/env/timeout/…_SHELL_WRAPPERS = frozenset(diff --git a/tests/test_transforms/test_read_command_detection.py b/tests/test_transforms/test_read_command_detection.pyindex 03758634e..7419a2c36 100644--- a/tests/test_transforms/test_read_command_detection.py+++ b/tests/test_transforms/test_read_command_detection.py@@ -7,9 +7,16 @@ the first program was inspected, so `wc -l a.py && sed -n '1,60p' a.py` wasclassified as a `wc` run and the file content was word-dropped by Kompress."""+import json+import pytest-from headroom.transforms.content_router import _is_read_command+from headroom.tokenizers.estimator import EstimatingTokenCounter+from headroom.transforms.content_router import (+ ContentRouter,+ ContentRouterConfig,+ _is_read_command,+)@pytest.mark.parametrize(@@ -19,6 +26,19 @@ from headroom.transforms.content_router import _is_read_command"head -80 foo.py","tail -n 50 foo.py","sed -n '1,60p' foo.py",+ "sed -ne '1,60p' foo.py",+ "sed -En '1,60p' foo.py",+ "sed --quiet '1,60p' foo.py",+ "sed --silent '1,60p' foo.py",+ "sed --quiet --expression='1,60p' foo.py",+ "timeout 30 sed -ne '1,60p' foo.py",+ "sed -n --follow-symlinks '1,60p' foo.py",+ "sed -a -n '1,60p' foo.py", # BSD sed.+ "sed -n -e '1,60p' foo.py -l", # BSD line buffering (no argument).+ "sed -l -n '1,60p' foo.py",+ "sed -ln '1,60p' foo.py",+ "sed -l 80 -n '1,60p' foo.py", # GNU line length takes an argument.+ "sed -nl80 '1,60p' foo.py","cd /repo && cat foo.py","sudo cat foo.py","rtk cat foo.py",@@ -75,6 +95,19 @@ def test_chained_reads_are_detected(command: str) -> None:"cat foo.py > copy.py && echo done","sed -n '1,60p' foo.py >> out.txt","sed 's/a/b/' foo.py",+ "sed -n1,60p foo.py", # Invalid: a script cannot be attached to -n.+ "sed -ni '1,60p' foo.py",+ "sed -n -i '1,60p' foo.py",+ "sed -n -I .bak '1,60p' foo.py", # BSD in-place editing.+ "sed -n -I '' '1,60p' foo.py",+ "sed -n -l -i '' '1,60p' foo.py", # BSD -l must not hide in-place flags.+ "sed -n -l -I .bak '1,60p' foo.py",+ "sed -in '1,60p' foo.py",+ "sed --quiet --in-place '1,60p' foo.py",+ "sed --quiet --in-place=.bak '1,60p' foo.py",+ "sed -e 's/a/ -n /' foo.py", # Script text is not a quiet option.+ "sed -e ' -n ' foo.py",+ "sed -- -n foo.py", # Option parsing ends at --.# Lockfiles are regenerated, never byte-patched."cat uv.lock",# `-n` from a sibling segment must not qualify a bare `sed`.@@ -95,3 +128,73 @@ def test_heredoc_body_is_not_split_into_a_bogus_read() -> None:read — the command as a whole writes a file."""command = "cat > foo.py <<EOF\nprint(1); head -5 bar.py\nEOF"assert _is_read_command(command) is False++[email protected]("flag", ["-n", "-ne", "-En", "--quiet", "--silent"])[email protected]("comment", ["", " # avoid -i here", " # --unknown", " # author's note"])+def test_quiet_sed_output_reaches_read_protection(flag: str, comment: str, monkeypatch) -> None:+ monkeypatch.setenv("HEADROOM_PROTECT_READS", "1")+ monkeypatch.setenv("HEADROOM_SHELL_RESULT_MIN_CHARS", "2000")+ # Sixty source lines exceed the small-shell shortcut and reach read protection.+ content = "".join(+ f"def compute_{i}(value):\n"+ " return value * 2 # Preserve this source exactly for a later patch.\n"+ for i in range(30)+ )+ assert len(content) >= 2000+ messages = [+ {"role": "user", "content": "Read the module."},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "read-1",+ "type": "function",+ "function": {+ "name": "Bash",+ "arguments": json.dumps({"command": f"sed {flag} '1,60p' foo.py{comment}"}),+ },+ }+ ],+ },+ {"role": "tool", "tool_call_id": "read-1", "content": content},+ ]+ router = ContentRouter(+ ContentRouterConfig(+ enable_code_aware=False,+ enable_kompress=False,+ protect_analysis_context=False,+ protect_recent_reads_fraction=0.0,+ )+ )+ result = router.apply(messages, EstimatingTokenCounter(), protect_recent=0)+ assert "router:shell_small_result_verbatim" not in result.transforms_applied+ assert "router:read_protected" in result.transforms_applied+ assert result.messages[2]["content"] == content+++ ("command", "expected"),+ [+ ("sed -n '1,60p' foo.py # avoid -i here", True),+ ("sed -n '1,60p' foo.py # --unknown", True),+ ("sed -n '1,60p' foo.py # author's note", True),+ ("sed -n '1,60p' foo.py # unclosed \"quote", True),+ ("sed -n '1,60p' 'foo#bar.py'", True),+ ('sed -n "1,60p" "foo#bar.py"', True),+ ("sed -n '1,60p' foo#bar.py", True),+ (r"sed -n '1,60p' foo\#bar.py", True),+ ("sed -n '/# marker/p' foo.py", True),+ # Literal hashes must not hide real in-place options that follow them.+ ("sed -n '1,60p' foo#bar.py -i", False),+ ("sed -n '1,60p' 'foo#bar.py' -i", False),+ (r"sed -n '1,60p' foo\#bar.py -i", False),+ (r"sed -n '1,60p' foo\ #bar.py -i", False),+ ("sed -n '/# marker/p' foo.py -i", False),+ ("sed -n '1,60p' foo.py -i # --quiet", False),+ ("sed -n 'unterminated # quote", False),+ ],+)+def test_sed_shell_comments_and_literal_hashes(command: str, expected: bool) -> None:+ assert _is_read_command(command) is expected