Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(transforms): classify grep -A/-B/-C context lines as search results (#3599)
lorenzozanef085c4740296
diff --git a/docs/content/docs/text-and-logs.mdx b/docs/content/docs/text-and-logs.mdxindex 2b46baa14..0cc8c8ddd 100644--- a/docs/content/docs/text-and-logs.mdx+++ b/docs/content/docs/text-and-logs.mdx@@ -182,6 +182,7 @@ The ContentRouter selects the right compressor automatically. Here's when each f| Content Pattern | Compressor | Detection Signal ||---|---|---|| `file:line:content` lines | SearchCompressor | grep/ripgrep output format |+| `file-line-content` context lines (`grep -A/-B/-C`) | SearchCompressor | grep/ripgrep context-line format (same section carving as match lines; `file:line:content` and context lines may mix with `--` separators) || pytest, npm, cargo markers | LogCompressor | Build tool output patterns || `---/+++` and `@@` markers | DiffCompressor | Unified diff format || Prose, documentation | KompressCompressor | ContentRouter's default for `PLAIN_TEXT` (`CompressionStrategy.TEXT` and `KOMPRESS` share the same ML compressor dispatch -- see `headroom/transforms/content_router.py:3524-3540`) |diff --git a/headroom/transforms/content_detector.py b/headroom/transforms/content_detector.pyindex bb8325e42..55a855013 100644--- a/headroom/transforms/content_detector.py+++ b/headroom/transforms/content_detector.py@@ -50,6 +50,16 @@ _SEARCH_RESULT_PATTERN = re.compile(r"^[^\s:]+:\d+:" # file:line: format (grep -n style))+# ``path-NN-content`` shape of grep -A/-B/-C context lines (#3580). The path+# group is non-greedy so the earliest ``-digits-`` marker wins, mirroring the+# search parser that anchors on the first line-number marker in the line.+_GREP_CONTEXT_PATTERN = re.compile(r"^(?P<path>[^\s:]+?)-(?P<line>\d+)-")++# Same idea with the path/line separator left as ``:``: ``path:NN-content``.+# Real GNU grep emits dashes in both positions, but the reported repro builds+# context lines this way, so both shapes must route identically.+_GREP_COLON_DASH_PATTERN = re.compile(r"^(?P<path>[^\s:]+):(?P<line>\d+)-")+# A markdown table separator row, e.g. "| --- | :--: |" or "---|---".# Every cell must be dashes with optional alignment colons._MD_SEP_CELL = re.compile(r"^:?-{2,}:?$")@@ -457,6 +467,37 @@ def _try_detect_html(content: str) -> DetectionResult | None:)+def _prefix_looks_like_path(prefix: str) -> bool:+ """Shared path-shape guard for every grep-line shape.++ Rules out markup tags and ``key=value`` log prefixes; a single helper so+ the three shapes cannot drift apart.+ """+ return "<" not in prefix and ">" not in prefix and "=" not in prefix+++def _is_grep_context_line(line: str) -> bool:+ """True when a line looks like ``path-NN-content`` grep context output.++ GNU grep (and ripgrep / git grep) separate ``-A``/``-B``/``-C`` context+ lines with ``-`` where match lines use ``:``. Without this branch those+ lines read as prose, and code in them reaches the word-dropping Kompress+ compressor (#3580). The colon branch runs first; this is tried after.++ The prefix must additionally look like a file path: the same ``</>=``+ exclusions as the colon branch, plus it must contain ``/`` or ``.`` —+ which keeps dates (``2026-09-14``) and dashed prose (``version-2-release``)+ out while accepting real paths, including dashed names (``my-file.py``).+ """+ match = _GREP_CONTEXT_PATTERN.match(line)+ if not match:+ return False+ prefix = match.group("path")+ if not _prefix_looks_like_path(prefix):+ return False+ return "/" in prefix or "." in prefix++def _is_search_result_line(line: str) -> bool:"""True when a line looks like ``path:line:content`` grep output.@@ -467,11 +508,15 @@ def _is_search_result_line(line: str) -> bool:deleting the rest. So the pre-colon segment must additionally look likea file path: no angle brackets and no ``=`` (rules out markup tags and``key=value:12:`` log lines).++ ``grep -A``/``-B``/``-C`` context lines — both the real GNU shape+ (``path-NN-content``) and the reported ``path:NN-content`` shape — are+ accepted via the context predicates so code in them routes to the search+ compressor instead of the prose path (#3580)."""- if not _SEARCH_RESULT_PATTERN.match(line):- return False- prefix = line.split(":", 1)[0]- return "<" not in prefix and ">" not in prefix and "=" not in prefix+ if _SEARCH_RESULT_PATTERN.match(line) or _GREP_COLON_DASH_PATTERN.match(line):+ return _prefix_looks_like_path(line.split(":", 1)[0])+ return _is_grep_context_line(line)def _try_detect_search(content: str) -> DetectionResult | None:diff --git a/headroom/transforms/mixed_content.py b/headroom/transforms/mixed_content.pyindex f59e2ac09..5cfc73246 100644--- a/headroom/transforms/mixed_content.py+++ b/headroom/transforms/mixed_content.py@@ -6,7 +6,7 @@ import jsonimport refrom dataclasses import dataclass-from .content_detector import ContentType+from .content_detector import ContentType, _is_search_result_line@dataclass@@ -34,6 +34,19 @@ _SEARCH_RESULT_PATTERN = re.compile(r"^\S+:\d+:", re.MULTILINE)_PROSE_PATTERN = re.compile(r"[A-Z][a-z]+\s+\w+\s+\w+")+def _is_search_line(line: str) -> bool:+ """True for grep match lines and ``-A``/``-B``/``-C`` context lines.++ Unions the legacy splitter pattern (kept so every previously carved line+ still carves) with the detector's predicate, which additionally claims+ both context-line shapes (``path-NN-content`` and ``path:NN-content``).+ Sharing it keeps the splitter carving the same lines the detector claims+ (#3580); otherwise code in context lines would strand in PLAIN_TEXT+ sections and reach the prose compressor.+ """+ return bool(_SEARCH_RESULT_PATTERN.match(line)) or _is_search_result_line(line)++def is_mixed_content(content: str) -> bool:"""Detect if content contains multiple distinct content types."""return sum(mixed_content_indicators(content).values()) >= 2@@ -46,7 +59,8 @@ def mixed_content_indicators(content: str) -> dict[str, bool]:"has_json_blocks": bool(_JSON_BLOCK_START.search(content)),"has_embedded_json_with_text": _has_valid_json_block_with_text(content),"has_prose": len(_PROSE_PATTERN.findall(content)) > 5,- "has_search_results": bool(_SEARCH_RESULT_PATTERN.search(content)),+ "has_search_results": bool(_SEARCH_RESULT_PATTERN.search(content))+ or any(_is_search_result_line(line) for line in content.split("\n")),}@@ -189,10 +203,10 @@ def split_into_sections(content: str, *, isolate: tuple[str, ...] = ()) -> list[i = end_i + 1continue- if _SEARCH_RESULT_PATTERN.match(line):+ if _is_search_line(line):search_lines = []start_line = i- while i < len(lines) and _SEARCH_RESULT_PATTERN.match(lines[i]):+ while i < len(lines) and _is_search_line(lines[i]):search_lines.append(lines[i])i += 1sections.append(@@ -214,7 +228,7 @@ def split_into_sections(content: str, *, isolate: tuple[str, ...] = ()) -> list[if (_CODE_FENCE_PATTERN.match(next_line)or next_line.strip().startswith(("[", "{"))- or _SEARCH_RESULT_PATTERN.match(next_line)+ or _is_search_line(next_line)or (isolate and _carries_isolate(next_line))):breakdiff --git a/tests/test_mixed_content_sections.py b/tests/test_mixed_content_sections.pyindex e8933a39c..3d80f75dd 100644--- a/tests/test_mixed_content_sections.py+++ b/tests/test_mixed_content_sections.py@@ -49,6 +49,43 @@ def test_split_into_sections_preserves_typed_boundaries():assert sections[3].start_line == 5+def test_split_carves_grep_context_lines_into_search_sections():+ """Regression (#3580): prose + ``grep -A`` output keeps code out of text.++ Context lines (``path-NN-content``) must join the SEARCH_RESULTS section+ instead of stranding source code in a PLAIN_TEXT section, where the+ word-dropping prose compressor would corrupt it. ``--`` group separators+ ride along as ordinary text lines.+ """+ content = "\n".join(+ [+ "Some intro prose explaining the search.",+ "src/main.py-40- context before",+ "src/main.py:42:def process_data(items):",+ "src/main.py-43- context after",+ "--",+ "src/other.py:12: return True",+ "Trailing prose after the results.",+ ]+ )++ sections = split_into_sections(content)++ # The ``--`` group separator breaks the run, so the two grep groups keep+ # their own SEARCH_RESULTS sections instead of merging through it.+ assert [section.content_type for section in sections] == [+ ContentType.PLAIN_TEXT,+ ContentType.SEARCH_RESULTS,+ ContentType.PLAIN_TEXT,+ ContentType.SEARCH_RESULTS,+ ContentType.PLAIN_TEXT,+ ]+ assert "src/main.py-40- context before" in sections[1].content+ assert "src/main.py:42:def process_data(items):" in sections[1].content+ assert "src/main.py-43- context after" in sections[1].content+ assert "src/other.py:12: return True" in sections[3].content++def test_extract_json_block_ignores_delimiters_inside_strings():lines = ["[",diff --git a/tests/test_transforms_content_detection.py b/tests/test_transforms_content_detection.pyindex e74776292..acf00a09f 100644--- a/tests/test_transforms_content_detection.py+++ b/tests/test_transforms_content_detection.py@@ -4,6 +4,8 @@ import jsonfrom headroom.transforms.content_detector import (ContentType,+ _is_grep_context_line,+ _is_search_result_line,_try_detect_code,_try_detect_diff,_try_detect_html,@@ -213,6 +215,90 @@ def test_search_detection_rejects_tag_like_and_key_value_prefixes() -> None:assert _try_detect_search("timeout=30:12:retried\ntimeout=31:12:retried") is None+def test_search_detection_classifies_grep_context_lines() -> None:+ """Regression (#3580): ``grep -A/-B/-C`` context lines are search output.++ GNU grep separates context lines with ``-`` (``path-NN-content``) instead+ of the ``:`` match lines use. The dash alone flipped detection to+ PLAIN_TEXT, routing source code into the word-dropping Kompress prose+ compressor. Context lines must classify as SEARCH_RESULTS, exactly like+ the colon-separated match lines already do.+ """+ code = [+ 'env = payload.get("env")',+ "env_map = dict(env) if isinstance(env, dict) else {}",+ "previous = {name: env_map.get(name) for name in values}",+ 'payload["env"] = env_map',+ 'path.write_text(json.dumps(payload, indent=2), encoding="utf-8")',+ ]+ path = "./headroom/providers/claude/install.py"++ def build(sep: str) -> str:+ return "\n".join(+ f"{path}:{40 + i}{sep}{line}" for _ in range(14) for i, line in enumerate(code)+ )++ for sep in (":", "-"):+ result = _try_detect_search(build(sep))+ assert result is not None, f"separator {sep!r} not detected as search"+ assert result.content_type is ContentType.SEARCH_RESULTS+ assert result.metadata == {"matching_lines": 70, "total_lines": 70}+ assert detect_content_type(build(sep)).content_type is ContentType.SEARCH_RESULTS+++def test_grep_context_line_predicate_guards() -> None:+ """The ``path-NN-content`` shape must not claim dates, prose, or markup."""+ # Genuine context lines, including dashed file names and ripgrep output.+ assert _is_grep_context_line("src/main.py-40-some context before")+ assert _is_grep_context_line("src/my-file.py-12-x = 1")+ assert _is_grep_context_line("./headroom/providers/claude/install.py-40- x = 1")+ # Group separators and single dashes are never matches.+ assert not _is_grep_context_line("--")+ assert not _is_grep_context_line("-")+ # Dates and dashed prose have no path-shaped prefix (no ``/`` or ``.``).+ assert not _is_grep_context_line("2026-09-14")+ assert not _is_grep_context_line("2026-09-14-release-notes-here")+ assert not _is_grep_context_line("version-2-release")+ # Markup / key=value prefixes are excluded, mirroring the colon branch.+ assert not _is_grep_context_line('<log time="10-00-00">started</log>')+ assert not _is_grep_context_line("timeout=30-12-retried")+++def test_grep_colon_dash_shape_routes_like_context() -> None:+ """``path:NN-content`` (the reported repro shape) is search output too."""+ assert _is_search_result_line("src/main.py:40- context before")+ assert _is_search_result_line("./headroom/x.py:40-env = payload.get(1)")+ # Same prefix exclusions as the colon branch.+ assert not _is_search_result_line('<log time="10:00-00">started</log>')+ assert not _is_search_result_line("timeout=30:12-retried")+++def test_search_detection_mixed_match_and_context_block() -> None:+ """A ``grep -C`` block mixing ``:`` matches, ``-`` context, and ``--``."""+ block = "\n".join(+ [+ "src/main.py-40- context before",+ "src/main.py:42:def process_data(items):",+ "src/main.py-43- context after",+ "--",+ "src/other.py-10- more context",+ "src/other.py:12: return True",+ "src/other.py-13- trailing context",+ ]+ )+ result = _try_detect_search(block)+ assert result is not None+ assert result.content_type is ContentType.SEARCH_RESULTS+ # Six grep-shaped lines out of seven (``--`` is a separator, not a match).+ assert result.metadata == {"matching_lines": 6, "total_lines": 7}+ assert detect_content_type(block).content_type is ContentType.SEARCH_RESULTS+++def test_search_detection_single_context_line_stays_unclaimed() -> None:+ """One coincidental ``path-NN`` line must not classify prose (floor of 2)."""+ assert _try_detect_search("src/foo.py-12- foo()") is None++def test_log_detection_prefers_build_output_patterns() -> None:log_output = "\n".join([