Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(transforms): classify grep -A/-B/-C context lines as search results (#3599)

lorenzozanef085c4740296
diff --git a/docs/content/docs/text-and-logs.mdx b/docs/content/docs/text-and-logs.mdx
index 2b46baa14..0cc8c8ddd 100644
--- a/docs/content/docs/text-and-logs.mdx
+++ b/docs/content/docs/text-and-logs.mdx
@@ -182,6 +182,7 @@ The ContentRouter selects the right compressor automatically. Here's when each f
| Content Pattern | Compressor | Detection Signal |
|---|---|---|
| `file:line:content` lines | SearchCompressor | grep/ripgrep output format |
+| `file-line-content` context lines (`grep -A/-B/-C`) | SearchCompressor | grep/ripgrep context-line format (same section carving as match lines; `file:line:content` and context lines may mix with `--` separators) |
| pytest, npm, cargo markers | LogCompressor | Build tool output patterns |
| `---/+++` and `@@` markers | DiffCompressor | Unified diff format |
| Prose, documentation | KompressCompressor | ContentRouter's default for `PLAIN_TEXT` (`CompressionStrategy.TEXT` and `KOMPRESS` share the same ML compressor dispatch -- see `headroom/transforms/content_router.py:3524-3540`) |
diff --git a/headroom/transforms/content_detector.py b/headroom/transforms/content_detector.py
index bb8325e42..55a855013 100644
--- a/headroom/transforms/content_detector.py
+++ b/headroom/transforms/content_detector.py
@@ -50,6 +50,16 @@ _SEARCH_RESULT_PATTERN = re.compile(
r"^[^\s:]+:\d+:" # file:line: format (grep -n style)
)
+# ``path-NN-content`` shape of grep -A/-B/-C context lines (#3580). The path
+# group is non-greedy so the earliest ``-digits-`` marker wins, mirroring the
+# search parser that anchors on the first line-number marker in the line.
+_GREP_CONTEXT_PATTERN = re.compile(r"^(?P<path>[^\s:]+?)-(?P<line>\d+)-")
+
+# Same idea with the path/line separator left as ``:``: ``path:NN-content``.
+# Real GNU grep emits dashes in both positions, but the reported repro builds
+# context lines this way, so both shapes must route identically.
+_GREP_COLON_DASH_PATTERN = re.compile(r"^(?P<path>[^\s:]+):(?P<line>\d+)-")
+
# A markdown table separator row, e.g. "| --- | :--: |" or "---|---".
# Every cell must be dashes with optional alignment colons.
_MD_SEP_CELL = re.compile(r"^:?-{2,}:?$")
@@ -457,6 +467,37 @@ def _try_detect_html(content: str) -> DetectionResult | None:
)
+def _prefix_looks_like_path(prefix: str) -> bool:
+ """Shared path-shape guard for every grep-line shape.
+
+ Rules out markup tags and ``key=value`` log prefixes; a single helper so
+ the three shapes cannot drift apart.
+ """
+ return "<" not in prefix and ">" not in prefix and "=" not in prefix
+
+
+def _is_grep_context_line(line: str) -> bool:
+ """True when a line looks like ``path-NN-content`` grep context output.
+
+ GNU grep (and ripgrep / git grep) separate ``-A``/``-B``/``-C`` context
+ lines with ``-`` where match lines use ``:``. Without this branch those
+ lines read as prose, and code in them reaches the word-dropping Kompress
+ compressor (#3580). The colon branch runs first; this is tried after.
+
+ The prefix must additionally look like a file path: the same ``</>=``
+ exclusions as the colon branch, plus it must contain ``/`` or ``.`` —
+ which keeps dates (``2026-09-14``) and dashed prose (``version-2-release``)
+ out while accepting real paths, including dashed names (``my-file.py``).
+ """
+ match = _GREP_CONTEXT_PATTERN.match(line)
+ if not match:
+ return False
+ prefix = match.group("path")
+ if not _prefix_looks_like_path(prefix):
+ return False
+ return "/" in prefix or "." in prefix
+
+
def _is_search_result_line(line: str) -> bool:
"""True when a line looks like ``path:line:content`` grep output.
@@ -467,11 +508,15 @@ def _is_search_result_line(line: str) -> bool:
deleting the rest. So the pre-colon segment must additionally look like
a file path: no angle brackets and no ``=`` (rules out markup tags and
``key=value:12:`` log lines).
+
+ ``grep -A``/``-B``/``-C`` context lines — both the real GNU shape
+ (``path-NN-content``) and the reported ``path:NN-content`` shape — are
+ accepted via the context predicates so code in them routes to the search
+ compressor instead of the prose path (#3580).
"""
- if not _SEARCH_RESULT_PATTERN.match(line):
- return False
- prefix = line.split(":", 1)[0]
- return "<" not in prefix and ">" not in prefix and "=" not in prefix
+ if _SEARCH_RESULT_PATTERN.match(line) or _GREP_COLON_DASH_PATTERN.match(line):
+ return _prefix_looks_like_path(line.split(":", 1)[0])
+ return _is_grep_context_line(line)
def _try_detect_search(content: str) -> DetectionResult | None:
diff --git a/headroom/transforms/mixed_content.py b/headroom/transforms/mixed_content.py
index f59e2ac09..5cfc73246 100644
--- a/headroom/transforms/mixed_content.py
+++ b/headroom/transforms/mixed_content.py
@@ -6,7 +6,7 @@ import json
import re
from dataclasses import dataclass
-from .content_detector import ContentType
+from .content_detector import ContentType, _is_search_result_line
@dataclass
@@ -34,6 +34,19 @@ _SEARCH_RESULT_PATTERN = re.compile(r"^\S+:\d+:", re.MULTILINE)
_PROSE_PATTERN = re.compile(r"[A-Z][a-z]+\s+\w+\s+\w+")
+def _is_search_line(line: str) -> bool:
+ """True for grep match lines and ``-A``/``-B``/``-C`` context lines.
+
+ Unions the legacy splitter pattern (kept so every previously carved line
+ still carves) with the detector's predicate, which additionally claims
+ both context-line shapes (``path-NN-content`` and ``path:NN-content``).
+ Sharing it keeps the splitter carving the same lines the detector claims
+ (#3580); otherwise code in context lines would strand in PLAIN_TEXT
+ sections and reach the prose compressor.
+ """
+ return bool(_SEARCH_RESULT_PATTERN.match(line)) or _is_search_result_line(line)
+
+
def is_mixed_content(content: str) -> bool:
"""Detect if content contains multiple distinct content types."""
return sum(mixed_content_indicators(content).values()) >= 2
@@ -46,7 +59,8 @@ def mixed_content_indicators(content: str) -> dict[str, bool]:
"has_json_blocks": bool(_JSON_BLOCK_START.search(content)),
"has_embedded_json_with_text": _has_valid_json_block_with_text(content),
"has_prose": len(_PROSE_PATTERN.findall(content)) > 5,
- "has_search_results": bool(_SEARCH_RESULT_PATTERN.search(content)),
+ "has_search_results": bool(_SEARCH_RESULT_PATTERN.search(content))
+ or any(_is_search_result_line(line) for line in content.split("\n")),
}
@@ -189,10 +203,10 @@ def split_into_sections(content: str, *, isolate: tuple[str, ...] = ()) -> list[
i = end_i + 1
continue
- if _SEARCH_RESULT_PATTERN.match(line):
+ if _is_search_line(line):
search_lines = []
start_line = i
- while i < len(lines) and _SEARCH_RESULT_PATTERN.match(lines[i]):
+ while i < len(lines) and _is_search_line(lines[i]):
search_lines.append(lines[i])
i += 1
sections.append(
@@ -214,7 +228,7 @@ def split_into_sections(content: str, *, isolate: tuple[str, ...] = ()) -> list[
if (
_CODE_FENCE_PATTERN.match(next_line)
or next_line.strip().startswith(("[", "{"))
- or _SEARCH_RESULT_PATTERN.match(next_line)
+ or _is_search_line(next_line)
or (isolate and _carries_isolate(next_line))
):
break
diff --git a/tests/test_mixed_content_sections.py b/tests/test_mixed_content_sections.py
index e8933a39c..3d80f75dd 100644
--- a/tests/test_mixed_content_sections.py
+++ b/tests/test_mixed_content_sections.py
@@ -49,6 +49,43 @@ def test_split_into_sections_preserves_typed_boundaries():
assert sections[3].start_line == 5
+def test_split_carves_grep_context_lines_into_search_sections():
+ """Regression (#3580): prose + ``grep -A`` output keeps code out of text.
+
+ Context lines (``path-NN-content``) must join the SEARCH_RESULTS section
+ instead of stranding source code in a PLAIN_TEXT section, where the
+ word-dropping prose compressor would corrupt it. ``--`` group separators
+ ride along as ordinary text lines.
+ """
+ content = "\n".join(
+ [
+ "Some intro prose explaining the search.",
+ "src/main.py-40- context before",
+ "src/main.py:42:def process_data(items):",
+ "src/main.py-43- context after",
+ "--",
+ "src/other.py:12: return True",
+ "Trailing prose after the results.",
+ ]
+ )
+
+ sections = split_into_sections(content)
+
+ # The ``--`` group separator breaks the run, so the two grep groups keep
+ # their own SEARCH_RESULTS sections instead of merging through it.
+ assert [section.content_type for section in sections] == [
+ ContentType.PLAIN_TEXT,
+ ContentType.SEARCH_RESULTS,
+ ContentType.PLAIN_TEXT,
+ ContentType.SEARCH_RESULTS,
+ ContentType.PLAIN_TEXT,
+ ]
+ assert "src/main.py-40- context before" in sections[1].content
+ assert "src/main.py:42:def process_data(items):" in sections[1].content
+ assert "src/main.py-43- context after" in sections[1].content
+ assert "src/other.py:12: return True" in sections[3].content
+
+
def test_extract_json_block_ignores_delimiters_inside_strings():
lines = [
"[",
diff --git a/tests/test_transforms_content_detection.py b/tests/test_transforms_content_detection.py
index e74776292..acf00a09f 100644
--- a/tests/test_transforms_content_detection.py
+++ b/tests/test_transforms_content_detection.py
@@ -4,6 +4,8 @@ import json
from headroom.transforms.content_detector import (
ContentType,
+ _is_grep_context_line,
+ _is_search_result_line,
_try_detect_code,
_try_detect_diff,
_try_detect_html,
@@ -213,6 +215,90 @@ def test_search_detection_rejects_tag_like_and_key_value_prefixes() -> None:
assert _try_detect_search("timeout=30:12:retried\ntimeout=31:12:retried") is None
+def test_search_detection_classifies_grep_context_lines() -> None:
+ """Regression (#3580): ``grep -A/-B/-C`` context lines are search output.
+
+ GNU grep separates context lines with ``-`` (``path-NN-content``) instead
+ of the ``:`` match lines use. The dash alone flipped detection to
+ PLAIN_TEXT, routing source code into the word-dropping Kompress prose
+ compressor. Context lines must classify as SEARCH_RESULTS, exactly like
+ the colon-separated match lines already do.
+ """
+ code = [
+ 'env = payload.get("env")',
+ "env_map = dict(env) if isinstance(env, dict) else {}",
+ "previous = {name: env_map.get(name) for name in values}",
+ 'payload["env"] = env_map',
+ 'path.write_text(json.dumps(payload, indent=2), encoding="utf-8")',
+ ]
+ path = "./headroom/providers/claude/install.py"
+
+ def build(sep: str) -> str:
+ return "\n".join(
+ f"{path}:{40 + i}{sep}{line}" for _ in range(14) for i, line in enumerate(code)
+ )
+
+ for sep in (":", "-"):
+ result = _try_detect_search(build(sep))
+ assert result is not None, f"separator {sep!r} not detected as search"
+ assert result.content_type is ContentType.SEARCH_RESULTS
+ assert result.metadata == {"matching_lines": 70, "total_lines": 70}
+ assert detect_content_type(build(sep)).content_type is ContentType.SEARCH_RESULTS
+
+
+def test_grep_context_line_predicate_guards() -> None:
+ """The ``path-NN-content`` shape must not claim dates, prose, or markup."""
+ # Genuine context lines, including dashed file names and ripgrep output.
+ assert _is_grep_context_line("src/main.py-40-some context before")
+ assert _is_grep_context_line("src/my-file.py-12-x = 1")
+ assert _is_grep_context_line("./headroom/providers/claude/install.py-40- x = 1")
+ # Group separators and single dashes are never matches.
+ assert not _is_grep_context_line("--")
+ assert not _is_grep_context_line("-")
+ # Dates and dashed prose have no path-shaped prefix (no ``/`` or ``.``).
+ assert not _is_grep_context_line("2026-09-14")
+ assert not _is_grep_context_line("2026-09-14-release-notes-here")
+ assert not _is_grep_context_line("version-2-release")
+ # Markup / key=value prefixes are excluded, mirroring the colon branch.
+ assert not _is_grep_context_line('<log time="10-00-00">started</log>')
+ assert not _is_grep_context_line("timeout=30-12-retried")
+
+
+def test_grep_colon_dash_shape_routes_like_context() -> None:
+ """``path:NN-content`` (the reported repro shape) is search output too."""
+ assert _is_search_result_line("src/main.py:40- context before")
+ assert _is_search_result_line("./headroom/x.py:40-env = payload.get(1)")
+ # Same prefix exclusions as the colon branch.
+ assert not _is_search_result_line('<log time="10:00-00">started</log>')
+ assert not _is_search_result_line("timeout=30:12-retried")
+
+
+def test_search_detection_mixed_match_and_context_block() -> None:
+ """A ``grep -C`` block mixing ``:`` matches, ``-`` context, and ``--``."""
+ block = "\n".join(
+ [
+ "src/main.py-40- context before",
+ "src/main.py:42:def process_data(items):",
+ "src/main.py-43- context after",
+ "--",
+ "src/other.py-10- more context",
+ "src/other.py:12: return True",
+ "src/other.py-13- trailing context",
+ ]
+ )
+ result = _try_detect_search(block)
+ assert result is not None
+ assert result.content_type is ContentType.SEARCH_RESULTS
+ # Six grep-shaped lines out of seven (``--`` is a separator, not a match).
+ assert result.metadata == {"matching_lines": 6, "total_lines": 7}
+ assert detect_content_type(block).content_type is ContentType.SEARCH_RESULTS
+
+
+def test_search_detection_single_context_line_stays_unclaimed() -> None:
+ """One coincidental ``path-NN`` line must not classify prose (floor of 2)."""
+ assert _try_detect_search("src/foo.py-12- foo()") is None
+
+
def test_log_detection_prefers_build_output_patterns() -> None:
log_output = "\n".join(
[