Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(search_compressor): don't let a date in a path hijack the line-number parse (#2084)

Doyeon Baek36577d95471f
diff --git a/crates/headroom-core/src/transforms/search_compressor.rs b/crates/headroom-core/src/transforms/search_compressor.rs
index 5ffed0e3a..a6e4f329c 100644
--- a/crates/headroom-core/src/transforms/search_compressor.rs
+++ b/crates/headroom-core/src/transforms/search_compressor.rs
@@ -625,7 +625,134 @@ impl SearchCompressor {
/// Returns `None` for lines that don't match the shape (no
/// `<sep>\d+<sep>` found). Caller treats those as un-parseable and
/// drops them.
+///
+/// # Why the scan runs in tiers
+///
+/// Leftmost-wins alone misreads any *path segment* that itself contains
+/// a `<sep>\d+<sep>` triplet, which is extremely common:
+///
+/// ```text
+/// logs/2026-05-03/app.log:12:ERROR -> ("logs/2026", 5, "03/app.log:12:ERROR")
+/// advisories/CVE-2021-44228.md-9-ctx -> ("advisories/CVE", 2021, "44228.md-9-ctx")
+/// ```
+///
+/// That is silent corruption, not a drop — the bogus path becomes the
+/// grouping key in [`SearchCompressor::parse_search_results`], so the
+/// model is shown a file and a line number that do not exist, plus a
+/// body with a path fragment glued onto the front.
+///
+/// The tiers exploit what grep actually emits:
+///
+/// 1. **Colon tier** — leftmost `:\d+:` whose path part contains no
+/// whitespace. `:` is grep's *match*-line separator and a path
+/// practically never contains one (the Windows drive colon is
+/// already skipped above), so the leftmost is the right one. The
+/// no-whitespace guard keeps a `foo.rs:12:` reference *inside the
+/// body* of a `-`-separated context line from hijacking the parse.
+/// 2. **Dash tier** — `-` is grep's *context*-line separator, and unlike
+/// `:` it appears inside real paths (`2026-05-03`, `CVE-2021-44228`,
+/// `20240101-002-add_users.sql`), so the leftmost triplet is often
+/// still inside the path. The scan therefore walks rightward — but it
+/// only advances past a marker on *positive* evidence that the path
+/// continues through it: either the path so far ends in a segment with
+/// an extension (that marker is the boundary — stop), or the rest of
+/// the token still holds a `/` or a further extension (still inside
+/// the path — keep going). If the walk reaches the end of the token
+/// without ever finding that evidence, the line is genuinely ambiguous
+/// and the tier falls back to the leftmost marker — exactly what the
+/// permissive rule returns. On any line where it cannot prove it knows
+/// better, it agrees with the rule it refines.
+///
+/// One shape stays ambiguous *in principle*, and the tier knowingly
+/// takes the other side of it than the permissive rule does:
+///
+/// ```text
+/// advisories/CVE-2021-44228.md-9-ctx -> file is CVE-2021-44228.md, line 9
+/// Makefile-7-build.sh-2-stage -> file is Makefile, line 7
+/// ```
+///
+/// Both are `<stem>-<digits>-<name>.<ext>-<digits>-<rest>`. Nothing in
+/// the *text* separates them — only a filesystem could. The tier stops
+/// at the extension, so it reads the first correctly and the second
+/// wrong; the permissive rule takes the leftmost marker and does the
+/// reverse. That trade is deliberate: dated and CVE-style paths are
+/// everyday grep output, whereas the second shape needs a context line
+/// from an *extension-less* file whose body begins with a bare
+/// `name.ext-<digits>-` token. Both readings lose sometimes; this one
+/// loses far less often.
+/// 3. **Permissive tier** — the original leftmost-any rule, unchanged.
+/// Only reached when neither tier matched (e.g. a path containing a
+/// space), so behaviour for those lines is exactly as before.
fn parse_match_line(line: &str) -> Option<(&str, u64, &str)> {
+ scan_match_line(line, ScanTier::Colon)
+ .or_else(|| scan_match_line(line, ScanTier::Dash))
+ .or_else(|| scan_match_line(line, ScanTier::Permissive))
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ScanTier {
+ Colon,
+ Dash,
+ Permissive,
+}
+
+/// True when `tok` holds a dot that looks like a *file extension*: a run
+/// of 1–8 alphanumerics, at least one of them a letter, ending the token
+/// or bounded by a path/marker separator.
+///
+/// The letter requirement is what keeps a dotted *version* out of the
+/// extension test. `v1.2.3` and `rel.1.2` end in all-digit runs, so they
+/// are not extensions — which matters because they show up in the bodies
+/// of context lines from extension-less files (`git describe` output in a
+/// `Makefile`, a version bump in a `CHANGELOG`), and reading one as an
+/// extension is what lets the dash tier walk out of the path and into the
+/// body. `.md`, `.sql`, `.log`, `.7z` all still qualify.
+fn has_extension_dot(tok: &str) -> bool {
+ let b = tok.as_bytes();
+ b.iter().enumerate().any(|(i, &c)| {
+ if c != b'.' || i + 1 == b.len() {
+ return false;
+ }
+ let tail = &b[i + 1..];
+ let end = tail
+ .iter()
+ .position(|c| !c.is_ascii_alphanumeric())
+ .unwrap_or(tail.len());
+ let ext = &tail[..end];
+ (1..=8).contains(&ext.len()) && ext.iter().any(|c| c.is_ascii_alphabetic())
+ })
+}
+
+/// True when `path`'s final segment carries a file extension.
+///
+/// The dash tier uses this to tell "these digits are still inside the
+/// path" (`logs/2026`, no extension → keep scanning) from "these digits
+/// are the line-number marker" (`logs/2026-05-03/app.log` → stop).
+fn last_segment_has_extension(path: &str) -> bool {
+ has_extension_dot(path.rsplit(['/', '\\']).next().unwrap_or(path))
+}
+
+/// True when the token after this marker still carries *path structure* —
+/// a directory separator or an extension dot — before the next whitespace.
+///
+/// This is the positive evidence that the digits just matched were still
+/// inside the path rather than the line-number marker. Both forms occur:
+///
+/// ```text
+/// logs/2026-05-03/app.log -> the path continues via `/`
+/// migrations/20240101-002-add_users.sql -> ...and here via `.`, same directory
+/// ```
+///
+/// Its absence is *not* proof of the opposite, which is why the caller
+/// falls back to the leftmost marker rather than reading "no evidence" as
+/// "keep walking".
+fn path_continues(rest: &str) -> bool {
+ rest.split_whitespace()
+ .next()
+ .is_some_and(|tok| tok.contains('/') || tok.contains('\\') || has_extension_dot(tok))
+}
+
+fn scan_match_line(line: &str, tier: ScanTier) -> Option<(&str, u64, &str)> {
let bytes = line.as_bytes();
// Windows drive prefix: starts with [A-Za-z]:[\\/]
let scan_start = if bytes.len() >= 3
@@ -640,9 +767,24 @@ fn parse_match_line(line: &str) -> Option<(&str, u64, &str)> {
0
};
+ // Candidates as (path_end, digits_start, digits_end).
+ //
+ // `first` is the leftmost marker — the answer the permissive
+ // leftmost-wins rule gives, and the fallback when the dash tier
+ // cannot prove a better one. `chosen` is a marker positively
+ // confirmed as the path/body boundary.
+ let mut first: Option<(usize, usize, usize)> = None;
+ let mut chosen: Option<(usize, usize, usize)> = None;
+
let mut i = scan_start;
while i < bytes.len() {
- if bytes[i] == b':' || bytes[i] == b'-' {
+ let is_sep = bytes[i] == b':' || bytes[i] == b'-';
+ let tier_sep = match tier {
+ ScanTier::Colon => bytes[i] == b':',
+ ScanTier::Dash => bytes[i] == b'-',
+ ScanTier::Permissive => is_sep,
+ };
+ if tier_sep {
// Reject markers where the byte immediately before the
// first separator is itself a separator. That collapses
// adjacent-separator runs (`::` or `:-`) so a line like
@@ -654,29 +796,80 @@ fn parse_match_line(line: &str) -> Option<(&str, u64, &str)> {
i += 1;
continue;
}
+ // The typed tiers only consider markers that sit inside the
+ // leading whitespace-free run — grep paths don't contain
+ // whitespace, bodies routinely do.
+ if tier != ScanTier::Permissive && bytes[..i].iter().any(|b| b.is_ascii_whitespace()) {
+ break;
+ }
// Try this as the first separator. Walk through digits.
let digits_start = i + 1;
let mut j = digits_start;
while j < bytes.len() && bytes[j].is_ascii_digit() {
j += 1;
}
- if j > digits_start && j < bytes.len() && (bytes[j] == b':' || bytes[j] == b'-') {
+ // The closing separator must match the opening one: grep
+ // emits `file:12:body` or `file-12-body`, never a mix.
+ let closes = j > digits_start
+ && j < bytes.len()
+ && match tier {
+ ScanTier::Permissive => bytes[j] == b':' || bytes[j] == b'-',
+ _ => bytes[j] == bytes[i],
+ };
+ if closes {
// Found <sep><digits><sep>. Reject zero-length path
// (line starts with separator).
if i == 0 {
return None;
}
- let file = &line[..i];
- let line_no = std::str::from_utf8(&bytes[digits_start..j])
- .ok()
- .and_then(|s| s.parse::<u64>().ok())?;
- let content = &line[j + 1..];
- return Some((file, line_no, content));
+ if first.is_none() {
+ first = Some((i, digits_start, j));
+ }
+ if tier != ScanTier::Dash {
+ chosen = Some((i, digits_start, j));
+ break;
+ }
+ // Dash tier: advance past this marker only on *positive*
+ // evidence that the path runs through it. Stop when
+ // either
+ // (a) the path so far ends in a segment with an
+ // extension — that marker is the boundary
+ // (`logs/2026-05-03/app.log-12-…`), or
+ // (b) the rest of the token carries no further path
+ // structure — no `/` and no `.` — so nothing
+ // suggests the path continues
+ // (`CHANGELOG-12-2026-05-03`).
+ // Otherwise the digits are still inside the path
+ // (`logs/2026` + `03/app.log…`) and a later marker is the
+ // real one.
+ //
+ // Advancing on the *absence* of a stop signal is what
+ // breaks here: neither whitespace nor an extension is
+ // guaranteed to appear, so a walk bounded that way runs
+ // off the end of the path and latches onto a triplet in
+ // the *body* — silently, and for exactly the paths that
+ // carry no extension (`Makefile`, `Dockerfile`,
+ // `LICENSE`, `.github/workflows/ci`).
+ if last_segment_has_extension(&line[..i]) || !path_continues(&line[j + 1..]) {
+ chosen = Some((i, digits_start, j));
+ break;
+ }
+ i = j + 1;
+ continue;
}
}
i += 1;
}
- None
+
+ // Walked the whole token without ever confirming a boundary: the line
+ // is genuinely ambiguous (`Makefile-7-include-2-src/foo.mk`). Fall back
+ // to the leftmost marker — precisely what the leftmost-wins rule this
+ // tier refines would have returned.
+ let (path_end, digits_start, digits_end) = chosen.or(first)?;
+ let line_no = std::str::from_utf8(&bytes[digits_start..digits_end])
+ .ok()
+ .and_then(|s| s.parse::<u64>().ok())?;
+ Some((&line[..path_end], line_no, &line[digits_end + 1..]))
}
// ─── Internals ──────────────────────────────────────────────────────────
@@ -725,6 +918,138 @@ mod tests {
assert!(cjk_bigrams("a认b证").is_empty()); // isolated CJK chars -> no pair
}
+ #[test]
+ fn dash_tier_does_not_run_past_the_path_into_the_body() {
+ // The dash tier keeps the *last* `-<digits>-` marker so that a
+ // dated path (`logs/2026-05-03/…`) isn't split at its first
+ // triplet. But a context-line *body* can be whitespace-free and
+ // carry a triplet of its own — the walk must not follow it there.
+ assert_eq!(
+ parse_line("notes.md-3-see-4-here"),
+ Some(("notes.md".into(), 3, "see-4-here".into()))
+ );
+ assert_eq!(
+ parse_line("CHANGELOG.md-7-v1-2-beta"),
+ Some(("CHANGELOG.md".into(), 7, "v1-2-beta".into()))
+ );
+ }
+
+ #[test]
+ fn parses_context_line_for_extensionless_path() {
+ // Files with no extension are ordinary grep targets, and nothing in
+ // such a line ever gives the walk an extension to stop at. Bounding
+ // the walk on that signal alone splits the path *and* lifts the line
+ // number out of the body — silently. The tier must instead fall back
+ // to the leftmost marker for these.
+ for (line, file, no, body) in [
+ ("CHANGELOG-12-2026-05-03", "CHANGELOG", 12, "2026-05-03"),
+ (
+ "LICENSE-1-Copyright-2026-Acme",
+ "LICENSE",
+ 1,
+ "Copyright-2026-Acme",
+ ),
+ ("Makefile-7-build-2-stage", "Makefile", 7, "build-2-stage"),
+ (
+ "Dockerfile-3-FROM-18-alpine",
+ "Dockerfile",
+ 3,
+ "FROM-18-alpine",
+ ),
+ ("bin/run-3-exec-9-now", "bin/run", 3, "exec-9-now"),
+ (
+ ".github/workflows/ci-9-2026-01-02",
+ ".github/workflows/ci",
+ 9,
+ "2026-01-02",
+ ),
+ ] {
+ assert_eq!(
+ parse_line(line),
+ Some((file.into(), no, body.into())),
+ "extension-less path mis-parsed: {line}"
+ );
+ }
+ }
+
+ #[test]
+ fn dated_dir_still_wins_for_an_extensionless_file() {
+ // The case this tier exists for and the case above, on one line: the
+ // walk must cross the dated directory (the rest of the token still
+ // has a `/`) and stop at `Makefile`, which has no extension to
+ // announce itself with.
+ assert_eq!(
+ parse_line("logs/2026-05-03/Makefile-12-all:"),
+ Some(("logs/2026-05-03/Makefile".into(), 12, "all:".into()))
+ );
+ }
+
+ #[test]
+ fn ambiguous_dash_line_falls_back_to_the_leftmost_marker() {
+ // No extension anywhere and a `/` still ahead: the walk never gets
+ // positive evidence either way. It must then return what the
+ // leftmost-wins rule returns rather than guessing — this tier is
+ // never worse than the rule it refines.
+ assert_eq!(
+ parse_line("Makefile-7-include-2-src/foo.mk"),
+ Some(("Makefile".into(), 7, "include-2-src/foo.mk".into()))
+ );
+ }
+
+ #[test]
+ fn a_dotted_version_in_the_body_is_not_an_extension() {
+ // The evidence that "the path continues" must not fire on a dotted
+ // *version* sitting in the body of an extension-less file. The
+ // classic source is `git describe` output — `v1.2.3-4-gdeadbee`
+ // carries both a dot and a `-<digits>-` run, so reading its `.3`
+ // as an extension walks the parse clean out of the path and makes
+ // the marker the `-4-`, inventing the file `Makefile-9-VERSION=v1.2.3`.
+ // An extension needs a letter; `.3` and `.2` are not extensions.
+ for line in [
+ "Makefile-9-VERSION=v1.2.3-4-gdeadbee",
+ "Dockerfile-4-TAG=v2.0.1-3-gabc",
+ "CHANGELOG-2-rel.1.2-3-final",
+ "LICENSE-1-v1.0-2-clause",
+ ] {
+ let (file, _, _) = parse_line(line).expect("parses");
+ assert!(
+ !file.contains('.'),
+ "{line}: dotted version read as a path extension, got file {file:?}"
+ );
+ }
+
+ assert_eq!(
+ parse_line("Makefile-9-VERSION=v1.2.3-4-gdeadbee"),
+ Some(("Makefile".into(), 9, "VERSION=v1.2.3-4-gdeadbee".into()))
+ );
+ }
+
+ #[test]
+ fn a_real_extension_still_counts_even_with_digits_in_it() {
+ // The letter requirement must not cost us genuine extensions that
+ // carry digits (`.mp3`, `.7z`, `.h264`): the walk still has to cross
+ // the dated directory and stop at the file.
+ assert_eq!(
+ parse_line("media/2026-05-03/track-2-final.mp3-8-id3"),
+ Some(("media/2026-05-03/track-2-final.mp3".into(), 8, "id3".into()))
+ );
+ }
+
+ #[test]
+ fn dated_path_inside_a_context_body_is_not_hijacked() {
+ // Same shape as the paths this PR fixes, but on the *body* side of
+ // the marker. Left unbounded, the scan reports a path that never
+ // existed and a line number taken from a date.
+ assert_eq!(
+ parse_line("manifest.txt-5-logs/2026-05-03/app.log"),
+ Some(("manifest.txt".into(), 5, "logs/2026-05-03/app.log".into()))
+ );
+ assert_eq!(
+ parse_line("index.md-2-advisories/CVE-2021-44228.md"),
+ Some(("index.md".into(), 2, "advisories/CVE-2021-44228.md".into()))
+ );
+ }
+
#[test]
fn parses_ripgrep_context_line() {
assert_eq!(
@@ -767,6 +1092,93 @@ mod tests {
);
}
+ #[test]
+ fn date_stamped_path_is_not_misread_as_line_number_marker() {
+ // `logs/2026-05-03/...` contains `-05-`, a `<sep><digits><sep>`
+ // triplet INSIDE the path. Leftmost-wins parsing takes it as the
+ // line-number marker and silently reports a bogus file, a bogus
+ // line number, and a truncated body.
+ assert_eq!(
+ parse_line("logs/2026-05-03/app.log:12:ERROR boom"),
+ Some(("logs/2026-05-03/app.log".into(), 12, "ERROR boom".into()))
+ );
+ // Same for the ripgrep context-line form.
+ assert_eq!(
+ parse_line("logs/2026-11-23/app.log-11-context before"),
+ Some((
+ "logs/2026-11-23/app.log".into(),
+ 11,
+ "context before".into()
+ ))
+ );
+ // And for version/ordinal path segments (`v1-2-beta`).
+ assert_eq!(
+ parse_line("src/v1-2-beta/mod.rs:3:fn x() {}"),
+ Some(("src/v1-2-beta/mod.rs".into(), 3, "fn x() {}".into()))
+ );
+ // `NAME-<digits>-<digits>` paths (CVE advisories, timestamped
+ // migrations) hide a triplet in the middle of the path token.
+ assert_eq!(
+ parse_line("advisories/CVE-2021-44228.md:8:Log4Shell"),
+ Some(("advisories/CVE-2021-44228.md".into(), 8, "Log4Shell".into()))
+ );
+ assert_eq!(
+ parse_line("migrations/20240101-002-add_users.sql-9- id BIGINT"),
+ Some((
+ "migrations/20240101-002-add_users.sql".into(),
+ 9,
+ " id BIGINT".into()
+ ))
+ );
+ }
+
+ #[test]
+ fn date_stamped_paths_are_not_collapsed_into_one_bogus_file() {
+ // The misparse doesn't just corrupt one line — the bogus path
+ // becomes the grouping key, so two unrelated files merge into a
+ // single `logs/2026` bucket and the model sees a path that
+ // doesn't exist. This is what reaches the LLM.
+ let compressor = SearchCompressor::new(SearchCompressorConfig::default());
+ let content = "\
+logs/2026-05-03/app.log:12:ERROR boom
+logs/2026-05-04/app.log:7:ERROR bang";
+ let mut stats = SearchCompressorStats::default();
+ let parsed = compressor.parse_search_results(content, &mut stats);
+
+ assert_eq!(stats.lines_unparsed, 0);
+ let files: Vec<&String> = parsed.keys().collect();
+ assert_eq!(
+ files,
+ vec!["logs/2026-05-03/app.log", "logs/2026-05-04/app.log"]
+ );
+ assert_eq!(parsed["logs/2026-05-03/app.log"].matches[0].line_number, 12);
+ assert_eq!(
+ parsed["logs/2026-05-03/app.log"].matches[0].content,
+ "ERROR boom"
+ );
+ }
+
+ #[test]
+ fn body_line_reference_does_not_hijack_a_context_line() {
+ // A `-`-separated context line whose BODY quotes a `file:line:`
+ // reference must still bind to the context marker, not the
+ // reference. Guards the whitespace-bounded colon tier.
+ assert_eq!(
+ parse_line("src/main.py-44-see foo.rs:12:bar"),
+ Some(("src/main.py".into(), 44, "see foo.rs:12:bar".into()))
+ );
+ }
+
+ #[test]
+ fn digit_terminated_path_still_parses_ripgrep_context_line() {
+ // Rotated logs end in a digit, so the context separator is
+ // digit-preceded — the last-marker rule must still land on it.
+ assert_eq!(
+ parse_line("logs/app.log.1-42-rotated line"),
+ Some(("logs/app.log.1".into(), 42, "rotated line".into()))
+ );
+ }
+
#[test]
fn preserves_colons_in_match_content() {
// Standard grep behavior: stop at the second separator, the rest