Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(log_compressor): keep and name pytest short-summary failures (#3828)

charlieannad90b32008dc4
diff --git a/crates/headroom-core/src/transforms/log_compressor.rs b/crates/headroom-core/src/transforms/log_compressor.rs
index 1c6a0a68e..baf6ce3be 100644
--- a/crates/headroom-core/src/transforms/log_compressor.rs
+++ b/crates/headroom-core/src/transforms/log_compressor.rs
@@ -896,6 +896,85 @@ fn is_summary_line(line: &str) -> bool {
false
}
+/// What a scan of pytest's "short test summary info" sections found, keyed by
+/// line number.
+struct PytestShortSummary {
+ /// One-line `FAILED <nodeid>` / `ERROR <nodeid>` entries -> node id.
+ entries: BTreeMap<usize, String>,
+ /// The `===` line closing each section, i.e. pytest's run totals
+ /// (`=== 20 failed, 380 passed in 41.02s ===`).
+ totals_lines: BTreeSet<usize>,
+ /// The first section header, if any section was found.
+ first_header: Option<usize>,
+}
+
+/// Scan for pytest's "short test summary info" sections.
+///
+/// A section opens on a complete `=== ... short test summary info ... ===`
+/// separator (a trailing `\r` is ignored, so CRLF logs work) and closes at
+/// the next `===` line or EOF, so a diagnostic line that merely mentions the
+/// phrase does not open one. For `FAILED <nodeid> - <msg>` the id is the text
+/// before ` - `. `parse_lines`, `select_lines` and `format_output` all use it,
+/// so retention and omission naming agree on which lines are entries.
+fn scan_pytest_short_summary(lines: &[&str]) -> PytestShortSummary {
+ let mut scan = PytestShortSummary {
+ entries: BTreeMap::new(),
+ totals_lines: BTreeSet::new(),
+ first_header: None,
+ };
+ let mut in_short_summary = false;
+
+ for (line_number, line) in lines.iter().enumerate() {
+ let recognition_line = line.strip_suffix('\r').unwrap_or(line);
+ if recognition_line.starts_with("===") {
+ let opens = recognition_line.ends_with("===")
+ && recognition_line.contains("short test summary info");
+ if opens {
+ scan.first_header.get_or_insert(line_number);
+ } else if in_short_summary {
+ scan.totals_lines.insert(line_number);
+ }
+ in_short_summary = opens;
+ continue;
+ }
+ if !in_short_summary {
+ continue;
+ }
+
+ let node_id = line
+ .strip_prefix("FAILED ")
+ .or_else(|| line.strip_prefix("ERROR "))
+ .map(|rest| rest.split_once(" - ").map_or(rest, |(id, _)| id).trim());
+ if let Some(node_id) = node_id.filter(|id| !id.is_empty()) {
+ scan.entries.insert(line_number, node_id.to_string());
+ }
+ }
+
+ scan
+}
+
+/// Map line number -> node id for pytest's short-summary `FAILED` / `ERROR`
+/// entries; see [`scan_pytest_short_summary`].
+fn pytest_short_summary_entries(lines: &[&str]) -> BTreeMap<usize, String> {
+ scan_pytest_short_summary(lines).entries
+}
+
+/// Lines the global cap must keep while a pytest short summary competes for
+/// it: each section's totals line and the first `E ` assertion line before
+/// the summary. Short-summary entries win equal-score ties, so without this
+/// reserve a long summary crowds out the run totals and every error message.
+fn pytest_cap_reserve(log_lines: &[LogLine], scan: &PytestShortSummary) -> BTreeSet<usize> {
+ let mut reserved = scan.totals_lines.clone();
+ let first_error_detail = log_lines
+ .iter()
+ .take_while(|line| scan.first_header.map_or(true, |h| line.line_number < h))
+ .find(|line| line.content.starts_with("E "));
+ if let Some(line) = first_error_detail {
+ reserved.insert(line.line_number);
+ }
+ reserved
+}
+
/// Extract an exception-type / error-code label from a single line, if the
/// line looks like an exception header or error declaration. Conservative:
/// returns `None` rather than guessing on ambiguous lines, so a generic log
@@ -1144,11 +1223,12 @@ impl LogCompressor {
let mut out: Vec<LogLine> = Vec::with_capacity(lines.len());
let mut active: Option<TraceFlavor> = None;
let mut trace_lines = 0usize;
+ let short_summary_entries = pytest_short_summary_entries(lines);
for (i, line) in lines.iter().enumerate() {
let mut entry = LogLine::new(i, *line);
entry.level = self.levels.classify(line);
- entry.is_summary = is_summary_line(line);
+ entry.is_summary = is_summary_line(line) || short_summary_entries.contains_key(&i);
// Stack-trace state machine: open on a new flavor match, then
// mark subsequent lines until the flavor terminates or we hit
@@ -1207,6 +1287,8 @@ impl LogCompressor {
stats: &mut LogCompressorStats,
) -> Vec<LogLine> {
let all_strings: Vec<&str> = log_lines.iter().map(|l| l.content.as_str()).collect();
+ let short_summary = scan_pytest_short_summary(&all_strings);
+ let short_summary_entries = &short_summary.entries;
let adaptive_max =
compute_optimal_k(&all_strings, bias, 10, Some(self.config.max_total_lines));
@@ -1321,12 +1403,43 @@ impl LogCompressor {
let mut ordered: Vec<LogLine> = selected.into_iter().collect();
if ordered.len() > adaptive_max {
+ let reserved = if self.config.keep_summary_lines && !short_summary_entries.is_empty() {
+ pytest_cap_reserve(log_lines, &short_summary)
+ } else {
+ BTreeSet::new()
+ };
+ for line in log_lines
+ .iter()
+ .filter(|l| reserved.contains(&l.line_number))
+ {
+ if !ordered
+ .iter()
+ .any(|kept| kept.line_number == line.line_number)
+ {
+ ordered.push(line.clone());
+ }
+ }
stats.lines_dropped_by_global_cap += ordered.len() - adaptive_max;
- // Sort by score desc, take top adaptive_max, restore line order.
+ // Sort reserved lines first, then by score desc; take top adaptive_max,
+ // restore line order.
ordered.sort_by(|a, b| {
- b.score
- .partial_cmp(&a.score)
- .unwrap_or(std::cmp::Ordering::Equal)
+ reserved
+ .contains(&b.line_number)
+ .cmp(&reserved.contains(&a.line_number))
+ .then_with(|| {
+ b.score
+ .partial_cmp(&a.score)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ })
+ .then_with(|| {
+ if self.config.keep_summary_lines {
+ short_summary_entries
+ .contains_key(&b.line_number)
+ .cmp(&short_summary_entries.contains_key(&a.line_number))
+ } else {
+ std::cmp::Ordering::Equal
+ }
+ })
.then_with(|| a.line_number.cmp(&b.line_number))
});
ordered.truncate(adaptive_max);
@@ -1398,6 +1511,15 @@ impl LogCompressor {
selected: &[LogLine],
all_lines: &[LogLine],
) -> (String, BTreeMap<String, u64>) {
+ let all_strings: Vec<&str> = all_lines.iter().map(|l| l.content.as_str()).collect();
+ let short_summary_entries = pytest_short_summary_entries(&all_strings);
+ let selected_numbers: BTreeSet<usize> =
+ selected.iter().map(|line| line.line_number).collect();
+ let omitted_short_summary_ids: Vec<&str> = short_summary_entries
+ .iter()
+ .filter(|(line_number, _)| !selected_numbers.contains(line_number))
+ .map(|(_, node_id)| node_id.as_str())
+ .collect();
let mut stats: BTreeMap<String, u64> = BTreeMap::new();
stats.insert("errors".into(), count_level(all_lines, LogLevel::Error));
stats.insert("fails".into(), count_level(all_lines, LogLevel::Fail));
@@ -1423,10 +1545,27 @@ impl LogCompressor {
}
}
if !summary_parts.is_empty() {
+ let omitted_names = if omitted_short_summary_ids.is_empty() {
+ String::new()
+ } else {
+ let shown = omitted_short_summary_ids
+ .iter()
+ .take(5)
+ .copied()
+ .collect::<Vec<_>>()
+ .join(", ");
+ let overflow = omitted_short_summary_ids.len().saturating_sub(5);
+ if overflow > 0 {
+ format!("; omitted: {shown}, +{overflow} more")
+ } else {
+ format!("; omitted: {shown}")
+ }
+ };
output.push(format!(
- "[{} lines omitted: {}]",
+ "[{} lines omitted: {}{}]",
omitted,
- summary_parts.join(", ")
+ summary_parts.join(", "),
+ omitted_names
));
}
}
@@ -1534,6 +1673,45 @@ mod tests {
LogCompressor::new(LogCompressorConfig::default())
}
+ fn pytest_failure_log(failure_count: usize, mixed: bool) -> Vec<String> {
+ let mut lines = (0..50)
+ .map(|i| format!("setup output line {i}"))
+ .collect::<Vec<_>>();
+ lines.push(
+ "=========================== short test summary info ==========================="
+ .into(),
+ );
+ for i in 0..failure_count {
+ let status = if mixed && i % 4 == 0 {
+ "ERROR"
+ } else {
+ "FAILED"
+ };
+ let suffix = if i % 3 == 0 {
+ " - AssertionError: expected value"
+ } else {
+ ""
+ };
+ lines.push(format!(
+ "{status} tests/test_generated.py::test_case{i}{suffix}"
+ ));
+ }
+ lines.push(format!(
+ "========================= {failure_count} failed in 1.00s ========================="
+ ));
+ lines
+ }
+
+ fn malformed_short_summary_diagnostic_log() -> Vec<String> {
+ let mut lines = (0..50)
+ .map(|i| format!("INFO setup output {i}"))
+ .collect::<Vec<_>>();
+ lines.push("=== diagnostic: short test summary info unavailable".into());
+ lines.extend((0..10).map(|i| format!("FAILED outside.py::test_{i}")));
+ lines.extend((10..20).map(|i| format!("ERROR outside.py::test_{i}")));
+ lines
+ }
+
#[test]
fn detects_pytest_format() {
let c = cmp();
@@ -1602,6 +1780,323 @@ mod tests {
assert_eq!(lines[2].level, LogLevel::Unknown);
}
+ #[test]
+ fn recognizes_pytest_short_summary_entries_by_section_and_line_number() {
+ let lines = [
+ "FAILED outside.py::test_lookalike",
+ "================ short test summary info ================",
+ "FAILED tests/test_a.py::test_short",
+ "ERROR tests/test_b.py::test_long - RuntimeError: boom",
+ "not a summary entry",
+ "================ 2 failed in 0.1s ================",
+ "FAILED outside.py::test_after",
+ "=== short test summary info ===",
+ "FAILED tests/test_c.py::test_eof - assertion failed",
+ ];
+
+ let entries = pytest_short_summary_entries(&lines);
+ assert_eq!(
+ entries,
+ BTreeMap::from([
+ (2, "tests/test_a.py::test_short".to_string()),
+ (3, "tests/test_b.py::test_long".to_string()),
+ (8, "tests/test_c.py::test_eof".to_string()),
+ ])
+ );
+
+ let parsed = cmp().parse_lines(&lines);
+ assert!(!parsed[0].is_summary);
+ assert!(parsed[2].is_summary);
+ assert!(parsed[3].is_summary);
+ assert!(!parsed[6].is_summary);
+ assert!(parsed[8].is_summary);
+ }
+
+ #[test]
+ fn recognizes_complete_crlf_short_summary_separators_and_preserves_content() {
+ let lines = [
+ "=== diagnostic: short test summary info unavailable\r",
+ "FAILED outside.py::test_before\r",
+ "=== short test summary info ===\r",
+ "FAILED tests/test_a.py::test_short\r",
+ "ERROR tests/test_b.py::test_long - RuntimeError: boom\r",
+ "=== 2 failed in 0.1s ===\r",
+ "FAILED outside.py::test_after\r",
+ "=== short test summary info ===\r",
+ "FAILED tests/test_c.py::test_eof - AssertionError\r",
+ ];
+
+ let entries = pytest_short_summary_entries(&lines);
+ assert_eq!(
+ entries,
+ BTreeMap::from([
+ (3, "tests/test_a.py::test_short".to_string()),
+ (4, "tests/test_b.py::test_long".to_string()),
+ (8, "tests/test_c.py::test_eof".to_string()),
+ ])
+ );
+
+ let parsed = cmp().parse_lines(&lines);
+ assert!(!parsed[1].is_summary);
+ assert!(parsed[3].is_summary);
+ assert!(parsed[4].is_summary);
+ assert!(!parsed[6].is_summary);
+ assert!(parsed[8].is_summary);
+ assert_eq!(parsed[3].content, lines[3]);
+ }
+
+ #[test]
+ fn keeps_all_crlf_short_summary_entries_with_non_binding_cap() {
+ let contents = pytest_failure_log(20, true)
+ .into_iter()
+ .map(|line| format!("{line}\r"))
+ .collect::<Vec<_>>();
+ let lines = contents.iter().map(String::as_str).collect::<Vec<_>>();
+ let c = LogCompressor::new(LogCompressorConfig {
+ max_errors: 2,
+ error_context_lines: 0,
+ max_total_lines: 1_000,
+ min_lines_for_ccr: 50,
+ enable_ccr: false,
+ ..Default::default()
+ });
+
+ let parsed = c.parse_lines(&lines);
+ let mut selection_stats = LogCompressorStats::default();
+ let selected = c.select_lines(&parsed, 1_000.0, &mut selection_stats);
+ assert_eq!(selection_stats.lines_dropped_by_global_cap, 0);
+ for i in 0..20 {
+ let expected_id = format!("tests/test_generated.py::test_case{i}");
+ assert!(selected
+ .iter()
+ .any(|line| line.content.contains(&expected_id)));
+ }
+
+ let (result, compression_stats) = c.compress(&contents.join("\n"), 1_000.0);
+ assert_eq!(compression_stats.lines_dropped_by_global_cap, 0);
+ assert!(result.compressed_line_count < result.original_line_count);
+ for i in 0..20 {
+ let expected = format!("tests/test_generated.py::test_case{i}");
+ assert!(result
+ .compressed
+ .lines()
+ .any(|line| line.contains(&expected)));
+ }
+ }
+
+ #[test]
+ fn malformed_short_summary_diagnostic_does_not_recognize_protect_or_name_entries() {
+ let contents = malformed_short_summary_diagnostic_log();
+ let lines = contents.iter().map(String::as_str).collect::<Vec<_>>();
+ assert_eq!(lines.len(), 71);
+ assert!(pytest_short_summary_entries(&lines).is_empty());
+
+ for keep_summary_lines in [false, true] {
+ let c = LogCompressor::new(LogCompressorConfig {
+ max_errors: 2,
+ error_context_lines: 0,
+ keep_summary_lines,
+ max_total_lines: 1_000,
+ enable_ccr: false,
+ ..Default::default()
+ });
+ let parsed = c.parse_lines(&lines);
+ assert!(parsed[51..].iter().all(|line| !line.is_summary));
+
+ let mut selection_stats = LogCompressorStats::default();
+ let selected = c.select_lines(&parsed, 1_000.0, &mut selection_stats);
+ assert_eq!(selection_stats.lines_dropped_by_global_cap, 0);
+ let selected_lookalikes = selected
+ .iter()
+ .filter(|line| matches!(line.level, LogLevel::Error | LogLevel::Fail))
+ .map(|line| line.line_number)
+ .collect::<Vec<_>>();
+ assert_eq!(selected_lookalikes, vec![51, 60, 61, 70]);
+
+ let (output, _) = c.format_output(&selected, &parsed);
+ assert!(!output.contains("; omitted: "));
+ if !keep_summary_lines {
+ assert_eq!(
+ output,
+ "FAILED outside.py::test_0\nFAILED outside.py::test_9\nERROR outside.py::test_10\nERROR outside.py::test_19\n[67 lines omitted: 10 ERROR, 10 FAIL, 51 INFO]"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn keeps_all_short_summary_entries_with_non_binding_cap() {
+ for (failure_count, mixed) in [(15, false), (16, false), (20, true), (200, false)] {
+ let lines = pytest_failure_log(failure_count, mixed);
+ let refs = lines.iter().map(String::as_str).collect::<Vec<_>>();
+ let c = LogCompressor::new(LogCompressorConfig {
+ max_errors: 10,
+ error_context_lines: 0,
+ max_total_lines: 1_000,
+ min_lines_for_ccr: 50,
+ enable_ccr: false,
+ ..Default::default()
+ });
+
+ let parsed = c.parse_lines(&refs);
+ let mut selection_stats = LogCompressorStats::default();
+ let selected = c.select_lines(&parsed, 1_000.0, &mut selection_stats);
+ assert_eq!(selection_stats.lines_dropped_by_global_cap, 0);
+ let selected_contents = selected
+ .iter()
+ .map(|line| line.content.as_str())
+ .collect::<BTreeSet<_>>();
+ for i in 0..failure_count {
+ let expected_id = format!("tests/test_generated.py::test_case{i}");
+ assert!(
+ selected_contents.iter().any(|line| {
+ line.strip_prefix("FAILED ")
+ .or_else(|| line.strip_prefix("ERROR "))
+ .map(|rest| rest.split_once(" - ").map_or(rest, |(id, _)| id))
+ == Some(expected_id.as_str())
+ }),
+ "test_case{i} was not retained for {failure_count} failures"
+ );
+ }
+
+ let content = lines.join("\n");
+ let (result, compression_stats) = c.compress(&content, 1_000.0);
+ assert_eq!(compression_stats.lines_dropped_by_global_cap, 0);
+ assert!(result.compressed_line_count < result.original_line_count);
+ for i in 0..failure_count {
+ let expected_id = format!("tests/test_generated.py::test_case{i}");
+ assert!(
+ result.compressed.lines().any(|line| {
+ line.strip_prefix("FAILED ")
+ .or_else(|| line.strip_prefix("ERROR "))
+ .map(|rest| rest.split_once(" - ").map_or(rest, |(id, _)| id))
+ == Some(expected_id.as_str())
+ }),
+ "test_case{i} was not in compressed output for {failure_count} failures"
+ );
+ }
+ }
+ }
+
+ /// The issue #3814 reproduction shape, with a distinct `E ` message per
+ /// failure so tests can tell which failure's detail survived.
+ fn pytest_issue_log(failure_count: usize) -> String {
+ let mut lines = vec![
+ "============================= test session starts ============================="
+ .to_string(),
+ "collected 400 items".to_string(),
+ String::new(),
+ ];
+ lines.extend(
+ (0..40).map(|i| {
+ format!("tests/test_module_{i:02}.py .......................... [ {i:2}%]")
+ }),
+ );
+ lines.push(
+ "=================================== FAILURES =================================="
+ .into(),
+ );
+ for i in 1..=failure_count {
+ lines.extend([
+ format!("____________________ test_case{i:03} ____________________"),
+ String::new(),
+ "> assert result == expected".to_string(),
+ format!("E AssertionError: mismatch in test_case{i:03}"),
+ String::new(),
+ "tests/t.py:42: AssertionError".to_string(),
+ ]);
+ }
+ lines.push(
+ "=========================== short test summary info ==========================="
+ .into(),
+ );
+ lines.extend(
+ (1..=failure_count)
+ .map(|i| format!("FAILED tests/t.py::test_case{i:03} - AssertionError: mismatch")),
+ );
+ lines.push(format!(
+ "=============== {failure_count} failed, 380 passed in 41.02s =============="
+ ));
+ lines.join("\n")
+ }
+
+ #[test]
+ fn binding_cap_reserves_pytest_totals_and_first_error_line() {
+ for failure_count in [60, 200] {
+ let (result, stats) = cmp().compress(&pytest_issue_log(failure_count), 1.0);
+ assert!(stats.lines_dropped_by_global_cap > 0, "{failure_count}");
+ let out = &result.compressed;
+ assert!(
+ out.contains(&format!("{failure_count} failed, 380 passed")),
+ "totals line dropped at {failure_count}: {out}"
+ );
+ assert!(
+ out.contains("E AssertionError: mismatch in test_case001"),
+ "first failure's E line dropped at {failure_count}: {out}"
+ );
+
+ // Every entry is still kept or named: kept + listed + K == total.
+ let kept = out
+ .lines()
+ .filter(|l| l.starts_with("FAILED tests/t.py::"))
+ .count();
+ let marker = out.lines().last().unwrap();
+ let named = marker
+ .split("; omitted: ")
+ .nth(1)
+ .expect("omitted entries must be named")
+ .trim_end_matches(']');
+ let listed = named.split(", ").filter(|p| !p.starts_with('+')).count();
+ let overflow = named
+ .rsplit(", +")
+ .next()
+ .and_then(|tail| tail.strip_suffix(" more"))
+ .map_or(0, |k| k.parse::<usize>().unwrap());
+ assert_eq!(kept + listed + overflow, failure_count, "{marker}");
+ }
+ }
+
+ #[test]
+ fn global_cap_precedence_is_flag_dependent_and_deterministic() {
+ let mut contents = (0..10)
+ .map(|i| format!("FAILED traceback.py::test_early_{i}\r"))
+ .collect::<Vec<_>>();
+ contents.push("================ short test summary info ================\r".into());
+ contents
+ .extend((0..10).map(|i| format!("FAILED tests/test_summary.py::test_summary_{i}\r")));
+ contents.push("================ 10 failed in 0.1s ================\r".into());
+ let lines = contents.iter().map(String::as_str).collect::<Vec<_>>();
+
+ // keep_summary_lines=true: the totals line (21) is reserved, and the
+ // short-summary entries win the remaining equal-score ties.
+ for (keep_summary_lines, expected) in [
+ (true, (11..20).chain([21]).collect::<Vec<_>>()),
+ (false, (0..10).collect::<Vec<_>>()),
+ ] {
+ let c = LogCompressor::new(LogCompressorConfig {
+ max_errors: 20,
+ error_context_lines: 0,
+ keep_summary_lines,
+ max_total_lines: 10,
+ ..Default::default()
+ });
+ let parsed = c.parse_lines(&lines);
+ let mut stats = LogCompressorStats::default();
+ let selected = c.select_lines(&parsed, 1.0, &mut stats);
+ assert!(
+ stats.lines_dropped_by_global_cap > 0,
+ "keep_summary_lines={keep_summary_lines}"
+ );
+ assert_eq!(
+ selected
+ .iter()
+ .map(|line| line.line_number)
+ .collect::<Vec<_>>(),
+ expected
+ );
+ }
+ }
+
#[test]
fn fixed_in_3e5_chained_exception_traces_survive_blank_lines() {
// Python machine terminated stack trace on first blank line,
@@ -1850,6 +2345,115 @@ mod tests {
assert_eq!(stats["info"], 2);
}
+ #[test]
+ fn format_output_names_all_omitted_short_summary_entries_with_bounded_suffix() {
+ for count in [3, 5, 7] {
+ let mut contents = vec!["=== short test summary info ===\r".to_string()];
+ for i in 0..count {
+ contents.push(format!("FAILED tests/test_ids.py::test_{i}\r"));
+ }
+ contents.push("=== failures complete ===\r".to_string());
+ let refs = contents.iter().map(String::as_str).collect::<Vec<_>>();
+ let all_lines = cmp().parse_lines(&refs);
+ let selected = vec![all_lines[0].clone(), all_lines[count + 1].clone()];
+
+ let (output, _) = cmp().format_output(&selected, &all_lines);
+ let shown = (0..count.min(5))
+ .map(|i| format!("tests/test_ids.py::test_{i}"))
+ .collect::<Vec<_>>()
+ .join(", ");
+ let overflow = if count > 5 {
+ format!(", +{} more", count - 5)
+ } else {
+ String::new()
+ };
+ assert!(
+ output.ends_with(&format!(
+ "[{count} lines omitted: {count} FAIL, 1 INFO; omitted: {shown}{overflow}]"
+ )),
+ "{output}"
+ );
+ }
+ }
+
+ #[test]
+ fn format_output_omission_naming_tracks_identity_order_and_repeated_ids() {
+ let contents = [
+ "=== short test summary info ===",
+ "ERROR tests/test_ids.py::test_error - RuntimeError: boom",
+ "FAILED tests/test_ids.py::test_repeat",
+ "FAILED tests/test_ids.py::test_repeat - first failure",
+ "FAILED tests/test_ids.py::test_kept",
+ "=== failures complete ===",
+ ];
+ let all_lines = cmp().parse_lines(&contents);
+ let selected = vec![
+ all_lines[0].clone(),
+ all_lines[4].clone(),
+ all_lines[5].clone(),
+ ];
+
+ let (output, stats) = cmp().format_output(&selected, &all_lines);
+ assert!(
+ output.ends_with(
+ "[3 lines omitted: 1 ERROR, 3 FAIL, 1 INFO; omitted: tests/test_ids.py::test_error, tests/test_ids.py::test_repeat, tests/test_ids.py::test_repeat]"
+ ),
+ "{output}"
+ );
+ assert_eq!(stats["errors"], 1);
+ assert_eq!(stats["fails"], 3);
+ }
+
+ #[test]
+ fn keep_summary_lines_false_names_entries_omitted_by_category_selection() {
+ let contents = [
+ "=== short test summary info ===",
+ "FAILED tests/test_ids.py::test_0",
+ "FAILED tests/test_ids.py::test_1",
+ "FAILED tests/test_ids.py::test_2",
+ "FAILED tests/test_ids.py::test_3",
+ "=== failures complete ===",
+ ];
+ let c = LogCompressor::new(LogCompressorConfig {
+ max_errors: 2,
+ error_context_lines: 0,
+ keep_summary_lines: false,
+ max_total_lines: 100,
+ ..Default::default()
+ });
+ let all_lines = c.parse_lines(&contents);
+ let mut selection_stats = LogCompressorStats::default();
+ let selected = c.select_lines(&all_lines, 100.0, &mut selection_stats);
+ assert_eq!(selection_stats.lines_dropped_by_global_cap, 0);
+ assert_eq!(
+ selected
+ .iter()
+ .map(|line| line.line_number)
+ .collect::<Vec<_>>(),
+ vec![1, 4]
+ );
+
+ let (output, _) = c.format_output(&selected, &all_lines);
+ assert!(
+ output.ends_with(
+ "[4 lines omitted: 4 FAIL, 1 INFO; omitted: tests/test_ids.py::test_1, tests/test_ids.py::test_2]"
+ ),
+ "{output}"
+ );
+ }
+
+ #[test]
+ fn format_output_marker_is_unchanged_without_omitted_short_summary_entries() {
+ let c = cmp();
+ let mut error = LogLine::new(0, "ERROR a");
+ error.level = LogLevel::Error;
+ let mut info = LogLine::new(1, "INFO b");
+ info.level = LogLevel::Info;
+
+ let (output, _) = c.format_output(&[error.clone()], &[error, info]);
+ assert_eq!(output, "ERROR a\n[1 lines omitted: 1 ERROR, 1 INFO]");
+ }
+
#[test]
fn score_line_caps_at_one_point_zero() {
let line = LogLine {
diff --git a/headroom/transforms/log_compressor.py b/headroom/transforms/log_compressor.py
index 0fc798876..922a516d6 100644
--- a/headroom/transforms/log_compressor.py
+++ b/headroom/transforms/log_compressor.py
@@ -93,6 +93,89 @@ class LogLine:
return hash(self.line_number)
+def _scan_pytest_short_summary(
+ lines: list[tuple[int, str]],
+) -> tuple[dict[int, str], set[int], int | None]:
+ """Scan for pytest's "short test summary info" sections.
+
+ Mirrors Rust's ``scan_pytest_short_summary``: a section opens on a
+ complete ``=== ... short test summary info ... ===`` separator (a trailing
+ ``\\r`` is ignored) and closes at the next ``===`` line or EOF. For
+ ``FAILED <nodeid> - <msg>`` the id is the text before ``" - "``.
+
+ Args:
+ lines: ``(line_number, content)`` pairs in log order.
+
+ Returns:
+ ``(entries, totals_lines, first_header)``: line number -> node id for
+ each entry, the line numbers of the ``===`` lines closing a section
+ (pytest's run totals), and the first section header's line number.
+ """
+ entries: dict[int, str] = {}
+ totals_lines: set[int] = set()
+ first_header: int | None = None
+ in_short_summary = False
+
+ for line_number, line in lines:
+ recognition_line = line.removesuffix("\r")
+ if recognition_line.startswith("==="):
+ opens = (
+ recognition_line.endswith("===") and "short test summary info" in recognition_line
+ )
+ if opens:
+ if first_header is None:
+ first_header = line_number
+ elif in_short_summary:
+ totals_lines.add(line_number)
+ in_short_summary = opens
+ continue
+ if not in_short_summary:
+ continue
+
+ rest = next(
+ (
+ line.removeprefix(prefix)
+ for prefix in ("FAILED ", "ERROR ")
+ if line.startswith(prefix)
+ ),
+ None,
+ )
+ if rest is not None:
+ node_id = rest.split(" - ", 1)[0].strip()
+ if node_id:
+ entries[line_number] = node_id
+
+ return entries, totals_lines, first_header
+
+
+def _pytest_short_summary_entries(lines: list[tuple[int, str]]) -> dict[int, str]:
+ """Map line numbers to node ids of pytest short-summary FAILED/ERROR entries.
+
+ See ``_scan_pytest_short_summary``.
+ """
+ return _scan_pytest_short_summary(lines)[0]
+
+
+def _pytest_cap_reserve(
+ log_lines: list[LogLine], totals_lines: set[int], first_header: int | None
+) -> set[int]:
+ """Return the lines the global cap must keep while a short summary competes.
+
+ Mirrors Rust's ``pytest_cap_reserve``: each section's totals line and the
+ first ``E `` assertion line before the summary. Short-summary entries win
+ equal-score ties, so without this reserve a long summary crowds out the run
+ totals and every error message.
+ """
+ reserved = set(totals_lines)
+ for line in log_lines:
+ if first_header is not None and line.line_number >= first_header:
+ break
+ if line.content.startswith("E "):
+ reserved.add(line.line_number)
+ break
+ return reserved
+
+
@dataclass
class LogCompressorConfig:
"""Configuration for log compression."""
@@ -289,6 +372,7 @@ class LogCompressor:
log_lines: list[LogLine] = []
in_stack_trace = False
stack_trace_lines = 0
+ short_summary_entries = _pytest_short_summary_entries(list(enumerate(lines)))
for i, line in enumerate(lines):
log_line = LogLine(line_number=i, content=line)
@@ -314,6 +398,7 @@ class LogCompressor:
if pattern.search(line):
log_line.is_summary = True
break
+ log_line.is_summary = log_line.is_summary or i in short_summary_entries
log_line.score = self._score_line(log_line)
log_lines.append(log_line)
@@ -343,6 +428,9 @@ class LogCompressor:
from headroom.transforms.adaptive_sizer import compute_optimal_k
all_strings = [line.content for line in log_lines]
+ short_summary_entries, totals_lines, first_header = _scan_pytest_short_summary(
+ [(line.line_number, line.content) for line in log_lines]
+ )
adaptive_max = compute_optimal_k(
all_strings, bias=bias, min_k=10, max_k=self.config.max_total_lines
)
@@ -389,7 +477,27 @@ class LogCompressor:
selected = sorted(set(selected), key=lambda x: x.line_number)
if len(selected) > adaptive_max:
- selected = sorted(selected, key=lambda x: x.score, reverse=True)
+ if self.config.keep_summary_lines:
+ reserved: set[int] = set()
+ if short_summary_entries:
+ reserved = _pytest_cap_reserve(log_lines, totals_lines, first_header)
+ present = {line.line_number for line in selected}
+ selected.extend(
+ line
+ for line in log_lines
+ if line.line_number in reserved and line.line_number not in present
+ )
+ selected = sorted(
+ selected,
+ key=lambda line: (
+ -(line.line_number in reserved),
+ -line.score,
+ -(line.line_number in short_summary_entries),
+ line.line_number,
+ ),
+ )
+ else:
+ selected = sorted(selected, key=lambda line: (-line.score, line.line_number))
selected = selected[:adaptive_max]
selected = sorted(selected, key=lambda x: x.line_number)
@@ -461,6 +569,15 @@ class LogCompressor:
def _format_output(
self, selected: list[LogLine], all_lines: list[LogLine]
) -> tuple[str, dict[str, int]]:
+ short_summary_entries = _pytest_short_summary_entries(
+ [(line.line_number, line.content) for line in all_lines]
+ )
+ selected_numbers = {line.line_number for line in selected}
+ omitted_short_summary_ids = [
+ node_id
+ for line_number, node_id in short_summary_entries.items()
+ if line_number not in selected_numbers
+ ]
stats: dict[str, int] = {
"errors": sum(1 for line in all_lines if line.level == LogLevel.ERROR),
"fails": sum(1 for line in all_lines if line.level == LogLevel.FAIL),
@@ -483,7 +600,15 @@ class LogCompressor:
if count > 0:
summary_parts.append(f"{count} {label}")
if summary_parts:
- output_lines.append(f"[{omitted} lines omitted: {', '.join(summary_parts)}]")
+ omitted_names = ""
+ if omitted_short_summary_ids:
+ omitted_names = "; omitted: " + ", ".join(omitted_short_summary_ids[:5])
+ overflow = len(omitted_short_summary_ids) - 5
+ if overflow > 0:
+ omitted_names += f", +{overflow} more"
+ output_lines.append(
+ f"[{omitted} lines omitted: {', '.join(summary_parts)}{omitted_names}]"
+ )
return "\n".join(output_lines), stats
def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:
diff --git a/tests/test_transforms_log_compressor.py b/tests/test_transforms_log_compressor.py
index 777840186..6ce409594 100644
--- a/tests/test_transforms_log_compressor.py
+++ b/tests/test_transforms_log_compressor.py
@@ -50,6 +50,100 @@ def test_detect_parse_and_score_log_lines() -> None:
)
+def test_pytest_short_summary_helper_recognizes_only_section_entries() -> None:
+ compressor = LogCompressor()
+ lines = [
+ "FAILED outside.py::test_before",
+ "=== short test summary info ===",
+ "FAILED tests/test_a.py::test_short",
+ "ERROR tests/test_b.py::test_long - RuntimeError: boom",
+ "=== 2 failed in 0.1s ===",
+ "FAILED outside.py::test_after",
+ "=== short test summary info ===",
+ "FAILED tests/test_c.py::test_eof - AssertionError",
+ ]
+
+ parsed = compressor._parse_lines(lines)
+
+ assert parsed[0].is_summary is False
+ assert parsed[2].is_summary is True
+ assert parsed[3].is_summary is True
+ assert parsed[5].is_summary is False
+ assert parsed[7].is_summary is True
+
+
+def test_pytest_short_summary_helper_accepts_crlf_complete_separators() -> None:
+ compressor = LogCompressor()
+ lines = [
+ "=== diagnostic: short test summary info unavailable\r",
+ "FAILED outside.py::test_before\r",
+ "=== short test summary info ===\r",
+ "FAILED tests/test_a.py::test_short\r",
+ "ERROR tests/test_b.py::test_long - RuntimeError: boom\r",
+ "=== 2 failed in 0.1s ===\r",
+ "FAILED outside.py::test_after\r",
+ "=== short test summary info ===\r",
+ "FAILED tests/test_c.py::test_eof - AssertionError\r",
+ ]
+
+ parsed = compressor._parse_lines(lines)
+
+ assert parsed[1].is_summary is False
+ assert parsed[3].is_summary is True
+ assert parsed[4].is_summary is True
+ assert parsed[6].is_summary is False
+ assert parsed[8].is_summary is True
+ assert parsed[3].content == lines[3]
+
+
+def _malformed_short_summary_diagnostic_log() -> list[str]:
+ lines = [f"INFO setup output {i}" for i in range(50)]
+ lines.append("=== diagnostic: short test summary info unavailable")
+ lines.extend(f"FAILED outside.py::test_{i}" for i in range(10))
+ lines.extend(f"ERROR outside.py::test_{i}" for i in range(10, 20))
+ return lines
+
+
[email protected]("keep_summary_lines", [False, True])
+def test_malformed_short_summary_diagnostic_does_not_affect_legacy_helpers(
+ monkeypatch: pytest.MonkeyPatch, keep_summary_lines: bool
+) -> None:
+ compressor = LogCompressor(
+ LogCompressorConfig(
+ max_errors=2,
+ error_context_lines=0,
+ keep_summary_lines=keep_summary_lines,
+ max_total_lines=1_000,
+ enable_ccr=False,
+ )
+ )
+ monkeypatch.setitem(
+ __import__("sys").modules,
+ "headroom.transforms.adaptive_sizer",
+ SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 1_000),
+ )
+ parsed = compressor._parse_lines(_malformed_short_summary_diagnostic_log())
+
+ assert len(parsed) == 71
+ assert all(not line.is_summary for line in parsed[51:])
+ selected = compressor._select_lines(parsed)
+ selected_lookalikes = [
+ line.line_number for line in selected if line.level in (LogLevel.ERROR, LogLevel.FAIL)
+ ]
+ assert selected_lookalikes == [51, 60, 61, 70]
+
+ output, _ = compressor._format_output(selected, parsed)
+ assert "; omitted: " not in output
+ if not keep_summary_lines:
+ assert output == (
+ "FAILED outside.py::test_0\n"
+ "FAILED outside.py::test_9\n"
+ "ERROR outside.py::test_10\n"
+ "ERROR outside.py::test_19\n"
+ "[67 lines omitted: 10 ERROR, 10 FAIL, 51 INFO]"
+ )
+
+
def test_select_dedupe_add_context_and_format_output(monkeypatch: pytest.MonkeyPatch) -> None:
compressor = LogCompressor(
LogCompressorConfig(
@@ -116,6 +210,127 @@ def test_select_dedupe_add_context_and_format_output(monkeypatch: pytest.MonkeyP
assert output.endswith("[3 lines omitted: 1 ERROR, 1 FAIL, 2 WARN, 1 INFO]")
+ ("keep_summary_lines", "expected"),
+ # True: the totals line (7) is reserved, and entries win the remaining ties.
+ [(True, [4, 5, 7]), (False, [0, 1, 4])],
+)
+def test_short_summary_global_cap_precedence_matches_rust(
+ monkeypatch: pytest.MonkeyPatch, keep_summary_lines: bool, expected: list[int]
+) -> None:
+ compressor = LogCompressor(
+ LogCompressorConfig(
+ max_errors=10,
+ error_context_lines=0,
+ keep_summary_lines=keep_summary_lines,
+ max_total_lines=3,
+ )
+ )
+ monkeypatch.setitem(
+ __import__("sys").modules,
+ "headroom.transforms.adaptive_sizer",
+ SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 3),
+ )
+ parsed = compressor._parse_lines(
+ [
+ "FAILED traceback.py::test_early_one\r",
+ "FAILED traceback.py::test_early_two\r",
+ "ordinary output\r",
+ "=== short test summary info ===\r",
+ "FAILED tests/test_summary.py::test_one\r",
+ "ERROR tests/test_summary.py::test_two\r",
+ "FAILED tests/test_summary.py::test_three\r",
+ "=== 3 failed in 0.1s ===\r",
+ ]
+ )
+
+ assert [line.line_number for line in compressor._select_lines(parsed)] == expected
+
+
+def test_legacy_cap_reserves_totals_and_first_error_line(monkeypatch: pytest.MonkeyPatch) -> None:
+ compressor = LogCompressor(
+ LogCompressorConfig(max_errors=10, error_context_lines=0, max_total_lines=4)
+ )
+ monkeypatch.setitem(
+ __import__("sys").modules,
+ "headroom.transforms.adaptive_sizer",
+ SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 4),
+ )
+ parsed = compressor._parse_lines(
+ [
+ "E AssertionError: first",
+ "E AssertionError: second",
+ "=== short test summary info ===",
+ "FAILED t.py::a",
+ "FAILED t.py::b",
+ "FAILED t.py::c",
+ "FAILED t.py::d",
+ "=== 4 failed in 0.1s ===",
+ ]
+ )
+
+ # The first E line (0) and the totals line (7) are reserved; entries fill the rest.
+ assert [line.line_number for line in compressor._select_lines(parsed)] == [0, 3, 4, 7]
+
+
+ ("count", "suffix"),
+ [
+ (
+ 3,
+ "; omitted: tests/test_ids.py::test_0, tests/test_ids.py::test_1, "
+ "tests/test_ids.py::test_2",
+ ),
+ (
+ 5,
+ "; omitted: tests/test_ids.py::test_0, tests/test_ids.py::test_1, "
+ "tests/test_ids.py::test_2, tests/test_ids.py::test_3, tests/test_ids.py::test_4",
+ ),
+ (
+ 7,
+ "; omitted: tests/test_ids.py::test_0, tests/test_ids.py::test_1, "
+ "tests/test_ids.py::test_2, tests/test_ids.py::test_3, tests/test_ids.py::test_4, "
+ "+2 more",
+ ),
+ ],
+)
+def test_short_summary_omission_naming_is_bounded(count: int, suffix: str) -> None:
+ compressor = LogCompressor(LogCompressorConfig(keep_summary_lines=False))
+ contents = ["=== short test summary info ===\r"] + [
+ f"FAILED tests/test_ids.py::test_{i}\r" for i in range(count)
+ ]
+ contents.append("=== failures complete ===\r")
+ all_lines = compressor._parse_lines(contents)
+ selected = [all_lines[0], all_lines[-1]]
+
+ output, _ = compressor._format_output(selected, all_lines)
+
+ assert output.endswith(f"[{count} lines omitted: {count} FAIL, 1 INFO{suffix}]")
+
+
+def test_short_summary_omission_naming_uses_ids_and_preserves_duplicates() -> None:
+ compressor = LogCompressor(LogCompressorConfig(keep_summary_lines=False))
+ all_lines = compressor._parse_lines(
+ [
+ "=== short test summary info ===",
+ "ERROR tests/test_ids.py::test_error - RuntimeError: boom",
+ "FAILED tests/test_ids.py::test_repeat",
+ "FAILED tests/test_ids.py::test_repeat - AssertionError",
+ "FAILED tests/test_ids.py::test_kept",
+ "=== failures complete ===",
+ ]
+ )
+
+ output, stats = compressor._format_output([all_lines[0], all_lines[4], all_lines[5]], all_lines)
+
+ assert output.endswith(
+ "[3 lines omitted: 1 ERROR, 3 FAIL, 1 INFO; omitted: tests/test_ids.py::test_error, "
+ "tests/test_ids.py::test_repeat, tests/test_ids.py::test_repeat]"
+ )
+ assert stats["errors"] == 1
+ assert stats["fails"] == 3
+
+
def test_log_compressor_compress_and_ccr_paths() -> None:
"""Phase 3e.5: `compress()` is now a single Rust call, so this test
exercises end-to-end behavior instead of monkeypatching internal
@@ -143,6 +358,150 @@ def test_log_compressor_compress_and_ccr_paths() -> None:
assert too_short.cache_key is None
+def _pytest_reproduction(failure_count: int, line_ending: str = "\n") -> str:
+ lines = [f"pytest setup output {i}" for i in range(20)]
+ for i in range(20):
+ lines.extend(
+ [
+ f"FAILED traceback detail {i}",
+ f"E AssertionError: failure detail {i}",
+ ]
+ )
+ lines.append("=========================== short test summary info ===========================")
+ lines.extend(
+ f"FAILED tests/test_generated.py::test_case{i} - AssertionError: failure {i}"
+ for i in range(failure_count)
+ )
+ lines.append(
+ f"========================= {failure_count} failed in 1.00s ========================="
+ )
+ return line_ending.join(lines)
+
+
+def _retained_or_named(output: str, node_id: str) -> bool:
+ for line in output.splitlines():
+ if line.startswith(("FAILED ", "ERROR ")):
+ entry_id = line.split(" ", 1)[1].split(" - ", 1)[0]
+ if entry_id == node_id:
+ return True
+ if line.startswith("[") and "; omitted: " in line:
+ named = line.split("; omitted: ", 1)[1].removesuffix("]").split(", ")
+ if node_id in named:
+ return True
+ return False
+
+
[email protected](("failure_count", "first_regression"), [(16, 13), (20, 13)])
+def test_real_compress_preserves_or_names_issue_3814_middle_failures(
+ failure_count: int, first_regression: int
+) -> None:
+ content = _pytest_reproduction(failure_count)
+ assert len(content.splitlines()) >= 50
+ compressor = LogCompressor(LogCompressorConfig(enable_ccr=False))
+
+ result = compressor.compress(content)
+
+ assert result.compressed_line_count < result.original_line_count
+ assert result.compression_ratio < 1.0
+ last_regression = 13 if failure_count == 16 else 17
+ for i in range(first_regression, last_regression + 1):
+ node_id = f"tests/test_generated.py::test_case{i}"
+ assert _retained_or_named(result.compressed, node_id), node_id
+
+
+def _pytest_issue_log(failure_count: int) -> str:
+ """Build the issue #3814 reproduction with a distinct E message per failure."""
+ lines = [
+ "============================= test session starts =============================",
+ "collected 400 items",
+ "",
+ ]
+ lines += [
+ f"tests/test_module_{i:02d}.py .......................... [ {i:2d}%]" for i in range(40)
+ ]
+ lines.append("=================================== FAILURES ==================================")
+ for i in range(1, failure_count + 1):
+ lines += [
+ f"____________________ test_case{i:03d} ____________________",
+ "",
+ "> assert result == expected",
+ f"E AssertionError: mismatch in test_case{i:03d}",
+ "",
+ "tests/t.py:42: AssertionError",
+ ]
+ lines.append("=========================== short test summary info ===========================")
+ lines += [
+ f"FAILED tests/t.py::test_case{i:03d} - AssertionError: mismatch"
+ for i in range(1, failure_count + 1)
+ ]
+ lines.append(f"=============== {failure_count} failed, 380 passed in 41.02s ==============")
+ return "\n".join(lines)
+
+
[email protected]("failure_count", [60, 200])
+def test_real_compress_binding_cap_keeps_totals_and_first_error_line(failure_count: int) -> None:
+ out = LogCompressor(LogCompressorConfig(enable_ccr=False)).compress(
+ _pytest_issue_log(failure_count)
+ )
+ compressed = out.compressed
+
+ assert f"{failure_count} failed, 380 passed" in compressed
+ assert "E AssertionError: mismatch in test_case001" in compressed
+ # Every entry is still kept or named: kept + listed + K == total.
+ kept = sum(1 for line in compressed.splitlines() if line.startswith("FAILED tests/t.py::"))
+ named = compressed.splitlines()[-1].split("; omitted: ", 1)[1].removesuffix("]").split(", ")
+ overflow = int(named[-1][1:].removesuffix(" more")) if named[-1].startswith("+") else 0
+ listed = len(named) - (1 if overflow else 0)
+ assert kept + listed + overflow == failure_count
+
+
+def test_real_compress_keeps_all_crlf_short_summary_entries_with_non_binding_cap() -> None:
+ content = _pytest_reproduction(20, "\r\n")
+ assert len(content.splitlines()) >= 50
+ compressor = LogCompressor(
+ LogCompressorConfig(
+ max_errors=2,
+ error_context_lines=0,
+ max_total_lines=1_000,
+ enable_ccr=False,
+ )
+ )
+
+ result = compressor.compress(content, bias=1_000)
+
+ assert result.compressed_line_count < result.original_line_count
+ retained_ids = {
+ line.split(" ", 1)[1].split(" - ", 1)[0]
+ for line in result.compressed.splitlines()
+ if line.startswith(("FAILED ", "ERROR "))
+ }
+ assert {f"tests/test_generated.py::test_case{i}" for i in range(20)}.issubset(retained_ids)
+
+
+def test_real_compress_ignores_malformed_short_summary_diagnostic() -> None:
+ content = "\n".join(_malformed_short_summary_diagnostic_log())
+ compressor = LogCompressor(
+ LogCompressorConfig(
+ max_errors=2,
+ error_context_lines=0,
+ keep_summary_lines=False,
+ max_total_lines=1_000,
+ enable_ccr=False,
+ )
+ )
+
+ result = compressor.compress(content, bias=1_000)
+
+ assert result.compressed_line_count < result.original_line_count
+ assert result.compressed == (
+ "FAILED outside.py::test_0\n"
+ "FAILED outside.py::test_9\n"
+ "ERROR outside.py::test_10\n"
+ "ERROR outside.py::test_19\n"
+ "[67 lines omitted: 10 ERROR, 10 FAIL, 51 INFO]"
+ )
+
+
def test_store_in_ccr_and_result_properties(monkeypatch: pytest.MonkeyPatch) -> None:
compressor = LogCompressor()
monkeypatch.setitem(