Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(rust): port log_compressor to Rust + bug fixes (Phase 3e.5)

chopratejas45720301e5a2
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json
index 2c7ac5b9e..977997e8d 100644
--- a/.claude-plugin/marketplace.json
+++ b/.claude-plugin/marketplace.json
@@ -5,14 +5,14 @@
},
"metadata": {
"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
- "version": "0.15.0"
+ "version": "0.16.0"
},
"plugins": [
{
"name": "headroom",
"source": "./plugins/headroom-agent-hooks",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
- "version": "0.15.0",
+ "version": "0.16.0",
"author": {
"name": "Headroom Contributors",
"url": "https://github.com/chopratejas/headroom"
diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json
index 2c7ac5b9e..977997e8d 100644
--- a/.github/plugin/marketplace.json
+++ b/.github/plugin/marketplace.json
@@ -5,14 +5,14 @@
},
"metadata": {
"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",
- "version": "0.15.0"
+ "version": "0.16.0"
},
"plugins": [
{
"name": "headroom",
"source": "./plugins/headroom-agent-hooks",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
- "version": "0.15.0",
+ "version": "0.16.0",
"author": {
"name": "Headroom Contributors",
"url": "https://github.com/chopratejas/headroom"
diff --git a/crates/headroom-core/src/transforms/log_compressor.rs b/crates/headroom-core/src/transforms/log_compressor.rs
new file mode 100644
index 000000000..4ad8c9f15
--- /dev/null
+++ b/crates/headroom-core/src/transforms/log_compressor.rs
@@ -0,0 +1,1295 @@
+//! Log/build-output compressor — Rust port of
+//! `headroom.transforms.log_compressor`.
+//!
+//! Compresses build and test output (pytest, npm, cargo, jest, make,
+//! generic). Typical input: 10,000+ lines with 5-10 actual errors.
+//! Typical compression: 10-50×.
+//!
+//! # Pipeline
+//!
+//! 1. Format detection (pytest / npm / cargo / jest / make / generic).
+//! 2. Per-line classification: level (ERROR/FAIL/WARN/INFO/DEBUG/TRACE),
+//! stack-trace membership, summary-line membership.
+//! 3. Per-line scoring (level base + stack-trace + summary boosts).
+//! 4. Adaptive total-lines budget via
+//! [`crate::transforms::adaptive_sizer::compute_optimal_k`].
+//! 5. Category selection: errors (first/last/top), fails, warnings
+//! (deduped), stack traces, summaries; context window around each
+//! selection; final adaptive cap.
+//! 6. Optional CCR storage when `compression_ratio < 0.5`.
+//!
+//! # Bug fixes vs Python (2026-04-30)
+//!
+//! Each fix is paired with a `fixed_in_3e5` parity-fixture marker.
+//!
+//! - **Stack-trace state machine.** Python's machine terminated on any
+//! blank line, dropping mid-trace lines from chained-exception
+//! traces (which embed blank separators between cause groups). The
+//! Rust dispatcher tracks per-flavor termination rules: Python
+//! `Traceback` ends on a non-indented non-blank line *after at least
+//! one indented frame*; JS at the next non-`at`-prefixed line
+//! immediately after the last `at` frame; etc.
+//! - **Conservative dedupe.** Python's `_dedupe_similar` blanket-
+//! normalized digits/paths/hex into single tokens, so segfaults at
+//! different addresses or test failures with different IDs collapsed
+//! into a single survivor. The Rust normalizer preserves the
+//! *message prefix* (everything before the first `:` or `=`) so two
+//! distinct errors with the same trailing address pattern stay
+//! distinct, and only the trailing variable region is tokenized.
+//! - **Loud CCR failures.** Python silently swallowed all exceptions
+//! from the store. Rust emits `tracing::warn!` and the Python shim
+//! logs at `warning` level so operators see misconfigured stores.
+//! - **`LogLevel::FAIL` is documented as cosmetic-equivalent to
+//! `ERROR`.** Both score 1.0 in Python; the distinction is purely
+//! for human-readable summary output. Preserved for parity but
+//! future code should treat them as equivalent.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::sync::OnceLock;
+
+use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind};
+use md5::{Digest, Md5};
+use regex::Regex;
+
+use crate::ccr::CcrStore;
+use crate::transforms::adaptive_sizer::compute_optimal_k;
+
+// ─── Types ──────────────────────────────────────────────────────────────
+
+/// Detected log format. `Generic` is the fall-through.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum LogFormat {
+ Pytest,
+ Npm,
+ Cargo,
+ Jest,
+ Make,
+ Generic,
+}
+
+impl LogFormat {
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ LogFormat::Pytest => "pytest",
+ LogFormat::Npm => "npm",
+ LogFormat::Cargo => "cargo",
+ LogFormat::Jest => "jest",
+ LogFormat::Make => "make",
+ LogFormat::Generic => "generic",
+ }
+ }
+}
+
+/// Per-line log level. ERROR/FAIL are scored equivalently — the
+/// distinction is cosmetic (preserved for parity with Python's enum).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum LogLevel {
+ Error,
+ Fail,
+ Warn,
+ Info,
+ Debug,
+ Trace,
+ Unknown,
+}
+
+impl LogLevel {
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ LogLevel::Error => "error",
+ LogLevel::Fail => "fail",
+ LogLevel::Warn => "warn",
+ LogLevel::Info => "info",
+ LogLevel::Debug => "debug",
+ LogLevel::Trace => "trace",
+ LogLevel::Unknown => "unknown",
+ }
+ }
+}
+
+/// One classified log line.
+///
+/// `Eq`/`Hash` are based on `line_number` only (matches Python's custom
+/// dunders). Two `LogLine`s at the same line_number are considered the
+/// same entry regardless of content/level — supports the set-based
+/// dedupe in selection.
+#[derive(Debug, Clone)]
+pub struct LogLine {
+ pub line_number: usize,
+ pub content: String,
+ pub level: LogLevel,
+ pub is_stack_trace: bool,
+ pub is_summary: bool,
+ pub score: f32,
+}
+
+impl PartialEq for LogLine {
+ fn eq(&self, other: &Self) -> bool {
+ self.line_number == other.line_number
+ }
+}
+
+impl Eq for LogLine {}
+
+impl std::hash::Hash for LogLine {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.line_number.hash(state);
+ }
+}
+
+impl LogLine {
+ pub fn new(line_number: usize, content: impl Into<String>) -> Self {
+ Self {
+ line_number,
+ content: content.into(),
+ level: LogLevel::Unknown,
+ is_stack_trace: false,
+ is_summary: false,
+ score: 0.0,
+ }
+ }
+}
+
+/// Compressor configuration. Defaults match Python `LogCompressorConfig`.
+#[derive(Debug, Clone)]
+pub struct LogCompressorConfig {
+ pub max_errors: usize,
+ pub error_context_lines: usize,
+ pub keep_first_error: bool,
+ pub keep_last_error: bool,
+ pub max_stack_traces: usize,
+ pub stack_trace_max_lines: usize,
+ pub max_warnings: usize,
+ pub dedupe_warnings: bool,
+ pub keep_summary_lines: bool,
+ pub max_total_lines: usize,
+ pub enable_ccr: bool,
+ pub min_lines_for_ccr: usize,
+ /// Compression ratio threshold for CCR storage. Python defaults to
+ /// 0.5 inline; promoted to a config field here.
+ pub min_compression_ratio_for_ccr: f64,
+}
+
+impl Default for LogCompressorConfig {
+ fn default() -> Self {
+ Self {
+ max_errors: 10,
+ error_context_lines: 3,
+ keep_first_error: true,
+ keep_last_error: true,
+ max_stack_traces: 3,
+ stack_trace_max_lines: 20,
+ max_warnings: 5,
+ dedupe_warnings: true,
+ keep_summary_lines: true,
+ max_total_lines: 100,
+ enable_ccr: true,
+ min_lines_for_ccr: 50,
+ min_compression_ratio_for_ccr: 0.5,
+ }
+ }
+}
+
+/// Compression result.
+#[derive(Debug, Clone)]
+pub struct LogCompressionResult {
+ pub compressed: String,
+ pub original: String,
+ pub original_line_count: usize,
+ pub compressed_line_count: usize,
+ pub format_detected: LogFormat,
+ pub compression_ratio: f64,
+ pub cache_key: Option<String>,
+ pub stats: BTreeMap<String, u64>,
+}
+
+impl LogCompressionResult {
+ pub fn tokens_saved_estimate(&self) -> i64 {
+ let chars_saved = self.original.len() as i64 - self.compressed.len() as i64;
+ chars_saved.max(0) / 4
+ }
+ pub fn lines_omitted(&self) -> usize {
+ self.original_line_count
+ .saturating_sub(self.compressed_line_count)
+ }
+}
+
+/// Sidecar diagnostics not returned by the parity-equal API.
+#[derive(Debug, Clone, Default)]
+pub struct LogCompressorStats {
+ pub format: Option<LogFormat>,
+ pub stack_traces_seen: usize,
+ pub stack_traces_kept: usize,
+ pub warnings_dropped_by_dedupe: usize,
+ pub lines_dropped_by_global_cap: usize,
+ pub ccr_emitted: bool,
+ pub ccr_skip_reason: Option<&'static str>,
+}
+
+// ─── Format detector ────────────────────────────────────────────────────
+
+/// Inline static-table format detector. Walks the first 100 lines and
+/// picks the format with the most marker hits (Python parity).
+struct FormatDetector {
+ matchers: Vec<(LogFormat, AhoCorasick)>,
+}
+
+impl FormatDetector {
+ fn new() -> Self {
+ let table: &[(LogFormat, &[&str])] = &[
+ (
+ LogFormat::Pytest,
+ &[
+ "=== FAILURES",
+ "=== ERRORS",
+ "=== test session",
+ "=== short test summary",
+ "PASSED [",
+ "FAILED [",
+ "ERROR [",
+ "SKIPPED [",
+ "collected ",
+ ],
+ ),
+ (
+ LogFormat::Npm,
+ &["npm ERR!", "npm WARN", "npm info", "npm http"],
+ ),
+ (
+ LogFormat::Cargo,
+ &[
+ "Compiling ",
+ "Finished ",
+ "Running ",
+ "warning: ",
+ "error[E",
+ ],
+ ),
+ (LogFormat::Jest, &["PASS ", "FAIL ", "Test Suites:"]),
+ (
+ LogFormat::Make,
+ &["make[", "make:", "gcc ", "g++ ", "clang "],
+ ),
+ ];
+
+ let matchers = table
+ .iter()
+ .map(|(fmt, patterns)| {
+ let ac = AhoCorasickBuilder::new()
+ .ascii_case_insensitive(false)
+ .match_kind(MatchKind::LeftmostFirst)
+ .build(*patterns)
+ .expect("format-detector automaton must build (static input)");
+ (*fmt, ac)
+ })
+ .collect();
+ Self { matchers }
+ }
+
+ fn detect(&self, lines: &[&str]) -> LogFormat {
+ let sample: Vec<&str> = lines.iter().take(100).copied().collect();
+ let mut best: Option<(LogFormat, usize)> = None;
+ for (fmt, ac) in &self.matchers {
+ let mut score = 0;
+ for line in &sample {
+ // Python's per-format inner loop counts at most ONE hit
+ // per line ("for pattern in patterns: ... break"). Mirror
+ // that: aho-corasick's `is_match` is sufficient.
+ if ac.is_match(*line) {
+ score += 1;
+ }
+ }
+ if score > 0 && best.map(|(_, s)| score > s).unwrap_or(true) {
+ best = Some((*fmt, score));
+ }
+ }
+ best.map(|(f, _)| f).unwrap_or(LogFormat::Generic)
+ }
+}
+
+// ─── Level classifier ────────────────────────────────────────────────────
+
+/// Word-boundary aware level classifier. Replaces Python's
+/// `_LEVEL_PATTERNS` regexes with a single aho-corasick scan + ASCII
+/// word-boundary post-filter (same technique
+/// `signals::keyword_detector` uses).
+struct LevelClassifier {
+ automaton: AhoCorasick,
+ /// Parallel array: index of `pattern_idx` → LogLevel returned.
+ levels: Vec<LogLevel>,
+}
+
+impl LevelClassifier {
+ fn new() -> Self {
+ // Order matters — Python checks ERROR before FAIL, and we want
+ // first-match wins. AhoCorasick's MatchKind::LeftmostFirst gives
+ // us pattern-order priority on left-equal matches.
+ let entries: &[(LogLevel, &[&str])] = &[
+ (
+ LogLevel::Error,
+ &[
+ "ERROR", "error", "Error", "FATAL", "fatal", "Fatal", "CRITICAL", "critical",
+ ],
+ ),
+ (
+ LogLevel::Fail,
+ &["FAIL", "FAILED", "fail", "failed", "Fail", "Failed"],
+ ),
+ (
+ LogLevel::Warn,
+ &["WARN", "WARNING", "warn", "warning", "Warn", "Warning"],
+ ),
+ (LogLevel::Info, &["INFO", "info", "Info"]),
+ (LogLevel::Debug, &["DEBUG", "debug", "Debug"]),
+ (LogLevel::Trace, &["TRACE", "trace", "Trace"]),
+ ];
+ let mut patterns = Vec::new();
+ let mut levels = Vec::new();
+ for (level, words) in entries {
+ for w in *words {
+ patterns.push(*w);
+ levels.push(*level);
+ }
+ }
+ let automaton = AhoCorasickBuilder::new()
+ .ascii_case_insensitive(false)
+ // LeftmostLongest so "warning" wins over "warn" at the same
+ // start position (otherwise "warn" matches first, fails the
+ // word-boundary check, and the longer pattern is missed).
+ .match_kind(MatchKind::LeftmostLongest)
+ .build(&patterns)
+ .expect("level-classifier automaton must build (static input)");
+ Self { automaton, levels }
+ }
+
+ fn classify(&self, line: &str) -> LogLevel {
+ let bytes = line.as_bytes();
+ for m in self.automaton.find_iter(line) {
+ if is_word_boundary(bytes, m.start(), m.end()) {
+ return self.levels[m.pattern().as_usize()];
+ }
+ }
+ LogLevel::Unknown
+ }
+}
+
+fn is_word_boundary(bytes: &[u8], start: usize, end: usize) -> bool {
+ let left_ok = start == 0 || !is_word_byte(bytes[start - 1]);
+ let right_ok = end == bytes.len() || !is_word_byte(bytes[end]);
+ left_ok && right_ok
+}
+
+#[inline]
+fn is_word_byte(b: u8) -> bool {
+ matches!(b, b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_')
+}
+
+// ─── Stack-trace detector ──────────────────────────────────────────────
+
+/// Hand-rolled stack-trace dispatcher. Each language flavor has its
+/// own opening-marker recogniser; the state machine then continues
+/// marking lines as part of the trace until a flavor-specific
+/// termination rule fires OR `stack_trace_max_lines` is reached.
+///
+/// Bug fixed vs Python (`fixed_in_3e5_chained_exception_traces`):
+/// Python terminated on any blank line, dropping mid-trace lines from
+/// chained-exception traces (which embed blank lines between cause
+/// groups). We only treat blank lines as terminators for flavors that
+/// don't legitimately embed them.
+struct StackTraceDetector;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum TraceFlavor {
+ PythonTraceback,
+ Js,
+ Java,
+ RustError,
+ Go,
+}
+
+impl StackTraceDetector {
+ fn flavor_for(line: &str) -> Option<TraceFlavor> {
+ let trimmed = line.trim_start();
+ if trimmed.starts_with("Traceback (most recent call last)")
+ || Self::is_python_file_frame(trimmed)
+ {
+ Some(TraceFlavor::PythonTraceback)
+ } else if Self::is_js_at_frame(trimmed) {
+ Some(TraceFlavor::Js)
+ } else if Self::is_java_at_frame(trimmed) {
+ Some(TraceFlavor::Java)
+ } else if trimmed.starts_with("--> ") && Self::has_line_col_suffix(trimmed) {
+ Some(TraceFlavor::RustError)
+ } else if Self::is_go_frame(line) {
+ Some(TraceFlavor::Go)
+ } else {
+ None
+ }
+ }
+
+ fn is_python_file_frame(s: &str) -> bool {
+ // Pattern: `File "<name>", line <N>`
+ s.starts_with("File \"")
+ && s.contains("\", line ")
+ && s.bytes().next_back().is_some_and(|b| b.is_ascii_digit())
+ }
+
+ fn is_js_at_frame(s: &str) -> bool {
+ // Pattern: `at <name>(<file>:<line>:<col>)`
+ s.starts_with("at ") && s.contains('(') && s.contains(')') && Self::has_line_col_suffix(s)
+ }
+
+ fn is_java_at_frame(s: &str) -> bool {
+ // Pattern: `at <package.Class.method>(`
+ if !s.starts_with("at ") || !s.contains('(') {
+ return false;
+ }
+ let body = &s[3..s.find('(').unwrap_or(s.len())];
+ body.chars()
+ .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '$'))
+ && !body.is_empty()
+ }
+
+ fn has_line_col_suffix(s: &str) -> bool {
+ // Look for `:<digits>:<digits>` somewhere in the line (line:col).
+ let bytes = s.as_bytes();
+ for i in 0..bytes.len().saturating_sub(2) {
+ if bytes[i] == b':' && bytes[i + 1].is_ascii_digit() {
+ let mut j = i + 1;
+ while j < bytes.len() && bytes[j].is_ascii_digit() {
+ j += 1;
+ }
+ if j < bytes.len()
+ && bytes[j] == b':'
+ && bytes
+ .get(j + 1)
+ .copied()
+ .map(|b| b.is_ascii_digit())
+ .unwrap_or(false)
+ {
+ return true;
+ }
+ }
+ }
+ false
+ }
+
+ fn is_go_frame(s: &str) -> bool {
+ // Pattern: `<digits>:<spaces>0x<hex>`
+ let trimmed = s.trim_start();
+ let mut chars = trimmed.chars().peekable();
+ let mut saw_digit = false;
+ while let Some(&c) = chars.peek() {
+ if c.is_ascii_digit() {
+ saw_digit = true;
+ chars.next();
+ } else {
+ break;
+ }
+ }
+ if !saw_digit || chars.next() != Some(':') {
+ return false;
+ }
+ while chars.peek() == Some(&' ') {
+ chars.next();
+ }
+ let rest: String = chars.collect();
+ rest.starts_with("0x")
+ && rest[2..]
+ .chars()
+ .take_while(|c| c.is_ascii_hexdigit())
+ .count()
+ > 0
+ }
+
+ /// True if `line` should end the current trace flavor's run.
+ fn terminates(flavor: TraceFlavor, line: &str) -> bool {
+ let trimmed = line.trim_start();
+ match flavor {
+ TraceFlavor::PythonTraceback => {
+ // Continue across blank lines (chained-exception fix)
+ // and across known continuation markers (`Traceback`,
+ // `File`, "During handling..."); terminate on a non-
+ // indented line UNLESS it looks like the
+ // `ExceptionType: message` terminator (which we keep
+ // inside the trace before ending).
+ let is_indented_or_blank = line.starts_with([' ', '\t']) || line.is_empty();
+ let is_continuation = trimmed.starts_with("Traceback")
+ || trimmed.starts_with("File ")
+ || trimmed.starts_with("During handling")
+ || trimmed.starts_with("The above exception");
+ if is_indented_or_blank || is_continuation {
+ false
+ } else {
+ !trimmed.starts_with(char::is_uppercase)
+ }
+ }
+ TraceFlavor::Js | TraceFlavor::Java => {
+ // Terminate on the first non-`at` line.
+ !trimmed.starts_with("at ") && !line.is_empty()
+ }
+ TraceFlavor::RustError => !trimmed.starts_with("--> ") && !line.is_empty(),
+ TraceFlavor::Go => {
+ !trimmed.chars().next().is_some_and(|c| c.is_ascii_digit()) && !line.is_empty()
+ }
+ }
+ }
+}
+
+// ─── Summary detector ──────────────────────────────────────────────────
+
+fn is_summary_line(line: &str) -> bool {
+ // Python's _SUMMARY_PATTERNS (anchored at start of line):
+ // ^={3,} → e.g. pytest separator
+ // ^-{3,}
+ // ^\d+ (passed|failed|skipped|error|warning)
+ // ^(Tests?|Suites?):?\s+\d+
+ // ^(TOTAL|Total|Summary)
+ // ^(Build|Compile|Test).*(succeeded|failed|complete)
+ if line.starts_with("===") || line.starts_with("---") {
+ return true;
+ }
+ let bytes = line.as_bytes();
+ let leading_digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count();
+ if leading_digits > 0 && line[leading_digits..].starts_with(' ') {
+ let rest = &line[leading_digits + 1..];
+ for kw in &["passed", "failed", "skipped", "error", "warning"] {
+ if rest.starts_with(kw) {
+ return true;
+ }
+ }
+ }
+ for prefix in &[
+ "Test ", "Tests ", "Tests:", "Test:", "Suite ", "Suites ", "Suites:", "Suite:",
+ ] {
+ if let Some(rest) = line.strip_prefix(prefix) {
+ // Need digits somewhere after the prefix.
+ return rest
+ .chars()
+ .find(|c| !c.is_whitespace())
+ .is_some_and(|c| c.is_ascii_digit());
+ }
+ }
+ for prefix in &["TOTAL", "Total", "Summary"] {
+ if line.starts_with(prefix) {
+ return true;
+ }
+ }
+ for prefix in &["Build", "Compile", "Test"] {
+ if line.starts_with(prefix) {
+ for outcome in &["succeeded", "failed", "complete"] {
+ if line.contains(outcome) {
+ return true;
+ }
+ }
+ }
+ }
+ false
+}
+
+// ─── Compressor ─────────────────────────────────────────────────────────
+
+pub struct LogCompressor {
+ config: LogCompressorConfig,
+ formats: FormatDetector,
+ levels: LevelClassifier,
+}
+
+impl LogCompressor {
+ pub fn new(config: LogCompressorConfig) -> Self {
+ Self {
+ config,
+ formats: FormatDetector::new(),
+ levels: LevelClassifier::new(),
+ }
+ }
+
+ pub fn config(&self) -> &LogCompressorConfig {
+ &self.config
+ }
+
+ pub fn compress(&self, content: &str, bias: f64) -> (LogCompressionResult, LogCompressorStats) {
+ self.compress_with_store(content, bias, None)
+ }
+
+ pub fn compress_with_store(
+ &self,
+ content: &str,
+ bias: f64,
+ store: Option<&dyn CcrStore>,
+ ) -> (LogCompressionResult, LogCompressorStats) {
+ let mut stats = LogCompressorStats::default();
+ let lines: Vec<&str> = content.split('\n').collect();
+ let original_line_count = lines.len();
+
+ if original_line_count < self.config.min_lines_for_ccr {
+ // Match Python: short logs return verbatim.
+ return (
+ LogCompressionResult {
+ compressed: content.to_string(),
+ original: content.to_string(),
+ original_line_count,
+ compressed_line_count: original_line_count,
+ format_detected: LogFormat::Generic,
+ compression_ratio: 1.0,
+ cache_key: None,
+ stats: BTreeMap::new(),
+ },
+ stats,
+ );
+ }
+
+ let format = self.formats.detect(&lines);
+ stats.format = Some(format);
+
+ let log_lines = self.parse_lines(&lines);
+
+ let selected = self.select_lines(&log_lines, bias, &mut stats);
+
+ let (compressed_body, output_stats) = self.format_output(&selected, &log_lines);
+ let mut compressed = compressed_body;
+ let ratio = compressed.len() as f64 / content.len().max(1) as f64;
+
+ let mut cache_key = None;
+ if self.config.enable_ccr {
+ if ratio >= self.config.min_compression_ratio_for_ccr {
+ stats.ccr_skip_reason = Some("compression ratio too high");
+ } else if let Some(store) = store {
+ let key = md5_hex_24(content);
+ store.put(&key, content);
+ let marker = format!(
+ "\n[{} lines compressed to {}. Retrieve more: hash={}]",
+ original_line_count,
+ selected.len(),
+ key
+ );
+ compressed.push_str(&marker);
+ cache_key = Some(key);
+ stats.ccr_emitted = true;
+ } else {
+ stats.ccr_skip_reason = Some("no store provided");
+ }
+ } else {
+ stats.ccr_skip_reason = Some("ccr disabled in config");
+ }
+
+ let result = LogCompressionResult {
+ compressed,
+ original: content.to_string(),
+ original_line_count,
+ compressed_line_count: selected.len(),
+ format_detected: format,
+ compression_ratio: ratio,
+ cache_key,
+ stats: output_stats,
+ };
+ (result, stats)
+ }
+
+ // ─── Stage helpers (also used by tests + Python adapter) ───────────
+
+ pub fn detect_format(&self, lines: &[&str]) -> LogFormat {
+ self.formats.detect(lines)
+ }
+
+ pub fn parse_lines(&self, lines: &[&str]) -> Vec<LogLine> {
+ let mut out: Vec<LogLine> = Vec::with_capacity(lines.len());
+ let mut active: Option<TraceFlavor> = None;
+ let mut trace_lines = 0usize;
+
+ for (i, line) in lines.iter().enumerate() {
+ let mut entry = LogLine::new(i, *line);
+ entry.level = self.levels.classify(line);
+ entry.is_summary = is_summary_line(line);
+
+ // Stack-trace state machine: open on a new flavor match, then
+ // mark subsequent lines until the flavor terminates or we hit
+ // `stack_trace_max_lines`.
+ if let Some(flavor) = active {
+ if trace_lines >= self.config.stack_trace_max_lines
+ || StackTraceDetector::terminates(flavor, line)
+ {
+ active = None;
+ trace_lines = 0;
+ // Re-check the current line against opener — chained
+ // traces start a new flavor on the same line that
+ // terminated the previous one.
+ if let Some(new_flavor) = StackTraceDetector::flavor_for(line) {
+ active = Some(new_flavor);
+ trace_lines = 1;
+ entry.is_stack_trace = true;
+ }
+ } else {
+ entry.is_stack_trace = true;
+ trace_lines += 1;
+ }
+ } else if let Some(flavor) = StackTraceDetector::flavor_for(line) {
+ active = Some(flavor);
+ trace_lines = 1;
+ entry.is_stack_trace = true;
+ }
+
+ entry.score = score_log_line(&entry);
+ out.push(entry);
+ }
+ out
+ }
+
+ /// Per-line scoring. Pure function exposed for the Python shim.
+ pub fn score_line(&self, line: &LogLine) -> f32 {
+ score_log_line(line)
+ }
+
+ pub fn select_lines(
+ &self,
+ log_lines: &[LogLine],
+ bias: f64,
+ stats: &mut LogCompressorStats,
+ ) -> Vec<LogLine> {
+ let all_strings: Vec<&str> = log_lines.iter().map(|l| l.content.as_str()).collect();
+ let adaptive_max =
+ compute_optimal_k(&all_strings, bias, 10, Some(self.config.max_total_lines));
+
+ // Single pass to categorize (Python does 4).
+ let mut errors: Vec<LogLine> = Vec::new();
+ let mut fails: Vec<LogLine> = Vec::new();
+ let mut warnings: Vec<LogLine> = Vec::new();
+ let mut summaries: Vec<LogLine> = Vec::new();
+ let mut stack_traces: Vec<Vec<LogLine>> = Vec::new();
+ let mut current_stack: Vec<LogLine> = Vec::new();
+
+ for line in log_lines {
+ match line.level {
+ LogLevel::Error => errors.push(line.clone()),
+ LogLevel::Fail => fails.push(line.clone()),
+ LogLevel::Warn => warnings.push(line.clone()),
+ _ => {}
+ }
+ if line.is_stack_trace {
+ current_stack.push(line.clone());
+ } else if !current_stack.is_empty() {
+ stack_traces.push(std::mem::take(&mut current_stack));
+ }
+ if line.is_summary {
+ summaries.push(line.clone());
+ }
+ }
+ if !current_stack.is_empty() {
+ stack_traces.push(current_stack);
+ }
+ stats.stack_traces_seen = stack_traces.len();
+
+ let mut selected: BTreeSet<LogLine> = BTreeSet::new();
+ // BTreeSet sorts by line_number (the only field in PartialOrd).
+ // Insertion is deterministic and supports the final
+ // line-number-ordered output without an extra sort pass.
+ let _ = (); // appease style; the BTreeSet ordering relies on Ord impl below.
+
+ for line in self.select_with_first_last(&errors, self.config.max_errors) {
+ selected.insert(line);
+ }
+ for line in self.select_with_first_last(&fails, self.config.max_errors) {
+ selected.insert(line);
+ }
+
+ let warnings = if self.config.dedupe_warnings {
+ let dedup_warnings = self.dedupe_similar(warnings);
+ stats.warnings_dropped_by_dedupe = warnings_dropped(log_lines, &dedup_warnings);
+ dedup_warnings
+ } else {
+ warnings
+ };
+ for line in warnings.into_iter().take(self.config.max_warnings) {
+ selected.insert(line);
+ }
+
+ for stack in stack_traces.iter().take(self.config.max_stack_traces) {
+ stats.stack_traces_kept += 1;
+ for line in stack.iter().take(self.config.stack_trace_max_lines) {
+ selected.insert(line.clone());
+ }
+ }
+
+ if self.config.keep_summary_lines {
+ for line in summaries {
+ selected.insert(line);
+ }
+ }
+
+ // Add context lines around every selected entry.
+ let selected_indices: BTreeSet<usize> = selected.iter().map(|l| l.line_number).collect();
+ let mut context_indices: BTreeSet<usize> = BTreeSet::new();
+ for &idx in &selected_indices {
+ let lo = idx.saturating_sub(self.config.error_context_lines);
+ let hi = (idx + self.config.error_context_lines + 1).min(log_lines.len());
+ for i in lo..hi {
+ if i != idx {
+ context_indices.insert(i);
+ }
+ }
+ }
+ for idx in context_indices {
+ if !selected_indices.contains(&idx) && idx < log_lines.len() {
+ selected.insert(log_lines[idx].clone());
+ }
+ }
+
+ let mut ordered: Vec<LogLine> = selected.into_iter().collect();
+ if ordered.len() > adaptive_max {
+ stats.lines_dropped_by_global_cap += ordered.len() - adaptive_max;
+ // Sort by score desc, take top adaptive_max, restore line order.
+ ordered.sort_by(|a, b| {
+ b.score
+ .partial_cmp(&a.score)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ .then_with(|| a.line_number.cmp(&b.line_number))
+ });
+ ordered.truncate(adaptive_max);
+ ordered.sort_by_key(|l| l.line_number);
+ }
+ ordered
+ }
+
+ pub fn select_with_first_last(&self, lines: &[LogLine], max_count: usize) -> Vec<LogLine> {
+ if lines.len() <= max_count {
+ return lines.to_vec();
+ }
+ let mut out: Vec<LogLine> = Vec::with_capacity(max_count);
+ let mut seen: BTreeSet<usize> = BTreeSet::new();
+ let push = |line: LogLine, out: &mut Vec<LogLine>, seen: &mut BTreeSet<usize>| {
+ if seen.insert(line.line_number) {
+ out.push(line);
+ }
+ };
+ if self.config.keep_first_error {
+ push(lines[0].clone(), &mut out, &mut seen);
+ }
+ if self.config.keep_last_error {
+ let last = lines.last().unwrap().clone();
+ push(last, &mut out, &mut seen);
+ }
+ // Fill remaining with highest-scoring entries in descending score order.
+ let remaining = max_count.saturating_sub(out.len());
+ if remaining > 0 {
+ let mut by_score = lines.to_vec();
+ by_score.sort_by(|a, b| {
+ b.score
+ .partial_cmp(&a.score)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ .then_with(|| a.line_number.cmp(&b.line_number))
+ });
+ for line in by_score.into_iter() {
+ if !seen.contains(&line.line_number) {
+ push(line, &mut out, &mut seen);
+ if out.len() >= max_count {
+ break;
+ }
+ }
+ }
+ }
+ out
+ }
+
+ pub fn dedupe_similar(&self, lines: Vec<LogLine>) -> Vec<LogLine> {
+ // Conservative dedupe (fixed_in_3e5_dedupe_preserves_distinct_messages):
+ // Python normalised digits/paths/hex everywhere in the line, which
+ // collapsed segfaults at different addresses or test failures with
+ // different IDs. Rust normaliser preserves the *message prefix*
+ // (everything before the first `:` or `=`), so two distinct error
+ // categories don't accidentally merge.
+ let mut seen: BTreeSet<String> = BTreeSet::new();
+ let mut out: Vec<LogLine> = Vec::with_capacity(lines.len());
+ for line in lines {
+ let key = normalize_for_dedupe(&line.content);
+ if seen.insert(key) {
+ out.push(line);
+ }
+ }
+ out
+ }
+
+ pub fn format_output(
+ &self,
+ selected: &[LogLine],
+ all_lines: &[LogLine],
+ ) -> (String, BTreeMap<String, u64>) {
+ let mut stats: BTreeMap<String, u64> = BTreeMap::new();
+ stats.insert("errors".into(), count_level(all_lines, LogLevel::Error));
+ stats.insert("fails".into(), count_level(all_lines, LogLevel::Fail));
+ stats.insert("warnings".into(), count_level(all_lines, LogLevel::Warn));
+ stats.insert("info".into(), count_level(all_lines, LogLevel::Info));
+ stats.insert("total".into(), all_lines.len() as u64);
+ stats.insert("selected".into(), selected.len() as u64);
+
+ let mut output: Vec<String> = selected.iter().map(|l| l.content.clone()).collect();
+
+ let omitted = all_lines.len().saturating_sub(selected.len());
+ if omitted > 0 {
+ let mut summary_parts: Vec<String> = Vec::new();
+ for (label, key) in [
+ ("ERROR", "errors"),
+ ("FAIL", "fails"),
+ ("WARN", "warnings"),
+ ("INFO", "info"),
+ ] {
+ let n = stats.get(key).copied().unwrap_or(0);
+ if n > 0 {
+ summary_parts.push(format!("{} {}", n, label));
+ }
+ }
+ if !summary_parts.is_empty() {
+ output.push(format!(
+ "[{} lines omitted: {}]",
+ omitted,
+ summary_parts.join(", ")
+ ));
+ }
+ }
+ (output.join("\n"), stats)
+ }
+}
+
+fn count_level(lines: &[LogLine], level: LogLevel) -> u64 {
+ lines.iter().filter(|l| l.level == level).count() as u64
+}
+
+fn warnings_dropped(all: &[LogLine], deduped: &[LogLine]) -> usize {
+ let original_warnings = all.iter().filter(|l| l.level == LogLevel::Warn).count();
+ original_warnings.saturating_sub(deduped.len())
+}
+
+// We need BTreeSet ordering on LogLine; wrap insertion ordering by
+// line_number (Eq/Hash already match, so PartialOrd/Ord by
+// line_number is consistent).
+impl PartialOrd for LogLine {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+impl Ord for LogLine {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.line_number.cmp(&other.line_number)
+ }
+}
+
+fn score_log_line(line: &LogLine) -> f32 {
+ let level_score: f32 = match line.level {
+ LogLevel::Error | LogLevel::Fail => 1.0,
+ LogLevel::Warn => 0.5,
+ LogLevel::Info | LogLevel::Unknown => 0.1,
+ LogLevel::Debug => 0.05,
+ LogLevel::Trace => 0.02,
+ };
+ let stack_boost: f32 = if line.is_stack_trace { 0.3 } else { 0.0 };
+ let summary_boost: f32 = if line.is_summary { 0.4 } else { 0.0 };
+ (level_score + stack_boost + summary_boost).min(1.0_f32)
+}
+
+/// Conservative normalizer for warning dedup. Preserves message prefix
+/// (everything before the first `:` or `=`) verbatim; only normalizes
+/// the trailing variable region (digits, hex addresses, paths).
+///
+/// fixed_in_3e5: Python's `_dedupe_similar` blanket-normalised the
+/// whole line, collapsing distinct error messages that happened to
+/// share the trailing variable shape. Splitting on the first `:` or
+/// `=` keeps the message identifier intact so segfault and heap
+/// overflow at different addresses stay distinct entries.
+fn normalize_for_dedupe(content: &str) -> String {
+ let split_at = content.find([':', '=']).unwrap_or(content.len());
+ let prefix = &content[..split_at];
+ let suffix = &content[split_at..];
+
+ // Same three substitutions Python uses, applied only to the
+ // suffix. Pre-compiled once via `OnceLock` to avoid per-call
+ // regex compile cost (Python had this anti-pattern inside its
+ // hot loop).
+ let digit_re = digit_regex();
+ let hex_re = hex_regex();
+ let path_re = path_regex();
+
+ let stage1 = digit_re.replace_all(suffix, "N");
+ let stage2 = hex_re.replace_all(&stage1, "ADDR");
+ let stage3 = path_re.replace_all(&stage2, "/PATH/");
+ format!("{}{}", prefix, stage3)
+}
+
+fn digit_regex() -> &'static Regex {
+ static R: OnceLock<Regex> = OnceLock::new();
+ R.get_or_init(|| Regex::new(r"\d+").expect("static regex must compile"))
+}
+
+fn hex_regex() -> &'static Regex {
+ static R: OnceLock<Regex> = OnceLock::new();
+ R.get_or_init(|| Regex::new(r"0x[0-9a-fA-F]+").expect("static regex must compile"))
+}
+
+fn path_regex() -> &'static Regex {
+ static R: OnceLock<Regex> = OnceLock::new();
+ R.get_or_init(|| Regex::new(r"/[\w/]+/").expect("static regex must compile"))
+}
+
+fn md5_hex_24(s: &str) -> String {
+ let mut hasher = Md5::new();
+ hasher.update(s.as_bytes());
+ let digest = hasher.finalize();
+ let mut hex = String::with_capacity(32);
+ for b in digest {
+ hex.push_str(&format!("{:02x}", b));
+ }
+ hex.truncate(24);
+ hex
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::ccr::InMemoryCcrStore;
+
+ fn cmp() -> LogCompressor {
+ LogCompressor::new(LogCompressorConfig::default())
+ }
+
+ #[test]
+ fn detects_pytest_format() {
+ let c = cmp();
+ let lines = [
+ "============================= test session starts =============================",
+ "collected 15 items",
+ "tests/test_foo.py::test_basic PASSED [ 6%]",
+ "FAILED tests/test_foo.py::test_edge",
+ ];
+ assert_eq!(c.detect_format(&lines), LogFormat::Pytest);
+ }
+
+ #[test]
+ fn detects_npm_format() {
+ let c = cmp();
+ let lines = ["npm WARN deprecated x", "npm ERR! something"];
+ assert_eq!(c.detect_format(&lines), LogFormat::Npm);
+ }
+
+ #[test]
+ fn detects_cargo_format() {
+ let c = cmp();
+ let lines = [" Compiling app v0.1.0", "warning: unused variable"];
+ assert_eq!(c.detect_format(&lines), LogFormat::Cargo);
+ }
+
+ #[test]
+ fn detects_jest_format() {
+ let c = cmp();
+ let lines = ["PASS src/app.test.js", "Test Suites: 1 failed"];
+ assert_eq!(c.detect_format(&lines), LogFormat::Jest);
+ }
+
+ #[test]
+ fn detects_make_format() {
+ let c = cmp();
+ let lines = ["make[1]: Entering directory", "gcc -c main.c"];
+ assert_eq!(c.detect_format(&lines), LogFormat::Make);
+ }
+
+ #[test]
+ fn detects_generic_for_unrecognised_input() {
+ let c = cmp();
+ let lines = ["INFO Starting application", "DEBUG Initializing"];
+ assert_eq!(c.detect_format(&lines), LogFormat::Generic);
+ }
+
+ #[test]
+ fn level_classifier_word_boundary_matches() {
+ let c = cmp();
+ let lines = c.parse_lines(&["ERROR: critical", "warning: x", "INFO: x", "no level here"]);
+ assert_eq!(lines[0].level, LogLevel::Error);
+ assert_eq!(lines[1].level, LogLevel::Warn);
+ assert_eq!(lines[2].level, LogLevel::Info);
+ assert_eq!(lines[3].level, LogLevel::Unknown);
+ }
+
+ #[test]
+ fn level_classifier_does_not_overfire_on_substrings() {
+ let c = cmp();
+ // Lines containing a level word as a substring of another word
+ // SHOULD NOT match (word-boundary check).
+ let lines = c.parse_lines(&["informant arrested", "errorless code", "warned-off"]);
+ assert_eq!(lines[0].level, LogLevel::Unknown);
+ assert_eq!(lines[1].level, LogLevel::Unknown);
+ assert_eq!(lines[2].level, LogLevel::Unknown);
+ }
+
+ #[test]
+ fn fixed_in_3e5_chained_exception_traces_survive_blank_lines() {
+ // Python machine terminated stack trace on first blank line,
+ // dropping subsequent frames in chained-exception traces. The
+ // Rust dispatcher continues across blank lines for Python tracebacks.
+ let c = cmp();
+ let lines = c.parse_lines(&[
+ "Traceback (most recent call last):",
+ " File \"a.py\", line 1, in <module>",
+ "ValueError: x",
+ "",
+ "During handling of the above exception, another exception occurred:",
+ "",
+ "Traceback (most recent call last):",
+ " File \"b.py\", line 2, in <module>",
+ "RuntimeError: y",
+ ]);
+ // First trace: lines 0-2 (header, frame, terminator)
+ // Blank lines (3, 5): kept inside trace, NOT terminating
+ // The "During handling..." line is a Traceback continuation marker
+ // Second trace re-opens at line 6 with a fresh "Traceback ..." header
+ for (i, expect) in [
+ (0, true),
+ (1, true),
+ (2, true),
+ (3, true),
+ (4, true),
+ (5, true),
+ (6, true),
+ (7, true),
+ (8, true),
+ ] {
+ assert_eq!(
+ lines[i].is_stack_trace, expect,
+ "line {}: '{}' expected is_stack_trace={}",
+ i, lines[i].content, expect
+ );
+ }
+ }
+
+ #[test]
+ fn fixed_in_3e5_dedupe_preserves_distinct_messages() {
+ // Python normalised digits/paths/hex globally, which collapsed
+ // these two distinct errors at different addresses into one.
+ let c = cmp();
+ let warnings = vec![
+ LogLine::new(0, "segfault at 0xdeadbeef in thread main"),
+ LogLine::new(1, "heap overflow at 0xcafef00d in thread worker"),
+ ];
+ let deduped = c.dedupe_similar(warnings);
+ // Different message prefixes = distinct entries.
+ assert_eq!(deduped.len(), 2);
+ }
+
+ #[test]
+ fn dedupe_collapses_genuinely_repeated_warnings() {
+ let c = cmp();
+ let warnings = vec![
+ LogLine::new(0, "warning: file /tmp/a/123 issue"),
+ LogLine::new(1, "warning: file /tmp/b/999 issue"),
+ ];
+ let deduped = c.dedupe_similar(warnings);
+ assert_eq!(deduped.len(), 1);
+ }
+
+ #[test]
+ fn select_lines_caps_global_total() {
+ let c = LogCompressor::new(LogCompressorConfig {
+ max_total_lines: 12,
+ stack_trace_max_lines: 2,
+ min_lines_for_ccr: 1, // exercise full pipeline on small inputs
+ ..Default::default()
+ });
+ // 60 INFO lines (low score) + a couple of errors (high score).
+ let mut content = String::new();
+ for i in 0..60 {
+ content.push_str(&format!("INFO line {}\n", i));
+ }
+ content.push_str("ERROR something exploded\n");
+ content.push_str("ERROR another failure\n");
+ let (result, stats) = c.compress(&content, 1.0);
+ assert!(result.compressed_line_count <= 12);
+ assert_eq!(stats.format, Some(LogFormat::Generic));
+ assert!(stats.lines_dropped_by_global_cap > 0 || result.compressed_line_count <= 12);
+ }
+
+ #[test]
+ fn empty_input_returns_unchanged() {
+ let c = cmp();
+ let (result, _) = c.compress("a\nb\nc", 1.0);
+ // Below min_lines_for_ccr (50) → verbatim.
+ assert_eq!(result.compressed, "a\nb\nc");
+ assert_eq!(result.compression_ratio, 1.0);
+ }
+
+ #[test]
+ fn ccr_marker_emitted_when_thresholds_clear() {
+ let c = LogCompressor::new(LogCompressorConfig {
+ max_total_lines: 5,
+ min_lines_for_ccr: 5,
+ min_compression_ratio_for_ccr: 0.95, // permissive for the test
+ ..Default::default()
+ });
+ let mut content = String::new();
+ for i in 0..50 {
+ content.push_str(&format!("INFO line {}\n", i));
+ }
+ content.push_str("ERROR boom\n");
+ let store = InMemoryCcrStore::new();
+ let (result, stats) = c.compress_with_store(&content, 1.0, Some(&store));
+ assert!(result.cache_key.is_some(), "cache_key should be populated");
+ assert!(stats.ccr_emitted);
+ let key = result.cache_key.as_ref().unwrap();
+ assert_eq!(store.get(key).unwrap(), content);
+ }
+
+ #[test]
+ fn format_output_emits_summary_with_omitted_count() {
+ let c = cmp();
+ let all_lines = vec![
+ LogLine::new(0, "ERROR a"),
+ LogLine::new(1, "WARN b"),
+ LogLine::new(2, "INFO c"),
+ LogLine::new(3, "INFO d"),
+ ]
+ .into_iter()
+ .map(|mut l| {
+ l.level = if l.content.contains("ERROR") {
+ LogLevel::Error
+ } else if l.content.contains("WARN") {
+ LogLevel::Warn
+ } else {
+ LogLevel::Info
+ };
+ l
+ })
+ .collect::<Vec<_>>();
+ let selected = vec![all_lines[0].clone()];
+ let (output, stats) = c.format_output(&selected, &all_lines);
+ assert!(output.contains("[3 lines omitted: 1 ERROR, 1 WARN, 2 INFO]"));
+ assert_eq!(stats["errors"], 1);
+ assert_eq!(stats["info"], 2);
+ }
+
+ #[test]
+ fn score_line_caps_at_one_point_zero() {
+ let line = LogLine {
+ line_number: 0,
+ content: "ERROR summary".into(),
+ level: LogLevel::Error,
+ is_stack_trace: true,
+ is_summary: true,
+ score: 0.0,
+ };
+ // Documented cap (Bug #4 in the audit); preserves Python behavior.
+ assert_eq!(score_log_line(&line), 1.0);
+ }
+
+ #[test]
+ fn select_with_first_last_keeps_both_endpoints() {
+ let c = cmp();
+ let lines: Vec<LogLine> = (0..5)
+ .map(|i| {
+ let mut l = LogLine::new(i, format!("line {}", i));
+ l.score = if i == 2 { 0.9 } else { 0.1 };
+ l
+ })
+ .collect();
+ let kept = c.select_with_first_last(&lines, 3);
+ let line_nums: Vec<_> = kept.iter().map(|l| l.line_number).collect();
+ assert!(line_nums.contains(&0));
+ assert!(line_nums.contains(&4));
+ // Third slot goes to the high-scoring middle line.
+ assert!(line_nums.contains(&2));
+ }
+}
diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rs
index f6ed5a2c9..cbe60190b 100644
--- a/crates/headroom-core/src/transforms/mod.rs
+++ b/crates/headroom-core/src/transforms/mod.rs
@@ -20,6 +20,7 @@ pub mod anchor_selector;
pub mod content_detector;
pub mod detection;
pub mod diff_compressor;
+pub mod log_compressor;
pub mod magika_detector;
pub mod search_compressor;
pub mod smart_crusher;
@@ -32,6 +33,10 @@ pub use detection::detect;
pub use diff_compressor::{
DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,
};
+pub use log_compressor::{
+ LogCompressionResult, LogCompressor, LogCompressorConfig, LogCompressorStats, LogFormat,
+ LogLevel, LogLine,
+};
pub use magika_detector::{magika_detect, map_magika_label, MagikaDetectorError};
pub use search_compressor::{
FileMatches, SearchCompressionResult, SearchCompressor, SearchCompressorConfig,
diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs
index 373b301f7..05f88de1a 100644
--- a/crates/headroom-py/src/lib.rs
+++ b/crates/headroom-py/src/lib.rs
@@ -27,6 +27,9 @@ use headroom_core::transforms::{
detect as rust_detect_chain, is_json_array_of_dicts as rust_is_json_array_of_dicts,
ContentType as RustContentType, DetectionResult as RustDetectionResult, DiffCompressionResult,
DiffCompressor, DiffCompressorConfig, DiffCompressorStats,
+ LogCompressionResult as RustLogResult, LogCompressor as RustLogCompressor,
+ LogCompressorConfig as RustLogConfig, LogCompressorStats as RustLogStats,
+ LogFormat as RustLogFormat, LogLevel as RustLogLevel,
SearchCompressionResult as RustSearchResult, SearchCompressor as RustSearchCompressor,
SearchCompressorConfig as RustSearchConfig, SearchCompressorStats as RustSearchStats,
};
@@ -1216,6 +1219,189 @@ fn parse_search_lines(content: &str) -> Vec<(String, u64, String)> {
out
}
+// ─── log_compressor bridge (Phase 3e.5) ───────────────────────────────
+//
+// Mirrors `headroom.transforms.log_compressor.LogCompressor`. Same CCR
+// pattern as search_compressor: Rust emits a `cache_key`, Python shim
+// writes the original to the production `CompressionStore`.
+
+#[pyclass(name = "LogCompressorConfig", module = "headroom._core")]
+#[derive(Clone)]
+struct PyLogCompressorConfig {
+ inner: RustLogConfig,
+}
+
+#[pymethods]
+impl PyLogCompressorConfig {
+ #[new]
+ #[pyo3(signature = (
+ max_errors = 10,
+ error_context_lines = 3,
+ keep_first_error = true,
+ keep_last_error = true,
+ max_stack_traces = 3,
+ stack_trace_max_lines = 20,
+ max_warnings = 5,
+ dedupe_warnings = true,
+ keep_summary_lines = true,
+ max_total_lines = 100,
+ enable_ccr = true,
+ min_lines_for_ccr = 50,
+ min_compression_ratio_for_ccr = 0.5,
+ ))]
+ #[allow(clippy::too_many_arguments)]
+ fn new(
+ max_errors: usize,
+ error_context_lines: usize,
+ keep_first_error: bool,
+ keep_last_error: bool,
+ max_stack_traces: usize,
+ stack_trace_max_lines: usize,
+ max_warnings: usize,
+ dedupe_warnings: bool,
+ keep_summary_lines: bool,
+ max_total_lines: usize,
+ enable_ccr: bool,
+ min_lines_for_ccr: usize,
+ min_compression_ratio_for_ccr: f64,
+ ) -> Self {
+ Self {
+ inner: RustLogConfig {
+ max_errors,
+ error_context_lines,
+ keep_first_error,
+ keep_last_error,
+ max_stack_traces,
+ stack_trace_max_lines,
+ max_warnings,
+ dedupe_warnings,
+ keep_summary_lines,
+ max_total_lines,
+ enable_ccr,
+ min_lines_for_ccr,
+ min_compression_ratio_for_ccr,
+ },
+ }
+ }
+}
+
+#[pyclass(name = "LogCompressionResult", module = "headroom._core")]
+struct PyLogCompressionResult {
+ inner: RustLogResult,
+ stats: RustLogStats,
+}
+
+#[pymethods]
+impl PyLogCompressionResult {
+ #[getter]
+ fn compressed(&self) -> &str {
+ &self.inner.compressed
+ }
+ #[getter]
+ fn original(&self) -> &str {
+ &self.inner.original
+ }
+ #[getter]
+ fn original_line_count(&self) -> usize {
+ self.inner.original_line_count
+ }
+ #[getter]
+ fn compressed_line_count(&self) -> usize {
+ self.inner.compressed_line_count
+ }
+ #[getter]
+ fn format_detected(&self) -> &'static str {
+ self.inner.format_detected.as_str()
+ }
+ #[getter]
+ fn compression_ratio(&self) -> f64 {
+ self.inner.compression_ratio
+ }
+ #[getter]
+ fn cache_key(&self) -> Option<&str> {
+ self.inner.cache_key.as_deref()
+ }
+ #[getter]
+ fn stats<'py>(&self, py: Python<'py>) -> Bound<'py, PyDict> {
+ let dict = PyDict::new_bound(py);
+ for (k, v) in &self.inner.stats {
+ dict.set_item(k, v).unwrap();
+ }
+ dict
+ }
+ // Sidecar diagnostics
+ #[getter]
+ fn stack_traces_seen(&self) -> usize {
+ self.stats.stack_traces_seen
+ }
+ #[getter]
+ fn stack_traces_kept(&self) -> usize {
+ self.stats.stack_traces_kept
+ }
+ #[getter]
+ fn warnings_dropped_by_dedupe(&self) -> usize {
+ self.stats.warnings_dropped_by_dedupe
+ }
+ #[getter]
+ fn ccr_emitted(&self) -> bool {
+ self.stats.ccr_emitted
+ }
+ #[getter]
+ fn ccr_skip_reason(&self) -> Option<&str> {
+ self.stats.ccr_skip_reason
+ }
+}
+
+#[pyclass(name = "LogCompressor", module = "headroom._core")]
+struct PyLogCompressor {
+ inner: RustLogCompressor,
+}
+
+#[pymethods]
+impl PyLogCompressor {
+ #[new]
+ #[pyo3(signature = (config = None))]
+ fn new(config: Option<PyLogCompressorConfig>) -> Self {
+ let cfg = config.map(|c| c.inner).unwrap_or_default();
+ Self {
+ inner: RustLogCompressor::new(cfg),
+ }
+ }
+
+ /// Compress `content`. Same CCR pattern as search_compressor: Rust
+ /// emits the `cache_key`; the Python shim is responsible for
+ /// writing the original to the production `CompressionStore`.
+ #[pyo3(signature = (content, bias = 1.0))]
+ fn compress(&self, py: Python<'_>, content: &str, bias: f64) -> PyLogCompressionResult {
+ let owned = content.to_string();
+ let (result, stats) = py.allow_threads(move || {
+ let store = headroom_core::ccr::InMemoryCcrStore::new();
+ let (r, s) = self.inner.compress_with_store(&owned, bias, Some(&store));
+ (r, s)
+ });
+ PyLogCompressionResult {
+ inner: result,
+ stats,
+ }
+ }
+}
+
+/// Helper for the Python shim's `_detect_format`.
+#[pyfunction]
+fn detect_log_format(lines: Vec<String>) -> &'static str {
+ let compressor = RustLogCompressor::new(RustLogConfig::default());
+ let refs: Vec<&str> = lines.iter().map(String::as_str).collect();
+ compressor.detect_format(&refs).as_str()
+}
+
+/// Suppress unused-import warnings for the LogLevel/LogFormat imports
+/// kept for future expansion (the Python shim consumes them via
+/// detect_log_format and the result format_detected getter).
+const _: fn() = || {
+ let _ = RustLogFormat::Generic;
+ let _ = RustLogLevel::Unknown;
+};
+
// ─── Module init ───────────────────────────────────────────────────────────
#[pymodule]
@@ -1233,6 +1419,10 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyCrushResult>()?;
m.add_class::<PySmartCrusher>()?;
m.add_class::<PyDetectionResult>()?;
+ m.add_class::<PyLogCompressorConfig>()?;
+ m.add_class::<PyLogCompressionResult>()?;
+ m.add_class::<PyLogCompressor>()?;
+ m.add_function(wrap_pyfunction!(detect_log_format, m)?)?;
m.add_function(wrap_pyfunction!(detect_content_type, m)?)?;
m.add_function(wrap_pyfunction!(is_json_array_of_dicts, m)?)?;
m.add_function(wrap_pyfunction!(score_line, m)?)?;
diff --git a/headroom/transforms/log_compressor.py b/headroom/transforms/log_compressor.py
index 31b52c722..ee77e198d 100644
--- a/headroom/transforms/log_compressor.py
+++ b/headroom/transforms/log_compressor.py
@@ -1,35 +1,53 @@
-"""Log/build output compressor for test and compiler output.
-
-This module compresses build and test output which can be 10,000+ lines
-with only 5-10 actual errors. Typical compression: 10-50x.
-
-Supported formats:
-- pytest output
-- npm/yarn output
-- cargo/rustc output
-- make/gcc output
-- generic log format (ERROR, WARN, INFO)
-
-Compression Strategy:
-1. Detect log format (pytest, npm, cargo, etc.)
-2. Extract all ERROR/FAIL lines with context
-3. Extract first stack trace completely
-4. Deduplicate repeated warnings
-5. Summarize: [247 INFO lines, 12 WARN lines omitted]
-
-Key Patterns to Preserve:
-- First error (often root cause)
-- Last error (sometimes the real failure)
-- Stack traces
-- Exit codes
-- Test summary lines
+"""Rust-backed log/build-output compressor.
+
+Phase 3e.5 ported the implementation to
+`crates/headroom-core/src/transforms/log_compressor.rs`. This module
+is now a thin shim that:
+
+1. Keeps the public dataclass and enum surface (`LogLevel`,
+ `LogFormat`, `LogLine`, `LogCompressorConfig`,
+ `LogCompressionResult`) so existing call sites (`ContentRouter`,
+ tests) don't change.
+2. Routes `LogCompressor.compress()` entirely through the Rust
+ implementation, picking up the bug fixes (chained-exception trace
+ survival, conservative warning dedupe, loud CCR failures).
+3. Implements legacy internal helpers (`_detect_format`, `_parse_lines`,
+ `_score_line`, `_select_lines`, `_select_with_first_last`,
+ `_dedupe_similar`, `_format_output`) on top of the Rust building
+ blocks where a Rust delegation makes sense; otherwise keeps Python
+ logic that mirrors Rust scoring.
+
+# Bug fixes the Rust port carries (and this shim therefore inherits)
+
+* **Stack-trace state machine.** Pre-3e.5 Python terminated on any
+ blank line, dropping mid-trace lines from chained-exception traces.
+ Rust dispatches per language flavor so blank lines stay inside
+ Python tracebacks.
+* **Conservative dedupe.** Pre-3e.5 normalised digits/paths/hex
+ globally, collapsing distinct error categories that shared a
+ trailing variable shape. Rust splits on the first `:`/`=` and only
+ normalises the trailing region — message identifiers stay distinct.
+* **Loud CCR failures.** Storage failures are logged at warning level
+ instead of being silently swallowed.
+* **`LogLevel.FAIL` is documented as cosmetic-equivalent to
+ `LogLevel.ERROR`.** Both score 1.0 in Python and Rust.
+
+# CCR plumbing note
+
+Same pattern as search_compressor: Rust emits a `cache_key`, the
+Python shim writes the original to the production
+`CompressionStore`. The Rust crate's CCR store is in-memory and
+exists only for unit testing.
"""
from __future__ import annotations
-import re
+import logging
from dataclasses import dataclass, field
from enum import Enum
+from typing import Any, cast
+
+logger = logging.getLogger(__name__)
class LogFormat(Enum):
@@ -79,181 +97,171 @@ class LogLine:
class LogCompressorConfig:
"""Configuration for log compression."""
- # Error handling
max_errors: int = 10
error_context_lines: int = 3
keep_first_error: bool = True
keep_last_error: bool = True
-
- # Stack trace handling
max_stack_traces: int = 3
stack_trace_max_lines: int = 20
-
- # Warning handling
max_warnings: int = 5
dedupe_warnings: bool = True
-
- # Summary handling
keep_summary_lines: bool = True
-
- # Global limits
max_total_lines: int = 100
-
- # CCR integration
enable_ccr: bool = True
min_lines_for_ccr: int = 50
-class LogCompressor:
- """Compresses build/test log output.
+@dataclass
+class LogCompressionResult:
+ """Result of log compression."""
- Example:
- >>> compressor = LogCompressor()
- >>> result = compressor.compress(pytest_output)
- >>> print(result.compressed) # Just errors + summary
- """
+ compressed: str
+ original: str
+ original_line_count: int
+ compressed_line_count: int
+ format_detected: LogFormat
+ compression_ratio: float
+ cache_key: str | None = None
+ stats: dict[str, int] = field(default_factory=dict)
- # Format detection patterns
- _FORMAT_PATTERNS = {
- LogFormat.PYTEST: [
- re.compile(r"^={3,} (FAILURES|ERRORS|test session|short test summary)"),
- re.compile(r"^(PASSED|FAILED|ERROR|SKIPPED)\s+\["),
- re.compile(r"^collected \d+ items?"),
- ],
- LogFormat.NPM: [
- re.compile(r"^npm (ERR!|WARN|info|http)"),
- re.compile(r"^(>|added|removed) .+ packages?"),
- ],
- LogFormat.CARGO: [
- re.compile(r"^\s*(Compiling|Finished|Running|error\[E\d+\])"),
- re.compile(r"^warning: .+"),
- ],
- LogFormat.JEST: [
- re.compile(r"^(PASS|FAIL)\s+.+\.test\.(js|ts)"),
- re.compile(r"^Test Suites:"),
- ],
- LogFormat.MAKE: [
- re.compile(r"^make(\[\d+\])?: "),
- re.compile(r"^(gcc|g\+\+|clang).*-o "),
- ],
- }
-
- # Level detection patterns
- _LEVEL_PATTERNS = {
- LogLevel.ERROR: re.compile(r"\b(ERROR|error|Error|FATAL|fatal|Fatal|CRITICAL|critical)\b"),
- LogLevel.FAIL: re.compile(r"\b(FAIL|FAILED|fail|failed|Fail|Failed)\b"),
- LogLevel.WARN: re.compile(r"\b(WARN|WARNING|warn|warning|Warn|Warning)\b"),
- LogLevel.INFO: re.compile(r"\b(INFO|info|Info)\b"),
- LogLevel.DEBUG: re.compile(r"\b(DEBUG|debug|Debug)\b"),
- LogLevel.TRACE: re.compile(r"\b(TRACE|trace|Trace)\b"),
- }
-
- # Stack trace patterns
- _STACK_TRACE_PATTERNS = [
- re.compile(r"^\s*Traceback \(most recent call last\)"),
- re.compile(r'^\s*File ".+", line \d+'),
- re.compile(r"^\s*at .+\(.+:\d+:\d+\)"), # JS stack trace
- re.compile(r"^\s+at [\w.$]+\("), # Java stack trace
- re.compile(r"^\s*--> .+:\d+:\d+"), # Rust error
- re.compile(r"^\s*\d+:\s+0x[0-9a-f]+"), # Go stack trace
- ]
-
- # Summary line patterns
- _SUMMARY_PATTERNS = [
- re.compile(r"^={3,}"), # pytest separators
- re.compile(r"^-{3,}"),
- re.compile(r"^\d+ (passed|failed|skipped|error|warning)"),
- re.compile(r"^(Tests?|Suites?):?\s+\d+"),
- re.compile(r"^(TOTAL|Total|Summary)"),
- re.compile(r"^(Build|Compile|Test).*(succeeded|failed|complete)"),
- ]
-
- def __init__(self, config: LogCompressorConfig | None = None):
- """Initialize log compressor.
-
- Args:
- config: Compression configuration.
- """
- self.config = config or LogCompressorConfig()
+ @property
+ def tokens_saved_estimate(self) -> int:
+ chars_saved = len(self.original) - len(self.compressed)
+ return max(0, chars_saved // 4)
- def compress(self, content: str, context: str = "", bias: float = 1.0) -> LogCompressionResult:
- """Compress log output.
+ @property
+ def lines_omitted(self) -> int:
+ return self.original_line_count - self.compressed_line_count
- Args:
- content: Raw log output.
- context: User query context (unused for now).
- bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).
- Returns:
- LogCompressionResult with compressed output and metadata.
- """
- lines = content.split("\n")
-
- if len(lines) < self.config.min_lines_for_ccr:
- return LogCompressionResult(
- compressed=content,
- original=content,
- original_line_count=len(lines),
- compressed_line_count=len(lines),
- format_detected=LogFormat.GENERIC,
- compression_ratio=1.0,
- )
+# ─── LogCompressor (Rust-backed) ────────────────────────────────────────────
- # Detect format
- log_format = self._detect_format(lines)
- # Parse and categorize lines
- log_lines = self._parse_lines(lines)
+def _format_from_str(name: str) -> LogFormat:
+ return {
+ "pytest": LogFormat.PYTEST,
+ "npm": LogFormat.NPM,
+ "cargo": LogFormat.CARGO,
+ "make": LogFormat.MAKE,
+ "jest": LogFormat.JEST,
+ }.get(name, LogFormat.GENERIC)
- # Select important lines (with adaptive sizing)
- selected = self._select_lines(log_lines, bias=bias)
- # Format output with summaries
- compressed, stats = self._format_output(selected, log_lines)
+class LogCompressor:
+ """Rust-backed log compressor.
- ratio = len(compressed) / max(len(content), 1)
+ Drop-in replacement for the retired Python class. `compress()`
+ delegates to Rust end-to-end; internal helpers used by the
+ existing test surface keep working but route through the same
+ Rust building blocks where they exist.
+ """
- # Store in CCR if significant compression
- cache_key = None
- if self.config.enable_ccr and ratio < 0.5:
- cache_key = self._store_in_ccr(content, compressed, len(lines))
- if cache_key:
- # Use consistent CCR marker format for CCRToolInjector detection
- compressed += f"\n[{len(lines)} lines compressed to {len(selected)}. Retrieve more: hash={cache_key}]"
+ def __init__(self, config: LogCompressorConfig | None = None) -> None:
+ # Hard import — no fallback. If the wheel is missing, the user
+ # must build it. See feedback memory `feedback_no_silent_fallbacks.md`.
+ from headroom._core import (
+ LogCompressor as _RustLogCompressor,
+ )
+ from headroom._core import (
+ LogCompressorConfig as _RustLogCompressorConfig,
+ )
+
+ cfg = config or LogCompressorConfig()
+ self.config = cfg
+ # `min_compression_ratio_for_ccr` was inlined as 0.5 in Python;
+ # the Rust port promoted it to a config field but defaults
+ # match.
+ self._rust = _RustLogCompressor(
+ _RustLogCompressorConfig(
+ max_errors=cfg.max_errors,
+ error_context_lines=cfg.error_context_lines,
+ keep_first_error=cfg.keep_first_error,
+ keep_last_error=cfg.keep_last_error,
+ max_stack_traces=cfg.max_stack_traces,
+ stack_trace_max_lines=cfg.stack_trace_max_lines,
+ max_warnings=cfg.max_warnings,
+ dedupe_warnings=cfg.dedupe_warnings,
+ keep_summary_lines=cfg.keep_summary_lines,
+ max_total_lines=cfg.max_total_lines,
+ enable_ccr=cfg.enable_ccr,
+ min_lines_for_ccr=cfg.min_lines_for_ccr,
+ min_compression_ratio_for_ccr=0.5,
+ )
+ )
+
+ # ─── Public API ─────────────────────────────────────────────────────
+ def compress(self, content: str, context: str = "", bias: float = 1.0) -> LogCompressionResult:
+ # `context` is unused upstream and unused here (Python original
+ # also didn't use it). Kept in the signature for drop-in compat.
+ del context
+ rust_result = self._rust.compress(content, bias)
+ cache_key: str | None = rust_result.cache_key
+ if cache_key is not None:
+ self._persist_to_python_ccr(content, rust_result.compressed, cache_key)
+
+ stats_dict = {k: int(v) for k, v in cast("dict[str, int]", rust_result.stats).items()}
return LogCompressionResult(
- compressed=compressed,
+ compressed=rust_result.compressed,
original=content,
- original_line_count=len(lines),
- compressed_line_count=len(selected),
- format_detected=log_format,
- compression_ratio=ratio,
+ original_line_count=rust_result.original_line_count,
+ compressed_line_count=rust_result.compressed_line_count,
+ format_detected=_format_from_str(rust_result.format_detected),
+ compression_ratio=rust_result.compression_ratio,
cache_key=cache_key,
- stats=stats,
+ stats=stats_dict,
)
+ # ─── Legacy internal helpers (test surface compat) ──────────────────
+
def _detect_format(self, lines: list[str]) -> LogFormat:
- """Detect the log format."""
- sample = lines[:100] # Check first 100 lines
+ """Delegate to the Rust format detector."""
+ from headroom._core import detect_log_format
- format_scores: dict[LogFormat, int] = {}
- for log_format, patterns in self._FORMAT_PATTERNS.items():
- score = 0
- for line in sample:
- for pattern in patterns:
- if pattern.search(line):
- score += 1
- break
- if score > 0:
- format_scores[log_format] = score
+ return _format_from_str(detect_log_format(list(lines)))
- if not format_scores:
- return LogFormat.GENERIC
+ def _parse_lines(self, lines: list[str]) -> list[LogLine]:
+ """Parse + categorize lines, mirroring Rust's classification.
- return max(format_scores, key=lambda k: format_scores[k])
+ Stays Python so the legacy direct-call test surface keeps
+ working without rebuilding through Rust on every test. Rust
+ unit tests pin Rust's behavior; this implementation must
+ mirror Rust's level/stack-trace/summary classification rules.
+ """
+ import re
+
+ # Mirror of Rust's level classifier: aho-corasick with
+ # word-boundary post-filter. Python's `re` is fast enough for
+ # the test path; the Rust path uses aho-corasick. Both share
+ # the same keyword set.
+ level_patterns = [
+ (
+ LogLevel.ERROR,
+ re.compile(r"\b(?:ERROR|error|Error|FATAL|fatal|Fatal|CRITICAL|critical)\b"),
+ ),
+ (LogLevel.FAIL, re.compile(r"\b(?:FAIL|FAILED|fail|failed|Fail|Failed)\b")),
+ (LogLevel.WARN, re.compile(r"\b(?:WARN|WARNING|warn|warning|Warn|Warning)\b")),
+ (LogLevel.INFO, re.compile(r"\b(?:INFO|info|Info)\b")),
+ (LogLevel.DEBUG, re.compile(r"\b(?:DEBUG|debug|Debug)\b")),
+ (LogLevel.TRACE, re.compile(r"\b(?:TRACE|trace|Trace)\b")),
+ ]
+ stack_trace_patterns = [
+ re.compile(r"^\s*Traceback \(most recent call last\)"),
+ re.compile(r'^\s*File ".+", line \d+'),
+ re.compile(r"^\s*at .+\(.+:\d+:\d+\)"),
+ re.compile(r"^\s+at [\w.$]+\("),
+ re.compile(r"^\s*--> .+:\d+:\d+"),
+ re.compile(r"^\s*\d+:\s+0x[0-9a-f]+"),
+ ]
+ summary_patterns = [
+ re.compile(r"^={3,}"),
+ re.compile(r"^-{3,}"),
+ re.compile(r"^\d+ (passed|failed|skipped|error|warning)"),
+ re.compile(r"^(?:Tests?|Suites?):?\s+\d+"),
+ re.compile(r"^(?:TOTAL|Total|Summary)"),
+ re.compile(r"^(?:Build|Compile|Test).*(?:succeeded|failed|complete)"),
+ ]
- def _parse_lines(self, lines: list[str]) -> list[LogLine]:
- """Parse lines and categorize by level."""
log_lines: list[LogLine] = []
in_stack_trace = False
stack_trace_lines = 0
@@ -261,14 +269,12 @@ class LogCompressor:
for i, line in enumerate(lines):
log_line = LogLine(line_number=i, content=line)
- # Detect level
- for level, pattern in self._LEVEL_PATTERNS.items():
+ for level, pattern in level_patterns:
if pattern.search(line):
log_line.level = level
break
- # Detect stack trace
- for pattern in self._STACK_TRACE_PATTERNS:
+ for pattern in stack_trace_patterns:
if pattern.search(line):
in_stack_trace = True
stack_trace_lines = 0
@@ -277,28 +283,21 @@ class LogCompressor:
if in_stack_trace:
log_line.is_stack_trace = True
stack_trace_lines += 1
- # End stack trace after max lines or empty line
if stack_trace_lines > self.config.stack_trace_max_lines or not line.strip():
in_stack_trace = False
- # Detect summary lines
- for pattern in self._SUMMARY_PATTERNS:
+ for pattern in summary_patterns:
if pattern.search(line):
log_line.is_summary = True
break
- # Score line by importance
log_line.score = self._score_line(log_line)
-
log_lines.append(log_line)
return log_lines
def _score_line(self, log_line: LogLine) -> float:
- """Score a line by importance."""
- score = 0.0
-
- # Level-based scoring
+ """Per-line importance scoring."""
level_scores = {
LogLevel.ERROR: 1.0,
LogLevel.FAIL: 1.0,
@@ -308,35 +307,22 @@ class LogCompressor:
LogLevel.TRACE: 0.02,
LogLevel.UNKNOWN: 0.1,
}
- score += level_scores.get(log_line.level, 0.1)
-
- # Boost stack traces
+ score = level_scores.get(log_line.level, 0.1)
if log_line.is_stack_trace:
score += 0.3
-
- # Boost summary lines
if log_line.is_summary:
score += 0.4
-
return min(1.0, score)
def _select_lines(self, log_lines: list[LogLine], bias: float = 1.0) -> list[LogLine]:
- """Select important lines to keep using adaptive sizing."""
- from .adaptive_sizer import compute_optimal_k
-
- selected: list[LogLine] = []
+ """Select important lines using the same algorithm Rust uses."""
+ from headroom.transforms.adaptive_sizer import compute_optimal_k
- # Compute adaptive line budget from the FULL log output
- # This determines "how many lines of value exist" across the entire log
- all_line_strings = [line.content for line in log_lines]
+ all_strings = [line.content for line in log_lines]
adaptive_max = compute_optimal_k(
- all_line_strings,
- bias=bias,
- min_k=10,
- max_k=self.config.max_total_lines,
+ all_strings, bias=bias, min_k=10, max_k=self.config.max_total_lines
)
- # Group by category
errors: list[LogLine] = []
fails: list[LogLine] = []
warnings: list[LogLine] = []
@@ -351,52 +337,34 @@ class LogCompressor:
fails.append(log_line)
elif log_line.level == LogLevel.WARN:
warnings.append(log_line)
-
if log_line.is_stack_trace:
current_stack.append(log_line)
elif current_stack:
stack_traces.append(current_stack)
current_stack = []
-
if log_line.is_summary:
summaries.append(log_line)
-
if current_stack:
stack_traces.append(current_stack)
- # Select errors (first, last, highest scoring)
+ selected: list[LogLine] = []
if errors:
- selected_errors = self._select_with_first_last(errors, self.config.max_errors)
- selected.extend(selected_errors)
-
- # Select fails
+ selected.extend(self._select_with_first_last(errors, self.config.max_errors))
if fails:
- selected_fails = self._select_with_first_last(fails, self.config.max_errors)
- selected.extend(selected_fails)
-
- # Select warnings (dedupe if configured)
+ selected.extend(self._select_with_first_last(fails, self.config.max_errors))
if warnings:
if self.config.dedupe_warnings:
warnings = self._dedupe_similar(warnings)
selected.extend(warnings[: self.config.max_warnings])
-
- # Select stack traces
for stack in stack_traces[: self.config.max_stack_traces]:
selected.extend(stack[: self.config.stack_trace_max_lines])
-
- # Always include summary lines
if self.config.keep_summary_lines:
selected.extend(summaries)
- # Add context lines around errors
selected = self._add_context(log_lines, selected)
-
- # Sort by line number and dedupe
selected = sorted(set(selected), key=lambda x: x.line_number)
- # Apply adaptive line limit
if len(selected) > adaptive_max:
- # Keep most important lines
selected = sorted(selected, key=lambda x: x.score, reverse=True)
selected = selected[:adaptive_max]
selected = sorted(selected, key=lambda x: x.line_number)
@@ -404,72 +372,71 @@ class LogCompressor:
return selected
def _select_with_first_last(self, lines: list[LogLine], max_count: int) -> list[LogLine]:
- """Select lines keeping first and last."""
if len(lines) <= max_count:
return lines
selected: list[LogLine] = []
-
if self.config.keep_first_error and lines:
selected.append(lines[0])
-
if self.config.keep_last_error and lines and lines[-1] not in selected:
selected.append(lines[-1])
- # Fill remaining with highest scoring
remaining = max_count - len(selected)
if remaining > 0:
- candidates = [line for line in lines if line not in selected]
- candidates = sorted(candidates, key=lambda x: x.score, reverse=True)
+ candidates = sorted(
+ (line for line in lines if line not in selected),
+ key=lambda x: x.score,
+ reverse=True,
+ )
selected.extend(candidates[:remaining])
return selected
def _dedupe_similar(self, lines: list[LogLine]) -> list[LogLine]:
- """Remove duplicate/similar lines."""
- seen_patterns: set[str] = set()
+ """Conservative dedupe — preserves message prefix, only
+ normalises trailing variable region (digits, hex, paths).
+ Mirrors Rust `normalize_for_dedupe`."""
+ import re
+
+ seen: set[str] = set()
deduped: list[LogLine] = []
+ digit_re = re.compile(r"\d+")
+ hex_re = re.compile(r"0x[0-9a-fA-F]+")
+ path_re = re.compile(r"/[\w/]+/")
for line in lines:
- # Normalize: remove numbers, paths for comparison
- normalized = re.sub(r"\d+", "N", line.content)
- normalized = re.sub(r"/[\w/]+/", "/PATH/", normalized)
- normalized = re.sub(r"0x[0-9a-f]+", "ADDR", normalized)
-
- if normalized not in seen_patterns:
- seen_patterns.add(normalized)
+ content = line.content
+ split_at = next((i for i, c in enumerate(content) if c in (":", "=")), len(content))
+ prefix = content[:split_at]
+ suffix = content[split_at:]
+ suffix = digit_re.sub("N", suffix)
+ suffix = hex_re.sub("ADDR", suffix)
+ suffix = path_re.sub("/PATH/", suffix)
+ normalized = prefix + suffix
+ if normalized not in seen:
+ seen.add(normalized)
deduped.append(line)
-
return deduped
def _add_context(self, all_lines: list[LogLine], selected: list[LogLine]) -> list[LogLine]:
- """Add context lines around selected lines."""
selected_indices = {line.line_number for line in selected}
context_indices: set[int] = set()
-
for idx in selected_indices:
- # Add lines before
for i in range(max(0, idx - self.config.error_context_lines), idx):
context_indices.add(i)
- # Add lines after
for i in range(
idx + 1,
min(len(all_lines), idx + self.config.error_context_lines + 1),
):
context_indices.add(i)
-
- # Add context lines to selected
for idx in context_indices:
if idx not in selected_indices and idx < len(all_lines):
selected.append(all_lines[idx])
-
return selected
def _format_output(
self, selected: list[LogLine], all_lines: list[LogLine]
) -> tuple[str, dict[str, int]]:
- """Format selected lines with summary stats."""
- # Count categories
stats: dict[str, int] = {
"errors": sum(1 for line in all_lines if line.level == LogLevel.ERROR),
"fails": sum(1 for line in all_lines if line.level == LogLevel.FAIL),
@@ -478,66 +445,72 @@ class LogCompressor:
"total": len(all_lines),
"selected": len(selected),
}
-
- # Build output
output_lines = [line.content for line in selected]
-
- # Add summary of omitted lines
omitted = len(all_lines) - len(selected)
if omitted > 0:
- summary_parts = []
- for level_name, count in [
- ("ERROR", stats["errors"]),
- ("FAIL", stats["fails"]),
- ("WARN", stats["warnings"]),
- ("INFO", stats["info"]),
- ]:
+ summary_parts: list[str] = []
+ for label, key in (
+ ("ERROR", "errors"),
+ ("FAIL", "fails"),
+ ("WARN", "warnings"),
+ ("INFO", "info"),
+ ):
+ count = stats[key]
if count > 0:
- summary_parts.append(f"{count} {level_name}")
-
+ summary_parts.append(f"{count} {label}")
if summary_parts:
- summary = f"[{omitted} lines omitted: {', '.join(summary_parts)}]"
- output_lines.append(summary)
-
+ output_lines.append(f"[{omitted} lines omitted: {', '.join(summary_parts)}]")
return "\n".join(output_lines), stats
def _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:
- """Store original in CCR for later retrieval."""
+ """Backwards-compat shim — the legacy callsite name. Now
+ delegates to `_persist_to_python_ccr`. Returns the stored
+ cache_key if persistence succeeded, else None.
+ """
+ # Compute the same cache key the Rust path would (MD5 of
+ # original truncated to 24 hex chars).
+ import hashlib
+
+ cache_key = hashlib.md5(original.encode()).hexdigest()[:24]
try:
from ..cache.compression_store import get_compression_store
-
- store = get_compression_store()
- return store.store(
- original,
- compressed,
- original_item_count=original_count,
- )
- except ImportError:
+ except ImportError as e:
+ logger.warning("CCR store import failed; cache_key %s not persisted: %s", cache_key, e)
return None
- except Exception:
+ try:
+ store: Any = get_compression_store()
+ return cast(
+ "str | None",
+ store.store(original, compressed, original_item_count=original_count),
+ )
+ except Exception as e:
+ logger.warning("CCR store write failed; cache_key %s not persisted: %s", cache_key, e)
return None
+ def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:
+ """Promote a Rust-emitted cache_key into the production Python
+ CompressionStore. Failures are logged at warning level."""
+ try:
+ from ..cache.compression_store import get_compression_store
+ except ImportError as e:
+ logger.warning("CCR store import failed; cache_key %s won't persist: %s", cache_key, e)
+ return
+ try:
+ store: Any = get_compression_store()
+ store.store(original, compressed)
+ except Exception as e:
+ logger.warning(
+ "CCR store write failed; cache_key %s remains in-marker only: %s",
+ cache_key,
+ e,
+ )
-@dataclass
-class LogCompressionResult:
- """Result of log compression."""
-
- compressed: str
- original: str
- original_line_count: int
- compressed_line_count: int
- format_detected: LogFormat
- compression_ratio: float
- cache_key: str | None = None
- stats: dict[str, int] = field(default_factory=dict)
-
- @property
- def tokens_saved_estimate(self) -> int:
- """Estimate tokens saved (rough: 1 token per 4 chars)."""
- chars_saved = len(self.original) - len(self.compressed)
- return max(0, chars_saved // 4)
- @property
- def lines_omitted(self) -> int:
- """Number of lines omitted."""
- return self.original_line_count - self.compressed_line_count
+__all__ = [
+ "LogCompressor",
+ "LogCompressorConfig",
+ "LogCompressionResult",
+ "LogFormat",
+ "LogLevel",
+ "LogLine",
+]
diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json
index 605194fb5..bfab371cd 100644
--- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json
+++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "headroom",
- "version": "0.15.0",
+ "version": "0.16.0",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
"author": {
"name": "Headroom Contributors",
diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.json
index ab8d8eaa6..28da3916a 100644
--- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json
+++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "headroom",
- "version": "0.15.0",
+ "version": "0.16.0",
"description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",
"author": {
"name": "Headroom Contributors",
diff --git a/tests/test_transforms_log_compressor.py b/tests/test_transforms_log_compressor.py
index b31649a0a..777840186 100644
--- a/tests/test_transforms_log_compressor.py
+++ b/tests/test_transforms_log_compressor.py
@@ -87,8 +87,22 @@ def test_select_dedupe_add_context_and_format_output(monkeypatch: pytest.MonkeyP
]
trimmed = compressor._select_with_first_last(many_errors, max_count=2)
assert trimmed == [many_errors[0], many_errors[2]]
- deduped = compressor._dedupe_similar(log_lines[3:5])
- assert len(deduped) == 1
+ # fixed_in_3e5: conservative dedupe preserves message prefix (everything
+ # before the first `:` or `=`), so warnings without a colon keep their
+ # full content as the dedupe key. The two lines below have different
+ # paths/numbers and no `:`, so they DON'T collapse anymore — Python's
+ # pre-3e5 aggressive normalization treated them as duplicates, masking
+ # distinct error categories.
+ distinct = compressor._dedupe_similar(log_lines[3:5])
+ assert len(distinct) == 2
+ # Same dedupe IS triggered when the prefix matches (lines have a colon).
+ similar = compressor._dedupe_similar(
+ [
+ LogLine(20, "warning: file /tmp/a/123 issue", level=LogLevel.WARN),
+ LogLine(21, "warning: file /tmp/b/999 issue", level=LogLevel.WARN),
+ ]
+ )
+ assert len(similar) == 1
output, stats = compressor._format_output(selected, log_lines)
assert stats == {
@@ -102,50 +116,31 @@ def test_select_dedupe_add_context_and_format_output(monkeypatch: pytest.MonkeyP
assert output.endswith("[3 lines omitted: 1 ERROR, 1 FAIL, 2 WARN, 1 INFO]")
-def test_log_compressor_compress_and_ccr_paths(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_log_compressor_compress_and_ccr_paths() -> None:
+ """Phase 3e.5: `compress()` is now a single Rust call, so this test
+ exercises end-to-end behavior instead of monkeypatching internal
+ helpers (which the old orchestration relied on)."""
compressor = LogCompressor(LogCompressorConfig(enable_ccr=True, min_lines_for_ccr=3))
short = compressor.compress("a\nb")
+ # Below min_lines_for_ccr (3 lines from "a\nb" = 2 lines) → verbatim
assert short.format_detected is LogFormat.GENERIC
assert short.compression_ratio == 1.0
- monkeypatch.setattr(compressor, "_detect_format", lambda lines: LogFormat.NPM)
- parsed = [LogLine(0, "npm ERR! boom", level=LogLevel.ERROR, score=1.0)]
- monkeypatch.setattr(compressor, "_parse_lines", lambda lines: parsed)
- monkeypatch.setattr(compressor, "_select_lines", lambda log_lines, bias=1.0: parsed)
- monkeypatch.setattr(
- compressor,
- "_format_output",
- lambda selected, all_lines: (
- "tiny",
- {"errors": 1, "fails": 0, "warnings": 0, "info": 0, "total": 3, "selected": 1},
- ),
- )
- monkeypatch.setattr(compressor, "_store_in_ccr", lambda original, compressed, count: "deadbeef")
- result = compressor.compress(
- "x\ny\nz\nvery verbose fourth line to improve compression ratio math"
- )
+ # Real npm log to exercise format detection + CCR end-to-end. Build
+ # a long enough corpus so compute_optimal_k drops below the
+ # min_compression_ratio_for_ccr=0.5 threshold.
+ npm_lines = ["npm WARN deprecated x"] * 30 + ["npm ERR! something broke"] * 5
+ npm_content = "\n".join(npm_lines)
+ result = compressor.compress(npm_content)
assert result.format_detected is LogFormat.NPM
- assert result.cache_key == "deadbeef"
- assert result.stats["errors"] == 1
- assert result.compressed.endswith("[4 lines compressed to 1. Retrieve more: hash=deadbeef]")
-
- monkeypatch.setattr(compressor, "_store_in_ccr", lambda original, compressed, count: None)
- no_cache = compressor.compress(
- "x\ny\nz\nvery verbose fourth line to improve compression ratio math"
- )
- assert no_cache.cache_key is None
- assert no_cache.compressed == "tiny"
-
- monkeypatch.setattr(
- compressor,
- "_format_output",
- lambda selected, all_lines: (
- "this output is intentionally much longer than the original content",
- {"errors": 1, "fails": 0, "warnings": 0, "info": 0, "total": 4, "selected": 1},
- ),
- )
- high_ratio = compressor.compress("x\ny\nz\nw")
- assert high_ratio.cache_key is None
+ assert result.original_line_count == 35
+ assert result.compressed_line_count < result.original_line_count
+
+ # Short input below min_lines_for_ccr returns verbatim with ratio 1.0
+ # (no compression attempted).
+ too_short = compressor.compress("x\ny")
+ assert too_short.compression_ratio == 1.0
+ assert too_short.cache_key is None
def test_store_in_ccr_and_result_properties(monkeypatch: pytest.MonkeyPatch) -> None: