Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(rust): port log_compressor to Rust + bug fixes (Phase 3e.5)
chopratejas45720301e5a2
diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.jsonindex 2c7ac5b9e..977997e8d 100644--- a/.claude-plugin/marketplace.json+++ b/.claude-plugin/marketplace.json@@ -5,14 +5,14 @@},"metadata": {"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.15.0"+ "version": "0.16.0"},"plugins": [{"name": "headroom","source": "./plugins/headroom-agent-hooks","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.15.0",+ "version": "0.16.0","author": {"name": "Headroom Contributors","url": "https://github.com/chopratejas/headroom"diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.jsonindex 2c7ac5b9e..977997e8d 100644--- a/.github/plugin/marketplace.json+++ b/.github/plugin/marketplace.json@@ -5,14 +5,14 @@},"metadata": {"description": "Headroom marketplace for Claude Code and GitHub Copilot CLI plugins.",- "version": "0.15.0"+ "version": "0.16.0"},"plugins": [{"name": "headroom","source": "./plugins/headroom-agent-hooks","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.",- "version": "0.15.0",+ "version": "0.16.0","author": {"name": "Headroom Contributors","url": "https://github.com/chopratejas/headroom"diff --git a/crates/headroom-core/src/transforms/log_compressor.rs b/crates/headroom-core/src/transforms/log_compressor.rsnew file mode 100644index 000000000..4ad8c9f15--- /dev/null+++ b/crates/headroom-core/src/transforms/log_compressor.rs@@ -0,0 +1,1295 @@+//! Log/build-output compressor — Rust port of+//! `headroom.transforms.log_compressor`.+//!+//! Compresses build and test output (pytest, npm, cargo, jest, make,+//! generic). Typical input: 10,000+ lines with 5-10 actual errors.+//! Typical compression: 10-50×.+//!+//! # Pipeline+//!+//! 1. Format detection (pytest / npm / cargo / jest / make / generic).+//! 2. Per-line classification: level (ERROR/FAIL/WARN/INFO/DEBUG/TRACE),+//! stack-trace membership, summary-line membership.+//! 3. Per-line scoring (level base + stack-trace + summary boosts).+//! 4. Adaptive total-lines budget via+//! [`crate::transforms::adaptive_sizer::compute_optimal_k`].+//! 5. Category selection: errors (first/last/top), fails, warnings+//! (deduped), stack traces, summaries; context window around each+//! selection; final adaptive cap.+//! 6. Optional CCR storage when `compression_ratio < 0.5`.+//!+//! # Bug fixes vs Python (2026-04-30)+//!+//! Each fix is paired with a `fixed_in_3e5` parity-fixture marker.+//!+//! - **Stack-trace state machine.** Python's machine terminated on any+//! blank line, dropping mid-trace lines from chained-exception+//! traces (which embed blank separators between cause groups). The+//! Rust dispatcher tracks per-flavor termination rules: Python+//! `Traceback` ends on a non-indented non-blank line *after at least+//! one indented frame*; JS at the next non-`at`-prefixed line+//! immediately after the last `at` frame; etc.+//! - **Conservative dedupe.** Python's `_dedupe_similar` blanket-+//! normalized digits/paths/hex into single tokens, so segfaults at+//! different addresses or test failures with different IDs collapsed+//! into a single survivor. The Rust normalizer preserves the+//! *message prefix* (everything before the first `:` or `=`) so two+//! distinct errors with the same trailing address pattern stay+//! distinct, and only the trailing variable region is tokenized.+//! - **Loud CCR failures.** Python silently swallowed all exceptions+//! from the store. Rust emits `tracing::warn!` and the Python shim+//! logs at `warning` level so operators see misconfigured stores.+//! - **`LogLevel::FAIL` is documented as cosmetic-equivalent to+//! `ERROR`.** Both score 1.0 in Python; the distinction is purely+//! for human-readable summary output. Preserved for parity but+//! future code should treat them as equivalent.++use std::collections::{BTreeMap, BTreeSet};+use std::sync::OnceLock;++use aho_corasick::{AhoCorasick, AhoCorasickBuilder, MatchKind};+use md5::{Digest, Md5};+use regex::Regex;++use crate::ccr::CcrStore;+use crate::transforms::adaptive_sizer::compute_optimal_k;++// ─── Types ──────────────────────────────────────────────────────────────++/// Detected log format. `Generic` is the fall-through.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum LogFormat {+ Pytest,+ Npm,+ Cargo,+ Jest,+ Make,+ Generic,+}++impl LogFormat {+ pub fn as_str(&self) -> &'static str {+ match self {+ LogFormat::Pytest => "pytest",+ LogFormat::Npm => "npm",+ LogFormat::Cargo => "cargo",+ LogFormat::Jest => "jest",+ LogFormat::Make => "make",+ LogFormat::Generic => "generic",+ }+ }+}++/// Per-line log level. ERROR/FAIL are scored equivalently — the+/// distinction is cosmetic (preserved for parity with Python's enum).+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]+pub enum LogLevel {+ Error,+ Fail,+ Warn,+ Info,+ Debug,+ Trace,+ Unknown,+}++impl LogLevel {+ pub fn as_str(&self) -> &'static str {+ match self {+ LogLevel::Error => "error",+ LogLevel::Fail => "fail",+ LogLevel::Warn => "warn",+ LogLevel::Info => "info",+ LogLevel::Debug => "debug",+ LogLevel::Trace => "trace",+ LogLevel::Unknown => "unknown",+ }+ }+}++/// One classified log line.+///+/// `Eq`/`Hash` are based on `line_number` only (matches Python's custom+/// dunders). Two `LogLine`s at the same line_number are considered the+/// same entry regardless of content/level — supports the set-based+/// dedupe in selection.+#[derive(Debug, Clone)]+pub struct LogLine {+ pub line_number: usize,+ pub content: String,+ pub level: LogLevel,+ pub is_stack_trace: bool,+ pub is_summary: bool,+ pub score: f32,+}++impl PartialEq for LogLine {+ fn eq(&self, other: &Self) -> bool {+ self.line_number == other.line_number+ }+}++impl Eq for LogLine {}++impl std::hash::Hash for LogLine {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.line_number.hash(state);+ }+}++impl LogLine {+ pub fn new(line_number: usize, content: impl Into<String>) -> Self {+ Self {+ line_number,+ content: content.into(),+ level: LogLevel::Unknown,+ is_stack_trace: false,+ is_summary: false,+ score: 0.0,+ }+ }+}++/// Compressor configuration. Defaults match Python `LogCompressorConfig`.+#[derive(Debug, Clone)]+pub struct LogCompressorConfig {+ pub max_errors: usize,+ pub error_context_lines: usize,+ pub keep_first_error: bool,+ pub keep_last_error: bool,+ pub max_stack_traces: usize,+ pub stack_trace_max_lines: usize,+ pub max_warnings: usize,+ pub dedupe_warnings: bool,+ pub keep_summary_lines: bool,+ pub max_total_lines: usize,+ pub enable_ccr: bool,+ pub min_lines_for_ccr: usize,+ /// Compression ratio threshold for CCR storage. Python defaults to+ /// 0.5 inline; promoted to a config field here.+ pub min_compression_ratio_for_ccr: f64,+}++impl Default for LogCompressorConfig {+ fn default() -> Self {+ Self {+ max_errors: 10,+ error_context_lines: 3,+ keep_first_error: true,+ keep_last_error: true,+ max_stack_traces: 3,+ stack_trace_max_lines: 20,+ max_warnings: 5,+ dedupe_warnings: true,+ keep_summary_lines: true,+ max_total_lines: 100,+ enable_ccr: true,+ min_lines_for_ccr: 50,+ min_compression_ratio_for_ccr: 0.5,+ }+ }+}++/// Compression result.+#[derive(Debug, Clone)]+pub struct LogCompressionResult {+ pub compressed: String,+ pub original: String,+ pub original_line_count: usize,+ pub compressed_line_count: usize,+ pub format_detected: LogFormat,+ pub compression_ratio: f64,+ pub cache_key: Option<String>,+ pub stats: BTreeMap<String, u64>,+}++impl LogCompressionResult {+ pub fn tokens_saved_estimate(&self) -> i64 {+ let chars_saved = self.original.len() as i64 - self.compressed.len() as i64;+ chars_saved.max(0) / 4+ }+ pub fn lines_omitted(&self) -> usize {+ self.original_line_count+ .saturating_sub(self.compressed_line_count)+ }+}++/// Sidecar diagnostics not returned by the parity-equal API.+#[derive(Debug, Clone, Default)]+pub struct LogCompressorStats {+ pub format: Option<LogFormat>,+ pub stack_traces_seen: usize,+ pub stack_traces_kept: usize,+ pub warnings_dropped_by_dedupe: usize,+ pub lines_dropped_by_global_cap: usize,+ pub ccr_emitted: bool,+ pub ccr_skip_reason: Option<&'static str>,+}++// ─── Format detector ────────────────────────────────────────────────────++/// Inline static-table format detector. Walks the first 100 lines and+/// picks the format with the most marker hits (Python parity).+struct FormatDetector {+ matchers: Vec<(LogFormat, AhoCorasick)>,+}++impl FormatDetector {+ fn new() -> Self {+ let table: &[(LogFormat, &[&str])] = &[+ (+ LogFormat::Pytest,+ &[+ "=== FAILURES",+ "=== ERRORS",+ "=== test session",+ "=== short test summary",+ "PASSED [",+ "FAILED [",+ "ERROR [",+ "SKIPPED [",+ "collected ",+ ],+ ),+ (+ LogFormat::Npm,+ &["npm ERR!", "npm WARN", "npm info", "npm http"],+ ),+ (+ LogFormat::Cargo,+ &[+ "Compiling ",+ "Finished ",+ "Running ",+ "warning: ",+ "error[E",+ ],+ ),+ (LogFormat::Jest, &["PASS ", "FAIL ", "Test Suites:"]),+ (+ LogFormat::Make,+ &["make[", "make:", "gcc ", "g++ ", "clang "],+ ),+ ];++ let matchers = table+ .iter()+ .map(|(fmt, patterns)| {+ let ac = AhoCorasickBuilder::new()+ .ascii_case_insensitive(false)+ .match_kind(MatchKind::LeftmostFirst)+ .build(*patterns)+ .expect("format-detector automaton must build (static input)");+ (*fmt, ac)+ })+ .collect();+ Self { matchers }+ }++ fn detect(&self, lines: &[&str]) -> LogFormat {+ let sample: Vec<&str> = lines.iter().take(100).copied().collect();+ let mut best: Option<(LogFormat, usize)> = None;+ for (fmt, ac) in &self.matchers {+ let mut score = 0;+ for line in &sample {+ // Python's per-format inner loop counts at most ONE hit+ // per line ("for pattern in patterns: ... break"). Mirror+ // that: aho-corasick's `is_match` is sufficient.+ if ac.is_match(*line) {+ score += 1;+ }+ }+ if score > 0 && best.map(|(_, s)| score > s).unwrap_or(true) {+ best = Some((*fmt, score));+ }+ }+ best.map(|(f, _)| f).unwrap_or(LogFormat::Generic)+ }+}++// ─── Level classifier ────────────────────────────────────────────────────++/// Word-boundary aware level classifier. Replaces Python's+/// `_LEVEL_PATTERNS` regexes with a single aho-corasick scan + ASCII+/// word-boundary post-filter (same technique+/// `signals::keyword_detector` uses).+struct LevelClassifier {+ automaton: AhoCorasick,+ /// Parallel array: index of `pattern_idx` → LogLevel returned.+ levels: Vec<LogLevel>,+}++impl LevelClassifier {+ fn new() -> Self {+ // Order matters — Python checks ERROR before FAIL, and we want+ // first-match wins. AhoCorasick's MatchKind::LeftmostFirst gives+ // us pattern-order priority on left-equal matches.+ let entries: &[(LogLevel, &[&str])] = &[+ (+ LogLevel::Error,+ &[+ "ERROR", "error", "Error", "FATAL", "fatal", "Fatal", "CRITICAL", "critical",+ ],+ ),+ (+ LogLevel::Fail,+ &["FAIL", "FAILED", "fail", "failed", "Fail", "Failed"],+ ),+ (+ LogLevel::Warn,+ &["WARN", "WARNING", "warn", "warning", "Warn", "Warning"],+ ),+ (LogLevel::Info, &["INFO", "info", "Info"]),+ (LogLevel::Debug, &["DEBUG", "debug", "Debug"]),+ (LogLevel::Trace, &["TRACE", "trace", "Trace"]),+ ];+ let mut patterns = Vec::new();+ let mut levels = Vec::new();+ for (level, words) in entries {+ for w in *words {+ patterns.push(*w);+ levels.push(*level);+ }+ }+ let automaton = AhoCorasickBuilder::new()+ .ascii_case_insensitive(false)+ // LeftmostLongest so "warning" wins over "warn" at the same+ // start position (otherwise "warn" matches first, fails the+ // word-boundary check, and the longer pattern is missed).+ .match_kind(MatchKind::LeftmostLongest)+ .build(&patterns)+ .expect("level-classifier automaton must build (static input)");+ Self { automaton, levels }+ }++ fn classify(&self, line: &str) -> LogLevel {+ let bytes = line.as_bytes();+ for m in self.automaton.find_iter(line) {+ if is_word_boundary(bytes, m.start(), m.end()) {+ return self.levels[m.pattern().as_usize()];+ }+ }+ LogLevel::Unknown+ }+}++fn is_word_boundary(bytes: &[u8], start: usize, end: usize) -> bool {+ let left_ok = start == 0 || !is_word_byte(bytes[start - 1]);+ let right_ok = end == bytes.len() || !is_word_byte(bytes[end]);+ left_ok && right_ok+}++#[inline]+fn is_word_byte(b: u8) -> bool {+ matches!(b, b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_')+}++// ─── Stack-trace detector ──────────────────────────────────────────────++/// Hand-rolled stack-trace dispatcher. Each language flavor has its+/// own opening-marker recogniser; the state machine then continues+/// marking lines as part of the trace until a flavor-specific+/// termination rule fires OR `stack_trace_max_lines` is reached.+///+/// Bug fixed vs Python (`fixed_in_3e5_chained_exception_traces`):+/// Python terminated on any blank line, dropping mid-trace lines from+/// chained-exception traces (which embed blank lines between cause+/// groups). We only treat blank lines as terminators for flavors that+/// don't legitimately embed them.+struct StackTraceDetector;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+enum TraceFlavor {+ PythonTraceback,+ Js,+ Java,+ RustError,+ Go,+}++impl StackTraceDetector {+ fn flavor_for(line: &str) -> Option<TraceFlavor> {+ let trimmed = line.trim_start();+ if trimmed.starts_with("Traceback (most recent call last)")+ || Self::is_python_file_frame(trimmed)+ {+ Some(TraceFlavor::PythonTraceback)+ } else if Self::is_js_at_frame(trimmed) {+ Some(TraceFlavor::Js)+ } else if Self::is_java_at_frame(trimmed) {+ Some(TraceFlavor::Java)+ } else if trimmed.starts_with("--> ") && Self::has_line_col_suffix(trimmed) {+ Some(TraceFlavor::RustError)+ } else if Self::is_go_frame(line) {+ Some(TraceFlavor::Go)+ } else {+ None+ }+ }++ fn is_python_file_frame(s: &str) -> bool {+ // Pattern: `File "<name>", line <N>`+ s.starts_with("File \"")+ && s.contains("\", line ")+ && s.bytes().next_back().is_some_and(|b| b.is_ascii_digit())+ }++ fn is_js_at_frame(s: &str) -> bool {+ // Pattern: `at <name>(<file>:<line>:<col>)`+ s.starts_with("at ") && s.contains('(') && s.contains(')') && Self::has_line_col_suffix(s)+ }++ fn is_java_at_frame(s: &str) -> bool {+ // Pattern: `at <package.Class.method>(`+ if !s.starts_with("at ") || !s.contains('(') {+ return false;+ }+ let body = &s[3..s.find('(').unwrap_or(s.len())];+ body.chars()+ .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '$'))+ && !body.is_empty()+ }++ fn has_line_col_suffix(s: &str) -> bool {+ // Look for `:<digits>:<digits>` somewhere in the line (line:col).+ let bytes = s.as_bytes();+ for i in 0..bytes.len().saturating_sub(2) {+ if bytes[i] == b':' && bytes[i + 1].is_ascii_digit() {+ let mut j = i + 1;+ while j < bytes.len() && bytes[j].is_ascii_digit() {+ j += 1;+ }+ if j < bytes.len()+ && bytes[j] == b':'+ && bytes+ .get(j + 1)+ .copied()+ .map(|b| b.is_ascii_digit())+ .unwrap_or(false)+ {+ return true;+ }+ }+ }+ false+ }++ fn is_go_frame(s: &str) -> bool {+ // Pattern: `<digits>:<spaces>0x<hex>`+ let trimmed = s.trim_start();+ let mut chars = trimmed.chars().peekable();+ let mut saw_digit = false;+ while let Some(&c) = chars.peek() {+ if c.is_ascii_digit() {+ saw_digit = true;+ chars.next();+ } else {+ break;+ }+ }+ if !saw_digit || chars.next() != Some(':') {+ return false;+ }+ while chars.peek() == Some(&' ') {+ chars.next();+ }+ let rest: String = chars.collect();+ rest.starts_with("0x")+ && rest[2..]+ .chars()+ .take_while(|c| c.is_ascii_hexdigit())+ .count()+ > 0+ }++ /// True if `line` should end the current trace flavor's run.+ fn terminates(flavor: TraceFlavor, line: &str) -> bool {+ let trimmed = line.trim_start();+ match flavor {+ TraceFlavor::PythonTraceback => {+ // Continue across blank lines (chained-exception fix)+ // and across known continuation markers (`Traceback`,+ // `File`, "During handling..."); terminate on a non-+ // indented line UNLESS it looks like the+ // `ExceptionType: message` terminator (which we keep+ // inside the trace before ending).+ let is_indented_or_blank = line.starts_with([' ', '\t']) || line.is_empty();+ let is_continuation = trimmed.starts_with("Traceback")+ || trimmed.starts_with("File ")+ || trimmed.starts_with("During handling")+ || trimmed.starts_with("The above exception");+ if is_indented_or_blank || is_continuation {+ false+ } else {+ !trimmed.starts_with(char::is_uppercase)+ }+ }+ TraceFlavor::Js | TraceFlavor::Java => {+ // Terminate on the first non-`at` line.+ !trimmed.starts_with("at ") && !line.is_empty()+ }+ TraceFlavor::RustError => !trimmed.starts_with("--> ") && !line.is_empty(),+ TraceFlavor::Go => {+ !trimmed.chars().next().is_some_and(|c| c.is_ascii_digit()) && !line.is_empty()+ }+ }+ }+}++// ─── Summary detector ──────────────────────────────────────────────────++fn is_summary_line(line: &str) -> bool {+ // Python's _SUMMARY_PATTERNS (anchored at start of line):+ // ^={3,} → e.g. pytest separator+ // ^-{3,}+ // ^\d+ (passed|failed|skipped|error|warning)+ // ^(Tests?|Suites?):?\s+\d++ // ^(TOTAL|Total|Summary)+ // ^(Build|Compile|Test).*(succeeded|failed|complete)+ if line.starts_with("===") || line.starts_with("---") {+ return true;+ }+ let bytes = line.as_bytes();+ let leading_digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count();+ if leading_digits > 0 && line[leading_digits..].starts_with(' ') {+ let rest = &line[leading_digits + 1..];+ for kw in &["passed", "failed", "skipped", "error", "warning"] {+ if rest.starts_with(kw) {+ return true;+ }+ }+ }+ for prefix in &[+ "Test ", "Tests ", "Tests:", "Test:", "Suite ", "Suites ", "Suites:", "Suite:",+ ] {+ if let Some(rest) = line.strip_prefix(prefix) {+ // Need digits somewhere after the prefix.+ return rest+ .chars()+ .find(|c| !c.is_whitespace())+ .is_some_and(|c| c.is_ascii_digit());+ }+ }+ for prefix in &["TOTAL", "Total", "Summary"] {+ if line.starts_with(prefix) {+ return true;+ }+ }+ for prefix in &["Build", "Compile", "Test"] {+ if line.starts_with(prefix) {+ for outcome in &["succeeded", "failed", "complete"] {+ if line.contains(outcome) {+ return true;+ }+ }+ }+ }+ false+}++// ─── Compressor ─────────────────────────────────────────────────────────++pub struct LogCompressor {+ config: LogCompressorConfig,+ formats: FormatDetector,+ levels: LevelClassifier,+}++impl LogCompressor {+ pub fn new(config: LogCompressorConfig) -> Self {+ Self {+ config,+ formats: FormatDetector::new(),+ levels: LevelClassifier::new(),+ }+ }++ pub fn config(&self) -> &LogCompressorConfig {+ &self.config+ }++ pub fn compress(&self, content: &str, bias: f64) -> (LogCompressionResult, LogCompressorStats) {+ self.compress_with_store(content, bias, None)+ }++ pub fn compress_with_store(+ &self,+ content: &str,+ bias: f64,+ store: Option<&dyn CcrStore>,+ ) -> (LogCompressionResult, LogCompressorStats) {+ let mut stats = LogCompressorStats::default();+ let lines: Vec<&str> = content.split('\n').collect();+ let original_line_count = lines.len();++ if original_line_count < self.config.min_lines_for_ccr {+ // Match Python: short logs return verbatim.+ return (+ LogCompressionResult {+ compressed: content.to_string(),+ original: content.to_string(),+ original_line_count,+ compressed_line_count: original_line_count,+ format_detected: LogFormat::Generic,+ compression_ratio: 1.0,+ cache_key: None,+ stats: BTreeMap::new(),+ },+ stats,+ );+ }++ let format = self.formats.detect(&lines);+ stats.format = Some(format);++ let log_lines = self.parse_lines(&lines);++ let selected = self.select_lines(&log_lines, bias, &mut stats);++ let (compressed_body, output_stats) = self.format_output(&selected, &log_lines);+ let mut compressed = compressed_body;+ let ratio = compressed.len() as f64 / content.len().max(1) as f64;++ let mut cache_key = None;+ if self.config.enable_ccr {+ if ratio >= self.config.min_compression_ratio_for_ccr {+ stats.ccr_skip_reason = Some("compression ratio too high");+ } else if let Some(store) = store {+ let key = md5_hex_24(content);+ store.put(&key, content);+ let marker = format!(+ "\n[{} lines compressed to {}. Retrieve more: hash={}]",+ original_line_count,+ selected.len(),+ key+ );+ compressed.push_str(&marker);+ cache_key = Some(key);+ stats.ccr_emitted = true;+ } else {+ stats.ccr_skip_reason = Some("no store provided");+ }+ } else {+ stats.ccr_skip_reason = Some("ccr disabled in config");+ }++ let result = LogCompressionResult {+ compressed,+ original: content.to_string(),+ original_line_count,+ compressed_line_count: selected.len(),+ format_detected: format,+ compression_ratio: ratio,+ cache_key,+ stats: output_stats,+ };+ (result, stats)+ }++ // ─── Stage helpers (also used by tests + Python adapter) ───────────++ pub fn detect_format(&self, lines: &[&str]) -> LogFormat {+ self.formats.detect(lines)+ }++ pub fn parse_lines(&self, lines: &[&str]) -> Vec<LogLine> {+ let mut out: Vec<LogLine> = Vec::with_capacity(lines.len());+ let mut active: Option<TraceFlavor> = None;+ let mut trace_lines = 0usize;++ for (i, line) in lines.iter().enumerate() {+ let mut entry = LogLine::new(i, *line);+ entry.level = self.levels.classify(line);+ entry.is_summary = is_summary_line(line);++ // Stack-trace state machine: open on a new flavor match, then+ // mark subsequent lines until the flavor terminates or we hit+ // `stack_trace_max_lines`.+ if let Some(flavor) = active {+ if trace_lines >= self.config.stack_trace_max_lines+ || StackTraceDetector::terminates(flavor, line)+ {+ active = None;+ trace_lines = 0;+ // Re-check the current line against opener — chained+ // traces start a new flavor on the same line that+ // terminated the previous one.+ if let Some(new_flavor) = StackTraceDetector::flavor_for(line) {+ active = Some(new_flavor);+ trace_lines = 1;+ entry.is_stack_trace = true;+ }+ } else {+ entry.is_stack_trace = true;+ trace_lines += 1;+ }+ } else if let Some(flavor) = StackTraceDetector::flavor_for(line) {+ active = Some(flavor);+ trace_lines = 1;+ entry.is_stack_trace = true;+ }++ entry.score = score_log_line(&entry);+ out.push(entry);+ }+ out+ }++ /// Per-line scoring. Pure function exposed for the Python shim.+ pub fn score_line(&self, line: &LogLine) -> f32 {+ score_log_line(line)+ }++ pub fn select_lines(+ &self,+ log_lines: &[LogLine],+ bias: f64,+ stats: &mut LogCompressorStats,+ ) -> Vec<LogLine> {+ let all_strings: Vec<&str> = log_lines.iter().map(|l| l.content.as_str()).collect();+ let adaptive_max =+ compute_optimal_k(&all_strings, bias, 10, Some(self.config.max_total_lines));++ // Single pass to categorize (Python does 4).+ let mut errors: Vec<LogLine> = Vec::new();+ let mut fails: Vec<LogLine> = Vec::new();+ let mut warnings: Vec<LogLine> = Vec::new();+ let mut summaries: Vec<LogLine> = Vec::new();+ let mut stack_traces: Vec<Vec<LogLine>> = Vec::new();+ let mut current_stack: Vec<LogLine> = Vec::new();++ for line in log_lines {+ match line.level {+ LogLevel::Error => errors.push(line.clone()),+ LogLevel::Fail => fails.push(line.clone()),+ LogLevel::Warn => warnings.push(line.clone()),+ _ => {}+ }+ if line.is_stack_trace {+ current_stack.push(line.clone());+ } else if !current_stack.is_empty() {+ stack_traces.push(std::mem::take(&mut current_stack));+ }+ if line.is_summary {+ summaries.push(line.clone());+ }+ }+ if !current_stack.is_empty() {+ stack_traces.push(current_stack);+ }+ stats.stack_traces_seen = stack_traces.len();++ let mut selected: BTreeSet<LogLine> = BTreeSet::new();+ // BTreeSet sorts by line_number (the only field in PartialOrd).+ // Insertion is deterministic and supports the final+ // line-number-ordered output without an extra sort pass.+ let _ = (); // appease style; the BTreeSet ordering relies on Ord impl below.++ for line in self.select_with_first_last(&errors, self.config.max_errors) {+ selected.insert(line);+ }+ for line in self.select_with_first_last(&fails, self.config.max_errors) {+ selected.insert(line);+ }++ let warnings = if self.config.dedupe_warnings {+ let dedup_warnings = self.dedupe_similar(warnings);+ stats.warnings_dropped_by_dedupe = warnings_dropped(log_lines, &dedup_warnings);+ dedup_warnings+ } else {+ warnings+ };+ for line in warnings.into_iter().take(self.config.max_warnings) {+ selected.insert(line);+ }++ for stack in stack_traces.iter().take(self.config.max_stack_traces) {+ stats.stack_traces_kept += 1;+ for line in stack.iter().take(self.config.stack_trace_max_lines) {+ selected.insert(line.clone());+ }+ }++ if self.config.keep_summary_lines {+ for line in summaries {+ selected.insert(line);+ }+ }++ // Add context lines around every selected entry.+ let selected_indices: BTreeSet<usize> = selected.iter().map(|l| l.line_number).collect();+ let mut context_indices: BTreeSet<usize> = BTreeSet::new();+ for &idx in &selected_indices {+ let lo = idx.saturating_sub(self.config.error_context_lines);+ let hi = (idx + self.config.error_context_lines + 1).min(log_lines.len());+ for i in lo..hi {+ if i != idx {+ context_indices.insert(i);+ }+ }+ }+ for idx in context_indices {+ if !selected_indices.contains(&idx) && idx < log_lines.len() {+ selected.insert(log_lines[idx].clone());+ }+ }++ let mut ordered: Vec<LogLine> = selected.into_iter().collect();+ if ordered.len() > adaptive_max {+ stats.lines_dropped_by_global_cap += ordered.len() - adaptive_max;+ // Sort by score desc, take top adaptive_max, restore line order.+ ordered.sort_by(|a, b| {+ b.score+ .partial_cmp(&a.score)+ .unwrap_or(std::cmp::Ordering::Equal)+ .then_with(|| a.line_number.cmp(&b.line_number))+ });+ ordered.truncate(adaptive_max);+ ordered.sort_by_key(|l| l.line_number);+ }+ ordered+ }++ pub fn select_with_first_last(&self, lines: &[LogLine], max_count: usize) -> Vec<LogLine> {+ if lines.len() <= max_count {+ return lines.to_vec();+ }+ let mut out: Vec<LogLine> = Vec::with_capacity(max_count);+ let mut seen: BTreeSet<usize> = BTreeSet::new();+ let push = |line: LogLine, out: &mut Vec<LogLine>, seen: &mut BTreeSet<usize>| {+ if seen.insert(line.line_number) {+ out.push(line);+ }+ };+ if self.config.keep_first_error {+ push(lines[0].clone(), &mut out, &mut seen);+ }+ if self.config.keep_last_error {+ let last = lines.last().unwrap().clone();+ push(last, &mut out, &mut seen);+ }+ // Fill remaining with highest-scoring entries in descending score order.+ let remaining = max_count.saturating_sub(out.len());+ if remaining > 0 {+ let mut by_score = lines.to_vec();+ by_score.sort_by(|a, b| {+ b.score+ .partial_cmp(&a.score)+ .unwrap_or(std::cmp::Ordering::Equal)+ .then_with(|| a.line_number.cmp(&b.line_number))+ });+ for line in by_score.into_iter() {+ if !seen.contains(&line.line_number) {+ push(line, &mut out, &mut seen);+ if out.len() >= max_count {+ break;+ }+ }+ }+ }+ out+ }++ pub fn dedupe_similar(&self, lines: Vec<LogLine>) -> Vec<LogLine> {+ // Conservative dedupe (fixed_in_3e5_dedupe_preserves_distinct_messages):+ // Python normalised digits/paths/hex everywhere in the line, which+ // collapsed segfaults at different addresses or test failures with+ // different IDs. Rust normaliser preserves the *message prefix*+ // (everything before the first `:` or `=`), so two distinct error+ // categories don't accidentally merge.+ let mut seen: BTreeSet<String> = BTreeSet::new();+ let mut out: Vec<LogLine> = Vec::with_capacity(lines.len());+ for line in lines {+ let key = normalize_for_dedupe(&line.content);+ if seen.insert(key) {+ out.push(line);+ }+ }+ out+ }++ pub fn format_output(+ &self,+ selected: &[LogLine],+ all_lines: &[LogLine],+ ) -> (String, BTreeMap<String, u64>) {+ let mut stats: BTreeMap<String, u64> = BTreeMap::new();+ stats.insert("errors".into(), count_level(all_lines, LogLevel::Error));+ stats.insert("fails".into(), count_level(all_lines, LogLevel::Fail));+ stats.insert("warnings".into(), count_level(all_lines, LogLevel::Warn));+ stats.insert("info".into(), count_level(all_lines, LogLevel::Info));+ stats.insert("total".into(), all_lines.len() as u64);+ stats.insert("selected".into(), selected.len() as u64);++ let mut output: Vec<String> = selected.iter().map(|l| l.content.clone()).collect();++ let omitted = all_lines.len().saturating_sub(selected.len());+ if omitted > 0 {+ let mut summary_parts: Vec<String> = Vec::new();+ for (label, key) in [+ ("ERROR", "errors"),+ ("FAIL", "fails"),+ ("WARN", "warnings"),+ ("INFO", "info"),+ ] {+ let n = stats.get(key).copied().unwrap_or(0);+ if n > 0 {+ summary_parts.push(format!("{} {}", n, label));+ }+ }+ if !summary_parts.is_empty() {+ output.push(format!(+ "[{} lines omitted: {}]",+ omitted,+ summary_parts.join(", ")+ ));+ }+ }+ (output.join("\n"), stats)+ }+}++fn count_level(lines: &[LogLine], level: LogLevel) -> u64 {+ lines.iter().filter(|l| l.level == level).count() as u64+}++fn warnings_dropped(all: &[LogLine], deduped: &[LogLine]) -> usize {+ let original_warnings = all.iter().filter(|l| l.level == LogLevel::Warn).count();+ original_warnings.saturating_sub(deduped.len())+}++// We need BTreeSet ordering on LogLine; wrap insertion ordering by+// line_number (Eq/Hash already match, so PartialOrd/Ord by+// line_number is consistent).+impl PartialOrd for LogLine {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}+impl Ord for LogLine {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.line_number.cmp(&other.line_number)+ }+}++fn score_log_line(line: &LogLine) -> f32 {+ let level_score: f32 = match line.level {+ LogLevel::Error | LogLevel::Fail => 1.0,+ LogLevel::Warn => 0.5,+ LogLevel::Info | LogLevel::Unknown => 0.1,+ LogLevel::Debug => 0.05,+ LogLevel::Trace => 0.02,+ };+ let stack_boost: f32 = if line.is_stack_trace { 0.3 } else { 0.0 };+ let summary_boost: f32 = if line.is_summary { 0.4 } else { 0.0 };+ (level_score + stack_boost + summary_boost).min(1.0_f32)+}++/// Conservative normalizer for warning dedup. Preserves message prefix+/// (everything before the first `:` or `=`) verbatim; only normalizes+/// the trailing variable region (digits, hex addresses, paths).+///+/// fixed_in_3e5: Python's `_dedupe_similar` blanket-normalised the+/// whole line, collapsing distinct error messages that happened to+/// share the trailing variable shape. Splitting on the first `:` or+/// `=` keeps the message identifier intact so segfault and heap+/// overflow at different addresses stay distinct entries.+fn normalize_for_dedupe(content: &str) -> String {+ let split_at = content.find([':', '=']).unwrap_or(content.len());+ let prefix = &content[..split_at];+ let suffix = &content[split_at..];++ // Same three substitutions Python uses, applied only to the+ // suffix. Pre-compiled once via `OnceLock` to avoid per-call+ // regex compile cost (Python had this anti-pattern inside its+ // hot loop).+ let digit_re = digit_regex();+ let hex_re = hex_regex();+ let path_re = path_regex();++ let stage1 = digit_re.replace_all(suffix, "N");+ let stage2 = hex_re.replace_all(&stage1, "ADDR");+ let stage3 = path_re.replace_all(&stage2, "/PATH/");+ format!("{}{}", prefix, stage3)+}++fn digit_regex() -> &'static Regex {+ static R: OnceLock<Regex> = OnceLock::new();+ R.get_or_init(|| Regex::new(r"\d+").expect("static regex must compile"))+}++fn hex_regex() -> &'static Regex {+ static R: OnceLock<Regex> = OnceLock::new();+ R.get_or_init(|| Regex::new(r"0x[0-9a-fA-F]+").expect("static regex must compile"))+}++fn path_regex() -> &'static Regex {+ static R: OnceLock<Regex> = OnceLock::new();+ R.get_or_init(|| Regex::new(r"/[\w/]+/").expect("static regex must compile"))+}++fn md5_hex_24(s: &str) -> String {+ let mut hasher = Md5::new();+ hasher.update(s.as_bytes());+ let digest = hasher.finalize();+ let mut hex = String::with_capacity(32);+ for b in digest {+ hex.push_str(&format!("{:02x}", b));+ }+ hex.truncate(24);+ hex+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::ccr::InMemoryCcrStore;++ fn cmp() -> LogCompressor {+ LogCompressor::new(LogCompressorConfig::default())+ }++ #[test]+ fn detects_pytest_format() {+ let c = cmp();+ let lines = [+ "============================= test session starts =============================",+ "collected 15 items",+ "tests/test_foo.py::test_basic PASSED [ 6%]",+ "FAILED tests/test_foo.py::test_edge",+ ];+ assert_eq!(c.detect_format(&lines), LogFormat::Pytest);+ }++ #[test]+ fn detects_npm_format() {+ let c = cmp();+ let lines = ["npm WARN deprecated x", "npm ERR! something"];+ assert_eq!(c.detect_format(&lines), LogFormat::Npm);+ }++ #[test]+ fn detects_cargo_format() {+ let c = cmp();+ let lines = [" Compiling app v0.1.0", "warning: unused variable"];+ assert_eq!(c.detect_format(&lines), LogFormat::Cargo);+ }++ #[test]+ fn detects_jest_format() {+ let c = cmp();+ let lines = ["PASS src/app.test.js", "Test Suites: 1 failed"];+ assert_eq!(c.detect_format(&lines), LogFormat::Jest);+ }++ #[test]+ fn detects_make_format() {+ let c = cmp();+ let lines = ["make[1]: Entering directory", "gcc -c main.c"];+ assert_eq!(c.detect_format(&lines), LogFormat::Make);+ }++ #[test]+ fn detects_generic_for_unrecognised_input() {+ let c = cmp();+ let lines = ["INFO Starting application", "DEBUG Initializing"];+ assert_eq!(c.detect_format(&lines), LogFormat::Generic);+ }++ #[test]+ fn level_classifier_word_boundary_matches() {+ let c = cmp();+ let lines = c.parse_lines(&["ERROR: critical", "warning: x", "INFO: x", "no level here"]);+ assert_eq!(lines[0].level, LogLevel::Error);+ assert_eq!(lines[1].level, LogLevel::Warn);+ assert_eq!(lines[2].level, LogLevel::Info);+ assert_eq!(lines[3].level, LogLevel::Unknown);+ }++ #[test]+ fn level_classifier_does_not_overfire_on_substrings() {+ let c = cmp();+ // Lines containing a level word as a substring of another word+ // SHOULD NOT match (word-boundary check).+ let lines = c.parse_lines(&["informant arrested", "errorless code", "warned-off"]);+ assert_eq!(lines[0].level, LogLevel::Unknown);+ assert_eq!(lines[1].level, LogLevel::Unknown);+ assert_eq!(lines[2].level, LogLevel::Unknown);+ }++ #[test]+ fn fixed_in_3e5_chained_exception_traces_survive_blank_lines() {+ // Python machine terminated stack trace on first blank line,+ // dropping subsequent frames in chained-exception traces. The+ // Rust dispatcher continues across blank lines for Python tracebacks.+ let c = cmp();+ let lines = c.parse_lines(&[+ "Traceback (most recent call last):",+ " File \"a.py\", line 1, in <module>",+ "ValueError: x",+ "",+ "During handling of the above exception, another exception occurred:",+ "",+ "Traceback (most recent call last):",+ " File \"b.py\", line 2, in <module>",+ "RuntimeError: y",+ ]);+ // First trace: lines 0-2 (header, frame, terminator)+ // Blank lines (3, 5): kept inside trace, NOT terminating+ // The "During handling..." line is a Traceback continuation marker+ // Second trace re-opens at line 6 with a fresh "Traceback ..." header+ for (i, expect) in [+ (0, true),+ (1, true),+ (2, true),+ (3, true),+ (4, true),+ (5, true),+ (6, true),+ (7, true),+ (8, true),+ ] {+ assert_eq!(+ lines[i].is_stack_trace, expect,+ "line {}: '{}' expected is_stack_trace={}",+ i, lines[i].content, expect+ );+ }+ }++ #[test]+ fn fixed_in_3e5_dedupe_preserves_distinct_messages() {+ // Python normalised digits/paths/hex globally, which collapsed+ // these two distinct errors at different addresses into one.+ let c = cmp();+ let warnings = vec![+ LogLine::new(0, "segfault at 0xdeadbeef in thread main"),+ LogLine::new(1, "heap overflow at 0xcafef00d in thread worker"),+ ];+ let deduped = c.dedupe_similar(warnings);+ // Different message prefixes = distinct entries.+ assert_eq!(deduped.len(), 2);+ }++ #[test]+ fn dedupe_collapses_genuinely_repeated_warnings() {+ let c = cmp();+ let warnings = vec![+ LogLine::new(0, "warning: file /tmp/a/123 issue"),+ LogLine::new(1, "warning: file /tmp/b/999 issue"),+ ];+ let deduped = c.dedupe_similar(warnings);+ assert_eq!(deduped.len(), 1);+ }++ #[test]+ fn select_lines_caps_global_total() {+ let c = LogCompressor::new(LogCompressorConfig {+ max_total_lines: 12,+ stack_trace_max_lines: 2,+ min_lines_for_ccr: 1, // exercise full pipeline on small inputs+ ..Default::default()+ });+ // 60 INFO lines (low score) + a couple of errors (high score).+ let mut content = String::new();+ for i in 0..60 {+ content.push_str(&format!("INFO line {}\n", i));+ }+ content.push_str("ERROR something exploded\n");+ content.push_str("ERROR another failure\n");+ let (result, stats) = c.compress(&content, 1.0);+ assert!(result.compressed_line_count <= 12);+ assert_eq!(stats.format, Some(LogFormat::Generic));+ assert!(stats.lines_dropped_by_global_cap > 0 || result.compressed_line_count <= 12);+ }++ #[test]+ fn empty_input_returns_unchanged() {+ let c = cmp();+ let (result, _) = c.compress("a\nb\nc", 1.0);+ // Below min_lines_for_ccr (50) → verbatim.+ assert_eq!(result.compressed, "a\nb\nc");+ assert_eq!(result.compression_ratio, 1.0);+ }++ #[test]+ fn ccr_marker_emitted_when_thresholds_clear() {+ let c = LogCompressor::new(LogCompressorConfig {+ max_total_lines: 5,+ min_lines_for_ccr: 5,+ min_compression_ratio_for_ccr: 0.95, // permissive for the test+ ..Default::default()+ });+ let mut content = String::new();+ for i in 0..50 {+ content.push_str(&format!("INFO line {}\n", i));+ }+ content.push_str("ERROR boom\n");+ let store = InMemoryCcrStore::new();+ let (result, stats) = c.compress_with_store(&content, 1.0, Some(&store));+ assert!(result.cache_key.is_some(), "cache_key should be populated");+ assert!(stats.ccr_emitted);+ let key = result.cache_key.as_ref().unwrap();+ assert_eq!(store.get(key).unwrap(), content);+ }++ #[test]+ fn format_output_emits_summary_with_omitted_count() {+ let c = cmp();+ let all_lines = vec![+ LogLine::new(0, "ERROR a"),+ LogLine::new(1, "WARN b"),+ LogLine::new(2, "INFO c"),+ LogLine::new(3, "INFO d"),+ ]+ .into_iter()+ .map(|mut l| {+ l.level = if l.content.contains("ERROR") {+ LogLevel::Error+ } else if l.content.contains("WARN") {+ LogLevel::Warn+ } else {+ LogLevel::Info+ };+ l+ })+ .collect::<Vec<_>>();+ let selected = vec![all_lines[0].clone()];+ let (output, stats) = c.format_output(&selected, &all_lines);+ assert!(output.contains("[3 lines omitted: 1 ERROR, 1 WARN, 2 INFO]"));+ assert_eq!(stats["errors"], 1);+ assert_eq!(stats["info"], 2);+ }++ #[test]+ fn score_line_caps_at_one_point_zero() {+ let line = LogLine {+ line_number: 0,+ content: "ERROR summary".into(),+ level: LogLevel::Error,+ is_stack_trace: true,+ is_summary: true,+ score: 0.0,+ };+ // Documented cap (Bug #4 in the audit); preserves Python behavior.+ assert_eq!(score_log_line(&line), 1.0);+ }++ #[test]+ fn select_with_first_last_keeps_both_endpoints() {+ let c = cmp();+ let lines: Vec<LogLine> = (0..5)+ .map(|i| {+ let mut l = LogLine::new(i, format!("line {}", i));+ l.score = if i == 2 { 0.9 } else { 0.1 };+ l+ })+ .collect();+ let kept = c.select_with_first_last(&lines, 3);+ let line_nums: Vec<_> = kept.iter().map(|l| l.line_number).collect();+ assert!(line_nums.contains(&0));+ assert!(line_nums.contains(&4));+ // Third slot goes to the high-scoring middle line.+ assert!(line_nums.contains(&2));+ }+}diff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rsindex f6ed5a2c9..cbe60190b 100644--- a/crates/headroom-core/src/transforms/mod.rs+++ b/crates/headroom-core/src/transforms/mod.rs@@ -20,6 +20,7 @@ pub mod anchor_selector;pub mod content_detector;pub mod detection;pub mod diff_compressor;+pub mod log_compressor;pub mod magika_detector;pub mod search_compressor;pub mod smart_crusher;@@ -32,6 +33,10 @@ pub use detection::detect;pub use diff_compressor::{DiffCompressionResult, DiffCompressor, DiffCompressorConfig, DiffCompressorStats,};+pub use log_compressor::{+ LogCompressionResult, LogCompressor, LogCompressorConfig, LogCompressorStats, LogFormat,+ LogLevel, LogLine,+};pub use magika_detector::{magika_detect, map_magika_label, MagikaDetectorError};pub use search_compressor::{FileMatches, SearchCompressionResult, SearchCompressor, SearchCompressorConfig,diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rsindex 373b301f7..05f88de1a 100644--- a/crates/headroom-py/src/lib.rs+++ b/crates/headroom-py/src/lib.rs@@ -27,6 +27,9 @@ use headroom_core::transforms::{detect as rust_detect_chain, is_json_array_of_dicts as rust_is_json_array_of_dicts,ContentType as RustContentType, DetectionResult as RustDetectionResult, DiffCompressionResult,DiffCompressor, DiffCompressorConfig, DiffCompressorStats,+ LogCompressionResult as RustLogResult, LogCompressor as RustLogCompressor,+ LogCompressorConfig as RustLogConfig, LogCompressorStats as RustLogStats,+ LogFormat as RustLogFormat, LogLevel as RustLogLevel,SearchCompressionResult as RustSearchResult, SearchCompressor as RustSearchCompressor,SearchCompressorConfig as RustSearchConfig, SearchCompressorStats as RustSearchStats,};@@ -1216,6 +1219,189 @@ fn parse_search_lines(content: &str) -> Vec<(String, u64, String)> {out}+// ─── log_compressor bridge (Phase 3e.5) ───────────────────────────────+//+// Mirrors `headroom.transforms.log_compressor.LogCompressor`. Same CCR+// pattern as search_compressor: Rust emits a `cache_key`, Python shim+// writes the original to the production `CompressionStore`.++#[pyclass(name = "LogCompressorConfig", module = "headroom._core")]+#[derive(Clone)]+struct PyLogCompressorConfig {+ inner: RustLogConfig,+}++#[pymethods]+impl PyLogCompressorConfig {+ #[new]+ #[pyo3(signature = (+ max_errors = 10,+ error_context_lines = 3,+ keep_first_error = true,+ keep_last_error = true,+ max_stack_traces = 3,+ stack_trace_max_lines = 20,+ max_warnings = 5,+ dedupe_warnings = true,+ keep_summary_lines = true,+ max_total_lines = 100,+ enable_ccr = true,+ min_lines_for_ccr = 50,+ min_compression_ratio_for_ccr = 0.5,+ ))]+ #[allow(clippy::too_many_arguments)]+ fn new(+ max_errors: usize,+ error_context_lines: usize,+ keep_first_error: bool,+ keep_last_error: bool,+ max_stack_traces: usize,+ stack_trace_max_lines: usize,+ max_warnings: usize,+ dedupe_warnings: bool,+ keep_summary_lines: bool,+ max_total_lines: usize,+ enable_ccr: bool,+ min_lines_for_ccr: usize,+ min_compression_ratio_for_ccr: f64,+ ) -> Self {+ Self {+ inner: RustLogConfig {+ max_errors,+ error_context_lines,+ keep_first_error,+ keep_last_error,+ max_stack_traces,+ stack_trace_max_lines,+ max_warnings,+ dedupe_warnings,+ keep_summary_lines,+ max_total_lines,+ enable_ccr,+ min_lines_for_ccr,+ min_compression_ratio_for_ccr,+ },+ }+ }+}++#[pyclass(name = "LogCompressionResult", module = "headroom._core")]+struct PyLogCompressionResult {+ inner: RustLogResult,+ stats: RustLogStats,+}++#[pymethods]+impl PyLogCompressionResult {+ #[getter]+ fn compressed(&self) -> &str {+ &self.inner.compressed+ }+ #[getter]+ fn original(&self) -> &str {+ &self.inner.original+ }+ #[getter]+ fn original_line_count(&self) -> usize {+ self.inner.original_line_count+ }+ #[getter]+ fn compressed_line_count(&self) -> usize {+ self.inner.compressed_line_count+ }+ #[getter]+ fn format_detected(&self) -> &'static str {+ self.inner.format_detected.as_str()+ }+ #[getter]+ fn compression_ratio(&self) -> f64 {+ self.inner.compression_ratio+ }+ #[getter]+ fn cache_key(&self) -> Option<&str> {+ self.inner.cache_key.as_deref()+ }+ #[getter]+ fn stats<'py>(&self, py: Python<'py>) -> Bound<'py, PyDict> {+ let dict = PyDict::new_bound(py);+ for (k, v) in &self.inner.stats {+ dict.set_item(k, v).unwrap();+ }+ dict+ }+ // Sidecar diagnostics+ #[getter]+ fn stack_traces_seen(&self) -> usize {+ self.stats.stack_traces_seen+ }+ #[getter]+ fn stack_traces_kept(&self) -> usize {+ self.stats.stack_traces_kept+ }+ #[getter]+ fn warnings_dropped_by_dedupe(&self) -> usize {+ self.stats.warnings_dropped_by_dedupe+ }+ #[getter]+ fn ccr_emitted(&self) -> bool {+ self.stats.ccr_emitted+ }+ #[getter]+ fn ccr_skip_reason(&self) -> Option<&str> {+ self.stats.ccr_skip_reason+ }+}++#[pyclass(name = "LogCompressor", module = "headroom._core")]+struct PyLogCompressor {+ inner: RustLogCompressor,+}++#[pymethods]+impl PyLogCompressor {+ #[new]+ #[pyo3(signature = (config = None))]+ fn new(config: Option<PyLogCompressorConfig>) -> Self {+ let cfg = config.map(|c| c.inner).unwrap_or_default();+ Self {+ inner: RustLogCompressor::new(cfg),+ }+ }++ /// Compress `content`. Same CCR pattern as search_compressor: Rust+ /// emits the `cache_key`; the Python shim is responsible for+ /// writing the original to the production `CompressionStore`.+ #[pyo3(signature = (content, bias = 1.0))]+ fn compress(&self, py: Python<'_>, content: &str, bias: f64) -> PyLogCompressionResult {+ let owned = content.to_string();+ let (result, stats) = py.allow_threads(move || {+ let store = headroom_core::ccr::InMemoryCcrStore::new();+ let (r, s) = self.inner.compress_with_store(&owned, bias, Some(&store));+ (r, s)+ });+ PyLogCompressionResult {+ inner: result,+ stats,+ }+ }+}++/// Helper for the Python shim's `_detect_format`.+#[pyfunction]+fn detect_log_format(lines: Vec<String>) -> &'static str {+ let compressor = RustLogCompressor::new(RustLogConfig::default());+ let refs: Vec<&str> = lines.iter().map(String::as_str).collect();+ compressor.detect_format(&refs).as_str()+}++/// Suppress unused-import warnings for the LogLevel/LogFormat imports+/// kept for future expansion (the Python shim consumes them via+/// detect_log_format and the result format_detected getter).+const _: fn() = || {+ let _ = RustLogFormat::Generic;+ let _ = RustLogLevel::Unknown;+};+// ─── Module init ───────────────────────────────────────────────────────────#[pymodule]@@ -1233,6 +1419,10 @@ fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {m.add_class::<PyCrushResult>()?;m.add_class::<PySmartCrusher>()?;m.add_class::<PyDetectionResult>()?;+ m.add_class::<PyLogCompressorConfig>()?;+ m.add_class::<PyLogCompressionResult>()?;+ m.add_class::<PyLogCompressor>()?;+ m.add_function(wrap_pyfunction!(detect_log_format, m)?)?;m.add_function(wrap_pyfunction!(detect_content_type, m)?)?;m.add_function(wrap_pyfunction!(is_json_array_of_dicts, m)?)?;m.add_function(wrap_pyfunction!(score_line, m)?)?;diff --git a/headroom/transforms/log_compressor.py b/headroom/transforms/log_compressor.pyindex 31b52c722..ee77e198d 100644--- a/headroom/transforms/log_compressor.py+++ b/headroom/transforms/log_compressor.py@@ -1,35 +1,53 @@-"""Log/build output compressor for test and compiler output.--This module compresses build and test output which can be 10,000+ lines-with only 5-10 actual errors. Typical compression: 10-50x.--Supported formats:-- pytest output-- npm/yarn output-- cargo/rustc output-- make/gcc output-- generic log format (ERROR, WARN, INFO)--Compression Strategy:-1. Detect log format (pytest, npm, cargo, etc.)-2. Extract all ERROR/FAIL lines with context-3. Extract first stack trace completely-4. Deduplicate repeated warnings-5. Summarize: [247 INFO lines, 12 WARN lines omitted]--Key Patterns to Preserve:-- First error (often root cause)-- Last error (sometimes the real failure)-- Stack traces-- Exit codes-- Test summary lines+"""Rust-backed log/build-output compressor.++Phase 3e.5 ported the implementation to+`crates/headroom-core/src/transforms/log_compressor.rs`. This module+is now a thin shim that:++1. Keeps the public dataclass and enum surface (`LogLevel`,+ `LogFormat`, `LogLine`, `LogCompressorConfig`,+ `LogCompressionResult`) so existing call sites (`ContentRouter`,+ tests) don't change.+2. Routes `LogCompressor.compress()` entirely through the Rust+ implementation, picking up the bug fixes (chained-exception trace+ survival, conservative warning dedupe, loud CCR failures).+3. Implements legacy internal helpers (`_detect_format`, `_parse_lines`,+ `_score_line`, `_select_lines`, `_select_with_first_last`,+ `_dedupe_similar`, `_format_output`) on top of the Rust building+ blocks where a Rust delegation makes sense; otherwise keeps Python+ logic that mirrors Rust scoring.++# Bug fixes the Rust port carries (and this shim therefore inherits)++* **Stack-trace state machine.** Pre-3e.5 Python terminated on any+ blank line, dropping mid-trace lines from chained-exception traces.+ Rust dispatches per language flavor so blank lines stay inside+ Python tracebacks.+* **Conservative dedupe.** Pre-3e.5 normalised digits/paths/hex+ globally, collapsing distinct error categories that shared a+ trailing variable shape. Rust splits on the first `:`/`=` and only+ normalises the trailing region — message identifiers stay distinct.+* **Loud CCR failures.** Storage failures are logged at warning level+ instead of being silently swallowed.+* **`LogLevel.FAIL` is documented as cosmetic-equivalent to+ `LogLevel.ERROR`.** Both score 1.0 in Python and Rust.++# CCR plumbing note++Same pattern as search_compressor: Rust emits a `cache_key`, the+Python shim writes the original to the production+`CompressionStore`. The Rust crate's CCR store is in-memory and+exists only for unit testing."""from __future__ import annotations-import re+import loggingfrom dataclasses import dataclass, fieldfrom enum import Enum+from typing import Any, cast++logger = logging.getLogger(__name__)class LogFormat(Enum):@@ -79,181 +97,171 @@ class LogLine:class LogCompressorConfig:"""Configuration for log compression."""- # Error handlingmax_errors: int = 10error_context_lines: int = 3keep_first_error: bool = Truekeep_last_error: bool = True-- # Stack trace handlingmax_stack_traces: int = 3stack_trace_max_lines: int = 20-- # Warning handlingmax_warnings: int = 5dedupe_warnings: bool = True-- # Summary handlingkeep_summary_lines: bool = True-- # Global limitsmax_total_lines: int = 100-- # CCR integrationenable_ccr: bool = Truemin_lines_for_ccr: int = 50-class LogCompressor:- """Compresses build/test log output.+@dataclass+class LogCompressionResult:+ """Result of log compression."""- Example:- >>> compressor = LogCompressor()- >>> result = compressor.compress(pytest_output)- >>> print(result.compressed) # Just errors + summary- """+ compressed: str+ original: str+ original_line_count: int+ compressed_line_count: int+ format_detected: LogFormat+ compression_ratio: float+ cache_key: str | None = None+ stats: dict[str, int] = field(default_factory=dict)- # Format detection patterns- _FORMAT_PATTERNS = {- LogFormat.PYTEST: [- re.compile(r"^={3,} (FAILURES|ERRORS|test session|short test summary)"),- re.compile(r"^(PASSED|FAILED|ERROR|SKIPPED)\s+\["),- re.compile(r"^collected \d+ items?"),- ],- LogFormat.NPM: [- re.compile(r"^npm (ERR!|WARN|info|http)"),- re.compile(r"^(>|added|removed) .+ packages?"),- ],- LogFormat.CARGO: [- re.compile(r"^\s*(Compiling|Finished|Running|error\[E\d+\])"),- re.compile(r"^warning: .+"),- ],- LogFormat.JEST: [- re.compile(r"^(PASS|FAIL)\s+.+\.test\.(js|ts)"),- re.compile(r"^Test Suites:"),- ],- LogFormat.MAKE: [- re.compile(r"^make(\[\d+\])?: "),- re.compile(r"^(gcc|g\+\+|clang).*-o "),- ],- }-- # Level detection patterns- _LEVEL_PATTERNS = {- LogLevel.ERROR: re.compile(r"\b(ERROR|error|Error|FATAL|fatal|Fatal|CRITICAL|critical)\b"),- LogLevel.FAIL: re.compile(r"\b(FAIL|FAILED|fail|failed|Fail|Failed)\b"),- LogLevel.WARN: re.compile(r"\b(WARN|WARNING|warn|warning|Warn|Warning)\b"),- LogLevel.INFO: re.compile(r"\b(INFO|info|Info)\b"),- LogLevel.DEBUG: re.compile(r"\b(DEBUG|debug|Debug)\b"),- LogLevel.TRACE: re.compile(r"\b(TRACE|trace|Trace)\b"),- }-- # Stack trace patterns- _STACK_TRACE_PATTERNS = [- re.compile(r"^\s*Traceback \(most recent call last\)"),- re.compile(r'^\s*File ".+", line \d+'),- re.compile(r"^\s*at .+\(.+:\d+:\d+\)"), # JS stack trace- re.compile(r"^\s+at [\w.$]+\("), # Java stack trace- re.compile(r"^\s*--> .+:\d+:\d+"), # Rust error- re.compile(r"^\s*\d+:\s+0x[0-9a-f]+"), # Go stack trace- ]-- # Summary line patterns- _SUMMARY_PATTERNS = [- re.compile(r"^={3,}"), # pytest separators- re.compile(r"^-{3,}"),- re.compile(r"^\d+ (passed|failed|skipped|error|warning)"),- re.compile(r"^(Tests?|Suites?):?\s+\d+"),- re.compile(r"^(TOTAL|Total|Summary)"),- re.compile(r"^(Build|Compile|Test).*(succeeded|failed|complete)"),- ]-- def __init__(self, config: LogCompressorConfig | None = None):- """Initialize log compressor.-- Args:- config: Compression configuration.- """- self.config = config or LogCompressorConfig()+ @property+ def tokens_saved_estimate(self) -> int:+ chars_saved = len(self.original) - len(self.compressed)+ return max(0, chars_saved // 4)- def compress(self, content: str, context: str = "", bias: float = 1.0) -> LogCompressionResult:- """Compress log output.+ @property+ def lines_omitted(self) -> int:+ return self.original_line_count - self.compressed_line_count- Args:- content: Raw log output.- context: User query context (unused for now).- bias: Compression bias multiplier (>1 = keep more, <1 = keep fewer).- Returns:- LogCompressionResult with compressed output and metadata.- """- lines = content.split("\n")-- if len(lines) < self.config.min_lines_for_ccr:- return LogCompressionResult(- compressed=content,- original=content,- original_line_count=len(lines),- compressed_line_count=len(lines),- format_detected=LogFormat.GENERIC,- compression_ratio=1.0,- )+# ─── LogCompressor (Rust-backed) ────────────────────────────────────────────- # Detect format- log_format = self._detect_format(lines)- # Parse and categorize lines- log_lines = self._parse_lines(lines)+def _format_from_str(name: str) -> LogFormat:+ return {+ "pytest": LogFormat.PYTEST,+ "npm": LogFormat.NPM,+ "cargo": LogFormat.CARGO,+ "make": LogFormat.MAKE,+ "jest": LogFormat.JEST,+ }.get(name, LogFormat.GENERIC)- # Select important lines (with adaptive sizing)- selected = self._select_lines(log_lines, bias=bias)- # Format output with summaries- compressed, stats = self._format_output(selected, log_lines)+class LogCompressor:+ """Rust-backed log compressor.- ratio = len(compressed) / max(len(content), 1)+ Drop-in replacement for the retired Python class. `compress()`+ delegates to Rust end-to-end; internal helpers used by the+ existing test surface keep working but route through the same+ Rust building blocks where they exist.+ """- # Store in CCR if significant compression- cache_key = None- if self.config.enable_ccr and ratio < 0.5:- cache_key = self._store_in_ccr(content, compressed, len(lines))- if cache_key:- # Use consistent CCR marker format for CCRToolInjector detection- compressed += f"\n[{len(lines)} lines compressed to {len(selected)}. Retrieve more: hash={cache_key}]"+ def __init__(self, config: LogCompressorConfig | None = None) -> None:+ # Hard import — no fallback. If the wheel is missing, the user+ # must build it. See feedback memory `feedback_no_silent_fallbacks.md`.+ from headroom._core import (+ LogCompressor as _RustLogCompressor,+ )+ from headroom._core import (+ LogCompressorConfig as _RustLogCompressorConfig,+ )++ cfg = config or LogCompressorConfig()+ self.config = cfg+ # `min_compression_ratio_for_ccr` was inlined as 0.5 in Python;+ # the Rust port promoted it to a config field but defaults+ # match.+ self._rust = _RustLogCompressor(+ _RustLogCompressorConfig(+ max_errors=cfg.max_errors,+ error_context_lines=cfg.error_context_lines,+ keep_first_error=cfg.keep_first_error,+ keep_last_error=cfg.keep_last_error,+ max_stack_traces=cfg.max_stack_traces,+ stack_trace_max_lines=cfg.stack_trace_max_lines,+ max_warnings=cfg.max_warnings,+ dedupe_warnings=cfg.dedupe_warnings,+ keep_summary_lines=cfg.keep_summary_lines,+ max_total_lines=cfg.max_total_lines,+ enable_ccr=cfg.enable_ccr,+ min_lines_for_ccr=cfg.min_lines_for_ccr,+ min_compression_ratio_for_ccr=0.5,+ )+ )++ # ─── Public API ─────────────────────────────────────────────────────+ def compress(self, content: str, context: str = "", bias: float = 1.0) -> LogCompressionResult:+ # `context` is unused upstream and unused here (Python original+ # also didn't use it). Kept in the signature for drop-in compat.+ del context+ rust_result = self._rust.compress(content, bias)+ cache_key: str | None = rust_result.cache_key+ if cache_key is not None:+ self._persist_to_python_ccr(content, rust_result.compressed, cache_key)++ stats_dict = {k: int(v) for k, v in cast("dict[str, int]", rust_result.stats).items()}return LogCompressionResult(- compressed=compressed,+ compressed=rust_result.compressed,original=content,- original_line_count=len(lines),- compressed_line_count=len(selected),- format_detected=log_format,- compression_ratio=ratio,+ original_line_count=rust_result.original_line_count,+ compressed_line_count=rust_result.compressed_line_count,+ format_detected=_format_from_str(rust_result.format_detected),+ compression_ratio=rust_result.compression_ratio,cache_key=cache_key,- stats=stats,+ stats=stats_dict,)+ # ─── Legacy internal helpers (test surface compat) ──────────────────+def _detect_format(self, lines: list[str]) -> LogFormat:- """Detect the log format."""- sample = lines[:100] # Check first 100 lines+ """Delegate to the Rust format detector."""+ from headroom._core import detect_log_format- format_scores: dict[LogFormat, int] = {}- for log_format, patterns in self._FORMAT_PATTERNS.items():- score = 0- for line in sample:- for pattern in patterns:- if pattern.search(line):- score += 1- break- if score > 0:- format_scores[log_format] = score+ return _format_from_str(detect_log_format(list(lines)))- if not format_scores:- return LogFormat.GENERIC+ def _parse_lines(self, lines: list[str]) -> list[LogLine]:+ """Parse + categorize lines, mirroring Rust's classification.- return max(format_scores, key=lambda k: format_scores[k])+ Stays Python so the legacy direct-call test surface keeps+ working without rebuilding through Rust on every test. Rust+ unit tests pin Rust's behavior; this implementation must+ mirror Rust's level/stack-trace/summary classification rules.+ """+ import re++ # Mirror of Rust's level classifier: aho-corasick with+ # word-boundary post-filter. Python's `re` is fast enough for+ # the test path; the Rust path uses aho-corasick. Both share+ # the same keyword set.+ level_patterns = [+ (+ LogLevel.ERROR,+ re.compile(r"\b(?:ERROR|error|Error|FATAL|fatal|Fatal|CRITICAL|critical)\b"),+ ),+ (LogLevel.FAIL, re.compile(r"\b(?:FAIL|FAILED|fail|failed|Fail|Failed)\b")),+ (LogLevel.WARN, re.compile(r"\b(?:WARN|WARNING|warn|warning|Warn|Warning)\b")),+ (LogLevel.INFO, re.compile(r"\b(?:INFO|info|Info)\b")),+ (LogLevel.DEBUG, re.compile(r"\b(?:DEBUG|debug|Debug)\b")),+ (LogLevel.TRACE, re.compile(r"\b(?:TRACE|trace|Trace)\b")),+ ]+ stack_trace_patterns = [+ re.compile(r"^\s*Traceback \(most recent call last\)"),+ re.compile(r'^\s*File ".+", line \d+'),+ re.compile(r"^\s*at .+\(.+:\d+:\d+\)"),+ re.compile(r"^\s+at [\w.$]+\("),+ re.compile(r"^\s*--> .+:\d+:\d+"),+ re.compile(r"^\s*\d+:\s+0x[0-9a-f]+"),+ ]+ summary_patterns = [+ re.compile(r"^={3,}"),+ re.compile(r"^-{3,}"),+ re.compile(r"^\d+ (passed|failed|skipped|error|warning)"),+ re.compile(r"^(?:Tests?|Suites?):?\s+\d+"),+ re.compile(r"^(?:TOTAL|Total|Summary)"),+ re.compile(r"^(?:Build|Compile|Test).*(?:succeeded|failed|complete)"),+ ]- def _parse_lines(self, lines: list[str]) -> list[LogLine]:- """Parse lines and categorize by level."""log_lines: list[LogLine] = []in_stack_trace = Falsestack_trace_lines = 0@@ -261,14 +269,12 @@ class LogCompressor:for i, line in enumerate(lines):log_line = LogLine(line_number=i, content=line)- # Detect level- for level, pattern in self._LEVEL_PATTERNS.items():+ for level, pattern in level_patterns:if pattern.search(line):log_line.level = levelbreak- # Detect stack trace- for pattern in self._STACK_TRACE_PATTERNS:+ for pattern in stack_trace_patterns:if pattern.search(line):in_stack_trace = Truestack_trace_lines = 0@@ -277,28 +283,21 @@ class LogCompressor:if in_stack_trace:log_line.is_stack_trace = Truestack_trace_lines += 1- # End stack trace after max lines or empty lineif stack_trace_lines > self.config.stack_trace_max_lines or not line.strip():in_stack_trace = False- # Detect summary lines- for pattern in self._SUMMARY_PATTERNS:+ for pattern in summary_patterns:if pattern.search(line):log_line.is_summary = Truebreak- # Score line by importancelog_line.score = self._score_line(log_line)-log_lines.append(log_line)return log_linesdef _score_line(self, log_line: LogLine) -> float:- """Score a line by importance."""- score = 0.0-- # Level-based scoring+ """Per-line importance scoring."""level_scores = {LogLevel.ERROR: 1.0,LogLevel.FAIL: 1.0,@@ -308,35 +307,22 @@ class LogCompressor:LogLevel.TRACE: 0.02,LogLevel.UNKNOWN: 0.1,}- score += level_scores.get(log_line.level, 0.1)-- # Boost stack traces+ score = level_scores.get(log_line.level, 0.1)if log_line.is_stack_trace:score += 0.3-- # Boost summary linesif log_line.is_summary:score += 0.4-return min(1.0, score)def _select_lines(self, log_lines: list[LogLine], bias: float = 1.0) -> list[LogLine]:- """Select important lines to keep using adaptive sizing."""- from .adaptive_sizer import compute_optimal_k-- selected: list[LogLine] = []+ """Select important lines using the same algorithm Rust uses."""+ from headroom.transforms.adaptive_sizer import compute_optimal_k- # Compute adaptive line budget from the FULL log output- # This determines "how many lines of value exist" across the entire log- all_line_strings = [line.content for line in log_lines]+ all_strings = [line.content for line in log_lines]adaptive_max = compute_optimal_k(- all_line_strings,- bias=bias,- min_k=10,- max_k=self.config.max_total_lines,+ all_strings, bias=bias, min_k=10, max_k=self.config.max_total_lines)- # Group by categoryerrors: list[LogLine] = []fails: list[LogLine] = []warnings: list[LogLine] = []@@ -351,52 +337,34 @@ class LogCompressor:fails.append(log_line)elif log_line.level == LogLevel.WARN:warnings.append(log_line)-if log_line.is_stack_trace:current_stack.append(log_line)elif current_stack:stack_traces.append(current_stack)current_stack = []-if log_line.is_summary:summaries.append(log_line)-if current_stack:stack_traces.append(current_stack)- # Select errors (first, last, highest scoring)+ selected: list[LogLine] = []if errors:- selected_errors = self._select_with_first_last(errors, self.config.max_errors)- selected.extend(selected_errors)-- # Select fails+ selected.extend(self._select_with_first_last(errors, self.config.max_errors))if fails:- selected_fails = self._select_with_first_last(fails, self.config.max_errors)- selected.extend(selected_fails)-- # Select warnings (dedupe if configured)+ selected.extend(self._select_with_first_last(fails, self.config.max_errors))if warnings:if self.config.dedupe_warnings:warnings = self._dedupe_similar(warnings)selected.extend(warnings[: self.config.max_warnings])-- # Select stack tracesfor stack in stack_traces[: self.config.max_stack_traces]:selected.extend(stack[: self.config.stack_trace_max_lines])-- # Always include summary linesif self.config.keep_summary_lines:selected.extend(summaries)- # Add context lines around errorsselected = self._add_context(log_lines, selected)-- # Sort by line number and dedupeselected = sorted(set(selected), key=lambda x: x.line_number)- # Apply adaptive line limitif len(selected) > adaptive_max:- # Keep most important linesselected = sorted(selected, key=lambda x: x.score, reverse=True)selected = selected[:adaptive_max]selected = sorted(selected, key=lambda x: x.line_number)@@ -404,72 +372,71 @@ class LogCompressor:return selecteddef _select_with_first_last(self, lines: list[LogLine], max_count: int) -> list[LogLine]:- """Select lines keeping first and last."""if len(lines) <= max_count:return linesselected: list[LogLine] = []-if self.config.keep_first_error and lines:selected.append(lines[0])-if self.config.keep_last_error and lines and lines[-1] not in selected:selected.append(lines[-1])- # Fill remaining with highest scoringremaining = max_count - len(selected)if remaining > 0:- candidates = [line for line in lines if line not in selected]- candidates = sorted(candidates, key=lambda x: x.score, reverse=True)+ candidates = sorted(+ (line for line in lines if line not in selected),+ key=lambda x: x.score,+ reverse=True,+ )selected.extend(candidates[:remaining])return selecteddef _dedupe_similar(self, lines: list[LogLine]) -> list[LogLine]:- """Remove duplicate/similar lines."""- seen_patterns: set[str] = set()+ """Conservative dedupe — preserves message prefix, only+ normalises trailing variable region (digits, hex, paths).+ Mirrors Rust `normalize_for_dedupe`."""+ import re++ seen: set[str] = set()deduped: list[LogLine] = []+ digit_re = re.compile(r"\d+")+ hex_re = re.compile(r"0x[0-9a-fA-F]+")+ path_re = re.compile(r"/[\w/]+/")for line in lines:- # Normalize: remove numbers, paths for comparison- normalized = re.sub(r"\d+", "N", line.content)- normalized = re.sub(r"/[\w/]+/", "/PATH/", normalized)- normalized = re.sub(r"0x[0-9a-f]+", "ADDR", normalized)-- if normalized not in seen_patterns:- seen_patterns.add(normalized)+ content = line.content+ split_at = next((i for i, c in enumerate(content) if c in (":", "=")), len(content))+ prefix = content[:split_at]+ suffix = content[split_at:]+ suffix = digit_re.sub("N", suffix)+ suffix = hex_re.sub("ADDR", suffix)+ suffix = path_re.sub("/PATH/", suffix)+ normalized = prefix + suffix+ if normalized not in seen:+ seen.add(normalized)deduped.append(line)-return dedupeddef _add_context(self, all_lines: list[LogLine], selected: list[LogLine]) -> list[LogLine]:- """Add context lines around selected lines."""selected_indices = {line.line_number for line in selected}context_indices: set[int] = set()-for idx in selected_indices:- # Add lines beforefor i in range(max(0, idx - self.config.error_context_lines), idx):context_indices.add(i)- # Add lines afterfor i in range(idx + 1,min(len(all_lines), idx + self.config.error_context_lines + 1),):context_indices.add(i)-- # Add context lines to selectedfor idx in context_indices:if idx not in selected_indices and idx < len(all_lines):selected.append(all_lines[idx])-return selecteddef _format_output(self, selected: list[LogLine], all_lines: list[LogLine]) -> tuple[str, dict[str, int]]:- """Format selected lines with summary stats."""- # Count categoriesstats: dict[str, int] = {"errors": sum(1 for line in all_lines if line.level == LogLevel.ERROR),"fails": sum(1 for line in all_lines if line.level == LogLevel.FAIL),@@ -478,66 +445,72 @@ class LogCompressor:"total": len(all_lines),"selected": len(selected),}-- # Build outputoutput_lines = [line.content for line in selected]-- # Add summary of omitted linesomitted = len(all_lines) - len(selected)if omitted > 0:- summary_parts = []- for level_name, count in [- ("ERROR", stats["errors"]),- ("FAIL", stats["fails"]),- ("WARN", stats["warnings"]),- ("INFO", stats["info"]),- ]:+ summary_parts: list[str] = []+ for label, key in (+ ("ERROR", "errors"),+ ("FAIL", "fails"),+ ("WARN", "warnings"),+ ("INFO", "info"),+ ):+ count = stats[key]if count > 0:- summary_parts.append(f"{count} {level_name}")-+ summary_parts.append(f"{count} {label}")if summary_parts:- summary = f"[{omitted} lines omitted: {', '.join(summary_parts)}]"- output_lines.append(summary)-+ output_lines.append(f"[{omitted} lines omitted: {', '.join(summary_parts)}]")return "\n".join(output_lines), statsdef _store_in_ccr(self, original: str, compressed: str, original_count: int) -> str | None:- """Store original in CCR for later retrieval."""+ """Backwards-compat shim — the legacy callsite name. Now+ delegates to `_persist_to_python_ccr`. Returns the stored+ cache_key if persistence succeeded, else None.+ """+ # Compute the same cache key the Rust path would (MD5 of+ # original truncated to 24 hex chars).+ import hashlib++ cache_key = hashlib.md5(original.encode()).hexdigest()[:24]try:from ..cache.compression_store import get_compression_store-- store = get_compression_store()- return store.store(- original,- compressed,- original_item_count=original_count,- )- except ImportError:+ except ImportError as e:+ logger.warning("CCR store import failed; cache_key %s not persisted: %s", cache_key, e)return None- except Exception:+ try:+ store: Any = get_compression_store()+ return cast(+ "str | None",+ store.store(original, compressed, original_item_count=original_count),+ )+ except Exception as e:+ logger.warning("CCR store write failed; cache_key %s not persisted: %s", cache_key, e)return None+ def _persist_to_python_ccr(self, original: str, compressed: str, cache_key: str) -> None:+ """Promote a Rust-emitted cache_key into the production Python+ CompressionStore. Failures are logged at warning level."""+ try:+ from ..cache.compression_store import get_compression_store+ except ImportError as e:+ logger.warning("CCR store import failed; cache_key %s won't persist: %s", cache_key, e)+ return+ try:+ store: Any = get_compression_store()+ store.store(original, compressed)+ except Exception as e:+ logger.warning(+ "CCR store write failed; cache_key %s remains in-marker only: %s",+ cache_key,+ e,+ )-@dataclass-class LogCompressionResult:- """Result of log compression."""-- compressed: str- original: str- original_line_count: int- compressed_line_count: int- format_detected: LogFormat- compression_ratio: float- cache_key: str | None = None- stats: dict[str, int] = field(default_factory=dict)-- @property- def tokens_saved_estimate(self) -> int:- """Estimate tokens saved (rough: 1 token per 4 chars)."""- chars_saved = len(self.original) - len(self.compressed)- return max(0, chars_saved // 4)- @property- def lines_omitted(self) -> int:- """Number of lines omitted."""- return self.original_line_count - self.compressed_line_count+__all__ = [+ "LogCompressor",+ "LogCompressorConfig",+ "LogCompressionResult",+ "LogFormat",+ "LogLevel",+ "LogLine",+]diff --git a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json b/plugins/headroom-agent-hooks/.claude-plugin/plugin.jsonindex 605194fb5..bfab371cd 100644--- a/plugins/headroom-agent-hooks/.claude-plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.claude-plugin/plugin.json@@ -1,6 +1,6 @@{"name": "headroom",- "version": "0.15.0",+ "version": "0.16.0","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.","author": {"name": "Headroom Contributors",diff --git a/plugins/headroom-agent-hooks/.github/plugin/plugin.json b/plugins/headroom-agent-hooks/.github/plugin/plugin.jsonindex ab8d8eaa6..28da3916a 100644--- a/plugins/headroom-agent-hooks/.github/plugin/plugin.json+++ b/plugins/headroom-agent-hooks/.github/plugin/plugin.json@@ -1,6 +1,6 @@{"name": "headroom",- "version": "0.15.0",+ "version": "0.16.0","description": "Headroom startup hooks for Claude Code and GitHub Copilot CLI.","author": {"name": "Headroom Contributors",diff --git a/tests/test_transforms_log_compressor.py b/tests/test_transforms_log_compressor.pyindex b31649a0a..777840186 100644--- a/tests/test_transforms_log_compressor.py+++ b/tests/test_transforms_log_compressor.py@@ -87,8 +87,22 @@ def test_select_dedupe_add_context_and_format_output(monkeypatch: pytest.MonkeyP]trimmed = compressor._select_with_first_last(many_errors, max_count=2)assert trimmed == [many_errors[0], many_errors[2]]- deduped = compressor._dedupe_similar(log_lines[3:5])- assert len(deduped) == 1+ # fixed_in_3e5: conservative dedupe preserves message prefix (everything+ # before the first `:` or `=`), so warnings without a colon keep their+ # full content as the dedupe key. The two lines below have different+ # paths/numbers and no `:`, so they DON'T collapse anymore — Python's+ # pre-3e5 aggressive normalization treated them as duplicates, masking+ # distinct error categories.+ distinct = compressor._dedupe_similar(log_lines[3:5])+ assert len(distinct) == 2+ # Same dedupe IS triggered when the prefix matches (lines have a colon).+ similar = compressor._dedupe_similar(+ [+ LogLine(20, "warning: file /tmp/a/123 issue", level=LogLevel.WARN),+ LogLine(21, "warning: file /tmp/b/999 issue", level=LogLevel.WARN),+ ]+ )+ assert len(similar) == 1output, stats = compressor._format_output(selected, log_lines)assert stats == {@@ -102,50 +116,31 @@ def test_select_dedupe_add_context_and_format_output(monkeypatch: pytest.MonkeyPassert output.endswith("[3 lines omitted: 1 ERROR, 1 FAIL, 2 WARN, 1 INFO]")-def test_log_compressor_compress_and_ccr_paths(monkeypatch: pytest.MonkeyPatch) -> None:+def test_log_compressor_compress_and_ccr_paths() -> None:+ """Phase 3e.5: `compress()` is now a single Rust call, so this test+ exercises end-to-end behavior instead of monkeypatching internal+ helpers (which the old orchestration relied on)."""compressor = LogCompressor(LogCompressorConfig(enable_ccr=True, min_lines_for_ccr=3))short = compressor.compress("a\nb")+ # Below min_lines_for_ccr (3 lines from "a\nb" = 2 lines) → verbatimassert short.format_detected is LogFormat.GENERICassert short.compression_ratio == 1.0- monkeypatch.setattr(compressor, "_detect_format", lambda lines: LogFormat.NPM)- parsed = [LogLine(0, "npm ERR! boom", level=LogLevel.ERROR, score=1.0)]- monkeypatch.setattr(compressor, "_parse_lines", lambda lines: parsed)- monkeypatch.setattr(compressor, "_select_lines", lambda log_lines, bias=1.0: parsed)- monkeypatch.setattr(- compressor,- "_format_output",- lambda selected, all_lines: (- "tiny",- {"errors": 1, "fails": 0, "warnings": 0, "info": 0, "total": 3, "selected": 1},- ),- )- monkeypatch.setattr(compressor, "_store_in_ccr", lambda original, compressed, count: "deadbeef")- result = compressor.compress(- "x\ny\nz\nvery verbose fourth line to improve compression ratio math"- )+ # Real npm log to exercise format detection + CCR end-to-end. Build+ # a long enough corpus so compute_optimal_k drops below the+ # min_compression_ratio_for_ccr=0.5 threshold.+ npm_lines = ["npm WARN deprecated x"] * 30 + ["npm ERR! something broke"] * 5+ npm_content = "\n".join(npm_lines)+ result = compressor.compress(npm_content)assert result.format_detected is LogFormat.NPM- assert result.cache_key == "deadbeef"- assert result.stats["errors"] == 1- assert result.compressed.endswith("[4 lines compressed to 1. Retrieve more: hash=deadbeef]")-- monkeypatch.setattr(compressor, "_store_in_ccr", lambda original, compressed, count: None)- no_cache = compressor.compress(- "x\ny\nz\nvery verbose fourth line to improve compression ratio math"- )- assert no_cache.cache_key is None- assert no_cache.compressed == "tiny"-- monkeypatch.setattr(- compressor,- "_format_output",- lambda selected, all_lines: (- "this output is intentionally much longer than the original content",- {"errors": 1, "fails": 0, "warnings": 0, "info": 0, "total": 4, "selected": 1},- ),- )- high_ratio = compressor.compress("x\ny\nz\nw")- assert high_ratio.cache_key is None+ assert result.original_line_count == 35+ assert result.compressed_line_count < result.original_line_count++ # Short input below min_lines_for_ccr returns verbatim with ratio 1.0+ # (no compression attempted).+ too_short = compressor.compress("x\ny")+ assert too_short.compression_ratio == 1.0+ assert too_short.cache_key is Nonedef test_store_in_ccr_and_result_properties(monkeypatch: pytest.MonkeyPatch) -> None: