Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(pipeline): store the pipeline input in CCR behind offload markers (#4576)
Parideboy4d7765c84b06
diff --git a/crates/headroom-core/src/transforms/diff_compressor.rs b/crates/headroom-core/src/transforms/diff_compressor.rsindex 510240ce0..5a200404b 100644--- a/crates/headroom-core/src/transforms/diff_compressor.rs+++ b/crates/headroom-core/src/transforms/diff_compressor.rs@@ -277,6 +277,20 @@ impl DiffCompressor {content: &str,context: &str,store: Option<&dyn CcrStore>,+ ) -> (DiffCompressionResult, DiffCompressorStats) {+ self.compress_with_ccr_source(content, context, store, content)+ }++ /// Like [`Self::compress_with_store`], but the CCR entry and its+ /// hash come from `ccr_source` rather than `content`. Use when+ /// `content` is a derived form of the original (e.g. after a+ /// reformat) and retrieval must return the original.+ pub fn compress_with_ccr_source(+ &self,+ content: &str,+ context: &str,+ store: Option<&dyn CcrStore>,+ ccr_source: &str,) -> (DiffCompressionResult, DiffCompressorStats) {let start = Instant::now();let mut stats = DiffCompressorStats::default();@@ -473,7 +487,7 @@ impl DiffCompressor {if self.config.enable_ccr&& (compressed_line_count as f64) < (original_line_count as f64) * savings_threshold{- let key = md5_hex_24(content);+ let key = md5_hex_24(ccr_source);compressed_output.push('\n');compressed_output.push_str(&format!("[{} lines compressed to {}. Retrieve full diff: hash={}]",@@ -485,7 +499,7 @@ impl DiffCompressor {// (typically the Python shim) is responsible — see the// method-level docs.if let Some(s) = store {- s.put(&key, content);+ s.put(&key, ccr_source);}cache_key = Some(key);stats.cache_key_emitted = true;diff --git a/crates/headroom-core/src/transforms/log_compressor.rs b/crates/headroom-core/src/transforms/log_compressor.rsindex fc7ff605a..338d4804b 100644--- a/crates/headroom-core/src/transforms/log_compressor.rs+++ b/crates/headroom-core/src/transforms/log_compressor.rs@@ -1139,6 +1139,20 @@ impl LogCompressor {content: &str,bias: f64,store: Option<&dyn CcrStore>,+ ) -> (LogCompressionResult, LogCompressorStats) {+ self.compress_with_ccr_source(content, bias, store, content)+ }++ /// Like [`Self::compress_with_store`], but the CCR entry and its+ /// hash come from `ccr_source` rather than `content`. Use when+ /// `content` is a derived form of the original (e.g. after a+ /// reformat) and retrieval must return the original.+ pub fn compress_with_ccr_source(+ &self,+ content: &str,+ bias: f64,+ store: Option<&dyn CcrStore>,+ ccr_source: &str,) -> (LogCompressionResult, LogCompressorStats) {let mut stats = LogCompressorStats::default();let lines: Vec<&str> = content.split('\n').collect();@@ -1177,8 +1191,8 @@ impl LogCompressor {if ratio >= self.config.min_compression_ratio_for_ccr {stats.ccr_skip_reason = Some("compression ratio too high");} else if let Some(store) = store {- let key = md5_hex_24(content);- store.put(&key, content);+ let key = md5_hex_24(ccr_source);+ store.put(&key, ccr_source);// Descriptive so the reader/agent can tell whether the// dropped content is relevant before deciding whether to// retrieve it (rather than just "how much" vanished).diff --git a/crates/headroom-core/src/transforms/pipeline/mod.rs b/crates/headroom-core/src/transforms/pipeline/mod.rsindex c6bde6073..38883a907 100644--- a/crates/headroom-core/src/transforms/pipeline/mod.rs+++ b/crates/headroom-core/src/transforms/pipeline/mod.rs@@ -52,8 +52,10 @@//! Reformats://! - [`reformats::JsonMinifier`] — JSON whitespace stripping.//! - [`reformats::LogTemplate`] — Drain-style template miner for-//! build/log output. Lossless — emits `[Template Tn: ...] (Nx)` +-//! variant table, every original line reconstructible.+//! build/log output. Tokens preserved, whitespace normalized —+//! emits `[Template Tn: ...] (Nx)` + variant table; the exact+//! original is recoverable only through CCR when an offload runs+//! after the reformat.//!//! Offloads://! - [`offloads::JsonOffload`] — wraps `SmartCrusher` for JSON arraysdiff --git a/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs b/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rsindex b3ccad1d5..d4441e19e 100644--- a/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs+++ b/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs@@ -117,6 +117,7 @@ impl OffloadTransform for DiffNoise {content: &str,_ctx: &CompressionContext,store: &dyn CcrStore,+ ccr_source: &str,) -> Result<OffloadOutput, TransformError> {let segments = parse_segments(content);if segments.is_empty() {@@ -173,8 +174,8 @@ impl OffloadTransform for DiffNoise {}// CCR: hash original, stash, append marker.- let key = md5_hex_24(content);- store.put(&key, content);+ let key = md5_hex_24(ccr_source);+ store.put(&key, ccr_source);output.push_str(&format!("\n[diff_noise CCR: hash={key}]"));Ok(OffloadOutput::from_lengths(content.len(), output, key))@@ -432,7 +433,7 @@ mod tests {]);let store = InMemoryCcrStore::new();let r = offload()- .apply(&diff, &CompressionContext::default(), &store)+ .apply(&diff, &CompressionContext::default(), &store, &diff).expect("must compress");assert!(r.bytes_saved > 0);assert!(r.output.contains("[diff_noise: lockfile hunks dropped"));@@ -451,7 +452,7 @@ mod tests {let diff = build_diff(&[("src/main.rs", &body_refs)]);let store = InMemoryCcrStore::new();let r = offload()- .apply(&diff, &CompressionContext::default(), &store)+ .apply(&diff, &CompressionContext::default(), &store, &diff).expect("must compress");assert!(r.output@@ -468,7 +469,7 @@ mod tests {let diff = build_diff(&[("src/main.rs", &body_refs)]);let store = InMemoryCcrStore::new();let err = offload()- .apply(&diff, &CompressionContext::default(), &store)+ .apply(&diff, &CompressionContext::default(), &store, &diff).expect_err("nothing droppable");match err {TransformError::Skipped { .. } => {}@@ -502,7 +503,7 @@ mod tests {diff.push_str(&build_diff(&[("yarn.lock", &lock_refs)]));let store = InMemoryCcrStore::new();let r = offload()- .apply(&diff, &CompressionContext::default(), &store)+ .apply(&diff, &CompressionContext::default(), &store, &diff).expect("compresses");// Pre-diff prelude must survive.assert!(r.output.contains("Subject: bump deps"));@@ -514,7 +515,7 @@ mod tests {assert_eq!(offload().estimate_bloat(""), 0.0);let store = InMemoryCcrStore::new();let err = offload()- .apply("", &CompressionContext::default(), &store)+ .apply("", &CompressionContext::default(), &store, "").expect_err("must skip");match err {TransformError::Skipped { .. } => {}diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rsindex d5a5df487..d1eb872a4 100644--- a/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs+++ b/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs@@ -132,15 +132,14 @@ impl OffloadTransform for DiffOffload {content: &str,ctx: &CompressionContext,store: &dyn CcrStore,+ ccr_source: &str,) -> Result<OffloadOutput, TransformError> {- // `compress_with_store` (added in the audit-cleanup PR) writes- // the original to `store` under the same `cache_key` it embeds- // in the marker. Earlier versions of this offload had to- // double-store post-hoc because the compressor lacked a store- // parameter — that hack is gone.- let (result, _) = self- .compressor- .compress_with_store(content, &ctx.query, Some(store));+ // The compressor writes `ccr_source` to `store` under the same+ // `cache_key` it embeds in the marker, so retrieval returns the+ // pipeline input rather than the reformatted `content`.+ let (result, _) =+ self.compressor+ .compress_with_ccr_source(content, &ctx.query, Some(store), ccr_source);let Some(key) = result.cache_key else {return Err(TransformError::skipped(@@ -254,7 +253,7 @@ mod tests {let diff = build_diff(1, 200, 5);let store = InMemoryCcrStore::new();let r = offload()- .apply(&diff, &CompressionContext::default(), &store)+ .apply(&diff, &CompressionContext::default(), &store, &diff).expect("should compress");assert!(!r.cache_key.is_empty());// Bug-fix verification: DiffCompressor itself wouldn't have@@ -268,7 +267,7 @@ mod tests {let diff = build_diff(1, 5, 2);let store = InMemoryCcrStore::new();let err = offload()- .apply(&diff, &CompressionContext::default(), &store)+ .apply(&diff, &CompressionContext::default(), &store, &diff).expect_err("must skip");match err {TransformError::Skipped { transform, .. } => assert_eq!(transform, "diff_offload"),diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rsindex 54bb62ded..99a1d22d3 100644--- a/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs+++ b/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs@@ -147,12 +147,15 @@ impl OffloadTransform for JsonOffload {content: &str,ctx: &CompressionContext,store: &dyn CcrStore,+ ccr_source: &str,) -> Result<OffloadOutput, TransformError> {let prose = &self.prose;let prose_hook = |leaf: &str, query: &str| {let leaf_ctx = CompressionContext::with_query(query);+ // Each leaf is its own CCR source: its marker resolves to the+ // leaf text the crusher saw.prose- .apply(leaf, &leaf_ctx, store)+ .apply(leaf, &leaf_ctx, store, leaf).ok().map(|output| (output.output, output.cache_key))};@@ -173,9 +176,10 @@ impl OffloadTransform for JsonOffload {// through the orchestrator-supplied store, append a marker so// the LLM sees a resolvable handle. SmartCrusher's per-array// markers in `result.compressed` stay informational — the LLM- // retrieves the full original via this outer hash.- let key = md5_hex_24(content);- store.put(&key, content);+ // retrieves the full original via this outer hash. The hash and+ // stored payload come from `ccr_source` (the pipeline input).+ let key = md5_hex_24(ccr_source);+ store.put(&key, ccr_source);let mut output = result.compressed;output.push_str("\n[json_offload CCR: hash=");output.push_str(&key);@@ -298,7 +302,7 @@ mod tests {let input = serde_json::json!([{"description": diff}]).to_string();let store = InMemoryCcrStore::new();let output = offload()- .apply(&input, &CompressionContext::default(), &store)+ .apply(&input, &CompressionContext::default(), &store, &input).expect("diff-shaped leaf should still process through the direct offload path");let marker = output.output@@ -318,7 +322,12 @@ mod tests {let input = serde_json::json!([{"summary": html}]).to_string();let store = InMemoryCcrStore::new();let result = offload()- .apply(&input, &CompressionContext::with_query("recovery"), &store)+ .apply(+ &input,+ &CompressionContext::with_query("recovery"),+ &store,+ &input,+ ).expect("structured html should still process");assert!(result.output.contains(",html,"));}@@ -424,7 +433,7 @@ mod tests {let arr = build_tabular_array(500);let store = InMemoryCcrStore::new();let r = offload()- .apply(&arr, &CompressionContext::default(), &store)+ .apply(&arr, &CompressionContext::default(), &store, &arr).expect("smart crusher should compress");assert!(r.bytes_saved > 0);assert!(!r.cache_key.is_empty());@@ -441,7 +450,7 @@ mod tests {let arr = build_tabular_array(2);let store = InMemoryCcrStore::new();let err = offload()- .apply(&arr, &CompressionContext::default(), &store)+ .apply(&arr, &CompressionContext::default(), &store, &arr).expect_err("must skip");match err {TransformError::Skipped { transform, .. } => assert_eq!(transform, "json_offload"),@@ -454,7 +463,12 @@ mod tests {fn apply_skipped_for_non_json_input() {let store = InMemoryCcrStore::new();let err = offload()- .apply("not json at all", &CompressionContext::default(), &store)+ .apply(+ "not json at all",+ &CompressionContext::default(),+ &store,+ "not json at all",+ ).expect_err("must skip non-json");match err {TransformError::Skipped { .. } => {}@@ -487,7 +501,7 @@ mod tests {let store = InMemoryCcrStore::new();let ctx = CompressionContext::with_query("needle");let r = offload()- .apply(&s, &ctx, &store)+ .apply(&s, &ctx, &store, &s).expect("crusher should run");// Output should reference "needle" — SmartCrusher's anchor logic// should keep that row.@@ -503,10 +517,10 @@ mod tests {let store_a = InMemoryCcrStore::new();let store_b = InMemoryCcrStore::new();let r_a = offload()- .apply(&arr, &CompressionContext::default(), &store_a)+ .apply(&arr, &CompressionContext::default(), &store_a, &arr).expect("ok");let r_b = offload()- .apply(&arr, &CompressionContext::default(), &store_b)+ .apply(&arr, &CompressionContext::default(), &store_b, &arr).expect("ok");assert_eq!(r_a.cache_key, r_b.cache_key,diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rsindex e23bd6a78..96fcdc3b9 100644--- a/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs+++ b/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs@@ -145,10 +145,11 @@ impl OffloadTransform for LogOffload {content: &str,_ctx: &CompressionContext,store: &dyn CcrStore,+ ccr_source: &str,) -> Result<OffloadOutput, TransformError> {- let (result, stats) = self- .compressor- .compress_with_store(content, self.bias, Some(store));+ let (result, stats) =+ self.compressor+ .compress_with_ccr_source(content, self.bias, Some(store), ccr_source);// The trait contract says `cache_key` is required. If the// underlying compressor decided post-hoc that the offload@@ -258,7 +259,7 @@ mod tests {let log: String = line.repeat(200);let store = InMemoryCcrStore::new();let r = offload()- .apply(&log, &CompressionContext::default(), &store)+ .apply(&log, &CompressionContext::default(), &store, &log).expect("offload should produce a key on bloaty input");assert!(!r.cache_key.is_empty());assert_eq!(store.get(&r.cache_key).as_deref(), Some(log.as_str()));@@ -273,7 +274,7 @@ mod tests {let log = "INFO: a\nINFO: b\nINFO: c\nINFO: d\nINFO: e";let store = InMemoryCcrStore::new();let err = offload()- .apply(log, &CompressionContext::default(), &store)+ .apply(log, &CompressionContext::default(), &store, log).expect_err("must skip");match err {TransformError::Skipped { transform, .. } => assert_eq!(transform, "log_offload"),diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/prose_field.rs b/crates/headroom-core/src/transforms/pipeline/offloads/prose_field.rsindex 6494f8cd1..db88c78d3 100644--- a/crates/headroom-core/src/transforms/pipeline/offloads/prose_field.rs+++ b/crates/headroom-core/src/transforms/pipeline/offloads/prose_field.rs@@ -34,7 +34,9 @@ impl ProseFieldOffload {&& detect_content_type(content).content_type == ContentType::PlainText}- fn compress(&self, content: &str, query: &str) -> Option<(String, String)> {+ /// Returns `(output, key)`. The key hashes `ccr_source`, the payload+ /// the caller will store; `content` is only the text compressed.+ fn compress(&self, content: &str, query: &str, ccr_source: &str) -> Option<(String, String)> {if !self.eligible(content) {return None;}@@ -45,7 +47,7 @@ impl ProseFieldOffload {return None;}- let key = compute_key(content.as_bytes());+ let key = compute_key(ccr_source.as_bytes());let output = format!("{}\n{}", result.compressed, marker_for(&key));(output.len() < content.len()).then_some((output, key))}@@ -80,14 +82,15 @@ impl OffloadTransform for ProseFieldOffload {content: &str,ctx: &CompressionContext,store: &dyn CcrStore,+ ccr_source: &str,) -> Result<OffloadOutput, TransformError> {- let Some((output, key)) = self.compress(content, &ctx.query) else {+ let Some((output, key)) = self.compress(content, &ctx.query, ccr_source) else {return Err(TransformError::skipped(NAME,"prose compression not worth it",));};- store.put(&key, content);+ store.put(&key, ccr_source);Ok(OffloadOutput::from_lengths(content.len(), output, key))}@@ -123,7 +126,7 @@ mod tests {fn short_plain_text_is_byte_identical() {let input = "A short note.";let store = InMemoryCcrStore::new();- let result = offload().apply(input, &CompressionContext::default(), &store);+ let result = offload().apply(input, &CompressionContext::default(), &store, input);assert!(result.is_err());assert!(store.is_empty());}@@ -132,7 +135,7 @@ mod tests {fn long_low_segment_text_is_byte_identical() {let input = "A".repeat(300);let store = InMemoryCcrStore::new();- let result = offload().apply(&input, &CompressionContext::default(), &store);+ let result = offload().apply(&input, &CompressionContext::default(), &store, &input);assert!(result.is_err());assert!(store.is_empty());}@@ -159,8 +162,8 @@ mod tests {fn query_changes_selection_deterministically() {let input = prose();let crusher = offload();- let a = crusher.compress(&input, "recovery").unwrap();- let b = crusher.compress(&input, "recovery").unwrap();+ let a = crusher.compress(&input, "recovery", &input).unwrap();+ let b = crusher.compress(&input, "recovery", &input).unwrap();assert_eq!(a, b);assert!(a.0.contains("recovery"));assert!(!adiff --git a/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rsindex 5600f7fa2..df74537b4 100644--- a/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs+++ b/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs@@ -116,10 +116,15 @@ impl OffloadTransform for SearchOffload {content: &str,ctx: &CompressionContext,store: &dyn CcrStore,+ ccr_source: &str,) -> Result<OffloadOutput, TransformError> {- let (result, stats) =- self.compressor- .compress_with_store(content, &ctx.query, self.bias, Some(store));+ let (result, stats) = self.compressor.compress_with_ccr_source(+ content,+ &ctx.query,+ self.bias,+ Some(store),+ ccr_source,+ );let Some(key) = result.cache_key else {let reason = stats.ccr_skip_reason.unwrap_or("no cache_key emitted");@@ -309,7 +314,7 @@ mod tests {.join("\n");let store = InMemoryCcrStore::new();let r = offload()- .apply(&s, &CompressionContext::default(), &store)+ .apply(&s, &CompressionContext::default(), &store, &s).expect("offload should produce a key on clustered input");assert!(!r.cache_key.is_empty());assert_eq!(store.get(&r.cache_key).as_deref(), Some(s.as_str()));@@ -321,7 +326,7 @@ mod tests {let s = "only.py:1:trivial";let store = InMemoryCcrStore::new();let err = offload()- .apply(s, &CompressionContext::default(), &store)+ .apply(s, &CompressionContext::default(), &store, s).expect_err("must skip");match err {TransformError::Skipped { transform, .. } => assert_eq!(transform, "search_offload"),diff --git a/crates/headroom-core/src/transforms/pipeline/orchestrator.rs b/crates/headroom-core/src/transforms/pipeline/orchestrator.rsindex 66592e09f..c9a196457 100644--- a/crates/headroom-core/src/transforms/pipeline/orchestrator.rs+++ b/crates/headroom-core/src/transforms/pipeline/orchestrator.rs@@ -153,7 +153,9 @@ impl CompressionPipeline {);continue;}- match offload.apply(¤t, ctx, store) {+ // CCR stores the pipeline input (`content`), not the+ // reformatted `current`, so retrieval returns the original.+ match offload.apply(¤t, ctx, store, content) {Ok(out) => {if out.bytes_saved == 0 {tracing::trace!(@@ -422,11 +424,12 @@ mod tests {content: &str,_ctx: &CompressionContext,store: &dyn CcrStore,+ ccr_source: &str,) -> Result<OffloadOutput, TransformError> {// Always halve; emit a cache_key derived from name.let half = &content[..content.len() / 2];let key = format!("test_{}_key", self.name);- store.put(&key, content);+ store.put(&key, ccr_source);Ok(OffloadOutput::from_lengths(content.len(),half.to_string(),@@ -644,6 +647,7 @@ mod tests {_content: &str,_ctx: &CompressionContext,_store: &dyn CcrStore,+ _ccr_source: &str,) -> Result<OffloadOutput, TransformError> {Err(TransformError::internal("always_internal_err", "by design"))}@@ -699,7 +703,7 @@ mod tests {#[test]fn end_to_end_log_template_collapses_then_log_offload_can_run() {- // LogTemplate runs first (lossless reformat), then LogOffload+ // LogTemplate runs first (token-preserving reformat), then LogOffload// sees the collapsed output and may further drop low-priority// lines. Both should appear in steps_applied if both fire.let cfg = PipelineConfig::default();@@ -733,6 +737,42 @@ mod tests {assert!(r.output.len() < log.len());}+ #[test]+ fn ccr_stores_pipeline_input_not_reformatted_text() {+ // LogTemplate rewrites the log before LogOffload runs. The CCR+ // entry behind LogOffload's marker must still be the original+ // pipeline input, so retrieval gives back the untouched log.+ let cfg = PipelineConfig::default();+ let p = CompressionPipeline::builder()+ .with_reformat(LogTemplate::new(cfg.reformat.log_template))+ .with_offload(LogOffload::new(cfg.bloat.log))+ .with_config(cfg)+ .build();+ let s = store();+ let mut input = String::new();+ for i in 0..300 {+ input.push_str(&format!(+ "2025-01-15T12:{:02}:{:02} INFO processing job {i}\n",+ i / 60 % 60,+ i % 60+ ));+ }+ for i in 0..300 {+ input.push_str(&format!(+ "2025-01-15T13:{:02}:{:02} DEBUG heartbeat ok\n",+ i / 60 % 60,+ i % 60+ ));+ }+ input.push_str("2025-01-15T15:00:00 ERROR database connection lost\n");++ let r = p.run(&input, ContentType::BuildOutput, &ctx(), &s);+ assert!(r.steps_applied.iter().any(|n| n == "log_template"));+ assert!(r.steps_applied.iter().any(|n| n == "log_offload"));+ let key = r.cache_keys.first().expect("log_offload emits a cache key");+ assert_eq!(s.get(key).as_deref(), Some(input.as_str()));+ }+#[test]fn end_to_end_diff_noise_drops_lockfile_then_diff_offload_handles_rest() {let cfg = PipelineConfig::default();diff --git a/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs b/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rsindex 9e668abd9..0c2a11311 100644--- a/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs+++ b/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs@@ -23,9 +23,17 @@//! ...//! ```//!-//! Every original line is reconstructible from `template + variants`,-//! so this is **lossless** — no CCR retrieval needed. The win is the-//! template prefix (often 30+ chars) emitted once instead of N times.+//! Every line is reconstructible from `template + variants` at token+//! level: the tokens are kept, but whitespace is normalized. Runs of+//! spaces, tabs, indentation, and `\r` are not preserved, so the+//! output is not a byte-exact copy of the input. The exact original is+//! recoverable only through CCR, when an offload runs after this+//! reformat. The win is the template prefix (often 30+ chars) emitted+//! once instead of N times.+//!+//! Constant tokens that are `<*>` behind zero or more backslashes are+//! escaped with one more leading backslash (`<*>` → `\<*>`), so a+//! header position reads as a wildcard only when it is exactly `<*>`.//!//! Order is preserved: only *consecutive* runs collapse, so the//! temporal flow of the log stays intact for the LLM.@@ -244,7 +252,7 @@ impl LogTemplate {out.push(' ');}match slot {- Some(constant) => out.push_str(constant),+ Some(constant) => push_constant(out, constant),None => out.push_str(WILDCARD),}}@@ -289,6 +297,17 @@ impl Run {}}+/// Write a constant template token. A literal `<*>` would print the+/// same as the wildcard, and a literal `\<*>` (any number of leading+/// backslashes) would read as an escape, so each gets one extra+/// leading backslash.+fn push_constant(out: &mut String, token: &str) {+ if token.trim_start_matches('\\') == WILDCARD {+ out.push('\\');+ }+ out.push_str(token);+}+/// Whitespace-split tokenizer. Empty result = blank line.////// Uses `str::split_whitespace` semantics: collapses runs of whitespace,@@ -357,7 +376,7 @@ mod tests {r.output.chars().take(200).collect::<String>());assert!(r.output.contains("(50 occurrences)"));- // Variants should still be in output (lossless guarantee).+ // Variants should still be in output (token-level guarantee).assert!(r.output.contains("worker-7"));}@@ -384,9 +403,10 @@ mod tests {}#[test]- fn lossless_round_trip_via_template_and_variants() {- // The flushed output must be reconstructible: each template- // line + its variant rows reproduces the original input lines.+ fn token_round_trip_via_template_and_variants() {+ // The flushed output must be reconstructible at token level:+ // each template line + its variant rows reproduces the original+ // tokens. Whitespace is normalized, so compare token lists.let mut log = String::new();for i in 0..25 {log.push_str(&format!("TOK1 TOK2 var{i} TOK3\n"));@@ -420,7 +440,14 @@ mod tests {full[var_pos] = var_tokens[0];reconstructed.push(full.join(" "));}- let original: Vec<String> = log.lines().map(|s| s.to_string()).collect();+ let original: Vec<Vec<String>> = log+ .lines()+ .map(|s| s.split_whitespace().map(str::to_string).collect())+ .collect();+ let reconstructed: Vec<Vec<String>> = reconstructed+ .iter()+ .map(|s: &String| s.split_whitespace().map(str::to_string).collect())+ .collect();assert_eq!(reconstructed, original);}@@ -463,7 +490,7 @@ mod tests {}// These all share token-count and "event-" prefix is varying.// Similarity might trigger collapse — that's fine if it does- // (still lossless), but we mainly assert no panic + no+ // (still token-level preserving), but we mainly assert no panic + no// information loss.let r = reformat().apply(&log).expect("processes");// Either way: every variant value must survive.@@ -542,4 +569,29 @@ mod tests {let r = reformat().apply(&log).expect("processes");assert!(!r.output.contains("[Template"));}++ #[test]+ fn literal_wildcard_constant_header_differs_from_variable_position() {+ // "<*>" as a literal constant and "x{i}" as a varying token must+ // not print the same template header.+ let mut cfg = cfg();+ cfg.min_lines = 10;+ let r = LogTemplate::new(cfg);+ let mut literal = String::new();+ let mut varying = String::new();+ for i in 0..30 {+ literal.push_str(&format!("svc up <*> ok {i}\n"));+ varying.push_str(&format!("svc up x{i} ok {i}\n"));+ }+ let header = |log: &str| {+ let out = r.apply(log).expect("collapses").output;+ out.lines().next().expect("has header").to_string()+ };+ let literal_header = header(&literal);+ let varying_header = header(&varying);+ assert!(literal_header.starts_with("[Template T1:"));+ assert!(varying_header.starts_with("[Template T1:"));+ assert_ne!(literal_header, varying_header);+ assert!(literal_header.contains("\\<*>"));+ }}diff --git a/crates/headroom-core/src/transforms/pipeline/traits.rs b/crates/headroom-core/src/transforms/pipeline/traits.rsindex a1087f45e..5ff8c9d10 100644--- a/crates/headroom-core/src/transforms/pipeline/traits.rs+++ b/crates/headroom-core/src/transforms/pipeline/traits.rs@@ -211,13 +211,20 @@ pub trait OffloadTransform: Send + Sync {/// Run the offload. The orchestrator only calls this when/// `estimate_bloat(content) ≥ threshold`. On success, the- /// returned `cache_key` MUST resolve to the original payload- /// in `store`.+ /// returned `cache_key` MUST resolve to `ccr_source` in `store`.+ ///+ /// `content` is the text to compress: the output of the prior+ /// reformat/offload stages. `ccr_source` is the original pipeline+ /// input, before any reformat ran. Offloads key and store+ /// `ccr_source`, not `content`, so a retrieval marker resolves to+ /// exactly what the caller passed in. Storing `content` would+ /// capture reformatted text instead.fn apply(&self,content: &str,ctx: &CompressionContext,store: &dyn CcrStore,+ ccr_source: &str,) -> Result<OffloadOutput, TransformError>;/// Calibrated 0.0–1.0 quality score for telemetry. Future PR4@@ -264,9 +271,10 @@ mod tests {content: &str,_ctx: &CompressionContext,store: &dyn CcrStore,+ ccr_source: &str,) -> Result<OffloadOutput, TransformError> {- let key = format!("test_key_{:024x}", content.len());- store.put(&key, content);+ let key = format!("test_key_{:024x}", ccr_source.len());+ store.put(&key, ccr_source);Ok(OffloadOutput::from_lengths(content.len(),content.to_string(),@@ -322,7 +330,7 @@ mod tests {let store = InMemoryCcrStore::new();let t = TestOffload { bloat: 0.9 };let r = t- .apply("hello", &CompressionContext::default(), &store)+ .apply("hello", &CompressionContext::default(), &store, "hello").expect("offload writes");// Trait contract: cache_key is required and resolves in the store.assert!(!r.cache_key.is_empty());diff --git a/crates/headroom-core/src/transforms/search_compressor.rs b/crates/headroom-core/src/transforms/search_compressor.rsindex 7be7f417d..f83d3d32e 100644--- a/crates/headroom-core/src/transforms/search_compressor.rs+++ b/crates/headroom-core/src/transforms/search_compressor.rs@@ -295,6 +295,21 @@ impl SearchCompressor {context: &str,bias: f64,store: Option<&dyn CcrStore>,+ ) -> (SearchCompressionResult, SearchCompressorStats) {+ self.compress_with_ccr_source(content, context, bias, store, content)+ }++ /// Like [`Self::compress_with_store`], but the CCR entry and its+ /// hash come from `ccr_source` rather than `content`. Use when+ /// `content` is a derived form of the original (e.g. after a+ /// reformat) and retrieval must return the original.+ pub fn compress_with_ccr_source(+ &self,+ content: &str,+ context: &str,+ bias: f64,+ store: Option<&dyn CcrStore>,+ ccr_source: &str,) -> (SearchCompressionResult, SearchCompressorStats) {let mut stats = SearchCompressorStats::default();let parsed = self.parse_search_results(content, &mut stats);@@ -334,8 +349,8 @@ impl SearchCompressor {} else if ratio >= self.config.min_compression_ratio_for_ccr {stats.ccr_skip_reason = Some("compression ratio too high");} else if let Some(store) = store {- let key = md5_hex_24(content);- store.put(&key, content);+ let key = md5_hex_24(ccr_source);+ store.put(&key, ccr_source);let marker = format!("\n[{} matches compressed to {}. Retrieve more: hash={}]",original_count, compressed_count, key