Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(pipeline): store the pipeline input in CCR behind offload markers (#4576)

Parideboy4d7765c84b06
diff --git a/crates/headroom-core/src/transforms/diff_compressor.rs b/crates/headroom-core/src/transforms/diff_compressor.rs
index 510240ce0..5a200404b 100644
--- a/crates/headroom-core/src/transforms/diff_compressor.rs
+++ b/crates/headroom-core/src/transforms/diff_compressor.rs
@@ -277,6 +277,20 @@ impl DiffCompressor {
content: &str,
context: &str,
store: Option<&dyn CcrStore>,
+ ) -> (DiffCompressionResult, DiffCompressorStats) {
+ self.compress_with_ccr_source(content, context, store, content)
+ }
+
+ /// Like [`Self::compress_with_store`], but the CCR entry and its
+ /// hash come from `ccr_source` rather than `content`. Use when
+ /// `content` is a derived form of the original (e.g. after a
+ /// reformat) and retrieval must return the original.
+ pub fn compress_with_ccr_source(
+ &self,
+ content: &str,
+ context: &str,
+ store: Option<&dyn CcrStore>,
+ ccr_source: &str,
) -> (DiffCompressionResult, DiffCompressorStats) {
let start = Instant::now();
let mut stats = DiffCompressorStats::default();
@@ -473,7 +487,7 @@ impl DiffCompressor {
if self.config.enable_ccr
&& (compressed_line_count as f64) < (original_line_count as f64) * savings_threshold
{
- let key = md5_hex_24(content);
+ let key = md5_hex_24(ccr_source);
compressed_output.push('\n');
compressed_output.push_str(&format!(
"[{} lines compressed to {}. Retrieve full diff: hash={}]",
@@ -485,7 +499,7 @@ impl DiffCompressor {
// (typically the Python shim) is responsible — see the
// method-level docs.
if let Some(s) = store {
- s.put(&key, content);
+ s.put(&key, ccr_source);
}
cache_key = Some(key);
stats.cache_key_emitted = true;
diff --git a/crates/headroom-core/src/transforms/log_compressor.rs b/crates/headroom-core/src/transforms/log_compressor.rs
index fc7ff605a..338d4804b 100644
--- a/crates/headroom-core/src/transforms/log_compressor.rs
+++ b/crates/headroom-core/src/transforms/log_compressor.rs
@@ -1139,6 +1139,20 @@ impl LogCompressor {
content: &str,
bias: f64,
store: Option<&dyn CcrStore>,
+ ) -> (LogCompressionResult, LogCompressorStats) {
+ self.compress_with_ccr_source(content, bias, store, content)
+ }
+
+ /// Like [`Self::compress_with_store`], but the CCR entry and its
+ /// hash come from `ccr_source` rather than `content`. Use when
+ /// `content` is a derived form of the original (e.g. after a
+ /// reformat) and retrieval must return the original.
+ pub fn compress_with_ccr_source(
+ &self,
+ content: &str,
+ bias: f64,
+ store: Option<&dyn CcrStore>,
+ ccr_source: &str,
) -> (LogCompressionResult, LogCompressorStats) {
let mut stats = LogCompressorStats::default();
let lines: Vec<&str> = content.split('\n').collect();
@@ -1177,8 +1191,8 @@ impl LogCompressor {
if ratio >= self.config.min_compression_ratio_for_ccr {
stats.ccr_skip_reason = Some("compression ratio too high");
} else if let Some(store) = store {
- let key = md5_hex_24(content);
- store.put(&key, content);
+ let key = md5_hex_24(ccr_source);
+ store.put(&key, ccr_source);
// Descriptive so the reader/agent can tell whether the
// dropped content is relevant before deciding whether to
// retrieve it (rather than just "how much" vanished).
diff --git a/crates/headroom-core/src/transforms/pipeline/mod.rs b/crates/headroom-core/src/transforms/pipeline/mod.rs
index c6bde6073..38883a907 100644
--- a/crates/headroom-core/src/transforms/pipeline/mod.rs
+++ b/crates/headroom-core/src/transforms/pipeline/mod.rs
@@ -52,8 +52,10 @@
//! Reformats:
//! - [`reformats::JsonMinifier`] — JSON whitespace stripping.
//! - [`reformats::LogTemplate`] — Drain-style template miner for
-//! build/log output. Lossless — emits `[Template Tn: ...] (Nx)` +
-//! variant table, every original line reconstructible.
+//! build/log output. Tokens preserved, whitespace normalized —
+//! emits `[Template Tn: ...] (Nx)` + variant table; the exact
+//! original is recoverable only through CCR when an offload runs
+//! after the reformat.
//!
//! Offloads:
//! - [`offloads::JsonOffload`] — wraps `SmartCrusher` for JSON arrays
diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs b/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs
index b3ccad1d5..d4441e19e 100644
--- a/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs
+++ b/crates/headroom-core/src/transforms/pipeline/offloads/diff_noise.rs
@@ -117,6 +117,7 @@ impl OffloadTransform for DiffNoise {
content: &str,
_ctx: &CompressionContext,
store: &dyn CcrStore,
+ ccr_source: &str,
) -> Result<OffloadOutput, TransformError> {
let segments = parse_segments(content);
if segments.is_empty() {
@@ -173,8 +174,8 @@ impl OffloadTransform for DiffNoise {
}
// CCR: hash original, stash, append marker.
- let key = md5_hex_24(content);
- store.put(&key, content);
+ let key = md5_hex_24(ccr_source);
+ store.put(&key, ccr_source);
output.push_str(&format!("\n[diff_noise CCR: hash={key}]"));
Ok(OffloadOutput::from_lengths(content.len(), output, key))
@@ -432,7 +433,7 @@ mod tests {
]);
let store = InMemoryCcrStore::new();
let r = offload()
- .apply(&diff, &CompressionContext::default(), &store)
+ .apply(&diff, &CompressionContext::default(), &store, &diff)
.expect("must compress");
assert!(r.bytes_saved > 0);
assert!(r.output.contains("[diff_noise: lockfile hunks dropped"));
@@ -451,7 +452,7 @@ mod tests {
let diff = build_diff(&[("src/main.rs", &body_refs)]);
let store = InMemoryCcrStore::new();
let r = offload()
- .apply(&diff, &CompressionContext::default(), &store)
+ .apply(&diff, &CompressionContext::default(), &store, &diff)
.expect("must compress");
assert!(r
.output
@@ -468,7 +469,7 @@ mod tests {
let diff = build_diff(&[("src/main.rs", &body_refs)]);
let store = InMemoryCcrStore::new();
let err = offload()
- .apply(&diff, &CompressionContext::default(), &store)
+ .apply(&diff, &CompressionContext::default(), &store, &diff)
.expect_err("nothing droppable");
match err {
TransformError::Skipped { .. } => {}
@@ -502,7 +503,7 @@ mod tests {
diff.push_str(&build_diff(&[("yarn.lock", &lock_refs)]));
let store = InMemoryCcrStore::new();
let r = offload()
- .apply(&diff, &CompressionContext::default(), &store)
+ .apply(&diff, &CompressionContext::default(), &store, &diff)
.expect("compresses");
// Pre-diff prelude must survive.
assert!(r.output.contains("Subject: bump deps"));
@@ -514,7 +515,7 @@ mod tests {
assert_eq!(offload().estimate_bloat(""), 0.0);
let store = InMemoryCcrStore::new();
let err = offload()
- .apply("", &CompressionContext::default(), &store)
+ .apply("", &CompressionContext::default(), &store, "")
.expect_err("must skip");
match err {
TransformError::Skipped { .. } => {}
diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs
index d5a5df487..d1eb872a4 100644
--- a/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs
+++ b/crates/headroom-core/src/transforms/pipeline/offloads/diff_offload.rs
@@ -132,15 +132,14 @@ impl OffloadTransform for DiffOffload {
content: &str,
ctx: &CompressionContext,
store: &dyn CcrStore,
+ ccr_source: &str,
) -> Result<OffloadOutput, TransformError> {
- // `compress_with_store` (added in the audit-cleanup PR) writes
- // the original to `store` under the same `cache_key` it embeds
- // in the marker. Earlier versions of this offload had to
- // double-store post-hoc because the compressor lacked a store
- // parameter — that hack is gone.
- let (result, _) = self
- .compressor
- .compress_with_store(content, &ctx.query, Some(store));
+ // The compressor writes `ccr_source` to `store` under the same
+ // `cache_key` it embeds in the marker, so retrieval returns the
+ // pipeline input rather than the reformatted `content`.
+ let (result, _) =
+ self.compressor
+ .compress_with_ccr_source(content, &ctx.query, Some(store), ccr_source);
let Some(key) = result.cache_key else {
return Err(TransformError::skipped(
@@ -254,7 +253,7 @@ mod tests {
let diff = build_diff(1, 200, 5);
let store = InMemoryCcrStore::new();
let r = offload()
- .apply(&diff, &CompressionContext::default(), &store)
+ .apply(&diff, &CompressionContext::default(), &store, &diff)
.expect("should compress");
assert!(!r.cache_key.is_empty());
// Bug-fix verification: DiffCompressor itself wouldn't have
@@ -268,7 +267,7 @@ mod tests {
let diff = build_diff(1, 5, 2);
let store = InMemoryCcrStore::new();
let err = offload()
- .apply(&diff, &CompressionContext::default(), &store)
+ .apply(&diff, &CompressionContext::default(), &store, &diff)
.expect_err("must skip");
match err {
TransformError::Skipped { transform, .. } => assert_eq!(transform, "diff_offload"),
diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs
index 54bb62ded..99a1d22d3 100644
--- a/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs
+++ b/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs
@@ -147,12 +147,15 @@ impl OffloadTransform for JsonOffload {
content: &str,
ctx: &CompressionContext,
store: &dyn CcrStore,
+ ccr_source: &str,
) -> Result<OffloadOutput, TransformError> {
let prose = &self.prose;
let prose_hook = |leaf: &str, query: &str| {
let leaf_ctx = CompressionContext::with_query(query);
+ // Each leaf is its own CCR source: its marker resolves to the
+ // leaf text the crusher saw.
prose
- .apply(leaf, &leaf_ctx, store)
+ .apply(leaf, &leaf_ctx, store, leaf)
.ok()
.map(|output| (output.output, output.cache_key))
};
@@ -173,9 +176,10 @@ impl OffloadTransform for JsonOffload {
// through the orchestrator-supplied store, append a marker so
// the LLM sees a resolvable handle. SmartCrusher's per-array
// markers in `result.compressed` stay informational — the LLM
- // retrieves the full original via this outer hash.
- let key = md5_hex_24(content);
- store.put(&key, content);
+ // retrieves the full original via this outer hash. The hash and
+ // stored payload come from `ccr_source` (the pipeline input).
+ let key = md5_hex_24(ccr_source);
+ store.put(&key, ccr_source);
let mut output = result.compressed;
output.push_str("\n[json_offload CCR: hash=");
output.push_str(&key);
@@ -298,7 +302,7 @@ mod tests {
let input = serde_json::json!([{"description": diff}]).to_string();
let store = InMemoryCcrStore::new();
let output = offload()
- .apply(&input, &CompressionContext::default(), &store)
+ .apply(&input, &CompressionContext::default(), &store, &input)
.expect("diff-shaped leaf should still process through the direct offload path");
let marker = output
.output
@@ -318,7 +322,12 @@ mod tests {
let input = serde_json::json!([{"summary": html}]).to_string();
let store = InMemoryCcrStore::new();
let result = offload()
- .apply(&input, &CompressionContext::with_query("recovery"), &store)
+ .apply(
+ &input,
+ &CompressionContext::with_query("recovery"),
+ &store,
+ &input,
+ )
.expect("structured html should still process");
assert!(result.output.contains(",html,"));
}
@@ -424,7 +433,7 @@ mod tests {
let arr = build_tabular_array(500);
let store = InMemoryCcrStore::new();
let r = offload()
- .apply(&arr, &CompressionContext::default(), &store)
+ .apply(&arr, &CompressionContext::default(), &store, &arr)
.expect("smart crusher should compress");
assert!(r.bytes_saved > 0);
assert!(!r.cache_key.is_empty());
@@ -441,7 +450,7 @@ mod tests {
let arr = build_tabular_array(2);
let store = InMemoryCcrStore::new();
let err = offload()
- .apply(&arr, &CompressionContext::default(), &store)
+ .apply(&arr, &CompressionContext::default(), &store, &arr)
.expect_err("must skip");
match err {
TransformError::Skipped { transform, .. } => assert_eq!(transform, "json_offload"),
@@ -454,7 +463,12 @@ mod tests {
fn apply_skipped_for_non_json_input() {
let store = InMemoryCcrStore::new();
let err = offload()
- .apply("not json at all", &CompressionContext::default(), &store)
+ .apply(
+ "not json at all",
+ &CompressionContext::default(),
+ &store,
+ "not json at all",
+ )
.expect_err("must skip non-json");
match err {
TransformError::Skipped { .. } => {}
@@ -487,7 +501,7 @@ mod tests {
let store = InMemoryCcrStore::new();
let ctx = CompressionContext::with_query("needle");
let r = offload()
- .apply(&s, &ctx, &store)
+ .apply(&s, &ctx, &store, &s)
.expect("crusher should run");
// Output should reference "needle" — SmartCrusher's anchor logic
// should keep that row.
@@ -503,10 +517,10 @@ mod tests {
let store_a = InMemoryCcrStore::new();
let store_b = InMemoryCcrStore::new();
let r_a = offload()
- .apply(&arr, &CompressionContext::default(), &store_a)
+ .apply(&arr, &CompressionContext::default(), &store_a, &arr)
.expect("ok");
let r_b = offload()
- .apply(&arr, &CompressionContext::default(), &store_b)
+ .apply(&arr, &CompressionContext::default(), &store_b, &arr)
.expect("ok");
assert_eq!(
r_a.cache_key, r_b.cache_key,
diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs
index e23bd6a78..96fcdc3b9 100644
--- a/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs
+++ b/crates/headroom-core/src/transforms/pipeline/offloads/log_offload.rs
@@ -145,10 +145,11 @@ impl OffloadTransform for LogOffload {
content: &str,
_ctx: &CompressionContext,
store: &dyn CcrStore,
+ ccr_source: &str,
) -> Result<OffloadOutput, TransformError> {
- let (result, stats) = self
- .compressor
- .compress_with_store(content, self.bias, Some(store));
+ let (result, stats) =
+ self.compressor
+ .compress_with_ccr_source(content, self.bias, Some(store), ccr_source);
// The trait contract says `cache_key` is required. If the
// underlying compressor decided post-hoc that the offload
@@ -258,7 +259,7 @@ mod tests {
let log: String = line.repeat(200);
let store = InMemoryCcrStore::new();
let r = offload()
- .apply(&log, &CompressionContext::default(), &store)
+ .apply(&log, &CompressionContext::default(), &store, &log)
.expect("offload should produce a key on bloaty input");
assert!(!r.cache_key.is_empty());
assert_eq!(store.get(&r.cache_key).as_deref(), Some(log.as_str()));
@@ -273,7 +274,7 @@ mod tests {
let log = "INFO: a\nINFO: b\nINFO: c\nINFO: d\nINFO: e";
let store = InMemoryCcrStore::new();
let err = offload()
- .apply(log, &CompressionContext::default(), &store)
+ .apply(log, &CompressionContext::default(), &store, log)
.expect_err("must skip");
match err {
TransformError::Skipped { transform, .. } => assert_eq!(transform, "log_offload"),
diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/prose_field.rs b/crates/headroom-core/src/transforms/pipeline/offloads/prose_field.rs
index 6494f8cd1..db88c78d3 100644
--- a/crates/headroom-core/src/transforms/pipeline/offloads/prose_field.rs
+++ b/crates/headroom-core/src/transforms/pipeline/offloads/prose_field.rs
@@ -34,7 +34,9 @@ impl ProseFieldOffload {
&& detect_content_type(content).content_type == ContentType::PlainText
}
- fn compress(&self, content: &str, query: &str) -> Option<(String, String)> {
+ /// Returns `(output, key)`. The key hashes `ccr_source`, the payload
+ /// the caller will store; `content` is only the text compressed.
+ fn compress(&self, content: &str, query: &str, ccr_source: &str) -> Option<(String, String)> {
if !self.eligible(content) {
return None;
}
@@ -45,7 +47,7 @@ impl ProseFieldOffload {
return None;
}
- let key = compute_key(content.as_bytes());
+ let key = compute_key(ccr_source.as_bytes());
let output = format!("{}\n{}", result.compressed, marker_for(&key));
(output.len() < content.len()).then_some((output, key))
}
@@ -80,14 +82,15 @@ impl OffloadTransform for ProseFieldOffload {
content: &str,
ctx: &CompressionContext,
store: &dyn CcrStore,
+ ccr_source: &str,
) -> Result<OffloadOutput, TransformError> {
- let Some((output, key)) = self.compress(content, &ctx.query) else {
+ let Some((output, key)) = self.compress(content, &ctx.query, ccr_source) else {
return Err(TransformError::skipped(
NAME,
"prose compression not worth it",
));
};
- store.put(&key, content);
+ store.put(&key, ccr_source);
Ok(OffloadOutput::from_lengths(content.len(), output, key))
}
@@ -123,7 +126,7 @@ mod tests {
fn short_plain_text_is_byte_identical() {
let input = "A short note.";
let store = InMemoryCcrStore::new();
- let result = offload().apply(input, &CompressionContext::default(), &store);
+ let result = offload().apply(input, &CompressionContext::default(), &store, input);
assert!(result.is_err());
assert!(store.is_empty());
}
@@ -132,7 +135,7 @@ mod tests {
fn long_low_segment_text_is_byte_identical() {
let input = "A".repeat(300);
let store = InMemoryCcrStore::new();
- let result = offload().apply(&input, &CompressionContext::default(), &store);
+ let result = offload().apply(&input, &CompressionContext::default(), &store, &input);
assert!(result.is_err());
assert!(store.is_empty());
}
@@ -159,8 +162,8 @@ mod tests {
fn query_changes_selection_deterministically() {
let input = prose();
let crusher = offload();
- let a = crusher.compress(&input, "recovery").unwrap();
- let b = crusher.compress(&input, "recovery").unwrap();
+ let a = crusher.compress(&input, "recovery", &input).unwrap();
+ let b = crusher.compress(&input, "recovery", &input).unwrap();
assert_eq!(a, b);
assert!(a.0.contains("recovery"));
assert!(!a
diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs
index 5600f7fa2..df74537b4 100644
--- a/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs
+++ b/crates/headroom-core/src/transforms/pipeline/offloads/search_offload.rs
@@ -116,10 +116,15 @@ impl OffloadTransform for SearchOffload {
content: &str,
ctx: &CompressionContext,
store: &dyn CcrStore,
+ ccr_source: &str,
) -> Result<OffloadOutput, TransformError> {
- let (result, stats) =
- self.compressor
- .compress_with_store(content, &ctx.query, self.bias, Some(store));
+ let (result, stats) = self.compressor.compress_with_ccr_source(
+ content,
+ &ctx.query,
+ self.bias,
+ Some(store),
+ ccr_source,
+ );
let Some(key) = result.cache_key else {
let reason = stats.ccr_skip_reason.unwrap_or("no cache_key emitted");
@@ -309,7 +314,7 @@ mod tests {
.join("\n");
let store = InMemoryCcrStore::new();
let r = offload()
- .apply(&s, &CompressionContext::default(), &store)
+ .apply(&s, &CompressionContext::default(), &store, &s)
.expect("offload should produce a key on clustered input");
assert!(!r.cache_key.is_empty());
assert_eq!(store.get(&r.cache_key).as_deref(), Some(s.as_str()));
@@ -321,7 +326,7 @@ mod tests {
let s = "only.py:1:trivial";
let store = InMemoryCcrStore::new();
let err = offload()
- .apply(s, &CompressionContext::default(), &store)
+ .apply(s, &CompressionContext::default(), &store, s)
.expect_err("must skip");
match err {
TransformError::Skipped { transform, .. } => assert_eq!(transform, "search_offload"),
diff --git a/crates/headroom-core/src/transforms/pipeline/orchestrator.rs b/crates/headroom-core/src/transforms/pipeline/orchestrator.rs
index 66592e09f..c9a196457 100644
--- a/crates/headroom-core/src/transforms/pipeline/orchestrator.rs
+++ b/crates/headroom-core/src/transforms/pipeline/orchestrator.rs
@@ -153,7 +153,9 @@ impl CompressionPipeline {
);
continue;
}
- match offload.apply(&current, ctx, store) {
+ // CCR stores the pipeline input (`content`), not the
+ // reformatted `current`, so retrieval returns the original.
+ match offload.apply(&current, ctx, store, content) {
Ok(out) => {
if out.bytes_saved == 0 {
tracing::trace!(
@@ -422,11 +424,12 @@ mod tests {
content: &str,
_ctx: &CompressionContext,
store: &dyn CcrStore,
+ ccr_source: &str,
) -> Result<OffloadOutput, TransformError> {
// Always halve; emit a cache_key derived from name.
let half = &content[..content.len() / 2];
let key = format!("test_{}_key", self.name);
- store.put(&key, content);
+ store.put(&key, ccr_source);
Ok(OffloadOutput::from_lengths(
content.len(),
half.to_string(),
@@ -644,6 +647,7 @@ mod tests {
_content: &str,
_ctx: &CompressionContext,
_store: &dyn CcrStore,
+ _ccr_source: &str,
) -> Result<OffloadOutput, TransformError> {
Err(TransformError::internal("always_internal_err", "by design"))
}
@@ -699,7 +703,7 @@ mod tests {
#[test]
fn end_to_end_log_template_collapses_then_log_offload_can_run() {
- // LogTemplate runs first (lossless reformat), then LogOffload
+ // LogTemplate runs first (token-preserving reformat), then LogOffload
// sees the collapsed output and may further drop low-priority
// lines. Both should appear in steps_applied if both fire.
let cfg = PipelineConfig::default();
@@ -733,6 +737,42 @@ mod tests {
assert!(r.output.len() < log.len());
}
+ #[test]
+ fn ccr_stores_pipeline_input_not_reformatted_text() {
+ // LogTemplate rewrites the log before LogOffload runs. The CCR
+ // entry behind LogOffload's marker must still be the original
+ // pipeline input, so retrieval gives back the untouched log.
+ let cfg = PipelineConfig::default();
+ let p = CompressionPipeline::builder()
+ .with_reformat(LogTemplate::new(cfg.reformat.log_template))
+ .with_offload(LogOffload::new(cfg.bloat.log))
+ .with_config(cfg)
+ .build();
+ let s = store();
+ let mut input = String::new();
+ for i in 0..300 {
+ input.push_str(&format!(
+ "2025-01-15T12:{:02}:{:02} INFO processing job {i}\n",
+ i / 60 % 60,
+ i % 60
+ ));
+ }
+ for i in 0..300 {
+ input.push_str(&format!(
+ "2025-01-15T13:{:02}:{:02} DEBUG heartbeat ok\n",
+ i / 60 % 60,
+ i % 60
+ ));
+ }
+ input.push_str("2025-01-15T15:00:00 ERROR database connection lost\n");
+
+ let r = p.run(&input, ContentType::BuildOutput, &ctx(), &s);
+ assert!(r.steps_applied.iter().any(|n| n == "log_template"));
+ assert!(r.steps_applied.iter().any(|n| n == "log_offload"));
+ let key = r.cache_keys.first().expect("log_offload emits a cache key");
+ assert_eq!(s.get(key).as_deref(), Some(input.as_str()));
+ }
+
#[test]
fn end_to_end_diff_noise_drops_lockfile_then_diff_offload_handles_rest() {
let cfg = PipelineConfig::default();
diff --git a/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs b/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs
index 9e668abd9..0c2a11311 100644
--- a/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs
+++ b/crates/headroom-core/src/transforms/pipeline/reformats/log_template.rs
@@ -23,9 +23,17 @@
//! ...
//! ```
//!
-//! Every original line is reconstructible from `template + variants`,
-//! so this is **lossless** — no CCR retrieval needed. The win is the
-//! template prefix (often 30+ chars) emitted once instead of N times.
+//! Every line is reconstructible from `template + variants` at token
+//! level: the tokens are kept, but whitespace is normalized. Runs of
+//! spaces, tabs, indentation, and `\r` are not preserved, so the
+//! output is not a byte-exact copy of the input. The exact original is
+//! recoverable only through CCR, when an offload runs after this
+//! reformat. The win is the template prefix (often 30+ chars) emitted
+//! once instead of N times.
+//!
+//! Constant tokens that are `<*>` behind zero or more backslashes are
+//! escaped with one more leading backslash (`<*>` → `\<*>`), so a
+//! header position reads as a wildcard only when it is exactly `<*>`.
//!
//! Order is preserved: only *consecutive* runs collapse, so the
//! temporal flow of the log stays intact for the LLM.
@@ -244,7 +252,7 @@ impl LogTemplate {
out.push(' ');
}
match slot {
- Some(constant) => out.push_str(constant),
+ Some(constant) => push_constant(out, constant),
None => out.push_str(WILDCARD),
}
}
@@ -289,6 +297,17 @@ impl Run {
}
}
+/// Write a constant template token. A literal `<*>` would print the
+/// same as the wildcard, and a literal `\<*>` (any number of leading
+/// backslashes) would read as an escape, so each gets one extra
+/// leading backslash.
+fn push_constant(out: &mut String, token: &str) {
+ if token.trim_start_matches('\\') == WILDCARD {
+ out.push('\\');
+ }
+ out.push_str(token);
+}
+
/// Whitespace-split tokenizer. Empty result = blank line.
///
/// Uses `str::split_whitespace` semantics: collapses runs of whitespace,
@@ -357,7 +376,7 @@ mod tests {
r.output.chars().take(200).collect::<String>()
);
assert!(r.output.contains("(50 occurrences)"));
- // Variants should still be in output (lossless guarantee).
+ // Variants should still be in output (token-level guarantee).
assert!(r.output.contains("worker-7"));
}
@@ -384,9 +403,10 @@ mod tests {
}
#[test]
- fn lossless_round_trip_via_template_and_variants() {
- // The flushed output must be reconstructible: each template
- // line + its variant rows reproduces the original input lines.
+ fn token_round_trip_via_template_and_variants() {
+ // The flushed output must be reconstructible at token level:
+ // each template line + its variant rows reproduces the original
+ // tokens. Whitespace is normalized, so compare token lists.
let mut log = String::new();
for i in 0..25 {
log.push_str(&format!("TOK1 TOK2 var{i} TOK3\n"));
@@ -420,7 +440,14 @@ mod tests {
full[var_pos] = var_tokens[0];
reconstructed.push(full.join(" "));
}
- let original: Vec<String> = log.lines().map(|s| s.to_string()).collect();
+ let original: Vec<Vec<String>> = log
+ .lines()
+ .map(|s| s.split_whitespace().map(str::to_string).collect())
+ .collect();
+ let reconstructed: Vec<Vec<String>> = reconstructed
+ .iter()
+ .map(|s: &String| s.split_whitespace().map(str::to_string).collect())
+ .collect();
assert_eq!(reconstructed, original);
}
@@ -463,7 +490,7 @@ mod tests {
}
// These all share token-count and "event-" prefix is varying.
// Similarity might trigger collapse — that's fine if it does
- // (still lossless), but we mainly assert no panic + no
+ // (still token-level preserving), but we mainly assert no panic + no
// information loss.
let r = reformat().apply(&log).expect("processes");
// Either way: every variant value must survive.
@@ -542,4 +569,29 @@ mod tests {
let r = reformat().apply(&log).expect("processes");
assert!(!r.output.contains("[Template"));
}
+
+ #[test]
+ fn literal_wildcard_constant_header_differs_from_variable_position() {
+ // "<*>" as a literal constant and "x{i}" as a varying token must
+ // not print the same template header.
+ let mut cfg = cfg();
+ cfg.min_lines = 10;
+ let r = LogTemplate::new(cfg);
+ let mut literal = String::new();
+ let mut varying = String::new();
+ for i in 0..30 {
+ literal.push_str(&format!("svc up <*> ok {i}\n"));
+ varying.push_str(&format!("svc up x{i} ok {i}\n"));
+ }
+ let header = |log: &str| {
+ let out = r.apply(log).expect("collapses").output;
+ out.lines().next().expect("has header").to_string()
+ };
+ let literal_header = header(&literal);
+ let varying_header = header(&varying);
+ assert!(literal_header.starts_with("[Template T1:"));
+ assert!(varying_header.starts_with("[Template T1:"));
+ assert_ne!(literal_header, varying_header);
+ assert!(literal_header.contains("\\<*>"));
+ }
}
diff --git a/crates/headroom-core/src/transforms/pipeline/traits.rs b/crates/headroom-core/src/transforms/pipeline/traits.rs
index a1087f45e..5ff8c9d10 100644
--- a/crates/headroom-core/src/transforms/pipeline/traits.rs
+++ b/crates/headroom-core/src/transforms/pipeline/traits.rs
@@ -211,13 +211,20 @@ pub trait OffloadTransform: Send + Sync {
/// Run the offload. The orchestrator only calls this when
/// `estimate_bloat(content) ≥ threshold`. On success, the
- /// returned `cache_key` MUST resolve to the original payload
- /// in `store`.
+ /// returned `cache_key` MUST resolve to `ccr_source` in `store`.
+ ///
+ /// `content` is the text to compress: the output of the prior
+ /// reformat/offload stages. `ccr_source` is the original pipeline
+ /// input, before any reformat ran. Offloads key and store
+ /// `ccr_source`, not `content`, so a retrieval marker resolves to
+ /// exactly what the caller passed in. Storing `content` would
+ /// capture reformatted text instead.
fn apply(
&self,
content: &str,
ctx: &CompressionContext,
store: &dyn CcrStore,
+ ccr_source: &str,
) -> Result<OffloadOutput, TransformError>;
/// Calibrated 0.0–1.0 quality score for telemetry. Future PR4
@@ -264,9 +271,10 @@ mod tests {
content: &str,
_ctx: &CompressionContext,
store: &dyn CcrStore,
+ ccr_source: &str,
) -> Result<OffloadOutput, TransformError> {
- let key = format!("test_key_{:024x}", content.len());
- store.put(&key, content);
+ let key = format!("test_key_{:024x}", ccr_source.len());
+ store.put(&key, ccr_source);
Ok(OffloadOutput::from_lengths(
content.len(),
content.to_string(),
@@ -322,7 +330,7 @@ mod tests {
let store = InMemoryCcrStore::new();
let t = TestOffload { bloat: 0.9 };
let r = t
- .apply("hello", &CompressionContext::default(), &store)
+ .apply("hello", &CompressionContext::default(), &store, "hello")
.expect("offload writes");
// Trait contract: cache_key is required and resolves in the store.
assert!(!r.cache_key.is_empty());
diff --git a/crates/headroom-core/src/transforms/search_compressor.rs b/crates/headroom-core/src/transforms/search_compressor.rs
index 7be7f417d..f83d3d32e 100644
--- a/crates/headroom-core/src/transforms/search_compressor.rs
+++ b/crates/headroom-core/src/transforms/search_compressor.rs
@@ -295,6 +295,21 @@ impl SearchCompressor {
context: &str,
bias: f64,
store: Option<&dyn CcrStore>,
+ ) -> (SearchCompressionResult, SearchCompressorStats) {
+ self.compress_with_ccr_source(content, context, bias, store, content)
+ }
+
+ /// Like [`Self::compress_with_store`], but the CCR entry and its
+ /// hash come from `ccr_source` rather than `content`. Use when
+ /// `content` is a derived form of the original (e.g. after a
+ /// reformat) and retrieval must return the original.
+ pub fn compress_with_ccr_source(
+ &self,
+ content: &str,
+ context: &str,
+ bias: f64,
+ store: Option<&dyn CcrStore>,
+ ccr_source: &str,
) -> (SearchCompressionResult, SearchCompressorStats) {
let mut stats = SearchCompressorStats::default();
let parsed = self.parse_search_results(content, &mut stats);
@@ -334,8 +349,8 @@ impl SearchCompressor {
} else if ratio >= self.config.min_compression_ratio_for_ccr {
stats.ccr_skip_reason = Some("compression ratio too high");
} else if let Some(store) = store {
- let key = md5_hex_24(content);
- store.put(&key, content);
+ let key = md5_hex_24(ccr_source);
+ store.put(&key, ccr_source);
let marker = format!(
"\n[{} matches compressed to {}. Retrieve more: hash={}]",
original_count, compressed_count, key