Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(rust): wire SmartCrusher as JsonOffload in the pipeline (Phase 3g PR2)
chopratejas2a0582dee1d9
diff --git a/crates/headroom-core/config/pipeline.toml b/crates/headroom-core/config/pipeline.tomlindex 650ef2eed..811123778 100644--- a/crates/headroom-core/config/pipeline.toml+++ b/crates/headroom-core/config/pipeline.toml@@ -103,6 +103,22 @@ similarity_threshold = 0.4# verbatim instead. 2 = "must share at least 2 anchor tokens."min_constant_tokens = 2+# ─── JSON-array offload config (SmartCrusher wrapper) ─────────────+#+# JsonOffload wraps SmartCrusher: a JSON array of dicts (rows) with+# repeated schema is highly bloaty — most fields repeat their keys per+# row. SmartCrusher dedups, drops near-duplicate rows, and emits a CCR+# marker for retrieval. The bloat estimator below cheaply spots the+# "array-of-dicts shape with N+ rows" signal without parsing JSON.++[offload.json]+# Don't fire below this many detected row separators (`},{` boundaries+# scaled across the input). 5 = "less than 5 rows isn't worth it."+min_array_rows = 5+# At this row count, the bloat score saturates to 1.0. Tabular inputs+# with this many rows are reliably bloat-heavy candidates for CCR.+saturation_rows = 50+# ─── Diff-noise offload config ─────────────────────────────────────## Drop hunks for files that match these glob-like suffixes (literaldiff --git a/crates/headroom-core/src/transforms/mod.rs b/crates/headroom-core/src/transforms/mod.rsindex 93274a4df..2f2f3a9b5 100644--- a/crates/headroom-core/src/transforms/mod.rs+++ b/crates/headroom-core/src/transforms/mod.rs@@ -42,8 +42,8 @@ pub use log_compressor::{pub use magika_detector::{magika_detect, map_magika_label, MagikaDetectorError};pub use pipeline::{CompressionContext, CompressionPipeline, CompressionPipelineBuilder, DiffNoise, DiffOffload,- JsonMinifier, LogOffload, LogTemplate, OffloadOutput, OffloadTransform, PipelineConfig,- PipelineResult, ReformatOutput, ReformatTransform, TransformError,+ JsonMinifier, JsonOffload, LogOffload, LogTemplate, OffloadOutput, OffloadTransform,+ PipelineConfig, PipelineResult, ReformatOutput, ReformatTransform, TransformError,};pub use search_compressor::{FileMatches, SearchCompressionResult, SearchCompressor, SearchCompressorConfig,diff --git a/crates/headroom-core/src/transforms/pipeline/config.rs b/crates/headroom-core/src/transforms/pipeline/config.rsindex 0591a1f45..a44a56a7d 100644--- a/crates/headroom-core/src/transforms/pipeline/config.rs+++ b/crates/headroom-core/src/transforms/pipeline/config.rs@@ -182,9 +182,20 @@ pub struct LogTemplateConfig {/// Offload-transform configs (one struct per offload that needs them).#[derive(Debug, Clone, Deserialize)]pub struct OffloadConfigs {+ pub json: JsonOffloadConfig,pub diff_noise: DiffNoiseConfig,}+/// Knobs for the [`crate::transforms::pipeline::offloads::JsonOffload`]+/// (SmartCrusher wrapper). The estimator scans byte-prefix-cheaply for+/// JSON array-of-objects shape; SmartCrusher itself does the heavy work+/// when the orchestrator decides to fire.+#[derive(Debug, Clone, Copy, Deserialize)]+pub struct JsonOffloadConfig {+ pub min_array_rows: usize,+ pub saturation_rows: usize,+}+/// Knobs for the [`crate::transforms::pipeline::offloads::DiffNoise`]/// offload. Lockfile suffixes are matched against the new-file path/// at the end of each `diff --git` header.@@ -264,6 +275,10 @@ mod tests {similarity_threshold = 0.8min_constant_tokens = 3+ [offload.json]+ min_array_rows = 3+ saturation_rows = 25+[offload.diff_noise]min_lines = 20lockfile_suffixes = ["custom.lock"]@@ -285,6 +300,8 @@ mod tests {let cfg = PipelineConfig::default();assert_eq!(cfg.reformat.log_template.min_lines, 20);assert_eq!(cfg.reformat.log_template.min_run, 3);+ assert_eq!(cfg.offload.json.min_array_rows, 5);+ assert_eq!(cfg.offload.json.saturation_rows, 50);assert!(!cfg.offload.diff_noise.lockfile_suffixes.is_empty());assert!(cfg.offloaddiff --git a/crates/headroom-core/src/transforms/pipeline/mod.rs b/crates/headroom-core/src/transforms/pipeline/mod.rsindex f29c5ee99..54fe4a667 100644--- a/crates/headroom-core/src/transforms/pipeline/mod.rs+++ b/crates/headroom-core/src/transforms/pipeline/mod.rs@@ -56,6 +56,11 @@//! variant table, every original line reconstructible.//!//! Offloads:+//! - [`offloads::JsonOffload`] — wraps `SmartCrusher` for JSON arrays+//! of dicts. Estimator counts row separators; apply delegates the+//! heavy work to SmartCrusher (schema dedup, row sampling,+//! anchor-aware selection) and adds a wrapper-level CCR marker+//! that resolves in the orchestrator's store.//! - [`offloads::LogOffload`] — wraps the existing `LogCompressor`,//! gates on per-line bloat heuristic.//! - [`offloads::DiffOffload`] — wraps the existing `DiffCompressor`,@@ -69,11 +74,7 @@//! `rg`/`grep`, the marginal value didn't justify default registration.//!//! Deferred to later PRs:-//! - **JSON Offload** — already lives at-//! `crate::transforms::smart_crusher::SmartCrusher`. Phase 3g PR3-//! refactors it to implement the [`OffloadTransform`] contract so-//! the orchestrator dispatches JSON arrays through it.-//! - **ProseFieldCompressor** — Phase 3g PR4. Compresses prose-shaped+//! - **ProseFieldCompressor** — Phase 3g PR3. Compresses prose-shaped//! string fields inside structured payloads.//!//! [`estimate_bloat`]: traits::OffloadTransform::estimate_bloat@@ -85,14 +86,15 @@ pub mod reformats;pub mod traits;pub use config::{- BloatConfigs, ConfigError, DiffBloatConfig, DiffNoiseConfig, LogBloatConfig, LogTemplateConfig,- OffloadConfigs, OrchestratorConfig, PipelineConfig, ReformatConfigs, SearchBloatConfig,+ BloatConfigs, ConfigError, DiffBloatConfig, DiffNoiseConfig, JsonOffloadConfig, LogBloatConfig,+ LogTemplateConfig, OffloadConfigs, OrchestratorConfig, PipelineConfig, ReformatConfigs,+ SearchBloatConfig,};// `SearchOffload` is intentionally NOT in the top-level re-export// (deprecated from default pipeline; reach via the explicit module// path if you want to opt in). See `offloads::search_offload` head// docs for rationale.-pub use offloads::{DiffNoise, DiffOffload, LogOffload};+pub use offloads::{DiffNoise, DiffOffload, JsonOffload, LogOffload};pub use orchestrator::{CompressionPipeline, CompressionPipelineBuilder, PipelineResult};pub use reformats::{JsonMinifier, LogTemplate};pub use traits::{diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs b/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rsnew file mode 100644index 000000000..eae284835--- /dev/null+++ b/crates/headroom-core/src/transforms/pipeline/offloads/json_offload.rs@@ -0,0 +1,400 @@+//! `JsonOffload` — wraps [`SmartCrusher`] as an [`OffloadTransform`].+//!+//! # What this offload does+//!+//! JSON arrays of dicts (rows) are the worst tool-output shape for+//! token bloat: every row repeats its field names, and many tools+//! return long arrays with near-duplicate or low-information rows+//! (audit logs, search index dumps, paginated API results). SmartCrusher+//! is the existing Rust port that handles this — schema dedup, row+//! sampling, anchor-aware row selection, lossless tabular compaction —+//! and emits CCR markers for any rows it drops.+//!+//! `JsonOffload` plugs SmartCrusher into the pipeline's+//! [`OffloadTransform`] contract:+//!+//! 1. `estimate_bloat` is a CHEAP byte scan that spots the+//! array-of-objects shape and counts row separators (`}` followed+//! by `,` or `]`). No JSON parse — that's reserved for `apply`.+//! 2. `apply` delegates to `SmartCrusher::crush(content, ctx.query)`+//! and adds a wrapper-level CCR marker. The orchestrator-supplied+//! store gets the original payload under the wrapper hash.+//!+//! # Why a wrapper-level CCR hash on top of SmartCrusher's internal one+//!+//! SmartCrusher mints PER-SUB-ARRAY hashes (`<<ccr:abc 42_rows_offloaded>>`)+//! that resolve in its **internal** store. The pipeline orchestrator+//! has its **own** store (passed to `apply` via the trait). To honor+//! the trait contract — `cache_key` MUST resolve in the orchestrator's+//! store — `JsonOffload` hashes the WHOLE input and stashes it via the+//! orchestrator store under that hash, returning that hash as the+//! `cache_key`.+//!+//! For retrieval the LLM uses the wrapper's outer hash (recovers the+//! whole original JSON). SmartCrusher's per-array markers in the+//! compressed body are informational hints — not directly resolvable+//! through the pipeline-level store, but the LLM doesn't need them+//! when the wrapper hash gives back the full payload.+//!+//! # No regex+//!+//! Per project convention. Estimator is byte-prefix scan ++//! `str::matches` over fixed substrings. SmartCrusher itself uses+//! `serde_json` for parsing, never regex.+//!+//! [`SmartCrusher`]: crate::transforms::smart_crusher::SmartCrusher+//! [`OffloadTransform`]: crate::transforms::pipeline::traits::OffloadTransform++use md5::{Digest, Md5};++use crate::ccr::CcrStore;+use crate::transforms::pipeline::config::JsonOffloadConfig;+use crate::transforms::pipeline::traits::{+ CompressionContext, OffloadOutput, OffloadTransform, TransformError,+};+use crate::transforms::smart_crusher::{SmartCrusher, SmartCrusherConfig};+use crate::transforms::ContentType;++const NAME: &str = "json_offload";+/// SmartCrusher has 50+ parity fixtures and shadow-validated against+/// Python — high confidence.+const CONFIDENCE: f32 = 0.85;++pub struct JsonOffload {+ crusher: SmartCrusher,+ config: JsonOffloadConfig,+}++impl JsonOffload {+ /// Default constructor — builds SmartCrusher with the OSS default+ /// composition (scorer + constraints + compaction stage + internal+ /// CCR store). The internal store handles SmartCrusher's per-array+ /// markers; the wrapper still emits its own outer marker that+ /// resolves in the orchestrator-supplied store.+ pub fn new(config: JsonOffloadConfig) -> Self {+ Self {+ crusher: SmartCrusher::new(SmartCrusherConfig::default()),+ config,+ }+ }++ /// Custom constructor — used by tests that want a stubbed crusher+ /// or a custom SmartCrusher config.+ pub fn with_crusher(crusher: SmartCrusher, config: JsonOffloadConfig) -> Self {+ Self { crusher, config }+ }+}++impl OffloadTransform for JsonOffload {+ fn name(&self) -> &'static str {+ NAME+ }++ fn applies_to(&self) -> &[ContentType] {+ &[ContentType::JsonArray]+ }++ fn estimate_bloat(&self, content: &str) -> f32 {+ if content.is_empty() {+ return 0.0;+ }+ let trimmed = content.trim_start();+ if !trimmed.starts_with('[') {+ return 0.0;+ }+ // Cheap structural scan: count `},{` or `}, {` occurrences as+ // row boundaries. Doesn't parse JSON — that's reserved for+ // `apply`. Conservative: counts only adjacent-row patterns,+ // not the trailing `}` before `]` (so a 5-row array reports 4+ // separators, but for our threshold logic that rounds correctly).+ let separators = count_row_separators(content);+ if separators < self.config.min_array_rows.saturating_sub(1) {+ return 0.0;+ }+ let saturation = self.config.saturation_rows.saturating_sub(1).max(1);+ (separators as f32 / saturation as f32).clamp(0.0, 1.0)+ }++ fn apply(+ &self,+ content: &str,+ ctx: &CompressionContext,+ store: &dyn CcrStore,+ ) -> Result<OffloadOutput, TransformError> {+ let result = self.crusher.crush(content, &ctx.query, 0.0);+ if !result.was_modified {+ return Err(TransformError::skipped(+ NAME,+ "smart crusher returned passthrough",+ ));+ }+ if result.compressed.len() >= content.len() {+ return Err(TransformError::skipped(NAME, "no savings after crush"));+ }++ // Wrapper-level CCR: hash the WHOLE original input, stash it+ // through the orchestrator-supplied store, append a marker so+ // the LLM sees a resolvable handle. SmartCrusher's per-array+ // markers in `result.compressed` stay informational — the LLM+ // retrieves the full original via this outer hash.+ let key = md5_hex_24(content);+ store.put(&key, content);+ let mut output = result.compressed;+ output.push_str("\n[json_offload CCR: hash=");+ output.push_str(&key);+ output.push(']');++ Ok(OffloadOutput::from_lengths(content.len(), output, key))+ }++ fn confidence(&self) -> f32 {+ CONFIDENCE+ }+}++/// Count plausible row-boundary patterns in `content`. Used by the+/// bloat estimator as a cheap proxy for JSON array length. Patterns+/// counted (no overlap):+///+/// * `},{` — compact JSON, no whitespace.+/// * `}, {` — pretty-printed JSON with single-space.+/// * `},\n` — pretty-printed JSON with newline-after-comma.+///+/// We don't attempt to be perfect — the estimator just needs an+/// order-of-magnitude signal. False positives on `},` strings inside+/// quoted values are tolerated; their rate is low and the eventual+/// `apply` does the rigorous JSON parse anyway.+fn count_row_separators(content: &str) -> usize {+ content.matches("},{").count()+ + content.matches("}, {").count()+ + content.matches("},\n").count()+}++/// MD5 of `content`, hex-encoded, truncated to 24 chars. Matches the+/// CCR convention used by other offload wrappers.+fn md5_hex_24(content: &str) -> String {+ let mut h = Md5::new();+ h.update(content.as_bytes());+ let digest = h.finalize();+ let mut hex = String::with_capacity(32);+ for b in digest.iter() {+ hex.push_str(&format!("{:02x}", b));+ }+ hex.truncate(24);+ hex+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::ccr::InMemoryCcrStore;+ use crate::transforms::pipeline::config::PipelineConfig;++ fn cfg() -> JsonOffloadConfig {+ PipelineConfig::default().offload.json+ }++ fn offload() -> JsonOffload {+ JsonOffload::new(cfg())+ }++ /// Build a JSON array of N similar dicts with id + name + value.+ /// Compact JSON (no extra whitespace) so byte counts are predictable.+ fn build_tabular_array(n: usize) -> String {+ let mut s = String::from("[");+ for i in 0..n {+ if i > 0 {+ s.push(',');+ }+ s.push_str(&format!(+ "{{\"id\":{},\"name\":\"item-{}\",\"value\":{}}}",+ i,+ i,+ i * 100+ ));+ }+ s.push(']');+ s+ }++ #[test]+ fn name_and_applies_to() {+ let o = offload();+ assert_eq!(o.name(), "json_offload");+ assert_eq!(o.applies_to(), &[ContentType::JsonArray]);+ }++ #[test]+ fn estimate_bloat_empty_input_zero() {+ assert_eq!(offload().estimate_bloat(""), 0.0);+ }++ #[test]+ fn estimate_bloat_non_array_input_zero() {+ // Object, not array.+ assert_eq!(offload().estimate_bloat(r#"{"a":1,"b":2}"#), 0.0);+ // Plain text.+ assert_eq!(offload().estimate_bloat("just words here"), 0.0);+ // Number.+ assert_eq!(offload().estimate_bloat("42"), 0.0);+ }++ #[test]+ fn estimate_bloat_below_min_rows_zero() {+ let arr = build_tabular_array(3); // 2 separators, default min_array_rows=5 → need ≥4+ assert_eq!(offload().estimate_bloat(&arr), 0.0);+ }++ #[test]+ fn estimate_bloat_at_saturation_is_one() {+ // 100 rows → 99 separators. saturation_rows=50 → 49 saturates.+ let arr = build_tabular_array(100);+ let score = offload().estimate_bloat(&arr);+ assert!(score >= 0.99, "expected ~1.0 at saturation, got {score}");+ }++ #[test]+ fn estimate_bloat_scales_linearly_in_middle_range() {+ // 25 rows → 24 separators. saturation=49 → score ≈ 0.49.+ let arr = build_tabular_array(25);+ let score = offload().estimate_bloat(&arr);+ assert!(+ (0.4..=0.6).contains(&score),+ "expected mid-range score, got {score}"+ );+ }++ #[test]+ fn estimate_bloat_handles_whitespace_after_array_open() {+ // Pretty-printed JSON: `[ {`, then `}, {`, etc.+ let mut s = String::from("[\n");+ for i in 0..30 {+ if i > 0 {+ s.push_str(",\n");+ }+ s.push_str(&format!(+ " {{\"id\":{i},\"name\":\"item-{i}\",\"value\":{}}}",+ i * 100+ ));+ }+ s.push_str("\n]");+ let score = offload().estimate_bloat(&s);+ assert!(+ score > 0.4,+ "pretty-printed array should still score, got {score}"+ );+ }++ #[test]+ fn estimate_bloat_handles_huge_input_safely() {+ let arr = build_tabular_array(50_000);+ // Just must not panic; saturation handles this fine.+ let score = offload().estimate_bloat(&arr);+ assert!((0.99..=1.0).contains(&score));+ }++ #[test]+ fn apply_compresses_large_tabular_array_and_stores_original() {+ // Big enough that SmartCrusher will engage.+ let arr = build_tabular_array(500);+ let store = InMemoryCcrStore::new();+ let r = offload()+ .apply(&arr, &CompressionContext::default(), &store)+ .expect("smart crusher should compress");+ assert!(r.bytes_saved > 0);+ assert!(!r.cache_key.is_empty());+ // Wrapper marker present.+ assert!(r.output.contains("[json_offload CCR: hash="));+ // Original recoverable through the orchestrator's store.+ assert_eq!(store.get(&r.cache_key).as_deref(), Some(arr.as_str()));+ }++ #[test]+ fn apply_skipped_when_smart_crusher_passes_through() {+ // 2-row array: below SmartCrusher's `min_items_to_analyze` so+ // it returns passthrough — wrapper must surface that as Skipped.+ let arr = build_tabular_array(2);+ let store = InMemoryCcrStore::new();+ let err = offload()+ .apply(&arr, &CompressionContext::default(), &store)+ .expect_err("must skip");+ match err {+ TransformError::Skipped { transform, .. } => assert_eq!(transform, "json_offload"),+ _ => panic!("expected Skipped, got {err:?}"),+ }+ assert_eq!(store.len(), 0);+ }++ #[test]+ fn apply_skipped_for_non_json_input() {+ let store = InMemoryCcrStore::new();+ let err = offload()+ .apply("not json at all", &CompressionContext::default(), &store)+ .expect_err("must skip non-json");+ match err {+ TransformError::Skipped { .. } => {}+ _ => panic!("expected Skipped, got {err:?}"),+ }+ }++ #[test]+ fn apply_propagates_query_anchors_into_smart_crusher() {+ // Construct a tabular array where one row has an anchor that+ // matches the query. SmartCrusher should be biased to keep+ // that row. We don't assert exactly what survives — just that+ // it ran with the query in scope.+ let mut s = String::from("[");+ for i in 0..50 {+ if i > 0 {+ s.push(',');+ }+ let name = if i == 17 {+ "needle".to_string()+ } else {+ format!("hay-{i}")+ };+ s.push_str(&format!(+ "{{\"id\":{i},\"name\":\"{name}\",\"score\":{}}}",+ i % 7+ ));+ }+ s.push(']');+ let store = InMemoryCcrStore::new();+ let ctx = CompressionContext::with_query("needle");+ let r = offload()+ .apply(&s, &ctx, &store)+ .expect("crusher should run");+ // Output should reference "needle" — SmartCrusher's anchor logic+ // should keep that row.+ assert!(+ r.output.contains("needle"),+ "anchor row should survive crush"+ );+ }++ #[test]+ fn cache_key_is_stable_across_calls_for_same_input() {+ let arr = build_tabular_array(100);+ let store_a = InMemoryCcrStore::new();+ let store_b = InMemoryCcrStore::new();+ let r_a = offload()+ .apply(&arr, &CompressionContext::default(), &store_a)+ .expect("ok");+ let r_b = offload()+ .apply(&arr, &CompressionContext::default(), &store_b)+ .expect("ok");+ assert_eq!(+ r_a.cache_key, r_b.cache_key,+ "cache_key should be a deterministic hash of input"+ );+ }++ #[test]+ fn count_row_separators_handles_compact_and_pretty() {+ assert_eq!(count_row_separators(""), 0);+ assert_eq!(count_row_separators("[]"), 0);+ assert_eq!(count_row_separators(r#"[{"a":1}]"#), 0);+ assert_eq!(count_row_separators(r#"[{"a":1},{"a":2}]"#), 1);+ assert_eq!(count_row_separators(r#"[{"a":1}, {"a":2}, {"a":3}]"#), 2);+ }+}diff --git a/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs b/crates/headroom-core/src/transforms/pipeline/offloads/mod.rsindex f80948a7d..cf01bb1cd 100644--- a/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs+++ b/crates/headroom-core/src/transforms/pipeline/offloads/mod.rs@@ -19,11 +19,13 @@pub mod diff_noise;pub mod diff_offload;+pub mod json_offload;pub mod log_offload;pub mod search_offload;pub use diff_noise::DiffNoise;pub use diff_offload::DiffOffload;+pub use json_offload::JsonOffload;pub use log_offload::LogOffload;// `SearchOffload` is intentionally NOT re-exported here. The// orchestrator-default registration omits it; keep the type accessiblediff --git a/crates/headroom-core/src/transforms/pipeline/orchestrator.rs b/crates/headroom-core/src/transforms/pipeline/orchestrator.rsindex da76ed8c7..69df00f4b 100644--- a/crates/headroom-core/src/transforms/pipeline/orchestrator.rs+++ b/crates/headroom-core/src/transforms/pipeline/orchestrator.rs@@ -332,7 +332,7 @@ mod tests {use super::*;use crate::ccr::InMemoryCcrStore;use crate::transforms::pipeline::offloads::{- DiffNoise, DiffOffload, LogOffload, SearchOffload,+ DiffNoise, DiffOffload, JsonOffload, LogOffload, SearchOffload,};use crate::transforms::pipeline::reformats::{JsonMinifier, LogTemplate};use crate::transforms::pipeline::traits::{OffloadOutput, ReformatOutput};@@ -768,6 +768,59 @@ mod tests {assert!(!r.cache_keys.is_empty());}+ #[test]+ fn end_to_end_json_minifier_then_json_offload_on_tabular_array() {+ // JsonMinifier (lossless reformat) runs first to strip+ // pretty-printing. If the array is large enough JsonOffload+ // (SmartCrusher wrapper) then engages.+ let cfg = PipelineConfig::default();+ let p = CompressionPipeline::builder()+ .with_reformat(JsonMinifier)+ .with_offload(JsonOffload::new(cfg.offload.json))+ .with_config(cfg)+ .build();+ let s = store();+ // Pretty-printed 200-row tabular array.+ let mut input = String::from("[\n");+ for i in 0..200 {+ if i > 0 {+ input.push_str(",\n");+ }+ input.push_str(&format!(+ " {{\"id\": {i}, \"name\": \"event-{i}\", \"value\": {}}}",+ i * 100+ ));+ }+ input.push_str("\n]");+ let r = p.run(&input, ContentType::JsonArray, &ctx(), &s);+ assert!(r.bytes_saved > 0);+ assert!(+ r.steps_applied.iter().any(|n| n == "json_offload"),+ "json_offload must engage on 200-row tabular array, got {:?}",+ r.steps_applied+ );+ assert!(!r.cache_keys.is_empty());+ // Original recoverable through the orchestrator's store.+ let key = r.cache_keys.last().unwrap();+ assert!(s.get(key).is_some(), "wrapper hash must resolve in store");+ }++ #[test]+ fn end_to_end_json_offload_skipped_for_small_array() {+ let cfg = PipelineConfig::default();+ let p = CompressionPipeline::builder()+ .with_offload(JsonOffload::new(cfg.offload.json))+ .with_config(cfg)+ .build();+ let s = store();+ // 3 rows — below default min_array_rows=5. Estimator returns 0+ // → orchestrator skips without calling SmartCrusher.+ let input = r#"[{"id":1,"v":1},{"id":2,"v":2},{"id":3,"v":3}]"#;+ let r = p.run(input, ContentType::JsonArray, &ctx(), &s);+ assert!(r.steps_applied.is_empty());+ assert_eq!(s.len(), 0);+ }+#[test]fn end_to_end_real_log_with_template_run_collapses_then_passes_through() {// Just LogTemplate (no LogOffload) — verify reformat alone