Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Marker-free lossless_only mode + gate opaque-blob CCR markers behind enable_ccr_marker (#1129)
Nadia Ujovich7c93c50c2cfb
diff --git a/crates/headroom-core/src/transforms/smart_crusher/config.rs b/crates/headroom-core/src/transforms/smart_crusher/config.rsindex 8096b7624..69956e9e4 100644--- a/crates/headroom-core/src/transforms/smart_crusher/config.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/config.rs@@ -80,6 +80,13 @@ pub struct SmartCrusherConfig {/// still emit always; they have no Python equivalent and no/// production caller has asked for them to be suppressed.pub enable_ccr_marker: bool,+ /// Strict lossless mode. When `true`, lossless tabular compaction+ /// still applies, but any path that would otherwise need a CCR+ /// marker — the lossy row-drop sentinel AND opaque-blob offload —+ /// leaves the content uncompacted instead. The result is always+ /// marker-free and byte-recoverable: rows are never dropped and+ /// opaque cells render inline. Default `false` (markers allowed).+ pub lossless_only: bool,/// Compaction heuristic: a field is "core" if it appears in at/// least this fraction of rows. Mirrors/// `CompactConfig::core_field_fraction`. Default 0.8.@@ -103,6 +110,22 @@ pub struct SmartCrusherConfig {pub compaction_max_buckets: usize,}+impl SmartCrusherConfig {+ /// Whether opaque blobs should be offloaded to a `<<ccr:…>>` marker.+ ///+ /// Single source of truth for the opaque-marker gate. Markers are+ /// emitted only when CCR markers are enabled AND strict lossless+ /// mode is off — `lossless_only` forbids any offload because the+ /// marker would break the marker-free / byte-recoverable guarantee.+ /// Both compaction-stage construction (`new` /+ /// `with_compaction_format`) and the top-level `process_string` path+ /// derive `ClassifyConfig::emit_opaque_markers` from this method so+ /// the three call sites can never drift apart.+ pub fn opaque_markers_enabled(&self) -> bool {+ self.enable_ccr_marker && !self.lossless_only+ }+}+impl Default for SmartCrusherConfig {fn default() -> Self {// These defaults must match smart_crusher.py:934-957 byte-for-byte.@@ -127,6 +150,7 @@ impl Default for SmartCrusherConfig {relevance_threshold: 0.3,lossless_min_savings_ratio: 0.15,enable_ccr_marker: true,+ lossless_only: false,compaction_core_field_fraction: 0.8,compaction_heterogeneous_core_ratio: 0.6,compaction_max_flatten_inner_keys: 6,@@ -164,6 +188,7 @@ mod tests {assert_eq!(c.relevance_threshold, 0.3);assert_eq!(c.lossless_min_savings_ratio, 0.15);assert!(c.enable_ccr_marker);+ assert!(!c.lossless_only);assert_eq!(c.compaction_core_field_fraction, 0.8);assert_eq!(c.compaction_heterogeneous_core_ratio, 0.6);assert_eq!(c.compaction_max_flatten_inner_keys, 6);diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rsindex 6af0fdc15..0c45f163a 100644--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs@@ -167,7 +167,7 @@ impl SmartCrusher {// the row-drop path), so `enable_ccr_marker=false` yields// marker-free, lossless output. Fixes #1091.classify: ClassifyConfig {- emit_opaque_markers: config.enable_ccr_marker,+ emit_opaque_markers: config.opaque_markers_enabled(),..ClassifyConfig::default()},..CompactConfig::default()@@ -202,7 +202,8 @@ impl SmartCrusher {/// `"markdown-kv"`). `None` for unknown names — callers own the/// fallback/error policy. `"csv-schema"` is equivalent to `new`.pub fn with_compaction_format(config: SmartCrusherConfig, format_name: &str) -> Option<Self> {- let stage = CompactionStage::from_format_name(format_name)?;+ let mut stage = CompactionStage::from_format_name(format_name)?;+ stage.config.classify.emit_opaque_markers = config.opaque_markers_enabled();Some(SmartCrusherBuilder::new(config).with_default_oss_setup()@@ -605,7 +606,7 @@ impl SmartCrusher {// Gated by `enable_ccr_marker` so disabling markers stays lossless// here too (#1091).let cfg = ClassifyConfig {- emit_opaque_markers: self.config.enable_ccr_marker,+ emit_opaque_markers: self.config.opaque_markers_enabled(),..ClassifyConfig::default()};if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.to_string()), &cfg) {@@ -698,6 +699,24 @@ impl SmartCrusher {}}+ // ── Strict lossless-only mode ──+ //+ // The lossless attempt above either shipped or didn't. Either way+ // `lossless_only` forbids the lossy row-drop fallback: dropping+ // rows needs a CCR marker to stay recoverable, and the whole+ // point of this mode is a marker-free, byte-recoverable result.+ // Leave the array uncompacted instead.+ if self.config.lossless_only {+ return CrushArrayResult {+ items: items.to_vec(),+ strategy_info: "lossless_only:uncompacted".to_string(),+ ccr_hash: None,+ dropped_summary: String::new(),+ compacted: None,+ compaction_kind: None,+ };+ }+// ── Lossy path: compress inline + cache full original via CCR ──//// The runtime caller (PyO3 bridge / proxy server) is expected@@ -705,6 +724,21 @@ impl SmartCrusher {// tool can serve dropped rows back to the LLM on demand.// **No data is lost** — "lossy" here means "compressed view// inline; full payload retrievable via CCR cache."+ //+ // Load-bearing invariant: a `lossless_only` crusher MUST NOT+ // reach this point — the early return above guarantees it. The+ // Python per-call override (`crush(..., lossless_only=True)`)+ // relies on this: it swaps in a separate Rust crusher whose CCR+ // store stays empty precisely because no lossless_only run ever+ // executes the store write below. If that early return is ever+ // removed, the alternate crusher's store would diverge and+ // retrieval could resolve markers the prompt can't reference.+ debug_assert!(+ !self.config.lossless_only,+ "lossy path reached under lossless_only — the early return \+ above must keep this codepath (and its CCR store write) \+ unreachable in strict lossless mode",+ );let effective_max_items = adaptive_k;let analysis = self.analyzer.analyze_array(items);@@ -1678,4 +1712,147 @@ mod tests {"default should write to ccr_store");}++ #[test]+ fn enable_ccr_marker_false_suppresses_opaque_markers() {+ // Opaque-blob path symmetry. A long string cell normally renders+ // as a `<<ccr:HASH,kind,size>>` marker in the lossless table.+ // With `enable_ccr_marker = false` it must render inline instead,+ // so no configuration leaks markers into a "lossless-only" prompt.+ let rows: Vec<Value> = (0..10)+ .map(|i| json!({"path": "a.py", "line": i, "content": "x".repeat(300)}))+ .collect();++ // ratio 0.0 forces the lossless table to ship, exercising the+ // compactor's opaque arm directly (not the lossy row-drop path).+ let off = SmartCrusher::new(SmartCrusherConfig {+ lossless_min_savings_ratio: 0.0,+ enable_ccr_marker: false,+ ..SmartCrusherConfig::default()+ });+ let rendered_off = off+ .crush_array(&rows, "", 1.0)+ .compacted+ .expect("lossless table should ship at ratio 0.0");+ assert!(+ !rendered_off.contains("<<ccr:"),+ "opaque marker leaked despite enable_ccr_marker=false: {rendered_off}"+ );+ assert!(+ rendered_off.contains(&"x".repeat(300)),+ "blob should be inline when markers are off: {rendered_off}"+ );++ // Default (markers on) still emits the opaque marker — the gate+ // is opt-out, not opt-in.+ let on = SmartCrusher::new(SmartCrusherConfig {+ lossless_min_savings_ratio: 0.0,+ ..SmartCrusherConfig::default()+ });+ let rendered_on = on+ .crush_array(&rows, "", 1.0)+ .compacted+ .expect("lossless table should ship at ratio 0.0");+ assert!(+ rendered_on.contains("<<ccr:"),+ "default should still emit the opaque marker: {rendered_on}"+ );+ }++ // ---------- lossless_only mode (PR part 2) ----------++ #[test]+ fn lossless_only_leaves_array_uncompacted_instead_of_dropping() {+ // When the lossless table can't win (forced via ratio 0.99),+ // lossless_only must NOT fall through to the lossy row-drop path.+ // The array passes through untouched, so it is marker-free and+ // byte-recoverable (every original row is preserved verbatim).+ let rows: Vec<Value> = (0..50)+ .map(|i| json!({"path": "a.py", "line": i, "content": "x".repeat(300)}))+ .collect();++ let crusher = SmartCrusher::new(SmartCrusherConfig {+ lossless_min_savings_ratio: 0.99, // force the would-be-lossy path+ lossless_only: true,+ ..SmartCrusherConfig::default()+ });+ let result = crusher.crush_array(&rows, "", 1.0);++ assert_eq!(result.items, rows, "lossless_only must not drop rows");+ assert!(result.ccr_hash.is_none(), "no hash under lossless_only");+ assert!(+ result.dropped_summary.is_empty(),+ "no drop sentinel under lossless_only: {:?}",+ result.dropped_summary+ );+ assert!(+ result.compacted.is_none(),+ "nothing shipped, nothing dropped"+ );+ }++ #[test]+ fn lossless_only_inlines_opaque_blobs_when_table_ships() {+ // When the lossless table DOES win, opaque cells render inline+ // (no marker) because lossless_only suppresses opaque offload.+ let rows: Vec<Value> = (0..10)+ .map(|i| json!({"path": "a.py", "line": i, "content": "x".repeat(300)}))+ .collect();+ let crusher = SmartCrusher::new(SmartCrusherConfig {+ lossless_min_savings_ratio: 0.0, // table ships+ lossless_only: true,+ ..SmartCrusherConfig::default()+ });+ let rendered = crusher+ .crush_array(&rows, "", 1.0)+ .compacted+ .expect("table should ship at ratio 0.0");+ assert!(+ !rendered.contains("<<ccr:"),+ "opaque marker leaked under lossless_only: {rendered}"+ );+ assert!(+ rendered.contains(&"x".repeat(300)),+ "blob should be inline under lossless_only: {rendered}"+ );+ }++ #[test]+ fn lossless_only_never_writes_to_ccr_store() {+ // Load-bearing invariant for the Python per-call override: a+ // lossless_only crusher MUST NOT write to the CCR store. Force+ // the would-be-lossy row-drop path (ratio 0.99) and assert the+ // store does not grow. This pins the early-return guard that the+ // `debug_assert` in `crush_array` documents — if that return is+ // ever removed, this test (and the alternate-crusher design)+ // breaks loudly.+ use crate::ccr::InMemoryCcrStore;+ use crate::transforms::smart_crusher::SmartCrusherBuilder;+ use std::sync::Arc;++ let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());+ let cfg = SmartCrusherConfig {+ lossless_min_savings_ratio: 0.99, // force the would-be-lossy path+ lossless_only: true,+ ..SmartCrusherConfig::default()+ };+ let c = SmartCrusherBuilder::new(cfg)+ .with_ccr_store(Arc::clone(&store))+ .build();+ let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();++ let store_len_before = store.len();+ let result = c.crush_array(&items, "", 1.0);++ assert_eq!(+ result.items, items,+ "lossless_only must keep every row (no drop)"+ );+ assert!(result.ccr_hash.is_none(), "no hash under lossless_only");+ assert_eq!(+ store.len(),+ store_len_before,+ "ccr_store grew under lossless_only — invariant violated"+ );+ }}diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rsindex 97c902449..b528362e5 100644--- a/crates/headroom-py/src/lib.rs+++ b/crates/headroom-py/src/lib.rs@@ -18,7 +18,9 @@ use std::collections::BTreeMap;use headroom_core::signals::{ImportanceCategory, ImportanceContext, KeywordDetector, KeywordRegistry, LineImportanceDetector,};-use headroom_core::transforms::smart_crusher::compaction::DocumentCompactor;+use headroom_core::transforms::smart_crusher::compaction::{+ ClassifyConfig, CompactConfig, DocumentCompactor,+};use headroom_core::transforms::smart_crusher::{CrushResult as RustCrushResult, SmartCrusher as RustSmartCrusher,SmartCrusherConfig as RustSmartCrusherConfig,@@ -475,6 +477,7 @@ impl PySmartCrusherConfig {relevance_threshold = 0.3,lossless_min_savings_ratio = 0.15,enable_ccr_marker = true,+ lossless_only = false,compaction_core_field_fraction = 0.8,compaction_heterogeneous_core_ratio = 0.6,compaction_max_flatten_inner_keys = 6,@@ -501,6 +504,7 @@ impl PySmartCrusherConfig {relevance_threshold: f64,lossless_min_savings_ratio: f64,enable_ccr_marker: bool,+ lossless_only: bool,compaction_core_field_fraction: f64,compaction_heterogeneous_core_ratio: f64,compaction_max_flatten_inner_keys: usize,@@ -527,6 +531,7 @@ impl PySmartCrusherConfig {relevance_threshold,lossless_min_savings_ratio,enable_ccr_marker,+ lossless_only,compaction_core_field_fraction,compaction_heterogeneous_core_ratio,compaction_max_flatten_inner_keys,@@ -605,6 +610,10 @@ impl PySmartCrusherConfig {self.inner.enable_ccr_marker}#[getter]+ fn lossless_only(&self) -> bool {+ self.inner.lossless_only+ }+ #[getter]fn lossless_min_savings_ratio(&self) -> f64 {self.inner.lossless_min_savings_ratio}@@ -854,7 +863,13 @@ impl PySmartCrusher {py.allow_threads(|| {let parsed: serde_json::Value = serde_json::from_str(&doc_json).unwrap_or_else(|e| panic!("doc_json must be JSON: {e}"));- let mut dc = DocumentCompactor::new();+ let mut dc = DocumentCompactor::new().with_config(CompactConfig {+ classify: ClassifyConfig {+ emit_opaque_markers: self.inner.config.opaque_markers_enabled(),+ ..ClassifyConfig::default()+ },+ ..CompactConfig::default()+ });if let Some(store) = self.inner.ccr_store() {dc = dc.with_ccr_store(store.clone());}diff --git a/headroom/config.py b/headroom/config.pyindex 0456f03b5..8dc3f35a8 100644--- a/headroom/config.py+++ b/headroom/config.py@@ -428,6 +428,14 @@ class SmartCrusherConfig:# transforms-level dataclass.lossless_min_savings_ratio: float = 0.15+ # Strict lossless mode. When True, lossless tabular compaction still+ # applies, but any path that would emit a CCR marker (lossy row-drop+ # OR opaque-blob offload) leaves the content uncompacted instead, so+ # the output is always marker-free and byte-recoverable. Mirrors the+ # Rust default. See also `CCRConfig` — with this on, no `<<ccr:…>>`+ # markers are produced regardless of CCR settings.+ lossless_only: bool = False+# Compaction heuristics (mirror Rust CompactConfig). A field is "core"# if present in at least this fraction of rows; arrays whose key sets# are mostly non-core are bucketed by a discriminator instead.diff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex f5dea7796..26d79b869 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -632,6 +632,15 @@ class HeadroomProxy(# to PASSTHROUGH instead of the default KOMPRESS fallback strategy.if config.disable_kompress_fallback:router_config.fallback_strategy = CompressionStrategy.PASSTHROUGH+ # `HEADROOM_LOSSLESS_ONLY=1` routes SmartCrusher through strict+ # marker-free mode: lossless tabular compaction still applies, but+ # any path that would emit a `<<ccr:…>>` marker (row-drop or+ # opaque-blob offload) leaves the content uncompacted instead — so+ # the session needs no CCR retrieval round-trips to stay recoverable.+ if "HEADROOM_LOSSLESS_ONLY" in os.environ:+ router_config.smart_crusher_lossless_only = _get_env_bool(+ "HEADROOM_LOSSLESS_ONLY", False+ )# A non-None exclude_tools replaces DEFAULT_EXCLUDE_TOOLS in# ContentRouter, so merge rather than assign.if config.exclude_tools:diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 702f0a1c2..9386e0497 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -689,6 +689,11 @@ class ContentRouterConfig:ccr_inject_marker: bool = True # Add retrieval markers to compressed contentsmart_crusher_max_items_after_crush: int | None = Nonesmart_crusher_with_compaction: bool = True+ # Strict lossless-only mode for SmartCrusher. None → leave the+ # crusher config's own value untouched; True/False force it. Wired+ # from the proxy's `HEADROOM_LOSSLESS_ONLY` env var so a real session+ # can run marker-free without constructing the crusher by hand.+ smart_crusher_lossless_only: bool | None = None# Tag protection: preserve custom/workflow XML tags from text compression.# When False (default), entire <custom-tag>content</custom-tag> blocks are@@ -1872,6 +1877,8 @@ class ContentRouter(Transform):crusher_config.max_items_after_crush = (self.config.smart_crusher_max_items_after_crush)+ if self.config.smart_crusher_lossless_only is not None:+ crusher_config.lossless_only = self.config.smart_crusher_lossless_onlyself._smart_crusher = SmartCrusher(config=crusher_config,ccr_config=ccr_config,diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.pyindex fe283e177..9823040e4 100644--- a/headroom/transforms/smart_crusher.py+++ b/headroom/transforms/smart_crusher.py@@ -187,6 +187,13 @@ class SmartCrusherConfig:# and KV experiments — KV repeats field names per row, so it clears# the gate less often than CSV.lossless_min_savings_ratio: float = 0.15+ # Strict lossless mode. When True, lossless tabular compaction still+ # applies, but any path that would otherwise emit a CCR marker — the+ # lossy row-drop sentinel AND opaque-blob offload — leaves the content+ # uncompacted instead. The output is always marker-free and fully+ # byte-recoverable: rows are never dropped and opaque cells render+ # inline. Default False (markers allowed). Mirrors the Rust default.+ lossless_only: bool = False# Compaction heuristics (mirror Rust CompactConfig; see# crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs).@@ -227,6 +234,7 @@ class SmartCrusher(Transform):with_compaction: bool = True,observer: Any = None,compaction_format: str | None = None,+ lossless_only: bool | None = None,):# Hard import — no Python fallback. If the wheel is missing the# caller must build it (scripts/build_rust_extension.sh) or@@ -242,6 +250,16 @@ class SmartCrusher(Transform):cfg = config or SmartCrusherConfig()self.config = cfgself._with_compaction = with_compaction+ # Strict lossless mode. An explicit `lossless_only=` kwarg wins+ # over the config field, so callers can flip it without rebuilding+ # a whole config. `crush(..., lossless_only=...)` overrides again+ # per call. getattr fallback: callers may pass the SDK-side+ # `headroom.config.SmartCrusherConfig`, which also carries it.+ self._lossless_only = (+ bool(getattr(cfg, "lossless_only", False))+ if lossless_only is None+ else bool(lossless_only)+ )# `observer`: see `headroom.transforms.observability`. The# legacy proxy pipeline uses SmartCrusher.apply() directly# (no ContentRouter); without an observer here, those@@ -315,39 +333,46 @@ class SmartCrusher(Transform):# Build the Rust crusher with every field from the Python# config, plus the relevance_threshold default (0.3) — the# Python dataclass doesn't carry that field; it lives on- # `RelevanceScorerConfig` instead.- rust_cfg = _RustSmartCrusherConfig(- enabled=cfg.enabled,- min_items_to_analyze=cfg.min_items_to_analyze,- min_tokens_to_crush=cfg.min_tokens_to_crush,- variance_threshold=cfg.variance_threshold,- uniqueness_threshold=cfg.uniqueness_threshold,- similarity_threshold=cfg.similarity_threshold,- max_items_after_crush=cfg.max_items_after_crush,- preserve_change_points=cfg.preserve_change_points,- factor_out_constants=cfg.factor_out_constants,- include_summaries=cfg.include_summaries,- use_feedback_hints=cfg.use_feedback_hints,- toin_confidence_threshold=cfg.toin_confidence_threshold,- dedup_identical_items=cfg.dedup_identical_items,- first_fraction=cfg.first_fraction,- last_fraction=cfg.last_fraction,- relevance_threshold=0.3,- enable_ccr_marker=(+ # `RelevanceScorerConfig` instead. Kept as a kwargs dict so the+ # per-call `crush(..., lossless_only=...)` override can rebuild an+ # alternate crusher with just that one field flipped.+ self._RustSmartCrusher = _RustSmartCrusher+ self._RustSmartCrusherConfig = _RustSmartCrusherConfig+ self._rust_cfg_kwargs = {+ "enabled": cfg.enabled,+ "min_items_to_analyze": cfg.min_items_to_analyze,+ "min_tokens_to_crush": cfg.min_tokens_to_crush,+ "variance_threshold": cfg.variance_threshold,+ "uniqueness_threshold": cfg.uniqueness_threshold,+ "similarity_threshold": cfg.similarity_threshold,+ "max_items_after_crush": cfg.max_items_after_crush,+ "preserve_change_points": cfg.preserve_change_points,+ "factor_out_constants": cfg.factor_out_constants,+ "include_summaries": cfg.include_summaries,+ "use_feedback_hints": cfg.use_feedback_hints,+ "toin_confidence_threshold": cfg.toin_confidence_threshold,+ "dedup_identical_items": cfg.dedup_identical_items,+ "first_fraction": cfg.first_fraction,+ "last_fraction": cfg.last_fraction,+ "relevance_threshold": 0.3,+ "enable_ccr_marker": (self._ccr_config.enabled and self._ccr_config.inject_retrieval_marker),+ "lossless_only": self._lossless_only,# getattr fallbacks: callers may pass the structurally-similar# `headroom.config.SmartCrusherConfig` (MCP server, SDK) or a# pre-existing config object that predates these fields.- lossless_min_savings_ratio=getattr(cfg, "lossless_min_savings_ratio", 0.15),- compaction_core_field_fraction=getattr(cfg, "compaction_core_field_fraction", 0.8),- compaction_heterogeneous_core_ratio=getattr(+ "lossless_min_savings_ratio": getattr(cfg, "lossless_min_savings_ratio", 0.15),+ "compaction_core_field_fraction": getattr(cfg, "compaction_core_field_fraction", 0.8),+ "compaction_heterogeneous_core_ratio": getattr(cfg, "compaction_heterogeneous_core_ratio", 0.6),- compaction_max_flatten_inner_keys=getattr(cfg, "compaction_max_flatten_inner_keys", 6),- compaction_min_buckets=getattr(cfg, "compaction_min_buckets", 2),- compaction_max_buckets=getattr(cfg, "compaction_max_buckets", 8),- )+ "compaction_max_flatten_inner_keys": getattr(+ cfg, "compaction_max_flatten_inner_keys", 6+ ),+ "compaction_min_buckets": getattr(cfg, "compaction_min_buckets", 2),+ "compaction_max_buckets": getattr(cfg, "compaction_max_buckets", 8),+ }# Default: lossless-first compaction (PR4). Lossless wins for# cleanly tabular input where it saves ≥ 30% bytes; otherwise# falls through to the lossy path with CCR-Dropped retrieval@@ -373,24 +398,59 @@ class SmartCrusher(Transform):f"expected one of: {', '.join(_SUPPORTED_COMPACTION_FORMATS)}")self._compaction_format = resolved_format if with_compaction else None- if not with_compaction:- self._rust = _RustSmartCrusher.without_compaction(rust_cfg)- elif resolved_format == "csv-schema":+ self._resolved_compaction_format = resolved_format+ # Cache of Rust crushers keyed by lossless_only, so a per-call+ # override builds the alternate at most once.+ self._rust_by_lossless_only: dict[bool, Any] = {}+ self._rust = self._build_rust(self._lossless_only)++ def _build_rust(self, lossless_only: bool) -> Any:+ """Build (and cache) the Rust crusher for a `lossless_only` value."""+ cached = self._rust_by_lossless_only.get(lossless_only)+ if cached is not None:+ return cached+ kwargs = dict(self._rust_cfg_kwargs)+ kwargs["lossless_only"] = lossless_only+ rust_cfg = self._RustSmartCrusherConfig(**kwargs)+ if not self._with_compaction:+ rust = self._RustSmartCrusher.without_compaction(rust_cfg)+ elif self._resolved_compaction_format == "csv-schema":# Keep the `new()` constructor for the default path so its- # byte-parity coverage stays on the exact production- # codepath.- self._rust = _RustSmartCrusher(rust_cfg)+ # byte-parity coverage stays on the exact production codepath.+ rust = self._RustSmartCrusher(rust_cfg)else:- self._rust = _RustSmartCrusher.with_compaction_format(rust_cfg, resolved_format)+ rust = self._RustSmartCrusher.with_compaction_format(+ rust_cfg, self._resolved_compaction_format+ )+ self._rust_by_lossless_only[lossless_only] = rust+ return rust- def crush(self, content: str, query: str = "", bias: float = 1.0) -> CrushResult:+ def crush(+ self,+ content: str,+ query: str = "",+ bias: float = 1.0,+ lossless_only: bool | None = None,+ ) -> CrushResult:"""Crush a single JSON content string.Mirrors the retired Python method. Returns a `CrushResult`dataclass so call sites that destructure with `asdict()` keepworking.++ `lossless_only` overrides the configured strict-lossless mode for+ this call only. When `True`, the output is guaranteed marker-free+ and byte-recoverable: lossless tabular compaction still applies,+ but any path that would need a CCR marker (row-drop or+ opaque-blob offload) leaves the content uncompacted instead.+ `None` (default) uses the instance's configured value."""- r = self._rust.crush(content, query, bias)+ rust = (+ self._rust+ if lossless_only is None or bool(lossless_only) == self._lossless_only+ else self._build_rust(bool(lossless_only))+ )+ r = rust.crush(content, query, bias)# Re-attach the TOIN learning loop. The retired Python class# recorded compressions into TOIN inline; the Rust port doesn't# know about TOIN, and `ContentRouter._record_to_toin` skips@@ -1017,6 +1077,7 @@ def smart_crush_tool_output(config: SmartCrusherConfig | None = None,ccr_config: CCRConfig | None = None,with_compaction: bool = True,+ lossless_only: bool | None = None,) -> tuple[str, bool, str]:"""Compress a single tool output. Returns `(crushed, was_modified, info)`.@@ -1024,6 +1085,14 @@ def smart_crush_tool_output(Defaults to the PR4 lossless-first behavior; pass`with_compaction=False` to exercise the legacy lossy-only path(still useful for retention-property tests).++ `lossless_only=True` forces strict lossless mode: the output is+ marker-free and byte-recoverable (no row drops, opaque blobs inline)."""- crusher = SmartCrusher(config=config, ccr_config=ccr_config, with_compaction=with_compaction)+ crusher = SmartCrusher(+ config=config,+ ccr_config=ccr_config,+ with_compaction=with_compaction,+ lossless_only=lossless_only,+ )return crusher._smart_crush_content(content)diff --git a/tests/test_agent_savings.py b/tests/test_agent_savings.pyindex 38dbe2b4c..1eeae1519 100644--- a/tests/test_agent_savings.py+++ b/tests/test_agent_savings.py@@ -358,6 +358,29 @@ def test_agent_90_router_uses_ccr_sampling_not_lossless_table() -> None:assert crusher._with_compaction is False+def test_router_lossless_only_flag_reaches_crusher() -> None:+ # HEADROOM_LOSSLESS_ONLY=1 sets this field on the proxy router; it+ # must flow through to the SmartCrusher so a real proxy session runs+ # strict marker-free mode.+ router = ContentRouter(ContentRouterConfig(smart_crusher_lossless_only=True))++ crusher = router._get_smart_crusher()++ assert crusher is not None+ assert crusher._lossless_only is True+++def test_router_lossless_only_defaults_off() -> None:+ # Unset (None) must not force the flag — default crushers stay in+ # the marker-emitting mode.+ router = ContentRouter(ContentRouterConfig())++ crusher = router._get_smart_crusher()++ assert crusher is not None+ assert crusher._lossless_only is False++def test_agent_90_router_json_tool_output_reaches_target_with_needle() -> None:needle = "CRITICAL_NEEDLE_42"rows = [diff --git a/tests/test_transforms/test_smart_crusher_bugs.py b/tests/test_transforms/test_smart_crusher_bugs.pyindex 3e7aa22e5..616806d99 100644--- a/tests/test_transforms/test_smart_crusher_bugs.py+++ b/tests/test_transforms/test_smart_crusher_bugs.py@@ -124,6 +124,45 @@ class TestRecursionDepthLimit:assert isinstance(parsed, list)+class TestLosslessOnlyMode:+ """`lossless_only` produces marker-free, byte-recoverable output.++ Strict mode: lossless tabular compaction still applies, but any path+ that would need a CCR marker (lossy row-drop OR opaque-blob offload)+ leaves the content uncompacted instead — so the result is always+ marker-free and decodes back to the original input without loss.+ """++ def _droppable_rows(self) -> list[dict]:+ return [{"path": "a.py", "line": i, "content": "x" * 300} for i in range(50)]++ def test_lossless_only_is_marker_free_and_byte_recoverable(self) -> None:+ rows = self._droppable_rows()+ config = SmartCrusherConfig(+ min_items_to_analyze=3,+ min_tokens_to_crush=0,+ lossless_min_savings_ratio=0.99, # force the would-be-lossy path+ lossless_only=True,+ )+ out = SmartCrusher(config=config).crush(json.dumps(rows))+ assert "<<ccr:" not in out.compressed+ assert json.loads(out.compressed) == rows++ def test_crush_kwarg_overrides_configured_mode(self) -> None:+ # Configured non-strict, but the per-call kwarg forces strict mode+ # for this call: marker-free and fully recoverable.+ rows = self._droppable_rows()+ config = SmartCrusherConfig(+ min_items_to_analyze=3,+ min_tokens_to_crush=0,+ lossless_min_savings_ratio=0.99,+ )+ crusher = SmartCrusher(config=config)+ out = crusher.crush(json.dumps(rows), lossless_only=True)+ assert "<<ccr:" not in out.compressed+ assert json.loads(out.compressed) == rows++# Stage 3c.1 lockstep bug-fix tests previously lived here; they probed# Python helpers (`_percentile_linear`, `_detect_sequential_pattern`,# `_detect_rare_status_values`, `_compute_k_split`) that were removed