Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Marker-free lossless_only mode + gate opaque-blob CCR markers behind enable_ccr_marker (#1129)

Nadia Ujovich7c93c50c2cfb
diff --git a/crates/headroom-core/src/transforms/smart_crusher/config.rs b/crates/headroom-core/src/transforms/smart_crusher/config.rs
index 8096b7624..69956e9e4 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/config.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/config.rs
@@ -80,6 +80,13 @@ pub struct SmartCrusherConfig {
/// still emit always; they have no Python equivalent and no
/// production caller has asked for them to be suppressed.
pub enable_ccr_marker: bool,
+ /// Strict lossless mode. When `true`, lossless tabular compaction
+ /// still applies, but any path that would otherwise need a CCR
+ /// marker — the lossy row-drop sentinel AND opaque-blob offload —
+ /// leaves the content uncompacted instead. The result is always
+ /// marker-free and byte-recoverable: rows are never dropped and
+ /// opaque cells render inline. Default `false` (markers allowed).
+ pub lossless_only: bool,
/// Compaction heuristic: a field is "core" if it appears in at
/// least this fraction of rows. Mirrors
/// `CompactConfig::core_field_fraction`. Default 0.8.
@@ -103,6 +110,22 @@ pub struct SmartCrusherConfig {
pub compaction_max_buckets: usize,
}
+impl SmartCrusherConfig {
+ /// Whether opaque blobs should be offloaded to a `<<ccr:…>>` marker.
+ ///
+ /// Single source of truth for the opaque-marker gate. Markers are
+ /// emitted only when CCR markers are enabled AND strict lossless
+ /// mode is off — `lossless_only` forbids any offload because the
+ /// marker would break the marker-free / byte-recoverable guarantee.
+ /// Both compaction-stage construction (`new` /
+ /// `with_compaction_format`) and the top-level `process_string` path
+ /// derive `ClassifyConfig::emit_opaque_markers` from this method so
+ /// the three call sites can never drift apart.
+ pub fn opaque_markers_enabled(&self) -> bool {
+ self.enable_ccr_marker && !self.lossless_only
+ }
+}
+
impl Default for SmartCrusherConfig {
fn default() -> Self {
// These defaults must match smart_crusher.py:934-957 byte-for-byte.
@@ -127,6 +150,7 @@ impl Default for SmartCrusherConfig {
relevance_threshold: 0.3,
lossless_min_savings_ratio: 0.15,
enable_ccr_marker: true,
+ lossless_only: false,
compaction_core_field_fraction: 0.8,
compaction_heterogeneous_core_ratio: 0.6,
compaction_max_flatten_inner_keys: 6,
@@ -164,6 +188,7 @@ mod tests {
assert_eq!(c.relevance_threshold, 0.3);
assert_eq!(c.lossless_min_savings_ratio, 0.15);
assert!(c.enable_ccr_marker);
+ assert!(!c.lossless_only);
assert_eq!(c.compaction_core_field_fraction, 0.8);
assert_eq!(c.compaction_heterogeneous_core_ratio, 0.6);
assert_eq!(c.compaction_max_flatten_inner_keys, 6);
diff --git a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
index 6af0fdc15..0c45f163a 100644
--- a/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
+++ b/crates/headroom-core/src/transforms/smart_crusher/crusher.rs
@@ -167,7 +167,7 @@ impl SmartCrusher {
// the row-drop path), so `enable_ccr_marker=false` yields
// marker-free, lossless output. Fixes #1091.
classify: ClassifyConfig {
- emit_opaque_markers: config.enable_ccr_marker,
+ emit_opaque_markers: config.opaque_markers_enabled(),
..ClassifyConfig::default()
},
..CompactConfig::default()
@@ -202,7 +202,8 @@ impl SmartCrusher {
/// `"markdown-kv"`). `None` for unknown names — callers own the
/// fallback/error policy. `"csv-schema"` is equivalent to `new`.
pub fn with_compaction_format(config: SmartCrusherConfig, format_name: &str) -> Option<Self> {
- let stage = CompactionStage::from_format_name(format_name)?;
+ let mut stage = CompactionStage::from_format_name(format_name)?;
+ stage.config.classify.emit_opaque_markers = config.opaque_markers_enabled();
Some(
SmartCrusherBuilder::new(config)
.with_default_oss_setup()
@@ -605,7 +606,7 @@ impl SmartCrusher {
// Gated by `enable_ccr_marker` so disabling markers stays lossless
// here too (#1091).
let cfg = ClassifyConfig {
- emit_opaque_markers: self.config.enable_ccr_marker,
+ emit_opaque_markers: self.config.opaque_markers_enabled(),
..ClassifyConfig::default()
};
if let CellClass::Opaque(kind) = classify_cell(&Value::String(s.to_string()), &cfg) {
@@ -698,6 +699,24 @@ impl SmartCrusher {
}
}
+ // ── Strict lossless-only mode ──
+ //
+ // The lossless attempt above either shipped or didn't. Either way
+ // `lossless_only` forbids the lossy row-drop fallback: dropping
+ // rows needs a CCR marker to stay recoverable, and the whole
+ // point of this mode is a marker-free, byte-recoverable result.
+ // Leave the array uncompacted instead.
+ if self.config.lossless_only {
+ return CrushArrayResult {
+ items: items.to_vec(),
+ strategy_info: "lossless_only:uncompacted".to_string(),
+ ccr_hash: None,
+ dropped_summary: String::new(),
+ compacted: None,
+ compaction_kind: None,
+ };
+ }
+
// ── Lossy path: compress inline + cache full original via CCR ──
//
// The runtime caller (PyO3 bridge / proxy server) is expected
@@ -705,6 +724,21 @@ impl SmartCrusher {
// tool can serve dropped rows back to the LLM on demand.
// **No data is lost** — "lossy" here means "compressed view
// inline; full payload retrievable via CCR cache."
+ //
+ // Load-bearing invariant: a `lossless_only` crusher MUST NOT
+ // reach this point — the early return above guarantees it. The
+ // Python per-call override (`crush(..., lossless_only=True)`)
+ // relies on this: it swaps in a separate Rust crusher whose CCR
+ // store stays empty precisely because no lossless_only run ever
+ // executes the store write below. If that early return is ever
+ // removed, the alternate crusher's store would diverge and
+ // retrieval could resolve markers the prompt can't reference.
+ debug_assert!(
+ !self.config.lossless_only,
+ "lossy path reached under lossless_only — the early return \
+ above must keep this codepath (and its CCR store write) \
+ unreachable in strict lossless mode",
+ );
let effective_max_items = adaptive_k;
let analysis = self.analyzer.analyze_array(items);
@@ -1678,4 +1712,147 @@ mod tests {
"default should write to ccr_store"
);
}
+
+ #[test]
+ fn enable_ccr_marker_false_suppresses_opaque_markers() {
+ // Opaque-blob path symmetry. A long string cell normally renders
+ // as a `<<ccr:HASH,kind,size>>` marker in the lossless table.
+ // With `enable_ccr_marker = false` it must render inline instead,
+ // so no configuration leaks markers into a "lossless-only" prompt.
+ let rows: Vec<Value> = (0..10)
+ .map(|i| json!({"path": "a.py", "line": i, "content": "x".repeat(300)}))
+ .collect();
+
+ // ratio 0.0 forces the lossless table to ship, exercising the
+ // compactor's opaque arm directly (not the lossy row-drop path).
+ let off = SmartCrusher::new(SmartCrusherConfig {
+ lossless_min_savings_ratio: 0.0,
+ enable_ccr_marker: false,
+ ..SmartCrusherConfig::default()
+ });
+ let rendered_off = off
+ .crush_array(&rows, "", 1.0)
+ .compacted
+ .expect("lossless table should ship at ratio 0.0");
+ assert!(
+ !rendered_off.contains("<<ccr:"),
+ "opaque marker leaked despite enable_ccr_marker=false: {rendered_off}"
+ );
+ assert!(
+ rendered_off.contains(&"x".repeat(300)),
+ "blob should be inline when markers are off: {rendered_off}"
+ );
+
+ // Default (markers on) still emits the opaque marker — the gate
+ // is opt-out, not opt-in.
+ let on = SmartCrusher::new(SmartCrusherConfig {
+ lossless_min_savings_ratio: 0.0,
+ ..SmartCrusherConfig::default()
+ });
+ let rendered_on = on
+ .crush_array(&rows, "", 1.0)
+ .compacted
+ .expect("lossless table should ship at ratio 0.0");
+ assert!(
+ rendered_on.contains("<<ccr:"),
+ "default should still emit the opaque marker: {rendered_on}"
+ );
+ }
+
+ // ---------- lossless_only mode (PR part 2) ----------
+
+ #[test]
+ fn lossless_only_leaves_array_uncompacted_instead_of_dropping() {
+ // When the lossless table can't win (forced via ratio 0.99),
+ // lossless_only must NOT fall through to the lossy row-drop path.
+ // The array passes through untouched, so it is marker-free and
+ // byte-recoverable (every original row is preserved verbatim).
+ let rows: Vec<Value> = (0..50)
+ .map(|i| json!({"path": "a.py", "line": i, "content": "x".repeat(300)}))
+ .collect();
+
+ let crusher = SmartCrusher::new(SmartCrusherConfig {
+ lossless_min_savings_ratio: 0.99, // force the would-be-lossy path
+ lossless_only: true,
+ ..SmartCrusherConfig::default()
+ });
+ let result = crusher.crush_array(&rows, "", 1.0);
+
+ assert_eq!(result.items, rows, "lossless_only must not drop rows");
+ assert!(result.ccr_hash.is_none(), "no hash under lossless_only");
+ assert!(
+ result.dropped_summary.is_empty(),
+ "no drop sentinel under lossless_only: {:?}",
+ result.dropped_summary
+ );
+ assert!(
+ result.compacted.is_none(),
+ "nothing shipped, nothing dropped"
+ );
+ }
+
+ #[test]
+ fn lossless_only_inlines_opaque_blobs_when_table_ships() {
+ // When the lossless table DOES win, opaque cells render inline
+ // (no marker) because lossless_only suppresses opaque offload.
+ let rows: Vec<Value> = (0..10)
+ .map(|i| json!({"path": "a.py", "line": i, "content": "x".repeat(300)}))
+ .collect();
+ let crusher = SmartCrusher::new(SmartCrusherConfig {
+ lossless_min_savings_ratio: 0.0, // table ships
+ lossless_only: true,
+ ..SmartCrusherConfig::default()
+ });
+ let rendered = crusher
+ .crush_array(&rows, "", 1.0)
+ .compacted
+ .expect("table should ship at ratio 0.0");
+ assert!(
+ !rendered.contains("<<ccr:"),
+ "opaque marker leaked under lossless_only: {rendered}"
+ );
+ assert!(
+ rendered.contains(&"x".repeat(300)),
+ "blob should be inline under lossless_only: {rendered}"
+ );
+ }
+
+ #[test]
+ fn lossless_only_never_writes_to_ccr_store() {
+ // Load-bearing invariant for the Python per-call override: a
+ // lossless_only crusher MUST NOT write to the CCR store. Force
+ // the would-be-lossy row-drop path (ratio 0.99) and assert the
+ // store does not grow. This pins the early-return guard that the
+ // `debug_assert` in `crush_array` documents — if that return is
+ // ever removed, this test (and the alternate-crusher design)
+ // breaks loudly.
+ use crate::ccr::InMemoryCcrStore;
+ use crate::transforms::smart_crusher::SmartCrusherBuilder;
+ use std::sync::Arc;
+
+ let store: Arc<dyn CcrStore> = Arc::new(InMemoryCcrStore::new());
+ let cfg = SmartCrusherConfig {
+ lossless_min_savings_ratio: 0.99, // force the would-be-lossy path
+ lossless_only: true,
+ ..SmartCrusherConfig::default()
+ };
+ let c = SmartCrusherBuilder::new(cfg)
+ .with_ccr_store(Arc::clone(&store))
+ .build();
+ let items: Vec<Value> = (0..50).map(|_| json!({"status": "ok"})).collect();
+
+ let store_len_before = store.len();
+ let result = c.crush_array(&items, "", 1.0);
+
+ assert_eq!(
+ result.items, items,
+ "lossless_only must keep every row (no drop)"
+ );
+ assert!(result.ccr_hash.is_none(), "no hash under lossless_only");
+ assert_eq!(
+ store.len(),
+ store_len_before,
+ "ccr_store grew under lossless_only — invariant violated"
+ );
+ }
}
diff --git a/crates/headroom-py/src/lib.rs b/crates/headroom-py/src/lib.rs
index 97c902449..b528362e5 100644
--- a/crates/headroom-py/src/lib.rs
+++ b/crates/headroom-py/src/lib.rs
@@ -18,7 +18,9 @@ use std::collections::BTreeMap;
use headroom_core::signals::{
ImportanceCategory, ImportanceContext, KeywordDetector, KeywordRegistry, LineImportanceDetector,
};
-use headroom_core::transforms::smart_crusher::compaction::DocumentCompactor;
+use headroom_core::transforms::smart_crusher::compaction::{
+ ClassifyConfig, CompactConfig, DocumentCompactor,
+};
use headroom_core::transforms::smart_crusher::{
CrushResult as RustCrushResult, SmartCrusher as RustSmartCrusher,
SmartCrusherConfig as RustSmartCrusherConfig,
@@ -475,6 +477,7 @@ impl PySmartCrusherConfig {
relevance_threshold = 0.3,
lossless_min_savings_ratio = 0.15,
enable_ccr_marker = true,
+ lossless_only = false,
compaction_core_field_fraction = 0.8,
compaction_heterogeneous_core_ratio = 0.6,
compaction_max_flatten_inner_keys = 6,
@@ -501,6 +504,7 @@ impl PySmartCrusherConfig {
relevance_threshold: f64,
lossless_min_savings_ratio: f64,
enable_ccr_marker: bool,
+ lossless_only: bool,
compaction_core_field_fraction: f64,
compaction_heterogeneous_core_ratio: f64,
compaction_max_flatten_inner_keys: usize,
@@ -527,6 +531,7 @@ impl PySmartCrusherConfig {
relevance_threshold,
lossless_min_savings_ratio,
enable_ccr_marker,
+ lossless_only,
compaction_core_field_fraction,
compaction_heterogeneous_core_ratio,
compaction_max_flatten_inner_keys,
@@ -605,6 +610,10 @@ impl PySmartCrusherConfig {
self.inner.enable_ccr_marker
}
#[getter]
+ fn lossless_only(&self) -> bool {
+ self.inner.lossless_only
+ }
+ #[getter]
fn lossless_min_savings_ratio(&self) -> f64 {
self.inner.lossless_min_savings_ratio
}
@@ -854,7 +863,13 @@ impl PySmartCrusher {
py.allow_threads(|| {
let parsed: serde_json::Value = serde_json::from_str(&doc_json)
.unwrap_or_else(|e| panic!("doc_json must be JSON: {e}"));
- let mut dc = DocumentCompactor::new();
+ let mut dc = DocumentCompactor::new().with_config(CompactConfig {
+ classify: ClassifyConfig {
+ emit_opaque_markers: self.inner.config.opaque_markers_enabled(),
+ ..ClassifyConfig::default()
+ },
+ ..CompactConfig::default()
+ });
if let Some(store) = self.inner.ccr_store() {
dc = dc.with_ccr_store(store.clone());
}
diff --git a/headroom/config.py b/headroom/config.py
index 0456f03b5..8dc3f35a8 100644
--- a/headroom/config.py
+++ b/headroom/config.py
@@ -428,6 +428,14 @@ class SmartCrusherConfig:
# transforms-level dataclass.
lossless_min_savings_ratio: float = 0.15
+ # Strict lossless mode. When True, lossless tabular compaction still
+ # applies, but any path that would emit a CCR marker (lossy row-drop
+ # OR opaque-blob offload) leaves the content uncompacted instead, so
+ # the output is always marker-free and byte-recoverable. Mirrors the
+ # Rust default. See also `CCRConfig` — with this on, no `<<ccr:…>>`
+ # markers are produced regardless of CCR settings.
+ lossless_only: bool = False
+
# Compaction heuristics (mirror Rust CompactConfig). A field is "core"
# if present in at least this fraction of rows; arrays whose key sets
# are mostly non-core are bucketed by a discriminator instead.
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index f5dea7796..26d79b869 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -632,6 +632,15 @@ class HeadroomProxy(
# to PASSTHROUGH instead of the default KOMPRESS fallback strategy.
if config.disable_kompress_fallback:
router_config.fallback_strategy = CompressionStrategy.PASSTHROUGH
+ # `HEADROOM_LOSSLESS_ONLY=1` routes SmartCrusher through strict
+ # marker-free mode: lossless tabular compaction still applies, but
+ # any path that would emit a `<<ccr:…>>` marker (row-drop or
+ # opaque-blob offload) leaves the content uncompacted instead — so
+ # the session needs no CCR retrieval round-trips to stay recoverable.
+ if "HEADROOM_LOSSLESS_ONLY" in os.environ:
+ router_config.smart_crusher_lossless_only = _get_env_bool(
+ "HEADROOM_LOSSLESS_ONLY", False
+ )
# A non-None exclude_tools replaces DEFAULT_EXCLUDE_TOOLS in
# ContentRouter, so merge rather than assign.
if config.exclude_tools:
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 702f0a1c2..9386e0497 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -689,6 +689,11 @@ class ContentRouterConfig:
ccr_inject_marker: bool = True # Add retrieval markers to compressed content
smart_crusher_max_items_after_crush: int | None = None
smart_crusher_with_compaction: bool = True
+ # Strict lossless-only mode for SmartCrusher. None → leave the
+ # crusher config's own value untouched; True/False force it. Wired
+ # from the proxy's `HEADROOM_LOSSLESS_ONLY` env var so a real session
+ # can run marker-free without constructing the crusher by hand.
+ smart_crusher_lossless_only: bool | None = None
# Tag protection: preserve custom/workflow XML tags from text compression.
# When False (default), entire <custom-tag>content</custom-tag> blocks are
@@ -1872,6 +1877,8 @@ class ContentRouter(Transform):
crusher_config.max_items_after_crush = (
self.config.smart_crusher_max_items_after_crush
)
+ if self.config.smart_crusher_lossless_only is not None:
+ crusher_config.lossless_only = self.config.smart_crusher_lossless_only
self._smart_crusher = SmartCrusher(
config=crusher_config,
ccr_config=ccr_config,
diff --git a/headroom/transforms/smart_crusher.py b/headroom/transforms/smart_crusher.py
index fe283e177..9823040e4 100644
--- a/headroom/transforms/smart_crusher.py
+++ b/headroom/transforms/smart_crusher.py
@@ -187,6 +187,13 @@ class SmartCrusherConfig:
# and KV experiments — KV repeats field names per row, so it clears
# the gate less often than CSV.
lossless_min_savings_ratio: float = 0.15
+ # Strict lossless mode. When True, lossless tabular compaction still
+ # applies, but any path that would otherwise emit a CCR marker — the
+ # lossy row-drop sentinel AND opaque-blob offload — leaves the content
+ # uncompacted instead. The output is always marker-free and fully
+ # byte-recoverable: rows are never dropped and opaque cells render
+ # inline. Default False (markers allowed). Mirrors the Rust default.
+ lossless_only: bool = False
# Compaction heuristics (mirror Rust CompactConfig; see
# crates/headroom-core/src/transforms/smart_crusher/compaction/compactor.rs).
@@ -227,6 +234,7 @@ class SmartCrusher(Transform):
with_compaction: bool = True,
observer: Any = None,
compaction_format: str | None = None,
+ lossless_only: bool | None = None,
):
# Hard import — no Python fallback. If the wheel is missing the
# caller must build it (scripts/build_rust_extension.sh) or
@@ -242,6 +250,16 @@ class SmartCrusher(Transform):
cfg = config or SmartCrusherConfig()
self.config = cfg
self._with_compaction = with_compaction
+ # Strict lossless mode. An explicit `lossless_only=` kwarg wins
+ # over the config field, so callers can flip it without rebuilding
+ # a whole config. `crush(..., lossless_only=...)` overrides again
+ # per call. getattr fallback: callers may pass the SDK-side
+ # `headroom.config.SmartCrusherConfig`, which also carries it.
+ self._lossless_only = (
+ bool(getattr(cfg, "lossless_only", False))
+ if lossless_only is None
+ else bool(lossless_only)
+ )
# `observer`: see `headroom.transforms.observability`. The
# legacy proxy pipeline uses SmartCrusher.apply() directly
# (no ContentRouter); without an observer here, those
@@ -315,39 +333,46 @@ class SmartCrusher(Transform):
# Build the Rust crusher with every field from the Python
# config, plus the relevance_threshold default (0.3) — the
# Python dataclass doesn't carry that field; it lives on
- # `RelevanceScorerConfig` instead.
- rust_cfg = _RustSmartCrusherConfig(
- enabled=cfg.enabled,
- min_items_to_analyze=cfg.min_items_to_analyze,
- min_tokens_to_crush=cfg.min_tokens_to_crush,
- variance_threshold=cfg.variance_threshold,
- uniqueness_threshold=cfg.uniqueness_threshold,
- similarity_threshold=cfg.similarity_threshold,
- max_items_after_crush=cfg.max_items_after_crush,
- preserve_change_points=cfg.preserve_change_points,
- factor_out_constants=cfg.factor_out_constants,
- include_summaries=cfg.include_summaries,
- use_feedback_hints=cfg.use_feedback_hints,
- toin_confidence_threshold=cfg.toin_confidence_threshold,
- dedup_identical_items=cfg.dedup_identical_items,
- first_fraction=cfg.first_fraction,
- last_fraction=cfg.last_fraction,
- relevance_threshold=0.3,
- enable_ccr_marker=(
+ # `RelevanceScorerConfig` instead. Kept as a kwargs dict so the
+ # per-call `crush(..., lossless_only=...)` override can rebuild an
+ # alternate crusher with just that one field flipped.
+ self._RustSmartCrusher = _RustSmartCrusher
+ self._RustSmartCrusherConfig = _RustSmartCrusherConfig
+ self._rust_cfg_kwargs = {
+ "enabled": cfg.enabled,
+ "min_items_to_analyze": cfg.min_items_to_analyze,
+ "min_tokens_to_crush": cfg.min_tokens_to_crush,
+ "variance_threshold": cfg.variance_threshold,
+ "uniqueness_threshold": cfg.uniqueness_threshold,
+ "similarity_threshold": cfg.similarity_threshold,
+ "max_items_after_crush": cfg.max_items_after_crush,
+ "preserve_change_points": cfg.preserve_change_points,
+ "factor_out_constants": cfg.factor_out_constants,
+ "include_summaries": cfg.include_summaries,
+ "use_feedback_hints": cfg.use_feedback_hints,
+ "toin_confidence_threshold": cfg.toin_confidence_threshold,
+ "dedup_identical_items": cfg.dedup_identical_items,
+ "first_fraction": cfg.first_fraction,
+ "last_fraction": cfg.last_fraction,
+ "relevance_threshold": 0.3,
+ "enable_ccr_marker": (
self._ccr_config.enabled and self._ccr_config.inject_retrieval_marker
),
+ "lossless_only": self._lossless_only,
# getattr fallbacks: callers may pass the structurally-similar
# `headroom.config.SmartCrusherConfig` (MCP server, SDK) or a
# pre-existing config object that predates these fields.
- lossless_min_savings_ratio=getattr(cfg, "lossless_min_savings_ratio", 0.15),
- compaction_core_field_fraction=getattr(cfg, "compaction_core_field_fraction", 0.8),
- compaction_heterogeneous_core_ratio=getattr(
+ "lossless_min_savings_ratio": getattr(cfg, "lossless_min_savings_ratio", 0.15),
+ "compaction_core_field_fraction": getattr(cfg, "compaction_core_field_fraction", 0.8),
+ "compaction_heterogeneous_core_ratio": getattr(
cfg, "compaction_heterogeneous_core_ratio", 0.6
),
- compaction_max_flatten_inner_keys=getattr(cfg, "compaction_max_flatten_inner_keys", 6),
- compaction_min_buckets=getattr(cfg, "compaction_min_buckets", 2),
- compaction_max_buckets=getattr(cfg, "compaction_max_buckets", 8),
- )
+ "compaction_max_flatten_inner_keys": getattr(
+ cfg, "compaction_max_flatten_inner_keys", 6
+ ),
+ "compaction_min_buckets": getattr(cfg, "compaction_min_buckets", 2),
+ "compaction_max_buckets": getattr(cfg, "compaction_max_buckets", 8),
+ }
# Default: lossless-first compaction (PR4). Lossless wins for
# cleanly tabular input where it saves ≥ 30% bytes; otherwise
# falls through to the lossy path with CCR-Dropped retrieval
@@ -373,24 +398,59 @@ class SmartCrusher(Transform):
f"expected one of: {', '.join(_SUPPORTED_COMPACTION_FORMATS)}"
)
self._compaction_format = resolved_format if with_compaction else None
- if not with_compaction:
- self._rust = _RustSmartCrusher.without_compaction(rust_cfg)
- elif resolved_format == "csv-schema":
+ self._resolved_compaction_format = resolved_format
+ # Cache of Rust crushers keyed by lossless_only, so a per-call
+ # override builds the alternate at most once.
+ self._rust_by_lossless_only: dict[bool, Any] = {}
+ self._rust = self._build_rust(self._lossless_only)
+
+ def _build_rust(self, lossless_only: bool) -> Any:
+ """Build (and cache) the Rust crusher for a `lossless_only` value."""
+ cached = self._rust_by_lossless_only.get(lossless_only)
+ if cached is not None:
+ return cached
+ kwargs = dict(self._rust_cfg_kwargs)
+ kwargs["lossless_only"] = lossless_only
+ rust_cfg = self._RustSmartCrusherConfig(**kwargs)
+ if not self._with_compaction:
+ rust = self._RustSmartCrusher.without_compaction(rust_cfg)
+ elif self._resolved_compaction_format == "csv-schema":
# Keep the `new()` constructor for the default path so its
- # byte-parity coverage stays on the exact production
- # codepath.
- self._rust = _RustSmartCrusher(rust_cfg)
+ # byte-parity coverage stays on the exact production codepath.
+ rust = self._RustSmartCrusher(rust_cfg)
else:
- self._rust = _RustSmartCrusher.with_compaction_format(rust_cfg, resolved_format)
+ rust = self._RustSmartCrusher.with_compaction_format(
+ rust_cfg, self._resolved_compaction_format
+ )
+ self._rust_by_lossless_only[lossless_only] = rust
+ return rust
- def crush(self, content: str, query: str = "", bias: float = 1.0) -> CrushResult:
+ def crush(
+ self,
+ content: str,
+ query: str = "",
+ bias: float = 1.0,
+ lossless_only: bool | None = None,
+ ) -> CrushResult:
"""Crush a single JSON content string.
Mirrors the retired Python method. Returns a `CrushResult`
dataclass so call sites that destructure with `asdict()` keep
working.
+
+ `lossless_only` overrides the configured strict-lossless mode for
+ this call only. When `True`, the output is guaranteed marker-free
+ and byte-recoverable: lossless tabular compaction still applies,
+ but any path that would need a CCR marker (row-drop or
+ opaque-blob offload) leaves the content uncompacted instead.
+ `None` (default) uses the instance's configured value.
"""
- r = self._rust.crush(content, query, bias)
+ rust = (
+ self._rust
+ if lossless_only is None or bool(lossless_only) == self._lossless_only
+ else self._build_rust(bool(lossless_only))
+ )
+ r = rust.crush(content, query, bias)
# Re-attach the TOIN learning loop. The retired Python class
# recorded compressions into TOIN inline; the Rust port doesn't
# know about TOIN, and `ContentRouter._record_to_toin` skips
@@ -1017,6 +1077,7 @@ def smart_crush_tool_output(
config: SmartCrusherConfig | None = None,
ccr_config: CCRConfig | None = None,
with_compaction: bool = True,
+ lossless_only: bool | None = None,
) -> tuple[str, bool, str]:
"""Compress a single tool output. Returns `(crushed, was_modified, info)`.
@@ -1024,6 +1085,14 @@ def smart_crush_tool_output(
Defaults to the PR4 lossless-first behavior; pass
`with_compaction=False` to exercise the legacy lossy-only path
(still useful for retention-property tests).
+
+ `lossless_only=True` forces strict lossless mode: the output is
+ marker-free and byte-recoverable (no row drops, opaque blobs inline).
"""
- crusher = SmartCrusher(config=config, ccr_config=ccr_config, with_compaction=with_compaction)
+ crusher = SmartCrusher(
+ config=config,
+ ccr_config=ccr_config,
+ with_compaction=with_compaction,
+ lossless_only=lossless_only,
+ )
return crusher._smart_crush_content(content)
diff --git a/tests/test_agent_savings.py b/tests/test_agent_savings.py
index 38dbe2b4c..1eeae1519 100644
--- a/tests/test_agent_savings.py
+++ b/tests/test_agent_savings.py
@@ -358,6 +358,29 @@ def test_agent_90_router_uses_ccr_sampling_not_lossless_table() -> None:
assert crusher._with_compaction is False
+def test_router_lossless_only_flag_reaches_crusher() -> None:
+ # HEADROOM_LOSSLESS_ONLY=1 sets this field on the proxy router; it
+ # must flow through to the SmartCrusher so a real proxy session runs
+ # strict marker-free mode.
+ router = ContentRouter(ContentRouterConfig(smart_crusher_lossless_only=True))
+
+ crusher = router._get_smart_crusher()
+
+ assert crusher is not None
+ assert crusher._lossless_only is True
+
+
+def test_router_lossless_only_defaults_off() -> None:
+ # Unset (None) must not force the flag — default crushers stay in
+ # the marker-emitting mode.
+ router = ContentRouter(ContentRouterConfig())
+
+ crusher = router._get_smart_crusher()
+
+ assert crusher is not None
+ assert crusher._lossless_only is False
+
+
def test_agent_90_router_json_tool_output_reaches_target_with_needle() -> None:
needle = "CRITICAL_NEEDLE_42"
rows = [
diff --git a/tests/test_transforms/test_smart_crusher_bugs.py b/tests/test_transforms/test_smart_crusher_bugs.py
index 3e7aa22e5..616806d99 100644
--- a/tests/test_transforms/test_smart_crusher_bugs.py
+++ b/tests/test_transforms/test_smart_crusher_bugs.py
@@ -124,6 +124,45 @@ class TestRecursionDepthLimit:
assert isinstance(parsed, list)
+class TestLosslessOnlyMode:
+ """`lossless_only` produces marker-free, byte-recoverable output.
+
+ Strict mode: lossless tabular compaction still applies, but any path
+ that would need a CCR marker (lossy row-drop OR opaque-blob offload)
+ leaves the content uncompacted instead — so the result is always
+ marker-free and decodes back to the original input without loss.
+ """
+
+ def _droppable_rows(self) -> list[dict]:
+ return [{"path": "a.py", "line": i, "content": "x" * 300} for i in range(50)]
+
+ def test_lossless_only_is_marker_free_and_byte_recoverable(self) -> None:
+ rows = self._droppable_rows()
+ config = SmartCrusherConfig(
+ min_items_to_analyze=3,
+ min_tokens_to_crush=0,
+ lossless_min_savings_ratio=0.99, # force the would-be-lossy path
+ lossless_only=True,
+ )
+ out = SmartCrusher(config=config).crush(json.dumps(rows))
+ assert "<<ccr:" not in out.compressed
+ assert json.loads(out.compressed) == rows
+
+ def test_crush_kwarg_overrides_configured_mode(self) -> None:
+ # Configured non-strict, but the per-call kwarg forces strict mode
+ # for this call: marker-free and fully recoverable.
+ rows = self._droppable_rows()
+ config = SmartCrusherConfig(
+ min_items_to_analyze=3,
+ min_tokens_to_crush=0,
+ lossless_min_savings_ratio=0.99,
+ )
+ crusher = SmartCrusher(config=config)
+ out = crusher.crush(json.dumps(rows), lossless_only=True)
+ assert "<<ccr:" not in out.compressed
+ assert json.loads(out.compressed) == rows
+
+
# Stage 3c.1 lockstep bug-fix tests previously lived here; they probed
# Python helpers (`_percentile_linear`, `_detect_sequential_pattern`,
# `_detect_rare_status_values`, `_compute_k_split`) that were removed